├── .gitattributes ├── .github └── workflows │ ├── docs-ci.yml │ └── pypi-release.yml ├── .gitignore ├── .readthedocs.yml ├── AUTHORS.rst ├── CHANGELOG.rst ├── CODE_OF_CONDUCT.rst ├── MANIFEST.in ├── Makefile ├── NOTICE ├── README.rst ├── apache-2.0.LICENSE ├── azure-pipelines.yml ├── configure ├── configure.bat ├── docs ├── Makefile ├── make.bat ├── scripts │ ├── doc8_style_check.sh │ └── sphinx_build_link_check.sh └── source │ ├── _static │ └── theme_overrides.css │ ├── api │ ├── license_expression.rst │ └── modules.rst │ ├── conf.py │ ├── index.rst │ └── readme_link.rst ├── etc ├── ci │ ├── azure-container-deb.yml │ ├── azure-container-rpm.yml │ ├── azure-posix.yml │ ├── azure-win.yml │ ├── install_sudo.sh │ ├── macports-ci │ ├── macports-ci.ABOUT │ └── mit.LICENSE └── scripts │ ├── README.rst │ ├── check_thirdparty.py │ ├── fetch_thirdparty.py │ ├── gen_pypi_simple.py │ ├── gen_pypi_simple.py.ABOUT │ ├── gen_pypi_simple.py.NOTICE │ ├── gen_requirements.py │ ├── gen_requirements_dev.py │ ├── requirements.txt │ ├── test_utils_pip_compatibility_tags.py │ ├── test_utils_pip_compatibility_tags.py.ABOUT │ ├── test_utils_pypi_supported_tags.py │ ├── test_utils_pypi_supported_tags.py.ABOUT │ ├── utils_dejacode.py │ ├── utils_pip_compatibility_tags.py │ ├── utils_pip_compatibility_tags.py.ABOUT │ ├── utils_pypi_supported_tags.py │ ├── utils_pypi_supported_tags.py.ABOUT │ ├── utils_requirements.py │ ├── utils_thirdparty.py │ └── utils_thirdparty.py.ABOUT ├── license-expression.ABOUT ├── pyproject.toml ├── requirements-dev.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── src └── license_expression │ ├── __init__.py │ ├── _pyahocorasick.ABOUT │ ├── _pyahocorasick.py │ └── data │ ├── cc-by-4.0.LICENSE │ ├── license_key_index.json.ABOUT │ └── scancode-licensedb-index.json ├── tests ├── data │ └── test_license_key_index.json ├── test__pyahocorasick.py ├── test_license_expression.py └── test_skeleton_codestyle.py └── tox.ini /.gitattributes: -------------------------------------------------------------------------------- 1 | # Ignore all Git auto CR/LF line endings conversions 2 | * -text 3 | pyproject.toml export-subst 4 | -------------------------------------------------------------------------------- /.github/workflows/docs-ci.yml: -------------------------------------------------------------------------------- 1 | name: CI Documentation 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-22.04 8 | 9 | strategy: 10 | max-parallel: 4 11 | matrix: 12 | python-version: [3.9] 13 | 14 | steps: 15 | - name: Checkout code 16 | uses: actions/checkout@v3 17 | 18 | - name: Set up Python ${{ matrix.python-version }} 19 | uses: actions/setup-python@v4 20 | with: 21 | python-version: ${{ matrix.python-version }} 22 | 23 | - name: Install Dependencies 24 | run: pip install -e .[docs] 25 | 26 | - name: Check Sphinx Documentation build minimally 27 | working-directory: ./docs 28 | run: python3 -m sphinx -E -W source build 29 | 30 | - name: Check for documentation style errors 31 | working-directory: ./docs 32 | run: ./scripts/doc8_style_check.sh 33 | 34 | 35 | -------------------------------------------------------------------------------- /.github/workflows/pypi-release.yml: -------------------------------------------------------------------------------- 1 | name: Create library release archives, create a GH release and publish PyPI wheel and sdist on tag in main branch 2 | 3 | 4 | # This is executed automatically on a tag in the main branch 5 | 6 | # Summary of the steps: 7 | # - build wheels and sdist 8 | # - upload wheels and sdist to PyPI 9 | # - create gh-release and upload wheels and dists there 10 | # TODO: smoke test wheels and sdist 11 | # TODO: add changelog to release text body 12 | 13 | # WARNING: this is designed only for packages building as pure Python wheels 14 | 15 | on: 16 | workflow_dispatch: 17 | push: 18 | tags: 19 | - "v*.*.*" 20 | 21 | jobs: 22 | build-pypi-distribs: 23 | name: Build and publish library to PyPI 24 | runs-on: ubuntu-24.04 25 | 26 | steps: 27 | - uses: actions/checkout@v4 28 | - name: Set up Python 29 | uses: actions/setup-python@v4 30 | with: 31 | python-version: 3.9 32 | 33 | - name: Install pypa/build 34 | run: python -m pip install build --user 35 | 36 | - name: Build a binary wheel and a source tarball 37 | run: python -m build --sdist --wheel --outdir dist/ 38 | 39 | - name: Upload built archives 40 | uses: actions/upload-artifact@v3 41 | with: 42 | name: pypi_archives 43 | path: dist/* 44 | 45 | 46 | create-gh-release: 47 | # Sets permissions of the GITHUB_TOKEN to allow release upload 48 | permissions: 49 | contents: write 50 | name: Create GH release 51 | needs: 52 | - build-pypi-distribs 53 | runs-on: ubuntu-24.04 54 | 55 | steps: 56 | - name: Download built archives 57 | uses: actions/download-artifact@v3 58 | with: 59 | name: pypi_archives 60 | path: dist 61 | 62 | - name: Create GH release 63 | uses: softprops/action-gh-release@v1 64 | with: 65 | draft: true 66 | files: dist/* 67 | 68 | 69 | create-pypi-release: 70 | name: Create PyPI release 71 | needs: 72 | - create-gh-release 73 | runs-on: ubuntu-24.04 74 | 75 | steps: 76 | - name: Download built archives 77 | uses: actions/download-artifact@v3 78 | with: 79 | name: pypi_archives 80 | path: dist 81 | 82 | - name: Publish to PyPI 83 | if: startsWith(github.ref, 'refs/tags') 84 | uses: pypa/gh-action-pypi-publish@release/v1 85 | with: 86 | password: ${{ secrets.PYPI_API_TOKEN }} 87 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python compiled files 2 | *.py[cod] 3 | 4 | # virtualenv and other misc bits 5 | /src/*.egg-info 6 | *.egg-info 7 | /dist 8 | /build 9 | /bin 10 | /lib 11 | /scripts 12 | /Scripts 13 | /Lib 14 | /pip-selfcheck.json 15 | /tmp 16 | /venv 17 | .Python 18 | /include 19 | /Include 20 | /local 21 | */local/* 22 | /local/ 23 | /share/ 24 | /tcl/ 25 | /.eggs/ 26 | 27 | # Installer logs 28 | pip-log.txt 29 | 30 | # Unit test / coverage reports 31 | .cache 32 | .coverage 33 | .coverage.* 34 | nosetests.xml 35 | htmlcov 36 | 37 | # Translations 38 | *.mo 39 | 40 | # IDEs 41 | .project 42 | .pydevproject 43 | .idea 44 | org.eclipse.core.resources.prefs 45 | .vscode 46 | .vs 47 | 48 | # Sphinx 49 | docs/_build 50 | docs/bin 51 | docs/build 52 | docs/include 53 | docs/Lib 54 | doc/pyvenv.cfg 55 | pyvenv.cfg 56 | 57 | # Various junk and temp files 58 | .DS_Store 59 | *~ 60 | .*.sw[po] 61 | .build 62 | .ve 63 | *.bak 64 | /.cache/ 65 | 66 | # pyenv 67 | /.python-version 68 | /man/ 69 | /.pytest_cache/ 70 | lib64 71 | tcl 72 | 73 | # Ignore Jupyter Notebook related temp files 74 | .ipynb_checkpoints/ 75 | /.tox/ 76 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # .readthedocs.yml 2 | # Read the Docs configuration file 3 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 4 | 5 | # Required 6 | version: 2 7 | 8 | # Build in latest ubuntu/python 9 | build: 10 | os: ubuntu-22.04 11 | tools: 12 | python: "3.11" 13 | 14 | # Build PDF & ePub 15 | formats: 16 | - epub 17 | - pdf 18 | 19 | # Where the Sphinx conf.py file is located 20 | sphinx: 21 | configuration: docs/source/conf.py 22 | 23 | # Setting the python version and doc build requirements 24 | python: 25 | install: 26 | - method: pip 27 | path: . 28 | extra_requirements: 29 | - docs 30 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | The following organizations or individuals have contributed to this code: 2 | 3 | - Ayan Sinha Mahapatra @AyanSinhaMahapatra 4 | - Carmen Bianca Bakker @carmenbianca 5 | - Chin-Yeung Li @chinyeungli 6 | - Dennis Clark @DennisClark 7 | - John Horan @johnmhoran 8 | - Jono Yang @JonoYang 9 | - Max Mehl @mxmehl 10 | - nexB Inc. @nexB 11 | - Peter Kolbus @pkolbus 12 | - Philippe Ombredanne @pombredanne 13 | - Sebastian Schuberth @sschuberth 14 | - Steven Esser @majurg 15 | - Thomas Druez @tdruez 16 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Changelog 2 | ========= 3 | 4 | v30.4.1 - 2025-01-10 5 | -------------------- 6 | 7 | This is a minor release without API changes: 8 | 9 | - Use latest skeleton 10 | - Update license list to latest ScanCode and SPDX 3.26 11 | 12 | 13 | v30.4.0 - 2024-10-21 14 | -------------------- 15 | 16 | This is a minor release without API changes: 17 | 18 | - Use latest skeleton 19 | - Update license list to latest ScanCode and SPDX 3.25 20 | - Drop support for Python 3.8 21 | 22 | v30.3.1 - 2024-08-13 23 | -------------------- 24 | 25 | This is a minor release without API changes: 26 | 27 | - Update link references of ownership from nexB to aboutcode-org 28 | 29 | v30.3.0 - 2024-03-18 30 | -------------------- 31 | 32 | This is a minor release without API changes: 33 | 34 | - Use latest skeleton 35 | - Update license list to latest ScanCode and SPDX 3.23 36 | - Drop support for Python 3.7 37 | 38 | v30.2.0 - 2023-11-29 39 | -------------------- 40 | 41 | This is a minor release without API changes: 42 | 43 | - Use latest skeleton 44 | - Update license list to latest ScanCode and SPDX 3.22 45 | - Add Python 3.12 support in CI 46 | 47 | 48 | v30.1.1 - 2023-01-16 49 | ---------------------- 50 | 51 | This is a minor dot release without API changes 52 | 53 | - Use latest skeleton 54 | - Update license list to latest ScanCode and SPDX 3.20 55 | 56 | 57 | v30.1.0 - 2023-01-16 58 | ---------------------- 59 | 60 | This is a minor release without API changes 61 | 62 | - Use latest skeleton (and updated configure script) 63 | - Update license list to latest ScanCode and SPDX 3.19 64 | - Use correct syntax for python_require 65 | - Drop using Travis and Appveyor 66 | - Drop support for Python 3.7 and add Python 3.11 in CI 67 | 68 | 69 | v30.0.0 - 2022-05-10 70 | ---------------------- 71 | 72 | This is a minor release with API changes 73 | 74 | - Use latest skeleton (and updated configure script) 75 | - Drop using calver 76 | - Improve error checking when combining licenses 77 | 78 | 79 | 80 | v21.6.14 - 2021-06-14 81 | ---------------------- 82 | 83 | Added 84 | ~~~~~ 85 | 86 | - Switch to calver for package versioning to better convey the currency of the 87 | bundled data. 88 | 89 | - Include https://scancode-licensedb.aboutcode.org/ licenses list with 90 | ScanCode (v21.6.7) and SPDX licenses (v3.13) keys. Add new functions to 91 | create Licensing using these licenses as LicenseSymbol. 92 | 93 | - Add new License.dedup() method to deduplicate and simplify license expressions 94 | without over simplifying. 95 | 96 | - Add new License.validate() method to return a new ExpressionInfo object with 97 | details on a license expression validation. 98 | 99 | 100 | Changed 101 | ~~~~~~~ 102 | - Drop support for Python 2. 103 | - Adopt the project skeleton from https://github.com/nexB/skeleton 104 | and its new configure script 105 | 106 | 107 | v1.2 - 2019-11-14 108 | ------------------ 109 | Added 110 | ~~~~~ 111 | - Add ability to render WITH expression wrapped in parenthesis 112 | 113 | Fixes 114 | ~~~~~ 115 | - Fix anomalous backslashes in strings 116 | 117 | Changed 118 | ~~~~~~~ 119 | - Update the thirdparty directory structure. 120 | 121 | 122 | v1.0 - 2019-10-16 123 | ------------------ 124 | Added 125 | ~~~~~ 126 | - New version of boolean.py library 127 | - Add ability to leave license expressions unsorted when simplifying 128 | 129 | Changed 130 | ~~~~~~~ 131 | - updated travis CI settings 132 | 133 | 134 | v0.999 - 2019-04-29 135 | -------------------- 136 | - Initial release 137 | - license-expression is small utility library to parse, compare and 138 | simplify and normalize license expressions. 139 | 140 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.rst: -------------------------------------------------------------------------------- 1 | Contributor Covenant Code of Conduct 2 | ==================================== 3 | 4 | Our Pledge 5 | ---------- 6 | 7 | In the interest of fostering an open and welcoming environment, we as 8 | contributors and maintainers pledge to making participation in our 9 | project and our community a harassment-free experience for everyone, 10 | regardless of age, body size, disability, ethnicity, gender identity and 11 | expression, level of experience, education, socio-economic status, 12 | nationality, personal appearance, race, religion, or sexual identity and 13 | orientation. 14 | 15 | Our Standards 16 | ------------- 17 | 18 | Examples of behavior that contributes to creating a positive environment 19 | include: 20 | 21 | - Using welcoming and inclusive language 22 | - Being respectful of differing viewpoints and experiences 23 | - Gracefully accepting constructive criticism 24 | - Focusing on what is best for the community 25 | - Showing empathy towards other community members 26 | 27 | Examples of unacceptable behavior by participants include: 28 | 29 | - The use of sexualized language or imagery and unwelcome sexual 30 | attention or advances 31 | - Trolling, insulting/derogatory comments, and personal or political 32 | attacks 33 | - Public or private harassment 34 | - Publishing others’ private information, such as a physical or 35 | electronic address, without explicit permission 36 | - Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | Our Responsibilities 40 | -------------------- 41 | 42 | Project maintainers are responsible for clarifying the standards of 43 | acceptable behavior and are expected to take appropriate and fair 44 | corrective action in response to any instances of unacceptable behavior. 45 | 46 | Project maintainers have the right and responsibility to remove, edit, 47 | or reject comments, commits, code, wiki edits, issues, and other 48 | contributions that are not aligned to this Code of Conduct, or to ban 49 | temporarily or permanently any contributor for other behaviors that they 50 | deem inappropriate, threatening, offensive, or harmful. 51 | 52 | Scope 53 | ----- 54 | 55 | This Code of Conduct applies both within project spaces and in public 56 | spaces when an individual is representing the project or its community. 57 | Examples of representing a project or community include using an 58 | official project e-mail address, posting via an official social media 59 | account, or acting as an appointed representative at an online or 60 | offline event. Representation of a project may be further defined and 61 | clarified by project maintainers. 62 | 63 | Enforcement 64 | ----------- 65 | 66 | Instances of abusive, harassing, or otherwise unacceptable behavior may 67 | be reported by contacting the project team at pombredanne@gmail.com 68 | or on the Gitter chat channel at https://gitter.im/aboutcode-org/discuss . 69 | All complaints will be reviewed and investigated and will result in a 70 | response that is deemed necessary and appropriate to the circumstances. 71 | The project team is obligated to maintain confidentiality with regard to 72 | the reporter of an incident. Further details of specific enforcement 73 | policies may be posted separately. 74 | 75 | Project maintainers who do not follow or enforce the Code of Conduct in 76 | good faith may face temporary or permanent repercussions as determined 77 | by other members of the project’s leadership. 78 | 79 | Attribution 80 | ----------- 81 | 82 | This Code of Conduct is adapted from the `Contributor Covenant`_ , 83 | version 1.4, available at 84 | https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 85 | 86 | .. _Contributor Covenant: https://www.contributor-covenant.org 87 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft src 2 | 3 | include *.LICENSE 4 | include NOTICE 5 | include *.ABOUT 6 | include *.toml 7 | include *.yml 8 | include *.rst 9 | include setup.* 10 | include configure* 11 | include requirements* 12 | include .git* 13 | 14 | global-exclude *.py[co] __pycache__ *.*~ 15 | 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: Apache-2.0 2 | # 3 | # Copyright (c) nexB Inc. and others. All rights reserved. 4 | # ScanCode is a trademark of nexB Inc. 5 | # SPDX-License-Identifier: Apache-2.0 6 | # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 7 | # See https://github.com/aboutcode-org/skeleton for support or download. 8 | # See https://aboutcode.org for more information about nexB OSS projects. 9 | # 10 | 11 | # Python version can be specified with `$ PYTHON_EXE=python3.x make conf` 12 | PYTHON_EXE?=python3 13 | VENV=venv 14 | ACTIVATE?=. ${VENV}/bin/activate; 15 | 16 | dev: 17 | @echo "-> Configure the development envt." 18 | ./configure --dev 19 | 20 | isort: 21 | @echo "-> Apply isort changes to ensure proper imports ordering" 22 | ${VENV}/bin/isort --sl -l 100 src tests setup.py 23 | 24 | black: 25 | @echo "-> Apply black code formatter" 26 | ${VENV}/bin/black -l 100 src tests setup.py 27 | 28 | doc8: 29 | @echo "-> Run doc8 validation" 30 | @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ 31 | 32 | valid: isort black 33 | 34 | check: 35 | @echo "-> Run pycodestyle (PEP8) validation" 36 | @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . 37 | @echo "-> Run isort imports ordering validation" 38 | @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . 39 | @echo "-> Run black validation" 40 | @${ACTIVATE} black --check --check -l 100 src tests setup.py 41 | 42 | clean: 43 | @echo "-> Clean the Python env" 44 | ./configure --clean 45 | 46 | test: 47 | @echo "-> Run the test suite" 48 | ${VENV}/bin/pytest -vvs 49 | 50 | docs: 51 | rm -rf docs/_build/ 52 | @${ACTIVATE} sphinx-build docs/ docs/_build/ 53 | 54 | .PHONY: conf dev check valid black isort clean test docs 55 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | # 2 | 3 | # Copyright (c) nexB Inc. and others. 4 | 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | # 8 | 9 | # Visit https://aboutcode.org and https://github.com/aboutcode-org/license-expression 10 | 11 | # for support and download. 12 | 13 | # 14 | 15 | # Licensed under the Apache License, Version 2.0 (the "License"); 16 | 17 | # you may not use this file except in compliance with the License. 18 | 19 | # You may obtain a copy of the License at 20 | 21 | # 22 | 23 | # http://www.apache.org/licenses/LICENSE-2.0 24 | 25 | # 26 | 27 | # Unless required by applicable law or agreed to in writing, software 28 | 29 | # distributed under the License is distributed on an "AS IS" BASIS, 30 | 31 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 32 | 33 | # See the License for the specific language governing permissions and 34 | 35 | # limitations under the License. 36 | 37 | # 38 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | license-expression 3 | ================== 4 | 5 | ``license-expression`` is a comprehensive utility library to parse, compare, 6 | simplify and normalize license expressions (such as SPDX license expressions) 7 | using boolean logic. 8 | 9 | - License: Apache-2.0 10 | - Python: 3.9+ 11 | - Homepage: https://github.com/aboutcode-org/license-expression/ 12 | - Install: `pip install license-expression` also available in most Linux distro. 13 | 14 | Software project licenses are often a combination of several free and open 15 | source software licenses. License expressions -- as specified by SPDX -- provide 16 | a concise and human readable way to express these licenses without having to 17 | read long license texts, while still being machine-readable. 18 | 19 | License expressions are used by key FOSS projects such as Linux; several 20 | packages ecosystem use them to document package licensing metadata such as 21 | npm and Rubygems; they are important when exchanging software data (such as with 22 | SPDX and SBOM in general) as a way to express licensing precisely. 23 | 24 | ``license-expression`` is a comprehensive utility library to parse, compare, 25 | simplify and normalize these license expressions (such as SPDX license expressions) 26 | using boolean logic like in: `GPL-2.0-or-later WITH Classpath-exception-2.0 AND MIT`. 27 | 28 | It includes the license keys from SPDX https://spdx.org/licenses/ (version 3.26) 29 | and ScanCode LicenseDB (from scancode-toolkit version 32.3.1, last published on 2025-01-10). 30 | See https://scancode-licensedb.aboutcode.org/ to get started quickly. 31 | 32 | ``license-expression`` is both powerful and simple to use and is a used as the 33 | license expression engine in several projects and products such as: 34 | 35 | - AboutCode-toolkit https://github.com/aboutcode-org/aboutcode-toolkit 36 | - AlekSIS (School Information System) https://edugit.org/AlekSIS/official/AlekSIS-Core 37 | - Barista https://github.com/Optum/barista 38 | - Conda forge tools https://github.com/conda-forge/conda-smithy 39 | - DejaCode https://dejacode.com 40 | - DeltaCode https://github.com/nexB/deltacode 41 | - FenixscanX https://github.com/SmartsYoung/FenixscanX 42 | - FetchCode https://github.com/aboutcode-org/fetchcode 43 | - Flict https://github.com/vinland-technology/flict and https://github.com/vinland-technology 44 | - license.sh https://github.com/webscopeio/license.sh 45 | - liferay_inbound_checker https://github.com/carmenbianca/liferay_inbound_checker 46 | - REUSE https://reuse.software/ and https://github.com/fsfe/reuse-tool 47 | - ScanCode-io https://github.com/aboutcode-org/scancode.io 48 | - ScanCode-toolkit https://github.com/aboutcode-org/scancode-toolkit 49 | - SecObserve https://github.com/MaibornWolff/SecObserve 50 | 51 | See also for details: 52 | - https://spdx.github.io/spdx-spec/appendix-IV-SPDX-license-expressions/ 53 | 54 | ``license-expression`` is also packaged for most Linux distributions. See below. 55 | 56 | Alternative: 57 | 58 | There is no known alternative library for Python, but there are several similar 59 | libraries in other languages (but not as powerful of course!): 60 | 61 | - JavaScript https://github.com/jslicense/spdx-expression-parse.js 62 | - Rust https://github.com/ehuss/license-exprs 63 | - Haskell https://github.com/phadej/spdx 64 | - Go https://github.com/kyoh86/go-spdx 65 | - Ada https://github.com/Fabien-Chouteau/spdx_ada 66 | - Java https://github.com/spdx/tools and https://github.com/aschet/spdx-license-expression-tools 67 | 68 | Build and tests status 69 | ====================== 70 | 71 | +--------------------------+------------------------+----------------------------------+ 72 | |**Linux & macOS (Travis)**| **Windows (AppVeyor)** |**Linux, Windows & macOS (Azure)**| 73 | +==========================+========================+==================================+ 74 | | | | | 75 | | |travis-badge-icon| | |appveyor-badge-icon| | |azure-badge-icon| | 76 | | | | | 77 | +--------------------------+------------------------+----------------------------------+ 78 | 79 | Source code and download 80 | ======================== 81 | 82 | - GitHub https://github.com/aboutcode-org/license-expression.git 83 | - PyPI https://pypi.python.org/pypi/license-expression 84 | 85 | Also available in several Linux distros: 86 | 87 | - Arch Linux https://archlinux.org/packages/extra/any/python-license-expression/ 88 | - Debian https://packages.debian.org/unstable/source/license-expression 89 | - DragonFly BSD https://github.com/DragonFlyBSD/DPorts/tree/master/textproc/py-license-expression 90 | - Fedora https://src.fedoraproject.org/rpms/python-license-expression/ 91 | - FreeBSD https://www.freshports.org/textproc/py-license-expression 92 | - NixOS https://github.com/NixOS/nixpkgs/blob/release-21.05/pkgs/development/python-modules/license-expression/default.nix 93 | - openSUSE https://build.opensuse.org/package/show/openSUSE:Factory/python-license-expression 94 | 95 | 96 | Support 97 | ======= 98 | 99 | - Submit bugs and questions at: https://github.com/aboutcode-org/license-expression/issues 100 | - Join the chat at: https://gitter.im/aboutcode-org/discuss 101 | 102 | Description 103 | =========== 104 | 105 | This module defines a mini language to parse, validate, simplify, normalize and 106 | compare license expressions using a boolean logic engine. 107 | 108 | This supports SPDX license expressions and also accepts other license naming 109 | conventions and license identifiers aliases to resolve and normalize any license 110 | expressions. 111 | 112 | Using boolean logic, license expressions can be tested for equality, containment, 113 | equivalence and can be normalized or simplified. 114 | 115 | It also bundles the SPDX License list (3.26 as of now) and the ScanCode license 116 | DB (based on latest ScanCode) to easily parse and validate expressions using 117 | the license symbols. 118 | 119 | 120 | Usage examples 121 | ============== 122 | 123 | The main entry point is the ``Licensing`` object that you can use to parse, 124 | validate, compare, simplify and normalize license expressions. 125 | 126 | Create an SPDX Licensing and parse expressions:: 127 | 128 | >>> from license_expression import get_spdx_licensing 129 | >>> licensing = get_spdx_licensing() 130 | >>> expression = ' GPL-2.0 or LGPL-2.1 and mit ' 131 | >>> parsed = licensing.parse(expression) 132 | >>> print(parsed.pretty()) 133 | OR( 134 | LicenseSymbol('GPL-2.0-only'), 135 | AND( 136 | LicenseSymbol('LGPL-2.1-only'), 137 | LicenseSymbol('MIT') 138 | ) 139 | ) 140 | 141 | >>> str(parsed) 142 | 'GPL-2.0-only OR (LGPL-2.1-only AND MIT)' 143 | 144 | >>> licensing.parse('unknwon with foo', validate=True, strict=True) 145 | license_expression.ExpressionParseError: A plain license symbol cannot be used 146 | as an exception in a "WITH symbol" statement. for token: "foo" at position: 13 147 | 148 | >>> licensing.parse('unknwon with foo', validate=True) 149 | license_expression.ExpressionError: Unknown license key(s): unknwon, foo 150 | 151 | >>> licensing.validate('foo and MIT and GPL-2.0+') 152 | ExpressionInfo( 153 | original_expression='foo and MIT and GPL-2.0+', 154 | normalized_expression=None, 155 | errors=['Unknown license key(s): foo'], 156 | invalid_symbols=['foo'] 157 | ) 158 | 159 | 160 | Create a simple Licensing and parse expressions:: 161 | 162 | >>> from license_expression import Licensing, LicenseSymbol 163 | >>> licensing = Licensing() 164 | >>> expression = ' GPL-2.0 or LGPL-2.1 and mit ' 165 | >>> parsed = licensing.parse(expression) 166 | >>> expression = ' GPL-2.0 or LGPL-2.1 and mit ' 167 | >>> expected = 'GPL-2.0-only OR (LGPL-2.1-only AND mit)' 168 | >>> assert str(parsed) == expected 169 | >>> assert parsed.render('{symbol.key}') == expected 170 | 171 | 172 | Create a Licensing with your own license symbols:: 173 | 174 | >>> expected = [ 175 | ... LicenseSymbol('GPL-2.0'), 176 | ... LicenseSymbol('LGPL-2.1'), 177 | ... LicenseSymbol('mit') 178 | ... ] 179 | >>> assert licensing.license_symbols(expression) == expected 180 | >>> assert licensing.license_symbols(parsed) == expected 181 | 182 | >>> symbols = ['GPL-2.0+', 'Classpath', 'BSD'] 183 | >>> licensing = Licensing(symbols) 184 | >>> expression = 'GPL-2.0+ with Classpath or (bsd)' 185 | >>> parsed = licensing.parse(expression) 186 | >>> expected = 'GPL-2.0+ WITH Classpath OR BSD' 187 | >>> assert parsed.render('{symbol.key}') == expected 188 | 189 | >>> expected = [ 190 | ... LicenseSymbol('GPL-2.0+'), 191 | ... LicenseSymbol('Classpath'), 192 | ... LicenseSymbol('BSD') 193 | ... ] 194 | >>> assert licensing.license_symbols(parsed) == expected 195 | >>> assert licensing.license_symbols(expression) == expected 196 | 197 | And expression can be deduplicated, to remove duplicate license subexpressions 198 | without changing the order and without consider license choices as simplifiable:: 199 | 200 | >>> expression2 = ' GPL-2.0 or (mit and LGPL 2.1) or bsd Or GPL-2.0 or (mit and LGPL 2.1)' 201 | >>> parsed2 = licensing.parse(expression2) 202 | >>> str(parsed2) 203 | 'GPL-2.0 OR (mit AND LGPL 2.1) OR BSD OR GPL-2.0 OR (mit AND LGPL 2.1)' 204 | >>> assert str(parsed2.simplify()) == 'BSD OR GPL-2.0 OR (LGPL 2.1 AND mit)' 205 | 206 | Expression can be simplified, treating them as boolean expressions:: 207 | 208 | >>> expression2 = ' GPL-2.0 or (mit and LGPL 2.1) or bsd Or GPL-2.0 or (mit and LGPL 2.1)' 209 | >>> parsed2 = licensing.parse(expression2) 210 | >>> str(parsed2) 211 | 'GPL-2.0 OR (mit AND LGPL 2.1) OR BSD OR GPL-2.0 OR (mit AND LGPL 2.1)' 212 | >>> assert str(parsed2.simplify()) == 'BSD OR GPL-2.0 OR (LGPL 2.1 AND mit)' 213 | 214 | Two expressions can be compared for equivalence and containment: 215 | 216 | >>> expr1 = licensing.parse(' GPL-2.0 or (LGPL 2.1 and mit) ') 217 | >>> expr2 = licensing.parse(' (mit and LGPL 2.1) or GPL-2.0 ') 218 | >>> licensing.is_equivalent(expr1, expr2) 219 | True 220 | >>> licensing.is_equivalent(' GPL-2.0 or (LGPL 2.1 and mit) ', 221 | ... ' (mit and LGPL 2.1) or GPL-2.0 ') 222 | True 223 | >>> expr1.simplify() == expr2.simplify() 224 | True 225 | >>> expr3 = licensing.parse(' GPL-2.0 or mit or LGPL 2.1') 226 | >>> licensing.is_equivalent(expr2, expr3) 227 | False 228 | >>> expr4 = licensing.parse('mit and LGPL 2.1') 229 | >>> expr4.simplify() in expr2.simplify() 230 | True 231 | >>> licensing.contains(expr2, expr4) 232 | True 233 | 234 | Development 235 | =========== 236 | 237 | - Checkout a clone from https://github.com/aboutcode-org/license-expression.git 238 | 239 | - Then run ``./configure --dev`` and then ``source tmp/bin/activate`` on Linux and POSIX. 240 | This will install all dependencies in a local virtualenv, including 241 | development deps. 242 | 243 | - On Windows run ``configure.bat --dev`` and then ``Scripts\bin\activate`` instead. 244 | 245 | - To run the tests, run ``pytest -vvs`` 246 | 247 | 248 | .. |travis-badge-icon| image:: https://api.travis-ci.org/nexB/license-expression.png?branch=master 249 | :target: https://travis-ci.org/nexB/license-expression 250 | :alt: Travis tests status 251 | :align: middle 252 | 253 | .. |appveyor-badge-icon| image:: https://ci.appveyor.com/api/projects/status/github/nexB/license-expression?svg=true 254 | :target: https://ci.appveyor.com/project/nexB/license-expression 255 | :alt: Appveyor tests status 256 | :align: middle 257 | 258 | .. |azure-badge-icon| image:: https://dev.azure.com/nexB/license-expression/_apis/build/status/nexB.license-expression?branchName=master 259 | :target: https://dev.azure.com/nexB/license-expression/_build/latest?definitionId=2&branchName=master 260 | :alt: Azure pipelines tests status 261 | :align: middle 262 | 263 | -------------------------------------------------------------------------------- /apache-2.0.LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # We use Azure to run the full tests suites on multiple Python 3.x 3 | # on multiple Windows, macOS and Linux versions all on 64 bits 4 | # These jobs are using VMs with Azure-provided Python builds 5 | ################################################################################ 6 | 7 | jobs: 8 | - template: etc/ci/azure-posix.yml 9 | parameters: 10 | job_name: ubuntu22_cpython 11 | image_name: ubuntu-22.04 12 | python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] 13 | test_suites: 14 | all: venv/bin/pytest -n 2 -vvs 15 | 16 | - template: etc/ci/azure-posix.yml 17 | parameters: 18 | job_name: ubuntu24_cpython 19 | image_name: ubuntu-24.04 20 | python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] 21 | test_suites: 22 | all: venv/bin/pytest -n 2 -vvs 23 | 24 | - template: etc/ci/azure-posix.yml 25 | parameters: 26 | job_name: macos13_cpython 27 | image_name: macOS-13 28 | python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] 29 | test_suites: 30 | all: venv/bin/pytest -n 2 -vvs 31 | 32 | - template: etc/ci/azure-posix.yml 33 | parameters: 34 | job_name: macos14_cpython 35 | image_name: macOS-14 36 | python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] 37 | test_suites: 38 | all: venv/bin/pytest -n 2 -vvs 39 | 40 | - template: etc/ci/azure-win.yml 41 | parameters: 42 | job_name: win2019_cpython 43 | image_name: windows-2019 44 | python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] 45 | test_suites: 46 | all: venv\Scripts\pytest -n 2 -vvs 47 | 48 | - template: etc/ci/azure-win.yml 49 | parameters: 50 | job_name: win2022_cpython 51 | image_name: windows-2022 52 | python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] 53 | test_suites: 54 | all: venv\Scripts\pytest -n 2 -vvs 55 | -------------------------------------------------------------------------------- /configure: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (c) nexB Inc. and others. All rights reserved. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 6 | # See https://github.com/aboutcode-org/ for support or download. 7 | # See https://aboutcode.org for more information about nexB OSS projects. 8 | # 9 | 10 | set -e 11 | #set -x 12 | 13 | ################################ 14 | # A configuration script to set things up: 15 | # create a virtualenv and install or update thirdparty packages. 16 | # Source this script for initial configuration 17 | # Use configure --help for details 18 | # 19 | # NOTE: please keep in sync with Windows script configure.bat 20 | # 21 | # This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz 22 | # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default 23 | ################################ 24 | CLI_ARGS=$1 25 | 26 | ################################ 27 | # Defaults. Change these variables to customize this script 28 | ################################ 29 | 30 | # Requirement arguments passed to pip and used by default or with --dev. 31 | REQUIREMENTS="--editable . --constraint requirements.txt" 32 | DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" 33 | DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" 34 | 35 | # where we create a virtualenv 36 | VIRTUALENV_DIR=venv 37 | 38 | # Cleanable files and directories to delete with the --clean option 39 | CLEANABLE="build dist venv .cache .eggs" 40 | 41 | # extra arguments passed to pip 42 | PIP_EXTRA_ARGS=" " 43 | 44 | # the URL to download virtualenv.pyz if needed 45 | VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz 46 | ################################ 47 | 48 | 49 | ################################ 50 | # Current directory where this script lives 51 | CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 52 | CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin 53 | 54 | 55 | ################################ 56 | # Install with or without and index. With "--no-index" this is using only local wheels 57 | # This is an offline mode with no index and no network operations 58 | # NO_INDEX="--no-index " 59 | NO_INDEX="" 60 | 61 | 62 | ################################ 63 | # Thirdparty package locations and index handling 64 | # Find packages from the local thirdparty directory if present 65 | THIRDPARDIR=$CFG_ROOT_DIR/thirdparty 66 | if [[ "$(echo $THIRDPARDIR/*.whl)x" != "$THIRDPARDIR/*.whlx" ]]; then 67 | PIP_EXTRA_ARGS="$NO_INDEX --find-links $THIRDPARDIR" 68 | fi 69 | 70 | 71 | ################################ 72 | # Set the quiet flag to empty if not defined 73 | if [[ "$CFG_QUIET" == "" ]]; then 74 | CFG_QUIET=" " 75 | fi 76 | 77 | 78 | ################################ 79 | # Find a proper Python to run 80 | # Use environment variables or a file if available. 81 | # Otherwise the latest Python by default. 82 | find_python() { 83 | if [[ "$PYTHON_EXECUTABLE" == "" ]]; then 84 | # check for a file named PYTHON_EXECUTABLE 85 | if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then 86 | PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") 87 | else 88 | PYTHON_EXECUTABLE=python3 89 | fi 90 | fi 91 | } 92 | 93 | 94 | ################################ 95 | create_virtualenv() { 96 | # create a virtualenv for Python 97 | # Note: we do not use the bundled Python 3 "venv" because its behavior and 98 | # presence is not consistent across Linux distro and sometimes pip is not 99 | # included either by default. The virtualenv.pyz app cures all these issues. 100 | 101 | VENV_DIR="$1" 102 | if [ ! -f "$CFG_BIN_DIR/python" ]; then 103 | 104 | mkdir -p "$CFG_ROOT_DIR/$VENV_DIR" 105 | 106 | if [ -f "$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" ]; then 107 | VIRTUALENV_PYZ="$CFG_ROOT_DIR/etc/thirdparty/virtualenv.pyz" 108 | else 109 | VIRTUALENV_PYZ="$CFG_ROOT_DIR/$VENV_DIR/virtualenv.pyz" 110 | wget -O "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" 2>/dev/null || curl -o "$VIRTUALENV_PYZ" "$VIRTUALENV_PYZ_URL" 111 | fi 112 | 113 | $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ 114 | --wheel embed --pip embed --setuptools embed \ 115 | --seeder pip \ 116 | --never-download \ 117 | --no-periodic-update \ 118 | --no-vcs-ignore \ 119 | $CFG_QUIET \ 120 | "$CFG_ROOT_DIR/$VENV_DIR" 121 | fi 122 | } 123 | 124 | 125 | ################################ 126 | install_packages() { 127 | # install requirements in virtualenv 128 | # note: --no-build-isolation means that pip/wheel/setuptools will not 129 | # be reinstalled a second time and reused from the virtualenv and this 130 | # speeds up the installation. 131 | # We always have the PEP517 build dependencies installed already. 132 | 133 | "$CFG_BIN_DIR/pip" install \ 134 | --upgrade \ 135 | --no-build-isolation \ 136 | $CFG_QUIET \ 137 | $PIP_EXTRA_ARGS \ 138 | $1 139 | } 140 | 141 | 142 | ################################ 143 | cli_help() { 144 | echo An initial configuration script 145 | echo " usage: ./configure [options]" 146 | echo 147 | echo The default is to configure for regular use. Use --dev for development. 148 | echo 149 | echo The options are: 150 | echo " --clean: clean built and installed files and exit." 151 | echo " --dev: configure the environment for development." 152 | echo " --help: display this help message and exit." 153 | echo 154 | echo By default, the python interpreter version found in the path is used. 155 | echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to 156 | echo configure another Python executable interpreter to use. If this is not 157 | echo set, a file named PYTHON_EXECUTABLE containing a single line with the 158 | echo path of the Python executable to use will be checked last. 159 | set +e 160 | exit 161 | } 162 | 163 | 164 | ################################ 165 | clean() { 166 | # Remove cleanable file and directories and files from the root dir. 167 | echo "* Cleaning ..." 168 | for cln in $CLEANABLE; 169 | do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; 170 | done 171 | set +e 172 | exit 173 | } 174 | 175 | 176 | ################################ 177 | # Main command line entry point 178 | CFG_REQUIREMENTS=$REQUIREMENTS 179 | 180 | # We are using getopts to parse option arguments that start with "-" 181 | while getopts :-: optchar; do 182 | case "${optchar}" in 183 | -) 184 | case "${OPTARG}" in 185 | help ) cli_help;; 186 | clean ) find_python && clean;; 187 | dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; 188 | docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; 189 | esac;; 190 | esac 191 | done 192 | 193 | 194 | PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" 195 | 196 | find_python 197 | create_virtualenv "$VIRTUALENV_DIR" 198 | install_packages "$CFG_REQUIREMENTS" 199 | . "$CFG_BIN_DIR/activate" 200 | 201 | 202 | set +e 203 | -------------------------------------------------------------------------------- /configure.bat: -------------------------------------------------------------------------------- 1 | @echo OFF 2 | @setlocal 3 | 4 | @rem Copyright (c) nexB Inc. and others. All rights reserved. 5 | @rem SPDX-License-Identifier: Apache-2.0 6 | @rem See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 7 | @rem See https://github.com/aboutcode-org/ for support or download. 8 | @rem See https://aboutcode.org for more information about nexB OSS projects. 9 | 10 | 11 | @rem ################################ 12 | @rem # A configuration script to set things up: 13 | @rem # create a virtualenv and install or update thirdparty packages. 14 | @rem # Source this script for initial configuration 15 | @rem # Use configure --help for details 16 | 17 | @rem # NOTE: please keep in sync with POSIX script configure 18 | 19 | @rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz 20 | @rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default 21 | @rem ################################ 22 | 23 | 24 | @rem ################################ 25 | @rem # Defaults. Change these variables to customize this script 26 | @rem ################################ 27 | 28 | @rem # Requirement arguments passed to pip and used by default or with --dev. 29 | set "REQUIREMENTS=--editable . --constraint requirements.txt" 30 | set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" 31 | set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" 32 | 33 | @rem # where we create a virtualenv 34 | set "VIRTUALENV_DIR=venv" 35 | 36 | @rem # Cleanable files and directories to delete with the --clean option 37 | set "CLEANABLE=build dist venv .cache .eggs" 38 | 39 | @rem # extra arguments passed to pip 40 | set "PIP_EXTRA_ARGS= " 41 | 42 | @rem # the URL to download virtualenv.pyz if needed 43 | set VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz 44 | @rem ################################ 45 | 46 | 47 | @rem ################################ 48 | @rem # Current directory where this script lives 49 | set CFG_ROOT_DIR=%~dp0 50 | set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" 51 | 52 | 53 | @rem ################################ 54 | @rem # Thirdparty package locations and index handling 55 | @rem # Find packages from the local thirdparty directory 56 | if exist "%CFG_ROOT_DIR%\thirdparty" ( 57 | set PIP_EXTRA_ARGS=--find-links "%CFG_ROOT_DIR%\thirdparty" 58 | ) 59 | 60 | 61 | @rem ################################ 62 | @rem # Set the quiet flag to empty if not defined 63 | if not defined CFG_QUIET ( 64 | set "CFG_QUIET= " 65 | ) 66 | 67 | 68 | @rem ################################ 69 | @rem # Main command line entry point 70 | set "CFG_REQUIREMENTS=%REQUIREMENTS%" 71 | 72 | :again 73 | if not "%1" == "" ( 74 | if "%1" EQU "--help" (goto cli_help) 75 | if "%1" EQU "--clean" (goto clean) 76 | if "%1" EQU "--dev" ( 77 | set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" 78 | ) 79 | if "%1" EQU "--docs" ( 80 | set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" 81 | ) 82 | shift 83 | goto again 84 | ) 85 | 86 | set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%" 87 | 88 | 89 | @rem ################################ 90 | @rem # Find a proper Python to run 91 | @rem # Use environment variables or a file if available. 92 | @rem # Otherwise the latest Python by default. 93 | if not defined PYTHON_EXECUTABLE ( 94 | @rem # check for a file named PYTHON_EXECUTABLE 95 | if exist "%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" ( 96 | set /p PYTHON_EXECUTABLE=<"%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" 97 | ) else ( 98 | set "PYTHON_EXECUTABLE=py" 99 | ) 100 | ) 101 | 102 | 103 | @rem ################################ 104 | :create_virtualenv 105 | @rem # create a virtualenv for Python 106 | @rem # Note: we do not use the bundled Python 3 "venv" because its behavior and 107 | @rem # presence is not consistent across Linux distro and sometimes pip is not 108 | @rem # included either by default. The virtualenv.pyz app cures all these issues. 109 | 110 | if not exist "%CFG_BIN_DIR%\python.exe" ( 111 | if not exist "%CFG_BIN_DIR%" ( 112 | mkdir "%CFG_BIN_DIR%" 113 | ) 114 | 115 | if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( 116 | %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ 117 | --wheel embed --pip embed --setuptools embed ^ 118 | --seeder pip ^ 119 | --never-download ^ 120 | --no-periodic-update ^ 121 | --no-vcs-ignore ^ 122 | %CFG_QUIET% ^ 123 | "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" 124 | ) else ( 125 | if not exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ( 126 | curl -o "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" %VIRTUALENV_PYZ_URL% 127 | 128 | if %ERRORLEVEL% neq 0 ( 129 | exit /b %ERRORLEVEL% 130 | ) 131 | ) 132 | %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ 133 | --wheel embed --pip embed --setuptools embed ^ 134 | --seeder pip ^ 135 | --never-download ^ 136 | --no-periodic-update ^ 137 | --no-vcs-ignore ^ 138 | %CFG_QUIET% ^ 139 | "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" 140 | ) 141 | ) 142 | 143 | if %ERRORLEVEL% neq 0 ( 144 | exit /b %ERRORLEVEL% 145 | ) 146 | 147 | 148 | @rem ################################ 149 | :install_packages 150 | @rem # install requirements in virtualenv 151 | @rem # note: --no-build-isolation means that pip/wheel/setuptools will not 152 | @rem # be reinstalled a second time and reused from the virtualenv and this 153 | @rem # speeds up the installation. 154 | @rem # We always have the PEP517 build dependencies installed already. 155 | 156 | "%CFG_BIN_DIR%\pip" install ^ 157 | --upgrade ^ 158 | --no-build-isolation ^ 159 | %CFG_QUIET% ^ 160 | %PIP_EXTRA_ARGS% ^ 161 | %CFG_REQUIREMENTS% 162 | 163 | 164 | @rem ################################ 165 | :create_bin_junction 166 | @rem # Create junction to bin to have the same directory between linux and windows 167 | if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( 168 | rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" 169 | ) 170 | mklink /J "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" 171 | 172 | if %ERRORLEVEL% neq 0 ( 173 | exit /b %ERRORLEVEL% 174 | ) 175 | 176 | exit /b 0 177 | 178 | 179 | @rem ################################ 180 | :cli_help 181 | echo An initial configuration script 182 | echo " usage: configure [options]" 183 | echo " " 184 | echo The default is to configure for regular use. Use --dev for development. 185 | echo " " 186 | echo The options are: 187 | echo " --clean: clean built and installed files and exit." 188 | echo " --dev: configure the environment for development." 189 | echo " --help: display this help message and exit." 190 | echo " " 191 | echo By default, the python interpreter version found in the path is used. 192 | echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to 193 | echo configure another Python executable interpreter to use. If this is not 194 | echo set, a file named PYTHON_EXECUTABLE containing a single line with the 195 | echo path of the Python executable to use will be checked last. 196 | exit /b 0 197 | 198 | 199 | @rem ################################ 200 | :clean 201 | @rem # Remove cleanable file and directories and files from the root dir. 202 | echo "* Cleaning ..." 203 | for %%F in (%CLEANABLE%) do ( 204 | rmdir /s /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 205 | del /f /q "%CFG_ROOT_DIR%\%%F" >nul 2>&1 206 | ) 207 | exit /b 0 208 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SPHINXAUTOBUILD = sphinx-autobuild 9 | SOURCEDIR = source 10 | BUILDDIR = build 11 | 12 | # Put it first so that "make" without argument is like "make help". 13 | help: 14 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 15 | 16 | .PHONY: help Makefile 17 | 18 | # Run the development server using sphinx-autobuild 19 | docs: 20 | @echo 21 | @echo "Starting up the docs server..." 22 | @echo 23 | $(SPHINXAUTOBUILD) --port 8000 --watch ${SOURCEDIR} $(SOURCEDIR) "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) 24 | 25 | # Catch-all target: route all unknown targets to Sphinx using the new 26 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 27 | %: Makefile 28 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 29 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | if "%SPHINXAUTOBUILD%" == "" ( 11 | set SPHINXAUTOBUILD=sphinx-autobuild 12 | ) 13 | set SOURCEDIR=source 14 | set BUILDDIR=build 15 | 16 | if "%1" == "" goto help 17 | 18 | if "%1" == "docs" goto docs 19 | 20 | %SPHINXBUILD% >NUL 2>NUL 21 | if errorlevel 9009 ( 22 | echo. 23 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 24 | echo.installed, then set the SPHINXBUILD environment variable to point 25 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 26 | echo.may add the Sphinx directory to PATH. 27 | echo. 28 | echo.If you don't have Sphinx installed, grab it from 29 | echo.http://sphinx-doc.org/ 30 | exit /b 1 31 | ) 32 | 33 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 34 | goto end 35 | 36 | :docs 37 | @echo 38 | @echo Starting up the docs server... 39 | @echo 40 | %SPHINXAUTOBUILD% --port 8000 --watch %SOURCEDIR% %SOURCEDIR% %BUILDDIR%\html %SPHINXOPTS% %O% 41 | goto end 42 | 43 | :help 44 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 45 | 46 | :end 47 | popd 48 | -------------------------------------------------------------------------------- /docs/scripts/doc8_style_check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # halt script on error 3 | set -e 4 | # Check for Style Code Violations 5 | doc8 --max-line-length 100 source --ignore D000 --quiet -------------------------------------------------------------------------------- /docs/scripts/sphinx_build_link_check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # halt script on error 3 | set -e 4 | # Build locally, and then check links 5 | sphinx-build -E -W -b linkcheck source build -------------------------------------------------------------------------------- /docs/source/_static/theme_overrides.css: -------------------------------------------------------------------------------- 1 | /* this is the container for the pages */ 2 | .wy-nav-content { 3 | max-width: 100%; 4 | padding: 0px 40px 0px 0px; 5 | margin-top: 0px; 6 | } 7 | 8 | .wy-nav-content-wrap { 9 | border-right: solid 1px; 10 | } 11 | 12 | div.rst-content { 13 | max-width: 1300px; 14 | border: 0; 15 | padding: 10px 80px 10px 80px; 16 | margin-left: 50px; 17 | } 18 | 19 | @media (max-width: 768px) { 20 | div.rst-content { 21 | max-width: 1300px; 22 | border: 0; 23 | padding: 0px 10px 10px 10px; 24 | margin-left: 0px; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /docs/source/api/license_expression.rst: -------------------------------------------------------------------------------- 1 | license\_expression package 2 | =========================== 3 | 4 | Module contents 5 | --------------- 6 | 7 | .. automodule:: license_expression 8 | :members: 9 | :undoc-members: 10 | :show-inheritance: 11 | -------------------------------------------------------------------------------- /docs/source/api/modules.rst: -------------------------------------------------------------------------------- 1 | license_expression 2 | ================== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | license_expression 8 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. 11 | 12 | import pathlib 13 | import sys 14 | 15 | srcdir = pathlib.Path(__file__).resolve().parents[2].joinpath('src') 16 | sys.path.insert(0, srcdir.as_posix()) 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = "nexb-skeleton" 21 | copyright = "nexB Inc. and others." 22 | author = "AboutCode.org authors and contributors" 23 | 24 | 25 | # -- General configuration --------------------------------------------------- 26 | 27 | # Add any Sphinx extension module names here, as strings. They can be 28 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 29 | # ones. 30 | extensions = [ 31 | "sphinx.ext.autodoc", 32 | "sphinx.ext.intersphinx", 33 | "sphinxcontrib.apidoc", 34 | "sphinx_reredirects", 35 | "sphinx_rtd_theme", 36 | "sphinx_rtd_dark_mode", 37 | "sphinx.ext.extlinks", 38 | "sphinx_copybutton", 39 | ] 40 | 41 | # FIXME: including AND, NOT and OR will result in endless recursion 42 | autodoc_default_options = { 43 | 'exclude-members': 'AND, NOT, OR', 44 | } 45 | 46 | 47 | # Redirects for olds pages 48 | # See https://documatt.gitlab.io/sphinx-reredirects/usage.html 49 | redirects = {} 50 | 51 | # This points to aboutcode.readthedocs.io 52 | # In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot 53 | # Link was created at commit - https://github.com/aboutcode-org/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 54 | 55 | intersphinx_mapping = { 56 | "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), 57 | "scancode-workbench": ( 58 | "https://scancode-workbench.readthedocs.io/en/develop/", 59 | None, 60 | ), 61 | } 62 | 63 | # Setting for sphinxcontrib.apidoc to automatically create API documentation. 64 | apidoc_module_dir = srcdir.joinpath('license_expression').as_posix() 65 | 66 | # Reference to other Sphinx documentations 67 | intersphinx_mapping = { 68 | "python": ("https://docs.python.org/3", None), 69 | "boolean.py": ("https://booleanpy.readthedocs.io/en/latest/", None), 70 | } 71 | 72 | # Add any paths that contain templates here, relative to this directory. 73 | templates_path = ["_templates"] 74 | 75 | # List of patterns, relative to source directory, that match files and 76 | # directories to ignore when looking for source files. 77 | # This pattern also affects html_static_path and html_extra_path. 78 | exclude_patterns = [] 79 | 80 | 81 | # -- Options for HTML output ------------------------------------------------- 82 | 83 | # The theme to use for HTML and HTML Help pages. See the documentation for 84 | # a list of builtin themes. 85 | # 86 | html_theme = "sphinx_rtd_theme" 87 | 88 | # Add any paths that contain custom static files (such as style sheets) here, 89 | # relative to this directory. They are copied after the builtin static files, 90 | # so a file named "default.css" will overwrite the builtin "default.css". 91 | html_static_path = ["_static"] 92 | 93 | master_doc = "index" 94 | 95 | html_context = { 96 | "display_github": True, 97 | "github_user": "nexB", 98 | "github_repo": "nexb-skeleton", 99 | "github_version": "develop", # branch 100 | "conf_py_path": "/docs/source/", # path in the checkout to the docs root 101 | } 102 | 103 | html_css_files = [ 104 | "theme_overrides.css", 105 | ] 106 | 107 | 108 | # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. 109 | html_show_sphinx = True 110 | 111 | # Define CSS and HTML abbreviations used in .rst files. These are examples. 112 | # .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text` 113 | rst_prolog = """ 114 | .. |psf| replace:: Python Software Foundation 115 | 116 | .. # define a hard line break for HTML 117 | .. |br| raw:: html 118 | 119 |
120 | 121 | .. role:: red 122 | 123 | .. role:: img-title 124 | 125 | .. role:: img-title-para 126 | 127 | """ 128 | 129 | # -- Options for LaTeX output ------------------------------------------------- 130 | 131 | latex_elements = {"classoptions": ",openany,oneside"} 132 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to license-expressions's documentation! 2 | =============================================== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | readme_link 9 | API 10 | 11 | Indices and tables 12 | ================== 13 | 14 | * :ref:`genindex` 15 | * :ref:`modindex` 16 | * :ref:`search` 17 | -------------------------------------------------------------------------------- /docs/source/readme_link.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../../README.rst 2 | -------------------------------------------------------------------------------- /etc/ci/azure-container-deb.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | job_name: '' 3 | container: '' 4 | python_path: '' 5 | python_version: '' 6 | package_manager: apt-get 7 | install_python: '' 8 | install_packages: | 9 | set -e -x 10 | sudo apt-get -y update 11 | sudo apt-get -y install \ 12 | build-essential \ 13 | xz-utils zlib1g bzip2 libbz2-1.0 tar \ 14 | sqlite3 libxml2-dev libxslt1-dev \ 15 | software-properties-common openssl 16 | test_suite: '' 17 | test_suite_label: '' 18 | 19 | 20 | jobs: 21 | - job: ${{ parameters.job_name }} 22 | 23 | pool: 24 | vmImage: 'ubuntu-16.04' 25 | 26 | container: 27 | image: ${{ parameters.container }} 28 | options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' 29 | 30 | steps: 31 | - checkout: self 32 | fetchDepth: 10 33 | 34 | - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} 35 | displayName: Install sudo 36 | 37 | - script: ${{ parameters.install_packages }} 38 | displayName: Install required packages 39 | 40 | - script: ${{ parameters.install_python }} 41 | displayName: 'Install Python ${{ parameters.python_version }}' 42 | 43 | - script: ${{ parameters.python_path }} --version 44 | displayName: 'Show Python version' 45 | 46 | - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev 47 | displayName: 'Run Configure' 48 | 49 | - script: ${{ parameters.test_suite }} 50 | displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' 51 | -------------------------------------------------------------------------------- /etc/ci/azure-container-rpm.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | job_name: '' 3 | image_name: 'ubuntu-16.04' 4 | container: '' 5 | python_path: '' 6 | python_version: '' 7 | package_manager: yum 8 | install_python: '' 9 | install_packages: | 10 | set -e -x 11 | sudo yum groupinstall -y "Development Tools" 12 | sudo yum install -y \ 13 | openssl openssl-devel \ 14 | sqlite-devel zlib-devel xz-devel bzip2-devel \ 15 | bzip2 tar unzip zip \ 16 | libxml2-devel libxslt-devel 17 | test_suite: '' 18 | test_suite_label: '' 19 | 20 | 21 | jobs: 22 | - job: ${{ parameters.job_name }} 23 | 24 | pool: 25 | vmImage: ${{ parameters.image_name }} 26 | 27 | container: 28 | image: ${{ parameters.container }} 29 | options: '--name ${{ parameters.job_name }} -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -v /usr/bin/docker:/tmp/docker:ro' 30 | 31 | steps: 32 | - checkout: self 33 | fetchDepth: 10 34 | 35 | - script: /tmp/docker exec -t -e LANG=C.UTF-8 -e LC_ALL=C.UTF-8 -u 0 ${{ parameters.job_name }} $(Build.SourcesDirectory)/etc/ci/install_sudo.sh ${{ parameters.package_manager }} 36 | displayName: Install sudo 37 | 38 | - script: ${{ parameters.install_packages }} 39 | displayName: Install required packages 40 | 41 | - script: ${{ parameters.install_python }} 42 | displayName: 'Install Python ${{ parameters.python_version }}' 43 | 44 | - script: ${{ parameters.python_path }} --version 45 | displayName: 'Show Python version' 46 | 47 | - script: PYTHON_EXE=${{ parameters.python_path }} ./configure --dev 48 | displayName: 'Run Configure' 49 | 50 | - script: ${{ parameters.test_suite }} 51 | displayName: 'Run ${{ parameters.test_suite_label }} tests with py${{ parameters.python_version }} on ${{ parameters.job_name }}' 52 | -------------------------------------------------------------------------------- /etc/ci/azure-posix.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | job_name: '' 3 | image_name: '' 4 | python_versions: [] 5 | test_suites: {} 6 | python_architecture: x64 7 | 8 | jobs: 9 | - job: ${{ parameters.job_name }} 10 | 11 | pool: 12 | vmImage: ${{ parameters.image_name }} 13 | 14 | strategy: 15 | matrix: 16 | ${{ each tsuite in parameters.test_suites }}: 17 | ${{ tsuite.key }}: 18 | test_suite_label: ${{ tsuite.key }} 19 | test_suite: ${{ tsuite.value }} 20 | 21 | steps: 22 | - checkout: self 23 | fetchDepth: 10 24 | 25 | - ${{ each pyver in parameters.python_versions }}: 26 | - task: UsePythonVersion@0 27 | inputs: 28 | versionSpec: '${{ pyver }}' 29 | architecture: '${{ parameters.python_architecture }}' 30 | displayName: '${{ pyver }} - Install Python' 31 | 32 | - script: | 33 | python${{ pyver }} --version 34 | echo "python${{ pyver }}" > PYTHON_EXECUTABLE 35 | ./configure --clean && ./configure --dev 36 | displayName: '${{ pyver }} - Configure' 37 | 38 | - script: $(test_suite) 39 | displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' 40 | -------------------------------------------------------------------------------- /etc/ci/azure-win.yml: -------------------------------------------------------------------------------- 1 | parameters: 2 | job_name: '' 3 | image_name: '' 4 | python_versions: [] 5 | test_suites: {} 6 | python_architecture: x64 7 | 8 | jobs: 9 | - job: ${{ parameters.job_name }} 10 | 11 | pool: 12 | vmImage: ${{ parameters.image_name }} 13 | 14 | strategy: 15 | matrix: 16 | ${{ each tsuite in parameters.test_suites }}: 17 | ${{ tsuite.key }}: 18 | test_suite_label: ${{ tsuite.key }} 19 | test_suite: ${{ tsuite.value }} 20 | 21 | steps: 22 | - checkout: self 23 | fetchDepth: 10 24 | 25 | - ${{ each pyver in parameters.python_versions }}: 26 | - task: UsePythonVersion@0 27 | inputs: 28 | versionSpec: '${{ pyver }}' 29 | architecture: '${{ parameters.python_architecture }}' 30 | displayName: '${{ pyver }} - Install Python' 31 | 32 | - script: | 33 | python --version 34 | echo | set /p=python> PYTHON_EXECUTABLE 35 | configure --clean && configure --dev 36 | displayName: '${{ pyver }} - Configure' 37 | 38 | - script: $(test_suite) 39 | displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' 40 | -------------------------------------------------------------------------------- /etc/ci/install_sudo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | 5 | if [[ "$1" == "apt-get" ]]; then 6 | apt-get update -y 7 | apt-get -o DPkg::Options::="--force-confold" install -y sudo 8 | 9 | elif [[ "$1" == "yum" ]]; then 10 | yum install -y sudo 11 | 12 | elif [[ "$1" == "dnf" ]]; then 13 | dnf install -y sudo 14 | 15 | fi 16 | -------------------------------------------------------------------------------- /etc/ci/macports-ci: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # Copyright (c) 2019 Giovanni Bussi 4 | 5 | # Permission is hereby granted, free of charge, to any person obtaining a copy 6 | # of this software and associated documentation files (the "Software"), to deal 7 | # in the Software without restriction, including without limitation the rights 8 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | # copies of the Software, and to permit persons to whom the Software is 10 | # furnished to do so, subject to the following conditions: 11 | 12 | # The above copyright notice and this permission notice shall be included in all 13 | # copies or substantial portions of the Software. 14 | 15 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | # SOFTWARE. 22 | 23 | export COLUMNS=80 24 | 25 | if [ "$GITHUB_ACTIONS" = true ] ; then 26 | echo "COLUMNS=$COLUMNS" >> "$GITHUB_ENV" 27 | fi 28 | 29 | # file to be source at the end of subshell: 30 | export MACPORTS_CI_SOURCEME="$(mktemp)" 31 | 32 | ( 33 | # start subshell 34 | # this allows to use the script in two ways: 35 | # 1. as ./macports-ci 36 | # 2. as source ./macports-ci 37 | # as of now, choice 2 only changes the env var COLUMNS. 38 | 39 | MACPORTS_VERSION=2.6.4 40 | MACPORTS_PREFIX=/opt/local 41 | MACPORTS_SYNC=tarball 42 | 43 | action=$1 44 | shift 45 | 46 | case "$action" in 47 | (install) 48 | 49 | echo "macports-ci: install" 50 | 51 | KEEP_BREW=yes 52 | 53 | for opt 54 | do 55 | case "$opt" in 56 | (--source) SOURCE=yes ;; 57 | (--binary) SOURCE=no ;; 58 | (--keep-brew) KEEP_BREW=yes ;; 59 | (--remove-brew) KEEP_BREW=no ;; 60 | (--version=*) MACPORTS_VERSION="${opt#--version=}" ;; 61 | (--prefix=*) MACPORTS_PREFIX="${opt#--prefix=}" ;; 62 | (--sync=*) MACPORTS_SYNC="${opt#--sync=}" ;; 63 | (*) echo "macports-ci: unknown option $opt" 64 | exit 1 ;; 65 | esac 66 | done 67 | 68 | if test "$KEEP_BREW" = no ; then 69 | echo "macports-ci: removing homebrew" 70 | pushd "$(mktemp -d)" 71 | curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/uninstall > uninstall 72 | chmod +x uninstall 73 | ./uninstall --force 74 | popd 75 | else 76 | echo "macports-ci: keeping HomeBrew" 77 | fi 78 | 79 | echo "macports-ci: prefix=$MACPORTS_PREFIX" 80 | 81 | if test "$MACPORTS_PREFIX" != /opt/local ; then 82 | echo "macports-ci: Installing on non standard prefix $MACPORTS_PREFIX can be only made from sources" 83 | SOURCE=yes 84 | fi 85 | 86 | if test "$SOURCE" = yes ; then 87 | echo "macports-ci: Installing from source" 88 | else 89 | echo "macports-ci: Installing from binary" 90 | fi 91 | 92 | echo "macports-ci: Sync mode=$MACPORTS_SYNC" 93 | 94 | pushd "$(mktemp -d)" 95 | 96 | OSX_VERSION="$(sw_vers -productVersion | grep -o '^[0-9][0-9]*\.[0-9][0-9]*')" 97 | 98 | if test "$OSX_VERSION" == 10.10 ; then 99 | OSX_NAME=Yosemite 100 | elif test "$OSX_VERSION" == 10.11 ; then 101 | OSX_NAME=ElCapitan 102 | elif test "$OSX_VERSION" == 10.12 ; then 103 | OSX_NAME=Sierra 104 | elif test "$OSX_VERSION" == 10.13 ; then 105 | OSX_NAME=HighSierra 106 | elif test "$OSX_VERSION" == 10.14 ; then 107 | OSX_NAME=Mojave 108 | elif test "$OSX_VERSION" == 10.15 ; then 109 | OSX_NAME=Catalina 110 | else 111 | echo "macports-ci: Unknown OSX version $OSX_VERSION" 112 | exit 1 113 | fi 114 | 115 | echo "macports-ci: OSX version $OSX_VERSION $OSX_NAME" 116 | 117 | MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg 118 | 119 | # this is a workaround needed because binary installer MacPorts-2.6.3-10.12-Sierra.pkg is broken 120 | if [ "$SOURCE" != yes ] && [ "$MACPORTS_PKG" = "MacPorts-2.6.3-10.12-Sierra.pkg" ] ; then 121 | echo "macports-ci: WARNING $MACPORTS_PKG installer is broken" 122 | echo "macports-ci: reverting to 2.6.2 installer followed by selfupdate" 123 | MACPORTS_VERSION=2.6.2 124 | MACPORTS_PKG=MacPorts-${MACPORTS_VERSION}-${OSX_VERSION}-${OSX_NAME}.pkg 125 | fi 126 | 127 | URL="https://distfiles.macports.org/MacPorts" 128 | URL="https://github.com/macports/macports-base/releases/download/v$MACPORTS_VERSION/" 129 | 130 | echo "macports-ci: Base URL is $URL" 131 | 132 | if test "$SOURCE" = yes ; then 133 | # download source: 134 | curl -LO $URL/MacPorts-${MACPORTS_VERSION}.tar.bz2 135 | tar xjf MacPorts-${MACPORTS_VERSION}.tar.bz2 136 | cd MacPorts-${MACPORTS_VERSION} 137 | # install 138 | ./configure --prefix="$MACPORTS_PREFIX" --with-applications-dir="$MACPORTS_PREFIX/Applications" >/dev/null && 139 | sudo make install >/dev/null 140 | else 141 | 142 | # download installer: 143 | curl -LO $URL/$MACPORTS_PKG 144 | # install: 145 | sudo installer -verbose -pkg $MACPORTS_PKG -target / 146 | fi 147 | 148 | # update: 149 | export PATH="$MACPORTS_PREFIX/bin:$PATH" 150 | 151 | echo "PATH=\"$MACPORTS_PREFIX/bin:\$PATH\"" > "$MACPORTS_CI_SOURCEME" 152 | 153 | if [ "$GITHUB_ACTIONS" = true ] ; then 154 | echo "$MACPORTS_PREFIX/bin" >> "$GITHUB_PATH" 155 | fi 156 | 157 | 158 | SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf 159 | 160 | case "$MACPORTS_SYNC" in 161 | (rsync) 162 | echo "macports-ci: Using rsync" 163 | ;; 164 | (github) 165 | echo "macports-ci: Using github" 166 | pushd "$MACPORTS_PREFIX"/var/macports/sources 167 | sudo mkdir -p github.com/macports/macports-ports/ 168 | sudo chown -R $USER:admin github.com 169 | git clone https://github.com/macports/macports-ports.git github.com/macports/macports-ports/ 170 | awk '{if($NF=="[default]") print "file:///opt/local/var/macports/sources/github.com/macports/macports-ports/"; else print}' "$SOURCES" > $HOME/$$.tmp 171 | sudo mv -f $HOME/$$.tmp "$SOURCES" 172 | popd 173 | ;; 174 | (tarball) 175 | echo "macports-ci: Using tarball" 176 | awk '{if($NF=="[default]") print "https://distfiles.macports.org/ports.tar.gz [default]"; else print}' "$SOURCES" > $$.tmp 177 | sudo mv -f $$.tmp "$SOURCES" 178 | ;; 179 | (*) 180 | echo "macports-ci: Unknown sync mode $MACPORTS_SYNC" 181 | ;; 182 | esac 183 | 184 | i=1 185 | # run through a while to retry upon failure 186 | while true 187 | do 188 | echo "macports-ci: Trying to selfupdate (iteration $i)" 189 | # here I test for the presence of a known portfile 190 | # this check confirms that ports were installed 191 | # notice that port -N selfupdate && break is not sufficient as a test 192 | # (sometime it returns a success even though ports have not been installed) 193 | # for some misterious reasons, running without "-d" does not work in some case 194 | sudo port -d -N selfupdate 2>&1 | grep -v DEBUG | awk '{if($1!="x")print}' 195 | port info xdrfile > /dev/null && break || true 196 | sleep 5 197 | i=$((i+1)) 198 | if ((i>20)) ; then 199 | echo "macports-ci: Failed after $i iterations" 200 | exit 1 201 | fi 202 | done 203 | 204 | echo "macports-ci: Selfupdate successful after $i iterations" 205 | 206 | dir="$PWD" 207 | popd 208 | sudo rm -fr $dir 209 | 210 | ;; 211 | 212 | (localports) 213 | 214 | echo "macports-ci: localports" 215 | 216 | for opt 217 | do 218 | case "$opt" in 219 | (*) ports="$opt" ;; 220 | esac 221 | done 222 | 223 | if ! test -d "$ports" ; then 224 | echo "macports-ci: Please provide a port directory" 225 | exit 1 226 | fi 227 | 228 | w=$(which port) 229 | 230 | MACPORTS_PREFIX="${w%/bin/port}" 231 | 232 | cd "$ports" 233 | 234 | ports="$(pwd)" 235 | 236 | echo "macports-ci: Portdir fullpath: $ports" 237 | SOURCES="${MACPORTS_PREFIX}"/etc/macports/sources.conf 238 | 239 | awk -v repo="file://$ports" '{if($NF=="[default]") print repo; print}' "$SOURCES" > $$.tmp 240 | sudo mv -f $$.tmp "$SOURCES" 241 | 242 | portindex 243 | 244 | ;; 245 | 246 | (ccache) 247 | w=$(which port) 248 | MACPORTS_PREFIX="${w%/bin/port}" 249 | 250 | echo "macports-ci: ccache" 251 | 252 | ccache_do=install 253 | 254 | for opt 255 | do 256 | case "$opt" in 257 | (--save) ccache_do=save ;; 258 | (--install) ccache_do=install ;; 259 | (*) echo "macports-ci: ccache: unknown option $opt" 260 | exit 1 ;; 261 | esac 262 | done 263 | 264 | 265 | case "$ccache_do" in 266 | (install) 267 | # first install ccache 268 | sudo port -N install ccache 269 | # then tell macports to use it 270 | CONF="${MACPORTS_PREFIX}"/etc/macports/macports.conf 271 | awk '{if(match($0,"configureccache")) print "configureccache yes" ; else print }' "$CONF" > $$.tmp 272 | sudo mv -f $$.tmp "$CONF" 273 | 274 | # notice that cache size is set to 512Mb, same as it is set by Travis-CI on linux 275 | # might be changed in the future 276 | test -f "$HOME"/.macports-ci-ccache/ccache.conf && 277 | sudo rm -fr "$MACPORTS_PREFIX"/var/macports/build/.ccache && 278 | sudo mkdir -p "$MACPORTS_PREFIX"/var/macports/build/.ccache && 279 | sudo cp -a "$HOME"/.macports-ci-ccache/* "$MACPORTS_PREFIX"/var/macports/build/.ccache/ && 280 | sudo echo "max_size = 512M" > "$MACPORTS_PREFIX"/var/macports/build/.ccache/ccache.conf && 281 | sudo chown -R macports:admin "$MACPORTS_PREFIX"/var/macports/build/.ccache 282 | 283 | ;; 284 | (save) 285 | 286 | sudo rm -fr "$HOME"/.macports-ci-ccache 287 | sudo mkdir -p "$HOME"/.macports-ci-ccache 288 | sudo cp -a "$MACPORTS_PREFIX"/var/macports/build/.ccache/* "$HOME"/.macports-ci-ccache/ 289 | 290 | esac 291 | 292 | CCACHE_DIR="$MACPORTS_PREFIX"/var/macports/build/.ccache/ ccache -s 293 | 294 | ;; 295 | 296 | (*) 297 | echo "macports-ci: unknown action $action" 298 | 299 | esac 300 | 301 | ) 302 | 303 | # allows setting env var if necessary: 304 | source "$MACPORTS_CI_SOURCEME" 305 | -------------------------------------------------------------------------------- /etc/ci/macports-ci.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: macports-ci 2 | name: macports-ci 3 | version: c9676e67351a3a519e37437e196cd0ee9c2180b8 4 | download_url: https://raw.githubusercontent.com/GiovanniBussi/macports-ci/c9676e67351a3a519e37437e196cd0ee9c2180b8/macports-ci 5 | description: Simplify MacPorts setup on Travis-CI 6 | homepage_url: https://github.com/GiovanniBussi/macports-ci 7 | license_expression: mit 8 | copyright: Copyright (c) Giovanni Bussi 9 | attribute: yes 10 | checksum_md5: 5d31d479132502f80acdaed78bed9e23 11 | checksum_sha1: 74b15643bd1a528d91b4a7c2169c6fc656f549c2 12 | package_url: pkg:github/giovannibussi/macports-ci@c9676e67351a3a519e37437e196cd0ee9c2180b8#macports-ci 13 | licenses: 14 | - key: mit 15 | name: MIT License 16 | file: mit.LICENSE 17 | -------------------------------------------------------------------------------- /etc/ci/mit.LICENSE: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 2 | 3 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 4 | 5 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /etc/scripts/README.rst: -------------------------------------------------------------------------------- 1 | This directory contains the tools to manage a directory of thirdparty Python 2 | package source, wheels and metadata pin, build, update, document and publish to 3 | a PyPI-like repo (GitHub release). 4 | 5 | NOTE: These are tested to run ONLY on Linux. 6 | 7 | 8 | Thirdparty packages management scripts 9 | ====================================== 10 | 11 | Pre-requisites 12 | -------------- 13 | 14 | * There are two run "modes": 15 | 16 | * To generate or update pip requirement files, you need to start with a clean 17 | virtualenv as instructed below (This is to avoid injecting requirements 18 | specific to the tools used here in the main requirements). 19 | 20 | * For other usages, the tools here can run either in their own isolated 21 | virtualenv or in the the main configured development virtualenv. 22 | These requireements need to be installed:: 23 | 24 | pip install --requirement etc/scripts/requirements.txt 25 | 26 | TODO: we need to pin the versions of these tools 27 | 28 | 29 | 30 | Generate or update pip requirement files 31 | ---------------------------------------- 32 | 33 | Scripts 34 | ~~~~~~~ 35 | 36 | **gen_requirements.py**: create/update requirements files from currently 37 | installed requirements. 38 | 39 | **gen_requirements_dev.py** does the same but can subtract the main requirements 40 | to get extra requirements used in only development. 41 | 42 | 43 | Usage 44 | ~~~~~ 45 | 46 | The sequence of commands to run are: 47 | 48 | 49 | * Start with these to generate the main pip requirements file:: 50 | 51 | ./configure --clean 52 | ./configure 53 | python etc/scripts/gen_requirements.py --site-packages-dir 54 | 55 | * You can optionally install or update extra main requirements after the 56 | ./configure step such that these are included in the generated main requirements. 57 | 58 | * Optionally, generate a development pip requirements file by running these:: 59 | 60 | ./configure --clean 61 | ./configure --dev 62 | python etc/scripts/gen_requirements_dev.py --site-packages-dir 63 | 64 | * You can optionally install or update extra dev requirements after the 65 | ./configure step such that these are included in the generated dev 66 | requirements. 67 | 68 | Notes: we generate development requirements after the main as this step requires 69 | the main requirements.txt to be up-to-date first. See **gen_requirements.py and 70 | gen_requirements_dev.py** --help for details. 71 | 72 | Note: this does NOT hash requirements for now. 73 | 74 | Note: Be aware that if you are using "conditional" requirements (e.g. only for 75 | OS or Python versions) in setup.py/setp.cfg/requirements.txt as these are NOT 76 | yet supported. 77 | 78 | 79 | Populate a thirdparty directory with wheels, sources, .ABOUT and license files 80 | ------------------------------------------------------------------------------ 81 | 82 | Scripts 83 | ~~~~~~~ 84 | 85 | * **fetch_thirdparty.py** will fetch package wheels, source sdist tarballs 86 | and their ABOUT, LICENSE and NOTICE files to populate a local directory from 87 | a list of PyPI simple URLs (typically PyPI.org proper and our self-hosted PyPI) 88 | using pip requirements file(s), specifiers or pre-existing packages files. 89 | Fetch wheels for specific python version and operating system combinations. 90 | 91 | * **check_thirdparty.py** will check a thirdparty directory for errors. 92 | 93 | 94 | Upgrade virtualenv app 95 | ---------------------- 96 | 97 | The bundled virtualenv.pyz has to be upgraded by hand and is stored under 98 | etc/thirdparty 99 | 100 | * Fetch https://github.com/pypa/get-virtualenv/raw//public/virtualenv.pyz 101 | for instance https://github.com/pypa/get-virtualenv/raw/20.2.2/public/virtualenv.pyz 102 | and save to thirdparty and update the ABOUT and LICENSE files as needed. 103 | 104 | * This virtualenv app contains also bundled pip, wheel and setuptools that are 105 | essential for the installation to work. 106 | 107 | 108 | Other files 109 | =========== 110 | 111 | The other files and scripts are test, support and utility modules used by the 112 | main scripts documented here. 113 | -------------------------------------------------------------------------------- /etc/scripts/check_thirdparty.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) nexB Inc. and others. All rights reserved. 5 | # ScanCode is a trademark of nexB Inc. 6 | # SPDX-License-Identifier: Apache-2.0 7 | # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 8 | # See https://github.com/aboutcode-org/skeleton for support or download. 9 | # See https://aboutcode.org for more information about nexB OSS projects. 10 | # 11 | import click 12 | 13 | import utils_thirdparty 14 | 15 | 16 | @click.command() 17 | @click.option( 18 | "-d", 19 | "--dest", 20 | type=click.Path(exists=True, readable=True, 21 | path_type=str, file_okay=False), 22 | required=True, 23 | help="Path to the thirdparty directory to check.", 24 | ) 25 | @click.option( 26 | "-w", 27 | "--wheels", 28 | is_flag=True, 29 | help="Check missing wheels.", 30 | ) 31 | @click.option( 32 | "-s", 33 | "--sdists", 34 | is_flag=True, 35 | help="Check missing source sdists tarballs.", 36 | ) 37 | @click.help_option("-h", "--help") 38 | def check_thirdparty_dir( 39 | dest, 40 | wheels, 41 | sdists, 42 | ): 43 | """ 44 | Check a thirdparty directory for problems and print these on screen. 45 | """ 46 | # check for problems 47 | print(f"==> CHECK FOR PROBLEMS") 48 | utils_thirdparty.find_problems( 49 | dest_dir=dest, 50 | report_missing_sources=sdists, 51 | report_missing_wheels=wheels, 52 | ) 53 | 54 | 55 | if __name__ == "__main__": 56 | check_thirdparty_dir() 57 | -------------------------------------------------------------------------------- /etc/scripts/fetch_thirdparty.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) nexB Inc. and others. All rights reserved. 5 | # ScanCode is a trademark of nexB Inc. 6 | # SPDX-License-Identifier: Apache-2.0 7 | # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 8 | # See https://github.com/aboutcode-org/skeleton for support or download. 9 | # See https://aboutcode.org for more information about nexB OSS projects. 10 | # 11 | 12 | import itertools 13 | import os 14 | import sys 15 | from collections import defaultdict 16 | 17 | import click 18 | 19 | import utils_thirdparty 20 | import utils_requirements 21 | 22 | TRACE = False 23 | TRACE_DEEP = False 24 | 25 | 26 | @click.command() 27 | @click.option( 28 | "-r", 29 | "--requirements", 30 | "requirements_files", 31 | type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), 32 | metavar="REQUIREMENT-FILE", 33 | multiple=True, 34 | required=False, 35 | help="Path to pip requirements file(s) listing thirdparty packages.", 36 | ) 37 | @click.option( 38 | "--spec", 39 | "--specifier", 40 | "specifiers", 41 | type=str, 42 | metavar="SPECIFIER", 43 | multiple=True, 44 | required=False, 45 | help="Thirdparty package name==version specification(s) as in django==1.2.3. " 46 | "With --latest-version a plain package name is also acceptable.", 47 | ) 48 | @click.option( 49 | "-l", 50 | "--latest-version", 51 | is_flag=True, 52 | help="Get the latest version of all packages, ignoring any specified versions.", 53 | ) 54 | @click.option( 55 | "-d", 56 | "--dest", 57 | "dest_dir", 58 | type=click.Path(exists=True, readable=True, 59 | path_type=str, file_okay=False), 60 | metavar="DIR", 61 | default=utils_thirdparty.THIRDPARTY_DIR, 62 | show_default=True, 63 | help="Path to the detsination directory where to save downloaded wheels, " 64 | "sources, ABOUT and LICENSE files..", 65 | ) 66 | @click.option( 67 | "-w", 68 | "--wheels", 69 | is_flag=True, 70 | help="Download wheels.", 71 | ) 72 | @click.option( 73 | "-s", 74 | "--sdists", 75 | is_flag=True, 76 | help="Download source sdists tarballs.", 77 | ) 78 | @click.option( 79 | "-p", 80 | "--python-version", 81 | "python_versions", 82 | type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), 83 | metavar="PYVER", 84 | default=utils_thirdparty.PYTHON_VERSIONS, 85 | show_default=True, 86 | multiple=True, 87 | help="Python version(s) to use for wheels.", 88 | ) 89 | @click.option( 90 | "-o", 91 | "--operating-system", 92 | "operating_systems", 93 | type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), 94 | metavar="OS", 95 | default=tuple(utils_thirdparty.PLATFORMS_BY_OS), 96 | multiple=True, 97 | show_default=True, 98 | help="OS(ses) to use for wheels: one of linux, mac or windows.", 99 | ) 100 | @click.option( 101 | "--index-url", 102 | "index_urls", 103 | type=str, 104 | metavar="INDEX", 105 | default=utils_thirdparty.PYPI_INDEX_URLS, 106 | show_default=True, 107 | multiple=True, 108 | help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", 109 | ) 110 | @click.option( 111 | "--use-cached-index", 112 | is_flag=True, 113 | help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", 114 | ) 115 | @click.option( 116 | "--sdist-only", 117 | "sdist_only", 118 | type=str, 119 | metavar="SDIST", 120 | default=tuple(), 121 | show_default=False, 122 | multiple=True, 123 | help="Package name(s) that come only in sdist format (no wheels). " 124 | "The command will not fail and exit if no wheel exists for these names", 125 | ) 126 | @click.option( 127 | "--wheel-only", 128 | "wheel_only", 129 | type=str, 130 | metavar="WHEEL", 131 | default=tuple(), 132 | show_default=False, 133 | multiple=True, 134 | help="Package name(s) that come only in wheel format (no sdist). " 135 | "The command will not fail and exit if no sdist exists for these names", 136 | ) 137 | @click.option( 138 | "--no-dist", 139 | "no_dist", 140 | type=str, 141 | metavar="DIST", 142 | default=tuple(), 143 | show_default=False, 144 | multiple=True, 145 | help="Package name(s) that do not come either in wheel or sdist format. " 146 | "The command will not fail and exit if no distribution exists for these names", 147 | ) 148 | @click.help_option("-h", "--help") 149 | def fetch_thirdparty( 150 | requirements_files, 151 | specifiers, 152 | latest_version, 153 | dest_dir, 154 | python_versions, 155 | operating_systems, 156 | wheels, 157 | sdists, 158 | index_urls, 159 | use_cached_index, 160 | sdist_only, 161 | wheel_only, 162 | no_dist, 163 | ): 164 | """ 165 | Download to --dest THIRDPARTY_DIR the PyPI wheels, source distributions, 166 | and their ABOUT metadata, license and notices files. 167 | 168 | Download the PyPI packages listed in the combination of: 169 | - the pip requirements --requirements REQUIREMENT-FILE(s), 170 | - the pip name==version --specifier SPECIFIER(s) 171 | - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. 172 | 173 | Download wheels with the --wheels option for the ``--python-version`` 174 | PYVER(s) and ``--operating_system`` OS(s) combinations defaulting to all 175 | supported combinations. 176 | 177 | Download sdists tarballs with the --sdists option. 178 | 179 | Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels 180 | and sources fetched. 181 | 182 | Download from the provided PyPI simple --index-url INDEX(s) URLs. 183 | """ 184 | if not (wheels or sdists): 185 | print("Error: one or both of --wheels and --sdists is required.") 186 | sys.exit(1) 187 | 188 | print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") 189 | 190 | existing_packages_by_nv = { 191 | (package.name, package.version): package 192 | for package in utils_thirdparty.get_local_packages(directory=dest_dir) 193 | } 194 | 195 | required_name_versions = set(existing_packages_by_nv.keys()) 196 | 197 | for req_file in requirements_files: 198 | nvs = utils_requirements.load_requirements( 199 | requirements_file=req_file, 200 | with_unpinned=latest_version, 201 | ) 202 | required_name_versions.update(nvs) 203 | 204 | for specifier in specifiers: 205 | nv = utils_requirements.get_required_name_version( 206 | requirement=specifier, 207 | with_unpinned=latest_version, 208 | ) 209 | required_name_versions.add(nv) 210 | 211 | if latest_version: 212 | names = set(name for name, _version in sorted(required_name_versions)) 213 | required_name_versions = {(n, None) for n in names} 214 | 215 | if not required_name_versions: 216 | print("Error: no requirements requested.") 217 | sys.exit(1) 218 | 219 | if TRACE_DEEP: 220 | print("required_name_versions:") 221 | for n, v in required_name_versions: 222 | print(f" {n} @ {v}") 223 | 224 | # create the environments matrix we need for wheels 225 | environments = None 226 | if wheels: 227 | evts = itertools.product(python_versions, operating_systems) 228 | environments = [utils_thirdparty.Environment.from_pyver_and_os( 229 | pyv, os) for pyv, os in evts] 230 | 231 | # Collect PyPI repos 232 | repos = [] 233 | for index_url in index_urls: 234 | index_url = index_url.strip("/") 235 | existing = utils_thirdparty.DEFAULT_PYPI_REPOS_BY_URL.get(index_url) 236 | if existing: 237 | existing.use_cached_index = use_cached_index 238 | repos.append(existing) 239 | else: 240 | repo = utils_thirdparty.PypiSimpleRepository( 241 | index_url=index_url, 242 | use_cached_index=use_cached_index, 243 | ) 244 | repos.append(repo) 245 | 246 | wheels_or_sdist_not_found = defaultdict(list) 247 | 248 | for name, version in sorted(required_name_versions): 249 | nv = name, version 250 | print(f"Processing: {name} @ {version}") 251 | if wheels: 252 | for environment in environments: 253 | 254 | if TRACE: 255 | print(f" ==> Fetching wheel for envt: {environment}") 256 | 257 | fetched = utils_thirdparty.download_wheel( 258 | name=name, 259 | version=version, 260 | environment=environment, 261 | dest_dir=dest_dir, 262 | repos=repos, 263 | ) 264 | if not fetched: 265 | wheels_or_sdist_not_found[f"{name}=={version}"].append( 266 | environment) 267 | if TRACE: 268 | print(f" NOT FOUND") 269 | 270 | if (sdists or 271 | (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) 272 | ): 273 | if TRACE: 274 | print(f" ==> Fetching sdist: {name}=={version}") 275 | 276 | fetched = utils_thirdparty.download_sdist( 277 | name=name, 278 | version=version, 279 | dest_dir=dest_dir, 280 | repos=repos, 281 | ) 282 | if not fetched: 283 | wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") 284 | if TRACE: 285 | print(f" NOT FOUND") 286 | 287 | mia = [] 288 | for nv, dists in wheels_or_sdist_not_found.items(): 289 | name, _, version = nv.partition("==") 290 | if name in no_dist: 291 | continue 292 | sdist_missing = sdists and "sdist" in dists and not name in wheel_only 293 | if sdist_missing: 294 | mia.append(f"SDist missing: {nv} {dists}") 295 | wheels_missing = wheels and any( 296 | d for d in dists if d != "sdist") and not name in sdist_only 297 | if wheels_missing: 298 | mia.append(f"Wheels missing: {nv} {dists}") 299 | 300 | if mia: 301 | for m in mia: 302 | print(m) 303 | raise Exception(mia) 304 | 305 | print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") 306 | utils_thirdparty.fetch_abouts_and_licenses( 307 | dest_dir=dest_dir, use_cached_index=use_cached_index) 308 | utils_thirdparty.clean_about_files(dest_dir=dest_dir) 309 | 310 | # check for problems 311 | print(f"==> CHECK FOR PROBLEMS") 312 | utils_thirdparty.find_problems( 313 | dest_dir=dest_dir, 314 | report_missing_sources=sdists, 315 | report_missing_wheels=wheels, 316 | ) 317 | 318 | 319 | if __name__ == "__main__": 320 | fetch_thirdparty() 321 | -------------------------------------------------------------------------------- /etc/scripts/gen_pypi_simple.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # SPDX-License-Identifier: BSD-2-Clause-Views AND MIT 5 | # Copyright (c) 2010 David Wolever . All rights reserved. 6 | # originally from https://github.com/wolever/pip2pi 7 | 8 | import hashlib 9 | import os 10 | import re 11 | import shutil 12 | from collections import defaultdict 13 | from html import escape 14 | from pathlib import Path 15 | from typing import NamedTuple 16 | 17 | """ 18 | Generate a PyPI simple index froma directory. 19 | """ 20 | 21 | 22 | class InvalidDistributionFilename(Exception): 23 | pass 24 | 25 | 26 | def get_package_name_from_filename(filename): 27 | """ 28 | Return the normalized package name extracted from a package ``filename``. 29 | Normalization is done according to distribution name rules. 30 | Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: 31 | 32 | >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") 33 | 'foo' 34 | >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl") 35 | 'foo-bar' 36 | >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") 37 | 'cython' 38 | >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") 39 | 'python-ldap' 40 | >>> try: 41 | ... get_package_name_from_filename("foo.whl") 42 | ... except InvalidDistributionFilename: 43 | ... pass 44 | >>> try: 45 | ... get_package_name_from_filename("foo.png") 46 | ... except InvalidDistributionFilename: 47 | ... pass 48 | """ 49 | if not filename or not filename.endswith(dist_exts): 50 | raise InvalidDistributionFilename(filename) 51 | 52 | filename = os.path.basename(filename) 53 | 54 | if filename.endswith(sdist_exts): 55 | name_ver = None 56 | extension = None 57 | 58 | for ext in sdist_exts: 59 | if filename.endswith(ext): 60 | name_ver, extension, _ = filename.rpartition(ext) 61 | break 62 | 63 | if not extension or not name_ver: 64 | raise InvalidDistributionFilename(filename) 65 | 66 | name, _, version = name_ver.rpartition("-") 67 | 68 | if not (name and version): 69 | raise InvalidDistributionFilename(filename) 70 | 71 | elif filename.endswith(wheel_ext): 72 | 73 | wheel_info = get_wheel_from_filename(filename) 74 | 75 | if not wheel_info: 76 | raise InvalidDistributionFilename(filename) 77 | 78 | name = wheel_info.group("name") 79 | version = wheel_info.group("version") 80 | 81 | if not (name and version): 82 | raise InvalidDistributionFilename(filename) 83 | 84 | elif filename.endswith(app_ext): 85 | name_ver, extension, _ = filename.rpartition(".pyz") 86 | 87 | if "-" in filename: 88 | name, _, version = name_ver.rpartition("-") 89 | else: 90 | name = name_ver 91 | 92 | if not name: 93 | raise InvalidDistributionFilename(filename) 94 | 95 | name = normalize_name(name) 96 | return name 97 | 98 | 99 | def normalize_name(name): 100 | """ 101 | Return a normalized package name per PEP503, and copied from 102 | https://www.python.org/dev/peps/pep-0503/#id4 103 | """ 104 | return name and re.sub(r"[-_.]+", "-", name).lower() or name 105 | 106 | 107 | def build_per_package_index(pkg_name, packages, base_url): 108 | """ 109 | Return an HTML document as string representing the index for a package 110 | """ 111 | document = [] 112 | header = f""" 113 | 114 | 115 | 116 | Links for {pkg_name} 117 | 118 | """ 119 | document.append(header) 120 | 121 | for package in sorted(packages, key=lambda p: p.archive_file): 122 | document.append(package.simple_index_entry(base_url)) 123 | 124 | footer = """ 125 | 126 | """ 127 | document.append(footer) 128 | return "\n".join(document) 129 | 130 | 131 | def build_links_package_index(packages_by_package_name, base_url): 132 | """ 133 | Return an HTML document as string which is a links index of all packages 134 | """ 135 | document = [] 136 | header = f""" 137 | 138 | 139 | Links for all packages 140 | 141 | """ 142 | document.append(header) 143 | 144 | for _name, packages in sorted(packages_by_package_name.items(), key=lambda i: i[0]): 145 | for package in sorted(packages, key=lambda p: p.archive_file): 146 | document.append(package.simple_index_entry(base_url)) 147 | 148 | footer = """ 149 | 150 | """ 151 | document.append(footer) 152 | return "\n".join(document) 153 | 154 | 155 | class Package(NamedTuple): 156 | name: str 157 | index_dir: Path 158 | archive_file: Path 159 | checksum: str 160 | 161 | @classmethod 162 | def from_file(cls, name, index_dir, archive_file): 163 | with open(archive_file, "rb") as f: 164 | checksum = hashlib.sha256(f.read()).hexdigest() 165 | return cls( 166 | name=name, 167 | index_dir=index_dir, 168 | archive_file=archive_file, 169 | checksum=checksum, 170 | ) 171 | 172 | def simple_index_entry(self, base_url): 173 | return ( 174 | f' ' 175 | f"{self.archive_file.name}
" 176 | ) 177 | 178 | 179 | def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): 180 | """ 181 | Using a ``directory`` directory of wheels and sdists, create the a PyPI 182 | simple directory index at ``directory``/simple/ populated with the proper 183 | PyPI simple index directory structure crafted using symlinks. 184 | 185 | WARNING: The ``directory``/simple/ directory is removed if it exists. 186 | NOTE: in addition to the a PyPI simple index.html there is also a links.html 187 | index file generated which is suitable to use with pip's --find-links 188 | """ 189 | 190 | directory = Path(directory) 191 | 192 | index_dir = directory / "simple" 193 | if index_dir.exists(): 194 | shutil.rmtree(str(index_dir), ignore_errors=True) 195 | 196 | index_dir.mkdir(parents=True) 197 | packages_by_package_name = defaultdict(list) 198 | 199 | # generate the main simple index.html 200 | simple_html_index = [ 201 | "", 202 | "PyPI Simple Index", 203 | '' '', 204 | ] 205 | 206 | for pkg_file in directory.iterdir(): 207 | 208 | pkg_filename = pkg_file.name 209 | 210 | if ( 211 | not pkg_file.is_file() 212 | or not pkg_filename.endswith(dist_exts) 213 | or pkg_filename.startswith(".") 214 | ): 215 | continue 216 | 217 | pkg_name = get_package_name_from_filename( 218 | filename=pkg_filename, 219 | ) 220 | pkg_index_dir = index_dir / pkg_name 221 | pkg_index_dir.mkdir(parents=True, exist_ok=True) 222 | pkg_indexed_file = pkg_index_dir / pkg_filename 223 | 224 | link_target = Path("../..") / pkg_filename 225 | pkg_indexed_file.symlink_to(link_target) 226 | 227 | if pkg_name not in packages_by_package_name: 228 | esc_name = escape(pkg_name) 229 | simple_html_index.append(f'{esc_name}
') 230 | 231 | packages_by_package_name[pkg_name].append( 232 | Package.from_file( 233 | name=pkg_name, 234 | index_dir=pkg_index_dir, 235 | archive_file=pkg_file, 236 | ) 237 | ) 238 | 239 | # finalize main index 240 | simple_html_index.append("") 241 | index_html = index_dir / "index.html" 242 | index_html.write_text("\n".join(simple_html_index)) 243 | 244 | # also generate the simple index.html of each package, listing all its versions. 245 | for pkg_name, packages in packages_by_package_name.items(): 246 | per_package_index = build_per_package_index( 247 | pkg_name=pkg_name, 248 | packages=packages, 249 | base_url=base_url, 250 | ) 251 | pkg_index_dir = packages[0].index_dir 252 | ppi_html = pkg_index_dir / "index.html" 253 | ppi_html.write_text(per_package_index) 254 | 255 | # also generate the a links.html page with all packages. 256 | package_links = build_links_package_index( 257 | packages_by_package_name=packages_by_package_name, 258 | base_url=base_url, 259 | ) 260 | links_html = index_dir / "links.html" 261 | links_html.write_text(package_links) 262 | 263 | 264 | """ 265 | name: pip-wheel 266 | version: 20.3.1 267 | download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py 268 | copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) 269 | license_expression: mit 270 | notes: the wheel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py 271 | 272 | Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) 273 | 274 | Permission is hereby granted, free of charge, to any person obtaining 275 | a copy of this software and associated documentation files (the 276 | "Software"), to deal in the Software without restriction, including 277 | without limitation the rights to use, copy, modify, merge, publish, 278 | distribute, sublicense, and/or sell copies of the Software, and to 279 | permit persons to whom the Software is furnished to do so, subject to 280 | the following conditions: 281 | 282 | The above copyright notice and this permission notice shall be 283 | included in all copies or substantial portions of the Software. 284 | 285 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 286 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 287 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 288 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 289 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 290 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 291 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 292 | """ 293 | get_wheel_from_filename = re.compile( 294 | r"""^(?P(?P.+?)-(?P.*?)) 295 | ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) 296 | \.whl)$""", 297 | re.VERBOSE, 298 | ).match 299 | 300 | sdist_exts = ( 301 | ".tar.gz", 302 | ".tar.bz2", 303 | ".zip", 304 | ".tar.xz", 305 | ) 306 | 307 | wheel_ext = ".whl" 308 | app_ext = ".pyz" 309 | dist_exts = sdist_exts + (wheel_ext, app_ext) 310 | 311 | if __name__ == "__main__": 312 | import sys 313 | 314 | pkg_dir = sys.argv[1] 315 | build_pypi_index(pkg_dir) 316 | -------------------------------------------------------------------------------- /etc/scripts/gen_pypi_simple.py.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: gen_pypi_simple.py 2 | name: gen_pypi_simple.py 3 | license_expression: bsd-2-clause-views and mit 4 | copyright: Copyright (c) nexB Inc. 5 | Copyright (c) 2010 David Wolever 6 | Copyright (c) The pip developers 7 | notes: Originally from https://github.com/wolever/pip2pi and modified extensivley 8 | Also partially derived from pip code 9 | -------------------------------------------------------------------------------- /etc/scripts/gen_pypi_simple.py.NOTICE: -------------------------------------------------------------------------------- 1 | SPDX-License-Identifier: BSD-2-Clause-Views AND mit 2 | 3 | Copyright (c) nexB Inc. 4 | Copyright (c) 2010 David Wolever 5 | Copyright (c) The pip developers 6 | 7 | 8 | Original code: copyright 2010 David Wolever . All rights reserved. 9 | 10 | Redistribution and use in source and binary forms, with or without 11 | modification, are permitted provided that the following conditions are met: 12 | 13 | 1. Redistributions of source code must retain the above copyright notice, 14 | this list of conditions and the following disclaimer. 15 | 16 | 2. Redistributions in binary form must reproduce the above copyright notice, 17 | this list of conditions and the following disclaimer in the documentation 18 | and/or other materials provided with the distribution. 19 | 20 | THIS SOFTWARE IS PROVIDED BY ``AS IS'' AND ANY EXPRESS OR 21 | IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 22 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO 23 | EVENT SHALL OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 24 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 25 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 27 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 28 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 29 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | The views and conclusions contained in the software and documentation are those 32 | of the authors and should not be interpreted as representing official policies, 33 | either expressed or implied, of David Wolever. 34 | 35 | 36 | Original code: Copyright (c) 2008-2020 The pip developers 37 | 38 | Permission is hereby granted, free of charge, to any person obtaining 39 | a copy of this software and associated documentation files (the 40 | "Software"), to deal in the Software without restriction, including 41 | without limitation the rights to use, copy, modify, merge, publish, 42 | distribute, sublicense, and/or sell copies of the Software, and to 43 | permit persons to whom the Software is furnished to do so, subject to 44 | the following conditions: 45 | 46 | The above copyright notice and this permission notice shall be 47 | included in all copies or substantial portions of the Software. 48 | 49 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 50 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 51 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 52 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 53 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 54 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 55 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 56 | 57 | -------------------------------------------------------------------------------- /etc/scripts/gen_requirements.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) nexB Inc. and others. All rights reserved. 5 | # ScanCode is a trademark of nexB Inc. 6 | # SPDX-License-Identifier: Apache-2.0 7 | # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 8 | # See https://github.com/aboutcode-org/skeleton for support or download. 9 | # See https://aboutcode.org for more information about nexB OSS projects. 10 | # 11 | import argparse 12 | import pathlib 13 | 14 | import utils_requirements 15 | 16 | """ 17 | Utilities to manage requirements files. 18 | NOTE: this should use ONLY the standard library and not import anything else 19 | because this is used for boostrapping with no requirements installed. 20 | """ 21 | 22 | 23 | def gen_requirements(): 24 | description = """ 25 | Create or replace the `--requirements-file` file FILE requirements file with all 26 | locally installed Python packages.all Python packages found installed in `--site-packages-dir` 27 | """ 28 | parser = argparse.ArgumentParser(description=description) 29 | 30 | parser.add_argument( 31 | "-s", 32 | "--site-packages-dir", 33 | dest="site_packages_dir", 34 | type=pathlib.Path, 35 | required=True, 36 | metavar="DIR", 37 | help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", 38 | ) 39 | parser.add_argument( 40 | "-r", 41 | "--requirements-file", 42 | type=pathlib.Path, 43 | metavar="FILE", 44 | default="requirements.txt", 45 | help="Path to the requirements file to update or create.", 46 | ) 47 | 48 | args = parser.parse_args() 49 | 50 | utils_requirements.lock_requirements( 51 | site_packages_dir=args.site_packages_dir, 52 | requirements_file=args.requirements_file, 53 | ) 54 | 55 | 56 | if __name__ == "__main__": 57 | gen_requirements() 58 | -------------------------------------------------------------------------------- /etc/scripts/gen_requirements_dev.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) nexB Inc. and others. All rights reserved. 5 | # ScanCode is a trademark of nexB Inc. 6 | # SPDX-License-Identifier: Apache-2.0 7 | # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 8 | # See https://github.com/aboutcode-org/skeleton for support or download. 9 | # See https://aboutcode.org for more information about nexB OSS projects. 10 | # 11 | import argparse 12 | import pathlib 13 | 14 | import utils_requirements 15 | 16 | """ 17 | Utilities to manage requirements files. 18 | NOTE: this should use ONLY the standard library and not import anything else 19 | because this is used for boostrapping with no requirements installed. 20 | """ 21 | 22 | 23 | def gen_dev_requirements(): 24 | description = """ 25 | Create or overwrite the `--dev-requirements-file` pip requirements FILE with 26 | all Python packages found installed in `--site-packages-dir`. Exclude 27 | package names also listed in the --main-requirements-file pip requirements 28 | FILE (that are assume to the production requirements and therefore to always 29 | be present in addition to the development requirements). 30 | """ 31 | parser = argparse.ArgumentParser(description=description) 32 | 33 | parser.add_argument( 34 | "-s", 35 | "--site-packages-dir", 36 | type=pathlib.Path, 37 | required=True, 38 | metavar="DIR", 39 | help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', 40 | ) 41 | parser.add_argument( 42 | "-d", 43 | "--dev-requirements-file", 44 | type=pathlib.Path, 45 | metavar="FILE", 46 | default="requirements-dev.txt", 47 | help="Path to the dev requirements file to update or create.", 48 | ) 49 | parser.add_argument( 50 | "-r", 51 | "--main-requirements-file", 52 | type=pathlib.Path, 53 | default="requirements.txt", 54 | metavar="FILE", 55 | help="Path to the main requirements file. Its requirements will be excluded " 56 | "from the generated dev requirements.", 57 | ) 58 | args = parser.parse_args() 59 | 60 | utils_requirements.lock_dev_requirements( 61 | dev_requirements_file=args.dev_requirements_file, 62 | main_requirements_file=args.main_requirements_file, 63 | site_packages_dir=args.site_packages_dir, 64 | ) 65 | 66 | 67 | if __name__ == "__main__": 68 | gen_dev_requirements() 69 | -------------------------------------------------------------------------------- /etc/scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | aboutcode_toolkit 2 | attrs 3 | commoncode 4 | click 5 | requests 6 | saneyaml 7 | pip 8 | setuptools 9 | twine 10 | wheel 11 | build 12 | packvers 13 | -------------------------------------------------------------------------------- /etc/scripts/test_utils_pip_compatibility_tags.py: -------------------------------------------------------------------------------- 1 | """Generate and work with PEP 425 Compatibility Tags. 2 | 3 | copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py 4 | download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py 5 | 6 | Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining 9 | a copy of this software and associated documentation files (the 10 | "Software"), to deal in the Software without restriction, including 11 | without limitation the rights to use, copy, modify, merge, publish, 12 | distribute, sublicense, and/or sell copies of the Software, and to 13 | permit persons to whom the Software is furnished to do so, subject to 14 | the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be 17 | included in all copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 23 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 24 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | """ 27 | 28 | from unittest.mock import patch 29 | import sysconfig 30 | 31 | import pytest 32 | 33 | import utils_pip_compatibility_tags 34 | 35 | 36 | @pytest.mark.parametrize( 37 | "version_info, expected", 38 | [ 39 | ((2,), "2"), 40 | ((2, 8), "28"), 41 | ((3,), "3"), 42 | ((3, 6), "36"), 43 | # Test a tuple of length 3. 44 | ((3, 6, 5), "36"), 45 | # Test a 2-digit minor version. 46 | ((3, 10), "310"), 47 | ], 48 | ) 49 | def test_version_info_to_nodot(version_info, expected): 50 | actual = utils_pip_compatibility_tags.version_info_to_nodot(version_info) 51 | assert actual == expected 52 | 53 | 54 | class Testcompatibility_tags(object): 55 | def mock_get_config_var(self, **kwd): 56 | """ 57 | Patch sysconfig.get_config_var for arbitrary keys. 58 | """ 59 | get_config_var = sysconfig.get_config_var 60 | 61 | def _mock_get_config_var(var): 62 | if var in kwd: 63 | return kwd[var] 64 | return get_config_var(var) 65 | 66 | return _mock_get_config_var 67 | 68 | def test_no_hyphen_tag(self): 69 | """ 70 | Test that no tag contains a hyphen. 71 | """ 72 | import pip._internal.utils.compatibility_tags 73 | 74 | mock_gcf = self.mock_get_config_var(SOABI="cpython-35m-darwin") 75 | 76 | with patch("sysconfig.get_config_var", mock_gcf): 77 | supported = pip._internal.utils.compatibility_tags.get_supported() 78 | 79 | for tag in supported: 80 | assert "-" not in tag.interpreter 81 | assert "-" not in tag.abi 82 | assert "-" not in tag.platform 83 | 84 | 85 | class TestManylinux2010Tags(object): 86 | @pytest.mark.parametrize( 87 | "manylinux2010,manylinux1", 88 | [ 89 | ("manylinux2010_x86_64", "manylinux1_x86_64"), 90 | ("manylinux2010_i686", "manylinux1_i686"), 91 | ], 92 | ) 93 | def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): 94 | """ 95 | Specifying manylinux2010 implies manylinux1. 96 | """ 97 | groups = {} 98 | supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinux2010]) 99 | for tag in supported: 100 | groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) 101 | 102 | for arches in groups.values(): 103 | if arches == ["any"]: 104 | continue 105 | assert arches[:2] == [manylinux2010, manylinux1] 106 | 107 | 108 | class TestManylinux2014Tags(object): 109 | @pytest.mark.parametrize( 110 | "manylinuxA,manylinuxB", 111 | [ 112 | ("manylinux2014_x86_64", ["manylinux2010_x86_64", "manylinux1_x86_64"]), 113 | ("manylinux2014_i686", ["manylinux2010_i686", "manylinux1_i686"]), 114 | ], 115 | ) 116 | def test_manylinuxA_implies_manylinuxB(self, manylinuxA, manylinuxB): 117 | """ 118 | Specifying manylinux2014 implies manylinux2010/manylinux1. 119 | """ 120 | groups = {} 121 | supported = utils_pip_compatibility_tags.get_supported(platforms=[manylinuxA]) 122 | for tag in supported: 123 | groups.setdefault((tag.interpreter, tag.abi), []).append(tag.platform) 124 | 125 | expected_arches = [manylinuxA] 126 | expected_arches.extend(manylinuxB) 127 | for arches in groups.values(): 128 | if arches == ["any"]: 129 | continue 130 | assert arches[:3] == expected_arches 131 | -------------------------------------------------------------------------------- /etc/scripts/test_utils_pip_compatibility_tags.py.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: test_utils_pip_compatibility_tags.py 2 | 3 | type: github 4 | namespace: pypa 5 | name: pip 6 | version: 20.3.1 7 | subpath: tests/unit/test_utils_compatibility_tags.py 8 | 9 | package_url: pkg:github/pypa/pip@20.3.1#tests/unit/test_utils_compatibility_tags.py 10 | 11 | download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py 12 | copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) 13 | license_expression: mit 14 | notes: subset copied from pip for tag handling 15 | -------------------------------------------------------------------------------- /etc/scripts/test_utils_pypi_supported_tags.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import pytest 14 | 15 | from utils_pypi_supported_tags import validate_platforms_for_pypi 16 | 17 | """ 18 | Wheel platform checking tests 19 | 20 | Copied and modified on 2020-12-24 from 21 | https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/tests/unit/forklift/test_legacy.py 22 | """ 23 | 24 | 25 | def validate_wheel_filename_for_pypi(filename): 26 | """ 27 | Validate if the filename is a PyPI/warehouse-uploadable wheel file name 28 | with supported platform tags. Return a list of unsupported platform tags or 29 | an empty list if all tags are supported. 30 | """ 31 | from utils_thirdparty import Wheel 32 | 33 | wheel = Wheel.from_filename(filename) 34 | return validate_platforms_for_pypi(wheel.platforms) 35 | 36 | 37 | @pytest.mark.parametrize( 38 | "plat", 39 | [ 40 | "any", 41 | "win32", 42 | "win_amd64", 43 | "win_ia64", 44 | "manylinux1_i686", 45 | "manylinux1_x86_64", 46 | "manylinux2010_i686", 47 | "manylinux2010_x86_64", 48 | "manylinux2014_i686", 49 | "manylinux2014_x86_64", 50 | "manylinux2014_aarch64", 51 | "manylinux2014_armv7l", 52 | "manylinux2014_ppc64", 53 | "manylinux2014_ppc64le", 54 | "manylinux2014_s390x", 55 | "manylinux_2_5_i686", 56 | "manylinux_2_12_x86_64", 57 | "manylinux_2_17_aarch64", 58 | "manylinux_2_17_armv7l", 59 | "manylinux_2_17_ppc64", 60 | "manylinux_2_17_ppc64le", 61 | "manylinux_3_0_s390x", 62 | "macosx_10_6_intel", 63 | "macosx_10_13_x86_64", 64 | "macosx_11_0_x86_64", 65 | "macosx_10_15_arm64", 66 | "macosx_11_10_universal2", 67 | # A real tag used by e.g. some numpy wheels 68 | ( 69 | "macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64." 70 | "macosx_10_10_intel.macosx_10_10_x86_64" 71 | ), 72 | ], 73 | ) 74 | def test_is_valid_pypi_wheel_return_true_for_supported_wheel(plat): 75 | filename = f"foo-1.2.3-cp34-none-{plat}.whl" 76 | assert not validate_wheel_filename_for_pypi(filename) 77 | 78 | 79 | @pytest.mark.parametrize( 80 | "plat", 81 | [ 82 | "linux_x86_64", 83 | "linux_x86_64.win32", 84 | "macosx_9_2_x86_64", 85 | "macosx_12_2_arm64", 86 | "macosx_10_15_amd64", 87 | ], 88 | ) 89 | def test_is_valid_pypi_wheel_raise_exception_for_aunsupported_wheel(plat): 90 | filename = f"foo-1.2.3-cp34-none-{plat}.whl" 91 | invalid = validate_wheel_filename_for_pypi(filename) 92 | assert invalid 93 | -------------------------------------------------------------------------------- /etc/scripts/test_utils_pypi_supported_tags.py.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: test_utils_pypi_supported_tags.py 2 | 3 | type: github 4 | namespace: pypa 5 | name: warehouse 6 | version: 37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d 7 | subpath: tests/unit/forklift/test_legacy.py 8 | 9 | package_url: pkg:github/pypa/warehouse@37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d#tests/unit/forklift/test_legacy.py 10 | 11 | download_url: https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/tests/unit/forklift/test_legacy.py 12 | copyright: Copyright (c) The warehouse developers 13 | homepage_url: https://warehouse.readthedocs.io 14 | license_expression: apache-2.0 15 | notes: Test for wheel platform checking copied and heavily modified on 16 | 2020-12-24 from warehouse. This contains the basic functions to check if a 17 | wheel file name is would be supported for uploading to PyPI. 18 | -------------------------------------------------------------------------------- /etc/scripts/utils_dejacode.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) nexB Inc. and others. All rights reserved. 5 | # ScanCode is a trademark of nexB Inc. 6 | # SPDX-License-Identifier: Apache-2.0 7 | # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 8 | # See https://github.com/aboutcode-org/skeleton for support or download. 9 | # See https://aboutcode.org for more information about nexB OSS projects. 10 | # 11 | import io 12 | import os 13 | import zipfile 14 | 15 | import requests 16 | import saneyaml 17 | 18 | from packvers import version as packaging_version 19 | 20 | """ 21 | Utility to create and retrieve package and ABOUT file data from DejaCode. 22 | """ 23 | 24 | DEJACODE_API_KEY = os.environ.get("DEJACODE_API_KEY", "") 25 | DEJACODE_API_URL = os.environ.get("DEJACODE_API_URL", "") 26 | 27 | DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" 28 | DEJACODE_API_HEADERS = { 29 | "Authorization": "Token {}".format(DEJACODE_API_KEY), 30 | "Accept": "application/json; indent=4", 31 | } 32 | 33 | 34 | def can_do_api_calls(): 35 | if not DEJACODE_API_KEY and DEJACODE_API_URL: 36 | print( 37 | "DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") 38 | return False 39 | else: 40 | return True 41 | 42 | 43 | def fetch_dejacode_packages(params): 44 | """ 45 | Return a list of package data mappings calling the package API with using 46 | `params` or an empty list. 47 | """ 48 | if not can_do_api_calls(): 49 | return [] 50 | 51 | response = requests.get( 52 | DEJACODE_API_URL_PACKAGES, 53 | params=params, 54 | headers=DEJACODE_API_HEADERS, 55 | ) 56 | 57 | return response.json()["results"] 58 | 59 | 60 | def get_package_data(distribution): 61 | """ 62 | Return a mapping of package data or None for a Distribution `distribution`. 63 | """ 64 | results = fetch_dejacode_packages(distribution.identifiers()) 65 | 66 | len_results = len(results) 67 | 68 | if len_results == 1: 69 | return results[0] 70 | 71 | elif len_results > 1: 72 | print( 73 | f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") 74 | else: 75 | print("Could not find package:", distribution.download_url) 76 | 77 | 78 | def update_with_dejacode_data(distribution): 79 | """ 80 | Update the Distribution `distribution` with DejaCode package data. Return 81 | True if data was updated. 82 | """ 83 | package_data = get_package_data(distribution) 84 | if package_data: 85 | return distribution.update(package_data, keep_extra=False) 86 | 87 | print(f"No package found for: {distribution}") 88 | 89 | 90 | def update_with_dejacode_about_data(distribution): 91 | """ 92 | Update the Distribution `distribution` wiht ABOUT code data fetched from 93 | DejaCode. Return True if data was updated. 94 | """ 95 | package_data = get_package_data(distribution) 96 | if package_data: 97 | package_api_url = package_data["api_url"] 98 | about_url = f"{package_api_url}about" 99 | response = requests.get(about_url, headers=DEJACODE_API_HEADERS) 100 | # note that this is YAML-formatted 101 | about_text = response.json()["about_data"] 102 | about_data = saneyaml.load(about_text) 103 | 104 | return distribution.update(about_data, keep_extra=True) 105 | 106 | print(f"No package found for: {distribution}") 107 | 108 | 109 | def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): 110 | """ 111 | Fetch and save in `dest_dir` the .ABOUT, .LICENSE and .NOTICE files fetched 112 | from DejaCode for a Distribution `distribution`. Return True if files were 113 | fetched. 114 | """ 115 | package_data = get_package_data(distribution) 116 | if package_data: 117 | package_api_url = package_data["api_url"] 118 | about_url = f"{package_api_url}about_files" 119 | response = requests.get(about_url, headers=DEJACODE_API_HEADERS) 120 | about_zip = response.content 121 | with io.BytesIO(about_zip) as zf: 122 | with zipfile.ZipFile(zf) as zi: 123 | zi.extractall(path=dest_dir) 124 | return True 125 | 126 | print(f"No package found for: {distribution}") 127 | 128 | 129 | def find_latest_dejacode_package(distribution): 130 | """ 131 | Return a mapping of package data for the closest version to 132 | a Distribution `distribution` or None. 133 | Return the newest of the packages if prefer_newest is True. 134 | Filter out version-specific attributes. 135 | """ 136 | ids = distribution.purl_identifiers(skinny=True) 137 | packages = fetch_dejacode_packages(params=ids) 138 | if not packages: 139 | return 140 | 141 | for package_data in packages: 142 | matched = ( 143 | package_data["download_url"] == distribution.download_url 144 | and package_data["version"] == distribution.version 145 | and package_data["filename"] == distribution.filename 146 | ) 147 | 148 | if matched: 149 | return package_data 150 | 151 | # there was no exact match, find the latest version 152 | # TODO: consider the closest version rather than the latest 153 | # or the version that has the best data 154 | with_versions = [(packaging_version.parse(p["version"]), p) 155 | for p in packages] 156 | with_versions = sorted(with_versions) 157 | latest_version, latest_package_version = sorted(with_versions)[-1] 158 | print( 159 | f"Found DejaCode latest version: {latest_version} " f"for dist: {distribution.package_url}", 160 | ) 161 | 162 | return latest_package_version 163 | 164 | 165 | def create_dejacode_package(distribution): 166 | """ 167 | Create a new DejaCode Package a Distribution `distribution`. 168 | Return the new or existing package data. 169 | """ 170 | if not can_do_api_calls(): 171 | return 172 | 173 | existing_package_data = get_package_data(distribution) 174 | if existing_package_data: 175 | return existing_package_data 176 | 177 | print(f"Creating new DejaCode package for: {distribution}") 178 | 179 | new_package_payload = { 180 | # Trigger data collection, scan, and purl 181 | "collect_data": 1, 182 | } 183 | 184 | fields_to_carry_over = [ 185 | "download_url" "type", 186 | "namespace", 187 | "name", 188 | "version", 189 | "qualifiers", 190 | "subpath", 191 | "license_expression", 192 | "copyright", 193 | "description", 194 | "homepage_url", 195 | "primary_language", 196 | "notice_text", 197 | ] 198 | 199 | for field in fields_to_carry_over: 200 | value = getattr(distribution, field, None) 201 | if value: 202 | new_package_payload[field] = value 203 | 204 | response = requests.post( 205 | DEJACODE_API_URL_PACKAGES, 206 | data=new_package_payload, 207 | headers=DEJACODE_API_HEADERS, 208 | ) 209 | new_package_data = response.json() 210 | if response.status_code != 201: 211 | raise Exception(f"Error, cannot create package for: {distribution}") 212 | 213 | print(f'New Package created at: {new_package_data["absolute_url"]}') 214 | return new_package_data 215 | -------------------------------------------------------------------------------- /etc/scripts/utils_pip_compatibility_tags.py: -------------------------------------------------------------------------------- 1 | """Generate and work with PEP 425 Compatibility Tags. 2 | 3 | copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py 4 | download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py 5 | 6 | Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining 9 | a copy of this software and associated documentation files (the 10 | "Software"), to deal in the Software without restriction, including 11 | without limitation the rights to use, copy, modify, merge, publish, 12 | distribute, sublicense, and/or sell copies of the Software, and to 13 | permit persons to whom the Software is furnished to do so, subject to 14 | the following conditions: 15 | 16 | The above copyright notice and this permission notice shall be 17 | included in all copies or substantial portions of the Software. 18 | 19 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 20 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 21 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 22 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 23 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 24 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 25 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 26 | """ 27 | 28 | import re 29 | 30 | from packvers.tags import ( 31 | compatible_tags, 32 | cpython_tags, 33 | generic_tags, 34 | interpreter_name, 35 | interpreter_version, 36 | mac_platforms, 37 | ) 38 | 39 | _osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") 40 | 41 | 42 | def version_info_to_nodot(version_info): 43 | # type: (Tuple[int, ...]) -> str 44 | # Only use up to the first two numbers. 45 | return "".join(map(str, version_info[:2])) 46 | 47 | 48 | def _mac_platforms(arch): 49 | # type: (str) -> List[str] 50 | match = _osx_arch_pat.match(arch) 51 | if match: 52 | name, major, minor, actual_arch = match.groups() 53 | mac_version = (int(major), int(minor)) 54 | arches = [ 55 | # Since we have always only checked that the platform starts 56 | # with "macosx", for backwards-compatibility we extract the 57 | # actual prefix provided by the user in case they provided 58 | # something like "macosxcustom_". It may be good to remove 59 | # this as undocumented or deprecate it in the future. 60 | "{}_{}".format(name, arch[len("macosx_") :]) 61 | for arch in mac_platforms(mac_version, actual_arch) 62 | ] 63 | else: 64 | # arch pattern didn't match (?!) 65 | arches = [arch] 66 | return arches 67 | 68 | 69 | def _custom_manylinux_platforms(arch): 70 | # type: (str) -> List[str] 71 | arches = [arch] 72 | arch_prefix, arch_sep, arch_suffix = arch.partition("_") 73 | if arch_prefix == "manylinux2014": 74 | # manylinux1/manylinux2010 wheels run on most manylinux2014 systems 75 | # with the exception of wheels depending on ncurses. PEP 599 states 76 | # manylinux1/manylinux2010 wheels should be considered 77 | # manylinux2014 wheels: 78 | # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels 79 | if arch_suffix in {"i686", "x86_64"}: 80 | arches.append("manylinux2010" + arch_sep + arch_suffix) 81 | arches.append("manylinux1" + arch_sep + arch_suffix) 82 | elif arch_prefix == "manylinux2010": 83 | # manylinux1 wheels run on most manylinux2010 systems with the 84 | # exception of wheels depending on ncurses. PEP 571 states 85 | # manylinux1 wheels should be considered manylinux2010 wheels: 86 | # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels 87 | arches.append("manylinux1" + arch_sep + arch_suffix) 88 | return arches 89 | 90 | 91 | def _get_custom_platforms(arch): 92 | # type: (str) -> List[str] 93 | arch_prefix, _arch_sep, _arch_suffix = arch.partition("_") 94 | if arch.startswith("macosx"): 95 | arches = _mac_platforms(arch) 96 | elif arch_prefix in ["manylinux2014", "manylinux2010"]: 97 | arches = _custom_manylinux_platforms(arch) 98 | else: 99 | arches = [arch] 100 | return arches 101 | 102 | 103 | def _expand_allowed_platforms(platforms): 104 | # type: (Optional[List[str]]) -> Optional[List[str]] 105 | if not platforms: 106 | return None 107 | 108 | seen = set() 109 | result = [] 110 | 111 | for p in platforms: 112 | if p in seen: 113 | continue 114 | additions = [c for c in _get_custom_platforms(p) if c not in seen] 115 | seen.update(additions) 116 | result.extend(additions) 117 | 118 | return result 119 | 120 | 121 | def _get_python_version(version): 122 | # type: (str) -> PythonVersion 123 | if len(version) > 1: 124 | return int(version[0]), int(version[1:]) 125 | else: 126 | return (int(version[0]),) 127 | 128 | 129 | def _get_custom_interpreter(implementation=None, version=None): 130 | # type: (Optional[str], Optional[str]) -> str 131 | if implementation is None: 132 | implementation = interpreter_name() 133 | if version is None: 134 | version = interpreter_version() 135 | return "{}{}".format(implementation, version) 136 | 137 | 138 | def get_supported( 139 | version=None, # type: Optional[str] 140 | platforms=None, # type: Optional[List[str]] 141 | impl=None, # type: Optional[str] 142 | abis=None, # type: Optional[List[str]] 143 | ): 144 | # type: (...) -> List[Tag] 145 | """Return a list of supported tags for each version specified in 146 | `versions`. 147 | 148 | :param version: a string version, of the form "33" or "32", 149 | or None. The version will be assumed to support our ABI. 150 | :param platforms: specify a list of platforms you want valid 151 | tags for, or None. If None, use the local system platform. 152 | :param impl: specify the exact implementation you want valid 153 | tags for, or None. If None, use the local interpreter impl. 154 | :param abis: specify a list of abis you want valid 155 | tags for, or None. If None, use the local interpreter abi. 156 | """ 157 | supported = [] # type: List[Tag] 158 | 159 | python_version = None # type: Optional[PythonVersion] 160 | if version is not None: 161 | python_version = _get_python_version(version) 162 | 163 | interpreter = _get_custom_interpreter(impl, version) 164 | 165 | platforms = _expand_allowed_platforms(platforms) 166 | 167 | is_cpython = (impl or interpreter_name()) == "cp" 168 | if is_cpython: 169 | supported.extend( 170 | cpython_tags( 171 | python_version=python_version, 172 | abis=abis, 173 | platforms=platforms, 174 | ) 175 | ) 176 | else: 177 | supported.extend( 178 | generic_tags( 179 | interpreter=interpreter, 180 | abis=abis, 181 | platforms=platforms, 182 | ) 183 | ) 184 | supported.extend( 185 | compatible_tags( 186 | python_version=python_version, 187 | interpreter=interpreter, 188 | platforms=platforms, 189 | ) 190 | ) 191 | 192 | return supported 193 | -------------------------------------------------------------------------------- /etc/scripts/utils_pip_compatibility_tags.py.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: utils_pip_compatibility_tags.py 2 | 3 | type: github 4 | namespace: pypa 5 | name: pip 6 | version: 20.3.1 7 | subpath: src/pip/_internal/utils/compatibility_tags.py 8 | 9 | package_url: pkg:github/pypa/pip@20.3.1#src/pip/_internal/utils/compatibility_tags.py 10 | 11 | download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py 12 | copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) 13 | license_expression: mit 14 | notes: subset copied from pip for tag handling -------------------------------------------------------------------------------- /etc/scripts/utils_pypi_supported_tags.py: -------------------------------------------------------------------------------- 1 | # Licensed under the Apache License, Version 2.0 (the "License"); 2 | # you may not use this file except in compliance with the License. 3 | # You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | import re 14 | 15 | """ 16 | Wheel platform checking 17 | 18 | Copied and modified on 2020-12-24 from 19 | https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/warehouse/forklift/legacy.py 20 | 21 | This contains the basic functions to check if a wheel file name is would be 22 | supported for uploading to PyPI. 23 | """ 24 | 25 | # These platforms can be handled by a simple static list: 26 | _allowed_platforms = { 27 | "any", 28 | "win32", 29 | "win_amd64", 30 | "win_ia64", 31 | "manylinux1_x86_64", 32 | "manylinux1_i686", 33 | "manylinux2010_x86_64", 34 | "manylinux2010_i686", 35 | "manylinux2014_x86_64", 36 | "manylinux2014_i686", 37 | "manylinux2014_aarch64", 38 | "manylinux2014_armv7l", 39 | "manylinux2014_ppc64", 40 | "manylinux2014_ppc64le", 41 | "manylinux2014_s390x", 42 | "linux_armv6l", 43 | "linux_armv7l", 44 | } 45 | # macosx is a little more complicated: 46 | _macosx_platform_re = re.compile(r"macosx_(?P\d+)_(\d+)_(?P.*)") 47 | _macosx_arches = { 48 | "ppc", 49 | "ppc64", 50 | "i386", 51 | "x86_64", 52 | "arm64", 53 | "intel", 54 | "fat", 55 | "fat32", 56 | "fat64", 57 | "universal", 58 | "universal2", 59 | } 60 | _macosx_major_versions = { 61 | "10", 62 | "11", 63 | } 64 | 65 | # manylinux pep600 is a little more complicated: 66 | _manylinux_platform_re = re.compile(r"manylinux_(\d+)_(\d+)_(?P.*)") 67 | _manylinux_arches = { 68 | "x86_64", 69 | "i686", 70 | "aarch64", 71 | "armv7l", 72 | "ppc64", 73 | "ppc64le", 74 | "s390x", 75 | } 76 | 77 | 78 | def is_supported_platform_tag(platform_tag): 79 | """ 80 | Return True if the ``platform_tag`` is supported on PyPI. 81 | """ 82 | if platform_tag in _allowed_platforms: 83 | return True 84 | m = _macosx_platform_re.match(platform_tag) 85 | if m and m.group("major") in _macosx_major_versions and m.group("arch") in _macosx_arches: 86 | return True 87 | m = _manylinux_platform_re.match(platform_tag) 88 | if m and m.group("arch") in _manylinux_arches: 89 | return True 90 | return False 91 | 92 | 93 | def validate_platforms_for_pypi(platforms): 94 | """ 95 | Validate if the wheel platforms are supported platform tags on Pypi. Return 96 | a list of unsupported platform tags or an empty list if all tags are 97 | supported. 98 | """ 99 | 100 | # Check that if it's a binary wheel, it's on a supported platform 101 | invalid_tags = [] 102 | for plat in platforms: 103 | if not is_supported_platform_tag(plat): 104 | invalid_tags.append(plat) 105 | return invalid_tags 106 | -------------------------------------------------------------------------------- /etc/scripts/utils_pypi_supported_tags.py.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: utils_pypi_supported_tags.py 2 | 3 | type: github 4 | namespace: pypa 5 | name: warehouse 6 | version: 37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d 7 | subpath: warehouse/forklift/legacy.py 8 | 9 | package_url: pkg:github/pypa/warehouse@37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d#warehouse/forklift/legacy.py 10 | 11 | download_url: https://github.com/pypa/warehouse/blob/37a83dd342d9e3b3ab4f6bde47ca30e6883e2c4d/warehouse/forklift/legacy.py 12 | copyright: Copyright (c) The warehouse developers 13 | homepage_url: https://warehouse.readthedocs.io 14 | license_expression: apache-2.0 15 | notes: Wheel platform checking copied and heavily modified on 2020-12-24 from 16 | warehouse. This contains the basic functions to check if a wheel file name is 17 | would be supported for uploading to PyPI. 18 | -------------------------------------------------------------------------------- /etc/scripts/utils_requirements.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) nexB Inc. and others. All rights reserved. 5 | # ScanCode is a trademark of nexB Inc. 6 | # SPDX-License-Identifier: Apache-2.0 7 | # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 8 | # See https://github.com/aboutcode-org/skeleton for support or download. 9 | # See https://aboutcode.org for more information about nexB OSS projects. 10 | # 11 | 12 | import os 13 | import re 14 | import subprocess 15 | 16 | """ 17 | Utilities to manage requirements files and call pip. 18 | NOTE: this should use ONLY the standard library and not import anything else 19 | because this is used for boostrapping with no requirements installed. 20 | """ 21 | 22 | 23 | def load_requirements(requirements_file="requirements.txt", with_unpinned=False): 24 | """ 25 | Yield package (name, version) tuples for each requirement in a `requirement` 26 | file. Only accept requirements pinned to an exact version. 27 | """ 28 | with open(requirements_file) as reqs: 29 | req_lines = reqs.read().splitlines(False) 30 | return get_required_name_versions(req_lines, with_unpinned=with_unpinned) 31 | 32 | 33 | def get_required_name_versions(requirement_lines, with_unpinned=False): 34 | """ 35 | Yield required (name, version) tuples given a`requirement_lines` iterable of 36 | requirement text lines. Only accept requirements pinned to an exact version. 37 | """ 38 | 39 | for req_line in requirement_lines: 40 | req_line = req_line.strip() 41 | if not req_line or req_line.startswith("#"): 42 | continue 43 | if req_line.startswith("-") or (not with_unpinned and not "==" in req_line): 44 | print(f"Requirement line is not supported: ignored: {req_line}") 45 | continue 46 | yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) 47 | 48 | 49 | def get_required_name_version(requirement, with_unpinned=False): 50 | """ 51 | Return a (name, version) tuple given a`requirement` specifier string. 52 | Requirement version must be pinned. If ``with_unpinned`` is True, unpinned 53 | requirements are accepted and only the name portion is returned. 54 | 55 | For example: 56 | >>> assert get_required_name_version("foo==1.2.3") == ("foo", "1.2.3") 57 | >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") 58 | >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") 59 | >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") 60 | >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") 61 | >>> try: 62 | ... assert not get_required_name_version("foo", with_unpinned=False) 63 | ... except Exception as e: 64 | ... assert "Requirement version must be pinned" in str(e) 65 | """ 66 | requirement = requirement and "".join(requirement.lower().split()) 67 | assert requirement, f"specifier is required is empty:{requirement!r}" 68 | name, operator, version = split_req(requirement) 69 | assert name, f"Name is required: {requirement}" 70 | is_pinned = operator == "==" 71 | if with_unpinned: 72 | version = "" 73 | else: 74 | assert is_pinned and version, f"Requirement version must be pinned: {requirement}" 75 | return name, version 76 | 77 | 78 | def lock_requirements(requirements_file="requirements.txt", site_packages_dir=None): 79 | """ 80 | Freeze and lock current installed requirements and save this to the 81 | `requirements_file` requirements file. 82 | """ 83 | with open(requirements_file, "w") as fo: 84 | fo.write(get_installed_reqs(site_packages_dir=site_packages_dir)) 85 | 86 | 87 | def lock_dev_requirements( 88 | dev_requirements_file="requirements-dev.txt", 89 | main_requirements_file="requirements.txt", 90 | site_packages_dir=None, 91 | ): 92 | """ 93 | Freeze and lock current installed development-only requirements and save 94 | this to the `dev_requirements_file` requirements file. Development-only is 95 | achieved by subtracting requirements from the `main_requirements_file` 96 | requirements file from the current requirements using package names (and 97 | ignoring versions). 98 | """ 99 | main_names = {n for n, _v in load_requirements(main_requirements_file)} 100 | all_reqs = get_installed_reqs(site_packages_dir=site_packages_dir) 101 | all_req_lines = all_reqs.splitlines(False) 102 | all_req_nvs = get_required_name_versions(all_req_lines) 103 | dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} 104 | 105 | new_reqs = "\n".join( 106 | f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) 107 | with open(dev_requirements_file, "w") as fo: 108 | fo.write(new_reqs) 109 | 110 | 111 | def get_installed_reqs(site_packages_dir): 112 | """ 113 | Return the installed pip requirements as text found in `site_packages_dir` 114 | as a text. 115 | """ 116 | if not os.path.exists(site_packages_dir): 117 | raise Exception( 118 | f"site_packages directory: {site_packages_dir!r} does not exists") 119 | # Also include these packages in the output with --all: wheel, distribute, 120 | # setuptools, pip 121 | args = ["pip", "freeze", "--exclude-editable", 122 | "--all", "--path", site_packages_dir] 123 | return subprocess.check_output(args, encoding="utf-8") 124 | 125 | 126 | comparators = ( 127 | "===", 128 | "~=", 129 | "!=", 130 | "==", 131 | "<=", 132 | ">=", 133 | ">", 134 | "<", 135 | ) 136 | 137 | _comparators_re = r"|".join(comparators) 138 | version_splitter = re.compile(rf"({_comparators_re})") 139 | 140 | 141 | def split_req(req): 142 | """ 143 | Return a three-tuple of (name, comparator, version) given a ``req`` 144 | requirement specifier string. Each segment may be empty. Spaces are removed. 145 | 146 | For example: 147 | >>> assert split_req("foo==1.2.3") == ("foo", "==", "1.2.3"), split_req("foo==1.2.3") 148 | >>> assert split_req("foo") == ("foo", "", ""), split_req("foo") 149 | >>> assert split_req("==1.2.3") == ("", "==", "1.2.3"), split_req("==1.2.3") 150 | >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") 151 | >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") 152 | """ 153 | assert req 154 | # do not allow multiple constraints and tags 155 | assert not any(c in req for c in ",;") 156 | req = "".join(req.split()) 157 | if not any(c in req for c in comparators): 158 | return req, "", "" 159 | segments = version_splitter.split(req, maxsplit=1) 160 | return tuple(segments) 161 | -------------------------------------------------------------------------------- /etc/scripts/utils_thirdparty.py.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: utils_thirdparty.py 2 | package_url: pkg:github.com/pypa/pip/@20.3.1#src/pip/_internal/models/wheel.py 3 | type: github 4 | namespace: pypa 5 | name: pip 6 | version: 20.3.1 7 | subpath: src/pip/_internal/models/wheel.py 8 | 9 | download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py 10 | copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) 11 | license_expression: mit 12 | notes: copied from pip-20.3.1 pip/_internal/models/wheel.py 13 | The models code has been heavily inspired from the ISC-licensed packaging-dists 14 | https://github.com/uranusjr/packaging-dists by Tzu-ping Chung 15 | -------------------------------------------------------------------------------- /license-expression.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: . 2 | name: license-expression 3 | 4 | copyright: Copyright (c) nexB Inc. and others. 5 | 6 | license_expression: apache-2.0 7 | license_file: apache-2.0.LICENSE 8 | 9 | homepage_url: https://github.com/aboutcode-org/license-expression 10 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools >= 50", "wheel", "setuptools_scm[toml] >= 6"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.setuptools_scm] 6 | # this is used populated when creating a git archive 7 | # and when there is .git dir and/or there is no git installed 8 | fallback_version = "9999.afa6dce-2025-04-03" 9 | 10 | [tool.pytest.ini_options] 11 | norecursedirs = [ 12 | ".git", 13 | "bin", 14 | "dist", 15 | "build", 16 | "_build", 17 | "dist", 18 | "etc", 19 | "local", 20 | "ci", 21 | "docs", 22 | "man", 23 | "share", 24 | "samples", 25 | ".cache", 26 | ".settings", 27 | "Include", 28 | "include", 29 | "Lib", 30 | "lib", 31 | "lib64", 32 | "Lib64", 33 | "Scripts", 34 | "thirdparty", 35 | "tmp", 36 | "venv", 37 | "tests/data", 38 | ".eggs", 39 | "src/*/data", 40 | "tests/*/data" 41 | ] 42 | 43 | python_files = "*.py" 44 | 45 | python_classes = "Test" 46 | python_functions = "test" 47 | 48 | addopts = [ 49 | "-rfExXw", 50 | "--strict-markers", 51 | "--doctest-modules" 52 | ] 53 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aboutcode-org/license-expression/afa6dce1a526681e3b01a0088e8ff0cf811cf263/requirements-dev.txt -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aboutcode-org/license-expression/afa6dce1a526681e3b01a0088e8ff0cf811cf263/requirements.txt -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = license-expression 3 | version = 30.4.1 4 | license = Apache-2.0 5 | 6 | # description must be on ONE line https://github.com/pypa/setuptools/issues/1390 7 | description = license-expression is a comprehensive utility library to parse, compare, simplify and normalize license expressions (such as SPDX license expressions) using boolean logic. 8 | long_description = file:README.rst 9 | long_description_content_type = text/x-rst 10 | url = https://github.com/aboutcode-org/license-expression 11 | 12 | author = nexB. Inc. and others 13 | author_email = info@aboutcode.org 14 | 15 | classifiers = 16 | Development Status :: 5 - Production/Stable 17 | Intended Audience :: Developers 18 | Programming Language :: Python :: 3 19 | Programming Language :: Python :: 3 :: Only 20 | Topic :: Software Development 21 | Topic :: Utilities 22 | 23 | keywords = 24 | open source 25 | license expression 26 | license 27 | spdx 28 | boolean 29 | parse expression 30 | normalize expression 31 | compare expression 32 | licence 33 | 34 | license_files = 35 | apache-2.0.LICENSE 36 | NOTICE 37 | AUTHORS.rst 38 | CHANGELOG.rst 39 | CODE_OF_CONDUCT.rst 40 | 41 | [options] 42 | package_dir = 43 | =src 44 | packages = find: 45 | include_package_data = true 46 | zip_safe = false 47 | 48 | setup_requires = setuptools_scm[toml] >= 4 49 | 50 | python_requires = >=3.9 51 | 52 | install_requires = 53 | boolean.py >= 4.0 54 | 55 | 56 | [options.packages.find] 57 | where = src 58 | 59 | 60 | [options.extras_require] 61 | testing = 62 | pytest >= 6, != 7.0.0 63 | pytest-xdist >= 2 64 | # do not use this as this triggers a bug 65 | # in setuptools_scm:aboutcode-toolkit >= 6.0.0 66 | twine 67 | black 68 | isort 69 | 70 | docs = 71 | Sphinx>=5.0.2 72 | sphinx-rtd-theme>=1.0.0 73 | sphinxcontrib-apidoc >= 0.4.0 74 | sphinx-reredirects >= 0.1.2 75 | doc8>=0.11.2 76 | sphinx-autobuild 77 | sphinx-rtd-dark-mode>=1.3.0 78 | sphinx-copybutton 79 | 80 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import setuptools 4 | 5 | if __name__ == "__main__": 6 | setuptools.setup() 7 | -------------------------------------------------------------------------------- /src/license_expression/_pyahocorasick.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: _pyahocorasick.py 2 | download_url: https://github.com/WojciechMula/pyahocorasick/tree/ec2fb9cb393f571fd4316ea98ed7b65992f16127/py 3 | name: pyahocorasick-python 4 | version: ec2fb9 5 | 6 | homepage_url: https://github.com/WojciechMula/pyahocorasick 7 | license_expression: public-domain 8 | 9 | copyright: originally authored by Wojciech Mula, modified by the license_expression authors. 10 | 11 | notes: this is a vendored subset of the full pyahocorasick containing only the pure 12 | python part with an implementation modified to return non-overlapping matches and 13 | non-matches. 14 | It has many limitation and in particular it does not pickle well and is much slower 15 | than the full C-based implementation but is convenient to use as a vendored, pure 16 | Python library. 17 | 18 | owner: nexB Inc. 19 | author: Wojciech Mula http://0x80.pl/ 20 | 21 | vcs_tool: git 22 | vcs_repository: https://github.com/WojciechMula/pyahocorasick.git 23 | 24 | -------------------------------------------------------------------------------- /src/license_expression/_pyahocorasick.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # SPDX-License-Identifier: LicenseRef-scancode-public-domain 4 | # See https://github.com/aboutcode-org/license-expression for support or download. 5 | # See https://aboutcode.org for more information about nexB OSS projects. 6 | # 7 | """ 8 | Aho-Corasick string search algorithm in pure Python 9 | 10 | Original Author: Wojciech Muła, wojciech_mula@poczta.onet.pl 11 | WWW : http://0x80.pl 12 | License : public domain 13 | 14 | This is the pure Python Aho-Corasick automaton from pyahocorasick modified for 15 | use in the license_expression library for advanced tokenization: 16 | 17 | - add support for unicode strings. 18 | - case insensitive search using sequence of words and not characters 19 | - improve returned results with the actual start,end and matched string. 20 | - support returning non-matched parts of a string 21 | """ 22 | from collections import deque 23 | from collections import OrderedDict 24 | import logging 25 | import re 26 | 27 | TRACE = False 28 | 29 | logger = logging.getLogger(__name__) 30 | 31 | 32 | def logger_debug(*args): 33 | pass 34 | 35 | 36 | if TRACE: 37 | 38 | def logger_debug(*args): 39 | return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) 40 | 41 | import sys 42 | logging.basicConfig(stream=sys.stdout) 43 | logger.setLevel(logging.DEBUG) 44 | 45 | # used to distinguish from None 46 | nil = object() 47 | 48 | 49 | class TrieNode(object): 50 | """ 51 | Node of the Trie/Aho-Corasick automaton. 52 | """ 53 | __slots__ = ['token', 'output', 'fail', 'children'] 54 | 55 | def __init__(self, token, output=nil): 56 | # token of a tokens string added to the Trie as a string 57 | self.token = token 58 | 59 | # an output function (in the Aho-Corasick meaning) for this node: this 60 | # is an object that contains the original key string and any 61 | # additional value data associated to that key. Or "nil" for a node that 62 | # is not a terminal leave for a key. It will be returned with a match. 63 | self.output = output 64 | 65 | # failure link used by the Aho-Corasick automaton and its search procedure 66 | self.fail = nil 67 | 68 | # children of this node as a mapping of char->node 69 | self.children = {} 70 | 71 | def __repr__(self): 72 | if self.output is not nil: 73 | return 'TrieNode(%r, %r)' % (self.token, self.output) 74 | else: 75 | return 'TrieNode(%r)' % self.token 76 | 77 | 78 | class Trie(object): 79 | """ 80 | A Trie and Aho-Corasick automaton. This behaves more or less like a mapping of 81 | key->value. This is the main entry point. 82 | """ 83 | 84 | def __init__(self): 85 | """ 86 | Initialize a new Trie. 87 | """ 88 | self.root = TrieNode('') 89 | 90 | # set of any unique tokens in the trie, updated on each addition we keep 91 | # track of the set of tokens added to the trie to build the automaton 92 | # these are needed to created the first level children failure links 93 | self._known_tokens = set() 94 | 95 | # Flag set to True once a Trie has been converted to an Aho-Corasick automaton 96 | self._converted = False 97 | 98 | def add(self, tokens_string, value=None): 99 | """ 100 | Add a new tokens_string and its associated value to the trie. If the 101 | tokens_string already exists in the Trie, its value is replaced with the 102 | provided value, typically a Token object. If a value is not provided, 103 | the tokens_string is used as value. 104 | 105 | A tokens_string is any string. It will be tokenized when added 106 | to the Trie. 107 | """ 108 | if self._converted: 109 | raise Exception('This Trie has been converted to an Aho-Corasick ' 110 | 'automaton and cannot be modified.') 111 | 112 | if not tokens_string or not isinstance(tokens_string, str): 113 | return 114 | 115 | tokens = [t for t in get_tokens(tokens_string) if t.strip()] 116 | 117 | # we keep track of the set of tokens added to the trie to build the 118 | # automaton these are needed to created the first level children failure 119 | # links 120 | 121 | self._known_tokens.update(tokens) 122 | 123 | node = self.root 124 | for token in tokens: 125 | try: 126 | node = node.children[token] 127 | except KeyError: 128 | child = TrieNode(token) 129 | node.children[token] = child 130 | node = child 131 | 132 | node.output = (tokens_string, value or tokens_string) 133 | 134 | def __get_node(self, tokens_string): 135 | """ 136 | Return a node for this tokens_string or None if the trie does not 137 | contain the tokens_string. Private function retrieving a final node of 138 | the Trie for a given tokens_string. 139 | """ 140 | if not tokens_string or not isinstance(tokens_string, str): 141 | return 142 | 143 | tokens = [t for t in get_tokens(tokens_string) if t.strip()] 144 | node = self.root 145 | for token in tokens: 146 | try: 147 | node = node.children[token] 148 | except KeyError: 149 | return None 150 | return node 151 | 152 | def get(self, tokens_string, default=nil): 153 | """ 154 | Return the output value found associated with a `tokens_string`. If 155 | there is no such tokens_string in the Trie, return the default value 156 | (other than nil). If `default` is not provided or is `nil`, raise a 157 | KeyError. 158 | """ 159 | node = self.__get_node(tokens_string) 160 | output = nil 161 | if node: 162 | output = node.output 163 | 164 | if output is nil: 165 | if default is nil: 166 | raise KeyError(tokens_string) 167 | else: 168 | return default 169 | else: 170 | return output 171 | 172 | def keys(self): 173 | """ 174 | Yield all keys stored in this trie. 175 | """ 176 | return (key for key, _ in self.items()) 177 | 178 | def values(self): 179 | """ 180 | Yield all values associated with keys stored in this trie. 181 | """ 182 | return (value for _, value in self.items()) 183 | 184 | def items(self): 185 | """ 186 | Yield tuple of all (key, value) stored in this trie. 187 | """ 188 | items = [] 189 | 190 | def walk(node, tokens): 191 | """ 192 | Walk the trie, depth first. 193 | """ 194 | tokens = [t for t in tokens + [node.token] if t] 195 | if node.output is not nil: 196 | items.append((node.output[0], node.output[1],)) 197 | 198 | for child in node.children.values(): 199 | if child is not node: 200 | walk(child, tokens) 201 | 202 | walk(self.root, tokens=[]) 203 | 204 | return iter(items) 205 | 206 | def exists(self, tokens_string): 207 | """ 208 | Return True if the key is present in this trie. 209 | """ 210 | node = self.__get_node(tokens_string) 211 | if node: 212 | return bool(node.output != nil) 213 | return False 214 | 215 | def is_prefix(self, tokens_string): 216 | """ 217 | Return True if tokens_string is a prefix of any existing tokens_string in the trie. 218 | """ 219 | return bool(self.__get_node(tokens_string) is not None) 220 | 221 | def make_automaton(self): 222 | """ 223 | Convert this trie to an Aho-Corasick automaton. 224 | Note that this is an error to add new keys to a Trie once it has been 225 | converted to an Automaton. 226 | """ 227 | queue = deque() 228 | 229 | # 1. create root children for each known items range (e.g. all unique 230 | # characters from all the added tokens), failing to root. 231 | # And build a queue of these 232 | for token in self._known_tokens: 233 | if token in self.root.children: 234 | node = self.root.children[token] 235 | # e.g. f(s) = 0, Aho-Corasick-wise 236 | node.fail = self.root 237 | queue.append(node) 238 | else: 239 | self.root.children[token] = self.root 240 | 241 | # 2. using the queue of all possible top level items/chars, walk the trie and 242 | # add failure links to nodes as needed 243 | while queue: 244 | current_node = queue.popleft() 245 | for node in current_node.children.values(): 246 | queue.append(node) 247 | state = current_node.fail 248 | while node.token not in state.children: 249 | state = state.fail 250 | node.fail = state.children.get(node.token, self.root) 251 | 252 | # Mark the trie as converted so it cannot be modified anymore 253 | self._converted = True 254 | 255 | def iter(self, tokens_string, include_unmatched=False, include_space=False): 256 | """ 257 | Yield Token objects for matched strings by performing the Aho-Corasick 258 | search procedure. 259 | 260 | The Token start and end positions in the searched string are such that 261 | the matched string is "tokens_string[start:end+1]". And the start is 262 | computed from the end_index collected by the Aho-Corasick search 263 | procedure such that 264 | "start=end_index - n + 1" where n is the length of a matched string. 265 | 266 | The Token.value is an object associated with a matched string. 267 | 268 | For example: 269 | >>> a = Trie() 270 | >>> a.add('BCDEF') 271 | >>> a.add('CDE') 272 | >>> a.add('DEFGH') 273 | >>> a.add('EFGH') 274 | >>> a.add('KL') 275 | >>> a.make_automaton() 276 | >>> tokens_string = 'a bcdef ghij kl m' 277 | >>> strings = Token.sort(a.iter(tokens_string)) 278 | >>> expected = [ 279 | ... Token(2, 6, u'bcdef', u'BCDEF'), 280 | ... Token(13, 14, u'kl', u'KL') 281 | ... ] 282 | 283 | >>> strings == expected 284 | True 285 | 286 | >>> list(a.iter('')) == [] 287 | True 288 | 289 | >>> list(a.iter(' ')) == [] 290 | True 291 | """ 292 | if not tokens_string: 293 | return 294 | 295 | tokens = get_tokens(tokens_string) 296 | state = self.root 297 | 298 | if TRACE: 299 | logger_debug('Trie.iter() with:', repr(tokens_string)) 300 | logger_debug(' tokens:', tokens) 301 | 302 | end_pos = -1 303 | for token_string in tokens: 304 | end_pos += len(token_string) 305 | if TRACE: 306 | logger_debug() 307 | logger_debug('token_string', repr(token_string)) 308 | logger_debug(' end_pos', end_pos) 309 | 310 | if not include_space and not token_string.strip(): 311 | if TRACE: 312 | logger_debug(' include_space skipped') 313 | continue 314 | 315 | if token_string not in self._known_tokens: 316 | state = self.root 317 | if TRACE: 318 | logger_debug(' unmatched') 319 | if include_unmatched: 320 | n = len(token_string) 321 | start_pos = end_pos - n + 1 322 | tok = Token( 323 | start=start_pos, 324 | end=end_pos, 325 | string=tokens_string[start_pos: end_pos + 1], 326 | value=None 327 | ) 328 | if TRACE: 329 | logger_debug(' unmatched tok:', tok) 330 | yield tok 331 | continue 332 | 333 | yielded = False 334 | 335 | # search for a matching token_string in the children, starting at root 336 | while token_string not in state.children: 337 | state = state.fail 338 | 339 | # we have a matching starting token_string 340 | state = state.children.get(token_string, self.root) 341 | match = state 342 | while match is not nil: 343 | if match.output is not nil: 344 | matched_string, output_value = match.output 345 | if TRACE: 346 | logger_debug(' type output', repr( 347 | output_value), type(matched_string)) 348 | n = len(matched_string) 349 | start_pos = end_pos - n + 1 350 | if TRACE: 351 | logger_debug(' start_pos', start_pos) 352 | yield Token(start_pos, end_pos, tokens_string[start_pos: end_pos + 1], output_value) 353 | yielded = True 354 | match = match.fail 355 | if not yielded and include_unmatched: 356 | if TRACE: 357 | logger_debug(' unmatched but known token') 358 | n = len(token_string) 359 | start_pos = end_pos - n + 1 360 | tok = Token(start_pos, end_pos, 361 | tokens_string[start_pos: end_pos + 1], None) 362 | if TRACE: 363 | logger_debug(' unmatched tok 2:', tok) 364 | yield tok 365 | 366 | logger_debug() 367 | 368 | def tokenize(self, string, include_unmatched=True, include_space=False): 369 | """ 370 | tokenize a string for matched and unmatched sub-sequences and yield non- 371 | overlapping Token objects performing a modified Aho-Corasick search 372 | procedure: 373 | 374 | - return both matched and unmatched sub-sequences. 375 | - do not return matches with positions that are contained or overlap with 376 | another match: 377 | - discard smaller matches contained in a larger match. 378 | - when there is overlap (but not containment), the matches are sorted by 379 | start and biggest length and then: 380 | - we return the largest match of two overlaping matches 381 | - if they have the same length, keep the match starting the earliest and 382 | return the non-overlapping portion of the other discarded match as a 383 | non-match. 384 | 385 | Each Token contains the start and end position, the corresponding string 386 | and an associated value object. 387 | 388 | For example: 389 | >>> a = Trie() 390 | >>> a.add('BCDEF') 391 | >>> a.add('CDE') 392 | >>> a.add('DEFGH') 393 | >>> a.add('EFGH') 394 | >>> a.add('KL') 395 | >>> a.make_automaton() 396 | >>> string = 'a bcdef ghij kl' 397 | >>> tokens = list(a.tokenize(string, include_space=True)) 398 | 399 | >>> expected = [ 400 | ... Token(0, 0, u'a', None), 401 | ... Token(1, 1, u' ', None), 402 | ... Token(2, 6, u'bcdef', u'BCDEF'), 403 | ... Token(7, 7, u' ', None), 404 | ... Token(8, 11, u'ghij', None), 405 | ... Token(12, 12, u' ', None), 406 | ... Token(13, 14, u'kl', u'KL') 407 | ... ] 408 | >>> tokens == expected 409 | True 410 | """ 411 | tokens = self.iter(string, 412 | include_unmatched=include_unmatched, include_space=include_space) 413 | tokens = list(tokens) 414 | if TRACE: 415 | logger_debug('tokenize.tokens:', tokens) 416 | if not include_space: 417 | tokens = [t for t in tokens if t.string.strip()] 418 | tokens = filter_overlapping(tokens) 419 | return tokens 420 | 421 | 422 | def filter_overlapping(tokens): 423 | """ 424 | Return a new list from an iterable of `tokens` discarding contained and 425 | overlaping Tokens using these rules: 426 | 427 | - skip a token fully contained in another token. 428 | - keep the biggest, left-most token of two overlapping tokens and skip the other 429 | 430 | For example: 431 | >>> tokens = [ 432 | ... Token(0, 0, 'a'), 433 | ... Token(1, 5, 'bcdef'), 434 | ... Token(2, 4, 'cde'), 435 | ... Token(3, 7, 'defgh'), 436 | ... Token(4, 7, 'efgh'), 437 | ... Token(8, 9, 'ij'), 438 | ... Token(10, 13, 'klmn'), 439 | ... Token(11, 15, 'lmnop'), 440 | ... Token(16, 16, 'q'), 441 | ... ] 442 | 443 | >>> expected = [ 444 | ... Token(0, 0, 'a'), 445 | ... Token(1, 5, 'bcdef'), 446 | ... Token(8, 9, 'ij'), 447 | ... Token(11, 15, 'lmnop'), 448 | ... Token(16, 16, 'q'), 449 | ... ] 450 | 451 | >>> filtered = list(filter_overlapping(tokens)) 452 | >>> filtered == expected 453 | True 454 | """ 455 | tokens = Token.sort(tokens) 456 | 457 | # compare pair of tokens in the sorted sequence: current and next 458 | i = 0 459 | while i < len(tokens) - 1: 460 | j = i + 1 461 | while j < len(tokens): 462 | curr_tok = tokens[i] 463 | next_tok = tokens[j] 464 | 465 | logger_debug('curr_tok, i, next_tok, j:', curr_tok, i, next_tok, j) 466 | # disjoint tokens: break, there is nothing to do 467 | if next_tok.is_after(curr_tok): 468 | logger_debug(' break to next', curr_tok) 469 | break 470 | 471 | # contained token: discard the contained token 472 | if next_tok in curr_tok: 473 | logger_debug(' del next_tok contained:', next_tok) 474 | del tokens[j] 475 | continue 476 | 477 | # overlap: Keep the longest token and skip the smallest overlapping 478 | # tokens. In case of length tie: keep the left most 479 | if curr_tok.overlap(next_tok): 480 | if len(curr_tok) >= len(next_tok): 481 | logger_debug(' del next_tok smaller overlap:', next_tok) 482 | del tokens[j] 483 | continue 484 | else: 485 | logger_debug(' del curr_tok smaller overlap:', curr_tok) 486 | del tokens[i] 487 | break 488 | j += 1 489 | i += 1 490 | return tokens 491 | 492 | 493 | class Token(object): 494 | """ 495 | A Token is used to track the tokenization an expression with its 496 | start and end as index position in the original string and other attributes: 497 | 498 | - `start` and `end` are zero-based index in the original string S such that 499 | S[start:end+1] will yield `string`. 500 | - `string` is the matched substring from the original string for this Token. 501 | - `value` is the corresponding object for this token as one of: 502 | - a LicenseSymbol object 503 | - a "Keyword" object (and, or, with, left and right parens) 504 | - None if this is a space. 505 | """ 506 | 507 | __slots__ = 'start', 'end', 'string', 'value', 508 | 509 | def __init__(self, start, end, string='', value=None): 510 | self.start = start 511 | self.end = end 512 | self.string = string 513 | self.value = value 514 | 515 | def __repr__(self): 516 | return self.__class__.__name__ + '(%(start)r, %(end)r, %(string)r, %(value)r)' % self.as_dict() 517 | 518 | def as_dict(self): 519 | return OrderedDict([(s, getattr(self, s)) for s in self.__slots__]) 520 | 521 | def __len__(self): 522 | return self.end - self.start + 1 523 | 524 | def __eq__(self, other): 525 | return isinstance(other, Token) and ( 526 | self.start == other.start and 527 | self.end == other.end and 528 | self.string == other.string and 529 | self.value == other.value 530 | ) 531 | 532 | def __hash__(self): 533 | tup = self.start, self.end, self.string, self.value 534 | return hash(tup) 535 | 536 | @classmethod 537 | def sort(cls, tokens): 538 | """ 539 | Return a new sorted sequence of tokens given a sequence of tokens. The 540 | primary sort is on start and the secondary sort is on longer lengths. 541 | Therefore if two tokens have the same start, the longer token will sort 542 | first. 543 | 544 | For example: 545 | >>> tokens = [Token(0, 0), Token(5, 5), Token(1, 1), Token(2, 4), Token(2, 5)] 546 | >>> expected = [Token(0, 0), Token(1, 1), Token(2, 5), Token(2, 4), Token(5, 5)] 547 | >>> expected == Token.sort(tokens) 548 | True 549 | """ 550 | def key(s): return (s.start, -len(s),) 551 | return sorted(tokens, key=key) 552 | 553 | def is_after(self, other): 554 | """ 555 | Return True if this token is after the other token. 556 | 557 | For example: 558 | >>> Token(1, 2).is_after(Token(5, 6)) 559 | False 560 | >>> Token(5, 6).is_after(Token(5, 6)) 561 | False 562 | >>> Token(2, 3).is_after(Token(1, 2)) 563 | False 564 | >>> Token(5, 6).is_after(Token(3, 4)) 565 | True 566 | """ 567 | return self.start > other.end 568 | 569 | def is_before(self, other): 570 | return self.end < other.start 571 | 572 | def __contains__(self, other): 573 | """ 574 | Return True if this token contains the other token. 575 | 576 | For example: 577 | >>> Token(5, 7) in Token(5, 7) 578 | True 579 | >>> Token(6, 8) in Token(5, 7) 580 | False 581 | >>> Token(6, 6) in Token(4, 8) 582 | True 583 | >>> Token(3, 9) in Token(4, 8) 584 | False 585 | >>> Token(4, 8) in Token(3, 9) 586 | True 587 | """ 588 | return self.start <= other.start and other.end <= self.end 589 | 590 | def overlap(self, other): 591 | """ 592 | Return True if this token and the other token overlap. 593 | 594 | For example: 595 | >>> Token(1, 2).overlap(Token(5, 6)) 596 | False 597 | >>> Token(5, 6).overlap(Token(5, 6)) 598 | True 599 | >>> Token(4, 5).overlap(Token(5, 6)) 600 | True 601 | >>> Token(4, 5).overlap(Token(5, 7)) 602 | True 603 | >>> Token(4, 5).overlap(Token(6, 7)) 604 | False 605 | """ 606 | start = self.start 607 | end = self.end 608 | return (start <= other.start <= end) or (start <= other.end <= end) 609 | 610 | 611 | # tokenize to separate text from parens 612 | _tokenizer = re.compile(r''' 613 | (?P[^\s\(\)]+) 614 | | 615 | (?P\s+) 616 | | 617 | (?P[\(\)]) 618 | ''', 619 | re.VERBOSE | re.MULTILINE | re.UNICODE 620 | ) 621 | 622 | 623 | def get_tokens(tokens_string): 624 | """ 625 | Return an iterable of strings splitting on spaces and parens. 626 | """ 627 | return [match for match in _tokenizer.split(tokens_string.lower()) if match] 628 | -------------------------------------------------------------------------------- /src/license_expression/data/cc-by-4.0.LICENSE: -------------------------------------------------------------------------------- 1 | Attribution 4.0 International 2 | 3 | ======================================================================= 4 | 5 | Creative Commons Corporation ("Creative Commons") is not a law firm and 6 | does not provide legal services or legal advice. Distribution of 7 | Creative Commons public licenses does not create a lawyer-client or 8 | other relationship. Creative Commons makes its licenses and related 9 | information available on an "as-is" basis. Creative Commons gives no 10 | warranties regarding its licenses, any material licensed under their 11 | terms and conditions, or any related information. Creative Commons 12 | disclaims all liability for damages resulting from their use to the 13 | fullest extent possible. 14 | 15 | Using Creative Commons Public Licenses 16 | 17 | Creative Commons public licenses provide a standard set of terms and 18 | conditions that creators and other rights holders may use to share 19 | original works of authorship and other material subject to copyright 20 | and certain other rights specified in the public license below. The 21 | following considerations are for informational purposes only, are not 22 | exhaustive, and do not form part of our licenses. 23 | 24 | Considerations for licensors: Our public licenses are 25 | intended for use by those authorized to give the public 26 | permission to use material in ways otherwise restricted by 27 | copyright and certain other rights. Our licenses are 28 | irrevocable. Licensors should read and understand the terms 29 | and conditions of the license they choose before applying it. 30 | Licensors should also secure all rights necessary before 31 | applying our licenses so that the public can reuse the 32 | material as expected. Licensors should clearly mark any 33 | material not subject to the license. This includes other CC- 34 | licensed material, or material used under an exception or 35 | limitation to copyright. More considerations for licensors: 36 | wiki.creativecommons.org/Considerations_for_licensors 37 | 38 | Considerations for the public: By using one of our public 39 | licenses, a licensor grants the public permission to use the 40 | licensed material under specified terms and conditions. If 41 | the licensor's permission is not necessary for any reason--for 42 | example, because of any applicable exception or limitation to 43 | copyright--then that use is not regulated by the license. Our 44 | licenses grant only permissions under copyright and certain 45 | other rights that a licensor has authority to grant. Use of 46 | the licensed material may still be restricted for other 47 | reasons, including because others have copyright or other 48 | rights in the material. A licensor may make special requests, 49 | such as asking that all changes be marked or described. 50 | Although not required by our licenses, you are encouraged to 51 | respect those requests where reasonable. More considerations 52 | for the public: 53 | wiki.creativecommons.org/Considerations_for_licensees 54 | 55 | ======================================================================= 56 | 57 | Creative Commons Attribution 4.0 International Public License 58 | 59 | By exercising the Licensed Rights (defined below), You accept and agree 60 | to be bound by the terms and conditions of this Creative Commons 61 | Attribution 4.0 International Public License ("Public License"). To the 62 | extent this Public License may be interpreted as a contract, You are 63 | granted the Licensed Rights in consideration of Your acceptance of 64 | these terms and conditions, and the Licensor grants You such rights in 65 | consideration of benefits the Licensor receives from making the 66 | Licensed Material available under these terms and conditions. 67 | 68 | 69 | Section 1 -- Definitions. 70 | 71 | a. Adapted Material means material subject to Copyright and Similar 72 | Rights that is derived from or based upon the Licensed Material 73 | and in which the Licensed Material is translated, altered, 74 | arranged, transformed, or otherwise modified in a manner requiring 75 | permission under the Copyright and Similar Rights held by the 76 | Licensor. For purposes of this Public License, where the Licensed 77 | Material is a musical work, performance, or sound recording, 78 | Adapted Material is always produced where the Licensed Material is 79 | synched in timed relation with a moving image. 80 | 81 | b. Adapter's License means the license You apply to Your Copyright 82 | and Similar Rights in Your contributions to Adapted Material in 83 | accordance with the terms and conditions of this Public License. 84 | 85 | c. Copyright and Similar Rights means copyright and/or similar rights 86 | closely related to copyright including, without limitation, 87 | performance, broadcast, sound recording, and Sui Generis Database 88 | Rights, without regard to how the rights are labeled or 89 | categorized. For purposes of this Public License, the rights 90 | specified in Section 2(b)(1)-(2) are not Copyright and Similar 91 | Rights. 92 | 93 | d. Effective Technological Measures means those measures that, in the 94 | absence of proper authority, may not be circumvented under laws 95 | fulfilling obligations under Article 11 of the WIPO Copyright 96 | Treaty adopted on December 20, 1996, and/or similar international 97 | agreements. 98 | 99 | e. Exceptions and Limitations means fair use, fair dealing, and/or 100 | any other exception or limitation to Copyright and Similar Rights 101 | that applies to Your use of the Licensed Material. 102 | 103 | f. Licensed Material means the artistic or literary work, database, 104 | or other material to which the Licensor applied this Public 105 | License. 106 | 107 | g. Licensed Rights means the rights granted to You subject to the 108 | terms and conditions of this Public License, which are limited to 109 | all Copyright and Similar Rights that apply to Your use of the 110 | Licensed Material and that the Licensor has authority to license. 111 | 112 | h. Licensor means the individual(s) or entity(ies) granting rights 113 | under this Public License. 114 | 115 | i. Share means to provide material to the public by any means or 116 | process that requires permission under the Licensed Rights, such 117 | as reproduction, public display, public performance, distribution, 118 | dissemination, communication, or importation, and to make material 119 | available to the public including in ways that members of the 120 | public may access the material from a place and at a time 121 | individually chosen by them. 122 | 123 | j. Sui Generis Database Rights means rights other than copyright 124 | resulting from Directive 96/9/EC of the European Parliament and of 125 | the Council of 11 March 1996 on the legal protection of databases, 126 | as amended and/or succeeded, as well as other essentially 127 | equivalent rights anywhere in the world. 128 | 129 | k. You means the individual or entity exercising the Licensed Rights 130 | under this Public License. Your has a corresponding meaning. 131 | 132 | 133 | Section 2 -- Scope. 134 | 135 | a. License grant. 136 | 137 | 1. Subject to the terms and conditions of this Public License, 138 | the Licensor hereby grants You a worldwide, royalty-free, 139 | non-sublicensable, non-exclusive, irrevocable license to 140 | exercise the Licensed Rights in the Licensed Material to: 141 | 142 | a. reproduce and Share the Licensed Material, in whole or 143 | in part; and 144 | 145 | b. produce, reproduce, and Share Adapted Material. 146 | 147 | 2. Exceptions and Limitations. For the avoidance of doubt, where 148 | Exceptions and Limitations apply to Your use, this Public 149 | License does not apply, and You do not need to comply with 150 | its terms and conditions. 151 | 152 | 3. Term. The term of this Public License is specified in Section 153 | 6(a). 154 | 155 | 4. Media and formats; technical modifications allowed. The 156 | Licensor authorizes You to exercise the Licensed Rights in 157 | all media and formats whether now known or hereafter created, 158 | and to make technical modifications necessary to do so. The 159 | Licensor waives and/or agrees not to assert any right or 160 | authority to forbid You from making technical modifications 161 | necessary to exercise the Licensed Rights, including 162 | technical modifications necessary to circumvent Effective 163 | Technological Measures. For purposes of this Public License, 164 | simply making modifications authorized by this Section 2(a) 165 | (4) never produces Adapted Material. 166 | 167 | 5. Downstream recipients. 168 | 169 | a. Offer from the Licensor -- Licensed Material. Every 170 | recipient of the Licensed Material automatically 171 | receives an offer from the Licensor to exercise the 172 | Licensed Rights under the terms and conditions of this 173 | Public License. 174 | 175 | b. No downstream restrictions. You may not offer or impose 176 | any additional or different terms or conditions on, or 177 | apply any Effective Technological Measures to, the 178 | Licensed Material if doing so restricts exercise of the 179 | Licensed Rights by any recipient of the Licensed 180 | Material. 181 | 182 | 6. No endorsement. Nothing in this Public License constitutes or 183 | may be construed as permission to assert or imply that You 184 | are, or that Your use of the Licensed Material is, connected 185 | with, or sponsored, endorsed, or granted official status by, 186 | the Licensor or others designated to receive attribution as 187 | provided in Section 3(a)(1)(A)(i). 188 | 189 | b. Other rights. 190 | 191 | 1. Moral rights, such as the right of integrity, are not 192 | licensed under this Public License, nor are publicity, 193 | privacy, and/or other similar personality rights; however, to 194 | the extent possible, the Licensor waives and/or agrees not to 195 | assert any such rights held by the Licensor to the limited 196 | extent necessary to allow You to exercise the Licensed 197 | Rights, but not otherwise. 198 | 199 | 2. Patent and trademark rights are not licensed under this 200 | Public License. 201 | 202 | 3. To the extent possible, the Licensor waives any right to 203 | collect royalties from You for the exercise of the Licensed 204 | Rights, whether directly or through a collecting society 205 | under any voluntary or waivable statutory or compulsory 206 | licensing scheme. In all other cases the Licensor expressly 207 | reserves any right to collect such royalties. 208 | 209 | 210 | Section 3 -- License Conditions. 211 | 212 | Your exercise of the Licensed Rights is expressly made subject to the 213 | following conditions. 214 | 215 | a. Attribution. 216 | 217 | 1. If You Share the Licensed Material (including in modified 218 | form), You must: 219 | 220 | a. retain the following if it is supplied by the Licensor 221 | with the Licensed Material: 222 | 223 | i. identification of the creator(s) of the Licensed 224 | Material and any others designated to receive 225 | attribution, in any reasonable manner requested by 226 | the Licensor (including by pseudonym if 227 | designated); 228 | 229 | ii. a copyright notice; 230 | 231 | iii. a notice that refers to this Public License; 232 | 233 | iv. a notice that refers to the disclaimer of 234 | warranties; 235 | 236 | v. a URI or hyperlink to the Licensed Material to the 237 | extent reasonably practicable; 238 | 239 | b. indicate if You modified the Licensed Material and 240 | retain an indication of any previous modifications; and 241 | 242 | c. indicate the Licensed Material is licensed under this 243 | Public License, and include the text of, or the URI or 244 | hyperlink to, this Public License. 245 | 246 | 2. You may satisfy the conditions in Section 3(a)(1) in any 247 | reasonable manner based on the medium, means, and context in 248 | which You Share the Licensed Material. For example, it may be 249 | reasonable to satisfy the conditions by providing a URI or 250 | hyperlink to a resource that includes the required 251 | information. 252 | 253 | 3. If requested by the Licensor, You must remove any of the 254 | information required by Section 3(a)(1)(A) to the extent 255 | reasonably practicable. 256 | 257 | 4. If You Share Adapted Material You produce, the Adapter's 258 | License You apply must not prevent recipients of the Adapted 259 | Material from complying with this Public License. 260 | 261 | 262 | Section 4 -- Sui Generis Database Rights. 263 | 264 | Where the Licensed Rights include Sui Generis Database Rights that 265 | apply to Your use of the Licensed Material: 266 | 267 | a. for the avoidance of doubt, Section 2(a)(1) grants You the right 268 | to extract, reuse, reproduce, and Share all or a substantial 269 | portion of the contents of the database; 270 | 271 | b. if You include all or a substantial portion of the database 272 | contents in a database in which You have Sui Generis Database 273 | Rights, then the database in which You have Sui Generis Database 274 | Rights (but not its individual contents) is Adapted Material; and 275 | 276 | c. You must comply with the conditions in Section 3(a) if You Share 277 | all or a substantial portion of the contents of the database. 278 | 279 | For the avoidance of doubt, this Section 4 supplements and does not 280 | replace Your obligations under this Public License where the Licensed 281 | Rights include other Copyright and Similar Rights. 282 | 283 | 284 | Section 5 -- Disclaimer of Warranties and Limitation of Liability. 285 | 286 | a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE 287 | EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS 288 | AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF 289 | ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, 290 | IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, 291 | WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR 292 | PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, 293 | ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT 294 | KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT 295 | ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. 296 | 297 | b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE 298 | TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, 299 | NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, 300 | INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, 301 | COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR 302 | USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN 303 | ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR 304 | DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR 305 | IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. 306 | 307 | c. The disclaimer of warranties and limitation of liability provided 308 | above shall be interpreted in a manner that, to the extent 309 | possible, most closely approximates an absolute disclaimer and 310 | waiver of all liability. 311 | 312 | 313 | Section 6 -- Term and Termination. 314 | 315 | a. This Public License applies for the term of the Copyright and 316 | Similar Rights licensed here. However, if You fail to comply with 317 | this Public License, then Your rights under this Public License 318 | terminate automatically. 319 | 320 | b. Where Your right to use the Licensed Material has terminated under 321 | Section 6(a), it reinstates: 322 | 323 | 1. automatically as of the date the violation is cured, provided 324 | it is cured within 30 days of Your discovery of the 325 | violation; or 326 | 327 | 2. upon express reinstatement by the Licensor. 328 | 329 | For the avoidance of doubt, this Section 6(b) does not affect any 330 | right the Licensor may have to seek remedies for Your violations 331 | of this Public License. 332 | 333 | c. For the avoidance of doubt, the Licensor may also offer the 334 | Licensed Material under separate terms or conditions or stop 335 | distributing the Licensed Material at any time; however, doing so 336 | will not terminate this Public License. 337 | 338 | d. Sections 1, 5, 6, 7, and 8 survive termination of this Public 339 | License. 340 | 341 | 342 | Section 7 -- Other Terms and Conditions. 343 | 344 | a. The Licensor shall not be bound by any additional or different 345 | terms or conditions communicated by You unless expressly agreed. 346 | 347 | b. Any arrangements, understandings, or agreements regarding the 348 | Licensed Material not stated herein are separate from and 349 | independent of the terms and conditions of this Public License. 350 | 351 | 352 | Section 8 -- Interpretation. 353 | 354 | a. For the avoidance of doubt, this Public License does not, and 355 | shall not be interpreted to, reduce, limit, restrict, or impose 356 | conditions on any use of the Licensed Material that could lawfully 357 | be made without permission under this Public License. 358 | 359 | b. To the extent possible, if any provision of this Public License is 360 | deemed unenforceable, it shall be automatically reformed to the 361 | minimum extent necessary to make it enforceable. If the provision 362 | cannot be reformed, it shall be severed from this Public License 363 | without affecting the enforceability of the remaining terms and 364 | conditions. 365 | 366 | c. No term or condition of this Public License will be waived and no 367 | failure to comply consented to unless expressly agreed to by the 368 | Licensor. 369 | 370 | d. Nothing in this Public License constitutes or may be interpreted 371 | as a limitation upon, or waiver of, any privileges and immunities 372 | that apply to the Licensor or You, including from the legal 373 | processes of any jurisdiction or authority. 374 | 375 | 376 | ======================================================================= 377 | 378 | Creative Commons is not a party to its public 379 | licenses. Notwithstanding, Creative Commons may elect to apply one of 380 | its public licenses to material it publishes and in those instances 381 | will be considered the “Licensor.” The text of the Creative Commons 382 | public licenses is dedicated to the public domain under the CC0 Public 383 | Domain Dedication. Except for the limited purpose of indicating that 384 | material is shared under a Creative Commons public license or as 385 | otherwise permitted by the Creative Commons policies published at 386 | creativecommons.org/policies, Creative Commons does not authorize the 387 | use of the trademark "Creative Commons" or any other trademark or logo 388 | of Creative Commons without its prior written consent including, 389 | without limitation, in connection with any unauthorized modifications 390 | to any of its public licenses or any other arrangements, 391 | understandings, or agreements concerning use of licensed material. For 392 | the avoidance of doubt, this paragraph does not form part of the 393 | public licenses. 394 | 395 | Creative Commons may be contacted at creativecommons.org. 396 | -------------------------------------------------------------------------------- /src/license_expression/data/license_key_index.json.ABOUT: -------------------------------------------------------------------------------- 1 | about_resource: scancode-licensedb-index.json 2 | download_url: https://raw.githubusercontent.com/aboutcode-org/scancode-licensedb/1e9ff1927b89bae4ca1356de77aa29cc18916025/docs/index.json 3 | spdx_license_list_version: 3.26 4 | name: scancode-licensedb-index.json 5 | license_expression: cc-by-4.0 6 | copyright: Copyright (c) nexB Inc. and others. 7 | homepage_url: https://scancode-licensedb.aboutcode.org/ 8 | -------------------------------------------------------------------------------- /tests/data/test_license_key_index.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "license_key": "389-exception", 4 | "spdx_license_key": "389-exception", 5 | "other_spdx_license_keys": [], 6 | "is_exception": true, 7 | "json": "389-exception.json", 8 | "yml": "389-exception.yml", 9 | "html": "389-exception.html", 10 | "text": "389-exception.LICENSE" 11 | }, 12 | { 13 | "license_key": "3com-microcode", 14 | "spdx_license_key": "LicenseRef-scancode-3com-microcode", 15 | "other_spdx_license_keys": [], 16 | "is_exception": false, 17 | "json": "3com-microcode.json", 18 | "yml": "3com-microcode.yml", 19 | "html": "3com-microcode.html", 20 | "text": "3com-microcode.LICENSE" 21 | }, 22 | { 23 | "license_key": "3dslicer-1.0", 24 | "spdx_license_key": "LicenseRef-scancode-3dslicer-1.0", 25 | "other_spdx_license_keys": [], 26 | "is_exception": false, 27 | "json": "3dslicer-1.0.json", 28 | "yml": "3dslicer-1.0.yml", 29 | "html": "3dslicer-1.0.html", 30 | "text": "3dslicer-1.0.LICENSE" 31 | }, 32 | { 33 | "license_key": "aladdin-md5", 34 | "spdx_license_key": null, 35 | "other_spdx_license_keys": [], 36 | "is_exception": false, 37 | "is_deprecated": true, 38 | "json": "aladdin-md5.json", 39 | "yml": "aladdin-md5.yml", 40 | "html": "aladdin-md5.html", 41 | "text": "aladdin-md5.LICENSE" 42 | } 43 | ] -------------------------------------------------------------------------------- /tests/test__pyahocorasick.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # 3 | # SPDX-License-Identifier: LicenseRef-scancode-public-domain 4 | # See https://github.com/aboutcode-org/license-expression for support or download. 5 | # See https://aboutcode.org for more information about nexB OSS projects. 6 | 7 | """ 8 | Tests for Aho-Corasick string search algorithm. 9 | Original Author: Wojciech Muła, wojciech_mula@poczta.onet.pl 10 | WWW : http://0x80.pl 11 | License : public domain 12 | 13 | Modified for use in the license_expression library. 14 | """ 15 | 16 | import unittest 17 | 18 | from license_expression._pyahocorasick import Trie 19 | from license_expression._pyahocorasick import Token 20 | 21 | 22 | class TestTrie(unittest.TestCase): 23 | 24 | def test_add_can_get(self): 25 | t = Trie() 26 | t.add('python', 'value') 27 | assert ('python', 'value') == t.get('python') 28 | 29 | def test_add_existing_WordShouldReplaceAssociatedValue(self): 30 | t = Trie() 31 | t.add('python', 'value') 32 | assert ('python', 'value') == t.get('python') 33 | 34 | t.add('python', 'other') 35 | assert ('python', 'other') == t.get('python') 36 | 37 | def test_get_UnknowWordWithoutDefaultValueShouldRaiseException(self): 38 | t = Trie() 39 | with self.assertRaises(KeyError): 40 | t.get('python') 41 | 42 | def test_get_UnknowWordWithDefaultValueShouldReturnDefault(self): 43 | t = Trie() 44 | self.assertEqual(t.get('python', 'default'), 'default') 45 | 46 | def test_exists_ShouldDetectAddedWords(self): 47 | t = Trie() 48 | t.add('python', 'value') 49 | t.add('ada', 'value') 50 | 51 | self.assertTrue(t.exists('python')) 52 | self.assertTrue(t.exists('ada')) 53 | 54 | def test_exists_ShouldReturnFailOnUnknownWord(self): 55 | t = Trie() 56 | t.add('python', 'value') 57 | 58 | self.assertFalse(t.exists('ada')) 59 | 60 | def test_is_prefix_ShouldDetecAllPrefixesIncludingWord(self): 61 | t = Trie() 62 | t.add('python', 'value') 63 | t.add('ada lovelace', 'value') 64 | 65 | self.assertFalse(t.is_prefix('a')) 66 | self.assertFalse(t.is_prefix('ad')) 67 | self.assertTrue(t.is_prefix('ada')) 68 | 69 | self.assertFalse(t.is_prefix('p')) 70 | self.assertFalse(t.is_prefix('py')) 71 | self.assertFalse(t.is_prefix('pyt')) 72 | self.assertFalse(t.is_prefix('pyth')) 73 | self.assertFalse(t.is_prefix('pytho')) 74 | self.assertTrue(t.is_prefix('python')) 75 | 76 | self.assertFalse(t.is_prefix('lovelace')) 77 | 78 | def test_items_ShouldReturnAllItemsAlreadyAddedToTheTrie(self): 79 | t = Trie() 80 | 81 | t.add('python', 1) 82 | t.add('ada', 2) 83 | t.add('perl', 3) 84 | t.add('pascal', 4) 85 | t.add('php', 5) 86 | t.add('php that', 6) 87 | 88 | result = list(t.items()) 89 | self.assertIn(('python', 1), result) 90 | self.assertIn(('ada', 2), result) 91 | self.assertIn(('perl', 3), result) 92 | self.assertIn(('pascal', 4), result) 93 | self.assertIn(('php', 5), result) 94 | self.assertIn(('php that', 6), result) 95 | 96 | def test_keys_ShouldReturnAllKeysAlreadyAddedToTheTrie(self): 97 | t = Trie() 98 | 99 | t.add('python', 1) 100 | t.add('ada', 2) 101 | t.add('perl', 3) 102 | t.add('pascal', 4) 103 | t.add('php', 5) 104 | t.add('php that', 6) 105 | 106 | result = list(t.keys()) 107 | self.assertIn('python', result) 108 | self.assertIn('ada', result) 109 | self.assertIn('perl', result) 110 | self.assertIn('pascal', result) 111 | self.assertIn('php', result) 112 | self.assertIn('php that', result) 113 | 114 | def test_values_ShouldReturnAllValuesAlreadyAddedToTheTrie(self): 115 | t = Trie() 116 | 117 | t.add('python', 1) 118 | t.add('ada', 2) 119 | t.add('perl', 3) 120 | t.add('pascal', 4) 121 | t.add('php', 5) 122 | 123 | result = list(t.values()) 124 | self.assertIn(1, result) 125 | self.assertIn(2, result) 126 | self.assertIn(3, result) 127 | self.assertIn(4, result) 128 | self.assertIn(5, result) 129 | 130 | def test_iter_should_not_return_non_matches_by_default(self): 131 | 132 | def get_test_automaton(): 133 | words = 'he her hers his she hi him man himan'.split() 134 | t = Trie() 135 | for w in words: 136 | t.add(w, w) 137 | t.make_automaton() 138 | return t 139 | 140 | test_string = 'he she himan' 141 | 142 | t = get_test_automaton() 143 | result = list(t.iter(test_string)) 144 | assert 'he she himan'.split() == [r.value for r in result] 145 | 146 | def test_iter_should_can_return_non_matches_optionally(self): 147 | 148 | def get_test_automaton(): 149 | words = 'he her hers his she hi him man himan'.split() 150 | t = Trie() 151 | for w in words: 152 | t.add(w, w) 153 | t.make_automaton() 154 | return t 155 | 156 | test_string = ' he she junk himan other stuffs ' 157 | # 111111111122222222223333333 158 | # 0123456789012345678901234567890123456 159 | 160 | t = get_test_automaton() 161 | result = list( 162 | t.iter(test_string, include_unmatched=True, include_space=True)) 163 | expected = [ 164 | Token(0, 1, u' ', None), 165 | Token(2, 3, u'he', u'he'), 166 | Token(4, 4, u' ', None), 167 | Token(5, 7, u'she', u'she'), 168 | Token(8, 8, u' ', None), 169 | Token(9, 12, u'junk', None), 170 | Token(13, 14, u' ', None), 171 | Token(15, 19, u'himan', u'himan'), 172 | Token(20, 21, u' ', None), 173 | Token(22, 26, u'other', None), 174 | Token(27, 27, u' ', None), 175 | Token(28, 33, u'stuffs', None), 176 | Token(34, 36, u' ', None), 177 | ] 178 | 179 | assert expected == result 180 | 181 | def test_iter_vs_tokenize(self): 182 | 183 | def get_test_automaton(): 184 | words = '( AND ) OR'.split() 185 | t = Trie() 186 | for w in words: 187 | t.add(w, w) 188 | t.make_automaton() 189 | return t 190 | 191 | test_string = '((l-a + AND l-b) OR (l -c+))' 192 | 193 | t = get_test_automaton() 194 | result = list( 195 | t.iter(test_string, include_unmatched=True, include_space=True)) 196 | expected = [ 197 | Token(0, 0, u'(', u'('), 198 | Token(1, 1, u'(', u'('), 199 | Token(2, 4, u'l-a', None), 200 | Token(5, 5, u' ', None), 201 | Token(6, 6, u'+', None), 202 | Token(7, 7, u' ', None), 203 | Token(8, 10, u'AND', u'AND'), 204 | Token(11, 11, u' ', None), 205 | Token(12, 14, u'l-b', None), 206 | Token(15, 15, u')', u')'), 207 | Token(16, 16, u' ', None), 208 | Token(17, 18, u'OR', u'OR'), 209 | Token(19, 19, u' ', None), 210 | Token(20, 20, u'(', u'('), 211 | Token(21, 21, u'l', None), 212 | Token(22, 22, u' ', None), 213 | Token(23, 25, u'-c+', None), 214 | Token(26, 26, u')', u')'), 215 | Token(27, 27, u')', u')') 216 | ] 217 | 218 | assert expected == result 219 | 220 | result = list(t.tokenize( 221 | test_string, include_unmatched=True, include_space=True)) 222 | assert expected == result 223 | 224 | def test_tokenize_with_unmatched_and_space(self): 225 | 226 | def get_test_automaton(): 227 | words = '( AND ) OR'.split() 228 | t = Trie() 229 | for w in words: 230 | t.add(w, w) 231 | t.make_automaton() 232 | return t 233 | 234 | test_string = '((l-a + AND l-b) OR an (l -c+))' 235 | # 111111111122222222223 236 | # 0123456789012345678901234567890 237 | t = get_test_automaton() 238 | result = list(t.tokenize( 239 | test_string, include_unmatched=True, include_space=True)) 240 | expected = [ 241 | Token(0, 0, u'(', u'('), 242 | Token(1, 1, u'(', u'('), 243 | Token(2, 4, u'l-a', None), 244 | Token(5, 5, u' ', None), 245 | Token(6, 6, u'+', None), 246 | Token(7, 7, u' ', None), 247 | Token(8, 10, u'AND', u'AND'), 248 | Token(11, 11, u' ', None), 249 | Token(12, 14, u'l-b', None), 250 | Token(15, 15, u')', u')'), 251 | Token(16, 16, u' ', None), 252 | Token(17, 18, u'OR', u'OR'), 253 | Token(19, 19, u' ', None), 254 | Token(20, 21, u'an', None), 255 | Token(22, 22, u' ', None), 256 | Token(23, 23, u'(', u'('), 257 | Token(24, 24, u'l', None), 258 | Token(25, 25, u' ', None), 259 | Token(26, 28, u'-c+', None), 260 | Token(29, 29, u')', u')'), 261 | Token(30, 30, u')', u')') 262 | ] 263 | 264 | assert expected == result 265 | assert test_string == ''.join(t.string for t in result) 266 | 267 | def test_iter_with_unmatched_simple(self): 268 | t = Trie() 269 | t.add('And', 'And') 270 | t.make_automaton() 271 | test_string = 'AND an a And' 272 | result = list(t.iter(test_string)) 273 | assert ['And', 'And'] == [r.value for r in result] 274 | 275 | def test_iter_with_unmatched_simple2(self): 276 | t = Trie() 277 | t.add('AND', 'AND') 278 | t.make_automaton() 279 | test_string = 'AND an a and' 280 | result = list(t.iter(test_string)) 281 | assert ['AND', 'AND'] == [r.value for r in result] 282 | 283 | def test_iter_with_unmatched_simple3(self): 284 | t = Trie() 285 | t.add('AND', 'AND') 286 | t.make_automaton() 287 | test_string = 'AND an a andersom' 288 | result = list(t.iter(test_string)) 289 | assert ['AND'] == [r.value for r in result] 290 | 291 | def test_iter_simple(self): 292 | t = Trie() 293 | t.add('AND', 'AND') 294 | t.add('OR', 'OR') 295 | t.add('WITH', 'WITH') 296 | t.add('(', '(') 297 | t.add(')', ')') 298 | t.add('GPL-2.0', 'GPL-2.0') 299 | t.add('mit', 'MIT') 300 | t.add('Classpath', 'Classpath') 301 | t.make_automaton() 302 | test_string = '(GPL-2.0 with Classpath) or (gpl-2.0) and (classpath or gpl-2.0 OR mit) ' 303 | # 111111111122222222223333333333444444444455555555556666666666777 304 | # 0123456789012345678901234567890123456789012345678901234567890123456789012 305 | result = list(t.iter(test_string)) 306 | expected = [ 307 | Token(0, 0, u'(', u'('), 308 | Token(1, 7, u'GPL-2.0', u'GPL-2.0'), 309 | Token(9, 12, u'with', u'WITH'), 310 | Token(14, 22, u'Classpath', u'Classpath'), 311 | Token(23, 23, u')', u')'), 312 | Token(25, 26, u'or', u'OR'), 313 | Token(28, 28, u'(', u'('), 314 | Token(29, 35, u'gpl-2.0', u'GPL-2.0'), 315 | Token(36, 36, u')', u')'), 316 | Token(38, 40, u'and', u'AND'), 317 | Token(42, 42, u'(', u'('), 318 | Token(43, 51, u'classpath', u'Classpath'), 319 | Token(53, 54, u'or', u'OR'), 320 | Token(57, 63, u'gpl-2.0', u'GPL-2.0'), 321 | Token(65, 66, u'OR', u'OR'), 322 | Token(68, 70, u'mit', u'MIT'), 323 | Token(71, 71, u')', u')') 324 | ] 325 | 326 | assert expected == result 327 | -------------------------------------------------------------------------------- /tests/test_skeleton_codestyle.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright (c) nexB Inc. and others. All rights reserved. 3 | # ScanCode is a trademark of nexB Inc. 4 | # SPDX-License-Identifier: Apache-2.0 5 | # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. 6 | # See https://github.com/aboutcode-org/skeleton for support or download. 7 | # See https://aboutcode.org for more information about nexB OSS projects. 8 | # 9 | 10 | import subprocess 11 | import unittest 12 | import configparser 13 | 14 | 15 | class BaseTests(unittest.TestCase): 16 | def test_skeleton_codestyle(self): 17 | """ 18 | This test shouldn't run in proliferated repositories. 19 | """ 20 | setup_cfg = configparser.ConfigParser() 21 | setup_cfg.read("setup.cfg") 22 | if setup_cfg["metadata"]["name"] != "skeleton": 23 | return 24 | 25 | args = "venv/bin/black --check -l 100 setup.py etc tests" 26 | try: 27 | subprocess.check_output(args.split()) 28 | except subprocess.CalledProcessError as e: 29 | print("===========================================================") 30 | print(e.output) 31 | print("===========================================================") 32 | raise Exception( 33 | "Black style check failed; please format the code using:\n" 34 | " python -m black -l 100 setup.py etc tests", 35 | e.output, 36 | ) from e 37 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py36,py37,py38,py39 3 | 4 | [testenv] 5 | # install pytest in the virtualenv where commands will be executed 6 | deps = pytest 7 | commands = 8 | pytest -vvs 9 | 10 | --------------------------------------------------------------------------------