├── src └── flake8_requirements │ ├── __init__.py │ ├── modules.py │ └── checker.py ├── .github ├── dependabot.yaml └── workflows │ ├── codecov.yaml │ ├── publish.yaml │ └── check.yaml ├── test ├── test_setup.cfg ├── test_requirements.txt ├── test_poetry.py ├── test_setup.py ├── test_checker.py ├── test_pep621.py └── test_requirements.py ├── .gitignore ├── tox.ini ├── LICENSE.txt ├── pyproject.toml └── README.rst /src/flake8_requirements/__init__.py: -------------------------------------------------------------------------------- 1 | from .checker import Flake8Checker 2 | 3 | __all__ = ( 4 | 'Flake8Checker', 5 | ) 6 | -------------------------------------------------------------------------------- /.github/dependabot.yaml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | # Check for updates to GitHub Actions every month 5 | - package-ecosystem: "github-actions" 6 | directory: "/" 7 | schedule: 8 | interval: "monthly" 9 | -------------------------------------------------------------------------------- /test/test_setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = my_package 3 | description = My package description 4 | license = BSD 3-Clause License 5 | classifiers = 6 | License :: OSI Approved :: BSD License 7 | Programming Language :: Python :: 3 8 | 9 | [options] 10 | packages = find: 11 | install_requires = 12 | requests 13 | importlib; python_version == "2.6" 14 | tests_require = 15 | pytest 16 | 17 | [options.package_data] 18 | * = *.txt, *.rst 19 | hello = *.msg 20 | 21 | [options.extras_require] 22 | pdf = ReportLab>=1.2 23 | rest = docutils>=0.3 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # Unit test / coverage reports 29 | htmlcov/ 30 | .tox/ 31 | .coverage 32 | .coverage.* 33 | .cache 34 | nosetests.xml 35 | coverage.xml 36 | *,cover 37 | .hypothesis/ 38 | -------------------------------------------------------------------------------- /.github/workflows/codecov.yaml: -------------------------------------------------------------------------------- 1 | name: Code Coverage 2 | on: 3 | push: 4 | pull_request: 5 | branches: [ master ] 6 | jobs: 7 | coverage: 8 | if: github.repository_owner == 'arkq' 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v5 12 | - uses: actions/setup-python@v6 13 | with: 14 | python-version: '3.x' 15 | - name: Generate Coverage Report 16 | run: | 17 | pip install tox 18 | tox 19 | - name: Upload Coverage to Codecov 20 | uses: codecov/codecov-action@v5 21 | with: 22 | token: ${{ secrets.CODECOV_TOKEN }} 23 | files: .tox/coverage.xml 24 | disable_search: true 25 | -------------------------------------------------------------------------------- /.github/workflows/publish.yaml: -------------------------------------------------------------------------------- 1 | name: Upload Python Package to PyPI 2 | on: 3 | push: 4 | tags: 5 | - '*' 6 | jobs: 7 | deploy: 8 | if: github.repository_owner == 'arkq' 9 | runs-on: ubuntu-latest 10 | environment: 11 | name: pypi 12 | url: https://pypi.org/p/flake8-requirements 13 | permissions: 14 | id-token: write 15 | steps: 16 | - uses: actions/checkout@v5 17 | - uses: actions/setup-python@v6 18 | with: 19 | python-version: '3.x' 20 | - name: Install dependencies 21 | run: pip install build 22 | - name: Build 23 | run: python -m build 24 | - name: Publish to PyPI 25 | uses: pypa/gh-action-pypi-publish@release/v1 26 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | coverage 4 | py3 5 | isolated_build = true 6 | 7 | [testenv] 8 | description = Run the tests with pytest under {basepython}. 9 | setenv = 10 | COVERAGE_FILE = {toxworkdir}/.coverage.{envname} 11 | commands = 12 | pytest \ 13 | --cov="{envsitepackagesdir}/flake8_requirements" \ 14 | --cov-config="{toxinidir}/tox.ini" \ 15 | test 16 | deps = 17 | pytest 18 | pytest-cov 19 | 20 | [testenv:coverage] 21 | description = Combine coverage data and create final XML report. 22 | setenv = 23 | COVERAGE_FILE = {toxworkdir}/.coverage 24 | commands = 25 | coverage combine 26 | coverage report 27 | coverage xml -o "{toxworkdir}/coverage.xml" 28 | skip_install = true 29 | deps = coverage 30 | depends = py3 31 | 32 | [coverage:paths] 33 | source = src/flake8_requirements 34 | */.tox/*/lib/python*/site-packages/flake8_requirements 35 | */src/flake8_requirements 36 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2017-2022 Arkadiusz Bokowy 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0", "wheel", "build"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "flake8-requirements" 7 | # NOTE: Keep in sync with src/flake8_requirements/checker.py file. 8 | version = "2.3.0" 9 | description = "Package requirements checker, plugin for flake8" 10 | readme = "README.rst" 11 | authors = [ { name = "Arkadiusz Bokowy", email = "arkadiusz.bokowy@gmail.com" } ] 12 | requires-python = ">=3.8" 13 | classifiers = [ 14 | "Framework :: Flake8", 15 | "Intended Audience :: Developers", 16 | "License :: OSI Approved :: MIT License", 17 | "Programming Language :: Python", 18 | "Programming Language :: Python :: 3", 19 | "Programming Language :: Python :: 3 :: Only", 20 | "Topic :: Software Development :: Libraries :: Python Modules", 21 | "Topic :: Software Development :: Quality Assurance", 22 | ] 23 | dependencies = [ 24 | "flake8 >= 4.0.0", 25 | "tomli>=1.2.1; python_version < '3.11'", 26 | "packaging >= 20.0", 27 | ] 28 | 29 | [project.optional-dependencies] 30 | pyproject = ["Flake8-pyproject"] 31 | 32 | [project.urls] 33 | Homepage = "https://github.com/arkq/flake8-requirements" 34 | 35 | [project.entry-points."flake8.extension"] 36 | I90 = "flake8_requirements:Flake8Checker" 37 | 38 | [tool.doc8] 39 | max-line-length = 99 40 | 41 | [tool.isort] 42 | force_single_line = true 43 | -------------------------------------------------------------------------------- /test/test_requirements.txt: -------------------------------------------------------------------------------- 1 | ####### requirements.txt ####### 2 | 3 | ###### Requirements without Version Specifiers ###### 4 | nose 5 | 6 | ###### Requirements with Version Specifiers ###### 7 | apache == 0.6.9 # Version Matching. Must be version 0.6.9 8 | coverage[test, graph] ~= 3.1 # Compatible release. Same as >= 3.1, == 3.* 9 | graph >=1.2, <2.0 ; python_version < '3.8' 10 | 11 | ###### Global options ###### 12 | --find-links http://some.archives.com/archives 13 | --no-index 14 | 15 | ###### Requirements with in-line options ###### 16 | foo-project >= 1.2 --install-option="--prefix=/usr/local --no-compile" 17 | bar-project == 8.8 --hash=sha256:cecb534b7d0022683d030b048a2d679c6ff3df969fd7b847027f1ed8d739ac8c \ 18 | --hash=md5:a540092b44178949e8d63ddd7a74f95d 19 | 20 | ###### Requirements from a particular file ###### 21 | /opt/configuration.tar.gz # Local configuration 22 | /opt/blackBox-1.4.4-cp34-none-win32.whl # Local proprietary package 23 | http://example.com/snapshot-builds/exPackage_paint-1.4.8.dev1984+49a8814-cp34-none-win_amd64.whl 24 | 25 | ###### Requirements from a VCS ###### 26 | git+git://github.com/path/to/package-one@releases/tag/v3.1.4#egg=package-one 27 | git+git://github.com/path/to/package-two@master#egg=package-two&subdirectory=src 28 | 29 | ###### Install local project in a develop mode ###### 30 | --editable /opt/whiteBox # Local proprietary package 31 | 32 | ###### Install THIS project in a develop mode ###### 33 | --editable .[foo,bar] 34 | --editable . 35 | -------------------------------------------------------------------------------- /.github/workflows/check.yaml: -------------------------------------------------------------------------------- 1 | name: Check Python Package 2 | 3 | on: 4 | push: 5 | pull_request: 6 | branches: [ master ] 7 | 8 | permissions: 9 | actions: read 10 | contents: read 11 | security-events: write 12 | 13 | jobs: 14 | 15 | check: 16 | strategy: 17 | fail-fast: false 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v5 21 | - uses: actions/setup-python@v6 22 | with: 23 | python-version: '3.x' 24 | - name: Install dependencies 25 | run: pip install tox 26 | - name: Run Tests 27 | run: tox -e py3 28 | 29 | code-ql: 30 | strategy: 31 | fail-fast: false 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/checkout@v5 35 | - name: Initialize CodeQL 36 | uses: github/codeql-action/init@v3 37 | with: 38 | languages: python 39 | queries: security-and-quality 40 | - name: Perform CodeQL Analysis 41 | uses: github/codeql-action/analyze@v3 42 | 43 | doc8-lint: 44 | strategy: 45 | fail-fast: false 46 | runs-on: ubuntu-latest 47 | steps: 48 | - uses: actions/checkout@v5 49 | - name: Run reStructuredText Linter 50 | uses: deep-entertainment/doc8-action@v5 51 | with: 52 | scanPaths: ${{ github.workspace }} 53 | 54 | flake8-lint: 55 | strategy: 56 | fail-fast: false 57 | runs-on: ubuntu-latest 58 | steps: 59 | - uses: actions/checkout@v5 60 | - uses: actions/setup-python@v6 61 | with: 62 | python-version: '3.x' 63 | - name: Run flake8 Linter 64 | run: | 65 | pip install -e . flake8 66 | flake8 --count --show-source --statistics src test 67 | -------------------------------------------------------------------------------- /test/test_poetry.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | from unittest.mock import mock_open 4 | 5 | from flake8_requirements.checker import Flake8Checker 6 | from flake8_requirements.checker import ModuleSet 7 | from flake8_requirements.checker import memoize 8 | 9 | 10 | class PoetryTestCase(unittest.TestCase): 11 | 12 | def setUp(self): 13 | memoize.mem = {} 14 | 15 | def test_get_pyproject_toml_poetry(self): 16 | content = b"[tool.poetry]\nname='x'\n[tool.poetry.tag]\nx=0\n" 17 | with mock.patch('builtins.open', mock_open(read_data=content)): 18 | poetry = Flake8Checker.get_pyproject_toml_poetry() 19 | self.assertDictEqual(poetry, {'name': "x", 'tag': {'x': 0}}) 20 | 21 | def test_1st_party(self): 22 | content = b"[tool.poetry]\nname='book'\n" 23 | 24 | with mock.patch('builtins.open', mock_open()) as m: 25 | m.side_effect = ( 26 | IOError("No such file or directory: 'setup.py'"), 27 | IOError("No such file or directory: 'setup.cfg'"), 28 | mock_open(read_data=content).return_value, 29 | ) 30 | 31 | checker = Flake8Checker(None, None) 32 | mods = checker.get_mods_1st_party() 33 | self.assertEqual(mods, ModuleSet({"book": {}})) 34 | 35 | def test_3rd_party(self): 36 | content = b"[tool.poetry.dependencies]\ntools='1.0'\n" 37 | content += b"[tool.poetry.dev-dependencies]\ndev-tools='1.0'\n" 38 | 39 | with mock.patch('builtins.open', mock_open()) as m: 40 | m.side_effect = ( 41 | IOError("No such file or directory: 'setup.py'"), 42 | IOError("No such file or directory: 'setup.cfg'"), 43 | mock_open(read_data=content).return_value, 44 | ) 45 | 46 | checker = Flake8Checker(None, None) 47 | mods = checker.get_mods_3rd_party(False) 48 | self.assertEqual(mods, ModuleSet({"tools": {}, "dev_tools": {}})) 49 | 50 | def test_3rd_party_groups(self): 51 | content = b"[tool.poetry.dependencies]\ntools='1.0'\n" 52 | content += b"[tool.poetry.group.dev.dependencies]\ndev-tools='1.0'\n" 53 | 54 | with mock.patch('builtins.open', mock_open()) as m: 55 | m.side_effect = ( 56 | IOError("No such file or directory: 'setup.py'"), 57 | IOError("No such file or directory: 'setup.cfg'"), 58 | mock_open(read_data=content).return_value, 59 | ) 60 | 61 | checker = Flake8Checker(None, None) 62 | mods = checker.get_mods_3rd_party(False) 63 | self.assertEqual(mods, ModuleSet({"tools": {}, "dev_tools": {}})) 64 | -------------------------------------------------------------------------------- /test/test_setup.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import os 3 | import unittest 4 | from unittest import mock 5 | from unittest.mock import mock_open 6 | 7 | from flake8_requirements.checker import Flake8Checker 8 | from flake8_requirements.checker import SetupVisitor 9 | from flake8_requirements.checker import parse_requirements 10 | 11 | 12 | class SetupTestCase(unittest.TestCase): 13 | 14 | def test_detect_setup(self): 15 | code = "setup({})".format(",".join(( 16 | "name='A'", 17 | "version='1'", 18 | "author='A'", 19 | "packages=['']", 20 | "url='URL'", 21 | ))) 22 | setup = SetupVisitor(ast.parse(code), "") 23 | self.assertEqual(setup.redirected, True) 24 | self.assertDictEqual(setup.keywords, { 25 | 'name': 'A', 26 | 'version': '1', 27 | 'packages': [''], 28 | 'author': 'A', 29 | 'url': 'URL', 30 | }) 31 | 32 | code = "setup({})".format(",".join( 33 | "{}='{}'".format(x, x) 34 | for x in SetupVisitor.attributes 35 | )) 36 | setup = SetupVisitor(ast.parse(code), "") 37 | self.assertEqual(setup.redirected, True) 38 | self.assertDictEqual(setup.keywords, { 39 | x: x for x in SetupVisitor.attributes 40 | }) 41 | 42 | code = "setup({})".format(",".join(( 43 | "name='A'", 44 | "version='1'", 45 | "package='ABC'", 46 | "processing=True", 47 | "verbose=True", 48 | ))) 49 | setup = SetupVisitor(ast.parse(code), "") 50 | self.assertEqual(setup.redirected, False) 51 | 52 | def test_detect_setup_wrong_num_of_args(self): 53 | setup = SetupVisitor(ast.parse("setup(name='A')"), "") 54 | self.assertEqual(setup.redirected, False) 55 | 56 | def test_detect_setup_wrong_function(self): 57 | setup = SetupVisitor(ast.parse("setup(1, name='A')"), "") 58 | self.assertEqual(setup.redirected, False) 59 | 60 | def test_detect_setup_oops(self): 61 | setup = SetupVisitor(ast.parse("\n".join(( 62 | "from .myModule import setup", 63 | "setup({})".format(",".join(( 64 | "name='A'", 65 | "version='1'", 66 | "author='A'", 67 | "packages=['']", 68 | "url='URL'", 69 | ))), 70 | ))), "") 71 | self.assertEqual(setup.redirected, False) 72 | 73 | def test_get_requirements(self): 74 | setup = SetupVisitor(ast.parse("setup(**{})".format(str({ 75 | 'name': 'A', 76 | 'version': '1', 77 | 'packages': [''], 78 | 'install_requires': ["ABC > 1.0.0", "bar.cat > 2, < 3"], 79 | 'extras_require': { 80 | 'extra': ["extra < 10"], 81 | }, 82 | }))), "") 83 | self.assertEqual(setup.redirected, True) 84 | self.assertEqual( 85 | sorted(setup.get_requirements(), key=lambda x: x.name), 86 | sorted(parse_requirements([ 87 | "ABC > 1.0.0", 88 | "bar.cat > 2, < 3", 89 | "extra < 10", 90 | ]), key=lambda x: x.name), 91 | ) 92 | 93 | def test_get_setup_cfg_requirements(self): 94 | curdir = os.path.abspath(os.path.dirname(__file__)) 95 | with open(os.path.join(curdir, "test_setup.cfg")) as f: 96 | content = f.read() 97 | with mock.patch('builtins.open', mock_open(read_data=content)): 98 | checker = Flake8Checker(None, None) 99 | self.assertEqual( 100 | checker.get_setup_cfg_requirements(False), 101 | list(parse_requirements([ 102 | "requests", 103 | "importlib; python_version == \"2.6\"", 104 | "pytest", 105 | "ReportLab>=1.2", 106 | "docutils>=0.3", 107 | ])), 108 | ) 109 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | Package requirements checker 2 | ============================ 3 | 4 | This module provides a plug-in for `flake8 `_, which checks/validates 5 | package import requirements. It reports missing and/or not used project direct dependencies. 6 | 7 | This plug-in adds new flake8 warnings: 8 | 9 | - ``I900``: Package is not listed as a requirement. 10 | - ``I901``: Package is required but not used. (not implemented yet) 11 | 12 | Important notice 13 | ---------------- 14 | 15 | In order to collect project's dependencies, this checker evaluates Python code from the 16 | ``setup.py`` file stored in the project's root directory. Code evaluation is done with the `eval() 17 | `_ function. As a fall-back method, this 18 | checker also tries to load dependencies, in order, from the ``setup.cfg``, the ``pyproject.toml`` 19 | file from the `PEP 621 `_ project section, the ``pyproject.toml`` 20 | file from the `poetry `_ tool section, or from the 21 | ``requirements.txt`` text file in the project's root directory. 22 | 23 | At this point it is very important to be aware of the consequences of the above approach. One 24 | might inject malicious code into the ``setup.py`` file, which will be executed by this checker. 25 | Hence, this checker shall NEVER be use to check code from an unknown source! However, in most 26 | cases, one validates code from a known source (e.g. own code) and one will run script stored in 27 | the ``setup.py`` file anyway. The worst case scenario is, that this checker will execute the 28 | equivalent of the ``python setup.py``, which shall be idempotent (it's a horribly designed 29 | ``setup.py`` file if it's not). 30 | 31 | If you have noticed some side effects during the ``flake8`` check and your ``setup.py`` file is 32 | written in a standard way (e.g. `pypa-sampleproject 33 | `_), please fill out a bug report. 34 | 35 | Installation 36 | ------------ 37 | 38 | You can install, upgrade, or uninstall ``flake8-requirements`` with these commands:: 39 | 40 | $ pip install flake8-requirements 41 | $ pip install --upgrade flake8-requirements 42 | $ pip uninstall flake8-requirements 43 | 44 | Customization 45 | ------------- 46 | 47 | For projects with custom (private) dependencies, one can provide mapping between project name and 48 | provided modules. Such a mapping can be set on the command line during the flake8 invocation with 49 | the ``--known-modules`` option or alternatively in the ``[flake8]`` section of the configuration 50 | file, e.g. ``setup.cfg``. The syntax of the custom mapping looks like follows:: 51 | 52 | 1st-project-name:[module1,module2,...],2nd-project-name:[moduleA,moduleB,...],... 53 | 54 | If some local project lacks "name" attribute in the ``setup.py`` file (it is highly discouraged 55 | not to provide the "name" attribute, though), one can omit the project name in the mapping and do 56 | as follows:: 57 | 58 | :[localmodule1,localmodule2,...],1st-local-library:[moduleA,moduleB,...],... 59 | 60 | Real life example:: 61 | 62 | $ cat setup.cfg 63 | [flake8] 64 | max-line-length = 100 65 | known-modules = my-lib:[mylib.drm,mylib.encryption] 66 | 67 | If you use `Flake8-pyproject `_ 68 | (can include for installation using ``flake8-requirements[pyproject]``), 69 | you can also configure the known modules using a nicer syntax in ``pyproject.toml``:: 70 | 71 | $ cat pyproject.toml 72 | ... 73 | [tool.flake8] 74 | max-line-length = 100 75 | 76 | [tool.flake8.known-modules] 77 | my-lib = ["mylib.drm", "mylib.encryption"] 78 | 79 | Note that if the module's name contains dots, you have to quote it in pyproject.toml (e.g. 80 | ``"my_namespace.my_lib" = [...]``). 81 | 82 | It is also possible to scan host's site-packages directory for installed packages. This feature is 83 | disabled by default, but user can enable it with the ``--scan-host-site-packages`` command line 84 | option. Please note, however, that the location of the site-packages directory will be determined 85 | by the Python version used for flake8 execution. 86 | 87 | In order to read requirements from the text file, user shall provide the location of such a file 88 | with the ``--requirements-file`` option. If the given location is not an absolute path, then it 89 | has to be specified as a path relative to the project's root directory. 90 | 91 | If you use the ``-r`` flag in your requirements text file with more than one level of recursion 92 | (in other words, one file includes another, the included file includes yet another, and so on), 93 | add the ``--requirements-max-depth`` option to flake8 (for example, ``--requirements-max-depth=3`` 94 | to allow three levels of recursion). 95 | 96 | FAQ 97 | --- 98 | 99 | | **Q:** Package is added to the requirements, but flake8 still reports "I900 '' not listed 100 | as a requirement". 101 | | **A:** It happens when the name of the package is not the same as the name of the module. In such 102 | a case, you have to provide the mapping between the package name and the module name. See 103 | the "`Customization <#customization>`_" section for more details. If the package for which 104 | that happens is a well-known package, please fill out a bug report or add mapping to the 105 | `KNOWN_3RD_PARTIES `_ and submit a pull request. 106 | -------------------------------------------------------------------------------- /test/test_checker.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import unittest 3 | 4 | from flake8_requirements import checker 5 | 6 | 7 | class SetupVisitorMock(checker.SetupVisitor): 8 | 9 | def __init__(self): 10 | self.redirected = True 11 | self.keywords = { 12 | 'name': "flake8-requires", 13 | 'install_requires': [ 14 | "foo", 15 | "bar", 16 | "hyp-hen", 17 | "python-boom", 18 | "python-snake", 19 | "pillow", 20 | "space.module", 21 | ], 22 | } 23 | 24 | 25 | class Flake8Checker(checker.Flake8Checker): 26 | 27 | @classmethod 28 | def get_setup_py(cls): 29 | return SetupVisitorMock() 30 | 31 | @staticmethod 32 | def is_project_setup_py(project_root_dir, filename): 33 | return filename == "setup.py" 34 | 35 | 36 | class Flake8OptionManagerMock(dict): 37 | 38 | def add_option(self, name, **kw): 39 | self[name] = kw 40 | 41 | 42 | class Flake8Options: 43 | known_modules = "" 44 | requirements_file = None 45 | requirements_max_depth = 1 46 | scan_host_site_packages = False 47 | 48 | 49 | def check(code, filename="", options=None): 50 | if options is None: 51 | options = Flake8Options 52 | checker.memoize.mem = {} 53 | Flake8Checker.parse_options(options) 54 | return list(Flake8Checker(ast.parse(code), filename).run()) 55 | 56 | 57 | class Flake8CheckerTestCase(unittest.TestCase): 58 | 59 | def test_add_options(self): 60 | manager = Flake8OptionManagerMock() 61 | Flake8Checker.add_options(manager) 62 | self.assertEqual( 63 | sorted(manager.keys()), 64 | ['--known-modules', '--requirements-file', 65 | '--requirements-max-depth', '--scan-host-site-packages'], 66 | ) 67 | 68 | def test_stdlib(self): 69 | errors = check("import os\nfrom unittest import TestCase") 70 | self.assertEqual(len(errors), 0) 71 | 72 | def test_stdlib_case(self): 73 | errors = check("from cProfile import Profile") 74 | self.assertEqual(len(errors), 0) 75 | errors = check("from cprofile import Profile") 76 | self.assertEqual(len(errors), 1) 77 | self.assertEqual( 78 | errors[0][2], 79 | "I900 'cprofile' not listed as a requirement", 80 | ) 81 | 82 | def test_1st_party(self): 83 | errors = check("import flake8_requires") 84 | self.assertEqual(len(errors), 0) 85 | 86 | def test_3rd_party(self): 87 | errors = check("import foo\nfrom bar import Bar") 88 | self.assertEqual(len(errors), 0) 89 | 90 | def test_3rd_party_python_prefix(self): 91 | errors = check("from boom import blast") 92 | self.assertEqual(len(errors), 0) 93 | 94 | def test_3rd_party_python_prefix_no_strip(self): 95 | errors = check("import python_snake as snake") 96 | self.assertEqual(len(errors), 0) 97 | 98 | def test_3rd_party_missing(self): 99 | errors = check("import os\nfrom cat import Cat") 100 | self.assertEqual(len(errors), 1) 101 | self.assertEqual( 102 | errors[0][2], 103 | "I900 'cat' not listed as a requirement", 104 | ) 105 | 106 | def test_3rd_party_hyphen(self): 107 | errors = check("from hyp_hen import Hyphen") 108 | self.assertEqual(len(errors), 0) 109 | 110 | def test_3rd_party_known_module(self): 111 | errors = check("import PIL") 112 | self.assertEqual(len(errors), 0) 113 | 114 | def test_non_top_level_import(self): 115 | errors = check("def function():\n import cat") 116 | self.assertEqual(len(errors), 1) 117 | self.assertEqual( 118 | errors[0][2], 119 | "I900 'cat' not listed as a requirement", 120 | ) 121 | 122 | def test_namespace(self): 123 | errors = check("import space.module") 124 | self.assertEqual(len(errors), 0) 125 | errors = check("from space import module") 126 | self.assertEqual(len(errors), 0) 127 | errors = check("import space") 128 | self.assertEqual(len(errors), 1) 129 | 130 | def test_relative(self): 131 | errors = check("from . import local") 132 | self.assertEqual(len(errors), 0) 133 | errors = check("from ..local import local") 134 | self.assertEqual(len(errors), 0) 135 | 136 | def test_discover_host_3rd_party_modules(self): 137 | class Options(Flake8Options): 138 | scan_host_site_packages = True 139 | Flake8Checker.parse_options(Options) 140 | self.assertEqual( 141 | type(Flake8Checker.known_host_3rd_parties), 142 | dict, 143 | ) 144 | # Since flake8-requirements (this package) is a plugin for flake8, it 145 | # is very likely that one will have flake8 installed in the host 146 | # site-packages. However, that is not the case for our GitHub Actions 147 | # runners, so we can not enforce this assertion. 148 | if 'flake8' in Flake8Checker.known_host_3rd_parties: 149 | self.assertEqual( 150 | Flake8Checker.known_host_3rd_parties['flake8'], 151 | ['flake8'], 152 | ) 153 | 154 | def test_custom_mapping_parser(self): 155 | class Options(Flake8Options): 156 | known_modules = ":[pydrmcodec],mylib:[mylib.drm,mylib.ex]" 157 | Flake8Checker.parse_options(Options) 158 | self.assertEqual( 159 | Flake8Checker.known_modules, 160 | {"": ["pydrmcodec"], "mylib": ["mylib.drm", "mylib.ex"]}, 161 | ) 162 | 163 | def test_custom_mapping(self): 164 | class Options(Flake8Options): 165 | known_modules = "flake8-requires:[flake8req]" 166 | errors = check("from flake8req import mymodule", options=Options) 167 | self.assertEqual(len(errors), 0) 168 | 169 | def test_setup_py(self): 170 | errors = check("from setuptools import setup", "setup.py") 171 | self.assertEqual(len(errors), 0) 172 | # mods_3rd_party 173 | errors = check("from setuptools import setup", "xxx.py") 174 | self.assertEqual(len(errors), 1) 175 | self.assertEqual( 176 | errors[0][2], 177 | "I900 'setuptools' not listed as a requirement", 178 | ) 179 | -------------------------------------------------------------------------------- /test/test_pep621.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | from unittest import mock 3 | from unittest.mock import mock_open 4 | from unittest.mock import patch 5 | 6 | from flake8_requirements.checker import Flake8Checker 7 | from flake8_requirements.checker import ModuleSet 8 | from flake8_requirements.checker import memoize 9 | from flake8_requirements.checker import parse_requirements 10 | 11 | 12 | class Flake8Options: 13 | known_modules = "" 14 | requirements_file = None 15 | requirements_max_depth = 1 16 | scan_host_site_packages = False 17 | 18 | 19 | class Pep621TestCase(unittest.TestCase): 20 | 21 | content = b""" 22 | [project] 23 | name="test" 24 | dependencies=["tools==1.0"] 25 | 26 | [project.optional-dependencies] 27 | dev = ["dev-tools==1.0"] 28 | """ 29 | 30 | def setUp(self): 31 | memoize.mem = {} 32 | 33 | def tearDown(self): 34 | Flake8Checker.root_dir = "" 35 | 36 | def test_pyproject_custom_mapping_parser(self): 37 | class Options(Flake8Options): 38 | known_modules = {"mylib": ["mylib.drm", "mylib.ex"]} 39 | Flake8Checker.parse_options(Options) 40 | self.assertEqual( 41 | Flake8Checker.known_modules, 42 | {"mylib": ["mylib.drm", "mylib.ex"]}, 43 | ) 44 | 45 | def test_get_pyproject_toml_pep621(self): 46 | with mock.patch('builtins.open', mock_open(read_data=self.content)): 47 | pep621 = Flake8Checker.get_pyproject_toml_pep621() 48 | expected = { 49 | "name": "test", 50 | "dependencies": ["tools==1.0"], 51 | "optional-dependencies": { 52 | "dev": ["dev-tools==1.0"] 53 | }, 54 | } 55 | self.assertDictEqual(pep621, expected) 56 | 57 | def test_get_pyproject_toml_invalid(self): 58 | content = self.content + b"invalid" 59 | with mock.patch('builtins.open', mock_open(read_data=content)): 60 | self.assertDictEqual(Flake8Checker.get_pyproject_toml_pep621(), {}) 61 | 62 | def test_1st_party(self): 63 | with mock.patch('builtins.open', mock_open()) as m: 64 | m.side_effect = ( 65 | IOError("No such file or directory: 'setup.py'"), 66 | IOError("No such file or directory: 'setup.cfg'"), 67 | mock_open(read_data=self.content).return_value, 68 | ) 69 | 70 | checker = Flake8Checker(None, None) 71 | mods = checker.get_mods_1st_party() 72 | self.assertEqual(mods, ModuleSet({"test": {}})) 73 | 74 | def test_3rd_party(self): 75 | with mock.patch('builtins.open', mock_open()) as m: 76 | m.side_effect = ( 77 | IOError("No such file or directory: 'setup.py'"), 78 | IOError("No such file or directory: 'setup.cfg'"), 79 | mock_open(read_data=self.content).return_value, 80 | ) 81 | 82 | checker = Flake8Checker(None, None) 83 | mods = checker.get_mods_3rd_party(False) 84 | self.assertEqual(mods, ModuleSet({"tools": {}, "dev_tools": {}})) 85 | 86 | def test_dynamic_requirements(self): 87 | requirements_content = "package1\npackage2>=2.0" 88 | data = { 89 | "project": {"dynamic": ["dependencies"]}, 90 | "tool": { 91 | "setuptools": { 92 | "dynamic": {"dependencies": {"file": ["requirements.txt"]}} 93 | } 94 | }, 95 | } 96 | with patch( 97 | 'flake8_requirements.checker.Flake8Checker.get_pyproject_toml', 98 | return_value=data, 99 | ): 100 | with patch( 101 | 'builtins.open', mock_open(read_data=requirements_content) 102 | ): 103 | result = Flake8Checker.get_setuptools_dynamic_requirements() 104 | expected_results = ['package1', 'package2>=2.0'] 105 | parsed_results = [str(req) for req in result] 106 | self.assertEqual(parsed_results, expected_results) 107 | 108 | def test_dynamic_optional_dependencies(self): 109 | data = { 110 | "project": {"dynamic": ["dependencies", "optional-dependencies"]}, 111 | "tool": { 112 | "setuptools": { 113 | "dynamic": { 114 | "dependencies": {"file": ["requirements.txt"]}, 115 | "optional-dependencies": { 116 | "test": {"file": ["optional-requirements.txt"]} 117 | }, 118 | } 119 | } 120 | }, 121 | } 122 | requirements_content = """ 123 | package1 124 | package2>=2.0 125 | """ 126 | optional_requirements_content = "package3[extra] >= 3.0" 127 | with mock.patch( 128 | 'flake8_requirements.checker.Flake8Checker.get_pyproject_toml', 129 | return_value=data, 130 | ): 131 | with mock.patch('builtins.open', mock.mock_open()) as mocked_file: 132 | mocked_file.side_effect = [ 133 | mock.mock_open( 134 | read_data=requirements_content 135 | ).return_value, 136 | mock.mock_open( 137 | read_data=optional_requirements_content 138 | ).return_value, 139 | ] 140 | result = Flake8Checker.get_setuptools_dynamic_requirements() 141 | expected = list(parse_requirements( 142 | requirements_content.splitlines())) 143 | expected += list(parse_requirements( 144 | optional_requirements_content.splitlines())) 145 | 146 | self.assertEqual(len(result), len(expected)) 147 | for i in range(len(result)): 148 | self.assertEqual(result[i], expected[i]) 149 | 150 | def test_missing_requirements_file(self): 151 | data = { 152 | "project": {"dynamic": ["dependencies"]}, 153 | "tool": { 154 | "setuptools": { 155 | "dynamic": { 156 | "dependencies": { 157 | "file": ["nonexistent-requirements.txt"] 158 | } 159 | } 160 | } 161 | }, 162 | } 163 | with mock.patch( 164 | 'flake8_requirements.checker.Flake8Checker.get_pyproject_toml', 165 | return_value=data, 166 | ): 167 | result = Flake8Checker.get_setuptools_dynamic_requirements() 168 | self.assertEqual(result, []) 169 | -------------------------------------------------------------------------------- /test/test_requirements.py: -------------------------------------------------------------------------------- 1 | import os 2 | import unittest 3 | from collections import OrderedDict 4 | from unittest import mock 5 | from unittest.mock import mock_open 6 | 7 | from flake8_requirements.checker import Flake8Checker 8 | from flake8_requirements.checker import memoize 9 | from flake8_requirements.checker import parse_requirements 10 | 11 | 12 | def mock_open_with_name(read_data="", name="file.name"): 13 | """Mock open call with a specified `name` attribute.""" 14 | m = mock_open(read_data=read_data) 15 | m.return_value.name = name 16 | return m 17 | 18 | 19 | def mock_open_multiple(files=OrderedDict()): 20 | """Create a mock open object for multiple files.""" 21 | m = mock_open() 22 | m.side_effect = [ 23 | mock_open_with_name(read_data=content, name=name).return_value 24 | for name, content in files.items() 25 | ] 26 | return m 27 | 28 | 29 | class RequirementsTestCase(unittest.TestCase): 30 | 31 | def setUp(self): 32 | memoize.mem = {} 33 | 34 | def test_resolve_requirement(self): 35 | self.assertEqual( 36 | Flake8Checker.resolve_requirement("foo >= 1.0.0"), 37 | ["foo >= 1.0.0"], 38 | ) 39 | 40 | def test_resolve_requirement_with_option(self): 41 | self.assertEqual( 42 | Flake8Checker.resolve_requirement("foo-bar.v1==1.0 --hash=md5:."), 43 | ["foo-bar.v1==1.0"], 44 | ) 45 | 46 | def test_resolve_requirement_standalone_option(self): 47 | self.assertEqual( 48 | Flake8Checker.resolve_requirement("--extra-index-url"), 49 | [], 50 | ) 51 | 52 | def test_resolve_requirement_with_file_beyond_max_depth(self): 53 | with self.assertRaises(RuntimeError): 54 | Flake8Checker.resolve_requirement("-r requirements.txt") 55 | 56 | def test_resolve_requirement_with_file_empty(self): 57 | with mock.patch('builtins.open', mock_open()) as m: 58 | self.assertEqual( 59 | Flake8Checker.resolve_requirement("-r requirements.txt", 1), 60 | [], 61 | ) 62 | m.assert_called_once_with("requirements.txt") 63 | 64 | def test_resolve_requirement_with_file_content(self): 65 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 66 | ("requirements.txt", "foo >= 1.0.0\nbar <= 1.0.0\n"), 67 | )))): 68 | self.assertEqual( 69 | Flake8Checker.resolve_requirement("-r requirements.txt", 1), 70 | ["foo >= 1.0.0", "bar <= 1.0.0"], 71 | ) 72 | 73 | def test_resolve_requirement_with_file_content_line_continuation(self): 74 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 75 | ("requirements.txt", "foo[bar] \\\n>= 1.0.0\n"), 76 | )))): 77 | self.assertEqual( 78 | Flake8Checker.resolve_requirement("-r requirements.txt", 1), 79 | ["foo[bar] >= 1.0.0"], 80 | ) 81 | 82 | def test_resolve_requirement_with_file_content_line_continuation_2(self): 83 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 84 | ("requirements.txt", "foo \\\n>= 1.0.0 \\\n# comment \\\nbar \\"), 85 | )))): 86 | self.assertEqual( 87 | Flake8Checker.resolve_requirement("-r requirements.txt", 1), 88 | ["foo >= 1.0.0", "bar"], 89 | ) 90 | 91 | def test_resolve_requirement_with_file_recursion_beyond_max_depth(self): 92 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 93 | ("requirements.txt", "-r requirements.txt\n"), 94 | )))): 95 | with self.assertRaises(RuntimeError): 96 | Flake8Checker.resolve_requirement("-r requirements.txt", 1), 97 | 98 | def test_resolve_requirement_with_file_recursion(self): 99 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 100 | ("requirements.txt", "--requirement inner.txt\nbar <= 1.0.0\n"), 101 | ("inner.txt", "# inner\nbaz\n\nqux\n"), 102 | )))): 103 | self.assertEqual( 104 | Flake8Checker.resolve_requirement("-r requirements.txt", 2), 105 | ["baz", "qux", "bar <= 1.0.0"], 106 | ) 107 | 108 | def test_resolve_requirement_with_relative_include(self): 109 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 110 | ("requirements.txt", "-r requirements/production.txt"), 111 | ("requirements/production.txt", "-r node/one.txt\nfoo"), 112 | ("requirements/node/one.txt", "-r common.txt\n-r /abs/path.txt"), 113 | ("requirements/node/common.txt", "bar"), 114 | ("/abs/path.txt", "bis"), 115 | )))) as m: 116 | self.assertEqual( 117 | Flake8Checker.resolve_requirement("-r requirements.txt", 5), 118 | ["bar", "bis", "foo"], 119 | ) 120 | m.assert_has_calls([ 121 | mock.call("requirements.txt"), 122 | mock.call("requirements/production.txt"), 123 | mock.call("requirements/node/one.txt"), 124 | mock.call("requirements/node/common.txt"), 125 | mock.call("/abs/path.txt"), 126 | ]) 127 | 128 | def test_init_with_no_requirements(self): 129 | with mock.patch('builtins.open', mock_open()) as m: 130 | m.side_effect = IOError("No such file or directory"), 131 | checker = Flake8Checker(None, None) 132 | self.assertEqual(checker.get_requirements_txt(), ()) 133 | 134 | def test_init_with_user_requirements(self): 135 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 136 | ("requirements/base.txt", "foo >= 1.0.0\n-r inner.txt\n"), 137 | ("requirements/inner.txt", "bar\n"), 138 | )))) as m: 139 | try: 140 | Flake8Checker.requirements_file = "requirements/base.txt" 141 | checker = Flake8Checker(None, None) 142 | self.assertEqual( 143 | checker.get_requirements_txt(), 144 | tuple(parse_requirements([ 145 | "foo >= 1.0.0", 146 | "bar", 147 | ])), 148 | ) 149 | m.assert_has_calls([ 150 | mock.call("requirements/base.txt"), 151 | mock.call("requirements/inner.txt"), 152 | ]) 153 | finally: 154 | Flake8Checker.requirements_file = None 155 | 156 | def test_init_with_simple_requirements(self): 157 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 158 | ("requirements.txt", "foo >= 1.0.0\nbar <= 1.0.0\n"), 159 | )))): 160 | checker = Flake8Checker(None, None) 161 | self.assertEqual( 162 | checker.get_requirements_txt(), 163 | tuple(parse_requirements([ 164 | "foo >= 1.0.0", 165 | "bar <= 1.0.0", 166 | ])), 167 | ) 168 | 169 | def test_init_with_recursive_requirements_beyond_max_depth(self): 170 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 171 | ("requirements.txt", "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n"), 172 | ("inner.txt", "# inner\nbaz\n\nqux\n"), 173 | )))): 174 | with self.assertRaises(RuntimeError): 175 | try: 176 | Flake8Checker.requirements_max_depth = 0 177 | checker = Flake8Checker(None, None) 178 | checker.get_requirements_txt() 179 | finally: 180 | Flake8Checker.requirements_max_depth = 1 181 | 182 | def test_init_with_recursive_requirements(self): 183 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 184 | ("requirements.txt", "foo >= 1.0.0\n-r inner.txt\nbar <= 1.0.0\n"), 185 | ("inner.txt", "# inner\nbaz\n\nqux\n"), 186 | )))): 187 | checker = Flake8Checker(None, None) 188 | self.assertEqual( 189 | checker.get_requirements_txt(), 190 | tuple(parse_requirements([ 191 | "foo >= 1.0.0", 192 | "baz", 193 | "qux", 194 | "bar <= 1.0.0", 195 | ])), 196 | ) 197 | 198 | def test_init_misc(self): 199 | curdir = os.path.abspath(os.path.dirname(__file__)) 200 | with open(os.path.join(curdir, "test_requirements.txt")) as f: 201 | requirements_content = f.read() 202 | with mock.patch('builtins.open', mock_open_multiple(files=OrderedDict(( 203 | ("requirements.txt", requirements_content), 204 | )))): 205 | checker = Flake8Checker(None, None) 206 | self.assertEqual( 207 | checker.get_requirements_txt(), 208 | tuple(parse_requirements([ 209 | "nose", 210 | "apache == 0.6.9", 211 | "coverage[graph,test] ~= 3.1", 212 | "graph <2.0, >=1.2 ; python_version < '3.8'", 213 | "foo-project >= 1.2", 214 | "bar-project == 8.8", 215 | "configuration", 216 | "blackBox == 1.4.4", 217 | "exPackage_paint == 1.4.8.dev1984+49a8814", 218 | "package-one", 219 | "package-two", 220 | "whiteBox", 221 | ])), 222 | ) 223 | -------------------------------------------------------------------------------- /src/flake8_requirements/modules.py: -------------------------------------------------------------------------------- 1 | # List of all modules (standard library) available in Python 3. 2 | STDLIB_PY3 = ( 3 | "__future__", 4 | "__main__", 5 | "_dummy_thread", 6 | "_thread", 7 | "abc", 8 | "aifc", 9 | "argparse", 10 | "array", 11 | "ast", 12 | "asynchat", 13 | "asyncio", 14 | "asyncore", 15 | "atexit", 16 | "audioop", 17 | "base64", 18 | "bdb", 19 | "binascii", 20 | "binhex", 21 | "bisect", 22 | "builtins", 23 | "bz2", 24 | "cProfile", 25 | "calendar", 26 | "cgi", 27 | "cgitb", 28 | "chunk", 29 | "cmath", 30 | "cmd", 31 | "code", 32 | "codecs", 33 | "codeop", 34 | "collection", 35 | "collections", 36 | "colorsys", 37 | "compileall", 38 | "concurrent", 39 | "configparser", 40 | "contextlib", 41 | "copy", 42 | "copyreg", 43 | "crypt", 44 | "csv", 45 | "ctypes", 46 | "curses", 47 | "datetime", 48 | "dbm", 49 | "decimal", 50 | "difflib", 51 | "dis", 52 | "distutils", 53 | "doctest", 54 | "dummy_threading", 55 | "email", 56 | "encodings", 57 | "ensurepip", 58 | "enum", 59 | "errno", 60 | "faulthandler", 61 | "fcntl", 62 | "filecmp", 63 | "fileinput", 64 | "fnmatch", 65 | "formatter", 66 | "fpectl", 67 | "fractions", 68 | "ftplib", 69 | "functools", 70 | "gc", 71 | "getopt", 72 | "getpass", 73 | "gettext", 74 | "glob", 75 | "grp", 76 | "gzip", 77 | "hashlib", 78 | "heapq", 79 | "hmac", 80 | "html", 81 | "http", 82 | "imaplib", 83 | "imghdr", 84 | "imp", 85 | "importlib", 86 | "inspect", 87 | "io", 88 | "ipaddress", 89 | "itertools", 90 | "json", 91 | "keyword", 92 | "lib2to3", 93 | "linecache", 94 | "locale", 95 | "logging", 96 | "lzma", 97 | "macpath", 98 | "mailbox", 99 | "mailcap", 100 | "marshal", 101 | "math", 102 | "mimetypes", 103 | "mmap", 104 | "modulefinder", 105 | "msilib", 106 | "msvcrt", 107 | "multiprocessing", 108 | "netrc", 109 | "nis", 110 | "nntplib", 111 | "ntpath", 112 | "numbers", 113 | "operator", 114 | "optparse", 115 | "os", 116 | "ossaudiodev", 117 | "parser", 118 | "pathlib", 119 | "pdb", 120 | "pickle", 121 | "pickletools", 122 | "pipes", 123 | "pkgutil", 124 | "platform", 125 | "plistlib", 126 | "poplib", 127 | "posix", 128 | "posixpath", 129 | "pprint", 130 | "profile", 131 | "pstats", 132 | "pty", 133 | "pwd", 134 | "py_compile", 135 | "pyclbr", 136 | "pydoc", 137 | "queue", 138 | "quopri", 139 | "random", 140 | "re", 141 | "readline", 142 | "reprlib", 143 | "resource", 144 | "rlcompleter", 145 | "runpy", 146 | "sched", 147 | "select", 148 | "selectors", 149 | "shelve", 150 | "shlex", 151 | "shutil", 152 | "signal", 153 | "site", 154 | "smtpd", 155 | "smtplib", 156 | "sndhdr", 157 | "socket", 158 | "socketserver", 159 | "spwd", 160 | "sqlite3", 161 | "ssl", 162 | "stat", 163 | "statistics", 164 | "string", 165 | "stringprep", 166 | "struct", 167 | "subprocess", 168 | "sunau", 169 | "symbol", 170 | "symtable", 171 | "sys", 172 | "sysconfig", 173 | "syslog", 174 | "tabnanny", 175 | "tarfile", 176 | "telnetlib", 177 | "tempfile", 178 | "termios", 179 | "test", 180 | "textwrap", 181 | "threading", 182 | "time", 183 | "timeit", 184 | "tkinter", 185 | "token", 186 | "tokenize", 187 | "trace", 188 | "traceback", 189 | "tracemalloc", 190 | "tty", 191 | "turtle", 192 | "turtledemo", 193 | "types", 194 | "unicodedata", 195 | "unittest", 196 | "urllib", 197 | "uu", 198 | "uuid", 199 | "venv", 200 | "warnings", 201 | "wave", 202 | "weakref", 203 | "webbrowser", 204 | "winreg", 205 | "winsound", 206 | "wsgiref", 207 | "xdrlib", 208 | "xml", 209 | "xmlrpc", 210 | "zipfile", 211 | "zipimport", 212 | "zlib", 213 | # Modules since Python 3.5 214 | "typing", 215 | "zipapp", 216 | # Modules since Python 3.6 217 | "secrets", 218 | # Modules since Python 3.7 219 | "contextvars", 220 | "dataclasses", 221 | # Modules since Python 3.9 222 | "graphlib", 223 | "zoneinfo", 224 | # Modules since Python 3.11 225 | "tomllib", 226 | ) 227 | 228 | # Mapping for known 3rd party projects, which provide more than one module 229 | # or the name of the module is different than the project name itself. 230 | KNOWN_3RD_PARTIES = { 231 | "absl-py": ["absl"], 232 | # NOTE: The allure-pytest package does not provide allure module directly 233 | # but it depends on allure-python-commons which provides it. User 234 | # will most likely specify allure-pytest as a dependency, though. 235 | "allure-pytest": ["allure"], 236 | "ansicolors": ["colors"], 237 | "apache-airflow": ["airflow"], 238 | "appengine-python-standard": ["google.appengine"], 239 | "atlassian-python-api": ["atlassian"], 240 | "attrs": ["attr", "attrs"], 241 | "awesome-slugify": ["slugify"], 242 | "azure-common": ["azure.common"], 243 | "azure-core": ["azure.core"], 244 | "azure-graphrbac": ["azure.graphrbac"], 245 | "azure-identity": ["azure.identity"], 246 | "azure-keyvault": ["azure.keyvault"], 247 | "azure-keyvault-certificates": ["azure.keyvault.certificates"], 248 | "azure-keyvault-keys": ["azure.keyvault.keys"], 249 | "azure-keyvault-secrets": ["azure.keyvault.secrets"], 250 | "azure-mgmt-apimanagement": ["azure.mgmt.apimanagement"], 251 | "azure-mgmt-authorization": ["azure.mgmt.authorization"], 252 | "azure-mgmt-automation": ["azure.mgmt.automation"], 253 | "azure-mgmt-batch": ["azure.mgmt.batch"], 254 | "azure-mgmt-compute": ["azure.mgmt.compute"], 255 | "azure-mgmt-containerinstance": ["azure.mgmt.containerinstance"], 256 | "azure-mgmt-containerregistry": ["azure.mgmt.containerregistry"], 257 | "azure-mgmt-containerservice": ["azure.mgmt.containerservice"], 258 | "azure-mgmt-core": ["azure.mgmt.core"], 259 | "azure-mgmt-cosmosdb": ["azure.mgmt.cosmosdb"], 260 | "azure-mgmt-frontdoor": ["azure.mgmt.frontdoor"], 261 | "azure-mgmt-hybridkubernetes": ["azure.mgmt.hybridkubernetes"], 262 | "azure-mgmt-keyvault": ["azure.mgmt.keyvault"], 263 | "azure-mgmt-logic": ["azure.mgmt.logic"], 264 | "azure-mgmt-managementgroups": ["azure.mgmt.managementgroups"], 265 | "azure-mgmt-monitor": ["azure.mgmt.monitor"], 266 | "azure-mgmt-msi": ["azure.mgmt.msi"], 267 | "azure-mgmt-network": ["azure.mgmt.network"], 268 | "azure-mgmt-rdbms": ["azure.mgmt.rdbms"], 269 | "azure-mgmt-resource": ["azure.mgmt.resource"], 270 | "azure-mgmt-security": ["azure.mgmt.security"], 271 | "azure-mgmt-servicefabric": ["azure.mgmt.servicefabric"], 272 | "azure-mgmt-sql": ["azure.mgmt.sql"], 273 | "azure-mgmt-storage": ["azure.mgmt.storage"], 274 | "azure-mgmt-subscription": ["azure.mgmt.subscription"], 275 | "azure-mgmt-web": ["azure.mgmt.web"], 276 | "azure-storage-blob": ["azure.storage.blob"], 277 | "azure-storage-queue": ["azure.storage.queue"], 278 | "beautifulsoup4": ["bs4"], 279 | "bitvector": ["BitVector"], 280 | "cattrs": ["cattr", "cattrs"], 281 | "cx-oracle": ["cx_Oracle"], 282 | "databricks-connect": ["pyspark"], 283 | "django-ajax-selects": ["ajax_select"], 284 | "django-cors-headers": ["corsheaders"], 285 | "django-csp": ["csp"], 286 | "django-debug-toolbar": ["debug_toolbar"], 287 | "django-dotenv": ["dotenv"], 288 | "django-filter": ["django_filters"], 289 | "django-haystack": ["haystack"], 290 | "django-safedelete": ["safedelete"], 291 | "django-simple-history": ["simple_history"], 292 | "djangorestframework": ["rest_framework"], 293 | "enum34": ["enum"], 294 | "factory-boy": ["factory"], 295 | "ffmpeg-python": ["ffmpeg"], 296 | "fluent-logger": ["fluent"], 297 | "gitpython": ["git"], 298 | "google-api-core": ["google.api_core"], 299 | "google-api-python-client": ["apiclient", "googleapiclient"], 300 | "google-auth": ["google.auth", "google.oauth2"], 301 | "google-cloud-aiplatform": ["google.cloud.aiplatform"], 302 | "google-cloud-bigquery": ["google.cloud.bigquery"], 303 | "google-cloud-bigtable": ["google.cloud.bigtable"], 304 | "google-cloud-datastore": ["google.cloud.datastore"], 305 | "google-cloud-firestore": ["google.cloud.firestore"], 306 | "google-cloud-functions": [ 307 | "google.cloud.functions_v1", 308 | "google.cloud.functions", 309 | ], 310 | "google-cloud-iam": ["google.cloud.iam_credentials_v1"], 311 | "google-cloud-iot": ["google.cloud.iot_v1"], 312 | "google-cloud-logging": [ 313 | "google.cloud.logging_v2", 314 | "google.cloud.logging", 315 | ], 316 | "google-cloud-pubsub": [ 317 | "google.cloud.pubsub_v1", 318 | "google.cloud.pubsub", 319 | "google.pubsub_v1", 320 | "google.pubsub", 321 | ], 322 | "google-cloud-secret-manager": ["google.cloud.secretmanager"], 323 | "google-cloud-storage": ["google.cloud.storage"], 324 | "grpcio": ["grpc"], 325 | "grpcio-channelz": ["grpc_channelz"], 326 | "grpcio-gcp": ["grpc_gcp"], 327 | "grpcio-health-checking": ["grpc_health"], 328 | "grpcio-opentracing": ["grpc_opentracing"], 329 | "grpcio-reflection": ["grpc_reflection"], 330 | "grpcio-status": ["grpc_status"], 331 | "grpcio-testing": ["grpc_testing"], 332 | "grpcio-tools": ["grpc_tools"], 333 | "hydra-core": ["hydra"], 334 | "ipython": ["IPython"], 335 | "jack-client": ["jack"], 336 | "kafka-python": ["kafka"], 337 | "lark-parser": ["lark"], 338 | "mysql-python": ["MySQLdb"], 339 | "mysqlclient": ["_mysql", "MySQLdb"], 340 | "opencv-contrib-python": ["cv2"], 341 | "opencv-contrib-python-headless": ["cv2"], 342 | "opencv-python": ["cv2"], 343 | "opencv-python-headless": ["cv2"], 344 | "opensearch-py": ["opensearchpy"], 345 | "opentelemetry-api": ["opentelemetry"], 346 | "opentelemetry-exporter-otlp-proto-grpc": [ 347 | "opentelemetry.exporter.otlp.proto.grpc" 348 | ], 349 | "opentelemetry-exporter-otlp-proto-http": [ 350 | "opentelemetry.exporter.otlp.proto.http" 351 | ], 352 | "opentelemetry-instrumentation-aiohttp-client": [ 353 | "opentelemetry.instrumentation.aiohttp_client" 354 | ], 355 | "opentelemetry-instrumentation-botocore": [ 356 | "opentelemetry.instrumentation.botocore" 357 | ], 358 | "opentelemetry-instrumentation-django": [ 359 | "opentelemetry.instrumentation.django" 360 | ], 361 | "opentelemetry-instrumentation-elasticsearch": [ 362 | "opentelemetry.instrumentation.elasticsearch" 363 | ], 364 | "opentelemetry-instrumentation-grpc": [ 365 | "opentelemetry.instrumentation.grpc" 366 | ], 367 | "opentelemetry-instrumentation-httpx": [ 368 | "opentelemetry.instrumentation.httpx" 369 | ], 370 | "opentelemetry-instrumentation-jinja2": [ 371 | "opentelemetry.instrumentation.jinja2" 372 | ], 373 | "opentelemetry-instrumentation-psycopg2": [ 374 | "opentelemetry.instrumentation.psycopg2" 375 | ], 376 | "opentelemetry-instrumentation-pymongo": [ 377 | "opentelemetry.instrumentation.pymongo" 378 | ], 379 | "opentelemetry-instrumentation-requests": [ 380 | "opentelemetry.instrumentation.requests" 381 | ], 382 | "opentelemetry-sdk": ["opentelemetry.sdk"], 383 | "opentelemetry-test-utils": ["opentelemetry.test"], 384 | "paho-mqtt": ["paho"], 385 | "phonenumberslite": ["phonenumbers"], 386 | "pillow": ["PIL"], 387 | "pillow-simd": ["PIL"], 388 | "pip-tools": ["piptools"], 389 | "plotly": [ 390 | "jupyterlab_plotly", 391 | "plotly", 392 | "_plotly_utils", 393 | "_plotly_future_", 394 | ], 395 | "progressbar2": ["progressbar"], 396 | "protobuf": ["google.protobuf"], 397 | "psycopg2-binary": ["psycopg2"], 398 | "py-lru-cache": ["lru"], 399 | "pycrypto": ["Crypto"], 400 | "pycryptodome": ["Crypto"], 401 | "pygithub": ["github"], 402 | "pygobject": ["gi", "pygtkcompat"], 403 | "pyhamcrest": ["hamcrest"], 404 | "pyicu": ["icu"], 405 | "pyjwt": ["jwt"], 406 | "pymongo": ["bson", "gridfs", "pymongo"], 407 | "pymupdf": ["fitz"], 408 | "pyopenssl": ["OpenSSL"], 409 | "pypdf2": ["PyPDF2"], 410 | "pypi-kenlm": ["kenlm"], 411 | "pyside": ["PySide", "pysideuic"], 412 | "pyside2": ["PySide2"], 413 | "pyside6": ["PySide6"], 414 | "pytest": ["pytest", "_pytest"], 415 | "pytest-runner": ["ptr"], 416 | "python-levenshtein": ["Levenshtein"], 417 | "python-lsp-jsonrpc": ["pylsp_jsonrpc"], 418 | "pyturbojpeg": ["turbojpeg"], 419 | "pyyaml": ["yaml"], 420 | "scikit-fda": ["skfda"], 421 | "scikit-image": ["skimage"], 422 | "scikit-learn": ["sklearn"], 423 | "setuptools": ["pkg_resources", "setuptools"], 424 | "sorl-thumbnail": ["sorl"], 425 | "splunk-sdk": ["splunklib"], 426 | "streamlit-aggrid": ["st_aggrid"], 427 | "tensorboardx": ["tensorboardX"], 428 | "umap-learn": ["umap"], 429 | "xlwt-future": ["xlwt"], 430 | } 431 | -------------------------------------------------------------------------------- /src/flake8_requirements/checker.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import os 3 | import re 4 | import site 5 | import sys 6 | from collections import namedtuple 7 | from configparser import ConfigParser 8 | from functools import wraps 9 | from logging import getLogger 10 | 11 | from packaging.requirements import Requirement 12 | 13 | if sys.version_info >= (3, 11): 14 | import tomllib 15 | else: 16 | import tomli as tomllib 17 | 18 | from .modules import KNOWN_3RD_PARTIES 19 | from .modules import STDLIB_PY3 20 | 21 | # NOTE: Keep in sync with pyproject.toml file. 22 | __version__ = "2.3.0" 23 | __license__ = "MIT" 24 | 25 | LOG = getLogger('flake8.plugin.requirements') 26 | 27 | ERRORS = { 28 | 'I900': "I900 '{pkg}' not listed as a requirement", 29 | 'I901': "I901 '{pkg}' required but not used", 30 | } 31 | 32 | STDLIB = set() 33 | STDLIB.update(STDLIB_PY3) 34 | 35 | 36 | def memoize(f): 37 | """Cache value returned by the function.""" 38 | @wraps(f) 39 | def w(*args, **kw): 40 | k = (f, repr(args), repr(kw)) 41 | if k not in memoize.mem: 42 | memoize.mem[k] = f(*args, **kw) 43 | return memoize.mem[k] 44 | return w 45 | 46 | 47 | # Initialize cache memory block. 48 | memoize.mem = {} 49 | 50 | 51 | def modsplit(module): 52 | """Split module into submodules.""" 53 | return tuple(module.split(".")) 54 | 55 | 56 | def project2modules(project): 57 | """Convert project name into auto-detected module names.""" 58 | # Name unification in accordance with PEP 426. 59 | modules = [project.lower().replace("-", "_")] 60 | if modules[0].startswith("python_"): 61 | # Remove conventional "python-" prefix. 62 | modules.append(modules[0][7:]) 63 | return modules 64 | 65 | 66 | def filtercomments(lines): 67 | """Strip comments and empty lines.""" 68 | for line in map(lambda x: x.strip(), lines): 69 | if line and not line.startswith('#'): 70 | yield line 71 | 72 | 73 | def joinlines(lines): 74 | """Join line continuations and strip comments.""" 75 | joined_line = "" 76 | for line in map(lambda x: x.strip(), lines): 77 | comment = line.startswith("#") 78 | if line.endswith("\\") and not comment: 79 | joined_line += line[:-1] 80 | continue 81 | if not comment: 82 | joined_line += line 83 | if joined_line: 84 | yield joined_line 85 | joined_line = "" 86 | if joined_line: 87 | yield joined_line 88 | 89 | 90 | def parse_requirements(lines): 91 | """Parse requirement strings into Requirement objects.""" 92 | for line in lines: 93 | # Strip comments from the end of the line 94 | line = line.split('#')[0].strip() 95 | # Skip empty lines and lines that start with options 96 | if not line or line.startswith('-'): 97 | continue 98 | try: 99 | yield Requirement(line) 100 | except Exception: 101 | continue 102 | 103 | 104 | class ImportVisitor(ast.NodeVisitor): 105 | """Import statement visitor.""" 106 | 107 | # Convenience structure for storing import statement. 108 | Import = namedtuple('Import', ('line', 'offset', 'module')) 109 | 110 | def __init__(self, tree): 111 | """Initialize import statement visitor.""" 112 | self.imports = [] 113 | self.visit(tree) 114 | 115 | def visit_Import(self, node): 116 | self.imports.append(ImportVisitor.Import( 117 | node.lineno, 118 | node.col_offset, 119 | modsplit(node.names[0].name), 120 | )) 121 | 122 | def visit_ImportFrom(self, node): 123 | if node.level != 0: 124 | # Omit relative imports (local modules). 125 | return 126 | self.imports.append(ImportVisitor.Import( 127 | node.lineno, 128 | node.col_offset, 129 | # Module name which covers: 130 | # > from namespace import module 131 | modsplit(node.module) + modsplit(node.names[0].name), 132 | )) 133 | 134 | 135 | class SetupVisitor(ast.NodeVisitor): 136 | """Package setup visitor. 137 | 138 | Warning: 139 | This visitor class executes given Abstract Syntax Tree! 140 | 141 | """ 142 | 143 | # Set of keywords used by the setup() function. 144 | attributes = { 145 | # Attributes present in almost every setup(), however 146 | # due to very generic name the score is not very high. 147 | 'name': 0.7, 148 | 'version': 0.7, 149 | # One of these attributes is present in every setup(), 150 | # scoring depends on the name uniqueness. 151 | 'ext_modules': 1.0, 152 | 'packages': 0.8, 153 | 'py_modules': 1.0, 154 | # Mostly used (hence listed here) optional attributes. 155 | 'author': 0.5, 156 | 'author_email': 0.6, 157 | 'classifiers': 0.6, 158 | 'cmdclass': 0.6, 159 | 'convert_2to3_doctests': 1.0, 160 | 'dependency_links': 0.7, 161 | 'description': 0.5, 162 | 'download_url': 0.5, 163 | 'eager_resources': 0.7, 164 | 'entry_points': 0.7, 165 | 'exclude_package_data': 0.9, 166 | 'extras_require': 0.7, 167 | 'include_package_data': 0.9, 168 | 'install_requires': 0.7, 169 | 'keywords': 0.5, 170 | 'license': 0.5, 171 | 'long_description': 0.5, 172 | 'maintainer': 0.5, 173 | 'maintainer_email': 0.6, 174 | 'namespace_packages': 0.6, 175 | 'package_data': 0.6, 176 | 'package_dir': 0.6, 177 | 'platforms': 0.5, 178 | 'python_requires': 0.7, 179 | 'scripts': 0.5, 180 | 'setup_requires': 0.7, 181 | 'test_loader': 0.6, 182 | 'test_suite': 0.6, 183 | 'tests_require': 0.7, 184 | 'url': 0.5, 185 | 'use_2to3': 0.9, 186 | 'use_2to3_fixers': 1.0, 187 | 'zip_safe': 0.6, 188 | } 189 | 190 | def __init__(self, tree, cwd): 191 | """Initialize package setup visitor.""" 192 | self.redirected = False 193 | self.keywords = {} 194 | 195 | # Find setup() call and redirect it. 196 | self.visit(tree) 197 | 198 | if not self.redirected: 199 | return 200 | 201 | def setup(**kw): 202 | """Setup() arguments hijacking.""" 203 | self.keywords = kw 204 | 205 | # XXX: If evaluated script (setup.py) depends on local modules we 206 | # have to add its root directory to the import search path. 207 | # Note however, that this hack might break further imports 208 | # for OUR Python instance (we're changing our own sys.path)! 209 | sys.path.insert(0, cwd) 210 | 211 | try: 212 | tree = ast.fix_missing_locations(tree) 213 | eval(compile(tree, "", mode='exec'), { 214 | '__name__': "__main__", 215 | '__file__': os.path.join(cwd, "setup.py"), 216 | '__f8r_setup': setup, 217 | }) 218 | except BaseException as e: 219 | # XXX: Exception during setup.py evaluation might not necessary 220 | # mean "fatal error". This exception might occur if e.g. 221 | # we have hijacked local setup() function (due to matching 222 | # heuristic for function arguments). Anyway, we shall not 223 | # break flake8 execution due to our eval() usage. 224 | LOG.error("Couldn't evaluate setup.py: %r", e) 225 | self.redirected = False 226 | 227 | # Restore import search path. 228 | sys.path.pop(0) 229 | 230 | def get_requirements( 231 | self, install=True, extras=True, setup=False, tests=False): 232 | """Get package requirements.""" 233 | requires = [] 234 | if install: 235 | requires.extend(parse_requirements( 236 | self.keywords.get('install_requires', ()), 237 | )) 238 | if extras: 239 | for r in self.keywords.get('extras_require', {}).values(): 240 | requires.extend(parse_requirements(r)) 241 | if setup: 242 | requires.extend(parse_requirements( 243 | self.keywords.get('setup_requires', ()), 244 | )) 245 | if tests: 246 | requires.extend(parse_requirements( 247 | self.keywords.get('tests_require', ()), 248 | )) 249 | return requires 250 | 251 | def visit_Call(self, node): 252 | """Call visitor - used for finding setup() call.""" 253 | self.generic_visit(node) 254 | 255 | # Setup() is a keywords-only function. 256 | if node.args: 257 | return 258 | 259 | keywords = set() 260 | for k in node.keywords: 261 | if k.arg is not None: 262 | keywords.add(k.arg) 263 | # Simple case for dictionary expansion for Python >= 3.5. 264 | if k.arg is None and isinstance(k.value, ast.Dict): 265 | keywords.update(x.value for x in k.value.keys) 266 | # Simple case for dictionary expansion for Python <= 3.4. 267 | if getattr(node, 'kwargs', ()) and isinstance(node.kwargs, ast.Dict): 268 | keywords.update(x.value for x in node.kwargs.keys) 269 | 270 | # The bare minimum number of arguments seems to be around five, which 271 | # includes author, name, version, module/package and something extra. 272 | if len(keywords) < 5: 273 | return 274 | 275 | score = sum( 276 | self.attributes.get(x, 0) 277 | for x in keywords 278 | ) / len(keywords) 279 | 280 | if score < 0.5: 281 | LOG.debug( 282 | "Scoring for setup%r below 0.5: %.2f", 283 | tuple(keywords), 284 | score) 285 | return 286 | 287 | # Redirect call to our setup() tap function. 288 | node.func = ast.Name(id='__f8r_setup', ctx=node.func.ctx) 289 | self.redirected = True 290 | 291 | 292 | class ModuleSet(dict): 293 | """Radix-tree-like structure for modules lookup.""" 294 | 295 | requirement = None 296 | 297 | def add(self, module, requirement): 298 | for mod in module: 299 | self = self.setdefault(mod, ModuleSet()) 300 | self.requirement = requirement 301 | 302 | def __contains__(self, module): 303 | for mod in module: 304 | self = self.get(mod) 305 | if self is None: 306 | return False 307 | if self.requirement is not None: 308 | return True 309 | return False 310 | 311 | 312 | class Flake8Checker(object): 313 | """Package requirements checker.""" 314 | 315 | name = "flake8-requirements" 316 | version = __version__ 317 | 318 | # Build-in mapping for known 3rd party modules. 319 | known_3rd_parties = { 320 | k: v 321 | for k, v in KNOWN_3RD_PARTIES.items() 322 | for k in project2modules(k) 323 | } 324 | 325 | # Host-based mapping for 3rd party modules. 326 | known_host_3rd_parties = {} 327 | 328 | # Collect and report I901 errors 329 | error_I901_enabled = False 330 | 331 | # User defined project->modules mapping. 332 | known_modules = {} 333 | 334 | # User provided requirements file. 335 | requirements_file = None 336 | 337 | # Max depth to resolve recursive requirements. 338 | requirements_max_depth = 1 339 | 340 | # Root directory of the project. 341 | root_dir = "" 342 | 343 | def __init__(self, tree, filename, lines=None): 344 | """Initialize requirements checker.""" 345 | self.tree = tree 346 | self.filename = filename 347 | self.lines = lines 348 | 349 | @classmethod 350 | def add_options(cls, manager): 351 | """Register plug-in specific options.""" 352 | manager.add_option( 353 | "--known-modules", 354 | action='store', 355 | default="", 356 | parse_from_config=True, 357 | help=( 358 | "User defined mapping between a project name and a list of" 359 | " provided modules. For example: ``--known-modules=project:" 360 | "[Project],extra-project:[extras,utilities]``." 361 | )) 362 | manager.add_option( 363 | "--requirements-file", 364 | action='store', 365 | parse_from_config=True, 366 | help=( 367 | "Specify the name (location) of the requirements text file. " 368 | "Unless an absolute path is given, the file will be searched " 369 | "relative to the project's root directory. If this option is " 370 | "not specified, the plugin look up for requirements in " 371 | "(1) setup.py, (2) setup.cfg, (3) pyproject.toml, and (4) " 372 | "requirements.txt. If specified, look up will not take place." 373 | )) 374 | manager.add_option( 375 | "--requirements-max-depth", 376 | type=int, 377 | default=1, 378 | parse_from_config=True, 379 | help=( 380 | "Max depth to resolve recursive requirements. Defaults to 1 " 381 | "(one level of recursion allowed)." 382 | )) 383 | manager.add_option( 384 | "--scan-host-site-packages", 385 | action='store_true', 386 | parse_from_config=True, 387 | help=( 388 | "Scan host's site-packages directory for 3rd party projects, " 389 | "which provide more than one module or the name of the module" 390 | " is different than the project name itself." 391 | )) 392 | 393 | @classmethod 394 | def parse_options(cls, options): 395 | """Parse plug-in specific options.""" 396 | if isinstance(options.known_modules, dict): 397 | # Support for nicer known-modules using flake8-pyproject. 398 | cls.known_modules = { 399 | k: v 400 | for k, v in options.known_modules.items() 401 | for k in project2modules(k) 402 | } 403 | else: 404 | cls.known_modules = { 405 | k: v.split(",") 406 | for k, v in [ 407 | x.split(":[") 408 | for x in re.split(r"],?", options.known_modules)[:-1]] 409 | for k in project2modules(k) 410 | } 411 | cls.requirements_file = options.requirements_file 412 | cls.requirements_max_depth = options.requirements_max_depth 413 | if options.scan_host_site_packages: 414 | cls.known_host_3rd_parties = cls.discover_host_3rd_party_modules() 415 | cls.root_dir = cls.discover_project_root_dir(os.getcwd()) 416 | 417 | @staticmethod 418 | def discover_host_3rd_party_modules(): 419 | """Scan host site-packages for 3rd party modules.""" 420 | mapping = {} 421 | try: 422 | site_packages_dirs = site.getsitepackages() 423 | site_packages_dirs.append(site.getusersitepackages()) 424 | except AttributeError as e: 425 | LOG.error("Couldn't get site packages: %s", e) 426 | return mapping 427 | for site_dir in site_packages_dirs: 428 | try: 429 | dir_entries = os.listdir(site_dir) 430 | except IOError: 431 | continue 432 | for egg in (x for x in dir_entries if x.endswith(".egg-info")): 433 | pkg_info_path = os.path.join(site_dir, egg, "PKG-INFO") 434 | modules_path = os.path.join(site_dir, egg, "top_level.txt") 435 | if not os.path.isfile(pkg_info_path): 436 | continue 437 | with open(pkg_info_path) as f: 438 | name = next(iter( 439 | line.split(":")[1].strip() 440 | for line in filtercomments(f.readlines()) 441 | if line.lower().startswith("name:") 442 | ), "") 443 | with open(modules_path) as f: 444 | modules = list(filtercomments(f.readlines())) 445 | for name in project2modules(name): 446 | mapping[name] = modules 447 | return mapping 448 | 449 | @staticmethod 450 | def discover_project_root_dir(path): 451 | """Discover project's root directory starting from given path.""" 452 | root_files = ["pyproject.toml", "requirements.txt", "setup.py"] 453 | while path != os.path.abspath(os.sep): 454 | paths = [os.path.join(path, x) for x in root_files] 455 | if any(map(os.path.exists, paths)): 456 | LOG.info("Discovered root directory: %s", path) 457 | return path 458 | path = os.path.abspath(os.path.join(path, "..")) 459 | return "" 460 | 461 | @staticmethod 462 | def is_project_setup_py(project_root_dir, filename): 463 | """Determine whether given file is project's setup.py file.""" 464 | project_setup_py = os.path.join(project_root_dir, "setup.py") 465 | try: 466 | return os.path.samefile(filename, project_setup_py) 467 | except OSError: 468 | return False 469 | 470 | _requirement_match_option = re.compile( 471 | r"(-[\w-]+)(.*)").match 472 | 473 | _requirement_match_spec = re.compile( 474 | r"(.*?)\s+--(global-option|install-option|hash)").match 475 | 476 | _requirement_match_archive = re.compile( 477 | r"(.*)(\.(tar(\.(bz2|gz|lz|lzma|xz))?|tbz|tgz|tlz|txz|whl|zip))", 478 | re.IGNORECASE).match 479 | _requirement_match_archive_spec = re.compile( 480 | r"(\w+)(-[^-]+)?").match 481 | 482 | _requirement_match_vcs = re.compile( 483 | r"(git|hg|svn|bzr)\+(.*)").match 484 | _requirement_match_vcs_spec = re.compile( 485 | r".*egg=([\w\-\.]+)").match 486 | 487 | @classmethod 488 | def resolve_requirement(cls, requirement, max_depth=0, path=None): 489 | """Resolves flags like -r in an individual requirement line.""" 490 | 491 | option = None 492 | if match := cls._requirement_match_option(requirement): 493 | option = match.group(1) 494 | requirement = match.group(2).lstrip() 495 | 496 | editable = False 497 | if option in ("-e", "--editable"): 498 | editable = True 499 | # We do not care about installation mode. 500 | option = None 501 | 502 | if option in ("-r", "--requirement"): 503 | # Error out if we need to recurse deeper than allowed. 504 | if max_depth <= 0: 505 | msg = ( 506 | "Cannot resolve {}: " 507 | "Beyond max depth (--requirements-max-depth={})") 508 | raise RuntimeError(msg.format( 509 | requirement, cls.requirements_max_depth)) 510 | resolved = [] 511 | # Error out if requirements file cannot be opened. 512 | with open(os.path.join(path or cls.root_dir, requirement)) as f: 513 | for line in joinlines(f.readlines()): 514 | resolved.extend(cls.resolve_requirement( 515 | line, max_depth - 1, os.path.dirname(f.name))) 516 | return resolved 517 | 518 | if option: 519 | # Skip whole line if option was not processed earlier. 520 | return [] 521 | 522 | # Check for a requirement given as a VCS link. 523 | if match := cls._requirement_match_vcs(requirement): 524 | if match := cls._requirement_match_vcs_spec(match.group(2)): 525 | return [match.group(1)] 526 | 527 | # Check for a requirement given as a local archive file. 528 | if match := cls._requirement_match_archive(requirement): 529 | base = os.path.basename(match.group(1)) 530 | if match := cls._requirement_match_archive_spec(base): 531 | name, version = match.groups() 532 | return [ 533 | name if not version else 534 | "{} == {}".format(name, version[1:]) 535 | ] 536 | 537 | # Editable installation is made either from local path or from VCS 538 | # URL. In case of VCS, the URL should be already handled in the if 539 | # block above. Here we shall get a local project path. 540 | if editable: 541 | requirement = os.path.basename(requirement) 542 | if requirement.split()[0] == ".": 543 | requirement = "" 544 | 545 | # Extract requirement specifier (skip in-line options). 546 | if match := cls._requirement_match_spec(requirement): 547 | requirement = match.group(1) 548 | 549 | return [requirement.strip()] 550 | 551 | @classmethod 552 | @memoize 553 | def get_pyproject_toml(cls): 554 | """Try to load PEP 518 configuration file.""" 555 | pyproject_config_path = os.path.join(cls.root_dir, "pyproject.toml") 556 | try: 557 | with open(pyproject_config_path, mode="rb") as f: 558 | return tomllib.load(f) 559 | except (IOError, tomllib.TOMLDecodeError) as e: 560 | LOG.debug("Couldn't load pyproject: %s", e) 561 | return {} 562 | 563 | @classmethod 564 | def get_pyproject_toml_pep621(cls): 565 | """Try to get PEP 621 metadata.""" 566 | cfg_pep518 = cls.get_pyproject_toml() 567 | return cfg_pep518.get('project', {}) 568 | 569 | @classmethod 570 | def get_setuptools_dynamic_requirements(cls): 571 | """Retrieve dynamic requirements defined in setuptools config.""" 572 | cfg = cls.get_pyproject_toml() 573 | dynamic_keys = cfg.get('project', {}).get('dynamic', []) 574 | dynamic_config = ( 575 | cfg.get('tool', {}).get('setuptools', {}).get('dynamic', {}) 576 | ) 577 | requirements = [] 578 | files_to_parse = [] 579 | if 'dependencies' in dynamic_keys: 580 | files_to_parse.extend( 581 | dynamic_config.get('dependencies', {}).get('file', []) 582 | ) 583 | if 'optional-dependencies' in dynamic_keys: 584 | for element in dynamic_config.get( 585 | 'optional-dependencies', {} 586 | ).values(): 587 | files_to_parse.extend(element.get('file', [])) 588 | for file_path in files_to_parse: 589 | try: 590 | with open(file_path, 'r') as file: 591 | requirements.extend(parse_requirements(file.readlines())) 592 | except IOError as e: 593 | LOG.debug("Couldn't open requirements file: %s", e) 594 | return requirements 595 | 596 | @classmethod 597 | def get_pyproject_toml_pep621_requirements(cls): 598 | """Try to get PEP 621 metadata requirements.""" 599 | pep621 = cls.get_pyproject_toml_pep621() 600 | requirements = [] 601 | requirements.extend(parse_requirements( 602 | pep621.get("dependencies", ()))) 603 | for r in pep621.get("optional-dependencies", {}).values(): 604 | requirements.extend(parse_requirements(r)) 605 | if len(requirements) == 0: 606 | requirements = cls.get_setuptools_dynamic_requirements() 607 | return requirements 608 | 609 | @classmethod 610 | def get_pyproject_toml_poetry(cls): 611 | """Try to get poetry configuration.""" 612 | cfg_pep518 = cls.get_pyproject_toml() 613 | return cfg_pep518.get('tool', {}).get('poetry', {}) 614 | 615 | @classmethod 616 | def get_pyproject_toml_poetry_requirements(cls): 617 | """Try to get poetry configuration requirements.""" 618 | poetry = cls.get_pyproject_toml_poetry() 619 | requirements = [] 620 | requirements.extend(parse_requirements( 621 | poetry.get('dependencies', ()))) 622 | requirements.extend(parse_requirements( 623 | poetry.get('dev-dependencies', ()))) 624 | # Collect dependencies from groups (since poetry-1.2). 625 | for _, group in poetry.get('group', {}).items(): 626 | requirements.extend(parse_requirements( 627 | group.get('dependencies', ()))) 628 | return requirements 629 | 630 | @classmethod 631 | def get_requirements_txt(cls): 632 | """Try to load requirements from text file.""" 633 | path = cls.requirements_file or "requirements.txt" 634 | if not os.path.isabs(path): 635 | path = os.path.join(cls.root_dir, path) 636 | try: 637 | return tuple(parse_requirements(cls.resolve_requirement( 638 | "-r {}".format(path), cls.requirements_max_depth + 1))) 639 | except IOError as e: 640 | LOG.error("Couldn't load requirements: %s", e) 641 | return () 642 | 643 | @classmethod 644 | @memoize 645 | def get_setup_cfg(cls): 646 | """Try to load standard configuration file.""" 647 | config = ConfigParser() 648 | config.read_dict({ 649 | 'metadata': {'name': ""}, 650 | 'options': { 651 | 'install_requires': "", 652 | 'setup_requires': "", 653 | 'tests_require': ""}, 654 | 'options.extras_require': {}, 655 | }) 656 | if not config.read(os.path.join(cls.root_dir, "setup.cfg")): 657 | LOG.debug("Couldn't load setup configuration: setup.cfg") 658 | return config 659 | 660 | @classmethod 661 | def get_setup_cfg_requirements(cls, is_setup_py): 662 | """Try to load standard configuration file requirements.""" 663 | config = cls.get_setup_cfg() 664 | requirements = [] 665 | if requires := config.get('options', 'install_requires'): 666 | requirements.extend(parse_requirements(requires.splitlines())) 667 | if requires := config.get('options', 'tests_require'): 668 | requirements.extend(parse_requirements(requires.splitlines())) 669 | for _, r in config.items('options.extras_require'): 670 | if r: 671 | requirements.extend(parse_requirements(r.splitlines())) 672 | if is_setup_py: 673 | if requires := config.get('options', 'setup_requires'): 674 | requirements.extend(parse_requirements(requires.splitlines())) 675 | return requirements 676 | 677 | @classmethod 678 | @memoize 679 | def get_setup_py(cls): 680 | """Try to load standard setup file.""" 681 | try: 682 | with open(os.path.join(cls.root_dir, "setup.py")) as f: 683 | return SetupVisitor(ast.parse(f.read()), cls.root_dir) 684 | except IOError as e: 685 | LOG.debug("Couldn't load setup: %s", e) 686 | return SetupVisitor(ast.parse(""), cls.root_dir) 687 | 688 | @classmethod 689 | def get_setup_py_requirements(cls, is_setup_py): 690 | """Try to load standard setup file requirements.""" 691 | setup = cls.get_setup_py() 692 | if not setup.redirected: 693 | return [] 694 | return setup.get_requirements( 695 | setup=is_setup_py, 696 | tests=True, 697 | ) 698 | 699 | @classmethod 700 | @memoize 701 | def get_mods_1st_party(cls): 702 | mods_1st_party = ModuleSet() 703 | # Get 1st party modules (used for absolute imports). 704 | modules = project2modules( 705 | cls.get_setup_py().keywords.get('name') or 706 | cls.get_setup_cfg().get('metadata', 'name') or 707 | cls.get_pyproject_toml_pep621().get('name') or 708 | cls.get_pyproject_toml_poetry().get('name') or 709 | "") 710 | # Use known module mappings to correct auto-detected name. Please note 711 | # that we're using the first module name only, since all mappings shall 712 | # contain all possible auto-detected module names. 713 | if modules[0] in cls.known_modules: 714 | modules = cls.known_modules[modules[0]] 715 | for module in modules: 716 | mods_1st_party.add(modsplit(module), True) 717 | return mods_1st_party 718 | 719 | @classmethod 720 | @memoize 721 | def get_mods_3rd_party(cls, is_setup_py): 722 | mods_3rd_party = ModuleSet() 723 | # Get 3rd party module names based on requirements. 724 | for requirement in cls.get_mods_3rd_party_requirements(is_setup_py): 725 | modules = project2modules(requirement.name) 726 | # Use known module mappings to correct auto-detected module name. 727 | if modules[0] in cls.known_modules: 728 | modules = cls.known_modules[modules[0]] 729 | elif modules[0] in cls.known_3rd_parties: 730 | modules = cls.known_3rd_parties[modules[0]] 731 | elif modules[0] in cls.known_host_3rd_parties: 732 | modules = cls.known_host_3rd_parties[modules[0]] 733 | for module in modules: 734 | mods_3rd_party.add(modsplit(module), requirement) 735 | return mods_3rd_party 736 | 737 | @classmethod 738 | def get_mods_3rd_party_requirements(cls, is_setup_py): 739 | """Get list of 3rd party requirements.""" 740 | # Use user provided requirements text file. 741 | if cls.requirements_file: 742 | return cls.get_requirements_txt() 743 | return ( 744 | # Use requirements from setup if available. 745 | cls.get_setup_py_requirements(is_setup_py) or 746 | # Check setup configuration file for requirements. 747 | cls.get_setup_cfg_requirements(is_setup_py) or 748 | # Check PEP 621 metadata for requirements. 749 | cls.get_pyproject_toml_pep621_requirements() or 750 | # Check project configuration for requirements. 751 | cls.get_pyproject_toml_poetry_requirements() or 752 | # Fall-back to requirements.txt in our root directory. 753 | cls.get_requirements_txt() 754 | ) 755 | 756 | def check_I900(self, node): 757 | """Run missing requirement checker.""" 758 | if node.module[0] in STDLIB: 759 | return None 760 | is_setup_py = self.is_project_setup_py(self.root_dir, self.filename) 761 | if node.module in self.get_mods_3rd_party(is_setup_py): 762 | return None 763 | if node.module in self.get_mods_1st_party(): 764 | return None 765 | # When processing setup.py file, forcefully add setuptools to the 766 | # project requirements. Setuptools might be required to build the 767 | # project, even though it is not listed as a requirement - this 768 | # package is required to run setup.py, so listing it as a setup 769 | # requirement would be pointless. 770 | if (is_setup_py and 771 | node.module[0] in KNOWN_3RD_PARTIES["setuptools"]): 772 | return None 773 | return ERRORS['I900'].format(pkg=node.module[0]) 774 | 775 | def check_I901(self, node): 776 | """Run not-used requirement checker.""" 777 | if node.module[0] in STDLIB: 778 | return None 779 | # TODO: Implement this check. 780 | return None 781 | 782 | def run(self): 783 | """Run checker.""" 784 | 785 | checkers = [] 786 | checkers.append(self.check_I900) 787 | if self.error_I901_enabled: 788 | checkers.append(self.check_I901) 789 | 790 | for node in ImportVisitor(self.tree).imports: 791 | for err in filter(None, map(lambda c: c(node), checkers)): 792 | yield node.line, node.offset, err, type(self) 793 | --------------------------------------------------------------------------------