├── src └── esss_fix_format │ ├── __init__.py │ ├── hooks.py │ ├── hook_utils.py │ └── cli.py ├── tests ├── __init__.py └── test_esss_fix_format.py ├── bin ├── fix-format.bat └── fix-format ├── MANIFEST.in ├── .settings ├── org.eclipse.core.resources.prefs └── org.python.pydev.yaml ├── .pre-commit-hooks.yaml ├── pyproject.toml ├── .project ├── .pydevproject ├── environment.devenv.yml ├── .isort.cfg ├── .gitignore ├── LICENSE ├── .github └── workflows │ └── test.yml ├── .clang-format ├── setup.py ├── .pre-commit-config.yaml ├── CHANGELOG.rst └── README.rst /src/esss_fix_format/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | -------------------------------------------------------------------------------- /bin/fix-format.bat: -------------------------------------------------------------------------------- 1 | @python -m esss_fix_format.cli %* 2 | -------------------------------------------------------------------------------- /bin/fix-format: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python -m esss_fix_format.cli $* 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include CHANGELOG.rst 2 | include LICENSE 3 | include README.rst 4 | 5 | recursive-include tests * 6 | recursive-exclude * __pycache__ 7 | recursive-exclude * *.py[co] 8 | -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//esss_fix_format/cli.py=utf-8 3 | encoding//tests/test_esss_fix_format.py=utf-8 4 | encoding/setup.py=utf-8 5 | -------------------------------------------------------------------------------- /.pre-commit-hooks.yaml: -------------------------------------------------------------------------------- 1 | - id: esss_fix_format 2 | description: Simple code formatter and pre-commit checker used internally by ESSS 3 | name: ESSS Fix Format 4 | entry: fix-format --commit 5 | language: python 6 | files: \.(py|pyw)$ 7 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.black] 2 | line-length = 100 3 | 4 | [tool.mypy] 5 | files = "src,tests" 6 | ignore_missing_imports = true 7 | no_implicit_optional = true 8 | show_error_codes = true 9 | local_partial_types = true 10 | ignore_missing_imports_per_module = true 11 | disallow_untyped_defs = true 12 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | esss_fix_format 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /${PROJECT_DIR_NAME} 5 | 6 | python interpreter 7 | Default 8 | 9 | -------------------------------------------------------------------------------- /environment.devenv.yml: -------------------------------------------------------------------------------- 1 | {% set CONDA_PY = os.environ['CONDA_PY'] %} 2 | {% set PY = CONDA_PY | int %} 3 | name: esss-fix-format-py{{ PY }} 4 | 5 | dependencies: 6 | # run 7 | - boltons 8 | - clang-format 9 | - click>=6.0 10 | - isort>=5.0 11 | - python=3.{{ CONDA_PY[1:] }} 12 | - tomli 13 | 14 | - attrs <=20 # [PY==36] 15 | - platformdirs <=2.4 # [PY==36] 16 | - typing-extensions <4 # [PY==36] 17 | - typing-extensions <5.11 # [PY==36] 18 | 19 | # develop 20 | - black 21 | - pre-commit 22 | - pygments>=2.2.0 23 | - pytest>=3.8.0 24 | - pytest-mock>=1.10.0 25 | 26 | environment: 27 | PYTHONPATH: 28 | - {{ root }}/src 29 | PATH: 30 | - {{ root }}/bin 31 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | extra_standard_library=Bastion,CGIHTTPServer,DocXMLRPCServer,HTMLParser,MimeWriter,SimpleHTTPServer,UserDict,UserList,UserString,aifc,antigravity,ast,audiodev,bdb,binhex,cgi,chunk,code,codeop,colorsys,cookielib,copy_reg,dummy_thread,dummy_threading,formatter,fpformat,ftplib,genericpath,htmlentitydefs,htmllib,httplib,ihooks,imghdr,imputil,keyword,macpath,macurl2path,mailcap,markupbase,md5,mimetools,mimetypes,mimify,modulefinder,multifile,mutex,netrc,new,nntplib,ntpath,nturl2path,numbers,opcode,os2emxpath,pickletools,popen2,poplib,posixfile,posixpath,pty,py_compile,quopri,repr,rexec,rfc822,runpy,sets,sgmllib,sha,sndhdr,sre,sre_compile,sre_constants,sre_parse,ssl,stat,statvfs,stringold,stringprep,sunau,sunaudio,symbol,symtable,telnetlib,this,toaiff,token,tokenize,tty,types,user,uu,wave,xdrlib,xmllib 3 | known_third_party=six,six.moves,sip 4 | line_length=100 5 | multi_line_output=4 6 | use_parentheses=true 7 | -------------------------------------------------------------------------------- /.settings/org.python.pydev.yaml: -------------------------------------------------------------------------------- 1 | ADD_NEW_LINE_AT_END_OF_FILE: true 2 | AUTOPEP8_PARAMETERS: '' 3 | BLANK_LINES_INNER: 1 4 | BLANK_LINES_TOP_LEVEL: 2 5 | BREAK_IMPORTS_MODE: PARENTHESIS 6 | DATE_FIELD_FORMAT: yyyy-MM-dd 7 | DATE_FIELD_NAME: __updated__ 8 | DELETE_UNUSED_IMPORTS: false 9 | ENABLE_DATE_FIELD_ACTION: false 10 | FORMAT_BEFORE_SAVING: true 11 | FORMAT_ONLY_CHANGED_LINES: false 12 | FORMAT_WITH_AUTOPEP8: false 13 | FROM_IMPORTS_FIRST: false 14 | GROUP_IMPORTS: true 15 | IMPORT_ENGINE: IMPORT_ENGINE_ISORT 16 | MANAGE_BLANK_LINES: true 17 | MULTILINE_IMPORTS: true 18 | SAVE_ACTIONS_ONLY_ON_WORKSPACE_FILES: true 19 | SORT_IMPORTS_ON_SAVE: true 20 | SORT_NAMES_GROUPED: true 21 | SPACES_BEFORE_COMMENT: '2' 22 | SPACES_IN_START_COMMENT: '1' 23 | TRIM_EMPTY_LINES: true 24 | TRIM_MULTILINE_LITERALS: true 25 | USE_ASSIGN_WITH_PACES_INSIDER_PARENTESIS: false 26 | USE_OPERATORS_WITH_SPACE: true 27 | USE_SPACE_AFTER_COMMA: true 28 | USE_SPACE_FOR_PARENTESIS: false 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | .idea 62 | .pytest_cache 63 | /environment.yml 64 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | MIT License 3 | 4 | Copyright (c) 2019, ESSS 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 9 | 10 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 11 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: test 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | 8 | pull_request: 9 | 10 | # Cancel running jobs for the same workflow and branch. 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.ref }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | test: 17 | runs-on: ${{ matrix.os }} 18 | 19 | strategy: 20 | fail-fast: false 21 | matrix: 22 | conda_py: ["36", "310"] 23 | os: ["windows-latest", "ubuntu-latest"] 24 | 25 | steps: 26 | - uses: actions/checkout@v2 27 | - uses: conda-incubator/setup-miniconda@v2 28 | with: 29 | auto-activate-base: false 30 | activate-environment: '' 31 | channels: conda-forge,esss 32 | channel-priority: true 33 | - name: Install 34 | env: 35 | CONDA_PY: ${{ matrix.conda_py }} 36 | run: | 37 | conda config --system --set always_yes yes --set changeps1 no 38 | conda install -c conda-forge conda-devenv 39 | conda info -a 40 | conda devenv 41 | - name: Tests 42 | run: | 43 | conda run --live-stream -n esss-fix-format-py${{ matrix.conda_py }} pytest 44 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | BasedOnStyle: WebKit 3 | AlignAfterOpenBracket: AlwaysBreak 4 | AlignEscapedNewlines: Left 5 | AllowShortFunctionsOnASingleLine: None 6 | AllowShortIfStatementsOnASingleLine: 'false' 7 | AlwaysBreakTemplateDeclarations: 'true' 8 | BinPackArguments: 'false' 9 | BinPackParameters: 'false' 10 | BraceWrapping: { 11 | AfterClass: 'true' 12 | AfterControlStatement: 'false' 13 | AfterEnum : 'true' 14 | AfterFunction : 'true' 15 | AfterNamespace : 'false' 16 | AfterStruct : 'true' 17 | AfterUnion : 'true' 18 | BeforeCatch : 'false' 19 | BeforeElse : 'false' 20 | IndentBraces : 'false' 21 | AfterExternBlock : 'false' 22 | SplitEmptyFunction : 'true' 23 | SplitEmptyRecord : 'true' 24 | SplitEmptyNamespace : 'true' 25 | } 26 | BreakBeforeBinaryOperators: NonAssignment 27 | BreakBeforeBraces: Custom 28 | BreakBeforeInheritanceComma: 'true' 29 | BreakConstructorInitializers: BeforeComma 30 | ColumnLimit: '100' 31 | FixNamespaceComments: 'true' 32 | KeepEmptyLinesAtTheStartOfBlocks: 'false' 33 | NamespaceIndentation: None 34 | PenaltyBreakBeforeFirstCallParameter: 1 35 | PenaltyReturnTypeOnItsOwnLine: 200 36 | SpaceAfterCStyleCast: 'true' 37 | SpacesBeforeTrailingComments: '2' 38 | IndentPPDirectives: AfterHash 39 | 40 | ... 41 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import find_packages 2 | from setuptools import setup 3 | 4 | with open("README.rst") as readme_file: 5 | readme = readme_file.read() 6 | 7 | with open("CHANGELOG.rst") as changelog_file: 8 | changelog = changelog_file.read() 9 | 10 | requirements = [ 11 | "boltons", 12 | "black", 13 | "click>=6.0", 14 | "isort", 15 | "tomli", 16 | ] 17 | 18 | setup( 19 | name="esss_fix_format", 20 | version="3.0.0", 21 | description="ESSS code formatter and checker", 22 | long_description=readme + "\n\n" + changelog, 23 | author="ESSS", 24 | author_email="foss@esss.co", 25 | url="https://github.com/esss/esss_fix_format", 26 | packages=find_packages(where="src"), 27 | package_dir={"": "src"}, 28 | entry_points={ 29 | "console_scripts": [ 30 | "fix-format=esss_fix_format.cli:main", 31 | "ff=esss_fix_format.cli:main", 32 | ] 33 | }, 34 | include_package_data=True, 35 | install_requires=requirements, 36 | license="MIT license", 37 | zip_safe=False, 38 | keywords="esss_fix_format", 39 | classifiers=[ 40 | "Development Status :: 5 - Production/Stable", 41 | "Intended Audience :: Developers", 42 | "License :: OSI Approved :: MIT License", 43 | "Programming Language :: Python :: 3", 44 | "Programming Language :: Python :: 3.6", 45 | "Programming Language :: Python :: 3.7", 46 | ], 47 | ) 48 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_language_version: 2 | python: python3.10 3 | repos: 4 | - repo: https://github.com/PyCQA/autoflake 5 | rev: v2.3.1 6 | hooks: 7 | - id: autoflake 8 | name: autoflake 9 | args: ["--in-place", "--remove-unused-variables", "--remove-all-unused-imports"] 10 | language: python 11 | files: \.py$ 12 | - repo: https://github.com/pre-commit/pre-commit-hooks 13 | rev: v5.0.0 14 | hooks: 15 | - id: trailing-whitespace 16 | - id: end-of-file-fixer 17 | - id: debug-statements 18 | - repo: https://github.com/asottile/reorder-python-imports 19 | rev: v3.13.0 20 | hooks: 21 | - id: reorder-python-imports 22 | args: ['--application-directories=.:src:tests', --py36-plus] 23 | - repo: https://github.com/psf/black 24 | rev: 24.8.0 25 | hooks: 26 | - id: black 27 | args: [--safe, --quiet] 28 | language_version: python3 29 | - repo: https://github.com/asottile/blacken-docs 30 | rev: 1.18.0 31 | hooks: 32 | - id: blacken-docs 33 | additional_dependencies: [black==23.3.0] 34 | - repo: local 35 | hooks: 36 | - id: rst 37 | name: rst 38 | entry: rst-lint --encoding utf-8 39 | files: ^(CHANGELOG.rst|README.rst)$ 40 | language: python 41 | additional_dependencies: [pygments, restructuredtext_lint] 42 | - repo: https://github.com/pre-commit/mirrors-mypy 43 | rev: v1.11.2 44 | hooks: 45 | - id: mypy 46 | files: ^(src/|tests/) 47 | args: [] 48 | additional_dependencies: 49 | - boltons 50 | - black 51 | - click >=6.0 52 | - isort 53 | - tomli 54 | -------------------------------------------------------------------------------- /src/esss_fix_format/hooks.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import textwrap 3 | from typing import Dict 4 | 5 | 6 | class GitHook(metaclass=abc.ABCMeta): 7 | """ 8 | Base class to define a Git hook usable by `hooks` task. 9 | """ 10 | 11 | @abc.abstractmethod 12 | def name(self) -> str: 13 | """ 14 | :return: Name of hook. 15 | """ 16 | 17 | @abc.abstractmethod 18 | def script(self) -> str: 19 | """ 20 | :return: Script code. Omit the shebang, as it is added later by a post-process step when 21 | hooks are installed in project. 22 | """ 23 | 24 | 25 | class FixFormatGitHook(GitHook): 26 | """ 27 | A hook that prevents developer from committing unless it respects formats expected by 28 | our `fix-format` tool. 29 | """ 30 | 31 | def name(self) -> str: 32 | return "fix-format" 33 | 34 | def script(self) -> str: 35 | script = """\ 36 | if ! which fix-format >/dev/null 2>&1 37 | then 38 | echo "fix-format not found, install in an active environment with:" 39 | echo " conda install esss_fix_format" 40 | exit 1 41 | else 42 | git diff-index --diff-filter=ACM --name-only --cached HEAD | fix-format --check --stdin 43 | returncode=$? 44 | if [ "$returncode" != "0" ] 45 | then 46 | echo "" 47 | echo "fix-format check failed (status=$returncode)! To fix, execute:" 48 | echo " ff -c" 49 | exit 1 50 | fi 51 | fi 52 | """ 53 | return textwrap.dedent(script) 54 | 55 | 56 | def _add_hook(hook: GitHook) -> None: 57 | name = hook.name() 58 | if name not in _HOOKS: 59 | _HOOKS[name] = hook 60 | else: 61 | raise KeyError(f"A hook named '{name}' already exists") 62 | 63 | 64 | # All hooks available by default 65 | _HOOKS: Dict[str, GitHook] = {} 66 | _add_hook(FixFormatGitHook()) 67 | 68 | 69 | def get_default_hook(name: str) -> GitHook: 70 | """ 71 | :param name: Name of a hook. 72 | :rtype: 73 | :return: A Git hook object. 74 | """ 75 | return _HOOKS[name] 76 | -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | History 3 | ======= 4 | 5 | 4.0.0 (2022-05-29) 6 | ------------------ 7 | 8 | * Drop support for the ``pydev`` formatter. Now if the project does not contain a ``[tool.black]`` section, ``esss-fix-format`` 9 | will issue an error. 10 | 11 | 3.2.0 (2021-01-12) 12 | ------------------ 13 | 14 | * Add support to .cu (CUDA) files. 15 | * When running ``black`` on a large number of files, pass all files at once on Linux. Previously ``esss_fix_format`` would 16 | pass files in chunks due to a limitation in Windows command-line size, but now that workaround is done in Windows only. 17 | 18 | 3.1.0 (2020-12-18) 19 | ------------------ 20 | 21 | * Files ignored by git are also ignored by fix-format when paths are given explicitly 22 | in the command-line. 23 | 24 | 3.0.0 (2020-08-14) 25 | ------------------ 26 | 27 | * Upgrade to ``isort 5``. 28 | 29 | 2.1.2 (2019-08-26) 30 | ------------------ 31 | 32 | * Fix error when running on a Windows mounted drive. 33 | 34 | 2.1.1 (2019-08-23) 35 | ------------------ 36 | 37 | * Improve the error description when ``fix-format`` crash with non UTF-8 files. 38 | 39 | 2.1.0 (2019-08-09) 40 | ------------------ 41 | 42 | * ``fix-format`` now allows exclude patterns to be configured through ``pyproject.toml``. 43 | 44 | 2.0.1 (2019-06-26) 45 | ------------------ 46 | 47 | * Fix bug where ``pyproject.toml`` would not be found if a relative path was given in the command-line. 48 | 49 | 2.0.0 (2019-06-24) 50 | ------------------ 51 | 52 | * ``fix-format`` now only supports Python 3.6+. 53 | 54 | * ``fix-format`` can now run `black `__ as the 55 | code formatter for Python code. 56 | 57 | See ``README.rst`` for instructions on how to use it. 58 | 59 | * By default ``fix-format`` is now less chatty, showing only files that were/would be changed and a summary 60 | at the end, unless the new ``--verbose`` flag is given. 61 | 62 | 1.8.0 63 | ---------- 64 | 65 | * Ignore Python files generated by `Jupytext`_. 66 | 67 | .. _`Jupytext`: https://github.com/mwouts/jupytext 68 | 69 | 1.7.0 70 | ---------- 71 | 72 | * Check if .cpp file is non-ascii, ensure it has BOM at the beginning of the file. 73 | * Emmit message when clang-format is not installed (or usable). 74 | * Ensure python files do not include BOM. 75 | 76 | 1.6.0 77 | ------ 78 | 79 | * Provided ``ff --git-hooks`` to add pre-commit hooks which validate that the code is properly formatted 80 | before being committed. 81 | 82 | 1.5.x 83 | ------ 84 | 85 | * Adopted pydevf formatter (https://github.com/fabioz/PyDev.Formatter) 86 | 87 | 1.4.3 88 | ----- 89 | 90 | * Use absolute paths when calling isort to properly load per project isort config files. 91 | 92 | 1.4.2 93 | ----- 94 | 95 | * Fix handling of isort skipped files. 96 | 97 | * Use default isort options, for custor formating a configuration file should be used as 98 | described in `isort documentation`_. 99 | 100 | .. _`isort documentation`: https://github.com/timothycrosley/isort/wiki/isort-Settings 101 | 102 | 1.4.1 103 | ----- 104 | 105 | * Fix support for filenames in pattern (such as `CMakeLists.txt`), when in subdirectories. 106 | 107 | 1.4.0 108 | ----- 109 | 110 | * Add support for CMake files (`*.cmake`, `CMakeLists.txt`). 111 | 112 | 1.3.0 113 | ----- 114 | 115 | * Add support for Cython files (`*.pyx`, `*.pxd`). 116 | 117 | 1.2.4 118 | ----- 119 | 120 | * Fix `ff -c`/`ff --commit` in Python 3. 121 | 122 | 1.2.3 123 | ----- 124 | 125 | * Force to always use parentheses on multi-line imports. 126 | 127 | 1.2.2 128 | ----- 129 | 130 | * Workaround for isort bug where some stdlib modules are not recognized as such because of a 131 | non-standard Python location. 132 | 133 | 1.2.1 134 | ----- 135 | 136 | * Fixed bug where EOL wasn't preserved in files affected by isort. 137 | 138 | 139 | 1.2.0 140 | ----- 141 | 142 | * Add "-k" shortcut for "--check". 143 | 144 | * Display a summary of files which skipped checks. 145 | 146 | * Fixed error when an entire file was skipped due to a "isort:skip_file" 147 | instruction on the docstring. 148 | 149 | 1.1.1 150 | ----- 151 | 152 | * Display error summary at the end in case some error happens when fixing files. 153 | 154 | * Fix bug when a file contained a single empty line. 155 | 156 | 1.1.0 157 | ----- 158 | 159 | * Add support for passing directories in the command line. 160 | 161 | * No longer check files for a specific end-of-line. 162 | 163 | * Fixed `#1`_: `--commit` option was not considering git root directory when listing files. 164 | 165 | .. _`#1`: https://github.com/ESSS/esss_fix_format/issues/1 166 | 167 | 1.0.0 168 | ----- 169 | 170 | * First version. 171 | -------------------------------------------------------------------------------- /src/esss_fix_format/hook_utils.py: -------------------------------------------------------------------------------- 1 | import io 2 | import os.path 3 | from typing import Optional 4 | from typing import Union 5 | 6 | from esss_fix_format.hooks import get_default_hook 7 | from esss_fix_format.hooks import GitHook 8 | 9 | 10 | def find_ancestor_dir_with(filename: str, begin_in: Optional[str] = None) -> Optional[str]: 11 | """ 12 | Look in current and ancestor directories (parent, parent of parent, ...) for a file. 13 | 14 | :param filename: File to find. 15 | :param begin_in: Directory to start searching. 16 | 17 | :rtype: unicode 18 | :return: Absolute path to directory where file is located. 19 | """ 20 | if begin_in is None: 21 | begin_in = os.curdir 22 | 23 | base_directory = os.path.abspath(begin_in) 24 | while True: 25 | directory = base_directory 26 | if os.path.exists(os.path.join(directory, filename)): 27 | return directory 28 | 29 | parent_base_directory, current_dir_name = os.path.split(base_directory) 30 | if not current_dir_name: 31 | return None 32 | if not parent_base_directory: 33 | raise RuntimeError(f"Unable to find .git in the {begin_in} hierarchy.") 34 | base_directory = parent_base_directory 35 | 36 | 37 | def add_hook(parts_dir: str, git_hook: Union[str, GitHook]) -> None: 38 | """ 39 | Adds an individual hook file to a folder with hook parts. An added hook is going to be executed 40 | by Git hooks like `pre-commit`, for instance, when installed using `invoke hooks` in a project. 41 | 42 | When a hook is added: 43 | 44 | * it prefixes its part file name with an index, so order of insertion is same order hook parts 45 | are executed; 46 | * it prefixes hook file with a header containing the shebang and a message printed to let 47 | developers name of hook in progress. 48 | 49 | :param parts_dir: Folder containing hook files. 50 | :param git_hook: A git hook or its name. Name notation can only be used 51 | by hooks available by default though (see `hooks` to learn more). 52 | """ 53 | import glob 54 | import stat 55 | 56 | if not isinstance(git_hook, GitHook): 57 | git_hook = get_default_hook(git_hook) 58 | 59 | count = len(glob.glob(f"{parts_dir}/*")) 60 | part_path = os.path.join(parts_dir, "{:05d}_{}".format(count + 1, git_hook.name())) 61 | with io.open(part_path, "w", newline="") as f: 62 | f.write("#!/bin/bash\n") 63 | f.write("\n") 64 | f.write("echo \u001b[34mHook {} in progress ....\u001b[0m\n".format(git_hook.name())) 65 | f.write(git_hook.script()) 66 | 67 | os.chmod(part_path, os.stat(part_path).st_mode | stat.S_IXUSR) 68 | 69 | 70 | def add_default_pre_commit_hooks(pre_commit_parts_dir: str) -> None: 71 | """ 72 | :param pre_commit_parts_dir: Folder containing hook files. 73 | """ 74 | add_hook(pre_commit_parts_dir, "fix-format") 75 | 76 | 77 | def install_pre_commit_hook(git_dir: Optional[str] = None) -> None: 78 | """ 79 | Install Git hooks in a project. 80 | """ 81 | import stat 82 | import sys 83 | import textwrap 84 | 85 | # Creates a pre-commit file that runs other scripts located in `_pre-commit-parts` folder. 86 | # This folder is (re)created every time hooks are installed. It runs all parts, even if one 87 | # of them fails. If any part fails, pre-commit exits as failure. 88 | git_root = find_ancestor_dir_with(".git", git_dir) 89 | assert git_root is not None 90 | git_root = os.path.abspath(git_root) 91 | project_name = os.path.basename(git_root) 92 | print(f"{project_name} hooks") 93 | 94 | if not os.path.exists(os.path.join(git_root, ".git")): 95 | raise ValueError("Expected to find: {}".format(os.path.join(git_root, ".git"))) 96 | 97 | pre_commit_file = os.path.join(git_root, ".git", "hooks", "pre-commit") 98 | 99 | # when the repository is from a submodule, ".git" is actually a file; in that case we skip hook 100 | # installation 101 | # this was encountered when building the etk-simbr package 102 | if os.path.isfile(os.path.join(git_root, ".git")): 103 | print("Skipping hook installation, %s is a file" % os.path.join(git_root, ".git")) 104 | return 105 | 106 | pre_commit_parts_dir = os.path.join(git_root, ".git", "hooks", "_pre-commit-parts") 107 | if os.path.isdir(pre_commit_parts_dir): 108 | import shutil 109 | 110 | shutil.rmtree(pre_commit_parts_dir) 111 | os.makedirs(pre_commit_parts_dir) 112 | 113 | pre_commit_contents = textwrap.dedent( 114 | """\ 115 | #!/bin/bash 116 | # installed automatically by the "hooks" task, changes will be lost! 117 | 118 | echo `pwd` 119 | globalreturncode=0 120 | for i in `ls .git/hooks/_pre-commit-parts`; 121 | do 122 | .git/hooks/_pre-commit-parts/$i 123 | returncode=$? 124 | if [ "$returncode" != "0" ] 125 | then 126 | globalreturncode=1 127 | fi 128 | done 129 | exit $globalreturncode 130 | """ 131 | ) 132 | 133 | with io.open(pre_commit_file, "w", newline="") as f: 134 | f.write(pre_commit_contents) 135 | 136 | add_default_pre_commit_hooks(pre_commit_parts_dir) 137 | 138 | if sys.platform.startswith("linux"): 139 | os.chmod(pre_commit_file, os.stat(pre_commit_file).st_mode | stat.S_IXUSR) 140 | print("Pre-commit hook installed: %s" % pre_commit_file) 141 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | =============================== 2 | esss_fix_format 3 | =============================== 4 | 5 | Important 6 | --------- 7 | 8 | Since then we have moved to pre-commit and standard tools, so this tool is no longer being maintained. 9 | 10 | .. image:: https://github.com/ESSS/esss_fix_format/workflows/linux/badge.svg 11 | :target: https://github.com/ESSS/esss_fix_format/actions?query=workflow%3Alinux 12 | 13 | .. image:: https://github.com/ESSS/esss_fix_format/workflows/windows/badge.svg 14 | :target: https://github.com/ESSS/esss_fix_format/actions?query=workflow%3Awindows 15 | 16 | Simple code formatter and pre-commit checker used internally by ESSS. 17 | 18 | * Imports sorted using `isort `_ 19 | * Trim right spaces 20 | * Expand tabs 21 | * Formats Python code using `black `__ 22 | * Formats C++ code using `clang-format `_ if a ``.clang-format`` file is available 23 | 24 | 25 | Install 26 | ------- 27 | 28 | .. code-block:: sh 29 | 30 | conda install esss_fix_format 31 | 32 | Note: 33 | 34 | If executed from the root environment (or another environment) isort will classify modules incorrectly, 35 | so you should install and run it from the same environment you're using for your project. 36 | 37 | 38 | Usage 39 | ----- 40 | 41 | Use ``fix-format`` (or ``ff`` for short) to reorder imports and format source code automatically. 42 | 43 | 1. To format files and/or directories:: 44 | 45 | fix-format ... 46 | 47 | 48 | 2. Format only modified files in Git:: 49 | 50 | fix-format --commit 51 | 52 | Or more succinctly:: 53 | 54 | ff -c 55 | 56 | 57 | .. _black: 58 | 59 | Options 60 | ------- 61 | 62 | Options for ``fix-format`` are defined in the section ``[tool.esss_fix_format]]`` of a ``pyproject.toml`` file. The 63 | TOML file should be placed in an ancestor directory of the filenames passed on the command-line. 64 | 65 | 66 | Exclude 67 | ^^^^^^^ 68 | 69 | A list of file name patterns to be excluded from the formatting. Patterns are matched using python ``fnmatch``: 70 | 71 | .. code-block:: toml 72 | 73 | [tool.esss_fix_format] 74 | exclude = [ 75 | "src/generated/*.py", 76 | "tmp/*", 77 | ] 78 | 79 | 80 | Black 81 | ^^^^^ 82 | 83 | Since version ``4.0.0`` `black `__ is used as the 84 | code formatter for Python code. 85 | 86 | For consistentcy, ``fix-format`` requires a ``pyproject.toml`` at the root of your repository (recommended) or project. 87 | 88 | It is suggested to use minimal configuration, usually just line length: 89 | 90 | .. code-block:: toml 91 | 92 | [tool.black] 93 | line-length = 100 94 | 95 | A popular option is also ``skip-string-normalization = true``, which is recommended when migrating from 96 | other formatter to ``black``. 97 | 98 | See "Converting master to black" below for details. 99 | 100 | Migrating a project to use fix-format 101 | ------------------------------------- 102 | 103 | Follow this steps to re format an entire project and start using the pre-commit hook: 104 | 105 | 1. You should have ``ff`` available in your environment already: 106 | 107 | .. code-block:: sh 108 | 109 | $ ff --help 110 | Usage: ff-script.py [OPTIONS] [FILES_OR_DIRECTORIES]... 111 | 112 | Fixes and checks formatting according to ESSS standards. 113 | 114 | Options: 115 | -k, --check Check if files are correctly formatted. 116 | --stdin Read filenames from stdin (1 per line). 117 | -c, --commit Use modified files from git. 118 | --git-hooks Add git pre-commit hooks to the repo in the current dir. 119 | --help Show this message and exit. 120 | 121 | 122 | 2. For each file you don't want imports reordered add ``isort:skipfile`` to the docstring: 123 | 124 | .. code-block:: python 125 | 126 | """ 127 | isort:skip_file 128 | """ 129 | 130 | Commit using ``-n`` to skip the current hook. 131 | 132 | 3. If there are any sensitive imports in your code which you wouldn't like to ``ff`` to touch, use 133 | a comment to prevent ``isort`` from touching it: 134 | 135 | .. code-block:: python 136 | 137 | ConfigurePyroSettings() # must be called before importing Pyro4 138 | import Pyro4 # isort:skip 139 | 140 | 4. If you want to use ``clang-format`` to format C++ code, you should copy the ``.clang-format`` 141 | file from ``esss-fix-format`` to the root of your project. This is optional for now in order 142 | to allow incremental changes (if this file is not present, the legacy C++ formatter will 143 | be used): 144 | 145 | .. code-block:: sh 146 | 147 | $ cd /path/to/repo/root 148 | $ curl -O https://raw.githubusercontent.com/ESSS/esss_fix_format/master/.clang-format 149 | 150 | 5. If you want to use ``black`` to format Python code, add a ``pyproject.toml`` to the root of 151 | your repository; an example can be found in "Converting master to black" below. 152 | 153 | 6. Activate your project environment: 154 | 155 | .. code-block:: sh 156 | 157 | $ conda activate myproject-py36 158 | 159 | 7. Execute: 160 | 161 | .. code-block:: sh 162 | 163 | $ cd /path/to/repo/root 164 | $ ff . 165 | 166 | After it completes, make sure there are no problems with the files: 167 | 168 | .. code-block:: sh 169 | 170 | $ ff . --check 171 | 172 | .. note:: 173 | if the check fails, try running it again; there's a rare 174 | `bug in isort `_ that might 175 | require to run ``ff /path/to/repo/root`` twice. 176 | 177 | Commit: 178 | 179 | .. code-block:: sh 180 | 181 | $ git commit -anm "Apply fix-format on all files" --author="fix-format " 182 | 183 | 8. Push and run your branch on CI. 184 | 185 | 9. If all goes well, it's possible to install pre-commit hooks by using ``ff --git-hooks`` so 186 | that any commit will be checked locally before commiting. 187 | 188 | 10. Profit! 💰 189 | 190 | Migrating from PyDev formatter to black 191 | --------------------------------------- 192 | 193 | Migrating an existing code base from a formatter to another can be a bit of pain. This steps will 194 | help you diminish that pain as much as possible. 195 | 196 | 197 | Converting ``master`` to black 198 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 199 | 200 | The first step is converting your ``master`` branch to black. 201 | 202 | 1. Add a ``pyproject.toml`` project with this contents: 203 | 204 | .. code-block:: toml 205 | 206 | [tool.black] 207 | line-length = 100 208 | skip-string-normalization = true 209 | 210 | 2. If your project doesn't have a ``.isort.cfg`` file, create one at the project's *repository* 211 | root with the same contents as `the one `_ 212 | in the root of this repository. 213 | 214 | 3. Run the ``upsert-isort-config`` task to update it (it should be run regularly, specially when adding new 215 | dependencies to internal projects, known as "first party" dependencies); *or*, if the project needs special 216 | configurations due to dual package and source modes, add these lines (and do not run ``upsert-isort-config``): 217 | 218 | .. code-block:: ini 219 | 220 | [settings] 221 | profile=black 222 | no_sections=True 223 | force_alphabetical_sort=True 224 | 225 | This will use black-like grouping, and clump imports together regardless if they are standard library, 226 | third party, or local. This avoids getting different results if you have a different environment activated, 227 | or commiting from an IDE. 228 | 229 | 4. Commit, and save the commit hash, possible in a task that you created for this conversion: 230 | 231 | .. code-block:: sh 232 | 233 | $ git commit -anm "Add configuration files for black" 234 | 235 | 236 | 5. Execute on the root of the repository: 237 | 238 | .. code-block:: sh 239 | 240 | $ fix-format . 241 | 242 | 6. Ensure everything is fine: 243 | 244 | .. code-block:: sh 245 | 246 | $ fix-format --check . 247 | 248 | If you **don't** see any "reformatting" messages, it means everything is formatted correctly. 249 | 250 | 7. Commit and then open a PR: 251 | 252 | .. code-block:: sh 253 | 254 | $ git commit -anm "Convert source files to black" --author="fix-format " 255 | 256 | 257 | Porting an existing branch to black 258 | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 259 | 260 | Here we are in the situation where the ``master`` is already blacken, and you want 261 | to update your branch. There are two ways, and which way generates less conflicts really 262 | depends on the contents of the source branch. 263 | 264 | merge -> Fix format 265 | ''''''''''''''''''' 266 | 267 | 1. Merge with the target branch, resolve any conflicts and then commit normally. 268 | 269 | 2. Execute ``fix-format`` in the root of your repository: 270 | 271 | .. code-block:: sh 272 | 273 | $ fix-format . 274 | 275 | This should only change the files you have touched in your branch. 276 | 277 | 3. Commit and push: 278 | 279 | .. code-block:: sh 280 | 281 | $ git commit -anm "Convert source files to black" --author="fix-format " 282 | 283 | 284 | Fix format -> merge 285 | ''''''''''''''''''' 286 | 287 | 1. Cherry-pick the commit you saved earlier on top of your branch. 288 | 289 | 2. Execute ``fix-format`` in the root of your repository: 290 | 291 | .. code-block:: sh 292 | 293 | $ fix-format . 294 | 295 | (In very large repositories, this will be a problem on Windows because of the command-line size, do it 296 | in chunks). 297 | 298 | 3. Fix any conflicts and then commit: 299 | 300 | .. code-block:: sh 301 | 302 | $ git commit -anm "Convert source files to black" --author="fix-format " 303 | 304 | 305 | Developing 306 | ---------- 307 | 308 | Create a conda environment (using Python 3 here) and install it in development mode. 309 | 310 | **Make sure you have conda configured to use ``conda-forge`` and ``esss`` conda channels.** 311 | 312 | .. code-block:: sh 313 | 314 | $ conda install -n base conda-devenv 315 | $ conda devenv 316 | $ source activate esss-fix-format-py310 317 | $ pre-commit install 318 | $ pytest 319 | 320 | When implementing changes, please do it in a separate branch and open a PR. 321 | 322 | Releasing 323 | ^^^^^^^^^ 324 | 325 | The release is done internally at ESSS using our `conda-recipes` repository. 326 | 327 | 328 | License 329 | ------- 330 | 331 | Licensed under the MIT license. 332 | -------------------------------------------------------------------------------- /src/esss_fix_format/cli.py: -------------------------------------------------------------------------------- 1 | #!python 2 | import codecs 3 | import io 4 | import os 5 | import re 6 | import shutil 7 | import subprocess 8 | import sys 9 | from pathlib import Path 10 | from typing import Dict 11 | from typing import Iterable 12 | from typing import List 13 | from typing import Optional 14 | from typing import Sequence 15 | from typing import Set 16 | from typing import Tuple 17 | 18 | import boltons.iterutils 19 | import click 20 | from isort.exceptions import FileSkipComment 21 | 22 | CPP_PATTERNS = { 23 | "*.cpp", 24 | "*.c", 25 | "*.h", 26 | "*.hpp", 27 | "*.hxx", 28 | "*.cxx", 29 | "*.cu", 30 | } 31 | 32 | PATTERNS = { 33 | "*.py", 34 | "*.java", 35 | "*.js", 36 | "*.pyx", 37 | "*.pxd", 38 | "CMakeLists.txt", 39 | "*.cmake", 40 | }.union(CPP_PATTERNS) 41 | 42 | SKIP_DIRS = { 43 | ".git", 44 | ".hg", 45 | } 46 | 47 | 48 | def is_cpp(filename: Path) -> bool: 49 | """Return True if the filename is of a type that should be treated as C++ source.""" 50 | from fnmatch import fnmatch 51 | 52 | return any(fnmatch(os.path.basename(filename), p) for p in CPP_PATTERNS) 53 | 54 | 55 | def should_format( 56 | filename: Path, include_patterns: Iterable[str], exclude_patterns: Iterable[str] 57 | ) -> Tuple[bool, str]: 58 | """ 59 | Return a tuple (fmt, reason) where fmt is True if the filename should be formatted. 60 | 61 | :param filename: file name to verify if should be formatted or not 62 | 63 | :param include_patterns: list of file patterns to be included in the formatting 64 | 65 | :param exclude_patterns: list of file patterns to be excluded from formatting. Has precedence 66 | over `include_patterns` 67 | 68 | :return: a tuple (should_format, reason). 69 | """ 70 | from fnmatch import fnmatch 71 | 72 | if any(fnmatch(os.path.abspath(filename), pattern) for pattern in exclude_patterns): 73 | return False, "Excluded file" 74 | 75 | filename_no_ext, ext = os.path.splitext(filename) 76 | # ignore .py file that has a jupytext configured notebook with the same base name 77 | ipynb_filename = filename_no_ext + ".ipynb" 78 | if ext == ".py" and os.path.isfile(ipynb_filename): 79 | with open(ipynb_filename, "rb") as f: 80 | if b"jupytext" not in f.read(): 81 | return True, "" 82 | with open(filename, "rb") as f: 83 | if b"jupytext:" not in f.read(): 84 | return True, "" 85 | return False, "Jupytext generated file" 86 | 87 | if any(fnmatch(os.path.basename(filename), pattern) for pattern in include_patterns): 88 | return True, "" 89 | 90 | return False, "Unknown file type" 91 | 92 | 93 | def find_pyproject_toml(files_or_directories: Sequence[Path]) -> Optional[Path]: 94 | """ 95 | Searches for a valid pyproject.toml file based on the list of files/directories given. 96 | 97 | We find the common ancestor of all files/directories given, then search from there 98 | upwards for a "pyproject.toml" file with a "[tool.black]" section. 99 | """ 100 | if not files_or_directories: 101 | files_or_directories = [Path.cwd()] 102 | common = Path(os.path.commonpath(files_or_directories)).absolute() 103 | for p in [common] + list(common.parents): 104 | fn = p / "pyproject.toml" 105 | if fn.is_file(): 106 | return fn 107 | return None 108 | 109 | 110 | def read_exclude_patterns(pyproject_toml: Path) -> Sequence[str]: 111 | import tomli 112 | 113 | with pyproject_toml.open("rb") as f: 114 | toml_contents = tomli.load(f) 115 | ff_options = toml_contents.get("tool", {}).get("esss_fix_format", {}) 116 | excludes_option = ff_options.get("exclude", []) 117 | if not isinstance(excludes_option, list): 118 | raise TypeError( 119 | f"pyproject.toml excludes option must be a list, got {type(excludes_option)})" 120 | ) 121 | 122 | def ensure_abspath(p: str) -> str: 123 | return os.path.join(pyproject_toml.parent, p) if not os.path.isabs(p) else p 124 | 125 | excludes_option = [ensure_abspath(p) for p in excludes_option] 126 | return excludes_option 127 | 128 | 129 | def has_black_config(pyproject_toml: Optional[Path]) -> bool: 130 | if pyproject_toml is None: 131 | return False 132 | return pyproject_toml.is_file() and "[tool.black]" in pyproject_toml.read_text(encoding="UTF-8") 133 | 134 | 135 | # caches which directories have the `.clang-format` file, *in or above it*, to avoid hitting the 136 | # disk too many times 137 | __HAS_DOT_CLANG_FORMAT: Dict[str, bool] = dict() 138 | 139 | 140 | def should_use_clang_format(filename: Path) -> bool: 141 | filename = filename.absolute() 142 | path_components = str(filename).split(os.sep)[:-1] 143 | paths_to_try = tuple( 144 | os.sep.join(path_components[:i] + [".clang-format"]) 145 | for i in range(1, len(path_components) + 1) 146 | ) 147 | 148 | # From file directory, going upwards, find the first directory already cached 149 | has_it = False 150 | for i in range(len(paths_to_try) - 1, -1, -1): 151 | path = paths_to_try[i] 152 | if path in __HAS_DOT_CLANG_FORMAT: 153 | has_it = __HAS_DOT_CLANG_FORMAT[path] 154 | break 155 | 156 | if has_it: 157 | return True 158 | 159 | # Go downwards again, looking in the disk and filling the cache if found 160 | found = False 161 | for path in paths_to_try[i + 1 :]: 162 | if found: 163 | __HAS_DOT_CLANG_FORMAT[path] = True 164 | continue 165 | found = __HAS_DOT_CLANG_FORMAT[path] = os.path.isfile(path) 166 | 167 | return found 168 | 169 | 170 | @click.command() 171 | @click.argument( 172 | "files_or_directories", nargs=-1, type=click.Path(exists=True, dir_okay=True, writable=True) 173 | ) 174 | @click.option( 175 | "-k", "--check", default=False, is_flag=True, help="Check if files are correctly formatted." 176 | ) 177 | @click.option( 178 | "--stdin", default=False, is_flag=True, help="Read filenames from stdin (1 per line)." 179 | ) 180 | @click.option("-c", "--commit", default=False, is_flag=True, help="Use modified files from git.") 181 | @click.option( 182 | "--git-hooks", 183 | default=False, 184 | is_flag=True, 185 | help="Add git pre-commit hooks to the repo in the current dir.", 186 | ) 187 | @click.option( 188 | "-v", "--verbose", default=False, is_flag=True, help="Show skipped files in the output" 189 | ) 190 | def main( 191 | files_or_directories: Sequence[Path], 192 | check: bool, 193 | stdin: bool, 194 | commit: bool, 195 | git_hooks: bool, 196 | verbose: bool, 197 | ) -> None: 198 | """Fixes and checks formatting according to ESSS standards.""" 199 | 200 | if git_hooks: 201 | from esss_fix_format.hook_utils import install_pre_commit_hook 202 | 203 | install_pre_commit_hook() # uses the current directory by default. 204 | return 205 | 206 | sys.exit(_main(files_or_directories, check=check, stdin=stdin, commit=commit, verbose=verbose)) 207 | 208 | 209 | def _process_file( 210 | filename: Path, *, check: bool, verbose: bool 211 | ) -> Tuple[bool, Sequence[str], Optional[str]]: 212 | """ 213 | :returns: a tuple with (changed, errors, formatter): 214 | - `changed` is a boolean, True if the file was changed 215 | - `errors` is a list with 0 or more error messages 216 | - `formatter` is an optional string with the formatter used (None if it does not apply) 217 | """ 218 | import isort.settings 219 | 220 | # Initialize results variables 221 | changed = False 222 | errors = [] 223 | formatter = None 224 | 225 | if is_cpp(filename): 226 | with io.open(filename, "rb") as f: 227 | content_bytes = f.read() 228 | try: 229 | content = content_bytes.decode("UTF-8") 230 | except UnicodeDecodeError: 231 | msg = ": ERROR The file contents can not be decoded using UTF-8" 232 | error_msg = click.format_filename(filename) + msg 233 | click.secho(error_msg, fg="red") 234 | errors.append(error_msg) 235 | return changed, errors, formatter 236 | # Remove all ASCII-characters, by substituting all printable ASCII-characters 237 | # with NULL character. Here, ' ' is the first printable ASCII-character (code 32) 238 | # and '~' is the last printable ASCII-character (code 126). 239 | non_ascii = re.sub("[ -~]", "", content).strip() 240 | use_bom = len(non_ascii) > 0 241 | if use_bom and not content_bytes.startswith(codecs.BOM_UTF8): 242 | msg = ( 243 | ": ERROR Not a valid UTF-8 encoded file, since it contains" 244 | " non-ASCII characters. Ensure it has UTF-8 encoding with BOM." 245 | ) 246 | error_msg = click.format_filename(filename) + msg 247 | click.secho(error_msg, fg="red") 248 | errors.append(error_msg) 249 | return changed, errors, formatter 250 | 251 | if is_cpp(filename) and should_use_clang_format(filename): 252 | formatter = "clang-format" 253 | if check: 254 | output = subprocess.check_output( 255 | 'clang-format -output-replacements-xml "%s"' % filename, shell=True 256 | ) 257 | changed = b" Tuple[bool, bool]: 341 | """ 342 | Runs black on the given files (checking or formatting). 343 | 344 | Black's output is shown directly to the user, so even in events of errors it is 345 | expected that the users sees the error directly. 346 | 347 | We need this function to work differently than the rest of ``esss_fix_format`` (which processes 348 | each file individually) because we want to call black only once, reformatting all files 349 | given to it in the command-line. 350 | 351 | :return: a pair (would_be_formatted, black_failed) 352 | """ 353 | py_files = [x for x in files if should_format(x, ["*.py"], exclude_patterns)[0]] 354 | black_failed = False 355 | would_be_formatted = False 356 | if py_files: 357 | if check: 358 | click.secho(f"Checking black on {len(py_files)} files...", fg="cyan") 359 | else: 360 | click.secho(f"Running black on {len(py_files)} files...", fg="cyan") 361 | # On Windows there's a limit on the command-line size, so we call black in batches 362 | # this should only be an issue when executing fix-format over the entire repository, 363 | # not on day-to-day usage. 364 | # Once black grows a public API (https://github.com/psf/black/issues/779), we can 365 | # ditch running things in a subprocess altogether. 366 | if sys.platform.startswith("win"): 367 | chunk_size = 100 368 | file_iterator = boltons.iterutils.chunked(py_files, chunk_size) 369 | else: 370 | file_iterator = iter([py_files]) 371 | for files in file_iterator: 372 | args = ["black"] 373 | if check: 374 | args.append("--check") 375 | if verbose: 376 | args.append("--verbose") 377 | args.extend(str(x) for x in files) 378 | status = subprocess.call(args) 379 | if not black_failed: 380 | black_failed = not check and status != 0 381 | if not would_be_formatted: 382 | would_be_formatted = check and status == 1 383 | 384 | return would_be_formatted, black_failed 385 | 386 | 387 | def get_git_ignored_files(directory: Path) -> Set[Path]: 388 | """Return a set() of git-ignored files if ``directory`` is tracked by git.""" 389 | try: 390 | git_path = shutil.which("git") 391 | if git_path is None: 392 | return set() 393 | output = subprocess.check_output( 394 | [ 395 | git_path, 396 | "status", 397 | "--ignored", 398 | "--untracked-files=all", 399 | "--porcelain=2", 400 | str(directory), 401 | ], 402 | encoding="UTF-8", 403 | ) 404 | except subprocess.CalledProcessError: 405 | # Assume we are not in a directory tracked by git (should be rare in practice). 406 | return set() 407 | else: 408 | result = set() 409 | for line in output.splitlines(): 410 | if line.startswith("!"): 411 | # Use os.path.abspath() because it also normalizes the path, 412 | # something which Path() doesn't do for us. 413 | p = Path(os.path.abspath(line[1:].strip())) 414 | result.add(p) 415 | return result 416 | 417 | 418 | def _main( 419 | files_or_directories: Sequence[Path], *, check: bool, stdin: bool, commit: bool, verbose: bool 420 | ) -> int: 421 | files: List[Path] 422 | if stdin: 423 | files = [Path(x.strip()) for x in click.get_text_stream("stdin").readlines()] 424 | elif commit: 425 | files = list(get_files_from_git()) 426 | else: 427 | files = [] 428 | for file_or_dir in files_or_directories: 429 | if os.path.isdir(file_or_dir): 430 | git_ignored = get_git_ignored_files(file_or_dir) 431 | for root, dirs, names in os.walk(file_or_dir): 432 | for dirname in list(dirs): 433 | if dirname in SKIP_DIRS: 434 | dirs.remove(dirname) 435 | files.extend( 436 | Path(root, n) 437 | for n in names 438 | if should_format(Path(n), PATTERNS, []) 439 | and Path(root, n).absolute() not in git_ignored 440 | ) 441 | else: 442 | files.append(file_or_dir) 443 | 444 | files = sorted(Path(x) for x in files) 445 | errors = [] 446 | pyproject_toml = find_pyproject_toml(files) 447 | if pyproject_toml: 448 | exclude_patterns = read_exclude_patterns(pyproject_toml) 449 | else: 450 | exclude_patterns = [] 451 | 452 | if has_black_config(pyproject_toml): 453 | would_be_formatted, black_failed = run_black_on_python_files( 454 | files, check, exclude_patterns, verbose 455 | ) 456 | if black_failed: 457 | errors.append("Error formatting black (see console)") 458 | else: 459 | click.secho("pyproject.toml not found or not configured for black.", fg="red") 460 | click.secho("Create a pyproject.toml file and add a [tool.black] section.", fg="red") 461 | click.secho("Suggestion:", fg="red") 462 | click.secho("", fg="cyan") 463 | click.secho(" [tool.black]", fg="cyan") 464 | click.secho(" line-length = 100", fg="cyan") 465 | return 1 466 | 467 | changed_files = [] 468 | analysed_files = [] 469 | for filename in files: 470 | fmt, reason = should_format(filename, PATTERNS, exclude_patterns) 471 | if not fmt: 472 | if verbose: 473 | click.secho(click.format_filename(filename) + ": " + reason, fg="white") 474 | continue 475 | 476 | analysed_files.append(filename) 477 | changed, new_errors, formatter = _process_file(filename, check=check, verbose=verbose) 478 | errors.extend(new_errors) 479 | if changed: 480 | changed_files.append(filename) 481 | status, color = _get_status_and_color(check, changed) 482 | if changed or verbose: 483 | msg = click.format_filename(filename) + ": " + status 484 | if formatter is not None: 485 | msg += " (" + formatter + ")" 486 | click.secho(msg, fg=color) 487 | 488 | def banner(caption: str) -> str: 489 | caption = " %s " % caption 490 | fill = (100 - len(caption)) // 2 491 | h = "=" * fill 492 | return h + caption + h 493 | 494 | if errors: 495 | click.secho("") 496 | click.secho(banner("ERRORS"), fg="red") 497 | for error_msg in errors: 498 | click.secho(error_msg, fg="red") 499 | return 1 500 | 501 | # show a summary of what has been done 502 | verb = "would be " if check else "" 503 | if changed_files: 504 | first_sentence = f"{len(changed_files)} files {verb}changed, " 505 | else: 506 | first_sentence = "" 507 | click.secho( 508 | f"fix-format: {first_sentence}" 509 | f"{len(analysed_files) - len(changed_files)} files {verb}left unchanged.", 510 | bold=True, 511 | fg="green", 512 | ) 513 | 514 | if check and (changed_files or would_be_formatted): 515 | return 1 516 | 517 | return 0 518 | 519 | 520 | def _get_status_and_color(check: bool, changed: bool) -> Tuple[str, str]: 521 | """ 522 | Return a pair (status message, color) based if we are checking a file for correct 523 | formatting and if the file is supposed to be changed or not. 524 | """ 525 | if check: 526 | if changed: 527 | return "Failed", "red" 528 | else: 529 | return "OK", "green" 530 | else: 531 | if changed: 532 | return "Fixed", "green" 533 | else: 534 | return "Skipped", "yellow" 535 | 536 | 537 | def fix_whitespace(lines: Sequence[str], eol: str, ends_with_eol: bool) -> str: 538 | """ 539 | Fix whitespace issues in the given list of lines. 540 | 541 | :param lines: 542 | List of lines to fix spaces and indentations. 543 | :param eol: 544 | EOL of file. 545 | :param ends_with_eol: 546 | If file ends with EOL. 547 | 548 | :rtype: unicode 549 | :return: 550 | Returns the new contents. 551 | """ 552 | lines = _strip(lines) 553 | lines = [i.expandtabs(4) for i in lines] 554 | result = eol.join(lines) 555 | if ends_with_eol: 556 | result += eol 557 | return result 558 | 559 | 560 | def _strip(lines: Sequence[str]) -> Sequence[str]: 561 | """ 562 | Splits the given text, removing the original eol but returning the eol 563 | so it can be written again on disk using the original eol. 564 | 565 | :param unicode contents: full file text 566 | :return: a triple (lines, eol, ends_with_eol), where `lines` is a list of 567 | strings, `eol` the string to be used as newline and `ends_with_eol` 568 | a boolean which indicates if the last line ends with a new line or not. 569 | """ 570 | lines = [i.rstrip() for i in lines] 571 | return lines 572 | 573 | 574 | def _peek_eol(line: str) -> str: 575 | """ 576 | :param unicode line: A line in file. 577 | :rtype: unicode 578 | :return: EOL used by line. 579 | """ 580 | eol = "\n" 581 | if line: 582 | if line.endswith("\r"): 583 | eol = "\r" 584 | elif line.endswith("\r\n"): 585 | eol = "\r\n" 586 | return eol 587 | 588 | 589 | def get_files_from_git() -> Sequence[Path]: 590 | """Obtain from a list of modified files in the current repository.""" 591 | 592 | def get_files(cmd: str) -> Sequence[str]: 593 | output = subprocess.check_output(cmd, shell=True) 594 | return [os.fsdecode(x) for x in output.splitlines()] 595 | 596 | root = os.fsdecode(subprocess.check_output("git rev-parse --show-toplevel", shell=True).strip()) 597 | result: Set[str] = set() 598 | result.update(get_files("git diff --name-only --diff-filter=ACM --staged")) 599 | result.update(get_files("git diff --name-only --diff-filter=ACM")) 600 | result.update(get_files("git ls-files -o --full-name --exclude-standard")) 601 | return sorted(Path(root, x) for x in result) 602 | 603 | 604 | if __name__ == "__main__": 605 | main() 606 | -------------------------------------------------------------------------------- /tests/test_esss_fix_format.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import codecs 3 | import os 4 | import shutil 5 | import subprocess 6 | import sys 7 | import textwrap 8 | from pathlib import Path 9 | from typing import Optional 10 | from typing import Sequence 11 | 12 | import pytest 13 | from _pytest.pytester import LineMatcher 14 | from click.testing import CliRunner 15 | from pytest_mock import MockerFixture 16 | 17 | from esss_fix_format import cli 18 | 19 | 20 | @pytest.fixture 21 | def sort_cfg_to_tmp(tmp_path: Path) -> None: 22 | import shutil 23 | 24 | shutil.copyfile( 25 | os.path.join(os.path.dirname(__file__), "..", ".isort.cfg"), str(tmp_path / ".isort.cfg") 26 | ) 27 | 28 | 29 | @pytest.fixture 30 | def dot_clang_format_to_tmp(tmp_path: Path) -> None: 31 | import shutil 32 | 33 | shutil.copyfile( 34 | os.path.join(os.path.dirname(__file__), "..", ".clang-format"), 35 | str(tmp_path / ".clang-format"), 36 | ) 37 | 38 | 39 | @pytest.fixture 40 | def input_file(tmp_path: Path, sort_cfg_to_tmp: None) -> Path: 41 | # imports out-of-order included in example so isort detects as necessary to change 42 | source = textwrap.dedent( 43 | """\ 44 | import sys 45 | import os 46 | 47 | alpha 48 | bravo\\s\\t\\s 49 | charlie 50 | if 0: 51 | \\tdelta 52 | echo 53 | foxtrot 54 | golf #Comment 55 | hotel 56 | """.replace( 57 | "\\s", " " 58 | ).replace( 59 | "\\t", "\t" 60 | ) 61 | ) 62 | filename = tmp_path / "test.py" 63 | filename.write_text(source) 64 | 65 | return filename 66 | 67 | 68 | @pytest.fixture(autouse=True) 69 | def black_config(tmp_path: Path) -> Path: 70 | fn = tmp_path.joinpath("pyproject.toml") 71 | fn.write_text("[tool.black]\nline-length = 100") 72 | return fn 73 | 74 | 75 | def test_command_line_interface(input_file: Path) -> None: 76 | check_invalid_file(input_file) 77 | fix_invalid_file(input_file) 78 | 79 | check_valid_file(input_file) 80 | fix_valid_file(input_file) 81 | 82 | 83 | def test_no_black_config(input_file: Path, black_config: Path) -> None: 84 | os.remove(str(black_config)) 85 | output = run(["--check", "--verbose", str(input_file)], expected_exit=1) 86 | output.fnmatch_lines("pyproject.toml not found or not configured for black.") 87 | 88 | 89 | def test_directory_command_line(input_file: Path, tmp_path: Path) -> None: 90 | another_file = tmp_path.joinpath("subdir", "test2.py") 91 | another_file.parent.mkdir(parents=True) 92 | shutil.copy(input_file, another_file) 93 | 94 | output = run([str(tmp_path), "--verbose"], expected_exit=0) 95 | output.fnmatch_lines( 96 | [ 97 | str(another_file) + ": Fixed", 98 | str(input_file) + ": Fixed", 99 | "fix-format: 2 files changed, 0 files left unchanged.", 100 | ] 101 | ) 102 | 103 | 104 | @pytest.mark.xfail(reason="this is locking up during main(), but works on the cmdline", run=False) 105 | def test_stdin_input(input_file: Path) -> None: 106 | runner = CliRunner() 107 | result = runner.invoke(cli.main, args=["--stdin"], input=str(input_file) + "\n") 108 | assert result.exit_code == 0 109 | assert str(input_file) in result.output 110 | 111 | 112 | def test_fix_whitespace(input_file: Path) -> None: 113 | obtained = cli.fix_whitespace(input_file.read_text().splitlines(), eol="\n", ends_with_eol=True) 114 | expected = textwrap.dedent( 115 | """\ 116 | import sys 117 | import os 118 | 119 | alpha 120 | bravo 121 | charlie 122 | if 0: 123 | \\s\\s\\s\\sdelta 124 | echo 125 | foxtrot 126 | golf #Comment 127 | hotel 128 | """.replace( 129 | "\\s", " " 130 | ) 131 | ) 132 | assert obtained == expected 133 | 134 | 135 | def test_imports(tmp_path: Path, sort_cfg_to_tmp: None) -> None: 136 | source = textwrap.dedent( 137 | """\ 138 | import pytest 139 | import sys 140 | 141 | import io 142 | # my class 143 | class Test: 144 | pass 145 | """ 146 | ) 147 | filename = tmp_path.joinpath("test.py") 148 | filename.write_text(source) 149 | 150 | check_invalid_file(filename) 151 | fix_invalid_file(filename) 152 | 153 | expected = textwrap.dedent( 154 | """\ 155 | import io 156 | import sys 157 | 158 | import pytest 159 | 160 | 161 | # my class 162 | class Test: 163 | pass 164 | """ 165 | ) 166 | assert filename.read_text() == expected 167 | 168 | 169 | @pytest.mark.parametrize("verbose", [True, False]) 170 | def test_verbosity(tmp_path: Path, input_file: Path, verbose: bool) -> None: 171 | # already in tmp_path: a py file incorrectly formatted and .isort.cfg 172 | # prepare extra files: a CPP file and a py file already properly formatted 173 | isort_fn = tmp_path / ".isort.cfg" 174 | assert isort_fn.is_file() 175 | 176 | input_cpp = "namespace boost {} " 177 | cpp_fn = tmp_path / "foo.cpp" 178 | cpp_fn.write_text(input_cpp) 179 | 180 | py_ok = tmp_path / "aa.py" # to appear as first file and simplify handling the expected lines 181 | py_ok.write_text("import os\n") 182 | 183 | # run once with --check and test output 184 | args = ["--check", str(tmp_path)] 185 | if verbose: 186 | args.append("--verbose") 187 | output = run(args, expected_exit=1) 188 | expected_lines = [] 189 | if verbose: 190 | expected_lines = [ 191 | str(isort_fn) + ": Unknown file type", 192 | str(py_ok) + ": OK", 193 | ] 194 | expected_lines.extend( 195 | [ 196 | str(cpp_fn) + ": Failed (legacy formatter)", 197 | str(input_file) + ": Failed", 198 | "fix-format: 2 files would be changed, 1 files would be left unchanged.", 199 | ] 200 | ) 201 | output.fnmatch_lines(expected_lines) 202 | 203 | # run once fixing files and test output 204 | args = [str(tmp_path)] 205 | if verbose: 206 | args.append("--verbose") 207 | output = run(args, expected_exit=0) 208 | expected_lines = [] 209 | if verbose: 210 | expected_lines = [ 211 | str(isort_fn) + ": Unknown file type", 212 | str(py_ok) + ": Skipped", 213 | ] 214 | expected_lines += [ 215 | str(cpp_fn) + ": Fixed (legacy formatter)", 216 | str(input_file) + ": Fixed", 217 | "fix-format: 2 files changed, 1 files left unchanged.", 218 | ] 219 | output.fnmatch_lines(expected_lines) 220 | 221 | # run again with everything already fixed 222 | args = [str(tmp_path)] 223 | if verbose: 224 | args.append("--verbose") 225 | output = run(args, expected_exit=0) 226 | expected_lines = [] 227 | if verbose: 228 | expected_lines = [ 229 | str(isort_fn) + ": Unknown file type", 230 | str(py_ok) + ": Skipped", 231 | str(cpp_fn) + ": Skipped (legacy formatter)", 232 | str(input_file) + ": Skipped", 233 | ] 234 | expected_lines += [ 235 | "fix-format: 3 files left unchanged.", 236 | ] 237 | output.fnmatch_lines(expected_lines) 238 | 239 | 240 | def test_filename_without_wildcard(tmp_path: Path, sort_cfg_to_tmp: None) -> None: 241 | filename = tmp_path / "CMakeLists.txt" 242 | filename.write_text("\t#\n") 243 | output = run([str(filename), "--verbose"], expected_exit=0) 244 | output.fnmatch_lines(str(filename) + ": Fixed") 245 | 246 | 247 | @pytest.mark.parametrize("param", ["-c", "--commit"]) 248 | def test_fix_commit(input_file: Path, mocker: MockerFixture, param: str, tmp_path: Path) -> None: 249 | def check_output(cmd: str, *_: object, **__: object) -> bytes: 250 | if "--show-toplevel" in cmd: 251 | result = str(tmp_path) + "\n" 252 | else: 253 | result = input_file.name + "\n" 254 | return os.fsencode(result) 255 | 256 | m = mocker.patch.object(subprocess, "check_output", side_effect=check_output) 257 | output = run([param, "--verbose"], expected_exit=0) 258 | output.fnmatch_lines(str(input_file) + ": Fixed") 259 | assert m.call_args_list == [ 260 | mocker.call("git rev-parse --show-toplevel", shell=True), 261 | mocker.call("git diff --name-only --diff-filter=ACM --staged", shell=True), 262 | mocker.call("git diff --name-only --diff-filter=ACM", shell=True), 263 | mocker.call("git ls-files -o --full-name --exclude-standard", shell=True), 264 | ] 265 | 266 | 267 | def test_input_invalid_codec(tmp_path: Path, sort_cfg_to_tmp: None) -> None: 268 | """Display error summary when we fail to open a file""" 269 | filename = tmp_path / "test.py" 270 | filename.write_bytes("hello world".encode("UTF-16")) 271 | output = run([str(filename)], expected_exit=1) 272 | output.fnmatch_lines(str(filename) + ": ERROR (Unicode*") 273 | output.fnmatch_lines("*== ERRORS ==*") 274 | output.fnmatch_lines(str(filename) + ": ERROR (Unicode*") 275 | 276 | 277 | def test_empty_file(tmp_path: Path, sort_cfg_to_tmp: None) -> None: 278 | """Ensure files with a single empty line do not raise an error""" 279 | filename = tmp_path / "test.py" 280 | filename.write_text("\r\n") 281 | run([str(filename)], expected_exit=0) 282 | 283 | 284 | @pytest.mark.parametrize( 285 | "notebook_content, expected_exit", 286 | [ 287 | ('"jupytext": {"formats": "ipynb,py"} ”', 0), 288 | ("Not a j-u-p-y-t-e-x-t configured notebook", 1), 289 | (None, 1), 290 | ], 291 | ) 292 | def test_ignore_jupytext( 293 | tmp_path: Path, sort_cfg_to_tmp: None, notebook_content: str, expected_exit: int 294 | ) -> None: 295 | if notebook_content is not None: 296 | filename_ipynb = tmp_path / "test.ipynb" 297 | filename_ipynb.write_text(notebook_content, "UTF-8") 298 | 299 | filename_py = tmp_path / "test.py" 300 | py_content = textwrap.dedent( 301 | """\ 302 | # -*- coding: utf-8 -*- 303 | # --- 304 | # jupyter: 305 | # jupytext: 306 | # formats: ipynb,py:light 307 | # text_representation: 308 | # extension: .py 309 | # format_name: light 310 | # format_version: '1.3' 311 | # jupytext_version: 0.8.6 312 | # kernelspec: 313 | # display_name: Python 3 314 | # language: python 315 | # name: python3 316 | # --- 317 | # ” 318 | import matplotlib.pyplot as plt 319 | """ 320 | ) 321 | filename_py.write_text(py_content, "UTF-8") 322 | 323 | output = run([str(filename_py), "--check"], expected_exit=expected_exit) 324 | if expected_exit == 0: 325 | assert output.str() == "fix-format: 0 files would be left unchanged." 326 | else: 327 | output.fnmatch_lines( 328 | [ 329 | "*test.py: Failed", 330 | ] 331 | ) 332 | 333 | 334 | @pytest.mark.parametrize("check", [True, False]) 335 | def test_python_with_bom(tmp_path: Path, sort_cfg_to_tmp: None, check: bool) -> None: 336 | filename = tmp_path / "test.py" 337 | original_contents = codecs.BOM_UTF8 + b"import io\r\n" 338 | filename.write_bytes(original_contents) 339 | 340 | args = [str(filename)] 341 | if check: 342 | args = ["--check"] + args 343 | 344 | run(args, expected_exit=1) 345 | 346 | current_contents = filename.read_bytes() 347 | if check: 348 | assert current_contents == original_contents 349 | else: 350 | assert current_contents == original_contents[len(codecs.BOM_UTF8) :] 351 | 352 | 353 | @pytest.mark.parametrize( 354 | "source", 355 | [ 356 | "", 357 | '"""\nisort:skip_file\n"""\n\nimport sys\nimport os\n', 358 | "# isort:skip_file\nimport sys\nimport os\n", 359 | ], 360 | ids=[ 361 | "empty file", 362 | "module-level isort:skip_file docstring", 363 | "module-level isort:skip_file comment", 364 | ], 365 | ) 366 | def test_skip_entire_file(tmp_path: Path, sort_cfg_to_tmp: None, source: str) -> None: 367 | filename = tmp_path / "test.py" 368 | filename.write_text(source) 369 | output = run([str(filename), "--verbose"], expected_exit=0) 370 | output.fnmatch_lines(str(filename) + ": Skipped") 371 | assert filename.read_text() == source 372 | 373 | 374 | def test_isort_bug_with_comment_headers(tmp_path: Path, sort_cfg_to_tmp: None) -> None: 375 | source = textwrap.dedent( 376 | """\ 377 | ''' 378 | See README.md for usage. 379 | ''' 380 | import os 381 | 382 | #=============================== 383 | # Ask 384 | #=============================== 385 | import io 386 | 387 | 388 | def Ask(question, answers): 389 | pass 390 | """ 391 | ) 392 | filename = tmp_path / "test.py" 393 | filename.write_text(source) 394 | check_invalid_file(filename) 395 | fix_invalid_file(filename) 396 | check_valid_file(filename) 397 | 398 | 399 | def test_missing_builtins(tmp_path: Path, sort_cfg_to_tmp: None) -> None: 400 | source = textwrap.dedent( 401 | """\ 402 | import thirdparty 403 | import os 404 | import ftplib 405 | import numbers 406 | """ 407 | ) 408 | filename = tmp_path / "test.py" 409 | filename.write_text(source) 410 | check_invalid_file(filename) 411 | fix_invalid_file(filename) 412 | check_valid_file(filename) 413 | obtained = filename.read_text() 414 | assert obtained == textwrap.dedent( 415 | """\ 416 | import ftplib 417 | import numbers 418 | import os 419 | 420 | import thirdparty 421 | """ 422 | ) 423 | 424 | 425 | def test_no_isort_cfg(tmp_path: Path) -> None: 426 | filename = tmp_path / "test.py" 427 | filename.write_text("import os") 428 | try: 429 | output = run([str(filename)], expected_exit=1) 430 | except Exception: 431 | for p in tmp_path.parents: 432 | isort_cfg_file = p / ".isort.cfg" 433 | if isort_cfg_file.exists(): 434 | msg = "Test does not expect that .isort.cfg is in one of the tmp_path parents ({})" 435 | raise AssertionError(msg.format(isort_cfg_file)) 436 | raise 437 | output.fnmatch_lines( 438 | r"*ERROR .isort.cfg not available in repository (or line_length config < 80)." 439 | ) 440 | 441 | 442 | def test_isort_cfg_in_parent(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: 443 | """ 444 | This test checks that a configuration file is properly read from a parent directory. 445 | This need to be checked because isort itself can fail to do this when passed a relative path. 446 | """ 447 | # more than 81 character on the same line. 448 | source = ( 449 | "from shutil import copyfileobj, copyfile, copymode, copystat, copymode, ignore_patterns," 450 | " move, rmtree" 451 | ) 452 | filename = tmp_path.joinpath("subfolder", "test.py") 453 | filename.parent.mkdir(parents=True) 454 | filename.write_text(source) 455 | 456 | cfg_filename = tmp_path.joinpath(".isort.cfg") 457 | cfg_filename.write_text("[settings]\nline_length=81\nmulti_line_output=1\n") 458 | 459 | monkeypatch.chdir(os.path.dirname(str(filename))) 460 | output = run(["."], expected_exit=0) 461 | output.fnmatch_lines("*test.py: Fixed") 462 | obtained = filename.read_text() 463 | expected = "\n".join( 464 | [ 465 | "from shutil import (copyfile,", 466 | " copyfileobj,", 467 | " copymode,", 468 | " copystat,", 469 | " ignore_patterns,", 470 | " move,", 471 | " rmtree)", 472 | "", 473 | ] 474 | ) 475 | assert obtained == expected 476 | 477 | 478 | def test_install_pre_commit_hook(tmp_path: Path) -> None: 479 | tmp_path.joinpath(".git").mkdir() 480 | 481 | from esss_fix_format import hook_utils 482 | 483 | hook_utils.install_pre_commit_hook(str(tmp_path)) 484 | assert tmp_path.joinpath(".git", "hooks", "_pre-commit-parts").is_dir() 485 | 486 | 487 | def test_install_pre_commit_hook_command_line( 488 | tmp_path: Path, monkeypatch: pytest.MonkeyPatch 489 | ) -> None: 490 | tmp_path.joinpath(".git").mkdir() 491 | monkeypatch.chdir(str(tmp_path)) 492 | run(["--git-hooks"], 0) 493 | assert tmp_path.joinpath(".git", "hooks", "_pre-commit-parts").is_dir() 494 | 495 | 496 | def test_missing_bom_error_for_non_ascii_cpp(tmp_path: Path) -> None: 497 | """ 498 | Throws an error for not encoding with "UTF-8 with BOM" of non-ascii cpp file. 499 | """ 500 | source = "int ŢōŶ; " 501 | filename = tmp_path.joinpath("a.cpp") 502 | filename.write_text(source, encoding="UTF-8") 503 | output = run([str(filename)], expected_exit=1) 504 | output.fnmatch_lines( 505 | str(filename) + ": ERROR Not a valid UTF-8 encoded file, since it contains non-ASCII*" 506 | ) 507 | output.fnmatch_lines("*== ERRORS ==*") 508 | output.fnmatch_lines( 509 | str(filename) + ": ERROR Not a valid UTF-8 encoded file, since it contains non-ASCII*" 510 | ) 511 | 512 | 513 | def test_bom_encoded_for_non_ascii_cpp(tmp_path: Path, dot_clang_format_to_tmp: None) -> None: 514 | """ 515 | Formats non-ascii cpp as usual, if it has 'UTF-8 encoding with BOM' 516 | """ 517 | source = "int ŢōŶ; " 518 | filename = tmp_path.joinpath("a.cpp") 519 | filename.write_text(source, encoding="UTF-8-SIG") 520 | check_invalid_file(filename, formatter="clang-format") 521 | fix_invalid_file(filename, formatter="clang-format") 522 | check_valid_file(filename, formatter="clang-format") 523 | obtained = filename.read_text("UTF-8-SIG") 524 | assert obtained == "int ŢōŶ;" 525 | 526 | 527 | def test_use_legacy_formatter_when_there_is_no_dot_clang_format_for_valid(tmp_path: Path) -> None: 528 | """ 529 | Won't format C++ if there's no `.clang-format` file in the directory or any directory above. 530 | """ 531 | source = "int a;" 532 | filename = tmp_path.joinpath("a.cpp") 533 | filename.write_text(source) 534 | check_valid_file(filename, formatter="legacy formatter") 535 | obtained = filename.read_text() 536 | assert obtained == source 537 | 538 | 539 | def test_use_legacy_formatter_when_there_is_no_dot_clang_format_for_invalid(tmp_path: Path) -> None: 540 | source = "int a; " 541 | filename = tmp_path.joinpath("a.cpp") 542 | filename.write_text(source) 543 | check_invalid_file(filename, formatter="legacy formatter") 544 | fix_invalid_file(filename, formatter="legacy formatter") 545 | check_valid_file(filename, formatter="legacy formatter") 546 | obtained = filename.read_text() 547 | assert obtained == "int a;" 548 | 549 | 550 | def test_clang_format(tmp_path: Path, dot_clang_format_to_tmp: None) -> None: 551 | source = "int a; " 552 | filename = tmp_path.joinpath("a.cpp") 553 | filename.write_text(source) 554 | check_invalid_file(filename, formatter="clang-format") 555 | fix_invalid_file(filename, formatter="clang-format") 556 | check_valid_file(filename, formatter="clang-format") 557 | obtained = filename.read_text() 558 | assert obtained == "int a;" 559 | 560 | 561 | def test_missing_clang_format( 562 | tmp_path: Path, mocker: MockerFixture, dot_clang_format_to_tmp: None 563 | ) -> None: 564 | source = "int a; " 565 | filename = tmp_path.joinpath("a.cpp") 566 | filename.write_text(source) 567 | 568 | # Check for invalid format: 569 | # File will not pass in the format check 570 | check_invalid_file(filename, formatter="clang-format") 571 | 572 | expected_command = 'clang-format -i "main.cpp"' 573 | expected_error_code = 1 574 | 575 | # The '*' is used to indicate that there may be a '.' in 576 | # the message depending on the python version 577 | expected_error_message = "Command '%s' returned non-zero exit status 1*" % expected_command 578 | message_extra_details = 'Please check if "clang-format" is installed and accessible' 579 | 580 | mocker.patch.object( 581 | subprocess, 582 | "check_output", 583 | side_effect=subprocess.CalledProcessError(expected_error_code, expected_command), 584 | ) 585 | 586 | # Check if the command-line instruction returned an exception 587 | # of type CalledProcessError with the correct error message 588 | check_cli_error_output(filename, expected_error_message, message_extra_details) 589 | 590 | # test should skip file, so no changes are made 591 | obtained = filename.read_text() 592 | assert obtained == source 593 | 594 | 595 | def run(args: Sequence[str], expected_exit: int) -> LineMatcher: 596 | runner = CliRunner() 597 | result = runner.invoke(cli.main, args) 598 | msg = "exit code %d != %d.\nOutput: %s" % (result.exit_code, expected_exit, result.output) 599 | assert result.exit_code == expected_exit, msg 600 | return LineMatcher(result.output.splitlines()) 601 | 602 | 603 | def fix_valid_file(input_file: Path) -> None: 604 | output = run([str(input_file), "--verbose"], expected_exit=0) 605 | output.fnmatch_lines(str(input_file) + ": Skipped") 606 | 607 | 608 | def _get_formatter_msg(formatter: Optional[str]) -> str: 609 | return (" (%s)" % formatter) if formatter is not None else "" 610 | 611 | 612 | def check_valid_file(input_file: Path, formatter: Optional[str] = None) -> None: 613 | output = run(["--check", "--verbose", str(input_file)], expected_exit=0) 614 | output.fnmatch_lines(str(input_file) + ": OK" + _get_formatter_msg(formatter)) 615 | 616 | 617 | def fix_invalid_file(input_file: Path, formatter: Optional[str] = None) -> None: 618 | output = run([str(input_file), "--verbose"], expected_exit=0) 619 | output.fnmatch_lines(str(input_file) + ": Fixed" + _get_formatter_msg(formatter)) 620 | 621 | 622 | def check_cli_error_output( 623 | input_file: Path, expected_error_message: str, message_details: str 624 | ) -> None: 625 | output = run([str(input_file), "--verbose"], expected_exit=1) 626 | msg = f": ERROR (CalledProcessError: {expected_error_message}): {message_details}" 627 | output.fnmatch_lines(str(input_file) + msg) 628 | 629 | 630 | def check_invalid_file(input_file: Path, formatter: Optional[str] = None) -> None: 631 | output = run(["--check", "--verbose", str(input_file)], expected_exit=1) 632 | output.fnmatch_lines(str(input_file) + ": Failed" + _get_formatter_msg(formatter)) 633 | 634 | 635 | def test_find_pyproject_toml( 636 | tmp_path: Path, monkeypatch: pytest.MonkeyPatch, black_config: Path 637 | ) -> None: 638 | os.remove(black_config) 639 | (tmp_path / "pA/p2/p3").mkdir(parents=True) 640 | (tmp_path / "pA/p2/p3/foo.py").touch() 641 | (tmp_path / "pA/p2/p3/pyproject.toml").touch() 642 | (tmp_path / "pX/p9").mkdir(parents=True) 643 | (tmp_path / "pX/p9/pyproject.toml").touch() 644 | monkeypatch.chdir(tmp_path) 645 | 646 | assert cli.find_pyproject_toml([tmp_path / "pA/p2/p3/foo.py", tmp_path / "pX/p9"]) is None 647 | assert cli.find_pyproject_toml([tmp_path / "pA/p2/p3"]) 648 | assert cli.has_black_config(tmp_path / "pA/p2/p3/pyproject.toml") is False 649 | assert cli.find_pyproject_toml([tmp_path]) is None 650 | assert cli.find_pyproject_toml([]) is None 651 | 652 | (tmp_path / "pX/p9/pyproject.toml").write_text("[tool.black]") 653 | assert cli.find_pyproject_toml([tmp_path / "pA/p2/p3/foo.py", tmp_path / "pX/p9"]) is None 654 | assert cli.find_pyproject_toml([tmp_path / "pX/p9"]) == tmp_path / "pX/p9/pyproject.toml" 655 | assert cli.has_black_config(tmp_path / "pX/p9/pyproject.toml") is True 656 | 657 | root_toml = tmp_path / "pyproject.toml" 658 | (root_toml).write_text("[tool.black]") 659 | assert cli.find_pyproject_toml([tmp_path / "pA/p2/p3/foo.py", tmp_path / "pX/p9"]) == root_toml 660 | 661 | monkeypatch.chdir(str(tmp_path / "pA/p2")) 662 | assert cli.find_pyproject_toml([Path(".")]) == root_toml 663 | 664 | 665 | def test_black_integration(tmp_path: Path, sort_cfg_to_tmp: None) -> None: 666 | (tmp_path / "pyproject.toml").write_text("[tool.black]") 667 | input_source = "import six\n" "import os\n" "x = [1,\n" " 2,\n" " 3]\n" "\n" "\n" "\n" 668 | py_file = tmp_path / "foo.py" 669 | py_file.write_text(input_source) 670 | 671 | # also write a cpp file to ensure black doesn't try to touch it 672 | input_cpp = "namespace boost {};" 673 | cpp_file = tmp_path / "foo.cpp" 674 | cpp_file.write_text(input_cpp) 675 | 676 | output = run(["--check", str(tmp_path), "--verbose"], expected_exit=1) 677 | output.fnmatch_lines( 678 | [ 679 | "Checking black on 1 files...", 680 | "*foo.cpp: OK*", 681 | "fix-format: 1 files would be changed, 1 files would be left unchanged.", 682 | ] 683 | ) 684 | obtained = py_file.read_text() 685 | assert obtained == input_source 686 | 687 | for i in range(2): 688 | output = run([str(tmp_path), "--verbose"], expected_exit=0) 689 | output.fnmatch_lines( 690 | [ 691 | "Running black on 1 files...", 692 | "*foo.cpp: Skipped*", 693 | ] 694 | ) 695 | obtained = py_file.read_text() 696 | assert obtained == "import os\n" "\n" "import six\n" "\n" "x = [1, 2, 3]\n" 697 | 698 | 699 | def test_skip_git_directory(input_file: Path, tmp_path: Path) -> None: 700 | (tmp_path / ".git").mkdir() 701 | (tmp_path / ".git/dummy.py").touch() 702 | (tmp_path / ".git/dummy.cpp").touch() 703 | 704 | output = run([str(tmp_path)], expected_exit=0) 705 | output.fnmatch_lines(["fix-format: 1 files changed, 0 files left unchanged."]) 706 | 707 | 708 | def test_black_operates_on_chunks_on_windows( 709 | tmp_path: Path, mocker: MockerFixture, sort_cfg_to_tmp: None 710 | ) -> None: 711 | """Ensure black is being called in chunks of at most 100 files on Windows. 712 | 713 | On Windows there's a limit on command-line size, so we call black in chunks there. On Linux 714 | we don't have this problem, so we always pass all files at once. 715 | """ 716 | (tmp_path / "pyproject.toml").write_text("[tool.black]") 717 | for i in range(521): 718 | (tmp_path / f"{i:03}_foo.py").touch() 719 | 720 | return_codes = [1, 0, 1, 0, 1, 0] # make black return failures in some batches 721 | subprocess_call_mock = mocker.patch.object( 722 | subprocess, "call", autospec=True, side_effect=return_codes 723 | ) 724 | output = run([str(tmp_path), "--check"], expected_exit=1) 725 | output.fnmatch_lines( 726 | ["Checking black on 521 files...", "fix-format: 521 files would be left unchanged."] 727 | ) 728 | call_list = subprocess_call_mock.call_args_list 729 | if sys.platform.startswith("win"): 730 | expected = 6 # 521 files in batches of 100. 731 | else: 732 | expected = 1 # All files are passed at once. 733 | assert len(call_list) == expected 734 | 735 | 736 | def test_exclude_patterns(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: 737 | config_content = """[tool.esss_fix_format] 738 | exclude = [ 739 | "src/drafts/*.py", 740 | "tmp/*", 741 | ] 742 | """ 743 | config_file = tmp_path / "pyproject.toml" 744 | config_file.write_text(config_content) 745 | include_patterns = ["*.cpp", "*.py"] 746 | exclude_patterns = cli.read_exclude_patterns(config_file) 747 | monkeypatch.chdir(tmp_path) 748 | assert not cli.should_format(Path("src/drafts/foo.py"), include_patterns, exclude_patterns)[0] 749 | assert cli.should_format(Path("src/drafts/foo.cpp"), include_patterns, exclude_patterns)[0] 750 | assert not cli.should_format(Path("tmp/foo.cpp"), include_patterns, exclude_patterns)[0] 751 | assert cli.should_format(Path("src/python/foo.py"), include_patterns, exclude_patterns)[0] 752 | 753 | 754 | def test_invalid_exclude_patterns(tmp_path: Path) -> None: 755 | config_content = """[tool.esss_fix_format] 756 | exclude = "src/drafts/*.py" 757 | """ 758 | 759 | config_file = tmp_path / "pyproject.toml" 760 | config_file.write_text(config_content) 761 | pytest.raises(TypeError, cli.read_exclude_patterns, config_file) 762 | 763 | 764 | def test_git_ignored_files(tmp_path: Path) -> None: 765 | # Smoke test for the get_git_ignored_files() function 766 | root = Path(__file__).parent.parent 767 | assert root.joinpath(".git").is_dir() 768 | ignored = cli.get_git_ignored_files(root) 769 | assert root.joinpath("environment.yml") in ignored 770 | 771 | # tmp_path is not tracked by git 772 | assert cli.get_git_ignored_files(tmp_path) == set() 773 | 774 | 775 | def test_git_ignored_files_integration( 776 | tmp_path: Path, monkeypatch: pytest.MonkeyPatch, sort_cfg_to_tmp: None 777 | ) -> None: 778 | # Write a file which is not properly formatted. 779 | content = textwrap.dedent( 780 | """ 781 | x = [1, 782 | 2, 783 | 3, 784 | ] 785 | """ 786 | ) 787 | fn = tmp_path.joinpath("foo.py") 788 | fn.write_text(content) 789 | 790 | tmp_path.joinpath("pyproject.toml").write_text("[tool.black]") 791 | 792 | # Mock get_git_ignored_files() to ignore the badly formatted file. 793 | monkeypatch.setattr(cli, "get_git_ignored_files", lambda p: {fn}) 794 | monkeypatch.chdir(tmp_path) 795 | run([str(".")], expected_exit=0) 796 | 797 | # Ensure the file has been properly ignored. 798 | assert fn.read_text() == content 799 | 800 | 801 | def test_exclude_patterns_relative_path_fix( 802 | tmp_path: Path, monkeypatch: pytest.MonkeyPatch 803 | ) -> None: 804 | config_content = """[tool.esss_fix_format] 805 | exclude = [ 806 | "src/drafts/*.py", 807 | "tmp/*", 808 | ] 809 | """ 810 | 811 | config_file = tmp_path / "pyproject.toml" 812 | run_dir = tmp_path / "src" 813 | run_dir.mkdir() 814 | config_file.write_text(config_content) 815 | monkeypatch.chdir(run_dir) 816 | include_patterns = ["*.py"] 817 | exclude_patterns = cli.read_exclude_patterns(config_file) 818 | assert not cli.should_format(Path("drafts/foo.py"), include_patterns, exclude_patterns)[0] 819 | 820 | 821 | @pytest.mark.skipif(os.name != "nt", reason="'subst' in only available on Windows") 822 | def test_exclude_patterns_error_on_subst( 823 | tmp_path: Path, request: pytest.FixtureRequest, sort_cfg_to_tmp: None, black_config: Path 824 | ) -> None: 825 | import subprocess 826 | 827 | request.addfinalizer(lambda: subprocess.check_call(["subst", "/D", "Z:"])) 828 | subprocess.check_call(["subst", "Z:", str(tmp_path)]) 829 | 830 | config_content = """ 831 | [tool.esss_fix_format] 832 | exclude = [ 833 | "src/drafts/*.py", 834 | "tmp/*", 835 | ] 836 | [tool.black] 837 | line-length = 100 838 | """ 839 | black_config.write_text(config_content) 840 | (tmp_path / "foo.py").touch() 841 | run(["Z:", "--check"], expected_exit=0) 842 | 843 | 844 | def test_utf8_error_handling(tmp_path: Path) -> None: 845 | file_with_no_uft8 = tmp_path.joinpath("test.cpp") 846 | file_with_no_uft8.write_bytes("""é""".encode("UTF-16")) 847 | 848 | check_utf8_error(file_with_no_uft8) 849 | 850 | 851 | def check_utf8_error(file: Path) -> None: 852 | output = run(["--check", "--verbose", str(file)], expected_exit=1) 853 | output.fnmatch_lines(str(file) + ": ERROR The file contents can not be decoded using UTF-8") 854 | --------------------------------------------------------------------------------