├── .coveragerc ├── .editorconfig ├── .flake8 ├── .gitattributes ├── .github ├── dependabot.yml └── workflows │ ├── ci.yml │ ├── pre-commit-autoupdate.yml │ └── pre-commit.yml ├── .gitignore ├── .isort.cfg ├── .pre-commit-config.yaml ├── .pre-commit-hooks.yaml ├── .python-version ├── .style.yapf ├── .vimrc ├── .vscode ├── extensions.json └── settings.default.json ├── AUTHORS ├── CHANGELOG.md ├── CONTRIBUTING.md ├── CONTRIBUTORS ├── EDITOR SUPPORT.md ├── HACKING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── plugins ├── README.md ├── pre-commit.sh └── vim │ ├── autoload │ └── yapf.vim │ └── plugin │ └── yapf.vim ├── pylintrc ├── pyproject.toml ├── third_party ├── __init__.py └── yapf_third_party │ ├── __init__.py │ ├── _ylib2to3 │ ├── Grammar.txt │ ├── LICENSE │ ├── PatternGrammar.txt │ ├── README.rst │ ├── __init__.py │ ├── fixer_base.py │ ├── fixer_util.py │ ├── patcomp.py │ ├── pgen2 │ │ ├── __init__.py │ │ ├── conv.py │ │ ├── driver.py │ │ ├── grammar.py │ │ ├── literals.py │ │ ├── parse.py │ │ ├── pgen.py │ │ ├── token.py │ │ └── tokenize.py │ ├── pygram.py │ └── pytree.py │ └── yapf_diff │ ├── LICENSE │ ├── __init__.py │ └── yapf_diff.py ├── tox.ini ├── yapf ├── __init__.py ├── __main__.py ├── _version.py ├── pyparser │ ├── __init__.py │ ├── pyparser.py │ ├── pyparser_utils.py │ ├── pyparser_visitor.py.tmpl │ └── split_penalty_visitor.py ├── pytree │ ├── __init__.py │ ├── blank_line_calculator.py │ ├── comment_splicer.py │ ├── continuation_splicer.py │ ├── pytree_unwrapper.py │ ├── pytree_utils.py │ ├── pytree_visitor.py │ ├── split_penalty.py │ └── subtype_assigner.py └── yapflib │ ├── __init__.py │ ├── errors.py │ ├── file_resources.py │ ├── format_decision_state.py │ ├── format_token.py │ ├── identify_container.py │ ├── line_joiner.py │ ├── logical_line.py │ ├── object_state.py │ ├── reformatter.py │ ├── split_penalty.py │ ├── style.py │ ├── subtypes.py │ └── yapf_api.py └── yapftests ├── __init__.py ├── blank_line_calculator_test.py ├── comment_splicer_test.py ├── file_resources_test.py ├── format_decision_state_test.py ├── format_token_test.py ├── line_joiner_test.py ├── logical_line_test.py ├── main_test.py ├── pytree_unwrapper_test.py ├── pytree_utils_test.py ├── pytree_visitor_test.py ├── reformatter_basic_test.py ├── reformatter_buganizer_test.py ├── reformatter_facebook_test.py ├── reformatter_pep8_test.py ├── reformatter_python3_test.py ├── reformatter_style_config_test.py ├── split_penalty_test.py ├── style_test.py ├── subtype_assigner_test.py ├── utils.py ├── yapf_test.py └── yapf_test_helper.py /.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | omit = 3 | */__main__.py 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | 11 | # 2 space indentation 12 | [*.py] 13 | indent_style = space 14 | indent_size = 2 15 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | ignore = 3 | # 'toml' imported but unused 4 | F401, 5 | # closing bracket does not match visual indentation 6 | E124, 7 | # continuation line over-indented for hanging indent 8 | E126, 9 | # visually indented line with same indent as next logical line, 10 | E129, 11 | # line break before binary operator 12 | W503, 13 | # line break after binary operator 14 | W504 15 | 16 | indent-size = 2 17 | max-line-length = 80 18 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | .python-version eol=lf 2 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | 4 | - package-ecosystem: "github-actions" 5 | commit-message: 6 | include: "scope" 7 | prefix: "Actions" 8 | directory: "/" 9 | labels: 10 | - "enhancement" 11 | schedule: 12 | interval: "weekly" 13 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Test with pytest 5 | 6 | on: 7 | pull_request: 8 | push: 9 | 10 | jobs: 11 | build: 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | python-version: ["3.8", "3.11", "3.12"] # no particular need for 3.9 or 3.10 17 | os: [macos-latest, ubuntu-latest, windows-latest] 18 | steps: 19 | - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 20 | - name: Set up Python ${{ matrix.python-version }} 21 | uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v4.6.0 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | - name: Upgrade pip 25 | run: >- 26 | python -m pip install 27 | --upgrade 28 | --disable-pip-version-check 29 | pip 30 | - name: Perform package installs 31 | run: >- 32 | pip install 33 | . 34 | pytest 35 | pytest-cov 36 | - name: Test with pytest 37 | run: pytest 38 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit-autoupdate.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Sebastian Pipping 2 | # Licensed under the Apache License Version 2.0 3 | 4 | name: Keep pre-commit hooks up to date 5 | 6 | on: 7 | schedule: 8 | - cron: '0 16 * * 5' # Every Friday 4pm 9 | workflow_dispatch: 10 | 11 | # NOTE: This will drop all permissions from GITHUB_TOKEN except metadata read, 12 | # and then (re)add the ones listed below: 13 | permissions: 14 | contents: write 15 | pull-requests: write 16 | 17 | jobs: 18 | pre_commit_autoupdate: 19 | name: Detect outdated pre-commit hooks 20 | runs-on: ubuntu-22.04 21 | steps: 22 | - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 23 | 24 | - name: Set up Python 3.11 25 | uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 26 | with: 27 | python-version: 3.11 28 | 29 | - name: Install pre-commit 30 | run: |- 31 | pip install \ 32 | --disable-pip-version-check \ 33 | --no-warn-script-location \ 34 | --user \ 35 | pre-commit 36 | echo "PATH=${HOME}/.local/bin:${PATH}" >> "${GITHUB_ENV}" 37 | 38 | - name: Check for outdated hooks 39 | run: |- 40 | pre-commit autoupdate 41 | git diff -- .pre-commit-config.yaml 42 | 43 | - name: Create pull request from changes (if any) 44 | id: create-pull-request 45 | uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5 46 | with: 47 | author: 'pre-commit ' 48 | base: main 49 | body: |- 50 | For your consideration. 51 | 52 | :warning: Please **CLOSE AND RE-OPEN** this pull request so that [further workflow runs get triggered](https://github.com/peter-evans/create-pull-request/blob/main/docs/concepts-guidelines.md#triggering-further-workflow-runs) for this pull request. 53 | branch: precommit-autoupdate 54 | commit-message: "pre-commit: Autoupdate" 55 | delete-branch: true 56 | draft: true 57 | labels: enhancement 58 | title: "pre-commit: Autoupdate" 59 | 60 | - name: Log pull request URL 61 | if: "${{ steps.create-pull-request.outputs.pull-request-url }}" 62 | run: | 63 | echo "Pull request URL is: ${{ steps.create-pull-request.outputs.pull-request-url }}" 64 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2023 Sebastian Pipping 2 | # Licensed under the Apache License Version 2.0 3 | 4 | name: Run pre-commit 5 | 6 | # Drop permissions to minimum for security 7 | permissions: 8 | contents: read 9 | 10 | on: 11 | pull_request: 12 | push: 13 | schedule: 14 | - cron: '0 2 * * 5' # Every Friday at 2am 15 | workflow_dispatch: 16 | 17 | jobs: 18 | pre_commit_run: 19 | name: Run pre-commit 20 | runs-on: ubuntu-22.04 21 | steps: 22 | - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 23 | 24 | - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 25 | with: 26 | python-version: 3.11 27 | 28 | - name: Install yapf (to be available to pre-commit) 29 | run: |- 30 | pip install \ 31 | --disable-pip-version-check \ 32 | --no-warn-script-location \ 33 | --user \ 34 | . 35 | echo "PATH=${HOME}/.local/bin:${PATH}" >> "${GITHUB_ENV}" 36 | 37 | - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #==============================================================================# 2 | # This file specifies intentionally untracked files that git should ignore. 3 | # See: http://www.kernel.org/pub/software/scm/git/docs/gitignore.html 4 | # 5 | # This file is intentionally different from the output of `git svn show-ignore`, 6 | # as most of those are useless. 7 | #==============================================================================# 8 | 9 | #==============================================================================# 10 | # File extensions to be ignored anywhere in the tree. 11 | #==============================================================================# 12 | # Temp files created by most text editors. 13 | *~ 14 | # Merge files created by git. 15 | *.orig 16 | # Compiled python. 17 | *.pyc 18 | *.pickle 19 | # vim swap files 20 | .*.sw? 21 | .sw? 22 | # OS X specific files. 23 | .DS_store 24 | 25 | #==============================================================================# 26 | # Files to ignore 27 | #==============================================================================# 28 | /.coverage 29 | 30 | # Directories to ignore (do not add trailing '/'s, they skip symlinks). 31 | #==============================================================================# 32 | /build 33 | /dist 34 | /.tox 35 | /yapf.egg-info 36 | 37 | # IDEs 38 | /.idea 39 | /.vscode/settings.json 40 | 41 | # Virtual Environment 42 | /.venv*/ 43 | 44 | # Worktrees 45 | /.wt 46 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | force_single_line=true 3 | known_third_party=yapf_third_party 4 | known_yapftests=yapftests 5 | 6 | sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER,YAPFTESTS 7 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # File introduces automated checks triggered on git events 2 | # to enable run `pip install pre-commit && pre-commit install` 3 | 4 | repos: 5 | - repo: https://github.com/pycqa/isort 6 | rev: 6.0.0 7 | hooks: 8 | - id: isort 9 | name: isort (python) 10 | - repo: local 11 | hooks: 12 | - id: yapf 13 | name: yapf 14 | language: python 15 | entry: yapf 16 | args: [-i] 17 | types: [python] 18 | - repo: https://github.com/pycqa/flake8 19 | rev: 7.1.1 20 | hooks: 21 | - id: flake8 22 | - repo: https://github.com/pre-commit/pre-commit-hooks 23 | rev: v5.0.0 24 | hooks: 25 | - id: trailing-whitespace 26 | - id: check-docstring-first 27 | - id: check-added-large-files 28 | - id: check-yaml 29 | - id: debug-statements 30 | - id: check-merge-conflict 31 | - id: double-quote-string-fixer 32 | - id: end-of-file-fixer 33 | - repo: meta 34 | hooks: 35 | - id: check-hooks-apply 36 | - id: check-useless-excludes 37 | -------------------------------------------------------------------------------- /.pre-commit-hooks.yaml: -------------------------------------------------------------------------------- 1 | # File configures YAPF to be used as a git hook with https://github.com/pre-commit/pre-commit 2 | 3 | - id: yapf 4 | name: yapf 5 | description: "A formatter for Python files." 6 | entry: yapf 7 | args: [-i] #inplace 8 | language: python 9 | types: [python] 10 | 11 | - id: yapf-diff 12 | name: yapf-diff 13 | description: "A formatter for Python files. (formats only changes included in commit)" 14 | always_run: true 15 | language: python 16 | pass_filenames: false 17 | stages: [pre-commit] 18 | entry: | 19 | bash -c "git diff -U0 --no-color --relative HEAD \ 20 | | yapf-diff \ 21 | | tee >(git apply --allow-empty -p0)" 22 | -------------------------------------------------------------------------------- /.python-version: -------------------------------------------------------------------------------- 1 | 3.7.9 2 | 3.8.10 3 | 3.9.13 4 | 3.10.11 5 | 3.11.5 6 | -------------------------------------------------------------------------------- /.style.yapf: -------------------------------------------------------------------------------- 1 | [style] 2 | based_on_style = yapf 3 | -------------------------------------------------------------------------------- /.vimrc: -------------------------------------------------------------------------------- 1 | " Force indentation styles for this directory 2 | autocmd FileType python set shiftwidth=2 3 | autocmd FileType python set tabstop=2 4 | autocmd FileType python set softtabstop=2 5 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "eeyore.yapf", 4 | "dangmai.workspace-default-settings", 5 | "ms-python.flake8", 6 | "ms-python.isort", 7 | "ms-python.python", 8 | ], 9 | // These are remarked as extenstions you should disable for this workspace. 10 | // VSCode does not support disabling extensions via workspace config files. 11 | "unwantedRecommendations": [ 12 | "ms-python.black-formatter", 13 | "ms-python.pylint" 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /.vscode/settings.default.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.codeActionsOnSave": { 3 | "source.organizeImports": true 4 | }, 5 | "files.insertFinalNewline": true, 6 | "files.trimFinalNewlines": true, 7 | "[python]": { 8 | "diffEditor.ignoreTrimWhitespace": false, 9 | "editor.defaultFormatter": "eeyore.yapf", 10 | "editor.formatOnSaveMode": "file", 11 | "editor.formatOnSave": true, 12 | "editor.wordBasedSuggestions": false, 13 | "files.trimTrailingWhitespace": true, 14 | }, 15 | "python.analysis.extraPaths": [ 16 | "./third_party" 17 | ], 18 | "python.analysis.typeCheckingMode": "basic", 19 | "python.languageServer": "Pylance", 20 | "files.exclude": { 21 | "**/*$py.class": true 22 | }, 23 | "json.schemas": [ 24 | { 25 | "fileMatch": [ 26 | "/.vscode/settings.default.json" 27 | ], 28 | "url": "vscode://schemas/settings/folder" 29 | } 30 | ], 31 | "workspace-default-settings.runOnActivation": true, 32 | "workspace-default-settings.jsonIndentation": 4 33 | } 34 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the official list of YAPF authors for copyright purposes. 2 | # This file is distinct from the CONTRIBUTORS files. 3 | # See the latter for an explanation. 4 | 5 | # Names should be added to this file as: 6 | # Name or Organization 7 | # The email address is not required for organizations. 8 | 9 | Google Inc. 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | Want to contribute? Great! First, read this page (including the small print at 4 | the end). 5 | 6 | ## Before you contribute 7 | 8 | Before we can use your code, you must sign the [Google Individual Contributor 9 | License Agreement](https://developers.google.com/open-source/cla/individual?csw=1) 10 | (CLA), which you can do online. The CLA is necessary mainly because you own the 11 | copyright to your changes, even after your contribution becomes part of our 12 | codebase, so we need your permission to use and distribute your code. We also 13 | need to be sure of various other things—for instance that you'll tell us if you 14 | know that your code infringes on other people's patents. You don't have to sign 15 | the CLA until after you've submitted your code for review and a member has 16 | approved it, but you must do it before we can put your code into our codebase. 17 | Before you start working on a larger contribution, you should get in touch with 18 | us first through the issue tracker with your idea so that we can help out and 19 | possibly guide you. Coordinating up front makes it much easier to avoid 20 | frustration later on. 21 | 22 | ## Code reviews 23 | 24 | All submissions, including submissions by project members, require review. We 25 | use Github pull requests for this purpose. 26 | 27 | ## YAPF coding style 28 | 29 | YAPF follows the [Google Python Style Guide](https://google.github.io/styleguide/pyguide.html) 30 | with two exceptions: 31 | 32 | - 2 spaces for indentation rather than 4. 33 | - CamelCase for function and method names rather than `snake_case`. 34 | 35 | The rationale for this is that YAPF was initially developed at Google where 36 | these two exceptions are still part of the internal Python style guide. 37 | 38 | ## Getting started 39 | YAPF supports using tox 3 for creating a local dev environment, testing, and 40 | building redistributables. See [HACKING.md](HACKING.md) for more info. 41 | 42 | ```bash 43 | $ pipx run --spec='tox<4' tox --devenv .venv 44 | ``` 45 | 46 | ## Small print 47 | 48 | Contributions made by corporations are covered by a different agreement than 49 | the one above, the Software Grant and Corporate Contributor License Agreement. 50 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # People who have agreed to one of the CLAs and can contribute patches. 2 | # The AUTHORS file lists the copyright holders; this file 3 | # lists people. For example, Google employees are listed here 4 | # but not in AUTHORS, because Google holds the copyright. 5 | # 6 | # https://developers.google.com/open-source/cla/individual 7 | # https://developers.google.com/open-source/cla/corporate 8 | # 9 | # Names should be added to this file as: 10 | # Name 11 | 12 | Bill Wendling 13 | Eli Bendersky 14 | Sam Clegg 15 | Łukasz Langa 16 | Oleg Butuzov 17 | Mauricio Herrera Cuadra 18 | Kyle Gottfried 19 | -------------------------------------------------------------------------------- /EDITOR SUPPORT.md: -------------------------------------------------------------------------------- 1 | # Using YAPF with your editor 2 | 3 | YAPF is supported by multiple editors via community extensions or plugins. 4 | 5 | - [IntelliJ/PyCharm](#intellijpycharm) 6 | - [IPython](#ipython) 7 | - [VSCode](#vscode) 8 | 9 | ## IntelliJ/PyCharm 10 | 11 | Use the `File Watchers` plugin to run YAPF against a file when you perform a save. 12 | 13 | 1. Install the [File Watchers](https://www.jetbrains.com/help/idea/using-file-watchers.html) Plugin 14 | 1. Add the following `.idea/watcherTasks.xml` to your project. If you already have this file just add the `TaskOptions` section from below. This example uses Windows and a virtual environment, modify the `program` option as appropriate. 15 | ```xml 16 | 17 | 18 | 19 | 20 | 31 | 39 | 40 | 41 | ``` 42 | 43 | ## IPython 44 | 45 | IPython supports formatting lines automatically when you press the `` button to submit the current code block. 46 | 47 | Make sure that the YAPF module is available to the IPython runtime: 48 | 49 | ```shell 50 | pip install ipython yapf 51 | ``` 52 | 53 | pipx example: 54 | 55 | ```shell 56 | pipx install ipython 57 | pipx inject ipython yapf 58 | ``` 59 | 60 | Add following to `~/.ipython/profile_default/ipython_config.py`: 61 | 62 | ```python 63 | c.TerminalInteractiveShell.autoformatter = 'yapf' 64 | ``` 65 | 66 | ## VSCode 67 | 68 | VSCode has deprecated support for YAPF in its official Python extension [in favor of dedicated formatter extensions](https://github.com/microsoft/vscode-python/wiki/Migration-to-Python-Tools-Extensions). 69 | 70 | 1. Install EeyoreLee's [yapf](https://marketplace.visualstudio.com/items?itemName=eeyore.yapf) extension. 71 | 1. Install the yapf package from pip. 72 | ``` 73 | pip install yapf 74 | ``` 75 | 1. Add the following to VSCode's `settings.json`: 76 | ```jsonc 77 | "[python]": { 78 | "editor.formatOnSaveMode": "file", 79 | "editor.formatOnSave": true, 80 | "editor.defaultFormatter": "eeyore.yapf" # choose this extension 81 | }, 82 | ``` 83 | -------------------------------------------------------------------------------- /HACKING.md: -------------------------------------------------------------------------------- 1 | ## Running YAPF on itself 2 | 3 | - To run YAPF on all of YAPF: 4 | 5 | ```bash 6 | $ pipx run --spec=${PWD} --no-cache yapf -m -i -r yapf/ yapftests/ third_party/ 7 | ``` 8 | 9 | - To run YAPF on just the files changed in the current git branch: 10 | 11 | ```bash 12 | $ pipx run --spec=${PWD} --no-cache yapf -m -i $(git diff --name-only @{upstream}) 13 | ``` 14 | 15 | ## Testing and building redistributables locally 16 | 17 | YAPF uses tox 3 to test against multiple python versions and to build redistributables. 18 | 19 | Tox will opportunistically use pyenv environments when available. 20 | To configure pyenv run the following in bash: 21 | 22 | ```bash 23 | $ xargs -t -n1 pyenv install < .python-version 24 | ``` 25 | 26 | Test against all supported Python versions that are currently installed: 27 | ```bash 28 | $ pipx run --spec='tox<4' tox 29 | ``` 30 | 31 | Build and test the sdist and wheel against your default Python environment. The redistributables will be in the `dist` directory. 32 | ```bash 33 | $ pipx run --spec='tox<4' tox -e bdist_wheel -e sdist 34 | ``` 35 | 36 | ## Releasing a new version 37 | 38 | 1. Install all expected pyenv environements 39 | ```bash 40 | $ xargs -t -n1 pyenv install < .python-version 41 | ``` 42 | 43 | 1. Run tests against Python 3.7 - 3.11 with 44 | ```bash 45 | $ pipx run --spec='tox<4' tox 46 | ``` 47 | 48 | 1. Bump version in `yapf/_version.py`. 49 | 50 | 1. Build and test redistributables 51 | 52 | ```bash 53 | $ pipx run --spec='tox<4' tox -e bdist_wheel -e sdist 54 | ``` 55 | 56 | 1. Check that it looks OK. 57 | 1. Install it onto a virtualenv, 58 | 1. run tests, and 59 | 1. run yapf as a tool. 60 | 61 | 1. Push to PyPI: 62 | 63 | ```bash 64 | $ pipx run twine upload dist/* 65 | ``` 66 | 67 | 1. Test in a clean virtualenv that 'pip install yapf' works with the new 68 | version. 69 | 70 | 1. Commit the version bump and add tag with: 71 | 72 | ```bash 73 | $ git tag v$(VERSION_NUM) 74 | $ git push --tags 75 | ``` 76 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include HACKING.md LICENSE AUTHORS CHANGELOG.md CONTRIBUTING.md CONTRIBUTORS 2 | include .coveragerc .editorconfig .flake8 plugins/README.md 3 | include plugins/vim/autoload/yapf.vim plugins/vim/plugin/yapf.vim pylintrc 4 | include .style.yapf tox.ini .travis.yml .vimrc 5 | -------------------------------------------------------------------------------- /plugins/README.md: -------------------------------------------------------------------------------- 1 | # IDE Plugins 2 | 3 | ## Emacs 4 | 5 | The `Emacs` plugin is maintained separately. Installation directions can be 6 | found here: https://github.com/paetzke/py-yapf.el 7 | 8 | 9 | ## Vim 10 | 11 | The `vim` plugin allows you to reformat a range of code. Copy `plugin` and 12 | `autoload` directories into your `~/.vim` or use `:packadd` in Vim 8. Or use 13 | a plugin manager like Plug or Vundle: 14 | 15 | ```vim 16 | " Plug 17 | Plug 'google/yapf', { 'rtp': 'plugins/vim', 'for': 'python' } 18 | 19 | " Vundle 20 | Plugin 'google/yapf', { 'rtp': 'plugins/vim' } 21 | ``` 22 | 23 | You can add key bindings in the `.vimrc` file: 24 | 25 | ```vim 26 | map :call yapf#YAPF() 27 | imap :call yapf#YAPF() 28 | ``` 29 | 30 | Alternatively, you can call the command `YAPF`. If you omit the range, it will 31 | reformat the whole buffer. 32 | 33 | example: 34 | 35 | ```vim 36 | :YAPF " formats whole buffer 37 | :'<,'>YAPF " formats lines selected in visual mode 38 | ``` 39 | 40 | 41 | ## Sublime Text 42 | 43 | The `Sublime Text` plugin is also maintained separately. It is compatible with 44 | both Sublime Text 2 and 3. 45 | 46 | The plugin can be easily installed by using *Sublime Package Control*. Check 47 | the project page of the plugin for more information: https://github.com/jason-kane/PyYapf 48 | 49 | 50 | ## git Pre-Commit Hook 51 | 52 | The `git` pre-commit hook automatically formats your Python files before they 53 | are committed to your local repository. Any changes `yapf` makes to the files 54 | will stay unstaged so that you can diff them manually. 55 | 56 | To install, simply download the raw file and copy it into your git hooks 57 | directory: 58 | 59 | ```bash 60 | # From the root of your git project. 61 | $ curl -o pre-commit.sh https://raw.githubusercontent.com/google/yapf/main/plugins/pre-commit.sh 62 | $ chmod a+x pre-commit.sh 63 | $ mv pre-commit.sh .git/hooks/pre-commit 64 | ``` 65 | 66 | 67 | ## Textmate 2 68 | 69 | Plugin for `Textmate 2` requires `yapf` Python package installed on your 70 | system: 71 | 72 | ```bash 73 | $ pip install yapf 74 | ``` 75 | 76 | Also, you will need to activate `Python` bundle from `Preferences > Bundles`. 77 | 78 | Finally, create a `~/Library/Application Support/TextMate/Bundles/Python.tmbundle/Commands/YAPF.tmCommand` 79 | file with the following content: 80 | 81 | ```xml 82 | 83 | 84 | 85 | 86 | beforeRunningCommand 87 | saveActiveFile 88 | command 89 | #!/bin/bash 90 | 91 | TPY=${TM_PYTHON:-python} 92 | 93 | "$TPY" "/usr/local/bin/yapf" "$TM_FILEPATH" 94 | input 95 | document 96 | name 97 | YAPF 98 | scope 99 | source.python 100 | uuid 101 | 297D5A82-2616-4950-9905-BD2D1C94D2D4 102 | 103 | 104 | ``` 105 | 106 | You will see a new menu item `Bundles > Python > YAPF`. 107 | -------------------------------------------------------------------------------- /plugins/pre-commit.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Git pre-commit hook to check staged Python files for formatting issues with 4 | # yapf. 5 | # 6 | # INSTALLING: Copy this script into `.git/hooks/pre-commit`, and mark it as 7 | # executable. 8 | # 9 | # This requires that yapf is installed and runnable in the environment running 10 | # the pre-commit hook. 11 | # 12 | # When running, this first checks for unstaged changes to staged files, and if 13 | # there are any, it will exit with an error. Files with unstaged changes will be 14 | # printed. 15 | # 16 | # If all staged files have no unstaged changes, it will run yapf against them, 17 | # leaving the formatting changes unstaged. Changed files will be printed. 18 | # 19 | # BUGS: This does not leave staged changes alone when used with the -a flag to 20 | # git commit, due to the fact that git stages ALL unstaged files when that flag 21 | # is used. 22 | 23 | # Find all staged Python files, and exit early if there aren't any. 24 | PYTHON_FILES=() 25 | while IFS=$'\n' read -r line; do PYTHON_FILES+=("$line"); done \ 26 | < <(git diff --name-only --cached --diff-filter=AM | grep --color=never '.py$') 27 | if [ ${#PYTHON_FILES[@]} -eq 0 ]; then 28 | exit 0 29 | fi 30 | 31 | ########## PIP VERSION ############# 32 | # Verify that yapf is installed; if not, warn and exit. 33 | if ! command -v yapf >/dev/null; then 34 | echo 'yapf not on path; can not format. Please install yapf:' 35 | echo ' pip install yapf' 36 | exit 2 37 | fi 38 | ######### END PIP VERSION ########## 39 | 40 | ########## PIPENV VERSION ########## 41 | # if ! pipenv run yapf --version 2>/dev/null 2>&1; then 42 | # echo 'yapf not on path; can not format. Please install yapf:' 43 | # echo ' pipenv install yapf' 44 | # exit 2 45 | # fi 46 | ###### END PIPENV VERSION ########## 47 | 48 | 49 | # Check for unstaged changes to files in the index. 50 | CHANGED_FILES=() 51 | while IFS=$'\n' read -r line; do CHANGED_FILES+=("$line"); done \ 52 | < <(git diff --name-only "${PYTHON_FILES[@]}") 53 | if [ ${#CHANGED_FILES[@]} -gt 0 ]; then 54 | echo 'You have unstaged changes to some files in your commit; skipping ' 55 | echo 'auto-format. Please stage, stash, or revert these changes. You may ' 56 | echo 'find `git stash -k` helpful here.' 57 | echo 'Files with unstaged changes:' "${CHANGED_FILES[@]}" 58 | exit 1 59 | fi 60 | 61 | # Format all staged files, then exit with an error code if any have uncommitted 62 | # changes. 63 | echo 'Formatting staged Python files . . .' 64 | 65 | ########## PIP VERSION ############# 66 | yapf -i -r "${PYTHON_FILES[@]}" 67 | ######### END PIP VERSION ########## 68 | 69 | ########## PIPENV VERSION ########## 70 | # pipenv run yapf -i -r "${PYTHON_FILES[@]}" 71 | ###### END PIPENV VERSION ########## 72 | 73 | 74 | CHANGED_FILES=() 75 | while IFS=$'\n' read -r line; do CHANGED_FILES+=("$line"); done \ 76 | < <(git diff --name-only "${PYTHON_FILES[@]}") 77 | if [ ${#CHANGED_FILES[@]} -gt 0 ]; then 78 | echo 'Reformatted staged files. Please review and stage the changes.' 79 | echo 'Files updated: ' "${CHANGED_FILES[@]}" 80 | exit 1 81 | else 82 | exit 0 83 | fi 84 | -------------------------------------------------------------------------------- /plugins/vim/autoload/yapf.vim: -------------------------------------------------------------------------------- 1 | " Copyright 2015 Google Inc. All Rights Reserved. 2 | " 3 | " Licensed under the Apache License, Version 2.0 (the "License"); 4 | " you may not use this file except in compliance with the License. 5 | " You may obtain a copy of the License at 6 | " 7 | " http://www.apache.org/licenses/LICENSE-2.0 8 | " 9 | " Unless required by applicable law or agreed to in writing, software 10 | " distributed under the License is distributed on an "AS IS" BASIS, 11 | " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | " See the License for the specific language governing permissions and 13 | " limitations under the License. 14 | """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 15 | " VIM Autoload script for YAPF support 16 | " 17 | " Place this script in your ~/.vim/autoload directory. You can add accessors to 18 | " ~/.vimrc, e.g.: 19 | " 20 | " map :call yapf#YAPF() 21 | " imap :call yapf#YAPF() 22 | " 23 | function! yapf#YAPF() range 24 | " Determine range to format. 25 | let l:line_ranges = a:firstline . '-' . a:lastline 26 | let l:cmd = 'yapf --lines=' . l:line_ranges 27 | 28 | " Call YAPF with the current buffer 29 | if exists('*systemlist') 30 | let l:formatted_text = systemlist(l:cmd, join(getline(1, '$'), "\n") . "\n") 31 | else 32 | let l:formatted_text = 33 | \ split(system(l:cmd, join(getline(1, '$'), "\n") . "\n"), "\n") 34 | endif 35 | 36 | if v:shell_error 37 | echohl ErrorMsg 38 | echomsg printf('"%s" returned error: %s', l:cmd, l:formatted_text[-1]) 39 | echohl None 40 | return 41 | endif 42 | 43 | " Update the buffer. 44 | execute '1,' . string(line('$')) . 'delete' 45 | call setline(1, l:formatted_text) 46 | 47 | " Reset cursor to first line of the formatted range. 48 | call cursor(a:firstline, 1) 49 | endfunction 50 | -------------------------------------------------------------------------------- /plugins/vim/plugin/yapf.vim: -------------------------------------------------------------------------------- 1 | " Copyright 2015 Google Inc. All Rights Reserved. 2 | " 3 | " Licensed under the Apache License, Version 2.0 (the "License"); 4 | " you may not use this file except in compliance with the License. 5 | " You may obtain a copy of the License at 6 | " 7 | " http://www.apache.org/licenses/LICENSE-2.0 8 | " 9 | " Unless required by applicable law or agreed to in writing, software 10 | " distributed under the License is distributed on an "AS IS" BASIS, 11 | " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | " See the License for the specific language governing permissions and 13 | " limitations under the License. 14 | """"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" 15 | " VIM command for YAPF support 16 | " 17 | " Place this script in your ~/.vim/plugin directory. You can call the 18 | " command YAPF. If you omit the range, it will reformat the whole 19 | " buffer. 20 | " 21 | " example: 22 | " :YAPF " formats whole buffer 23 | " :'<,'>YAPF " formats lines selected in visual mode 24 | 25 | command! -range=% YAPF ,call yapf#YAPF() 26 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=58.5.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "yapf" 7 | description = "A formatter for Python code" 8 | authors = [{ name = "Google Inc." }] 9 | maintainers = [{ name = "Bill Wendling", email = "morbo@google.com" }] 10 | dynamic = ["version"] 11 | license = { file = "LICENSE" } 12 | readme = "README.md" 13 | requires-python = ">=3.7" 14 | classifiers = [ 15 | 'Development Status :: 4 - Beta', 16 | 'Environment :: Console', 17 | 'Intended Audience :: Developers', 18 | 'License :: OSI Approved :: Apache Software License', 19 | 'Operating System :: OS Independent', 20 | 'Programming Language :: Python', 21 | 'Programming Language :: Python :: 3 :: Only', 22 | 'Programming Language :: Python :: 3.7', 23 | 'Programming Language :: Python :: 3.8', 24 | 'Programming Language :: Python :: 3.9', 25 | 'Programming Language :: Python :: 3.10', 26 | 'Programming Language :: Python :: 3.11', 27 | 'Topic :: Software Development :: Libraries :: Python Modules', 28 | 'Topic :: Software Development :: Quality Assurance', 29 | ] 30 | dependencies = ['platformdirs>=3.5.1', 'tomli>=2.0.1; python_version<"3.11"'] 31 | 32 | [project.scripts] 33 | yapf = "yapf:run_main" 34 | yapf-diff = "yapf_third_party.yapf_diff.yapf_diff:main" 35 | 36 | [project.urls] 37 | # https://daniel.feldroy.com/posts/2023-08-pypi-project-urls-cheatsheet 38 | Home = 'https://github.com/google/yapf' 39 | Changelog = 'https://github.com/google/yapf/blob/main/CHANGELOG.md' 40 | Docs = 'https://github.com/google/yapf/blob/main/README.md#yapf' 41 | Issues = 'https://github.com/google/yapf/issues' 42 | 43 | [tool.distutils.bdist_wheel] 44 | python_tag = "py3" 45 | 46 | [tool.setuptools] 47 | include-package-data = true 48 | package-dir = { yapf_third_party = 'third_party/yapf_third_party' } 49 | 50 | [tool.setuptools.dynamic] 51 | version = { attr = "yapf._version.__version__" } 52 | 53 | [tool.setuptools.packages.find] 54 | where = [".", 'third_party'] 55 | include = ["yapf*", 'yapftests*'] 56 | 57 | [tool.setuptools.package-data] 58 | yapf_third_party = [ 59 | 'yapf_diff/LICENSE', 60 | '_ylib2to3/Grammar.txt', 61 | '_ylib2to3/PatternGrammar.txt', 62 | '_ylib2to3/LICENSE', 63 | ] 64 | -------------------------------------------------------------------------------- /third_party/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/yapf/12005095296072751e3e4c1f33a047d41b0ce18d/third_party/__init__.py -------------------------------------------------------------------------------- /third_party/yapf_third_party/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/yapf/12005095296072751e3e4c1f33a047d41b0ce18d/third_party/yapf_third_party/__init__.py -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/PatternGrammar.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2006 Google, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | # A grammar to describe tree matching patterns. 5 | # Not shown here: 6 | # - 'TOKEN' stands for any token (leaf node) 7 | # - 'any' stands for any node (leaf or interior) 8 | # With 'any' we can still specify the sub-structure. 9 | 10 | # The start symbol is 'Matcher'. 11 | 12 | Matcher: Alternatives ENDMARKER 13 | 14 | Alternatives: Alternative ('|' Alternative)* 15 | 16 | Alternative: (Unit | NegatedUnit)+ 17 | 18 | Unit: [NAME '='] ( STRING [Repeater] 19 | | NAME [Details] [Repeater] 20 | | '(' Alternatives ')' [Repeater] 21 | | '[' Alternatives ']' 22 | ) 23 | 24 | NegatedUnit: 'not' (STRING | NAME [Details] | '(' Alternatives ')') 25 | 26 | Repeater: '*' | '+' | '{' NUMBER [',' NUMBER] '}' 27 | 28 | Details: '<' Alternatives '>' 29 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/README.rst: -------------------------------------------------------------------------------- 1 | A fork of python's lib2to3 with select features backported from black's blib2to3. 2 | 3 | Reasons for forking: 4 | 5 | - black's fork of lib2to3 already considers newer features like Structured Pattern matching 6 | - lib2to3 itself is deprecated and no longer getting support 7 | 8 | Maintenance moving forward: 9 | - Most changes moving forward should only have to be done to the grammar files in this project. 10 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/__init__.py: -------------------------------------------------------------------------------- 1 | """fork of python's lib2to3 with some backports from black's blib2to3""" 2 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/fixer_base.py: -------------------------------------------------------------------------------- 1 | # Copyright 2006 Google, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | """Base class for fixers (optional, but recommended).""" 4 | 5 | # Python imports 6 | import itertools 7 | 8 | from . import pygram 9 | from .fixer_util import does_tree_import 10 | # Local imports 11 | from .patcomp import PatternCompiler 12 | 13 | 14 | class BaseFix(object): 15 | """Optional base class for fixers. 16 | 17 | The subclass name must be FixFooBar where FooBar is the result of 18 | removing underscores and capitalizing the words of the fix name. 19 | For example, the class name for a fixer named 'has_key' should be 20 | FixHasKey. 21 | """ 22 | 23 | PATTERN = None # Most subclasses should override with a string literal 24 | pattern = None # Compiled pattern, set by compile_pattern() 25 | pattern_tree = None # Tree representation of the pattern 26 | options = None # Options object passed to initializer 27 | filename = None # The filename (set by set_filename) 28 | numbers = itertools.count(1) # For new_name() 29 | used_names = set() # A set of all used NAMEs 30 | order = 'post' # Does the fixer prefer pre- or post-order traversal 31 | explicit = False # Is this ignored by refactor.py -f all? 32 | run_order = 5 # Fixers will be sorted by run order before execution 33 | # Lower numbers will be run first. 34 | _accept_type = None # [Advanced and not public] This tells RefactoringTool 35 | # which node type to accept when there's not a pattern. 36 | 37 | keep_line_order = False # For the bottom matcher: match with the 38 | # original line order 39 | BM_compatible = False # Compatibility with the bottom matching 40 | # module; every fixer should set this 41 | # manually 42 | 43 | # Shortcut for access to Python grammar symbols 44 | syms = pygram.python_symbols 45 | 46 | def __init__(self, options, log): 47 | """Initializer. Subclass may override. 48 | 49 | Args: 50 | options: a dict containing the options passed to RefactoringTool 51 | that could be used to customize the fixer through the command line. 52 | log: a list to append warnings and other messages to. 53 | """ 54 | self.options = options 55 | self.log = log 56 | self.compile_pattern() 57 | 58 | def compile_pattern(self): 59 | """Compiles self.PATTERN into self.pattern. 60 | 61 | Subclass may override if it doesn't want to use 62 | self.{pattern,PATTERN} in .match(). 63 | """ 64 | if self.PATTERN is not None: 65 | PC = PatternCompiler() 66 | self.pattern, self.pattern_tree = PC.compile_pattern( 67 | self.PATTERN, with_tree=True) 68 | 69 | def set_filename(self, filename): 70 | """Set the filename. 71 | 72 | The main refactoring tool should call this. 73 | """ 74 | self.filename = filename 75 | 76 | def match(self, node): 77 | """Returns match for a given parse tree node. 78 | 79 | Should return a true or false object (not necessarily a bool). 80 | It may return a non-empty dict of matching sub-nodes as 81 | returned by a matching pattern. 82 | 83 | Subclass may override. 84 | """ 85 | results = {'node': node} 86 | return self.pattern.match(node, results) and results 87 | 88 | def transform(self, node, results): 89 | """Returns the transformation for a given parse tree node. 90 | 91 | Args: 92 | node: the root of the parse tree that matched the fixer. 93 | results: a dict mapping symbolic names to part of the match. 94 | 95 | Returns: 96 | None, or a node that is a modified copy of the 97 | argument node. The node argument may also be modified in-place to 98 | effect the same change. 99 | 100 | Subclass *must* override. 101 | """ 102 | raise NotImplementedError() 103 | 104 | def new_name(self, template='xxx_todo_changeme'): 105 | """Return a string suitable for use as an identifier 106 | 107 | The new name is guaranteed not to conflict with other identifiers. 108 | """ 109 | name = template 110 | while name in self.used_names: 111 | name = template + str(next(self.numbers)) 112 | self.used_names.add(name) 113 | return name 114 | 115 | def log_message(self, message): 116 | if self.first_log: 117 | self.first_log = False 118 | self.log.append('### In file %s ###' % self.filename) 119 | self.log.append(message) 120 | 121 | def cannot_convert(self, node, reason=None): 122 | """Warn the user that a given chunk of code is not valid Python 3, 123 | but that it cannot be converted automatically. 124 | 125 | First argument is the top-level node for the code in question. 126 | Optional second argument is why it can't be converted. 127 | """ 128 | lineno = node.get_lineno() 129 | for_output = node.clone() 130 | for_output.prefix = '' 131 | msg = 'Line %d: could not convert: %s' 132 | self.log_message(msg % (lineno, for_output)) 133 | if reason: 134 | self.log_message(reason) 135 | 136 | def warning(self, node, reason): 137 | """Used for warning the user about possible uncertainty in the translation. 138 | 139 | First argument is the top-level node for the code in question. 140 | Optional second argument is why it can't be converted. 141 | """ 142 | lineno = node.get_lineno() 143 | self.log_message('Line %d: %s' % (lineno, reason)) 144 | 145 | def start_tree(self, tree, filename): 146 | """Some fixers need to maintain tree-wide state. 147 | 148 | This method is called once, at the start of tree fix-up. 149 | 150 | tree - the root node of the tree to be processed. 151 | filename - the name of the file the tree came from. 152 | """ 153 | self.used_names = tree.used_names 154 | self.set_filename(filename) 155 | self.numbers = itertools.count(1) 156 | self.first_log = True 157 | 158 | def finish_tree(self, tree, filename): 159 | """Some fixers need to maintain tree-wide state. 160 | 161 | This method is called once, at the conclusion of tree fix-up. 162 | 163 | tree - the root node of the tree to be processed. 164 | filename - the name of the file the tree came from. 165 | """ 166 | pass 167 | 168 | 169 | class ConditionalFix(BaseFix): 170 | """ Base class for fixers which not execute if an import is found. """ 171 | 172 | # This is the name of the import which, if found, will cause the test to be 173 | # skipped. 174 | skip_on = None 175 | 176 | def start_tree(self, *args): 177 | super(ConditionalFix, self).start_tree(*args) 178 | self._should_skip = None 179 | 180 | def should_skip(self, node): 181 | if self._should_skip is not None: 182 | return self._should_skip 183 | pkg = self.skip_on.split('.') 184 | name = pkg[-1] 185 | pkg = '.'.join(pkg[:-1]) 186 | self._should_skip = does_tree_import(pkg, name, node) 187 | return self._should_skip 188 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/patcomp.py: -------------------------------------------------------------------------------- 1 | # Copyright 2006 Google, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | """Pattern compiler. 4 | 5 | The grammar is taken from PatternGrammar.txt. 6 | 7 | The compiler compiles a pattern to a pytree.*Pattern instance. 8 | """ 9 | 10 | __author__ = 'Guido van Rossum ' 11 | 12 | # Python imports 13 | import io 14 | 15 | # Really local imports 16 | from . import pygram 17 | from . import pytree 18 | # Fairly local imports 19 | from .pgen2 import driver 20 | from .pgen2 import grammar 21 | from .pgen2 import literals 22 | from .pgen2 import parse 23 | from .pgen2 import token 24 | from .pgen2 import tokenize 25 | 26 | 27 | class PatternSyntaxError(Exception): 28 | pass 29 | 30 | 31 | def tokenize_wrapper(input): 32 | """Tokenizes a string suppressing significant whitespace.""" 33 | skip = {token.NEWLINE, token.INDENT, token.DEDENT} 34 | tokens = tokenize.generate_tokens(io.StringIO(input).readline) 35 | for quintuple in tokens: 36 | type, value, start, end, line_text = quintuple 37 | if type not in skip: 38 | yield quintuple 39 | 40 | 41 | class PatternCompiler(object): 42 | 43 | def __init__(self, grammar_file=None): 44 | """Initializer. 45 | 46 | Takes an optional alternative filename for the pattern grammar. 47 | """ 48 | if grammar_file is None: 49 | self.grammar = pygram.pattern_grammar 50 | self.syms = pygram.pattern_symbols 51 | else: 52 | self.grammar = driver.load_grammar(grammar_file) 53 | self.syms = pygram.Symbols(self.grammar) 54 | self.pygrammar = pygram.python_grammar 55 | self.pysyms = pygram.python_symbols 56 | self.driver = driver.Driver(self.grammar, convert=pattern_convert) 57 | 58 | def compile_pattern(self, input, debug=False, with_tree=False): 59 | """Compiles a pattern string to a nested pytree.*Pattern object.""" 60 | tokens = tokenize_wrapper(input) 61 | try: 62 | root = self.driver.parse_tokens(tokens, debug=debug) 63 | except parse.ParseError as e: 64 | raise PatternSyntaxError(str(e)) from None 65 | if with_tree: 66 | return self.compile_node(root), root 67 | else: 68 | return self.compile_node(root) 69 | 70 | def compile_node(self, node): 71 | """Compiles a node, recursively. 72 | 73 | This is one big switch on the node type. 74 | """ 75 | # XXX Optimize certain Wildcard-containing-Wildcard patterns 76 | # that can be merged 77 | if node.type == self.syms.Matcher: 78 | node = node.children[0] # Avoid unneeded recursion 79 | 80 | if node.type == self.syms.Alternatives: 81 | # Skip the odd children since they are just '|' tokens 82 | alts = [self.compile_node(ch) for ch in node.children[::2]] 83 | if len(alts) == 1: 84 | return alts[0] 85 | p = pytree.WildcardPattern([[a] for a in alts], min=1, max=1) 86 | return p.optimize() 87 | 88 | if node.type == self.syms.Alternative: 89 | units = [self.compile_node(ch) for ch in node.children] 90 | if len(units) == 1: 91 | return units[0] 92 | p = pytree.WildcardPattern([units], min=1, max=1) 93 | return p.optimize() 94 | 95 | if node.type == self.syms.NegatedUnit: 96 | pattern = self.compile_basic(node.children[1:]) 97 | p = pytree.NegatedPattern(pattern) 98 | return p.optimize() 99 | 100 | assert node.type == self.syms.Unit 101 | 102 | name = None 103 | nodes = node.children 104 | if len(nodes) >= 3 and nodes[1].type == token.EQUAL: 105 | name = nodes[0].value 106 | nodes = nodes[2:] 107 | repeat = None 108 | if len(nodes) >= 2 and nodes[-1].type == self.syms.Repeater: 109 | repeat = nodes[-1] 110 | nodes = nodes[:-1] 111 | 112 | # Now we've reduced it to: STRING | NAME [Details] | (...) | [...] 113 | pattern = self.compile_basic(nodes, repeat) 114 | 115 | if repeat is not None: 116 | assert repeat.type == self.syms.Repeater 117 | children = repeat.children 118 | child = children[0] 119 | if child.type == token.STAR: 120 | min = 0 121 | max = pytree.HUGE 122 | elif child.type == token.PLUS: 123 | min = 1 124 | max = pytree.HUGE 125 | elif child.type == token.LBRACE: 126 | assert children[-1].type == token.RBRACE 127 | assert len(children) in (3, 5) 128 | min = max = self.get_int(children[1]) 129 | if len(children) == 5: 130 | max = self.get_int(children[3]) 131 | else: 132 | assert False 133 | if min != 1 or max != 1: 134 | pattern = pattern.optimize() 135 | pattern = pytree.WildcardPattern([[pattern]], min=min, max=max) 136 | 137 | if name is not None: 138 | pattern.name = name 139 | return pattern.optimize() 140 | 141 | def compile_basic(self, nodes, repeat=None): 142 | # Compile STRING | NAME [Details] | (...) | [...] 143 | assert len(nodes) >= 1 144 | node = nodes[0] 145 | if node.type == token.STRING: 146 | value = str(literals.evalString(node.value)) 147 | return pytree.LeafPattern(_type_of_literal(value), value) 148 | elif node.type == token.NAME: 149 | value = node.value 150 | if value.isupper(): 151 | if value not in TOKEN_MAP: 152 | raise PatternSyntaxError('Invalid token: %r' % value) 153 | if nodes[1:]: 154 | raise PatternSyntaxError("Can't have details for token") 155 | return pytree.LeafPattern(TOKEN_MAP[value]) 156 | else: 157 | if value == 'any': 158 | type = None 159 | elif not value.startswith('_'): 160 | type = getattr(self.pysyms, value, None) 161 | if type is None: 162 | raise PatternSyntaxError('Invalid symbol: %r' % value) 163 | if nodes[1:]: # Details present 164 | content = [self.compile_node(nodes[1].children[1])] 165 | else: 166 | content = None 167 | return pytree.NodePattern(type, content) 168 | elif node.value == '(': 169 | return self.compile_node(nodes[1]) 170 | elif node.value == '[': 171 | assert repeat is None 172 | subpattern = self.compile_node(nodes[1]) 173 | return pytree.WildcardPattern([[subpattern]], min=0, max=1) 174 | assert False, node 175 | 176 | def get_int(self, node): 177 | assert node.type == token.NUMBER 178 | return int(node.value) 179 | 180 | 181 | # Map named tokens to the type value for a LeafPattern 182 | TOKEN_MAP = { 183 | 'NAME': token.NAME, 184 | 'STRING': token.STRING, 185 | 'NUMBER': token.NUMBER, 186 | 'TOKEN': None 187 | } 188 | 189 | 190 | def _type_of_literal(value): 191 | if value[0].isalpha(): 192 | return token.NAME 193 | elif value in grammar.opmap: 194 | return grammar.opmap[value] 195 | else: 196 | return None 197 | 198 | 199 | def pattern_convert(grammar, raw_node_info): 200 | """Converts raw node information to a Node or Leaf instance.""" 201 | type, value, context, children = raw_node_info 202 | if children or type in grammar.number2symbol: 203 | return pytree.Node(type, children, context=context) 204 | else: 205 | return pytree.Leaf(type, value, context=context) 206 | 207 | 208 | def compile_pattern(pattern): 209 | return PatternCompiler().compile_pattern(pattern) 210 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/pgen2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | """The pgen2 package.""" 4 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/pgen2/conv.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | """Convert graminit.[ch] spit out by pgen to Python code. 4 | 5 | Pgen is the Python parser generator. It is useful to quickly create a 6 | parser from a grammar file in Python's grammar notation. But I don't 7 | want my parsers to be written in C (yet), so I'm translating the 8 | parsing tables to Python data structures and writing a Python parse 9 | engine. 10 | 11 | Note that the token numbers are constants determined by the standard 12 | Python tokenizer. The standard token module defines these numbers and 13 | their names (the names are not used much). The token numbers are 14 | hardcoded into the Python tokenizer and into pgen. A Python 15 | implementation of the Python tokenizer is also available, in the 16 | standard tokenize module. 17 | 18 | On the other hand, symbol numbers (representing the grammar's 19 | non-terminals) are assigned by pgen based on the actual grammar 20 | input. 21 | 22 | Note: this module is pretty much obsolete; the pgen module generates 23 | equivalent grammar tables directly from the Grammar.txt input file 24 | without having to invoke the Python pgen C program. 25 | 26 | """ 27 | 28 | # Python imports 29 | import re 30 | 31 | # Local imports 32 | from pgen2 import grammar 33 | from pgen2 import token 34 | 35 | 36 | class Converter(grammar.Grammar): 37 | """Grammar subclass that reads classic pgen output files. 38 | 39 | The run() method reads the tables as produced by the pgen parser 40 | generator, typically contained in two C files, graminit.h and 41 | graminit.c. The other methods are for internal use only. 42 | 43 | See the base class for more documentation. 44 | 45 | """ 46 | 47 | def run(self, graminit_h, graminit_c): 48 | """Load the grammar tables from the text files written by pgen.""" 49 | self.parse_graminit_h(graminit_h) 50 | self.parse_graminit_c(graminit_c) 51 | self.finish_off() 52 | 53 | def parse_graminit_h(self, filename): 54 | """Parse the .h file written by pgen. (Internal) 55 | 56 | This file is a sequence of #define statements defining the 57 | nonterminals of the grammar as numbers. We build two tables 58 | mapping the numbers to names and back. 59 | 60 | """ 61 | try: 62 | f = open(filename) 63 | except OSError as err: 64 | print("Can't open %s: %s" % (filename, err)) 65 | return False 66 | self.symbol2number = {} 67 | self.number2symbol = {} 68 | lineno = 0 69 | for line in f: 70 | lineno += 1 71 | mo = re.match(r'^#define\s+(\w+)\s+(\d+)$', line) 72 | if not mo and line.strip(): 73 | print("%s(%s): can't parse %s" % (filename, lineno, line.strip())) 74 | else: 75 | symbol, number = mo.groups() 76 | number = int(number) 77 | assert symbol not in self.symbol2number 78 | assert number not in self.number2symbol 79 | self.symbol2number[symbol] = number 80 | self.number2symbol[number] = symbol 81 | return True 82 | 83 | def parse_graminit_c(self, filename): 84 | """Parse the .c file written by pgen. (Internal) 85 | 86 | The file looks as follows. The first two lines are always this: 87 | 88 | #include "pgenheaders.h" 89 | #include "grammar.h" 90 | 91 | After that come four blocks: 92 | 93 | 1) one or more state definitions 94 | 2) a table defining dfas 95 | 3) a table defining labels 96 | 4) a struct defining the grammar 97 | 98 | A state definition has the following form: 99 | - one or more arc arrays, each of the form: 100 | static arc arcs__[] = { 101 | {, }, 102 | ... 103 | }; 104 | - followed by a state array, of the form: 105 | static state states_[] = { 106 | {, arcs__}, 107 | ... 108 | }; 109 | 110 | """ 111 | try: 112 | f = open(filename) 113 | except OSError as err: 114 | print("Can't open %s: %s" % (filename, err)) 115 | return False 116 | # The code below essentially uses f's iterator-ness! 117 | lineno = 0 118 | 119 | # Expect the two #include lines 120 | lineno, line = lineno + 1, next(f) 121 | assert line == '#include "pgenheaders.h"\n', (lineno, line) 122 | lineno, line = lineno + 1, next(f) 123 | assert line == '#include "grammar.h"\n', (lineno, line) 124 | 125 | # Parse the state definitions 126 | lineno, line = lineno + 1, next(f) 127 | allarcs = {} 128 | states = [] 129 | while line.startswith('static arc '): 130 | while line.startswith('static arc '): 131 | mo = re.match(r'static arc arcs_(\d+)_(\d+)\[(\d+)\] = {$', line) 132 | assert mo, (lineno, line) 133 | n, m, k = list(map(int, mo.groups())) 134 | arcs = [] 135 | for _ in range(k): 136 | lineno, line = lineno + 1, next(f) 137 | mo = re.match(r'\s+{(\d+), (\d+)},$', line) 138 | assert mo, (lineno, line) 139 | i, j = list(map(int, mo.groups())) 140 | arcs.append((i, j)) 141 | lineno, line = lineno + 1, next(f) 142 | assert line == '};\n', (lineno, line) 143 | allarcs[(n, m)] = arcs 144 | lineno, line = lineno + 1, next(f) 145 | mo = re.match(r'static state states_(\d+)\[(\d+)\] = {$', line) 146 | assert mo, (lineno, line) 147 | s, t = list(map(int, mo.groups())) 148 | assert s == len(states), (lineno, line) 149 | state = [] 150 | for _ in range(t): 151 | lineno, line = lineno + 1, next(f) 152 | mo = re.match(r'\s+{(\d+), arcs_(\d+)_(\d+)},$', line) 153 | assert mo, (lineno, line) 154 | k, n, m = list(map(int, mo.groups())) 155 | arcs = allarcs[n, m] 156 | assert k == len(arcs), (lineno, line) 157 | state.append(arcs) 158 | states.append(state) 159 | lineno, line = lineno + 1, next(f) 160 | assert line == '};\n', (lineno, line) 161 | lineno, line = lineno + 1, next(f) 162 | self.states = states 163 | 164 | # Parse the dfas 165 | dfas = {} 166 | mo = re.match(r'static dfa dfas\[(\d+)\] = {$', line) 167 | assert mo, (lineno, line) 168 | ndfas = int(mo.group(1)) 169 | for i in range(ndfas): 170 | lineno, line = lineno + 1, next(f) 171 | mo = re.match(r'\s+{(\d+), "(\w+)", (\d+), (\d+), states_(\d+),$', line) 172 | assert mo, (lineno, line) 173 | symbol = mo.group(2) 174 | number, x, y, z = list(map(int, mo.group(1, 3, 4, 5))) 175 | assert self.symbol2number[symbol] == number, (lineno, line) 176 | assert self.number2symbol[number] == symbol, (lineno, line) 177 | assert x == 0, (lineno, line) 178 | state = states[z] 179 | assert y == len(state), (lineno, line) 180 | lineno, line = lineno + 1, next(f) 181 | mo = re.match(r'\s+("(?:\\\d\d\d)*")},$', line) 182 | assert mo, (lineno, line) 183 | first = {} 184 | rawbitset = eval(mo.group(1)) 185 | for i, c in enumerate(rawbitset): 186 | byte = ord(c) 187 | for j in range(8): 188 | if byte & (1 << j): 189 | first[i * 8 + j] = 1 190 | dfas[number] = (state, first) 191 | lineno, line = lineno + 1, next(f) 192 | assert line == '};\n', (lineno, line) 193 | self.dfas = dfas 194 | 195 | # Parse the labels 196 | labels = [] 197 | lineno, line = lineno + 1, next(f) 198 | mo = re.match(r'static label labels\[(\d+)\] = {$', line) 199 | assert mo, (lineno, line) 200 | nlabels = int(mo.group(1)) 201 | for i in range(nlabels): 202 | lineno, line = lineno + 1, next(f) 203 | mo = re.match(r'\s+{(\d+), (0|"\w+")},$', line) 204 | assert mo, (lineno, line) 205 | x, y = mo.groups() 206 | x = int(x) 207 | if y == '0': 208 | y = None 209 | else: 210 | y = eval(y) 211 | labels.append((x, y)) 212 | lineno, line = lineno + 1, next(f) 213 | assert line == '};\n', (lineno, line) 214 | self.labels = labels 215 | 216 | # Parse the grammar struct 217 | lineno, line = lineno + 1, next(f) 218 | assert line == 'grammar _PyParser_Grammar = {\n', (lineno, line) 219 | lineno, line = lineno + 1, next(f) 220 | mo = re.match(r'\s+(\d+),$', line) 221 | assert mo, (lineno, line) 222 | ndfas = int(mo.group(1)) 223 | assert ndfas == len(self.dfas) 224 | lineno, line = lineno + 1, next(f) 225 | assert line == '\tdfas,\n', (lineno, line) 226 | lineno, line = lineno + 1, next(f) 227 | mo = re.match(r'\s+{(\d+), labels},$', line) 228 | assert mo, (lineno, line) 229 | nlabels = int(mo.group(1)) 230 | assert nlabels == len(self.labels), (lineno, line) 231 | lineno, line = lineno + 1, next(f) 232 | mo = re.match(r'\s+(\d+)$', line) 233 | assert mo, (lineno, line) 234 | start = int(mo.group(1)) 235 | assert start in self.number2symbol, (lineno, line) 236 | self.start = start 237 | lineno, line = lineno + 1, next(f) 238 | assert line == '};\n', (lineno, line) 239 | try: 240 | lineno, line = lineno + 1, next(f) 241 | except StopIteration: 242 | pass 243 | else: 244 | assert 0, (lineno, line) 245 | 246 | def finish_off(self): 247 | """Create additional useful structures. (Internal).""" 248 | self.keywords = {} # map from keyword strings to arc labels 249 | self.tokens = {} # map from numeric token values to arc labels 250 | for ilabel, (type, value) in enumerate(self.labels): 251 | if type == token.NAME and value is not None: 252 | self.keywords[value] = ilabel 253 | elif value is None: 254 | self.tokens[type] = ilabel 255 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/pgen2/driver.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | 4 | # Modifications: 5 | # Copyright 2006 Google, Inc. All Rights Reserved. 6 | # Licensed to PSF under a Contributor Agreement. 7 | """Parser driver. 8 | 9 | This provides a high-level interface to parse a file into a syntax tree. 10 | 11 | """ 12 | 13 | __author__ = 'Guido van Rossum ' 14 | 15 | __all__ = ['Driver', 'load_grammar'] 16 | 17 | import io 18 | import logging 19 | import os 20 | import pkgutil 21 | import sys 22 | # Python imports 23 | from contextlib import contextmanager 24 | from dataclasses import dataclass 25 | from dataclasses import field 26 | from pathlib import Path 27 | from typing import Any 28 | from typing import Iterator 29 | from typing import List 30 | from typing import Optional 31 | 32 | from platformdirs import user_cache_dir 33 | 34 | from yapf._version import __version__ as yapf_version 35 | 36 | # Pgen imports 37 | from . import grammar 38 | from . import parse 39 | from . import pgen 40 | from . import token 41 | from . import tokenize 42 | 43 | 44 | @dataclass 45 | class ReleaseRange: 46 | start: int 47 | end: Optional[int] = None 48 | tokens: List[Any] = field(default_factory=list) 49 | 50 | def lock(self) -> None: 51 | total_eaten = len(self.tokens) 52 | self.end = self.start + total_eaten 53 | 54 | 55 | class TokenProxy: 56 | 57 | def __init__(self, generator: Any) -> None: 58 | self._tokens = generator 59 | self._counter = 0 60 | self._release_ranges: List[ReleaseRange] = [] 61 | 62 | @contextmanager 63 | def release(self) -> Iterator['TokenProxy']: 64 | release_range = ReleaseRange(self._counter) 65 | self._release_ranges.append(release_range) 66 | try: 67 | yield self 68 | finally: 69 | # Lock the last release range to the final position that 70 | # has been eaten. 71 | release_range.lock() 72 | 73 | def eat(self, point: int) -> Any: 74 | eaten_tokens = self._release_ranges[-1].tokens 75 | if point < len(eaten_tokens): 76 | return eaten_tokens[point] 77 | else: 78 | while point >= len(eaten_tokens): 79 | token = next(self._tokens) 80 | eaten_tokens.append(token) 81 | return token 82 | 83 | def __iter__(self) -> 'TokenProxy': 84 | return self 85 | 86 | def __next__(self) -> Any: 87 | # If the current position is already compromised (looked up) 88 | # return the eaten token, if not just go further on the given 89 | # token producer. 90 | for release_range in self._release_ranges: 91 | assert release_range.end is not None 92 | 93 | start, end = release_range.start, release_range.end 94 | if start <= self._counter < end: 95 | token = release_range.tokens[self._counter - start] 96 | break 97 | else: 98 | token = next(self._tokens) 99 | self._counter += 1 100 | return token 101 | 102 | def can_advance(self, to: int) -> bool: 103 | # Try to eat, fail if it can't. The eat operation is cached 104 | # so there wont be any additional cost of eating here 105 | try: 106 | self.eat(to) 107 | except StopIteration: 108 | return False 109 | else: 110 | return True 111 | 112 | 113 | class Driver(object): 114 | 115 | def __init__(self, grammar, convert=None, logger=None): 116 | self.grammar = grammar 117 | if logger is None: 118 | logger = logging.getLogger() 119 | self.logger = logger 120 | self.convert = convert 121 | 122 | def parse_tokens(self, tokens, debug=False): 123 | """Parse a series of tokens and return the syntax tree.""" 124 | # XXX Move the prefix computation into a wrapper around tokenize. 125 | p = parse.Parser(self.grammar, self.convert) 126 | proxy = TokenProxy(tokens) 127 | p.setup(proxy=proxy) 128 | lineno = 1 129 | column = 0 130 | type = value = start = end = line_text = None 131 | prefix = '' 132 | for quintuple in proxy: 133 | type, value, start, end, line_text = quintuple 134 | if start != (lineno, column): 135 | assert (lineno, column) <= start, ((lineno, column), start) 136 | s_lineno, s_column = start 137 | if lineno < s_lineno: 138 | prefix += '\n' * (s_lineno - lineno) 139 | lineno = s_lineno 140 | column = 0 141 | if column < s_column: 142 | prefix += line_text[column:s_column] 143 | column = s_column 144 | if type in (tokenize.COMMENT, tokenize.NL): 145 | prefix += value 146 | lineno, column = end 147 | if value.endswith('\n'): 148 | lineno += 1 149 | column = 0 150 | continue 151 | if type == token.OP: 152 | type = grammar.opmap[value] 153 | if debug: 154 | self.logger.debug('%s %r (prefix=%r)', token.tok_name[type], value, 155 | prefix) 156 | if p.addtoken(type, value, (prefix, start)): 157 | if debug: 158 | self.logger.debug('Stop.') 159 | break 160 | prefix = '' 161 | lineno, column = end 162 | if value.endswith('\n'): 163 | lineno += 1 164 | column = 0 165 | else: 166 | # We never broke out -- EOF is too soon (how can this happen???) 167 | raise parse.ParseError('incomplete input', type, value, (prefix, start)) 168 | return p.rootnode 169 | 170 | def parse_stream_raw(self, stream, debug=False): 171 | """Parse a stream and return the syntax tree.""" 172 | tokens = tokenize.generate_tokens(stream.readline) 173 | return self.parse_tokens(tokens, debug) 174 | 175 | def parse_stream(self, stream, debug=False): 176 | """Parse a stream and return the syntax tree.""" 177 | return self.parse_stream_raw(stream, debug) 178 | 179 | def parse_file(self, filename, encoding=None, debug=False): 180 | """Parse a file and return the syntax tree.""" 181 | with io.open(filename, 'r', encoding=encoding) as stream: 182 | return self.parse_stream(stream, debug) 183 | 184 | def parse_string(self, text, debug=False): 185 | """Parse a string and return the syntax tree.""" 186 | tokens = tokenize.generate_tokens(io.StringIO(text).readline) 187 | return self.parse_tokens(tokens, debug) 188 | 189 | 190 | def _generate_pickle_name(gt): 191 | # type:(str) -> str 192 | """Get the filepath to write a pickle file to 193 | given the path of a grammar textfile. 194 | 195 | The returned filepath should be in a user-specific cache directory. 196 | 197 | Args: 198 | gt (str): path to grammar text file 199 | 200 | Returns: 201 | str: path to pickle file 202 | """ 203 | 204 | grammar_textfile_name = os.path.basename(gt) 205 | head, tail = os.path.splitext(grammar_textfile_name) 206 | if tail == '.txt': 207 | tail = '' 208 | cache_dir = user_cache_dir( 209 | appname='YAPF', appauthor='Google', version=yapf_version) 210 | return cache_dir + os.sep + head + tail + '-py' + '.'.join( 211 | map(str, sys.version_info)) + '.pickle' 212 | 213 | 214 | def load_grammar(gt='Grammar.txt', 215 | gp=None, 216 | save=True, 217 | force=False, 218 | logger=None): 219 | # type:(str, str | None, bool, bool, logging.Logger | None) -> grammar.Grammar 220 | """Load the grammar (maybe from a pickle).""" 221 | if logger is None: 222 | logger = logging.getLogger() 223 | gp = _generate_pickle_name(gt) if gp is None else gp 224 | grammar_text = gt 225 | try: 226 | newer = _newer(gp, gt) 227 | except OSError as err: 228 | logger.debug('OSError, could not check if newer: %s', err.args) 229 | newer = True 230 | if not os.path.exists(gt): 231 | # Assume package data 232 | gt_basename = os.path.basename(gt) 233 | pd = pkgutil.get_data('yapf_third_party._ylib2to3', gt_basename) 234 | if pd is None: 235 | raise RuntimeError('Failed to load grammer %s from package' % gt_basename) 236 | grammar_text = io.StringIO(pd.decode(encoding='utf-8')) 237 | if force or not newer: 238 | g = pgen.generate_grammar(grammar_text) 239 | if save: 240 | try: 241 | Path(gp).parent.mkdir(parents=True, exist_ok=True) 242 | g.dump(gp) 243 | except OSError: 244 | # Ignore error, caching is not vital. 245 | pass 246 | else: 247 | g = grammar.Grammar() 248 | g.load(gp) 249 | return g 250 | 251 | 252 | def _newer(a, b): 253 | """Inquire whether file a was written since file b.""" 254 | if not os.path.exists(a): 255 | return False 256 | if not os.path.exists(b): 257 | return True 258 | return os.path.getmtime(a) >= os.path.getmtime(b) 259 | 260 | 261 | def load_packaged_grammar(package, grammar_source): 262 | """Normally, loads a pickled grammar by doing 263 | pkgutil.get_data(package, pickled_grammar) 264 | where *pickled_grammar* is computed from *grammar_source* by adding the 265 | Python version and using a ``.pickle`` extension. 266 | 267 | However, if *grammar_source* is an extant file, load_grammar(grammar_source) 268 | is called instead. This facilitates using a packaged grammar file when needed 269 | but preserves load_grammar's automatic regeneration behavior when possible. 270 | 271 | """ # noqa: E501 272 | if os.path.isfile(grammar_source): 273 | return load_grammar(grammar_source) 274 | pickled_name = _generate_pickle_name(os.path.basename(grammar_source)) 275 | data = pkgutil.get_data(package, pickled_name) 276 | g = grammar.Grammar() 277 | g.loads(data) 278 | return g 279 | 280 | 281 | def main(*args): 282 | """Main program, when run as a script: produce grammar pickle files. 283 | 284 | Calls load_grammar for each argument, a path to a grammar text file. 285 | """ 286 | if not args: 287 | args = sys.argv[1:] 288 | logging.basicConfig( 289 | level=logging.INFO, stream=sys.stdout, format='%(message)s') 290 | for gt in args: 291 | load_grammar(gt, save=True, force=True) 292 | return True 293 | 294 | 295 | if __name__ == '__main__': 296 | sys.exit(int(not main())) 297 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/pgen2/grammar.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | """This module defines the data structures used to represent a grammar. 4 | 5 | These are a bit arcane because they are derived from the data 6 | structures used by Python's 'pgen' parser generator. 7 | 8 | There's also a table here mapping operators to their names in the 9 | token module; the Python tokenize module reports all operators as the 10 | fallback token code OP, but the parser needs the actual token code. 11 | 12 | """ 13 | 14 | # Python imports 15 | import os 16 | import pickle 17 | import tempfile 18 | 19 | # Local imports 20 | from . import token 21 | 22 | 23 | class Grammar(object): 24 | """Pgen parsing tables conversion class. 25 | 26 | Once initialized, this class supplies the grammar tables for the 27 | parsing engine implemented by parse.py. The parsing engine 28 | accesses the instance variables directly. The class here does not 29 | provide initialization of the tables; several subclasses exist to 30 | do this (see the conv and pgen modules). 31 | 32 | The load() method reads the tables from a pickle file, which is 33 | much faster than the other ways offered by subclasses. The pickle 34 | file is written by calling dump() (after loading the grammar 35 | tables using a subclass). The report() method prints a readable 36 | representation of the tables to stdout, for debugging. 37 | 38 | The instance variables are as follows: 39 | 40 | symbol2number -- a dict mapping symbol names to numbers. Symbol 41 | numbers are always 256 or higher, to distinguish 42 | them from token numbers, which are between 0 and 43 | 255 (inclusive). 44 | 45 | number2symbol -- a dict mapping numbers to symbol names; 46 | these two are each other's inverse. 47 | 48 | states -- a list of DFAs, where each DFA is a list of 49 | states, each state is a list of arcs, and each 50 | arc is a (i, j) pair where i is a label and j is 51 | a state number. The DFA number is the index into 52 | this list. (This name is slightly confusing.) 53 | Final states are represented by a special arc of 54 | the form (0, j) where j is its own state number. 55 | 56 | dfas -- a dict mapping symbol numbers to (DFA, first) 57 | pairs, where DFA is an item from the states list 58 | above, and first is a set of tokens that can 59 | begin this grammar rule (represented by a dict 60 | whose values are always 1). 61 | 62 | labels -- a list of (x, y) pairs where x is either a token 63 | number or a symbol number, and y is either None 64 | or a string; the strings are keywords. The label 65 | number is the index in this list; label numbers 66 | are used to mark state transitions (arcs) in the 67 | DFAs. 68 | 69 | start -- the number of the grammar's start symbol. 70 | 71 | keywords -- a dict mapping keyword strings to arc labels. 72 | 73 | tokens -- a dict mapping token numbers to arc labels. 74 | 75 | """ 76 | 77 | def __init__(self): 78 | self.symbol2number = {} 79 | self.number2symbol = {} 80 | self.states = [] 81 | self.dfas = {} 82 | self.labels = [(0, 'EMPTY')] 83 | self.keywords = {} 84 | self.soft_keywords = {} 85 | self.tokens = {} 86 | self.symbol2label = {} 87 | self.start = 256 88 | 89 | def dump(self, filename): 90 | """Dump the grammar tables to a pickle file.""" 91 | # NOTE: 92 | # - We're writing a tempfile first so that there is no chance 93 | # for someone to read a half-written file from this very spot 94 | # while we're were not done writing. 95 | # - We're using ``os.rename`` to sure not copy data around (which 96 | # would get us back to square one with a reading-half-written file 97 | # race condition). 98 | # - We're making the tempfile go to the same directory as the eventual 99 | # target ``filename`` so that there is no chance of failing from 100 | # cross-file-system renames in ``os.rename``. 101 | # - We're using the same prefix and suffix for the tempfile so if we 102 | # ever have to leave a tempfile around for failure of deletion, 103 | # it will have a reasonable filename extension and its name will help 104 | # explain is nature. 105 | tempfile_dir = os.path.dirname(filename) 106 | tempfile_prefix, tempfile_suffix = os.path.splitext(filename) 107 | with tempfile.NamedTemporaryFile( 108 | mode='wb', 109 | suffix=tempfile_suffix, 110 | prefix=tempfile_prefix, 111 | dir=tempfile_dir, 112 | delete=False) as f: 113 | pickle.dump(self.__dict__, f.file, pickle.HIGHEST_PROTOCOL) 114 | try: 115 | os.rename(f.name, filename) 116 | except OSError: 117 | # This makes sure that we do not leave the tempfile around 118 | # unless we have to... 119 | try: 120 | os.remove(f.name) 121 | except OSError: 122 | pass 123 | raise 124 | 125 | def load(self, filename): 126 | """Load the grammar tables from a pickle file.""" 127 | with open(filename, 'rb') as f: 128 | d = pickle.load(f) 129 | self.__dict__.update(d) 130 | 131 | def loads(self, pkl): 132 | """Load the grammar tables from a pickle bytes object.""" 133 | self.__dict__.update(pickle.loads(pkl)) 134 | 135 | def copy(self): 136 | """ 137 | Copy the grammar. 138 | """ 139 | new = self.__class__() 140 | for dict_attr in ('symbol2number', 'number2symbol', 'dfas', 'keywords', 141 | 'soft_keywords', 'tokens', 'symbol2label'): 142 | setattr(new, dict_attr, getattr(self, dict_attr).copy()) 143 | new.labels = self.labels[:] 144 | new.states = self.states[:] 145 | new.start = self.start 146 | return new 147 | 148 | def report(self): 149 | """Dump the grammar tables to standard output, for debugging.""" 150 | from pprint import pprint 151 | print('s2n') 152 | pprint(self.symbol2number) 153 | print('n2s') 154 | pprint(self.number2symbol) 155 | print('states') 156 | pprint(self.states) 157 | print('dfas') 158 | pprint(self.dfas) 159 | print('labels') 160 | pprint(self.labels) 161 | print('start', self.start) 162 | 163 | 164 | # Map from operator to number (since tokenize doesn't do this) 165 | 166 | opmap_raw = """ 167 | ( LPAR 168 | ) RPAR 169 | [ LSQB 170 | ] RSQB 171 | : COLON 172 | , COMMA 173 | ; SEMI 174 | + PLUS 175 | - MINUS 176 | * STAR 177 | / SLASH 178 | | VBAR 179 | & AMPER 180 | < LESS 181 | > GREATER 182 | = EQUAL 183 | . DOT 184 | % PERCENT 185 | ` BACKQUOTE 186 | { LBRACE 187 | } RBRACE 188 | @ AT 189 | @= ATEQUAL 190 | == EQEQUAL 191 | != NOTEQUAL 192 | <> NOTEQUAL 193 | <= LESSEQUAL 194 | >= GREATEREQUAL 195 | ~ TILDE 196 | ^ CIRCUMFLEX 197 | << LEFTSHIFT 198 | >> RIGHTSHIFT 199 | ** DOUBLESTAR 200 | += PLUSEQUAL 201 | -= MINEQUAL 202 | *= STAREQUAL 203 | /= SLASHEQUAL 204 | %= PERCENTEQUAL 205 | &= AMPEREQUAL 206 | |= VBAREQUAL 207 | ^= CIRCUMFLEXEQUAL 208 | <<= LEFTSHIFTEQUAL 209 | >>= RIGHTSHIFTEQUAL 210 | **= DOUBLESTAREQUAL 211 | // DOUBLESLASH 212 | //= DOUBLESLASHEQUAL 213 | -> RARROW 214 | := COLONEQUAL 215 | """ 216 | 217 | opmap = {} 218 | for line in opmap_raw.splitlines(): 219 | if line: 220 | op, name = line.split() 221 | opmap[op] = getattr(token, name) 222 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/pgen2/literals.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | """Safely evaluate Python string literals without using eval().""" 4 | 5 | import re 6 | 7 | simple_escapes = { 8 | 'a': '\a', 9 | 'b': '\b', 10 | 'f': '\f', 11 | 'n': '\n', 12 | 'r': '\r', 13 | 't': '\t', 14 | 'v': '\v', 15 | "'": "'", 16 | '"': '"', 17 | '\\': '\\' 18 | } 19 | 20 | 21 | def escape(m): 22 | all, tail = m.group(0, 1) 23 | assert all.startswith('\\') 24 | esc = simple_escapes.get(tail) 25 | if esc is not None: 26 | return esc 27 | if tail.startswith('x'): 28 | hexes = tail[1:] 29 | if len(hexes) < 2: 30 | raise ValueError("invalid hex string escape ('\\%s')" % tail) 31 | try: 32 | i = int(hexes, 16) 33 | except ValueError: 34 | raise ValueError("invalid hex string escape ('\\%s')" % tail) from None 35 | else: 36 | try: 37 | i = int(tail, 8) 38 | except ValueError: 39 | raise ValueError("invalid octal string escape ('\\%s')" % tail) from None 40 | return chr(i) 41 | 42 | 43 | def evalString(s): 44 | assert s.startswith("'") or s.startswith('"'), repr(s[:1]) 45 | q = s[0] 46 | if s[:3] == q * 3: 47 | q = q * 3 48 | assert s.endswith(q), repr(s[-len(q):]) 49 | assert len(s) >= 2 * len(q) 50 | s = s[len(q):-len(q)] 51 | return re.sub(r"\\(\'|\"|\\|[abfnrtv]|x.{0,2}|[0-7]{1,3})", escape, s) 52 | 53 | 54 | def test(): 55 | for i in range(256): 56 | c = chr(i) 57 | s = repr(c) 58 | e = evalString(s) 59 | if e != c: 60 | print(i, c, s, e) 61 | 62 | 63 | if __name__ == '__main__': 64 | test() 65 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/pgen2/token.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | """Token constants (from "token.h").""" 3 | 4 | # Taken from Python (r53757) and modified to include some tokens 5 | # originally monkeypatched in by pgen2.tokenize 6 | 7 | # --start constants-- 8 | ENDMARKER = 0 9 | NAME = 1 10 | NUMBER = 2 11 | STRING = 3 12 | NEWLINE = 4 13 | INDENT = 5 14 | DEDENT = 6 15 | LPAR = 7 16 | RPAR = 8 17 | LSQB = 9 18 | RSQB = 10 19 | COLON = 11 20 | COMMA = 12 21 | SEMI = 13 22 | PLUS = 14 23 | MINUS = 15 24 | STAR = 16 25 | SLASH = 17 26 | VBAR = 18 27 | AMPER = 19 28 | LESS = 20 29 | GREATER = 21 30 | EQUAL = 22 31 | DOT = 23 32 | PERCENT = 24 33 | BACKQUOTE = 25 34 | LBRACE = 26 35 | RBRACE = 27 36 | EQEQUAL = 28 37 | NOTEQUAL = 29 38 | LESSEQUAL = 30 39 | GREATEREQUAL = 31 40 | TILDE = 32 41 | CIRCUMFLEX = 33 42 | LEFTSHIFT = 34 43 | RIGHTSHIFT = 35 44 | DOUBLESTAR = 36 45 | PLUSEQUAL = 37 46 | MINEQUAL = 38 47 | STAREQUAL = 39 48 | SLASHEQUAL = 40 49 | PERCENTEQUAL = 41 50 | AMPEREQUAL = 42 51 | VBAREQUAL = 43 52 | CIRCUMFLEXEQUAL = 44 53 | LEFTSHIFTEQUAL = 45 54 | RIGHTSHIFTEQUAL = 46 55 | DOUBLESTAREQUAL = 47 56 | DOUBLESLASH = 48 57 | DOUBLESLASHEQUAL = 49 58 | AT = 50 59 | ATEQUAL = 51 60 | OP = 52 61 | COMMENT = 53 62 | NL = 54 63 | RARROW = 55 64 | AWAIT = 56 65 | ASYNC = 57 66 | ERRORTOKEN = 58 67 | COLONEQUAL = 59 68 | N_TOKENS = 60 69 | NT_OFFSET = 256 70 | # --end constants-- 71 | 72 | tok_name = { 73 | _value: _name 74 | for _name, _value in globals().copy().items() 75 | if isinstance(_value, int) 76 | } 77 | 78 | 79 | def ISTERMINAL(x): 80 | return x < NT_OFFSET 81 | 82 | 83 | def ISNONTERMINAL(x): 84 | return x >= NT_OFFSET 85 | 86 | 87 | def ISEOF(x): 88 | return x == ENDMARKER 89 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/_ylib2to3/pygram.py: -------------------------------------------------------------------------------- 1 | # Copyright 2006 Google, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | """Export the Python grammar and symbols.""" 4 | 5 | # Python imports 6 | import os 7 | 8 | # Local imports 9 | from .pgen2 import driver 10 | 11 | # The grammar file 12 | _GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), 'Grammar.txt') 13 | _PATTERN_GRAMMAR_FILE = os.path.join( 14 | os.path.dirname(__file__), 'PatternGrammar.txt') 15 | 16 | 17 | class Symbols(object): 18 | 19 | def __init__(self, grammar): 20 | """Initializer. 21 | 22 | Creates an attribute for each grammar symbol (nonterminal), 23 | whose value is the symbol's type (an int >= 256). 24 | """ 25 | for name, symbol in grammar.symbol2number.items(): 26 | setattr(self, name, symbol) 27 | 28 | 29 | python_grammar = driver.load_grammar(_GRAMMAR_FILE) 30 | 31 | python_symbols = Symbols(python_grammar) 32 | 33 | python_grammar_no_print_statement = python_grammar.copy() 34 | del python_grammar_no_print_statement.keywords['print'] 35 | 36 | python_grammar_no_print_and_exec_statement = python_grammar_no_print_statement.copy() # yapf: disable # noqa: E501 37 | del python_grammar_no_print_and_exec_statement.keywords['exec'] 38 | 39 | pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE) 40 | pattern_symbols = Symbols(pattern_grammar) 41 | -------------------------------------------------------------------------------- /third_party/yapf_third_party/yapf_diff/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/yapf/12005095296072751e3e4c1f33a047d41b0ce18d/third_party/yapf_third_party/yapf_diff/__init__.py -------------------------------------------------------------------------------- /third_party/yapf_third_party/yapf_diff/yapf_diff.py: -------------------------------------------------------------------------------- 1 | # Modified copy of clang-format-diff.py that works with yapf. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License") with LLVM 4 | # Exceptions; you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://llvm.org/LICENSE.txt 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """ 15 | This script reads input from a unified diff and reformats all the changed 16 | lines. This is useful to reformat all the lines touched by a specific patch. 17 | Example usage for git/svn users: 18 | 19 | git diff -U0 --no-color --relative HEAD^ | yapf-diff -i 20 | svn diff --diff-cmd=diff -x-U0 | yapf-diff -p0 -i 21 | 22 | It should be noted that the filename contained in the diff is used unmodified 23 | to determine the source file to update. Users calling this script directly 24 | should be careful to ensure that the path in the diff is correct relative to the 25 | current working directory. 26 | """ 27 | 28 | import argparse 29 | import difflib 30 | import re 31 | import subprocess 32 | import sys 33 | from io import StringIO 34 | 35 | 36 | def main(): 37 | parser = argparse.ArgumentParser( 38 | description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) 39 | parser.add_argument( 40 | '-i', 41 | '--in-place', 42 | action='store_true', 43 | default=False, 44 | help='apply edits to files instead of displaying a diff') 45 | parser.add_argument( 46 | '-p', 47 | '--prefix', 48 | metavar='NUM', 49 | default=1, 50 | help='strip the smallest prefix containing P slashes') 51 | parser.add_argument( 52 | '--regex', 53 | metavar='PATTERN', 54 | default=None, 55 | help='custom pattern selecting file paths to reformat ' 56 | '(case sensitive, overrides -iregex)') 57 | parser.add_argument( 58 | '--iregex', 59 | metavar='PATTERN', 60 | default=r'.*\.(py)', 61 | help='custom pattern selecting file paths to reformat ' 62 | '(case insensitive, overridden by -regex)') 63 | parser.add_argument( 64 | '-v', 65 | '--verbose', 66 | action='store_true', 67 | help='be more verbose, ineffective without -i') 68 | parser.add_argument( 69 | '--style', 70 | help='specify formatting style: either a style name (for ' 71 | 'example "pep8" or "google"), or the name of a file with ' 72 | 'style settings. The default is pep8 unless a ' 73 | '.style.yapf or setup.cfg file located in one of the ' 74 | 'parent directories of the source file (or current ' 75 | 'directory for stdin)') 76 | parser.add_argument( 77 | '--binary', default='yapf', help='location of binary to use for yapf') 78 | args = parser.parse_args() 79 | 80 | # Extract changed lines for each file. 81 | filename = None 82 | lines_by_file = {} 83 | for line in sys.stdin: 84 | match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.prefix, line) 85 | if match: 86 | filename = match.group(2) 87 | if filename is None: 88 | continue 89 | 90 | if args.regex is not None: 91 | if not re.match('^%s$' % args.regex, filename): 92 | continue 93 | elif not re.match('^%s$' % args.iregex, filename, re.IGNORECASE): 94 | continue 95 | 96 | match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line) 97 | if match: 98 | start_line = int(match.group(1)) 99 | line_count = 1 100 | if match.group(3): 101 | line_count = int(match.group(3)) 102 | if line_count == 0: 103 | continue 104 | end_line = start_line + line_count - 1 105 | lines_by_file.setdefault(filename, []).extend( 106 | ['--lines', str(start_line) + '-' + str(end_line)]) 107 | 108 | # Reformat files containing changes in place. 109 | for filename, lines in lines_by_file.items(): 110 | if args.in_place and args.verbose: 111 | print('Formatting {}'.format(filename)) 112 | command = [args.binary, filename] 113 | if args.in_place: 114 | command.append('-i') 115 | command.extend(lines) 116 | if args.style: 117 | command.extend(['--style', args.style]) 118 | p = subprocess.Popen( 119 | command, 120 | stdout=subprocess.PIPE, 121 | stderr=None, 122 | stdin=subprocess.PIPE, 123 | universal_newlines=True) 124 | stdout, stderr = p.communicate() 125 | if p.returncode != 0: 126 | sys.exit(p.returncode) 127 | 128 | if not args.in_place: 129 | with open(filename) as f: 130 | code = f.readlines() 131 | formatted_code = StringIO(stdout).readlines() 132 | diff = difflib.unified_diff(code, formatted_code, filename, filename, 133 | '(before formatting)', '(after formatting)') 134 | diff_string = ''.join(diff) 135 | if len(diff_string) > 0: 136 | sys.stdout.write(diff_string) 137 | 138 | 139 | if __name__ == '__main__': 140 | main() 141 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | requires = 3 | tox<4 4 | tox-pyenv 5 | tox-wheel 6 | envlist = py37,py38,py39,py310,py311,py312 7 | # tox-wheel alias for `wheel_pep517 = true` 8 | isolated_build = True 9 | distshare = ./dist 10 | 11 | [testenv] 12 | wheel = True 13 | wheel_build_env = bdist_wheel 14 | commands = python -m unittest discover -p '*_test.py' yapftests/ 15 | 16 | [testenv:bdist_wheel] 17 | 18 | [testenv:sdist] 19 | wheel = False 20 | -------------------------------------------------------------------------------- /yapf/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Main entry point.""" 15 | # pylint: disable=invalid-name 16 | import yapf 17 | 18 | yapf.run_main() 19 | -------------------------------------------------------------------------------- /yapf/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.43.0' 2 | -------------------------------------------------------------------------------- /yapf/pyparser/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /yapf/pyparser/pyparser.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Bill Wendling, All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Simple Python Parser 15 | 16 | Parse Python code into a list of logical lines, represented by LogicalLine 17 | objects. This uses Python's tokenizer to generate the tokens. As such, YAPF must 18 | be run with the appropriate Python version---Python >=3.7 for Python 3.7 code, 19 | Python >=3.8 for Python 3.8 code, etc. 20 | 21 | This parser uses Python's native "tokenizer" module to generate a list of tokens 22 | for the source code. It then uses Python's native "ast" module to assign 23 | subtypes, calculate split penalties, etc. 24 | 25 | A "logical line" produced by Python's "tokenizer" module ends with a 26 | tokenize.NEWLINE, rather than a tokenize.NL, making it easy to separate them 27 | out. Comments all end with a tokentizer.NL, so we need to make sure we don't 28 | errantly pick up non-comment tokens when parsing comment blocks. 29 | 30 | ParseCode(): parse the code producing a list of logical lines. 31 | """ 32 | 33 | # TODO: Call from yapf_api.FormatCode. 34 | 35 | import ast 36 | import codecs 37 | import os 38 | import token 39 | import tokenize 40 | from io import StringIO 41 | from tokenize import TokenInfo 42 | 43 | from yapf.pyparser import split_penalty_visitor 44 | from yapf.yapflib import format_token 45 | from yapf.yapflib import logical_line 46 | 47 | CONTINUATION = token.N_TOKENS 48 | 49 | 50 | def ParseCode(unformatted_source, filename=''): 51 | """Parse a string of Python code into logical lines. 52 | 53 | This provides an alternative entry point to YAPF. 54 | 55 | Arguments: 56 | unformatted_source: (unicode) The code to format. 57 | filename: (unicode) The name of the file being reformatted. 58 | 59 | Returns: 60 | A list of LogicalLines. 61 | 62 | Raises: 63 | An exception is raised if there's an error during AST parsing. 64 | """ 65 | if not unformatted_source.endswith(os.linesep): 66 | unformatted_source += os.linesep 67 | 68 | try: 69 | ast_tree = ast.parse(unformatted_source, filename) 70 | ast.fix_missing_locations(ast_tree) 71 | readline = StringIO(unformatted_source).readline 72 | tokens = tokenize.generate_tokens(readline) 73 | except Exception: 74 | raise 75 | 76 | logical_lines = _CreateLogicalLines(tokens) 77 | 78 | # Process the logical lines. 79 | split_penalty_visitor.SplitPenalty(logical_lines).visit(ast_tree) 80 | 81 | return logical_lines 82 | 83 | 84 | def _CreateLogicalLines(tokens): 85 | """Separate tokens into logical lines. 86 | 87 | Arguments: 88 | tokens: (list of tokenizer.TokenInfo) Tokens generated by tokenizer. 89 | 90 | Returns: 91 | A list of LogicalLines. 92 | """ 93 | formatted_tokens = [] 94 | 95 | # Convert tokens into "TokenInfo" and add tokens for continuation markers. 96 | prev_tok = None 97 | for tok in tokens: 98 | tok = TokenInfo(*tok) 99 | 100 | if (prev_tok and prev_tok.line.rstrip().endswith('\\') and 101 | prev_tok.start[0] < tok.start[0]): 102 | ctok = TokenInfo( 103 | type=CONTINUATION, 104 | string='\\', 105 | start=(prev_tok.start[0], prev_tok.start[1] + 1), 106 | end=(prev_tok.end[0], prev_tok.end[0] + 2), 107 | line=prev_tok.line) 108 | ctok.lineno = ctok.start[0] 109 | ctok.column = ctok.start[1] 110 | ctok.value = '\\' 111 | formatted_tokens.append(format_token.FormatToken(ctok, 'CONTINUATION')) 112 | 113 | tok.lineno = tok.start[0] 114 | tok.column = tok.start[1] 115 | tok.value = tok.string 116 | formatted_tokens.append( 117 | format_token.FormatToken(tok, token.tok_name[tok.type])) 118 | prev_tok = tok 119 | 120 | # Generate logical lines. 121 | logical_lines, cur_logical_line = [], [] 122 | depth = 0 123 | for tok in formatted_tokens: 124 | if tok.type == tokenize.ENDMARKER: 125 | break 126 | 127 | if tok.type == tokenize.NEWLINE: 128 | # End of a logical line. 129 | logical_lines.append(logical_line.LogicalLine(depth, cur_logical_line)) 130 | cur_logical_line = [] 131 | elif tok.type == tokenize.INDENT: 132 | depth += 1 133 | elif tok.type == tokenize.DEDENT: 134 | depth -= 1 135 | elif tok.type == tokenize.NL: 136 | pass 137 | else: 138 | if (cur_logical_line and not tok.type == tokenize.COMMENT and 139 | cur_logical_line[0].type == tokenize.COMMENT): 140 | # We were parsing a comment block, but now we have real code to worry 141 | # about. Store the comment and carry on. 142 | logical_lines.append(logical_line.LogicalLine(depth, cur_logical_line)) 143 | cur_logical_line = [] 144 | 145 | cur_logical_line.append(tok) 146 | 147 | # Link the FormatTokens in each line together to form a doubly linked list. 148 | for line in logical_lines: 149 | previous = line.first 150 | bracket_stack = [previous] if previous.OpensScope() else [] 151 | for tok in line.tokens[1:]: 152 | tok.previous_token = previous 153 | previous.next_token = tok 154 | previous = tok 155 | 156 | # Set up the "matching_bracket" attribute. 157 | if tok.OpensScope(): 158 | bracket_stack.append(tok) 159 | elif tok.ClosesScope(): 160 | bracket_stack[-1].matching_bracket = tok 161 | tok.matching_bracket = bracket_stack.pop() 162 | 163 | return logical_lines 164 | -------------------------------------------------------------------------------- /yapf/pyparser/pyparser_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Bill Wendling, All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """PyParser-related utilities. 15 | 16 | This module collects various utilities related to the parse trees produced by 17 | the pyparser. 18 | 19 | GetLogicalLine: produces a list of tokens from the logical lines within a 20 | range. 21 | GetTokensInSubRange: produces a sublist of tokens from a current token list 22 | within a range. 23 | GetTokenIndex: Get the index of a token. 24 | GetNextTokenIndex: Get the index of the next token after a given position. 25 | GetPrevTokenIndex: Get the index of the previous token before a given 26 | position. 27 | TokenStart: Convenience function to return the token's start as a tuple. 28 | TokenEnd: Convenience function to return the token's end as a tuple. 29 | """ 30 | 31 | 32 | def GetLogicalLine(logical_lines, node): 33 | """Get a list of tokens within the node's range from the logical lines.""" 34 | start = TokenStart(node) 35 | end = TokenEnd(node) 36 | tokens = [] 37 | 38 | for line in logical_lines: 39 | if line.start > end: 40 | break 41 | if line.start <= start or line.end >= end: 42 | tokens.extend(GetTokensInSubRange(line.tokens, node)) 43 | 44 | return tokens 45 | 46 | 47 | def GetTokensInSubRange(tokens, node): 48 | """Get a subset of tokens representing the node.""" 49 | start = TokenStart(node) 50 | end = TokenEnd(node) 51 | tokens_in_range = [] 52 | 53 | for tok in tokens: 54 | tok_range = (tok.lineno, tok.column) 55 | if tok_range >= start and tok_range < end: 56 | tokens_in_range.append(tok) 57 | 58 | return tokens_in_range 59 | 60 | 61 | def GetTokenIndex(tokens, pos): 62 | """Get the index of the token at pos.""" 63 | for index, token in enumerate(tokens): 64 | if (token.lineno, token.column) == pos: 65 | return index 66 | 67 | return None 68 | 69 | 70 | def GetNextTokenIndex(tokens, pos): 71 | """Get the index of the next token after pos.""" 72 | for index, token in enumerate(tokens): 73 | if (token.lineno, token.column) >= pos: 74 | return index 75 | 76 | return None 77 | 78 | 79 | def GetPrevTokenIndex(tokens, pos): 80 | """Get the index of the previous token before pos.""" 81 | for index, token in enumerate(tokens): 82 | if index > 0 and (token.lineno, token.column) >= pos: 83 | return index - 1 84 | 85 | return None 86 | 87 | 88 | def TokenStart(node): 89 | return (node.lineno, node.col_offset) 90 | 91 | 92 | def TokenEnd(node): 93 | return (node.end_lineno, node.end_col_offset) 94 | 95 | 96 | ############################################################################# 97 | # Code for debugging # 98 | ############################################################################# 99 | 100 | 101 | def AstDump(node): 102 | import ast 103 | print(ast.dump(node, include_attributes=True, indent=4)) 104 | -------------------------------------------------------------------------------- /yapf/pytree/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /yapf/pytree/blank_line_calculator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Calculate the number of blank lines between top-level entities. 15 | 16 | Calculates how many blank lines we need between classes, functions, and other 17 | entities at the same level. 18 | 19 | CalculateBlankLines(): the main function exported by this module. 20 | 21 | Annotations: 22 | newlines: The number of newlines required before the node. 23 | """ 24 | 25 | from yapf_third_party._ylib2to3.pgen2 import token as grammar_token 26 | 27 | from yapf.pytree import pytree_utils 28 | from yapf.pytree import pytree_visitor 29 | from yapf.yapflib import style 30 | 31 | _NO_BLANK_LINES = 1 32 | _ONE_BLANK_LINE = 2 33 | _TWO_BLANK_LINES = 3 34 | 35 | _PYTHON_STATEMENTS = frozenset({ 36 | 'small_stmt', 'expr_stmt', 'print_stmt', 'del_stmt', 'pass_stmt', 37 | 'break_stmt', 'continue_stmt', 'return_stmt', 'raise_stmt', 'yield_stmt', 38 | 'import_stmt', 'global_stmt', 'exec_stmt', 'assert_stmt', 'if_stmt', 39 | 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt', 'nonlocal_stmt', 40 | 'async_stmt', 'simple_stmt' 41 | }) 42 | 43 | 44 | def CalculateBlankLines(tree): 45 | """Run the blank line calculator visitor over the tree. 46 | 47 | This modifies the tree in place. 48 | 49 | Arguments: 50 | tree: the top-level pytree node to annotate with subtypes. 51 | """ 52 | blank_line_calculator = _BlankLineCalculator() 53 | blank_line_calculator.Visit(tree) 54 | 55 | 56 | class _BlankLineCalculator(pytree_visitor.PyTreeVisitor): 57 | """_BlankLineCalculator - see file-level docstring for a description.""" 58 | 59 | def __init__(self): 60 | self.class_level = 0 61 | self.function_level = 0 62 | self.last_comment_lineno = 0 63 | self.last_was_decorator = False 64 | self.last_was_class_or_function = False 65 | 66 | def Visit_simple_stmt(self, node): # pylint: disable=invalid-name 67 | self.DefaultNodeVisit(node) 68 | if node.children[0].type == grammar_token.COMMENT: 69 | self.last_comment_lineno = node.children[0].lineno 70 | 71 | def Visit_decorator(self, node): # pylint: disable=invalid-name 72 | if (self.last_comment_lineno and 73 | self.last_comment_lineno == node.children[0].lineno - 1): 74 | _SetNumNewlines(node.children[0], _NO_BLANK_LINES) 75 | else: 76 | _SetNumNewlines(node.children[0], self._GetNumNewlines(node)) 77 | for child in node.children: 78 | self.Visit(child) 79 | self.last_was_decorator = True 80 | 81 | def Visit_classdef(self, node): # pylint: disable=invalid-name 82 | self.last_was_class_or_function = False 83 | index = self._SetBlankLinesBetweenCommentAndClassFunc(node) 84 | self.last_was_decorator = False 85 | self.class_level += 1 86 | for child in node.children[index:]: 87 | self.Visit(child) 88 | self.class_level -= 1 89 | self.last_was_class_or_function = True 90 | 91 | def Visit_funcdef(self, node): # pylint: disable=invalid-name 92 | self.last_was_class_or_function = False 93 | index = self._SetBlankLinesBetweenCommentAndClassFunc(node) 94 | if _AsyncFunction(node): 95 | index = self._SetBlankLinesBetweenCommentAndClassFunc( 96 | node.prev_sibling.parent) 97 | _SetNumNewlines(node.children[0], None) 98 | else: 99 | index = self._SetBlankLinesBetweenCommentAndClassFunc(node) 100 | self.last_was_decorator = False 101 | self.function_level += 1 102 | for child in node.children[index:]: 103 | self.Visit(child) 104 | self.function_level -= 1 105 | self.last_was_class_or_function = True 106 | 107 | def DefaultNodeVisit(self, node): 108 | """Override the default visitor for Node. 109 | 110 | This will set the blank lines required if the last entity was a class or 111 | function. 112 | 113 | Arguments: 114 | node: (pytree.Node) The node to visit. 115 | """ 116 | if self.last_was_class_or_function: 117 | if pytree_utils.NodeName(node) in _PYTHON_STATEMENTS: 118 | leaf = pytree_utils.FirstLeafNode(node) 119 | _SetNumNewlines(leaf, self._GetNumNewlines(leaf)) 120 | self.last_was_class_or_function = False 121 | super(_BlankLineCalculator, self).DefaultNodeVisit(node) 122 | 123 | def _SetBlankLinesBetweenCommentAndClassFunc(self, node): 124 | """Set the number of blanks between a comment and class or func definition. 125 | 126 | Class and function definitions have leading comments as children of the 127 | classdef and functdef nodes. 128 | 129 | Arguments: 130 | node: (pytree.Node) The classdef or funcdef node. 131 | 132 | Returns: 133 | The index of the first child past the comment nodes. 134 | """ 135 | index = 0 136 | while pytree_utils.IsCommentStatement(node.children[index]): 137 | # Standalone comments are wrapped in a simple_stmt node with the comment 138 | # node as its only child. 139 | self.Visit(node.children[index].children[0]) 140 | if not self.last_was_decorator: 141 | _SetNumNewlines(node.children[index].children[0], _ONE_BLANK_LINE) 142 | index += 1 143 | if (index and node.children[index].lineno - 1 144 | == node.children[index - 1].children[0].lineno): 145 | _SetNumNewlines(node.children[index], _NO_BLANK_LINES) 146 | else: 147 | if self.last_comment_lineno + 1 == node.children[index].lineno: 148 | num_newlines = _NO_BLANK_LINES 149 | else: 150 | num_newlines = self._GetNumNewlines(node) 151 | _SetNumNewlines(node.children[index], num_newlines) 152 | return index 153 | 154 | def _GetNumNewlines(self, node): 155 | if self.last_was_decorator: 156 | return _NO_BLANK_LINES 157 | elif self._IsTopLevel(node): 158 | return 1 + style.Get('BLANK_LINES_AROUND_TOP_LEVEL_DEFINITION') 159 | return _ONE_BLANK_LINE 160 | 161 | def _IsTopLevel(self, node): 162 | return (not (self.class_level or self.function_level) and 163 | _StartsInZerothColumn(node)) 164 | 165 | 166 | def _SetNumNewlines(node, num_newlines): 167 | pytree_utils.SetNodeAnnotation(node, pytree_utils.Annotation.NEWLINES, 168 | num_newlines) 169 | 170 | 171 | def _StartsInZerothColumn(node): 172 | return (pytree_utils.FirstLeafNode(node).column == 0 or 173 | (_AsyncFunction(node) and node.prev_sibling.column == 0)) 174 | 175 | 176 | def _AsyncFunction(node): 177 | return (node.prev_sibling and node.prev_sibling.type == grammar_token.ASYNC) 178 | -------------------------------------------------------------------------------- /yapf/pytree/continuation_splicer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Insert "continuation" nodes into lib2to3 tree. 15 | 16 | The "backslash-newline" continuation marker is shoved into the node's prefix. 17 | Pull them out and make it into nodes of their own. 18 | 19 | SpliceContinuations(): the main function exported by this module. 20 | """ 21 | 22 | from yapf_third_party._ylib2to3 import pytree 23 | 24 | from yapf.yapflib import format_token 25 | 26 | 27 | def SpliceContinuations(tree): 28 | """Given a pytree, splice the continuation marker into nodes. 29 | 30 | Arguments: 31 | tree: (pytree.Node) The tree to work on. The tree is modified by this 32 | function. 33 | """ 34 | 35 | def RecSplicer(node): 36 | """Inserts a continuation marker into the node.""" 37 | if isinstance(node, pytree.Leaf): 38 | if node.prefix.lstrip().startswith('\\\n'): 39 | new_lineno = node.lineno - node.prefix.count('\n') 40 | return pytree.Leaf( 41 | type=format_token.CONTINUATION, 42 | value=node.prefix, 43 | context=('', (new_lineno, 0))) 44 | return None 45 | num_inserted = 0 46 | for index, child in enumerate(node.children[:]): 47 | continuation_node = RecSplicer(child) 48 | if continuation_node: 49 | node.children.insert(index + num_inserted, continuation_node) 50 | num_inserted += 1 51 | 52 | RecSplicer(tree) 53 | -------------------------------------------------------------------------------- /yapf/pytree/pytree_visitor.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Generic visitor pattern for pytrees. 15 | 16 | The lib2to3 parser produces a "pytree" - syntax tree consisting of Node 17 | and Leaf types. This module implements a visitor pattern for such trees. 18 | 19 | It also exports a basic "dumping" visitor that dumps a textual representation of 20 | a pytree into a stream. 21 | 22 | PyTreeVisitor: a generic visitor pattern for pytrees. 23 | PyTreeDumper: a configurable "dumper" for displaying pytrees. 24 | DumpPyTree(): a convenience function to dump a pytree. 25 | """ 26 | 27 | import sys 28 | 29 | from yapf_third_party._ylib2to3 import pytree 30 | 31 | from yapf.pytree import pytree_utils 32 | 33 | 34 | class PyTreeVisitor(object): 35 | """Visitor pattern for pytree trees. 36 | 37 | Methods named Visit_XXX will be invoked when a node with type XXX is 38 | encountered in the tree. The type is either a token type (for Leaf nodes) or 39 | grammar symbols (for Node nodes). The return value of Visit_XXX methods is 40 | ignored by the visitor. 41 | 42 | Visitors can modify node contents but must not change the tree structure 43 | (e.g. add/remove children and move nodes around). 44 | 45 | This is a very common visitor pattern in Python code; it's also used in the 46 | Python standard library ast module for providing AST visitors. 47 | 48 | Note: this makes names that aren't style conformant, so such visitor methods 49 | need to be marked with # pylint: disable=invalid-name We don't have a choice 50 | here, because lib2to3 nodes have under_separated names. 51 | 52 | For more complex behavior, the visit, DefaultNodeVisit and DefaultLeafVisit 53 | methods can be overridden. Don't forget to invoke DefaultNodeVisit for nodes 54 | that may have children - otherwise the children will not be visited. 55 | """ 56 | 57 | def Visit(self, node): 58 | """Visit a node.""" 59 | method = 'Visit_{0}'.format(pytree_utils.NodeName(node)) 60 | if hasattr(self, method): 61 | # Found a specific visitor for this node 62 | getattr(self, method)(node) 63 | else: 64 | if isinstance(node, pytree.Leaf): 65 | self.DefaultLeafVisit(node) 66 | else: 67 | self.DefaultNodeVisit(node) 68 | 69 | def DefaultNodeVisit(self, node): 70 | """Default visitor for Node: visits the node's children depth-first. 71 | 72 | This method is invoked when no specific visitor for the node is defined. 73 | 74 | Arguments: 75 | node: the node to visit 76 | """ 77 | for child in node.children: 78 | self.Visit(child) 79 | 80 | def DefaultLeafVisit(self, leaf): 81 | """Default visitor for Leaf: no-op. 82 | 83 | This method is invoked when no specific visitor for the leaf is defined. 84 | 85 | Arguments: 86 | leaf: the leaf to visit 87 | """ 88 | pass 89 | 90 | 91 | def DumpPyTree(tree, target_stream=sys.stdout): 92 | """Convenience function for dumping a given pytree. 93 | 94 | This function presents a very minimal interface. For more configurability (for 95 | example, controlling how specific node types are displayed), use PyTreeDumper 96 | directly. 97 | 98 | Arguments: 99 | tree: the tree to dump. 100 | target_stream: the stream to dump the tree to. A file-like object. By 101 | default will dump into stdout. 102 | """ 103 | dumper = PyTreeDumper(target_stream) 104 | dumper.Visit(tree) 105 | 106 | 107 | class PyTreeDumper(PyTreeVisitor): 108 | """Visitor that dumps the tree to a stream. 109 | 110 | Implements the PyTreeVisitor interface. 111 | """ 112 | 113 | def __init__(self, target_stream=sys.stdout): 114 | """Create a tree dumper. 115 | 116 | Arguments: 117 | target_stream: the stream to dump the tree to. A file-like object. By 118 | default will dump into stdout. 119 | """ 120 | self._target_stream = target_stream 121 | self._current_indent = 0 122 | 123 | def _DumpString(self, s): 124 | self._target_stream.write('{0}{1}\n'.format(' ' * self._current_indent, s)) 125 | 126 | def DefaultNodeVisit(self, node): 127 | # Dump information about the current node, and then use the generic 128 | # DefaultNodeVisit visitor to dump each of its children. 129 | self._DumpString(pytree_utils.DumpNodeToString(node)) 130 | self._current_indent += 2 131 | super(PyTreeDumper, self).DefaultNodeVisit(node) 132 | self._current_indent -= 2 133 | 134 | def DefaultLeafVisit(self, leaf): 135 | self._DumpString(pytree_utils.DumpNodeToString(leaf)) 136 | -------------------------------------------------------------------------------- /yapf/yapflib/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /yapf/yapflib/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """YAPF error objects.""" 15 | 16 | from yapf_third_party._ylib2to3.pgen2 import tokenize 17 | 18 | 19 | def FormatErrorMsg(e): 20 | """Convert an exception into a standard format. 21 | 22 | The standard error message format is: 23 | 24 | ::: 25 | 26 | Arguments: 27 | e: An exception. 28 | 29 | Returns: 30 | A properly formatted error message string. 31 | """ 32 | if isinstance(e, SyntaxError): 33 | return '{}:{}:{}: {}'.format(e.filename, e.lineno, e.offset, e.msg) 34 | if isinstance(e, tokenize.TokenError): 35 | return '{}:{}:{}: {}'.format(e.filename, e.args[1][0], e.args[1][1], 36 | e.args[0]) 37 | return '{}:{}:{}: {}'.format(e.args[1][0], e.args[1][1], e.args[1][2], e.msg) 38 | 39 | 40 | class YapfError(Exception): 41 | """Parent class for user errors or input errors. 42 | 43 | Exceptions of this type are handled by the command line tool 44 | and result in clear error messages, as opposed to backtraces. 45 | """ 46 | pass 47 | -------------------------------------------------------------------------------- /yapf/yapflib/file_resources.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Interface to file resources. 15 | 16 | This module provides functions for interfacing with files: opening, writing, and 17 | querying. 18 | """ 19 | 20 | import codecs 21 | import fnmatch 22 | import os 23 | import re 24 | import sys 25 | from configparser import ConfigParser 26 | from tokenize import detect_encoding 27 | 28 | from yapf.yapflib import errors 29 | from yapf.yapflib import style 30 | 31 | if sys.version_info >= (3, 11): 32 | import tomllib 33 | else: 34 | import tomli as tomllib 35 | 36 | CR = '\r' 37 | LF = '\n' 38 | CRLF = '\r\n' 39 | 40 | 41 | def _GetExcludePatternsFromYapfIgnore(filename): 42 | """Get a list of file patterns to ignore from .yapfignore.""" 43 | ignore_patterns = [] 44 | if os.path.isfile(filename) and os.access(filename, os.R_OK): 45 | with open(filename, 'r') as fd: 46 | for line in fd: 47 | if line.strip() and not line.startswith('#'): 48 | ignore_patterns.append(line.strip()) 49 | 50 | if any(e.startswith('./') for e in ignore_patterns): 51 | raise errors.YapfError('path in .yapfignore should not start with ./') 52 | 53 | return ignore_patterns 54 | 55 | 56 | def _GetExcludePatternsFromPyprojectToml(filename): 57 | """Get a list of file patterns to ignore from pyproject.toml.""" 58 | ignore_patterns = [] 59 | 60 | if os.path.isfile(filename) and os.access(filename, os.R_OK): 61 | with open(filename, 'rb') as fd: 62 | pyproject_toml = tomllib.load(fd) 63 | ignore_patterns = pyproject_toml.get('tool', 64 | {}).get('yapfignore', 65 | {}).get('ignore_patterns', []) 66 | if any(e.startswith('./') for e in ignore_patterns): 67 | raise errors.YapfError('path in pyproject.toml should not start with ./') 68 | 69 | return ignore_patterns 70 | 71 | 72 | def GetExcludePatternsForDir(dirname): 73 | """Return patterns of files to exclude from ignorefile in a given directory. 74 | 75 | Looks for .yapfignore in the directory dirname. 76 | 77 | Arguments: 78 | dirname: (unicode) The name of the directory. 79 | 80 | Returns: 81 | A List of file patterns to exclude if ignore file is found, otherwise empty 82 | List. 83 | """ 84 | ignore_patterns = [] 85 | 86 | yapfignore_file = os.path.join(dirname, '.yapfignore') 87 | if os.path.exists(yapfignore_file): 88 | ignore_patterns += _GetExcludePatternsFromYapfIgnore(yapfignore_file) 89 | 90 | pyproject_toml_file = os.path.join(dirname, 'pyproject.toml') 91 | if os.path.exists(pyproject_toml_file): 92 | ignore_patterns += _GetExcludePatternsFromPyprojectToml(pyproject_toml_file) 93 | return ignore_patterns 94 | 95 | 96 | def GetDefaultStyleForDir(dirname, default_style=style.DEFAULT_STYLE): 97 | """Return default style name for a given directory. 98 | 99 | Looks for .style.yapf or setup.cfg or pyproject.toml in the parent 100 | directories. 101 | 102 | Arguments: 103 | dirname: (unicode) The name of the directory. 104 | default_style: The style to return if nothing is found. Defaults to the 105 | global default style ('pep8') unless otherwise specified. 106 | 107 | Returns: 108 | The filename if found, otherwise return the default style. 109 | """ 110 | dirname = os.path.abspath(dirname) 111 | while True: 112 | # See if we have a .style.yapf file. 113 | style_file = os.path.join(dirname, style.LOCAL_STYLE) 114 | if os.path.exists(style_file): 115 | return style_file 116 | 117 | # See if we have a setup.cfg file with a '[yapf]' section. 118 | config_file = os.path.join(dirname, style.SETUP_CONFIG) 119 | try: 120 | fd = open(config_file) 121 | except IOError: 122 | pass # It's okay if it's not there. 123 | else: 124 | with fd: 125 | config = ConfigParser() 126 | config.read_file(fd) 127 | if config.has_section('yapf'): 128 | return config_file 129 | 130 | # See if we have a pyproject.toml file with a '[tool.yapf]' section. 131 | config_file = os.path.join(dirname, style.PYPROJECT_TOML) 132 | try: 133 | fd = open(config_file, 'rb') 134 | except IOError: 135 | pass # It's okay if it's not there. 136 | else: 137 | with fd: 138 | pyproject_toml = tomllib.load(fd) 139 | style_dict = pyproject_toml.get('tool', {}).get('yapf', None) 140 | if style_dict is not None: 141 | return config_file 142 | 143 | if (not dirname or not os.path.basename(dirname) or 144 | dirname == os.path.abspath(os.path.sep)): 145 | break 146 | dirname = os.path.dirname(dirname) 147 | 148 | global_file = os.path.expanduser(style.GLOBAL_STYLE) 149 | if os.path.exists(global_file): 150 | return global_file 151 | 152 | return default_style 153 | 154 | 155 | def GetCommandLineFiles(command_line_file_list, recursive, exclude): 156 | """Return the list of files specified on the command line.""" 157 | return _FindPythonFiles(command_line_file_list, recursive, exclude) 158 | 159 | 160 | def WriteReformattedCode(filename, 161 | reformatted_code, 162 | encoding='', 163 | in_place=False): 164 | """Emit the reformatted code. 165 | 166 | Write the reformatted code into the file, if in_place is True. Otherwise, 167 | write to stdout. 168 | 169 | Arguments: 170 | filename: (unicode) The name of the unformatted file. 171 | reformatted_code: (unicode) The reformatted code. 172 | encoding: (unicode) The encoding of the file. 173 | in_place: (bool) If True, then write the reformatted code to the file. 174 | """ 175 | if in_place: 176 | with codecs.open(filename, mode='w', encoding=encoding) as fd: 177 | fd.write(reformatted_code) 178 | else: 179 | sys.stdout.buffer.write(reformatted_code.encode('utf-8')) 180 | 181 | 182 | def LineEnding(lines): 183 | """Retrieve the line ending of the original source.""" 184 | endings = {CRLF: 0, CR: 0, LF: 0} 185 | for line in lines: 186 | if line.endswith(CRLF): 187 | endings[CRLF] += 1 188 | elif line.endswith(CR): 189 | endings[CR] += 1 190 | elif line.endswith(LF): 191 | endings[LF] += 1 192 | return sorted((LF, CRLF, CR), key=endings.get, reverse=True)[0] 193 | 194 | 195 | def _FindPythonFiles(filenames, recursive, exclude): 196 | """Find all Python files.""" 197 | if exclude and any(e.startswith('./') for e in exclude): 198 | raise errors.YapfError("path in '--exclude' should not start with ./") 199 | exclude = exclude and [e.rstrip('/' + os.path.sep) for e in exclude] 200 | 201 | python_files = [] 202 | for filename in filenames: 203 | if filename != '.' and exclude and IsIgnored(filename, exclude): 204 | continue 205 | if os.path.isdir(filename): 206 | if not recursive: 207 | raise errors.YapfError( 208 | "directory specified without '--recursive' flag: %s" % filename) 209 | 210 | # TODO(morbo): Look into a version of os.walk that can handle recursion. 211 | excluded_dirs = [] 212 | for dirpath, dirnames, filelist in os.walk(filename): 213 | if dirpath != '.' and exclude and IsIgnored(dirpath, exclude): 214 | excluded_dirs.append(dirpath) 215 | continue 216 | elif any(dirpath.startswith(e) for e in excluded_dirs): 217 | continue 218 | for f in filelist: 219 | filepath = os.path.join(dirpath, f) 220 | if exclude and IsIgnored(filepath, exclude): 221 | continue 222 | if IsPythonFile(filepath): 223 | python_files.append(filepath) 224 | # To prevent it from scanning the contents excluded folders, os.walk() 225 | # lets you amend its list of child dirs `dirnames`. These edits must be 226 | # made in-place instead of creating a modified copy of `dirnames`. 227 | # list.remove() is slow and list.pop() is a headache. Instead clear 228 | # `dirnames` then repopulate it. 229 | dirnames_ = [dirnames.pop(0) for i in range(len(dirnames))] 230 | for dirname in dirnames_: 231 | dir_ = os.path.join(dirpath, dirname) 232 | if IsIgnored(dir_, exclude): 233 | excluded_dirs.append(dir_) 234 | else: 235 | dirnames.append(dirname) 236 | 237 | elif os.path.isfile(filename): 238 | python_files.append(filename) 239 | 240 | return python_files 241 | 242 | 243 | def IsIgnored(path, exclude): 244 | """Return True if filename matches any patterns in exclude.""" 245 | if exclude is None: 246 | return False 247 | path = path.lstrip(os.path.sep) 248 | while path.startswith('.' + os.path.sep): 249 | path = path[2:] 250 | return any(fnmatch.fnmatch(path, e.rstrip(os.path.sep)) for e in exclude) 251 | 252 | 253 | def IsPythonFile(filename): 254 | """Return True if filename is a Python file.""" 255 | if os.path.splitext(filename)[1] in frozenset({'.py', '.pyi'}): 256 | return True 257 | 258 | try: 259 | with open(filename, 'rb') as fd: 260 | encoding = detect_encoding(fd.readline)[0] 261 | 262 | # Check for correctness of encoding. 263 | with codecs.open(filename, mode='r', encoding=encoding) as fd: 264 | fd.read() 265 | except UnicodeDecodeError: 266 | encoding = 'latin-1' 267 | except (IOError, SyntaxError): 268 | # If we fail to detect encoding (or the encoding cookie is incorrect - which 269 | # will make detect_encoding raise SyntaxError), assume it's not a Python 270 | # file. 271 | return False 272 | 273 | try: 274 | with codecs.open(filename, mode='r', encoding=encoding) as fd: 275 | first_line = fd.readline(256) 276 | except IOError: 277 | return False 278 | 279 | return re.match(r'^#!.*\bpython[23]?\b', first_line) 280 | 281 | 282 | def FileEncoding(filename): 283 | """Return the file's encoding.""" 284 | with open(filename, 'rb') as fd: 285 | return detect_encoding(fd.readline)[0] 286 | -------------------------------------------------------------------------------- /yapf/yapflib/identify_container.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Identify containers for lib2to3 trees. 15 | 16 | This module identifies containers and the elements in them. Each element points 17 | to the opening bracket and vice-versa. 18 | 19 | IdentifyContainers(): the main function exported by this module. 20 | """ 21 | 22 | from yapf_third_party._ylib2to3.pgen2 import token as grammar_token 23 | 24 | from yapf.pytree import pytree_utils 25 | from yapf.pytree import pytree_visitor 26 | 27 | 28 | def IdentifyContainers(tree): 29 | """Run the identify containers visitor over the tree, modifying it in place. 30 | 31 | Arguments: 32 | tree: the top-level pytree node to annotate with subtypes. 33 | """ 34 | identify_containers = _IdentifyContainers() 35 | identify_containers.Visit(tree) 36 | 37 | 38 | class _IdentifyContainers(pytree_visitor.PyTreeVisitor): 39 | """_IdentifyContainers - see file-level docstring for detailed description.""" 40 | 41 | def Visit_trailer(self, node): # pylint: disable=invalid-name 42 | for child in node.children: 43 | self.Visit(child) 44 | 45 | if len(node.children) != 3: 46 | return 47 | if node.children[0].type != grammar_token.LPAR: 48 | return 49 | 50 | if pytree_utils.NodeName(node.children[1]) == 'arglist': 51 | for child in node.children[1].children: 52 | pytree_utils.SetOpeningBracket( 53 | pytree_utils.FirstLeafNode(child), node.children[0]) 54 | else: 55 | pytree_utils.SetOpeningBracket( 56 | pytree_utils.FirstLeafNode(node.children[1]), node.children[0]) 57 | 58 | def Visit_atom(self, node): # pylint: disable=invalid-name 59 | for child in node.children: 60 | self.Visit(child) 61 | 62 | if len(node.children) != 3: 63 | return 64 | if node.children[0].type != grammar_token.LPAR: 65 | return 66 | 67 | for child in node.children[1].children: 68 | pytree_utils.SetOpeningBracket( 69 | pytree_utils.FirstLeafNode(child), node.children[0]) 70 | -------------------------------------------------------------------------------- /yapf/yapflib/line_joiner.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Join logical lines together. 15 | 16 | Determine how many lines can be joined into one line. For instance, we could 17 | join these statements into one line: 18 | 19 | if a == 42: 20 | continue 21 | 22 | like this: 23 | 24 | if a == 42: continue 25 | 26 | There are a few restrictions: 27 | 28 | 1. The lines should have been joined in the original source. 29 | 2. The joined lines must not go over the column boundary if placed on the same 30 | line. 31 | 3. They need to be very simple statements. 32 | 33 | Note: Because we don't allow the use of a semicolon to separate statements, it 34 | follows that there can only be at most two lines to join. 35 | """ 36 | 37 | from yapf.yapflib import style 38 | 39 | _CLASS_OR_FUNC = frozenset({'def', 'class'}) 40 | 41 | 42 | def CanMergeMultipleLines(lines, last_was_merged=False): 43 | """Determine if multiple lines can be joined into one. 44 | 45 | Arguments: 46 | lines: (list of LogicalLine) This is a splice of LogicalLines from the full 47 | code base. 48 | last_was_merged: (bool) The last line was merged. 49 | 50 | Returns: 51 | True if two consecutive lines can be joined together. In reality, this will 52 | only happen if two consecutive lines can be joined, due to the style guide. 53 | """ 54 | # The indentation amount for the starting line (number of spaces). 55 | indent_amt = lines[0].depth * style.Get('INDENT_WIDTH') 56 | if len(lines) == 1 or indent_amt > style.Get('COLUMN_LIMIT'): 57 | return False 58 | 59 | if (len(lines) >= 3 and lines[2].depth >= lines[1].depth and 60 | lines[0].depth != lines[2].depth): 61 | # If lines[2]'s depth is greater than or equal to line[1]'s depth, we're not 62 | # looking at a single statement (e.g., if-then, while, etc.). A following 63 | # line with the same depth as the first line isn't part of the lines we 64 | # would want to combine. 65 | return False # Don't merge more than two lines together. 66 | 67 | if lines[0].first.value in _CLASS_OR_FUNC: 68 | # Don't join lines onto the starting line of a class or function. 69 | return False 70 | 71 | limit = style.Get('COLUMN_LIMIT') - indent_amt 72 | if lines[0].last.total_length < limit: 73 | limit -= lines[0].last.total_length 74 | 75 | if lines[0].first.value == 'if': 76 | return _CanMergeLineIntoIfStatement(lines, limit) 77 | if last_was_merged and lines[0].first.value in {'elif', 'else'}: 78 | return _CanMergeLineIntoIfStatement(lines, limit) 79 | 80 | # TODO(morbo): Other control statements? 81 | 82 | return False 83 | 84 | 85 | def _CanMergeLineIntoIfStatement(lines, limit): 86 | """Determine if we can merge a short if-then statement into one line. 87 | 88 | Two lines of an if-then statement can be merged if they were that way in the 89 | original source, fit on the line without going over the column limit, and are 90 | considered "simple" statements --- typically statements like 'pass', 91 | 'continue', and 'break'. 92 | 93 | Arguments: 94 | lines: (list of LogicalLine) The lines we are wanting to merge. 95 | limit: (int) The amount of space remaining on the line. 96 | 97 | Returns: 98 | True if the lines can be merged, False otherwise. 99 | """ 100 | if len(lines[1].tokens) == 1 and lines[1].last.is_multiline_string: 101 | # This might be part of a multiline shebang. 102 | return True 103 | if lines[0].lineno != lines[1].lineno: 104 | # Don't merge lines if the original lines weren't merged. 105 | return False 106 | if lines[1].last.total_length >= limit: 107 | # Don't merge lines if the result goes over the column limit. 108 | return False 109 | return style.Get('JOIN_MULTIPLE_LINES') 110 | -------------------------------------------------------------------------------- /yapf/yapflib/object_state.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Represents the state of Python objects being formatted. 15 | 16 | Objects (e.g., list comprehensions, dictionaries, etc.) have specific 17 | requirements on how they're formatted. These state objects keep track of these 18 | requirements. 19 | """ 20 | 21 | from functools import lru_cache 22 | 23 | from yapf.yapflib import style 24 | from yapf.yapflib import subtypes 25 | 26 | 27 | class ComprehensionState(object): 28 | """Maintains the state of list comprehension formatting decisions. 29 | 30 | A stack of ComprehensionState objects are kept to ensure that list 31 | comprehensions are wrapped with well-defined rules. 32 | 33 | Attributes: 34 | expr_token: The first token in the comprehension. 35 | for_token: The first 'for' token of the comprehension. 36 | opening_bracket: The opening bracket of the list comprehension. 37 | closing_bracket: The closing bracket of the list comprehension. 38 | has_split_at_for: Whether there is a newline immediately before the 39 | for_token. 40 | has_interior_split: Whether there is a newline within the comprehension. 41 | That is, a split somewhere after expr_token or before closing_bracket. 42 | """ 43 | 44 | def __init__(self, expr_token): 45 | self.expr_token = expr_token 46 | self.for_token = None 47 | self.has_split_at_for = False 48 | self.has_interior_split = False 49 | 50 | def HasTrivialExpr(self): 51 | """Returns whether the comp_expr is "trivial" i.e. is a single token.""" 52 | return self.expr_token.next_token.value == 'for' 53 | 54 | @property 55 | def opening_bracket(self): 56 | return self.expr_token.previous_token 57 | 58 | @property 59 | def closing_bracket(self): 60 | return self.opening_bracket.matching_bracket 61 | 62 | def Clone(self): 63 | clone = ComprehensionState(self.expr_token) 64 | clone.for_token = self.for_token 65 | clone.has_split_at_for = self.has_split_at_for 66 | clone.has_interior_split = self.has_interior_split 67 | return clone 68 | 69 | def __repr__(self): 70 | return ('[opening_bracket::%s, for_token::%s, has_split_at_for::%s,' 71 | ' has_interior_split::%s, has_trivial_expr::%s]' % 72 | (self.opening_bracket, self.for_token, self.has_split_at_for, 73 | self.has_interior_split, self.HasTrivialExpr())) 74 | 75 | def __eq__(self, other): 76 | return hash(self) == hash(other) 77 | 78 | def __ne__(self, other): 79 | return not self == other 80 | 81 | def __hash__(self, *args, **kwargs): 82 | return hash((self.expr_token, self.for_token, self.has_split_at_for, 83 | self.has_interior_split)) 84 | 85 | 86 | class ParameterListState(object): 87 | """Maintains the state of function parameter list formatting decisions. 88 | 89 | Attributes: 90 | opening_bracket: The opening bracket of the parameter list. 91 | closing_bracket: The closing bracket of the parameter list. 92 | has_typed_return: True if the function definition has a typed return. 93 | ends_in_comma: True if the parameter list ends in a comma. 94 | last_token: Returns the last token of the function declaration. 95 | has_default_values: True if the parameters have default values. 96 | has_split_before_first_param: Whether there is a newline before the first 97 | parameter. 98 | opening_column: The position of the opening parameter before a newline. 99 | parameters: A list of parameter objects (Parameter). 100 | split_before_closing_bracket: Split before the closing bracket. Sometimes 101 | needed if the indentation would collide. 102 | """ 103 | 104 | def __init__(self, opening_bracket, newline, opening_column): 105 | self.opening_bracket = opening_bracket 106 | self.has_split_before_first_param = newline 107 | self.opening_column = opening_column 108 | self.parameters = opening_bracket.parameters 109 | self.split_before_closing_bracket = False 110 | 111 | @property 112 | def closing_bracket(self): 113 | return self.opening_bracket.matching_bracket 114 | 115 | @property 116 | def has_typed_return(self): 117 | return self.closing_bracket.next_token.value == '->' 118 | 119 | @property 120 | @lru_cache() 121 | def has_default_values(self): 122 | return any(param.has_default_value for param in self.parameters) 123 | 124 | @property 125 | @lru_cache() 126 | def ends_in_comma(self): 127 | if not self.parameters: 128 | return False 129 | return self.parameters[-1].last_token.next_token.value == ',' 130 | 131 | @property 132 | @lru_cache() 133 | def last_token(self): 134 | token = self.opening_bracket.matching_bracket 135 | while not token.is_comment and token.next_token: 136 | token = token.next_token 137 | return token 138 | 139 | @lru_cache() 140 | def LastParamFitsOnLine(self, indent): 141 | """Return true if the last parameter fits on a single line.""" 142 | if not self.has_typed_return: 143 | return False 144 | if not self.parameters: 145 | return True 146 | total_length = self.last_token.total_length 147 | last_param = self.parameters[-1].first_token 148 | total_length -= last_param.total_length - len(last_param.value) 149 | return total_length + indent <= style.Get('COLUMN_LIMIT') 150 | 151 | @lru_cache() 152 | def SplitBeforeClosingBracket(self, indent): 153 | """Return true if there's a split before the closing bracket.""" 154 | if style.Get('DEDENT_CLOSING_BRACKETS'): 155 | return True 156 | if self.ends_in_comma: 157 | return True 158 | if not self.parameters: 159 | return False 160 | total_length = self.last_token.total_length 161 | last_param = self.parameters[-1].first_token 162 | total_length -= last_param.total_length - len(last_param.value) 163 | return total_length + indent > style.Get('COLUMN_LIMIT') 164 | 165 | def Clone(self): 166 | clone = ParameterListState(self.opening_bracket, 167 | self.has_split_before_first_param, 168 | self.opening_column) 169 | clone.split_before_closing_bracket = self.split_before_closing_bracket 170 | clone.parameters = [param.Clone() for param in self.parameters] 171 | return clone 172 | 173 | def __repr__(self): 174 | return ('[opening_bracket::%s, has_split_before_first_param::%s, ' 175 | 'opening_column::%d]' % 176 | (self.opening_bracket, self.has_split_before_first_param, 177 | self.opening_column)) 178 | 179 | def __eq__(self, other): 180 | return hash(self) == hash(other) 181 | 182 | def __ne__(self, other): 183 | return not self == other 184 | 185 | def __hash__(self, *args, **kwargs): 186 | return hash( 187 | (self.opening_bracket, self.has_split_before_first_param, 188 | self.opening_column, (hash(param) for param in self.parameters))) 189 | 190 | 191 | class Parameter(object): 192 | """A parameter in a parameter list. 193 | 194 | Attributes: 195 | first_token: (format_token.FormatToken) First token of parameter. 196 | last_token: (format_token.FormatToken) Last token of parameter. 197 | has_default_value: (boolean) True if the parameter has a default value 198 | """ 199 | 200 | def __init__(self, first_token, last_token): 201 | self.first_token = first_token 202 | self.last_token = last_token 203 | 204 | @property 205 | @lru_cache() 206 | def has_default_value(self): 207 | """Returns true if the parameter has a default value.""" 208 | tok = self.first_token 209 | while tok != self.last_token: 210 | if subtypes.DEFAULT_OR_NAMED_ASSIGN in tok.subtypes: 211 | return True 212 | tok = tok.matching_bracket if tok.OpensScope() else tok.next_token 213 | return False 214 | 215 | def Clone(self): 216 | return Parameter(self.first_token, self.last_token) 217 | 218 | def __repr__(self): 219 | return '[first_token::%s, last_token:%s]' % (self.first_token, 220 | self.last_token) 221 | 222 | def __eq__(self, other): 223 | return hash(self) == hash(other) 224 | 225 | def __ne__(self, other): 226 | return not self == other 227 | 228 | def __hash__(self, *args, **kwargs): 229 | return hash((self.first_token, self.last_token)) 230 | -------------------------------------------------------------------------------- /yapf/yapflib/split_penalty.py: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Bill Wendling, All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from yapf.yapflib import style 16 | 17 | # Generic split penalties 18 | UNBREAKABLE = 1000**5 19 | VERY_STRONGLY_CONNECTED = 5000 20 | STRONGLY_CONNECTED = 2500 21 | 22 | ############################################################################# 23 | # Grammar-specific penalties - should be <= 1000 # 24 | ############################################################################# 25 | 26 | # Lambdas shouldn't be split unless absolutely necessary or if 27 | # ALLOW_MULTILINE_LAMBDAS is True. 28 | LAMBDA = 1000 29 | MULTILINE_LAMBDA = 500 30 | 31 | ANNOTATION = 100 32 | ARGUMENT = 25 33 | 34 | # TODO: Assign real values. 35 | RETURN_TYPE = 1 36 | DOTTED_NAME = 40 37 | EXPR = 10 38 | DICT_KEY_EXPR = 20 39 | DICT_VALUE_EXPR = 11 40 | -------------------------------------------------------------------------------- /yapf/yapflib/subtypes.py: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Token subtypes used to improve formatting.""" 15 | 16 | NONE = 0 17 | UNARY_OPERATOR = 1 18 | BINARY_OPERATOR = 2 19 | SUBSCRIPT_COLON = 3 20 | SUBSCRIPT_BRACKET = 4 21 | DEFAULT_OR_NAMED_ASSIGN = 5 22 | DEFAULT_OR_NAMED_ASSIGN_ARG_LIST = 6 23 | VARARGS_LIST = 7 24 | VARARGS_STAR = 8 25 | KWARGS_STAR_STAR = 9 26 | ASSIGN_OPERATOR = 10 27 | DICTIONARY_KEY = 11 28 | DICTIONARY_KEY_PART = 12 29 | DICTIONARY_VALUE = 13 30 | DICT_SET_GENERATOR = 14 31 | COMP_EXPR = 15 32 | COMP_FOR = 16 33 | COMP_IF = 17 34 | FUNC_DEF = 18 35 | DECORATOR = 19 36 | TYPED_NAME = 20 37 | TYPED_NAME_ARG_LIST = 21 38 | SIMPLE_EXPRESSION = 22 39 | PARAMETER_START = 23 40 | PARAMETER_STOP = 24 41 | LAMBDEF = 25 42 | -------------------------------------------------------------------------------- /yapftests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | -------------------------------------------------------------------------------- /yapftests/blank_line_calculator_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for yapf.blank_line_calculator.""" 15 | 16 | import textwrap 17 | import unittest 18 | 19 | from yapf.yapflib import reformatter 20 | from yapf.yapflib import style 21 | from yapf.yapflib import yapf_api 22 | 23 | from yapftests import yapf_test_helper 24 | 25 | 26 | class BasicBlankLineCalculatorTest(yapf_test_helper.YAPFTest): 27 | 28 | @classmethod 29 | def setUpClass(cls): 30 | style.SetGlobalStyle(style.CreateYapfStyle()) 31 | 32 | def testDecorators(self): 33 | unformatted_code = textwrap.dedent("""\ 34 | @bork() 35 | 36 | def foo(): 37 | pass 38 | """) 39 | expected_formatted_code = textwrap.dedent("""\ 40 | @bork() 41 | def foo(): 42 | pass 43 | """) 44 | llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) 45 | self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) 46 | 47 | def testComplexDecorators(self): 48 | unformatted_code = textwrap.dedent("""\ 49 | import sys 50 | @bork() 51 | 52 | def foo(): 53 | pass 54 | @fork() 55 | 56 | class moo(object): 57 | @bar() 58 | @baz() 59 | 60 | def method(self): 61 | pass 62 | """) 63 | expected_formatted_code = textwrap.dedent("""\ 64 | import sys 65 | 66 | 67 | @bork() 68 | def foo(): 69 | pass 70 | 71 | 72 | @fork() 73 | class moo(object): 74 | 75 | @bar() 76 | @baz() 77 | def method(self): 78 | pass 79 | """) 80 | llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) 81 | self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) 82 | 83 | def testCodeAfterFunctionsAndClasses(self): 84 | unformatted_code = textwrap.dedent("""\ 85 | def foo(): 86 | pass 87 | top_level_code = True 88 | class moo(object): 89 | def method_1(self): 90 | pass 91 | ivar_a = 42 92 | ivar_b = 13 93 | def method_2(self): 94 | pass 95 | try: 96 | raise Error 97 | except Error as error: 98 | pass 99 | """) 100 | expected_formatted_code = textwrap.dedent("""\ 101 | def foo(): 102 | pass 103 | 104 | 105 | top_level_code = True 106 | 107 | 108 | class moo(object): 109 | 110 | def method_1(self): 111 | pass 112 | 113 | ivar_a = 42 114 | ivar_b = 13 115 | 116 | def method_2(self): 117 | pass 118 | 119 | 120 | try: 121 | raise Error 122 | except Error as error: 123 | pass 124 | """) 125 | llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) 126 | self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) 127 | 128 | def testCommentSpacing(self): 129 | unformatted_code = textwrap.dedent("""\ 130 | # This is the first comment 131 | # And it's multiline 132 | 133 | # This is the second comment 134 | 135 | def foo(): 136 | pass 137 | 138 | # multiline before a 139 | # class definition 140 | 141 | # This is the second comment 142 | 143 | class qux(object): 144 | pass 145 | 146 | 147 | # An attached comment. 148 | class bar(object): 149 | '''class docstring''' 150 | # Comment attached to 151 | # function 152 | def foo(self): 153 | '''Another docstring.''' 154 | # Another multiline 155 | # comment 156 | pass 157 | """) 158 | expected_formatted_code = textwrap.dedent("""\ 159 | # This is the first comment 160 | # And it's multiline 161 | 162 | # This is the second comment 163 | 164 | 165 | def foo(): 166 | pass 167 | 168 | 169 | # multiline before a 170 | # class definition 171 | 172 | # This is the second comment 173 | 174 | 175 | class qux(object): 176 | pass 177 | 178 | 179 | # An attached comment. 180 | class bar(object): 181 | '''class docstring''' 182 | 183 | # Comment attached to 184 | # function 185 | def foo(self): 186 | '''Another docstring.''' 187 | # Another multiline 188 | # comment 189 | pass 190 | """) 191 | llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) 192 | self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) 193 | 194 | def testCommentBeforeMethod(self): 195 | code = textwrap.dedent("""\ 196 | class foo(object): 197 | 198 | # pylint: disable=invalid-name 199 | def f(self): 200 | pass 201 | """) 202 | llines = yapf_test_helper.ParseAndUnwrap(code) 203 | self.assertCodeEqual(code, reformatter.Reformat(llines)) 204 | 205 | def testCommentsBeforeClassDefs(self): 206 | code = textwrap.dedent('''\ 207 | """Test.""" 208 | 209 | # Comment 210 | 211 | 212 | class Foo(object): 213 | pass 214 | ''') 215 | llines = yapf_test_helper.ParseAndUnwrap(code) 216 | self.assertCodeEqual(code, reformatter.Reformat(llines)) 217 | 218 | def testCommentsBeforeDecorator(self): 219 | code = textwrap.dedent("""\ 220 | # The @foo operator adds bork to a(). 221 | @foo() 222 | def a(): 223 | pass 224 | """) 225 | llines = yapf_test_helper.ParseAndUnwrap(code) 226 | self.assertCodeEqual(code, reformatter.Reformat(llines)) 227 | 228 | code = textwrap.dedent("""\ 229 | # Hello world 230 | 231 | 232 | @foo() 233 | def a(): 234 | pass 235 | """) 236 | llines = yapf_test_helper.ParseAndUnwrap(code) 237 | self.assertCodeEqual(code, reformatter.Reformat(llines)) 238 | 239 | def testCommentsAfterDecorator(self): 240 | code = textwrap.dedent("""\ 241 | class _(): 242 | 243 | def _(): 244 | pass 245 | 246 | @pytest.mark.xfail(reason="#709 and #710") 247 | # also 248 | #@pytest.mark.xfail(setuptools.tests.is_ascii, 249 | # reason="https://github.com/pypa/setuptools/issues/706") 250 | def test_unicode_filename_in_sdist(self, sdist_unicode, tmpdir, monkeypatch): 251 | pass 252 | """) # noqa 253 | llines = yapf_test_helper.ParseAndUnwrap(code) 254 | self.assertCodeEqual(code, reformatter.Reformat(llines)) 255 | 256 | def testInnerClasses(self): 257 | unformatted_code = textwrap.dedent("""\ 258 | class DeployAPIClient(object): 259 | class Error(Exception): pass 260 | 261 | class TaskValidationError(Error): pass 262 | 263 | class DeployAPIHTTPError(Error): pass 264 | """) 265 | expected_formatted_code = textwrap.dedent("""\ 266 | class DeployAPIClient(object): 267 | 268 | class Error(Exception): 269 | pass 270 | 271 | class TaskValidationError(Error): 272 | pass 273 | 274 | class DeployAPIHTTPError(Error): 275 | pass 276 | """) 277 | llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) 278 | self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) 279 | 280 | def testLinesOnRangeBoundary(self): 281 | unformatted_code = textwrap.dedent("""\ 282 | def A(): 283 | pass 284 | 285 | def B(): # 4 286 | pass # 5 287 | 288 | def C(): 289 | pass 290 | def D(): # 9 291 | pass # 10 292 | def E(): 293 | pass 294 | """) 295 | expected_formatted_code = textwrap.dedent("""\ 296 | def A(): 297 | pass 298 | 299 | 300 | def B(): # 4 301 | pass # 5 302 | 303 | def C(): 304 | pass 305 | 306 | 307 | def D(): # 9 308 | pass # 10 309 | def E(): 310 | pass 311 | """) 312 | code, changed = yapf_api.FormatCode( 313 | unformatted_code, lines=[(4, 5), (9, 10)]) 314 | self.assertCodeEqual(expected_formatted_code, code) 315 | self.assertTrue(changed) 316 | 317 | def testLinesRangeBoundaryNotOutside(self): 318 | unformatted_code = textwrap.dedent("""\ 319 | def A(): 320 | pass 321 | 322 | 323 | 324 | def B(): # 6 325 | pass # 7 326 | 327 | 328 | 329 | def C(): 330 | pass 331 | """) 332 | expected_formatted_code = textwrap.dedent("""\ 333 | def A(): 334 | pass 335 | 336 | 337 | 338 | def B(): # 6 339 | pass # 7 340 | 341 | 342 | 343 | def C(): 344 | pass 345 | """) 346 | code, changed = yapf_api.FormatCode(unformatted_code, lines=[(6, 7)]) 347 | self.assertCodeEqual(expected_formatted_code, code) 348 | self.assertFalse(changed) 349 | 350 | def testLinesRangeRemove(self): 351 | unformatted_code = textwrap.dedent("""\ 352 | def A(): 353 | pass 354 | 355 | 356 | 357 | def B(): # 6 358 | pass # 7 359 | 360 | 361 | 362 | 363 | def C(): 364 | pass 365 | """) 366 | expected_formatted_code = textwrap.dedent("""\ 367 | def A(): 368 | pass 369 | 370 | 371 | def B(): # 6 372 | pass # 7 373 | 374 | 375 | 376 | 377 | def C(): 378 | pass 379 | """) 380 | code, changed = yapf_api.FormatCode(unformatted_code, lines=[(5, 9)]) 381 | self.assertCodeEqual(expected_formatted_code, code) 382 | self.assertTrue(changed) 383 | 384 | def testLinesRangeRemoveSome(self): 385 | unformatted_code = textwrap.dedent("""\ 386 | def A(): 387 | pass 388 | 389 | 390 | 391 | 392 | def B(): # 7 393 | pass # 8 394 | 395 | 396 | 397 | 398 | def C(): 399 | pass 400 | """) 401 | expected_formatted_code = textwrap.dedent("""\ 402 | def A(): 403 | pass 404 | 405 | 406 | 407 | def B(): # 7 408 | pass # 8 409 | 410 | 411 | 412 | 413 | def C(): 414 | pass 415 | """) 416 | code, changed = yapf_api.FormatCode(unformatted_code, lines=[(6, 9)]) 417 | self.assertCodeEqual(expected_formatted_code, code) 418 | self.assertTrue(changed) 419 | 420 | 421 | if __name__ == '__main__': 422 | unittest.main() 423 | -------------------------------------------------------------------------------- /yapftests/format_decision_state_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for yapf.format_decision_state.""" 15 | 16 | import textwrap 17 | import unittest 18 | 19 | from yapf.pytree import pytree_utils 20 | from yapf.yapflib import format_decision_state 21 | from yapf.yapflib import logical_line 22 | from yapf.yapflib import style 23 | 24 | from yapftests import yapf_test_helper 25 | 26 | 27 | class FormatDecisionStateTest(yapf_test_helper.YAPFTest): 28 | 29 | @classmethod 30 | def setUpClass(cls): 31 | style.SetGlobalStyle(style.CreateYapfStyle()) 32 | 33 | def testSimpleFunctionDefWithNoSplitting(self): 34 | code = textwrap.dedent("""\ 35 | def f(a, b): 36 | pass 37 | """) 38 | llines = yapf_test_helper.ParseAndUnwrap(code) 39 | lline = logical_line.LogicalLine(0, _FilterLine(llines[0])) 40 | lline.CalculateFormattingInformation() 41 | 42 | # Add: 'f' 43 | state = format_decision_state.FormatDecisionState(lline, 0) 44 | state.MoveStateToNextToken() 45 | self.assertEqual('f', state.next_token.value) 46 | self.assertFalse(state.CanSplit(False)) 47 | 48 | # Add: '(' 49 | state.AddTokenToState(False, True) 50 | self.assertEqual('(', state.next_token.value) 51 | self.assertFalse(state.CanSplit(False)) 52 | self.assertFalse(state.MustSplit()) 53 | 54 | # Add: 'a' 55 | state.AddTokenToState(False, True) 56 | self.assertEqual('a', state.next_token.value) 57 | self.assertTrue(state.CanSplit(False)) 58 | self.assertFalse(state.MustSplit()) 59 | 60 | # Add: ',' 61 | state.AddTokenToState(False, True) 62 | self.assertEqual(',', state.next_token.value) 63 | self.assertFalse(state.CanSplit(False)) 64 | self.assertFalse(state.MustSplit()) 65 | 66 | # Add: 'b' 67 | state.AddTokenToState(False, True) 68 | self.assertEqual('b', state.next_token.value) 69 | self.assertTrue(state.CanSplit(False)) 70 | self.assertFalse(state.MustSplit()) 71 | 72 | # Add: ')' 73 | state.AddTokenToState(False, True) 74 | self.assertEqual(')', state.next_token.value) 75 | self.assertTrue(state.CanSplit(False)) 76 | self.assertFalse(state.MustSplit()) 77 | 78 | # Add: ':' 79 | state.AddTokenToState(False, True) 80 | self.assertEqual(':', state.next_token.value) 81 | self.assertFalse(state.CanSplit(False)) 82 | self.assertFalse(state.MustSplit()) 83 | 84 | clone = state.Clone() 85 | self.assertEqual(repr(state), repr(clone)) 86 | 87 | def testSimpleFunctionDefWithSplitting(self): 88 | code = textwrap.dedent("""\ 89 | def f(a, b): 90 | pass 91 | """) 92 | llines = yapf_test_helper.ParseAndUnwrap(code) 93 | lline = logical_line.LogicalLine(0, _FilterLine(llines[0])) 94 | lline.CalculateFormattingInformation() 95 | 96 | # Add: 'f' 97 | state = format_decision_state.FormatDecisionState(lline, 0) 98 | state.MoveStateToNextToken() 99 | self.assertEqual('f', state.next_token.value) 100 | self.assertFalse(state.CanSplit(False)) 101 | 102 | # Add: '(' 103 | state.AddTokenToState(True, True) 104 | self.assertEqual('(', state.next_token.value) 105 | self.assertFalse(state.CanSplit(False)) 106 | 107 | # Add: 'a' 108 | state.AddTokenToState(True, True) 109 | self.assertEqual('a', state.next_token.value) 110 | self.assertTrue(state.CanSplit(False)) 111 | 112 | # Add: ',' 113 | state.AddTokenToState(True, True) 114 | self.assertEqual(',', state.next_token.value) 115 | self.assertFalse(state.CanSplit(False)) 116 | 117 | # Add: 'b' 118 | state.AddTokenToState(True, True) 119 | self.assertEqual('b', state.next_token.value) 120 | self.assertTrue(state.CanSplit(False)) 121 | 122 | # Add: ')' 123 | state.AddTokenToState(True, True) 124 | self.assertEqual(')', state.next_token.value) 125 | self.assertTrue(state.CanSplit(False)) 126 | 127 | # Add: ':' 128 | state.AddTokenToState(True, True) 129 | self.assertEqual(':', state.next_token.value) 130 | self.assertFalse(state.CanSplit(False)) 131 | 132 | clone = state.Clone() 133 | self.assertEqual(repr(state), repr(clone)) 134 | 135 | 136 | def _FilterLine(lline): 137 | """Filter out nonsemantic tokens from the LogicalLines.""" 138 | return [ 139 | ft for ft in lline.tokens 140 | if ft.name not in pytree_utils.NONSEMANTIC_TOKENS 141 | ] 142 | 143 | 144 | if __name__ == '__main__': 145 | unittest.main() 146 | -------------------------------------------------------------------------------- /yapftests/format_token_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for yapf.format_token.""" 15 | 16 | import unittest 17 | 18 | from yapf_third_party._ylib2to3 import pytree 19 | from yapf_third_party._ylib2to3.pgen2 import token 20 | 21 | from yapf.yapflib import format_token 22 | 23 | from yapftests import yapf_test_helper 24 | 25 | 26 | class TabbedContinuationAlignPaddingTest(yapf_test_helper.YAPFTest): 27 | 28 | def testSpace(self): 29 | align_style = 'SPACE' 30 | 31 | pad = format_token._TabbedContinuationAlignPadding(0, align_style, 2) 32 | self.assertEqual(pad, '') 33 | 34 | pad = format_token._TabbedContinuationAlignPadding(2, align_style, 2) 35 | self.assertEqual(pad, ' ' * 2) 36 | 37 | pad = format_token._TabbedContinuationAlignPadding(5, align_style, 2) 38 | self.assertEqual(pad, ' ' * 5) 39 | 40 | def testFixed(self): 41 | align_style = 'FIXED' 42 | 43 | pad = format_token._TabbedContinuationAlignPadding(0, align_style, 4) 44 | self.assertEqual(pad, '') 45 | 46 | pad = format_token._TabbedContinuationAlignPadding(2, align_style, 4) 47 | self.assertEqual(pad, '\t') 48 | 49 | pad = format_token._TabbedContinuationAlignPadding(5, align_style, 4) 50 | self.assertEqual(pad, '\t' * 2) 51 | 52 | def testVAlignRight(self): 53 | align_style = 'VALIGN-RIGHT' 54 | 55 | pad = format_token._TabbedContinuationAlignPadding(0, align_style, 4) 56 | self.assertEqual(pad, '') 57 | 58 | pad = format_token._TabbedContinuationAlignPadding(2, align_style, 4) 59 | self.assertEqual(pad, '\t') 60 | 61 | pad = format_token._TabbedContinuationAlignPadding(4, align_style, 4) 62 | self.assertEqual(pad, '\t') 63 | 64 | pad = format_token._TabbedContinuationAlignPadding(5, align_style, 4) 65 | self.assertEqual(pad, '\t' * 2) 66 | 67 | 68 | class FormatTokenTest(yapf_test_helper.YAPFTest): 69 | 70 | def testSimple(self): 71 | tok = format_token.FormatToken( 72 | pytree.Leaf(token.STRING, "'hello world'"), 'STRING') 73 | self.assertEqual( 74 | "FormatToken(name=DOCSTRING, value='hello world', column=0, " 75 | 'lineno=0, splitpenalty=0)', str(tok)) 76 | self.assertTrue(tok.is_string) 77 | 78 | tok = format_token.FormatToken( 79 | pytree.Leaf(token.COMMENT, '# A comment'), 'COMMENT') 80 | self.assertEqual( 81 | 'FormatToken(name=COMMENT, value=# A comment, column=0, ' 82 | 'lineno=0, splitpenalty=0)', str(tok)) 83 | self.assertTrue(tok.is_comment) 84 | 85 | def testIsMultilineString(self): 86 | tok = format_token.FormatToken( 87 | pytree.Leaf(token.STRING, '"""hello"""'), 'STRING') 88 | self.assertTrue(tok.is_multiline_string) 89 | 90 | tok = format_token.FormatToken( 91 | pytree.Leaf(token.STRING, 'r"""hello"""'), 'STRING') 92 | self.assertTrue(tok.is_multiline_string) 93 | 94 | 95 | if __name__ == '__main__': 96 | unittest.main() 97 | -------------------------------------------------------------------------------- /yapftests/line_joiner_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for yapf.line_joiner.""" 15 | 16 | import textwrap 17 | import unittest 18 | 19 | from yapf.yapflib import line_joiner 20 | from yapf.yapflib import style 21 | 22 | from yapftests import yapf_test_helper 23 | 24 | 25 | class LineJoinerTest(yapf_test_helper.YAPFTest): 26 | 27 | @classmethod 28 | def setUpClass(cls): 29 | style.SetGlobalStyle(style.CreatePEP8Style()) 30 | 31 | def _CheckLineJoining(self, code, join_lines): 32 | """Check that the given LogicalLines are joined as expected. 33 | 34 | Arguments: 35 | code: The code to check to see if we can join it. 36 | join_lines: True if we expect the lines to be joined. 37 | """ 38 | llines = yapf_test_helper.ParseAndUnwrap(code) 39 | self.assertCodeEqual(line_joiner.CanMergeMultipleLines(llines), join_lines) 40 | 41 | def testSimpleSingleLineStatement(self): 42 | code = textwrap.dedent("""\ 43 | if isinstance(a, int): continue 44 | """) 45 | self._CheckLineJoining(code, join_lines=True) 46 | 47 | def testSimpleMultipleLineStatement(self): 48 | code = textwrap.dedent("""\ 49 | if isinstance(b, int): 50 | continue 51 | """) 52 | self._CheckLineJoining(code, join_lines=False) 53 | 54 | def testSimpleMultipleLineComplexStatement(self): 55 | code = textwrap.dedent("""\ 56 | if isinstance(c, int): 57 | while True: 58 | continue 59 | """) 60 | self._CheckLineJoining(code, join_lines=False) 61 | 62 | def testSimpleMultipleLineStatementWithComment(self): 63 | code = textwrap.dedent("""\ 64 | if isinstance(d, int): continue # We're pleased that d's an int. 65 | """) 66 | self._CheckLineJoining(code, join_lines=True) 67 | 68 | def testSimpleMultipleLineStatementWithLargeIndent(self): 69 | code = textwrap.dedent("""\ 70 | if isinstance(e, int): continue 71 | """) 72 | self._CheckLineJoining(code, join_lines=True) 73 | 74 | def testOverColumnLimit(self): 75 | code = textwrap.dedent("""\ 76 | if instance(bbbbbbbbbbbbbbbbbbbbbbbbb, int): cccccccccccccccccccccccccc = ddddddddddddddddddddd 77 | """) # noqa 78 | self._CheckLineJoining(code, join_lines=False) 79 | 80 | 81 | if __name__ == '__main__': 82 | unittest.main() 83 | -------------------------------------------------------------------------------- /yapftests/logical_line_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for yapf.logical_line.""" 15 | 16 | import textwrap 17 | import unittest 18 | 19 | from yapf_third_party._ylib2to3 import pytree 20 | from yapf_third_party._ylib2to3.pgen2 import token 21 | 22 | from yapf.pytree import split_penalty 23 | from yapf.yapflib import format_token 24 | from yapf.yapflib import logical_line 25 | 26 | from yapftests import yapf_test_helper 27 | 28 | 29 | class LogicalLineBasicTest(yapf_test_helper.YAPFTest): 30 | 31 | def testConstruction(self): 32 | toks = _MakeFormatTokenList([(token.DOT, '.', 'DOT'), 33 | (token.VBAR, '|', 'VBAR')]) 34 | lline = logical_line.LogicalLine(20, toks) 35 | self.assertEqual(20, lline.depth) 36 | self.assertEqual(['DOT', 'VBAR'], [tok.name for tok in lline.tokens]) 37 | 38 | def testFirstLast(self): 39 | toks = _MakeFormatTokenList([(token.DOT, '.', 'DOT'), 40 | (token.LPAR, '(', 'LPAR'), 41 | (token.VBAR, '|', 'VBAR')]) 42 | lline = logical_line.LogicalLine(20, toks) 43 | self.assertEqual(20, lline.depth) 44 | self.assertEqual('DOT', lline.first.name) 45 | self.assertEqual('VBAR', lline.last.name) 46 | 47 | def testAsCode(self): 48 | toks = _MakeFormatTokenList([(token.DOT, '.', 'DOT'), 49 | (token.LPAR, '(', 'LPAR'), 50 | (token.VBAR, '|', 'VBAR')]) 51 | lline = logical_line.LogicalLine(2, toks) 52 | self.assertEqual(' . ( |', lline.AsCode()) 53 | 54 | def testAppendToken(self): 55 | lline = logical_line.LogicalLine(0) 56 | lline.AppendToken(_MakeFormatTokenLeaf(token.LPAR, '(', 'LPAR')) 57 | lline.AppendToken(_MakeFormatTokenLeaf(token.RPAR, ')', 'RPAR')) 58 | self.assertEqual(['LPAR', 'RPAR'], [tok.name for tok in lline.tokens]) 59 | 60 | 61 | class LogicalLineFormattingInformationTest(yapf_test_helper.YAPFTest): 62 | 63 | def testFuncDef(self): 64 | code = textwrap.dedent("""\ 65 | def f(a, b): 66 | pass 67 | """) 68 | llines = yapf_test_helper.ParseAndUnwrap(code) 69 | 70 | f = llines[0].tokens[1] 71 | self.assertFalse(f.can_break_before) 72 | self.assertFalse(f.must_break_before) 73 | self.assertEqual(f.split_penalty, split_penalty.UNBREAKABLE) 74 | 75 | lparen = llines[0].tokens[2] 76 | self.assertFalse(lparen.can_break_before) 77 | self.assertFalse(lparen.must_break_before) 78 | self.assertEqual(lparen.split_penalty, split_penalty.UNBREAKABLE) 79 | 80 | 81 | def _MakeFormatTokenLeaf(token_type, token_value, name): 82 | return format_token.FormatToken(pytree.Leaf(token_type, token_value), name) 83 | 84 | 85 | def _MakeFormatTokenList(token_type_values): 86 | return [ 87 | _MakeFormatTokenLeaf(token_type, token_value, token_name) 88 | for token_type, token_value, token_name in token_type_values 89 | ] 90 | 91 | 92 | if __name__ == '__main__': 93 | unittest.main() 94 | -------------------------------------------------------------------------------- /yapftests/main_test.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Tests for yapf.__init__.main.""" 16 | 17 | import sys 18 | import unittest 19 | from contextlib import contextmanager 20 | from io import StringIO 21 | 22 | import yapf 23 | 24 | from yapftests import yapf_test_helper 25 | 26 | 27 | class IO(object): 28 | """IO is a thin wrapper around StringIO. 29 | 30 | This is strictly to wrap the Python 3 StringIO object so that it can supply a 31 | "buffer" attribute. 32 | """ 33 | 34 | class Buffer(object): 35 | 36 | def __init__(self): 37 | self.string_io = StringIO() 38 | 39 | def write(self, s): 40 | if isinstance(s, bytes): 41 | s = str(s, 'utf-8') 42 | self.string_io.write(s) 43 | 44 | def getvalue(self): 45 | return self.string_io.getvalue() 46 | 47 | def __init__(self): 48 | self.buffer = self.Buffer() 49 | 50 | def write(self, s): 51 | self.buffer.write(s) 52 | 53 | def getvalue(self): 54 | return self.buffer.getvalue() 55 | 56 | 57 | @contextmanager 58 | def captured_output(): 59 | new_out, new_err = IO(), IO() 60 | old_out, old_err = sys.stdout, sys.stderr 61 | try: 62 | sys.stdout, sys.stderr = new_out, new_err 63 | yield sys.stdout, sys.stderr 64 | finally: 65 | sys.stdout, sys.stderr = old_out, old_err 66 | 67 | 68 | @contextmanager 69 | def patched_input(code): 70 | """Monkey patch code as though it were coming from stdin.""" 71 | 72 | def lines(): 73 | for line in code.splitlines(): 74 | yield line 75 | raise EOFError() 76 | 77 | def patch_raw_input(lines=lines()): 78 | return next(lines) 79 | 80 | try: 81 | orig_raw_import = yapf._raw_input 82 | yapf._raw_input = patch_raw_input 83 | yield 84 | finally: 85 | yapf._raw_input = orig_raw_import 86 | 87 | 88 | class RunMainTest(yapf_test_helper.YAPFTest): 89 | 90 | def testShouldHandleYapfError(self): 91 | """run_main should handle YapfError and sys.exit(1).""" 92 | expected_message = 'yapf: input filenames did not match any python files\n' 93 | sys.argv = ['yapf', 'foo.c'] 94 | with captured_output() as (out, err): 95 | with self.assertRaises(SystemExit): 96 | yapf.run_main() 97 | self.assertEqual(out.getvalue(), '') 98 | self.assertEqual(err.getvalue(), expected_message) 99 | 100 | 101 | class MainTest(yapf_test_helper.YAPFTest): 102 | 103 | def testNoPythonFilesMatched(self): 104 | with self.assertRaisesRegex(yapf.errors.YapfError, 105 | 'did not match any python files'): 106 | yapf.main(['yapf', 'foo.c']) 107 | 108 | def testEchoInput(self): 109 | code = 'a = 1\nb = 2\n' 110 | with patched_input(code): 111 | with captured_output() as (out, _): 112 | ret = yapf.main([]) 113 | self.assertEqual(ret, 0) 114 | self.assertEqual(out.getvalue(), code) 115 | 116 | def testEchoInputWithStyle(self): 117 | code = 'def f(a = 1\n\n):\n return 2*a\n' 118 | yapf_code = 'def f(a=1):\n return 2 * a\n' 119 | with patched_input(code): 120 | with captured_output() as (out, _): 121 | ret = yapf.main(['-', '--style=yapf']) 122 | self.assertEqual(ret, 0) 123 | self.assertEqual(out.getvalue(), yapf_code) 124 | 125 | def testEchoBadInput(self): 126 | bad_syntax = ' a = 1\n' 127 | with patched_input(bad_syntax): 128 | with captured_output() as (_, _): 129 | with self.assertRaisesRegex(yapf.errors.YapfError, 'unexpected indent'): 130 | yapf.main([]) 131 | 132 | def testHelp(self): 133 | with captured_output() as (out, _): 134 | ret = yapf.main(['-', '--style-help', '--style=pep8']) 135 | self.assertEqual(ret, 0) 136 | help_message = out.getvalue() 137 | self.assertIn('indent_width=4', help_message) 138 | self.assertIn('The number of spaces required before a trailing comment.', 139 | help_message) 140 | -------------------------------------------------------------------------------- /yapftests/pytree_unwrapper_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for yapf.pytree_unwrapper.""" 15 | 16 | import textwrap 17 | import unittest 18 | 19 | from yapf.pytree import pytree_utils 20 | 21 | from yapftests import yapf_test_helper 22 | 23 | 24 | class PytreeUnwrapperTest(yapf_test_helper.YAPFTest): 25 | 26 | def _CheckLogicalLines(self, llines, list_of_expected): 27 | """Check that the given LogicalLines match expectations. 28 | 29 | Args: 30 | llines: list of LogicalLine 31 | list_of_expected: list of (depth, values) pairs. Non-semantic tokens are 32 | filtered out from the expected values. 33 | """ 34 | actual = [] 35 | for lline in llines: 36 | filtered_values = [ 37 | ft.value 38 | for ft in lline.tokens 39 | if ft.name not in pytree_utils.NONSEMANTIC_TOKENS 40 | ] 41 | actual.append((lline.depth, filtered_values)) 42 | 43 | self.assertEqual(list_of_expected, actual) 44 | 45 | def testSimpleFileScope(self): 46 | code = textwrap.dedent("""\ 47 | x = 1 48 | # a comment 49 | y = 2 50 | """) 51 | llines = yapf_test_helper.ParseAndUnwrap(code) 52 | self._CheckLogicalLines(llines, [ 53 | (0, ['x', '=', '1']), 54 | (0, ['# a comment']), 55 | (0, ['y', '=', '2']), 56 | ]) 57 | 58 | def testSimpleMultilineStatement(self): 59 | code = textwrap.dedent("""\ 60 | y = (1 + 61 | x) 62 | """) 63 | llines = yapf_test_helper.ParseAndUnwrap(code) 64 | self._CheckLogicalLines(llines, [ 65 | (0, ['y', '=', '(', '1', '+', 'x', ')']), 66 | ]) 67 | 68 | def testFileScopeWithInlineComment(self): 69 | code = textwrap.dedent("""\ 70 | x = 1 # a comment 71 | y = 2 72 | """) 73 | llines = yapf_test_helper.ParseAndUnwrap(code) 74 | self._CheckLogicalLines(llines, [ 75 | (0, ['x', '=', '1', '# a comment']), 76 | (0, ['y', '=', '2']), 77 | ]) 78 | 79 | def testSimpleIf(self): 80 | code = textwrap.dedent("""\ 81 | if foo: 82 | x = 1 83 | y = 2 84 | """) 85 | llines = yapf_test_helper.ParseAndUnwrap(code) 86 | self._CheckLogicalLines(llines, [ 87 | (0, ['if', 'foo', ':']), 88 | (1, ['x', '=', '1']), 89 | (1, ['y', '=', '2']), 90 | ]) 91 | 92 | def testSimpleIfWithComments(self): 93 | code = textwrap.dedent("""\ 94 | # c1 95 | if foo: # c2 96 | x = 1 97 | y = 2 98 | """) 99 | llines = yapf_test_helper.ParseAndUnwrap(code) 100 | self._CheckLogicalLines(llines, [ 101 | (0, ['# c1']), 102 | (0, ['if', 'foo', ':', '# c2']), 103 | (1, ['x', '=', '1']), 104 | (1, ['y', '=', '2']), 105 | ]) 106 | 107 | def testIfWithCommentsInside(self): 108 | code = textwrap.dedent("""\ 109 | if foo: 110 | # c1 111 | x = 1 # c2 112 | # c3 113 | y = 2 114 | """) 115 | llines = yapf_test_helper.ParseAndUnwrap(code) 116 | self._CheckLogicalLines(llines, [ 117 | (0, ['if', 'foo', ':']), 118 | (1, ['# c1']), 119 | (1, ['x', '=', '1', '# c2']), 120 | (1, ['# c3']), 121 | (1, ['y', '=', '2']), 122 | ]) 123 | 124 | def testIfElifElse(self): 125 | code = textwrap.dedent("""\ 126 | if x: 127 | x = 1 # c1 128 | elif y: # c2 129 | y = 1 130 | else: 131 | # c3 132 | z = 1 133 | """) 134 | llines = yapf_test_helper.ParseAndUnwrap(code) 135 | self._CheckLogicalLines(llines, [ 136 | (0, ['if', 'x', ':']), 137 | (1, ['x', '=', '1', '# c1']), 138 | (0, ['elif', 'y', ':', '# c2']), 139 | (1, ['y', '=', '1']), 140 | (0, ['else', ':']), 141 | (1, ['# c3']), 142 | (1, ['z', '=', '1']), 143 | ]) 144 | 145 | def testNestedCompoundTwoLevel(self): 146 | code = textwrap.dedent("""\ 147 | if x: 148 | x = 1 # c1 149 | while t: 150 | # c2 151 | j = 1 152 | k = 1 153 | """) 154 | llines = yapf_test_helper.ParseAndUnwrap(code) 155 | self._CheckLogicalLines(llines, [ 156 | (0, ['if', 'x', ':']), 157 | (1, ['x', '=', '1', '# c1']), 158 | (1, ['while', 't', ':']), 159 | (2, ['# c2']), 160 | (2, ['j', '=', '1']), 161 | (1, ['k', '=', '1']), 162 | ]) 163 | 164 | def testSimpleWhile(self): 165 | code = textwrap.dedent("""\ 166 | while x > 1: # c1 167 | # c2 168 | x = 1 169 | """) 170 | llines = yapf_test_helper.ParseAndUnwrap(code) 171 | self._CheckLogicalLines(llines, [ 172 | (0, ['while', 'x', '>', '1', ':', '# c1']), 173 | (1, ['# c2']), 174 | (1, ['x', '=', '1']), 175 | ]) 176 | 177 | def testSimpleTry(self): 178 | code = textwrap.dedent("""\ 179 | try: 180 | pass 181 | except: 182 | pass 183 | except: 184 | pass 185 | else: 186 | pass 187 | finally: 188 | pass 189 | """) 190 | llines = yapf_test_helper.ParseAndUnwrap(code) 191 | self._CheckLogicalLines(llines, [ 192 | (0, ['try', ':']), 193 | (1, ['pass']), 194 | (0, ['except', ':']), 195 | (1, ['pass']), 196 | (0, ['except', ':']), 197 | (1, ['pass']), 198 | (0, ['else', ':']), 199 | (1, ['pass']), 200 | (0, ['finally', ':']), 201 | (1, ['pass']), 202 | ]) 203 | 204 | def testSimpleFuncdef(self): 205 | code = textwrap.dedent("""\ 206 | def foo(x): # c1 207 | # c2 208 | return x 209 | """) 210 | llines = yapf_test_helper.ParseAndUnwrap(code) 211 | self._CheckLogicalLines(llines, [ 212 | (0, ['def', 'foo', '(', 'x', ')', ':', '# c1']), 213 | (1, ['# c2']), 214 | (1, ['return', 'x']), 215 | ]) 216 | 217 | def testTwoFuncDefs(self): 218 | code = textwrap.dedent("""\ 219 | def foo(x): # c1 220 | # c2 221 | return x 222 | 223 | def bar(): # c3 224 | # c4 225 | return x 226 | """) 227 | llines = yapf_test_helper.ParseAndUnwrap(code) 228 | self._CheckLogicalLines(llines, [ 229 | (0, ['def', 'foo', '(', 'x', ')', ':', '# c1']), 230 | (1, ['# c2']), 231 | (1, ['return', 'x']), 232 | (0, ['def', 'bar', '(', ')', ':', '# c3']), 233 | (1, ['# c4']), 234 | (1, ['return', 'x']), 235 | ]) 236 | 237 | def testSimpleClassDef(self): 238 | code = textwrap.dedent("""\ 239 | class Klass: # c1 240 | # c2 241 | p = 1 242 | """) 243 | llines = yapf_test_helper.ParseAndUnwrap(code) 244 | self._CheckLogicalLines(llines, [ 245 | (0, ['class', 'Klass', ':', '# c1']), 246 | (1, ['# c2']), 247 | (1, ['p', '=', '1']), 248 | ]) 249 | 250 | def testSingleLineStmtInFunc(self): 251 | code = textwrap.dedent("""\ 252 | def f(): return 37 253 | """) 254 | llines = yapf_test_helper.ParseAndUnwrap(code) 255 | self._CheckLogicalLines(llines, [ 256 | (0, ['def', 'f', '(', ')', ':']), 257 | (1, ['return', '37']), 258 | ]) 259 | 260 | def testMultipleComments(self): 261 | code = textwrap.dedent("""\ 262 | # Comment #1 263 | 264 | # Comment #2 265 | def f(): 266 | pass 267 | """) 268 | llines = yapf_test_helper.ParseAndUnwrap(code) 269 | self._CheckLogicalLines(llines, [ 270 | (0, ['# Comment #1']), 271 | (0, ['# Comment #2']), 272 | (0, ['def', 'f', '(', ')', ':']), 273 | (1, ['pass']), 274 | ]) 275 | 276 | def testSplitListWithComment(self): 277 | code = textwrap.dedent("""\ 278 | a = [ 279 | 'a', 280 | 'b', 281 | 'c', # hello world 282 | ] 283 | """) 284 | llines = yapf_test_helper.ParseAndUnwrap(code) 285 | self._CheckLogicalLines(llines, [(0, [ 286 | 'a', '=', '[', "'a'", ',', "'b'", ',', "'c'", ',', '# hello world', ']' 287 | ])]) 288 | 289 | 290 | class MatchBracketsTest(yapf_test_helper.YAPFTest): 291 | 292 | def _CheckMatchingBrackets(self, llines, list_of_expected): 293 | """Check that the tokens have the expected matching bracket. 294 | 295 | Arguments: 296 | llines: list of LogicalLine. 297 | list_of_expected: list of (index, index) pairs. The matching brackets at 298 | the indexes need to match. Non-semantic tokens are filtered out from the 299 | expected values. 300 | """ 301 | actual = [] 302 | for lline in llines: 303 | filtered_values = [(ft, ft.matching_bracket) 304 | for ft in lline.tokens 305 | if ft.name not in pytree_utils.NONSEMANTIC_TOKENS] 306 | if filtered_values: 307 | actual.append(filtered_values) 308 | 309 | for index, bracket_list in enumerate(list_of_expected): 310 | lline = actual[index] 311 | if not bracket_list: 312 | for value in lline: 313 | self.assertIsNone(value[1]) 314 | else: 315 | for open_bracket, close_bracket in bracket_list: 316 | self.assertEqual(lline[open_bracket][0], lline[close_bracket][1]) 317 | self.assertEqual(lline[close_bracket][0], lline[open_bracket][1]) 318 | 319 | def testFunctionDef(self): 320 | code = textwrap.dedent("""\ 321 | def foo(a, b=['w','d'], c=[42, 37]): 322 | pass 323 | """) 324 | llines = yapf_test_helper.ParseAndUnwrap(code) 325 | self._CheckMatchingBrackets(llines, [ 326 | [(2, 20), (7, 11), (15, 19)], 327 | [], 328 | ]) 329 | 330 | def testDecorator(self): 331 | code = textwrap.dedent("""\ 332 | @bar() 333 | def foo(a, b, c): 334 | pass 335 | """) 336 | llines = yapf_test_helper.ParseAndUnwrap(code) 337 | self._CheckMatchingBrackets(llines, [ 338 | [(2, 3)], 339 | [(2, 8)], 340 | [], 341 | ]) 342 | 343 | def testClassDef(self): 344 | code = textwrap.dedent("""\ 345 | class A(B, C, D): 346 | pass 347 | """) 348 | llines = yapf_test_helper.ParseAndUnwrap(code) 349 | self._CheckMatchingBrackets(llines, [ 350 | [(2, 8)], 351 | [], 352 | ]) 353 | 354 | 355 | if __name__ == '__main__': 356 | unittest.main() 357 | -------------------------------------------------------------------------------- /yapftests/pytree_utils_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for yapf.pytree_utils.""" 15 | 16 | import unittest 17 | 18 | from yapf_third_party._ylib2to3 import pygram 19 | from yapf_third_party._ylib2to3 import pytree 20 | from yapf_third_party._ylib2to3.pgen2 import token 21 | 22 | from yapf.pytree import pytree_utils 23 | 24 | from yapftests import yapf_test_helper 25 | 26 | # More direct access to the symbol->number mapping living within the grammar 27 | # module. 28 | _GRAMMAR_SYMBOL2NUMBER = pygram.python_grammar.symbol2number 29 | 30 | _FOO = 'foo' 31 | _FOO1 = 'foo1' 32 | _FOO2 = 'foo2' 33 | _FOO3 = 'foo3' 34 | _FOO4 = 'foo4' 35 | _FOO5 = 'foo5' 36 | 37 | 38 | class NodeNameTest(yapf_test_helper.YAPFTest): 39 | 40 | def testNodeNameForLeaf(self): 41 | leaf = pytree.Leaf(token.LPAR, '(') 42 | self.assertEqual('LPAR', pytree_utils.NodeName(leaf)) 43 | 44 | def testNodeNameForNode(self): 45 | leaf = pytree.Leaf(token.LPAR, '(') 46 | node = pytree.Node(pygram.python_grammar.symbol2number['suite'], [leaf]) 47 | self.assertEqual('suite', pytree_utils.NodeName(node)) 48 | 49 | 50 | class ParseCodeToTreeTest(yapf_test_helper.YAPFTest): 51 | 52 | def testParseCodeToTree(self): 53 | # Since ParseCodeToTree is a thin wrapper around underlying lib2to3 54 | # functionality, only a sanity test here... 55 | tree = pytree_utils.ParseCodeToTree('foo = 2\n') 56 | self.assertEqual('file_input', pytree_utils.NodeName(tree)) 57 | self.assertEqual(2, len(tree.children)) 58 | self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0])) 59 | 60 | def testPrintFunctionToTree(self): 61 | tree = pytree_utils.ParseCodeToTree( 62 | 'print("hello world", file=sys.stderr)\n') 63 | self.assertEqual('file_input', pytree_utils.NodeName(tree)) 64 | self.assertEqual(2, len(tree.children)) 65 | self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0])) 66 | 67 | def testPrintStatementToTree(self): 68 | with self.assertRaises(SyntaxError): 69 | pytree_utils.ParseCodeToTree('print "hello world"\n') 70 | 71 | def testClassNotLocal(self): 72 | with self.assertRaises(SyntaxError): 73 | pytree_utils.ParseCodeToTree('class nonlocal: pass\n') 74 | 75 | 76 | class InsertNodesBeforeAfterTest(yapf_test_helper.YAPFTest): 77 | 78 | def _BuildSimpleTree(self): 79 | # Builds a simple tree we can play with in the tests. 80 | # The tree looks like this: 81 | # 82 | # suite: 83 | # LPAR 84 | # LPAR 85 | # simple_stmt: 86 | # NAME('foo') 87 | # 88 | lpar1 = pytree.Leaf(token.LPAR, '(') 89 | lpar2 = pytree.Leaf(token.LPAR, '(') 90 | simple_stmt = pytree.Node(_GRAMMAR_SYMBOL2NUMBER['simple_stmt'], 91 | [pytree.Leaf(token.NAME, 'foo')]) 92 | return pytree.Node(_GRAMMAR_SYMBOL2NUMBER['suite'], 93 | [lpar1, lpar2, simple_stmt]) 94 | 95 | def _MakeNewNodeRPAR(self): 96 | return pytree.Leaf(token.RPAR, ')') 97 | 98 | def setUp(self): 99 | self._simple_tree = self._BuildSimpleTree() 100 | 101 | def testInsertNodesBefore(self): 102 | # Insert before simple_stmt and make sure it went to the right place 103 | pytree_utils.InsertNodesBefore([self._MakeNewNodeRPAR()], 104 | self._simple_tree.children[2]) 105 | self.assertEqual(4, len(self._simple_tree.children)) 106 | self.assertEqual('RPAR', 107 | pytree_utils.NodeName(self._simple_tree.children[2])) 108 | self.assertEqual('simple_stmt', 109 | pytree_utils.NodeName(self._simple_tree.children[3])) 110 | 111 | def testInsertNodesBeforeFirstChild(self): 112 | # Insert before the first child of its parent 113 | simple_stmt = self._simple_tree.children[2] 114 | foo_child = simple_stmt.children[0] 115 | pytree_utils.InsertNodesBefore([self._MakeNewNodeRPAR()], foo_child) 116 | self.assertEqual(3, len(self._simple_tree.children)) 117 | self.assertEqual(2, len(simple_stmt.children)) 118 | self.assertEqual('RPAR', pytree_utils.NodeName(simple_stmt.children[0])) 119 | self.assertEqual('NAME', pytree_utils.NodeName(simple_stmt.children[1])) 120 | 121 | def testInsertNodesAfter(self): 122 | # Insert after and make sure it went to the right place 123 | pytree_utils.InsertNodesAfter([self._MakeNewNodeRPAR()], 124 | self._simple_tree.children[2]) 125 | self.assertEqual(4, len(self._simple_tree.children)) 126 | self.assertEqual('simple_stmt', 127 | pytree_utils.NodeName(self._simple_tree.children[2])) 128 | self.assertEqual('RPAR', 129 | pytree_utils.NodeName(self._simple_tree.children[3])) 130 | 131 | def testInsertNodesAfterLastChild(self): 132 | # Insert after the last child of its parent 133 | simple_stmt = self._simple_tree.children[2] 134 | foo_child = simple_stmt.children[0] 135 | pytree_utils.InsertNodesAfter([self._MakeNewNodeRPAR()], foo_child) 136 | self.assertEqual(3, len(self._simple_tree.children)) 137 | self.assertEqual(2, len(simple_stmt.children)) 138 | self.assertEqual('NAME', pytree_utils.NodeName(simple_stmt.children[0])) 139 | self.assertEqual('RPAR', pytree_utils.NodeName(simple_stmt.children[1])) 140 | 141 | def testInsertNodesWhichHasParent(self): 142 | # Try to insert an existing tree node into another place and fail. 143 | with self.assertRaises(RuntimeError): 144 | pytree_utils.InsertNodesAfter([self._simple_tree.children[1]], 145 | self._simple_tree.children[0]) 146 | 147 | 148 | class AnnotationsTest(yapf_test_helper.YAPFTest): 149 | 150 | def setUp(self): 151 | self._leaf = pytree.Leaf(token.LPAR, '(') 152 | self._node = pytree.Node(_GRAMMAR_SYMBOL2NUMBER['simple_stmt'], 153 | [pytree.Leaf(token.NAME, 'foo')]) 154 | 155 | def testGetWhenNone(self): 156 | self.assertIsNone(pytree_utils.GetNodeAnnotation(self._leaf, _FOO)) 157 | 158 | def testSetWhenNone(self): 159 | pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 20) 160 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 20) 161 | 162 | def testSetAgain(self): 163 | pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 20) 164 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 20) 165 | pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 30) 166 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 30) 167 | 168 | def testMultiple(self): 169 | pytree_utils.SetNodeAnnotation(self._leaf, _FOO, 20) 170 | pytree_utils.SetNodeAnnotation(self._leaf, _FOO1, 1) 171 | pytree_utils.SetNodeAnnotation(self._leaf, _FOO2, 2) 172 | pytree_utils.SetNodeAnnotation(self._leaf, _FOO3, 3) 173 | pytree_utils.SetNodeAnnotation(self._leaf, _FOO4, 4) 174 | pytree_utils.SetNodeAnnotation(self._leaf, _FOO5, 5) 175 | 176 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO), 20) 177 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO1), 1) 178 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO2), 2) 179 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO3), 3) 180 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO4), 4) 181 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._leaf, _FOO5), 5) 182 | 183 | def testSubtype(self): 184 | pytree_utils.AppendNodeAnnotation(self._leaf, 185 | pytree_utils.Annotation.SUBTYPE, _FOO) 186 | 187 | self.assertSetEqual( 188 | pytree_utils.GetNodeAnnotation(self._leaf, 189 | pytree_utils.Annotation.SUBTYPE), {_FOO}) 190 | 191 | pytree_utils.RemoveSubtypeAnnotation(self._leaf, _FOO) 192 | 193 | self.assertSetEqual( 194 | pytree_utils.GetNodeAnnotation(self._leaf, 195 | pytree_utils.Annotation.SUBTYPE), set()) 196 | 197 | def testSetOnNode(self): 198 | pytree_utils.SetNodeAnnotation(self._node, _FOO, 20) 199 | self.assertEqual(pytree_utils.GetNodeAnnotation(self._node, _FOO), 20) 200 | 201 | 202 | if __name__ == '__main__': 203 | unittest.main() 204 | -------------------------------------------------------------------------------- /yapftests/pytree_visitor_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for yapf.pytree_visitor.""" 15 | 16 | import unittest 17 | from io import StringIO 18 | 19 | from yapf.pytree import pytree_utils 20 | from yapf.pytree import pytree_visitor 21 | 22 | from yapftests import yapf_test_helper 23 | 24 | 25 | class _NodeNameCollector(pytree_visitor.PyTreeVisitor): 26 | """A tree visitor that collects the names of all tree nodes into a list. 27 | 28 | Attributes: 29 | all_node_names: collected list of the names, available when the traversal 30 | is over. 31 | name_node_values: collects a list of NAME leaves (in addition to those going 32 | into all_node_names). 33 | """ 34 | 35 | def __init__(self): 36 | self.all_node_names = [] 37 | self.name_node_values = [] 38 | 39 | def DefaultNodeVisit(self, node): 40 | self.all_node_names.append(pytree_utils.NodeName(node)) 41 | super(_NodeNameCollector, self).DefaultNodeVisit(node) 42 | 43 | def DefaultLeafVisit(self, leaf): 44 | self.all_node_names.append(pytree_utils.NodeName(leaf)) 45 | 46 | def Visit_NAME(self, leaf): 47 | self.name_node_values.append(leaf.value) 48 | self.DefaultLeafVisit(leaf) 49 | 50 | 51 | _VISITOR_TEST_SIMPLE_CODE = """\ 52 | foo = bar 53 | baz = x 54 | """ 55 | 56 | _VISITOR_TEST_NESTED_CODE = """\ 57 | if x: 58 | if y: 59 | return z 60 | """ 61 | 62 | 63 | class PytreeVisitorTest(yapf_test_helper.YAPFTest): 64 | 65 | def testCollectAllNodeNamesSimpleCode(self): 66 | tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE) 67 | collector = _NodeNameCollector() 68 | collector.Visit(tree) 69 | expected_names = [ 70 | 'file_input', 71 | 'simple_stmt', 'expr_stmt', 'NAME', 'EQUAL', 'NAME', 'NEWLINE', 72 | 'simple_stmt', 'expr_stmt', 'NAME', 'EQUAL', 'NAME', 'NEWLINE', 73 | 'ENDMARKER', 74 | ] # yapf: disable 75 | self.assertEqual(expected_names, collector.all_node_names) 76 | 77 | expected_name_node_values = ['foo', 'bar', 'baz', 'x'] 78 | self.assertEqual(expected_name_node_values, collector.name_node_values) 79 | 80 | def testCollectAllNodeNamesNestedCode(self): 81 | tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_NESTED_CODE) 82 | collector = _NodeNameCollector() 83 | collector.Visit(tree) 84 | expected_names = [ 85 | 'file_input', 86 | 'if_stmt', 'NAME', 'NAME', 'COLON', 87 | 'suite', 'NEWLINE', 88 | 'INDENT', 'if_stmt', 'NAME', 'NAME', 'COLON', 'suite', 'NEWLINE', 89 | 'INDENT', 'simple_stmt', 'return_stmt', 'NAME', 'NAME', 'NEWLINE', 90 | 'DEDENT', 'DEDENT', 'ENDMARKER', 91 | ] # yapf: disable 92 | self.assertEqual(expected_names, collector.all_node_names) 93 | 94 | expected_name_node_values = ['if', 'x', 'if', 'y', 'return', 'z'] 95 | self.assertEqual(expected_name_node_values, collector.name_node_values) 96 | 97 | def testDumper(self): 98 | # PyTreeDumper is mainly a debugging utility, so only do basic sanity 99 | # checking. 100 | tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE) 101 | stream = StringIO() 102 | pytree_visitor.PyTreeDumper(target_stream=stream).Visit(tree) 103 | 104 | dump_output = stream.getvalue() 105 | self.assertIn('file_input [3 children]', dump_output) 106 | self.assertIn("NAME(Leaf(NAME, 'foo'))", dump_output) 107 | self.assertIn("EQUAL(Leaf(EQUAL, '='))", dump_output) 108 | 109 | def testDumpPyTree(self): 110 | # Similar sanity checking for the convenience wrapper DumpPyTree 111 | tree = pytree_utils.ParseCodeToTree(_VISITOR_TEST_SIMPLE_CODE) 112 | stream = StringIO() 113 | pytree_visitor.DumpPyTree(tree, target_stream=stream) 114 | 115 | dump_output = stream.getvalue() 116 | self.assertIn('file_input [3 children]', dump_output) 117 | self.assertIn("NAME(Leaf(NAME, 'foo'))", dump_output) 118 | self.assertIn("EQUAL(Leaf(EQUAL, '='))", dump_output) 119 | 120 | 121 | if __name__ == '__main__': 122 | unittest.main() 123 | -------------------------------------------------------------------------------- /yapftests/reformatter_style_config_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Style config tests for yapf.reformatter.""" 15 | 16 | import textwrap 17 | import unittest 18 | 19 | from yapf.yapflib import reformatter 20 | from yapf.yapflib import style 21 | 22 | from yapftests import yapf_test_helper 23 | 24 | 25 | class TestsForStyleConfig(yapf_test_helper.YAPFTest): 26 | 27 | def setUp(self): 28 | self.current_style = style.DEFAULT_STYLE 29 | 30 | def testSetGlobalStyle(self): 31 | try: 32 | style.SetGlobalStyle(style.CreateYapfStyle()) 33 | unformatted_code = textwrap.dedent("""\ 34 | for i in range(5): 35 | print('bar') 36 | """) 37 | expected_formatted_code = textwrap.dedent("""\ 38 | for i in range(5): 39 | print('bar') 40 | """) 41 | llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) 42 | self.assertCodeEqual(expected_formatted_code, 43 | reformatter.Reformat(llines)) 44 | finally: 45 | style.SetGlobalStyle(style.CreatePEP8Style()) 46 | style.DEFAULT_STYLE = self.current_style 47 | 48 | unformatted_code = textwrap.dedent("""\ 49 | for i in range(5): 50 | print('bar') 51 | """) 52 | expected_formatted_code = textwrap.dedent("""\ 53 | for i in range(5): 54 | print('bar') 55 | """) 56 | llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) 57 | self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) 58 | 59 | def testOperatorNoSpaceStyle(self): 60 | try: 61 | sympy_style = style.CreatePEP8Style() 62 | sympy_style['NO_SPACES_AROUND_SELECTED_BINARY_OPERATORS'] = \ 63 | style._StringSetConverter('*,/') 64 | style.SetGlobalStyle(sympy_style) 65 | unformatted_code = textwrap.dedent("""\ 66 | a = 1+2 * 3 - 4 / 5 67 | b = '0' * 1 68 | """) 69 | expected_formatted_code = textwrap.dedent("""\ 70 | a = 1 + 2*3 - 4/5 71 | b = '0'*1 72 | """) 73 | 74 | llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) 75 | self.assertCodeEqual(expected_formatted_code, 76 | reformatter.Reformat(llines)) 77 | finally: 78 | style.SetGlobalStyle(style.CreatePEP8Style()) 79 | style.DEFAULT_STYLE = self.current_style 80 | 81 | def testOperatorPrecedenceStyle(self): 82 | try: 83 | pep8_with_precedence = style.CreatePEP8Style() 84 | pep8_with_precedence['ARITHMETIC_PRECEDENCE_INDICATION'] = True 85 | style.SetGlobalStyle(pep8_with_precedence) 86 | unformatted_code = textwrap.dedent("""\ 87 | 1+2 88 | (1 + 2) * (3 - (4 / 5)) 89 | a = 1 * 2 + 3 / 4 90 | b = 1 / 2 - 3 * 4 91 | c = (1 + 2) * (3 - 4) 92 | d = (1 - 2) / (3 + 4) 93 | e = 1 * 2 - 3 94 | f = 1 + 2 + 3 + 4 95 | g = 1 * 2 * 3 * 4 96 | h = 1 + 2 - 3 + 4 97 | i = 1 * 2 / 3 * 4 98 | j = (1 * 2 - 3) + 4 99 | k = (1 * 2 * 3) + (4 * 5 * 6 * 7 * 8) 100 | """) 101 | expected_formatted_code = textwrap.dedent("""\ 102 | 1 + 2 103 | (1+2) * (3 - (4/5)) 104 | a = 1*2 + 3/4 105 | b = 1/2 - 3*4 106 | c = (1+2) * (3-4) 107 | d = (1-2) / (3+4) 108 | e = 1*2 - 3 109 | f = 1 + 2 + 3 + 4 110 | g = 1 * 2 * 3 * 4 111 | h = 1 + 2 - 3 + 4 112 | i = 1 * 2 / 3 * 4 113 | j = (1*2 - 3) + 4 114 | k = (1*2*3) + (4*5*6*7*8) 115 | """) 116 | 117 | llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) 118 | self.assertCodeEqual(expected_formatted_code, 119 | reformatter.Reformat(llines)) 120 | finally: 121 | style.SetGlobalStyle(style.CreatePEP8Style()) 122 | style.DEFAULT_STYLE = self.current_style 123 | 124 | def testNoSplitBeforeFirstArgumentStyle1(self): 125 | try: 126 | pep8_no_split_before_first = style.CreatePEP8Style() 127 | pep8_no_split_before_first['SPLIT_BEFORE_FIRST_ARGUMENT'] = False 128 | pep8_no_split_before_first['SPLIT_BEFORE_NAMED_ASSIGNS'] = False 129 | style.SetGlobalStyle(pep8_no_split_before_first) 130 | formatted_code = textwrap.dedent("""\ 131 | # Example from in-code MustSplit comments 132 | foo = outer_function_call(fitting_inner_function_call(inner_arg1, inner_arg2), 133 | outer_arg1, outer_arg2) 134 | 135 | foo = outer_function_call( 136 | not_fitting_inner_function_call(inner_arg1, inner_arg2), outer_arg1, 137 | outer_arg2) 138 | 139 | # Examples Issue#424 140 | a_super_long_version_of_print(argument1, argument2, argument3, argument4, 141 | argument5, argument6, argument7) 142 | 143 | CREDS_FILE = os.path.join(os.path.expanduser('~'), 144 | 'apis/super-secret-admin-creds.json') 145 | 146 | # Examples Issue#556 147 | i_take_a_lot_of_params(arg1, param1=very_long_expression1(), 148 | param2=very_long_expression2(), 149 | param3=very_long_expression3(), 150 | param4=very_long_expression4()) 151 | 152 | # Examples Issue#590 153 | plt.plot(numpy.linspace(0, 1, 10), numpy.linspace(0, 1, 10), marker="x", 154 | color="r") 155 | 156 | plt.plot(veryverylongvariablename, veryverylongvariablename, marker="x", 157 | color="r") 158 | """) # noqa 159 | llines = yapf_test_helper.ParseAndUnwrap(formatted_code) 160 | self.assertCodeEqual(formatted_code, reformatter.Reformat(llines)) 161 | finally: 162 | style.SetGlobalStyle(style.CreatePEP8Style()) 163 | style.DEFAULT_STYLE = self.current_style 164 | 165 | def testNoSplitBeforeFirstArgumentStyle2(self): 166 | try: 167 | pep8_no_split_before_first = style.CreatePEP8Style() 168 | pep8_no_split_before_first['SPLIT_BEFORE_FIRST_ARGUMENT'] = False 169 | pep8_no_split_before_first['SPLIT_BEFORE_NAMED_ASSIGNS'] = True 170 | style.SetGlobalStyle(pep8_no_split_before_first) 171 | formatted_code = textwrap.dedent("""\ 172 | # Examples Issue#556 173 | i_take_a_lot_of_params(arg1, 174 | param1=very_long_expression1(), 175 | param2=very_long_expression2(), 176 | param3=very_long_expression3(), 177 | param4=very_long_expression4()) 178 | 179 | # Examples Issue#590 180 | plt.plot(numpy.linspace(0, 1, 10), 181 | numpy.linspace(0, 1, 10), 182 | marker="x", 183 | color="r") 184 | 185 | plt.plot(veryverylongvariablename, 186 | veryverylongvariablename, 187 | marker="x", 188 | color="r") 189 | """) 190 | llines = yapf_test_helper.ParseAndUnwrap(formatted_code) 191 | self.assertCodeEqual(formatted_code, reformatter.Reformat(llines)) 192 | finally: 193 | style.SetGlobalStyle(style.CreatePEP8Style()) 194 | style.DEFAULT_STYLE = self.current_style 195 | 196 | 197 | if __name__ == '__main__': 198 | unittest.main() 199 | -------------------------------------------------------------------------------- /yapftests/split_penalty_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Tests for yapf.split_penalty.""" 15 | 16 | import sys 17 | import textwrap 18 | import unittest 19 | 20 | from yapf_third_party._ylib2to3 import pytree 21 | 22 | from yapf.pytree import pytree_utils 23 | from yapf.pytree import pytree_visitor 24 | from yapf.pytree import split_penalty 25 | from yapf.yapflib import style 26 | 27 | from yapftests import yapf_test_helper 28 | 29 | UNBREAKABLE = split_penalty.UNBREAKABLE 30 | VERY_STRONGLY_CONNECTED = split_penalty.VERY_STRONGLY_CONNECTED 31 | DOTTED_NAME = split_penalty.DOTTED_NAME 32 | STRONGLY_CONNECTED = split_penalty.STRONGLY_CONNECTED 33 | 34 | 35 | class SplitPenaltyTest(yapf_test_helper.YAPFTest): 36 | 37 | @classmethod 38 | def setUpClass(cls): 39 | style.SetGlobalStyle(style.CreateYapfStyle()) 40 | 41 | def _ParseAndComputePenalties(self, code, dumptree=False): 42 | """Parses the code and computes split penalties. 43 | 44 | Arguments: 45 | code: code to parse as a string 46 | dumptree: if True, the parsed pytree (after penalty assignment) is dumped 47 | to stderr. Useful for debugging. 48 | 49 | Returns: 50 | Parse tree. 51 | """ 52 | tree = pytree_utils.ParseCodeToTree(code) 53 | split_penalty.ComputeSplitPenalties(tree) 54 | if dumptree: 55 | pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) 56 | return tree 57 | 58 | def _CheckPenalties(self, tree, list_of_expected): 59 | """Check that the tokens in the tree have the correct penalties. 60 | 61 | Args: 62 | tree: the pytree. 63 | list_of_expected: list of (name, penalty) pairs. Non-semantic tokens are 64 | filtered out from the expected values. 65 | """ 66 | 67 | def FlattenRec(tree): 68 | if pytree_utils.NodeName(tree) in pytree_utils.NONSEMANTIC_TOKENS: 69 | return [] 70 | if isinstance(tree, pytree.Leaf): 71 | return [(tree.value, 72 | pytree_utils.GetNodeAnnotation( 73 | tree, pytree_utils.Annotation.SPLIT_PENALTY))] 74 | nodes = [] 75 | for node in tree.children: 76 | nodes += FlattenRec(node) 77 | return nodes 78 | 79 | self.assertEqual(list_of_expected, FlattenRec(tree)) 80 | 81 | def testUnbreakable(self): 82 | # Test function definitions. 83 | code = textwrap.dedent("""\ 84 | def foo(x): 85 | pass 86 | """) 87 | tree = self._ParseAndComputePenalties(code) 88 | self._CheckPenalties(tree, [ 89 | ('def', None), 90 | ('foo', UNBREAKABLE), 91 | ('(', UNBREAKABLE), 92 | ('x', None), 93 | (')', STRONGLY_CONNECTED), 94 | (':', UNBREAKABLE), 95 | ('pass', None), 96 | ]) 97 | 98 | # Test function definition with trailing comment. 99 | code = textwrap.dedent("""\ 100 | def foo(x): # trailing comment 101 | pass 102 | """) 103 | tree = self._ParseAndComputePenalties(code) 104 | self._CheckPenalties(tree, [ 105 | ('def', None), 106 | ('foo', UNBREAKABLE), 107 | ('(', UNBREAKABLE), 108 | ('x', None), 109 | (')', STRONGLY_CONNECTED), 110 | (':', UNBREAKABLE), 111 | ('pass', None), 112 | ]) 113 | 114 | # Test class definitions. 115 | code = textwrap.dedent("""\ 116 | class A: 117 | pass 118 | class B(A): 119 | pass 120 | """) 121 | tree = self._ParseAndComputePenalties(code) 122 | self._CheckPenalties(tree, [ 123 | ('class', None), 124 | ('A', UNBREAKABLE), 125 | (':', UNBREAKABLE), 126 | ('pass', None), 127 | ('class', None), 128 | ('B', UNBREAKABLE), 129 | ('(', UNBREAKABLE), 130 | ('A', None), 131 | (')', None), 132 | (':', UNBREAKABLE), 133 | ('pass', None), 134 | ]) 135 | 136 | # Test lambda definitions. 137 | code = textwrap.dedent("""\ 138 | lambda a, b: None 139 | """) 140 | tree = self._ParseAndComputePenalties(code) 141 | self._CheckPenalties(tree, [ 142 | ('lambda', None), 143 | ('a', VERY_STRONGLY_CONNECTED), 144 | (',', VERY_STRONGLY_CONNECTED), 145 | ('b', VERY_STRONGLY_CONNECTED), 146 | (':', VERY_STRONGLY_CONNECTED), 147 | ('None', VERY_STRONGLY_CONNECTED), 148 | ]) 149 | 150 | # Test dotted names. 151 | code = textwrap.dedent("""\ 152 | import a.b.c 153 | """) 154 | tree = self._ParseAndComputePenalties(code) 155 | self._CheckPenalties(tree, [ 156 | ('import', None), 157 | ('a', None), 158 | ('.', UNBREAKABLE), 159 | ('b', UNBREAKABLE), 160 | ('.', UNBREAKABLE), 161 | ('c', UNBREAKABLE), 162 | ]) 163 | 164 | def testStronglyConnected(self): 165 | # Test dictionary keys. 166 | code = textwrap.dedent("""\ 167 | a = { 168 | 'x': 42, 169 | y(lambda a: 23): 37, 170 | } 171 | """) 172 | tree = self._ParseAndComputePenalties(code) 173 | self._CheckPenalties(tree, [ 174 | ('a', None), 175 | ('=', None), 176 | ('{', None), 177 | ("'x'", None), 178 | (':', STRONGLY_CONNECTED), 179 | ('42', None), 180 | (',', None), 181 | ('y', None), 182 | ('(', UNBREAKABLE), 183 | ('lambda', STRONGLY_CONNECTED), 184 | ('a', VERY_STRONGLY_CONNECTED), 185 | (':', VERY_STRONGLY_CONNECTED), 186 | ('23', VERY_STRONGLY_CONNECTED), 187 | (')', VERY_STRONGLY_CONNECTED), 188 | (':', STRONGLY_CONNECTED), 189 | ('37', None), 190 | (',', None), 191 | ('}', None), 192 | ]) 193 | 194 | # Test list comprehension. 195 | code = textwrap.dedent("""\ 196 | [a for a in foo if a.x == 37] 197 | """) 198 | tree = self._ParseAndComputePenalties(code) 199 | self._CheckPenalties(tree, [ 200 | ('[', None), 201 | ('a', None), 202 | ('for', 0), 203 | ('a', STRONGLY_CONNECTED), 204 | ('in', STRONGLY_CONNECTED), 205 | ('foo', STRONGLY_CONNECTED), 206 | ('if', 0), 207 | ('a', STRONGLY_CONNECTED), 208 | ('.', VERY_STRONGLY_CONNECTED), 209 | ('x', DOTTED_NAME), 210 | ('==', STRONGLY_CONNECTED), 211 | ('37', STRONGLY_CONNECTED), 212 | (']', None), 213 | ]) 214 | 215 | def testFuncCalls(self): 216 | code = textwrap.dedent("""\ 217 | foo(1, 2, 3) 218 | """) 219 | tree = self._ParseAndComputePenalties(code) 220 | self._CheckPenalties(tree, [ 221 | ('foo', None), 222 | ('(', UNBREAKABLE), 223 | ('1', None), 224 | (',', UNBREAKABLE), 225 | ('2', None), 226 | (',', UNBREAKABLE), 227 | ('3', None), 228 | (')', VERY_STRONGLY_CONNECTED), 229 | ]) 230 | 231 | # Now a method call, which has more than one trailer 232 | code = textwrap.dedent("""\ 233 | foo.bar.baz(1, 2, 3) 234 | """) 235 | tree = self._ParseAndComputePenalties(code) 236 | self._CheckPenalties(tree, [ 237 | ('foo', None), 238 | ('.', VERY_STRONGLY_CONNECTED), 239 | ('bar', DOTTED_NAME), 240 | ('.', VERY_STRONGLY_CONNECTED), 241 | ('baz', DOTTED_NAME), 242 | ('(', STRONGLY_CONNECTED), 243 | ('1', None), 244 | (',', UNBREAKABLE), 245 | ('2', None), 246 | (',', UNBREAKABLE), 247 | ('3', None), 248 | (')', VERY_STRONGLY_CONNECTED), 249 | ]) 250 | 251 | # Test single generator argument. 252 | code = textwrap.dedent("""\ 253 | max(i for i in xrange(10)) 254 | """) 255 | tree = self._ParseAndComputePenalties(code) 256 | self._CheckPenalties(tree, [ 257 | ('max', None), 258 | ('(', UNBREAKABLE), 259 | ('i', 0), 260 | ('for', 0), 261 | ('i', STRONGLY_CONNECTED), 262 | ('in', STRONGLY_CONNECTED), 263 | ('xrange', STRONGLY_CONNECTED), 264 | ('(', UNBREAKABLE), 265 | ('10', STRONGLY_CONNECTED), 266 | (')', VERY_STRONGLY_CONNECTED), 267 | (')', VERY_STRONGLY_CONNECTED), 268 | ]) 269 | 270 | 271 | if __name__ == '__main__': 272 | unittest.main() 273 | -------------------------------------------------------------------------------- /yapftests/utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Copyright 2017 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | """Utilities for tests.""" 16 | 17 | import contextlib 18 | import io 19 | import os 20 | import sys 21 | import tempfile 22 | 23 | 24 | @contextlib.contextmanager 25 | def stdout_redirector(stream): # pylint: disable=invalid-name 26 | old_stdout = sys.stdout 27 | sys.stdout = stream 28 | try: 29 | yield 30 | finally: 31 | sys.stdout = old_stdout 32 | 33 | 34 | # NamedTemporaryFile is useless because on Windows the temporary file would be 35 | # created with O_TEMPORARY, which would not allow the file to be opened a 36 | # second time, even by the same process, unless the same flag is used. 37 | # Thus we provide a simplified version ourselves. 38 | # 39 | # Note: returns a tuple of (io.file_obj, file_path), instead of a file_obj with 40 | # a .name attribute 41 | # 42 | # Note: `buffering` is set to -1 despite documentation of NamedTemporaryFile 43 | # says None. This is probably a problem with the python documentation. 44 | @contextlib.contextmanager 45 | def NamedTempFile(mode='w+b', 46 | buffering=-1, 47 | encoding=None, 48 | errors=None, 49 | newline=None, 50 | suffix=None, 51 | prefix=None, 52 | dirname=None, 53 | text=False): 54 | """Context manager creating a new temporary file in text mode.""" 55 | (fd, fname) = tempfile.mkstemp( 56 | suffix=suffix, prefix=prefix, dir=dirname, text=text) 57 | f = io.open( 58 | fd, 59 | mode=mode, 60 | buffering=buffering, 61 | encoding=encoding, 62 | errors=errors, 63 | newline=newline) 64 | yield f, fname 65 | f.close() 66 | os.remove(fname) 67 | 68 | 69 | @contextlib.contextmanager 70 | def TempFileContents(dirname, 71 | contents, 72 | encoding='utf-8', 73 | newline='', 74 | suffix=None): 75 | # Note: NamedTempFile properly handles unicode encoding when using mode='w' 76 | with NamedTempFile( 77 | dirname=dirname, 78 | mode='w', 79 | encoding=encoding, 80 | newline=newline, 81 | suffix=suffix) as (f, fname): 82 | f.write(contents) 83 | f.flush() 84 | yield fname 85 | -------------------------------------------------------------------------------- /yapftests/yapf_test_helper.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | """Support module for tests for yapf.""" 15 | 16 | import difflib 17 | import sys 18 | import unittest 19 | 20 | from yapf.pytree import blank_line_calculator 21 | from yapf.pytree import comment_splicer 22 | from yapf.pytree import continuation_splicer 23 | from yapf.pytree import pytree_unwrapper 24 | from yapf.pytree import pytree_utils 25 | from yapf.pytree import pytree_visitor 26 | from yapf.pytree import split_penalty 27 | from yapf.pytree import subtype_assigner 28 | from yapf.yapflib import identify_container 29 | from yapf.yapflib import style 30 | 31 | 32 | class YAPFTest(unittest.TestCase): 33 | 34 | def __init__(self, *args): 35 | super(YAPFTest, self).__init__(*args) 36 | 37 | def assertCodeEqual(self, expected_code, code): 38 | if code != expected_code: 39 | msg = ['Code format mismatch:', 'Expected:'] 40 | linelen = style.Get('COLUMN_LIMIT') 41 | for line in expected_code.splitlines(): 42 | if len(line) > linelen: 43 | msg.append('!> %s' % line) 44 | else: 45 | msg.append(' > %s' % line) 46 | msg.append('Actual:') 47 | for line in code.splitlines(): 48 | if len(line) > linelen: 49 | msg.append('!> %s' % line) 50 | else: 51 | msg.append(' > %s' % line) 52 | msg.append('Diff:') 53 | msg.extend( 54 | difflib.unified_diff( 55 | code.splitlines(), 56 | expected_code.splitlines(), 57 | fromfile='actual', 58 | tofile='expected', 59 | lineterm='')) 60 | self.fail('\n'.join(msg)) 61 | 62 | 63 | def ParseAndUnwrap(code, dumptree=False): 64 | """Produces logical lines from the given code. 65 | 66 | Parses the code into a tree, performs comment splicing and runs the 67 | unwrapper. 68 | 69 | Arguments: 70 | code: code to parse as a string 71 | dumptree: if True, the parsed pytree (after comment splicing) is dumped 72 | to stderr. Useful for debugging. 73 | 74 | Returns: 75 | List of logical lines. 76 | """ 77 | tree = pytree_utils.ParseCodeToTree(code) 78 | comment_splicer.SpliceComments(tree) 79 | continuation_splicer.SpliceContinuations(tree) 80 | subtype_assigner.AssignSubtypes(tree) 81 | identify_container.IdentifyContainers(tree) 82 | split_penalty.ComputeSplitPenalties(tree) 83 | blank_line_calculator.CalculateBlankLines(tree) 84 | 85 | if dumptree: 86 | pytree_visitor.DumpPyTree(tree, target_stream=sys.stderr) 87 | 88 | llines = pytree_unwrapper.UnwrapPyTree(tree) 89 | for lline in llines: 90 | lline.CalculateFormattingInformation() 91 | 92 | return llines 93 | --------------------------------------------------------------------------------