├── .cargo └── config.toml ├── .editorconfig ├── .fixit.config.yaml ├── .flake8 ├── .gitattributes ├── .github ├── PULL_REQUEST_TEMPLATE.md ├── build-matrix.json ├── dependabot.yml └── workflows │ ├── build.yml │ ├── ci.yml │ ├── pypi_upload.yml │ └── zizmor.yml ├── .gitignore ├── .pyre_configuration ├── .readthedocs.yml ├── .watchmanconfig ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── MAINTAINERS.md ├── MANIFEST.in ├── README.rst ├── apt.txt ├── docs └── source │ ├── _static │ ├── custom.css │ ├── img │ │ ├── python_scopes.png │ │ └── python_scopes.svg │ └── logo │ │ ├── favicon.ico │ │ ├── favicon.svg │ │ ├── favicon_16px.png │ │ ├── favicon_32px.png │ │ ├── horizontal.svg │ │ ├── horizontal_white.svg │ │ ├── horizontal_white_sidebar.png │ │ ├── icon.svg │ │ ├── icon_white.svg │ │ ├── vertical.svg │ │ └── vertical_white.svg │ ├── _templates │ └── page.html │ ├── best_practices.rst │ ├── codemods.rst │ ├── codemods_tutorial.rst │ ├── conf.py │ ├── experimental.rst │ ├── helpers.rst │ ├── index.rst │ ├── matchers.rst │ ├── matchers_tutorial.ipynb │ ├── metadata.rst │ ├── metadata_tutorial.ipynb │ ├── motivation.rst │ ├── nodes.rst │ ├── parser.rst │ ├── scope_tutorial.ipynb │ ├── tutorial.ipynb │ ├── visitors.rst │ └── why_libcst.rst ├── libcst ├── __init__.py ├── _add_slots.py ├── _batched_visitor.py ├── _exceptions.py ├── _flatten_sentinel.py ├── _maybe_sentinel.py ├── _metadata_dependent.py ├── _nodes │ ├── __init__.py │ ├── base.py │ ├── deep_equals.py │ ├── expression.py │ ├── internal.py │ ├── module.py │ ├── op.py │ ├── statement.py │ ├── tests │ │ ├── __init__.py │ │ ├── base.py │ │ ├── test_assert.py │ │ ├── test_assign.py │ │ ├── test_atom.py │ │ ├── test_attribute.py │ │ ├── test_await.py │ │ ├── test_binary_op.py │ │ ├── test_boolean_op.py │ │ ├── test_call.py │ │ ├── test_classdef.py │ │ ├── test_comment.py │ │ ├── test_comparison.py │ │ ├── test_cst_node.py │ │ ├── test_del.py │ │ ├── test_dict.py │ │ ├── test_dict_comp.py │ │ ├── test_docstring.py │ │ ├── test_else.py │ │ ├── test_empty_line.py │ │ ├── test_flatten_behavior.py │ │ ├── test_for.py │ │ ├── test_funcdef.py │ │ ├── test_global.py │ │ ├── test_if.py │ │ ├── test_ifexp.py │ │ ├── test_import.py │ │ ├── test_indented_block.py │ │ ├── test_lambda.py │ │ ├── test_leaf_small_statements.py │ │ ├── test_list.py │ │ ├── test_match.py │ │ ├── test_matrix_multiply.py │ │ ├── test_module.py │ │ ├── test_namedexpr.py │ │ ├── test_newline.py │ │ ├── test_nonlocal.py │ │ ├── test_number.py │ │ ├── test_raise.py │ │ ├── test_removal_behavior.py │ │ ├── test_return.py │ │ ├── test_set.py │ │ ├── test_simple_comp.py │ │ ├── test_simple_statement.py │ │ ├── test_simple_string.py │ │ ├── test_simple_whitespace.py │ │ ├── test_small_statement.py │ │ ├── test_subscript.py │ │ ├── test_trailing_whitespace.py │ │ ├── test_try.py │ │ ├── test_tuple.py │ │ ├── test_type_alias.py │ │ ├── test_unary_op.py │ │ ├── test_while.py │ │ ├── test_with.py │ │ └── test_yield.py │ └── whitespace.py ├── _parser │ ├── __init__.py │ ├── base_parser.py │ ├── conversions │ │ ├── README.md │ │ ├── __init__.py │ │ ├── expression.py │ │ ├── module.py │ │ ├── params.py │ │ ├── statement.py │ │ └── terminals.py │ ├── custom_itertools.py │ ├── detect_config.py │ ├── entrypoints.py │ ├── grammar.py │ ├── parso │ │ ├── __init__.py │ │ ├── pgen2 │ │ │ ├── __init__.py │ │ │ ├── generator.py │ │ │ └── grammar_parser.py │ │ ├── python │ │ │ ├── __init__.py │ │ │ ├── py_token.py │ │ │ ├── token.py │ │ │ └── tokenize.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_fstring.py │ │ │ ├── test_tokenize.py │ │ │ └── test_utils.py │ │ └── utils.py │ ├── production_decorator.py │ ├── py_whitespace_parser.py │ ├── python_parser.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_config.py │ │ ├── test_detect_config.py │ │ ├── test_footer_behavior.py │ │ ├── test_node_identity.py │ │ ├── test_parse_errors.py │ │ ├── test_version_compare.py │ │ ├── test_whitespace_parser.py │ │ └── test_wrapped_tokenize.py │ ├── types │ │ ├── __init__.py │ │ ├── config.py │ │ ├── conversions.py │ │ ├── partials.py │ │ ├── production.py │ │ ├── py_config.py │ │ ├── py_token.py │ │ ├── py_whitespace_state.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ └── test_config.py │ │ ├── token.py │ │ └── whitespace_state.py │ ├── whitespace_parser.py │ └── wrapped_tokenize.py ├── _position.py ├── _removal_sentinel.py ├── _tabs.py ├── _type_enforce.py ├── _typed_visitor.py ├── _typed_visitor_base.py ├── _types.py ├── _visitors.py ├── codegen │ ├── __init__.py │ ├── gather.py │ ├── gen_matcher_classes.py │ ├── gen_type_mapping.py │ ├── gen_visitor_functions.py │ ├── generate.py │ ├── tests │ │ ├── __init__.py │ │ └── test_codegen_clean.py │ └── transforms.py ├── codemod │ ├── __init__.py │ ├── _cli.py │ ├── _codemod.py │ ├── _command.py │ ├── _context.py │ ├── _dummy_pool.py │ ├── _runner.py │ ├── _testing.py │ ├── _visitor.py │ ├── commands │ │ ├── __init__.py │ │ ├── add_pyre_directive.py │ │ ├── add_trailing_commas.py │ │ ├── convert_format_to_fstring.py │ │ ├── convert_namedtuple_to_dataclass.py │ │ ├── convert_percent_format_to_fstring.py │ │ ├── convert_type_comments.py │ │ ├── convert_union_to_or.py │ │ ├── ensure_import_present.py │ │ ├── fix_pyre_directives.py │ │ ├── fix_variadic_callable.py │ │ ├── noop.py │ │ ├── remove_pyre_directive.py │ │ ├── remove_unused_imports.py │ │ ├── rename.py │ │ ├── rename_typing_generic_aliases.py │ │ ├── strip_strings_from_types.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_add_pyre_directive.py │ │ │ ├── test_add_trailing_commas.py │ │ │ ├── test_convert_format_to_fstring.py │ │ │ ├── test_convert_namedtuple_to_dataclass.py │ │ │ ├── test_convert_percent_format_to_fstring.py │ │ │ ├── test_convert_type_comments.py │ │ │ ├── test_convert_union_to_or.py │ │ │ ├── test_ensure_import_present.py │ │ │ ├── test_fix_pyre_directives.py │ │ │ ├── test_fix_variadic_callable.py │ │ │ ├── test_noop.py │ │ │ ├── test_remove_pyre_directive.py │ │ │ ├── test_remove_unused_imports.py │ │ │ ├── test_rename.py │ │ │ ├── test_rename_typing_generic_aliases.py │ │ │ ├── test_strip_strings_from_types.py │ │ │ └── test_unnecessary_format_string.py │ │ └── unnecessary_format_string.py │ ├── tests │ │ ├── __init__.py │ │ ├── codemod_formatter_error_input.py.txt │ │ ├── test_codemod.py │ │ ├── test_codemod_cli.py │ │ ├── test_metadata.py │ │ └── test_runner.py │ └── visitors │ │ ├── __init__.py │ │ ├── _add_imports.py │ │ ├── _apply_type_annotations.py │ │ ├── _gather_comments.py │ │ ├── _gather_exports.py │ │ ├── _gather_global_names.py │ │ ├── _gather_imports.py │ │ ├── _gather_string_annotation_names.py │ │ ├── _gather_unused_imports.py │ │ ├── _imports.py │ │ ├── _remove_imports.py │ │ └── tests │ │ ├── __init__.py │ │ ├── test_add_imports.py │ │ ├── test_apply_type_annotations.py │ │ ├── test_gather_comments.py │ │ ├── test_gather_exports.py │ │ ├── test_gather_global_names.py │ │ ├── test_gather_imports.py │ │ ├── test_gather_string_annotation_names.py │ │ ├── test_gather_unused_imports.py │ │ └── test_remove_imports.py ├── display │ ├── __init__.py │ ├── graphviz.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_dump_graphviz.py │ │ └── test_dump_text.py │ └── text.py ├── helpers │ ├── __init__.py │ ├── _template.py │ ├── common.py │ ├── expression.py │ ├── module.py │ ├── node_fields.py │ ├── paths.py │ └── tests │ │ ├── __init__.py │ │ ├── test_expression.py │ │ ├── test_module.py │ │ ├── test_node_fields.py │ │ ├── test_paths.py │ │ └── test_template.py ├── matchers │ ├── __init__.py │ ├── _decorators.py │ ├── _matcher_base.py │ ├── _return_types.py │ ├── _visitors.py │ └── tests │ │ ├── __init__.py │ │ ├── test_decorators.py │ │ ├── test_extract.py │ │ ├── test_findall.py │ │ ├── test_matchers.py │ │ ├── test_matchers_with_metadata.py │ │ ├── test_replace.py │ │ └── test_visitors.py ├── metadata │ ├── __init__.py │ ├── accessor_provider.py │ ├── base_provider.py │ ├── expression_context_provider.py │ ├── file_path_provider.py │ ├── full_repo_manager.py │ ├── name_provider.py │ ├── parent_node_provider.py │ ├── position_provider.py │ ├── reentrant_codegen.py │ ├── scope_provider.py │ ├── span_provider.py │ ├── tests │ │ ├── __init__.py │ │ ├── test_accessor_provider.py │ │ ├── test_base_provider.py │ │ ├── test_expression_context_provider.py │ │ ├── test_file_path_provider.py │ │ ├── test_full_repo_manager.py │ │ ├── test_metadata_provider.py │ │ ├── test_metadata_wrapper.py │ │ ├── test_name_provider.py │ │ ├── test_parent_node_provider.py │ │ ├── test_position_provider.py │ │ ├── test_reentrant_codegen.py │ │ ├── test_scope_provider.py │ │ ├── test_span_provider.py │ │ └── test_type_inference_provider.py │ ├── type_inference_provider.py │ └── wrapper.py ├── py.typed ├── testing │ ├── __init__.py │ └── utils.py ├── tests │ ├── __init__.py │ ├── __main__.py │ ├── pyre │ │ ├── .pyre_configuration │ │ ├── simple_class.json │ │ └── simple_class.py │ ├── test_add_slots.py │ ├── test_batched_visitor.py │ ├── test_deep_clone.py │ ├── test_deep_replace.py │ ├── test_e2e.py │ ├── test_exceptions.py │ ├── test_fuzz.py │ ├── test_pyre_integration.py │ ├── test_roundtrip.py │ ├── test_tabs.py │ ├── test_type_enforce.py │ └── test_visitor.py └── tool.py ├── native ├── Cargo.lock ├── Cargo.toml ├── libcst │ ├── Cargo.toml │ ├── Grammar │ ├── LICENSE │ ├── README.md │ ├── benches │ │ └── parser_benchmark.rs │ ├── src │ │ ├── bin.rs │ │ ├── lib.rs │ │ ├── nodes │ │ │ ├── codegen.rs │ │ │ ├── expression.rs │ │ │ ├── inflate_helpers.rs │ │ │ ├── macros.rs │ │ │ ├── mod.rs │ │ │ ├── module.rs │ │ │ ├── op.rs │ │ │ ├── parser_config.rs │ │ │ ├── py_cached.rs │ │ │ ├── statement.rs │ │ │ ├── test_utils.rs │ │ │ ├── traits.rs │ │ │ └── whitespace.rs │ │ ├── parser │ │ │ ├── errors.rs │ │ │ ├── grammar.rs │ │ │ ├── mod.rs │ │ │ └── numbers.rs │ │ ├── py.rs │ │ └── tokenizer │ │ │ ├── core │ │ │ ├── LICENSE │ │ │ ├── README.md │ │ │ ├── mod.rs │ │ │ └── string_types.rs │ │ │ ├── debug_utils.rs │ │ │ ├── mod.rs │ │ │ ├── operators.rs │ │ │ ├── tests.rs │ │ │ ├── text_position │ │ │ ├── char_width.rs │ │ │ └── mod.rs │ │ │ └── whitespace_parser.rs │ └── tests │ │ ├── .gitattributes │ │ ├── fixtures │ │ ├── big_binary_operator.py │ │ ├── class_craziness.py │ │ ├── comments.py │ │ ├── comparisons.py │ │ ├── dangling_indent.py │ │ ├── decorated_function_without_body.py │ │ ├── dysfunctional_del.py │ │ ├── expr.py │ │ ├── expr_statement.py │ │ ├── fun_with_func_defs.py │ │ ├── global_nonlocal.py │ │ ├── import.py │ │ ├── indents_but_no_eol_before_eof.py │ │ ├── just_a_comment_without_nl.py │ │ ├── malicious_match.py │ │ ├── mixed_newlines.py │ │ ├── pep646.py │ │ ├── raise.py │ │ ├── smol_statements.py │ │ ├── spacious_spaces.py │ │ ├── starry_tries.py │ │ ├── suicidal_slices.py │ │ ├── super_strings.py │ │ ├── terrible_tries.py │ │ ├── trailing_comment_without_nl.py │ │ ├── trailing_whitespace.py │ │ ├── tuple_shenanigans.py │ │ ├── type_parameters.py │ │ ├── vast_emptiness.py │ │ ├── with_wickedness.py │ │ └── wonky_walrus.py │ │ └── parser_roundtrip.rs ├── libcst_derive │ ├── Cargo.toml │ ├── LICENSE │ ├── src │ │ ├── codegen.rs │ │ ├── cstnode.rs │ │ ├── inflate.rs │ │ ├── into_py.rs │ │ ├── lib.rs │ │ └── parenthesized_node.rs │ └── tests │ │ └── pass │ │ ├── minimal_cst.rs │ │ └── simple.rs └── roundtrip.sh ├── pyproject.toml ├── scripts ├── check_copyright.py └── regenerate-fixtures.py ├── setup.py ├── stubs ├── hypothesis.pyi ├── hypothesmith.pyi ├── libcst │ └── native.pyi ├── libcst_native │ ├── parser_config.pyi │ ├── token_type.pyi │ ├── tokenize.pyi │ ├── whitespace_parser.pyi │ └── whitespace_state.pyi ├── setuptools.pyi ├── tokenize.pyi └── typing_inspect.pyi └── zizmor.yml /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [target.x86_64-apple-darwin] 2 | rustflags = [ 3 | "-C", "link-arg=-undefined", 4 | "-C", "link-arg=dynamic_lookup", 5 | ] 6 | 7 | [target.aarch64-apple-darwin] 8 | rustflags = [ 9 | "-C", "link-arg=-undefined", 10 | "-C", "link-arg=dynamic_lookup", 11 | ] -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*.{py,pyi,rs,toml,md}] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 4 7 | indent_style = space 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | max_line_length = 88 11 | 12 | [*.rs] 13 | # https://github.com/rust-dev-tools/fmt-rfcs/blob/master/guide/guide.md 14 | max_line_length = 100 15 | -------------------------------------------------------------------------------- /.fixit.config.yaml: -------------------------------------------------------------------------------- 1 | block_list_patterns: 2 | - '@generated' 3 | - '@nolint' 4 | block_list_rules: ["UseFstringRule", "CompareSingletonPrimitivesByIsRule"] 5 | fixture_dir: ./fixtures 6 | formatter: ["black", "-"] 7 | packages: 8 | - fixit.rules 9 | repo_root: libcst 10 | rule_config: {} 11 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | *.svg binary 2 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | ## Summary 3 | 4 | ## Test Plan 5 | 6 | -------------------------------------------------------------------------------- /.github/build-matrix.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "vers": "x86_64", 4 | "os": "ubuntu-20.04" 5 | }, 6 | { 7 | "vers": "i686", 8 | "os": "ubuntu-20.04" 9 | }, 10 | { 11 | "vers": "arm64", 12 | "os": "macos-latest" 13 | }, 14 | { 15 | "vers": "auto64", 16 | "os": "macos-latest" 17 | }, 18 | { 19 | "vers": "auto64", 20 | "os": "windows-2019" 21 | }, 22 | { 23 | "vers": "aarch64", 24 | "os": [ 25 | "self-hosted", 26 | "linux", 27 | "ARM64" 28 | ], 29 | "on_ref_regex": "^refs/(heads/main|tags/.*)$" 30 | } 31 | ] -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file 2 | 3 | version: 2 4 | updates: 5 | - package-ecosystem: pip 6 | directory: "/" 7 | schedule: 8 | interval: weekly 9 | 10 | - package-ecosystem: cargo 11 | directory: "/native" 12 | schedule: 13 | interval: weekly 14 | 15 | - package-ecosystem: github-actions 16 | directory: "/" 17 | schedule: 18 | interval: weekly 19 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: 3 | workflow_call: 4 | 5 | jobs: 6 | # Build python wheels 7 | build: 8 | name: Build wheels on ${{ matrix.os }} 9 | runs-on: ${{ matrix.os }} 10 | strategy: 11 | fail-fast: false 12 | matrix: 13 | # macos-13 is an intel runner, macos-latest is apple silicon 14 | os: 15 | [ 16 | macos-13, 17 | macos-latest, 18 | ubuntu-latest, 19 | ubuntu-24.04-arm, 20 | windows-latest, 21 | windows-11-arm, 22 | ] 23 | env: 24 | SCCACHE_VERSION: 0.2.13 25 | GITHUB_WORKSPACE: "${{github.workspace}}" 26 | steps: 27 | - uses: actions/checkout@v4 28 | with: 29 | fetch-depth: 0 30 | persist-credentials: false 31 | - uses: actions/setup-python@v5 32 | with: 33 | python-version: "3.12" 34 | - uses: dtolnay/rust-toolchain@stable 35 | - name: Set MACOSX_DEPLOYMENT_TARGET for Intel MacOS 36 | if: matrix.os == 'macos-13' 37 | run: >- 38 | echo MACOSX_DEPLOYMENT_TARGET=10.12 >> $GITHUB_ENV 39 | - name: Disable scmtools local scheme 40 | if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} 41 | run: >- 42 | echo LIBCST_NO_LOCAL_SCHEME=1 >> $GITHUB_ENV 43 | - name: Enable building wheels for pre-release CPython versions 44 | if: github.event_name != 'release' 45 | run: echo CIBW_ENABLE=cpython-prerelease >> $GITHUB_ENV 46 | - name: Build wheels 47 | uses: pypa/cibuildwheel@v3.0.0b4 48 | - uses: actions/upload-artifact@v4 49 | with: 50 | path: wheelhouse/*.whl 51 | name: wheels-${{matrix.os}} 52 | -------------------------------------------------------------------------------- /.github/workflows/pypi_upload.yml: -------------------------------------------------------------------------------- 1 | name: pypi_upload 2 | 3 | on: 4 | release: 5 | types: [published] 6 | push: 7 | branches: [main] 8 | 9 | permissions: 10 | contents: read 11 | 12 | jobs: 13 | build: 14 | uses: Instagram/LibCST/.github/workflows/build.yml@main 15 | upload_release: 16 | name: Upload wheels to pypi 17 | runs-on: ubuntu-latest 18 | needs: build 19 | permissions: 20 | id-token: write 21 | steps: 22 | - uses: actions/checkout@v4 23 | with: 24 | fetch-depth: 0 25 | persist-credentials: false 26 | - name: Download binary wheels 27 | id: download 28 | uses: actions/download-artifact@v4 29 | with: 30 | pattern: wheels-* 31 | path: wheelhouse 32 | merge-multiple: true 33 | - uses: actions/setup-python@v5 34 | with: 35 | python-version: "3.10" 36 | - name: Install hatch 37 | run: pip install -U hatch 38 | - name: Build a source tarball 39 | env: 40 | LIBCST_NO_LOCAL_SCHEME: 1 41 | OUTDIR: ${{ steps.download.outputs.download-path }} 42 | run: >- 43 | hatch run python -m 44 | build 45 | --sdist 46 | --outdir "$OUTDIR" 47 | - name: Publish distribution 📦 to Test PyPI 48 | if: github.event_name == 'push' 49 | uses: pypa/gh-action-pypi-publish@release/v1 50 | with: 51 | repository-url: https://test.pypi.org/legacy/ 52 | packages-dir: ${{ steps.download.outputs.download-path }} 53 | - name: Publish distribution 📦 to PyPI 54 | if: github.event_name == 'release' 55 | uses: pypa/gh-action-pypi-publish@release/v1 56 | with: 57 | packages-dir: ${{ steps.download.outputs.download-path }} 58 | -------------------------------------------------------------------------------- /.github/workflows/zizmor.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Actions Security Analysis with zizmor 🌈 2 | 3 | on: 4 | push: 5 | branches: ["main"] 6 | pull_request: 7 | branches: ["**"] 8 | 9 | jobs: 10 | zizmor: 11 | name: zizmor latest via PyPI 12 | runs-on: ubuntu-latest 13 | permissions: 14 | security-events: write 15 | contents: read 16 | actions: read 17 | steps: 18 | - name: Checkout repository 19 | uses: actions/checkout@v4 20 | with: 21 | persist-credentials: false 22 | 23 | - name: Install the latest version of uv 24 | uses: astral-sh/setup-uv@v6 25 | 26 | - name: Run zizmor 🌈 27 | run: uvx zizmor --format sarif . > results.sarif 28 | env: 29 | GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | 31 | - name: Upload SARIF file 32 | uses: github/codeql-action/upload-sarif@v3 33 | with: 34 | sarif_file: results.sarif 35 | category: zizmor -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | *.swo 3 | *.pyc 4 | *.pyd 5 | *.pyo 6 | *.so 7 | *.egg-info/ 8 | .eggs/ 9 | .pyre/ 10 | __pycache__/ 11 | .tox/ 12 | docs/build/ 13 | dist/ 14 | docs/source/.ipynb_checkpoints/ 15 | build/ 16 | libcst/_version.py 17 | .coverage 18 | .hypothesis/ 19 | .python-version 20 | target/ 21 | venv/ 22 | .venv/ 23 | .idea/ 24 | -------------------------------------------------------------------------------- /.pyre_configuration: -------------------------------------------------------------------------------- 1 | { 2 | "exclude": [ 3 | ".*\/native\/.*" 4 | ], 5 | "source_directories": [ 6 | "." 7 | ], 8 | "search_path": [ 9 | "stubs", {"site-package": "setuptools_rust"} 10 | ], 11 | "workers": 3, 12 | "strict": true 13 | } 14 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | sphinx: 4 | configuration: docs/source/conf.py 5 | 6 | formats: all 7 | 8 | build: 9 | os: ubuntu-20.04 10 | tools: 11 | python: "3" 12 | rust: "1.70" 13 | apt_packages: 14 | - graphviz 15 | 16 | python: 17 | install: 18 | - method: pip 19 | path: . 20 | extra_requirements: 21 | - dev 22 | 23 | -------------------------------------------------------------------------------- /.watchmanconfig: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to LibCST 2 | We want to make contributing to this project as easy and transparent as 3 | possible. 4 | 5 | ## Our Development Process 6 | This github repo is the source of truth and all changes need to be reviewed in 7 | pull requests. 8 | 9 | ## Pull Requests 10 | We actively welcome your pull requests. 11 | 12 | ### Setup Your Environment 13 | 14 | 1. Install a [Rust toolchain](https://rustup.rs) and [hatch](https://hatch.pypa.io) 15 | 2. Fork the repo on your side 16 | 3. Clone the repo 17 | > git clone [your fork.git] libcst 18 | > cd libcst 19 | 4. Sync with the main libcst version package 20 | > git fetch --tags https://github.com/instagram/libcst 21 | 5. Setup the env 22 | > hatch env create 23 | 24 | You are now ready to create your own branch from main, and contribute. 25 | Please provide tests (using unittest), and update the documentation (both docstrings 26 | and sphinx doc), if applicable. 27 | 28 | ### Before Submitting Your Pull Request 29 | 30 | 1. Format your code 31 | > hatch run format 32 | 2. Run the type checker 33 | > hatch run typecheck 34 | 3. Test your changes 35 | > hatch run test 36 | 4. Check linters 37 | > hatch run lint 38 | 39 | ## Contributor License Agreement ("CLA") 40 | In order to accept your pull request, we need you to submit a CLA. You only need 41 | to do this once to work on any of Facebook's open source projects. 42 | 43 | Complete your CLA here: 44 | 45 | ## Issues 46 | We use GitHub issues to track public bugs. Please ensure your description is 47 | clear and has sufficient instructions to be able to reproduce the issue. 48 | 49 | Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe 50 | disclosure of security bugs. In those cases, please go through the process 51 | outlined on that page and do not file a public issue. 52 | 53 | ## Coding Style 54 | We use flake8 and ufmt to enforce coding style. 55 | 56 | ## License 57 | By contributing to LibCST, you agree that your contributions will be licensed 58 | under the MIT LICENSE file in the root directory of this source tree. 59 | -------------------------------------------------------------------------------- /MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # How to make a new release 2 | 3 | 1. Add a new entry to `CHANGELOG.md` (I normally use the [new release page](https://github.com/Instagram/LibCST/releases/new) to generate a changelog, then manually group) 4 | 1. Follow the existing format: `Fixed`, `Added`, `Updated`, `Deprecated`, `Removed`, `New Contributors` sections, and the full changelog link at the bottom. 5 | 1. Mention only user-visible changes - improvements to CI, tests, or development workflow aren't noteworthy enough 6 | 1. Version bumps are generally not worth mentioning with some notable exceptions (like pyo3) 7 | 1. Group related PRs into one bullet point if it makes sense 8 | 2. manually bump versions in `Cargo.toml` files in the repo 9 | 3. make a new PR with the above changes, get it reviewed and landed 10 | 4. make a new release on Github, create a new tag on publish, and copy the contents of the changelog entry in there 11 | 5. after publishing, check out the repo at the new tag, and run `cd native; cargo +nightly publish -Z package-workspace -p libcst_derive -p libcst` -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.rst LICENSE CODE_OF_CONDUCT.md CONTRIBUTING.md docs/source/*.rst libcst/py.typed 2 | 3 | include native/Cargo.toml 4 | recursive-include native * 5 | recursive-exclude native/target * -------------------------------------------------------------------------------- /apt.txt: -------------------------------------------------------------------------------- 1 | rustc 2 | cargo -------------------------------------------------------------------------------- /docs/source/_static/custom.css: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) Meta Platforms, Inc. and affiliates. 3 | * 4 | * This source code is licensed under the MIT license found in the 5 | * LICENSE file in the root directory of this source tree. 6 | */ 7 | 8 | .toggle { 9 | display: block; 10 | clear: both; 11 | } 12 | 13 | .toggle:after { 14 | content: "Show Code [+]"; 15 | } 16 | 17 | .toggle.open:before { 18 | content: "Hide Code [-]"; 19 | } 20 | .toggle.open:after { 21 | content: ""; 22 | } 23 | -------------------------------------------------------------------------------- /docs/source/_static/img/python_scopes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Instagram/LibCST/482a2e5f0997273e3fb272346dac275c04e84807/docs/source/_static/img/python_scopes.png -------------------------------------------------------------------------------- /docs/source/_static/logo/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Instagram/LibCST/482a2e5f0997273e3fb272346dac275c04e84807/docs/source/_static/logo/favicon.ico -------------------------------------------------------------------------------- /docs/source/_static/logo/favicon_16px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Instagram/LibCST/482a2e5f0997273e3fb272346dac275c04e84807/docs/source/_static/logo/favicon_16px.png -------------------------------------------------------------------------------- /docs/source/_static/logo/favicon_32px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Instagram/LibCST/482a2e5f0997273e3fb272346dac275c04e84807/docs/source/_static/logo/favicon_32px.png -------------------------------------------------------------------------------- /docs/source/_static/logo/horizontal_white_sidebar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Instagram/LibCST/482a2e5f0997273e3fb272346dac275c04e84807/docs/source/_static/logo/horizontal_white_sidebar.png -------------------------------------------------------------------------------- /docs/source/_templates/page.html: -------------------------------------------------------------------------------- 1 | {% extends "!page.html" %} 2 | 3 | {% block footer %} 4 | 14 | {% endblock %} 15 | -------------------------------------------------------------------------------- /docs/source/experimental.rst: -------------------------------------------------------------------------------- 1 | .. _libcst-experimental: 2 | 3 | ================= 4 | Experimental APIs 5 | ================= 6 | 7 | These APIs may change at any time (including in minor releases) with no notice. You 8 | probably shouldn't use them, but if you do, you should pin your application to an exact 9 | release of LibCST to avoid breakages. 10 | 11 | Reentrant Code Generation 12 | ------------------------- 13 | 14 | .. autoclass:: libcst.metadata.ExperimentalReentrantCodegenProvider 15 | .. autoclass:: libcst.metadata.CodegenPartial 16 | -------------------------------------------------------------------------------- /docs/source/helpers.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Helpers 3 | ======= 4 | 5 | Helpers are higher level functions built for reducing recurring code boilerplate. 6 | We add helpers as method of ``CSTNode`` or ``libcst.helpers`` package based on those principles: 7 | 8 | - ``CSTNode`` method: simple, read-only and only require data of the direct children of a CSTNode. 9 | - ``libcst.helpers``: node transforms or require recursively traversing the syntax tree. 10 | 11 | Construction Helpers 12 | -------------------- 13 | 14 | Functions that assist in creating a new LibCST tree. 15 | 16 | .. autofunction:: libcst.helpers.parse_template_module 17 | .. autofunction:: libcst.helpers.parse_template_expression 18 | .. autofunction:: libcst.helpers.parse_template_statement 19 | 20 | Transformation Helpers 21 | ---------------------- 22 | 23 | Functions that assist in transforming an existing LibCST node. 24 | 25 | .. autofunction:: libcst.helpers.insert_header_comments 26 | 27 | Traversing Helpers 28 | ------------------ 29 | 30 | Functions that assist in traversing an existing LibCST tree. 31 | 32 | .. autofunction:: libcst.helpers.get_full_name_for_node 33 | .. autofunction:: libcst.helpers.get_full_name_for_node_or_raise 34 | .. autofunction:: libcst.helpers.ensure_type 35 | 36 | Node fields filtering Helpers 37 | ----------------------------- 38 | 39 | Function that assist when handling CST nodes' fields. 40 | 41 | .. autofunction:: libcst.helpers.filter_node_fields 42 | 43 | And lower level functions: 44 | 45 | .. autofunction:: libcst.helpers.get_node_fields 46 | .. autofunction:: libcst.helpers.is_whitespace_node_field 47 | .. autofunction:: libcst.helpers.is_syntax_node_field 48 | .. autofunction:: libcst.helpers.is_default_node_field 49 | .. autofunction:: libcst.helpers.get_field_default_value 50 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | .. LibCST documentation master file, created by 2 | sphinx-quickstart on Wed Jul 17 17:05:21 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | ====== 7 | LibCST 8 | ====== 9 | 10 | .. include:: ../../README.rst 11 | :start-after: intro-start 12 | :end-before: intro-end 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | :caption: Introduction: 17 | 18 | why_libcst 19 | motivation 20 | 21 | 22 | .. toctree:: 23 | :maxdepth: 2 24 | :caption: Tutorial: 25 | 26 | Parsing and Visitors 27 | Metadata 28 | Scope Analysis 29 | Matchers 30 | Codemodding 31 | Best Practices 32 | 33 | 34 | .. toctree:: 35 | :maxdepth: 2 36 | :caption: Reference: 37 | 38 | parser 39 | nodes 40 | visitors 41 | metadata 42 | matchers 43 | codemods 44 | helpers 45 | experimental 46 | 47 | 48 | 49 | Indices and tables 50 | ================== 51 | 52 | * :ref:`genindex` 53 | * :ref:`modindex` 54 | * :ref:`search` 55 | 56 | 57 | .. include:: ../../README.rst 58 | :start-after: fb-docs-start 59 | :end-before: fb-docs-end 60 | -------------------------------------------------------------------------------- /docs/source/parser.rst: -------------------------------------------------------------------------------- 1 | Parsing 2 | ======= 3 | 4 | The parser functions accept source code and an optional configuration object, 5 | and will generate :class:`~libcst.CSTNode` objects. 6 | 7 | :func:`~libcst.parse_module` is the most useful function here, since it accepts 8 | the entire contents of a file and returns a new tree, but 9 | :func:`~libcst.parse_expression` and :func:`~libcst.parse_statement` are useful 10 | when inserting new nodes into the tree, because they're easier to use than the 11 | equivalent node constructors. 12 | 13 | >>> import libcst as cst 14 | >>> cst.parse_expression("1 + 2") 15 | BinaryOperation( 16 | left=Integer( 17 | value='1', 18 | lpar=[], 19 | rpar=[], 20 | ), 21 | operator=Add( 22 | whitespace_before=SimpleWhitespace( 23 | value=' ', 24 | ), 25 | whitespace_after=SimpleWhitespace( 26 | value=' ', 27 | ), 28 | ), 29 | right=Integer( 30 | value='2', 31 | lpar=[], 32 | rpar=[], 33 | ), 34 | lpar=[], 35 | rpar=[], 36 | ) 37 | 38 | 39 | .. autofunction:: libcst.parse_module 40 | .. autofunction:: libcst.parse_expression 41 | .. autofunction:: libcst.parse_statement 42 | .. autoclass:: libcst.PartialParserConfig 43 | 44 | Syntax Errors 45 | ------------- 46 | 47 | .. autoclass:: libcst.ParserSyntaxError 48 | :members: message, raw_line, raw_column, editor_line, editor_column 49 | :special-members: __str__ 50 | -------------------------------------------------------------------------------- /libcst/_add_slots.py: -------------------------------------------------------------------------------- 1 | # This file is derived from github.com/ericvsmith/dataclasses, and is Apache 2 licensed. 2 | # https://github.com/ericvsmith/dataclasses/blob/ae712dd993420d43444f188f452/LICENSE.txt 3 | # https://github.com/ericvsmith/dataclasses/blob/ae712dd993420d43444f/dataclass_tools.py 4 | # Changed: takes slots in base classes into account when creating slots 5 | 6 | import dataclasses 7 | from itertools import chain, filterfalse 8 | from typing import Any, Mapping, Type, TypeVar 9 | 10 | _T = TypeVar("_T") 11 | 12 | 13 | def add_slots(cls: Type[_T]) -> Type[_T]: 14 | # Need to create a new class, since we can't set __slots__ 15 | # after a class has been created. 16 | 17 | # Make sure __slots__ isn't already set. 18 | if "__slots__" in cls.__dict__: 19 | raise TypeError(f"{cls.__name__} already specifies __slots__") 20 | 21 | # Create a new dict for our new class. 22 | cls_dict = dict(cls.__dict__) 23 | field_names = tuple(f.name for f in dataclasses.fields(cls)) 24 | inherited_slots = set( 25 | chain.from_iterable( 26 | superclass.__dict__.get("__slots__", ()) for superclass in cls.mro() 27 | ) 28 | ) 29 | cls_dict["__slots__"] = tuple( 30 | filterfalse(inherited_slots.__contains__, field_names) 31 | ) 32 | for field_name in field_names: 33 | # Remove our attributes, if present. They'll still be 34 | # available in _MARKER. 35 | cls_dict.pop(field_name, None) 36 | # Remove __dict__ itself. 37 | cls_dict.pop("__dict__", None) 38 | 39 | # Create the class. 40 | qualname = getattr(cls, "__qualname__", None) 41 | 42 | # pyre-fixme[9]: cls has type `Type[Variable[_T]]`; used as `_T`. 43 | # pyre-fixme[19]: Expected 0 positional arguments. 44 | cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) 45 | if qualname is not None: 46 | cls.__qualname__ = qualname 47 | 48 | # Set __getstate__ and __setstate__ to workaround a bug with pickling frozen 49 | # dataclasses with slots. See https://bugs.python.org/issue36424 50 | 51 | def __getstate__(self: object) -> Mapping[str, Any]: 52 | return { 53 | field.name: getattr(self, field.name) 54 | for field in dataclasses.fields(self) 55 | if hasattr(self, field.name) 56 | } 57 | 58 | def __setstate__(self: object, state: Mapping[str, Any]) -> None: 59 | for fieldname, value in state.items(): 60 | object.__setattr__(self, fieldname, value) 61 | 62 | cls.__getstate__ = __getstate__ 63 | cls.__setstate__ = __setstate__ 64 | 65 | return cls 66 | -------------------------------------------------------------------------------- /libcst/_flatten_sentinel.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import sys 7 | 8 | # PEP 585 9 | if sys.version_info < (3, 9): 10 | from typing import Iterable, Sequence 11 | else: 12 | from collections.abc import Iterable, Sequence 13 | 14 | from libcst._types import CSTNodeT_co 15 | 16 | 17 | class FlattenSentinel(Sequence[CSTNodeT_co]): 18 | """ 19 | A :class:`FlattenSentinel` may be returned by a :meth:`CSTTransformer.on_leave` 20 | method when one wants to replace a node with multiple nodes. The replaced 21 | node must be contained in a `Sequence` attribute such as 22 | :attr:`~libcst.Module.body`. This is generally the case for 23 | :class:`~libcst.BaseStatement` and :class:`~libcst.BaseSmallStatement`. 24 | For example to insert a print before every return:: 25 | 26 | def leave_Return( 27 | self, original_node: cst.Return, updated_node: cst.Return 28 | ) -> Union[cst.Return, cst.RemovalSentinel, cst.FlattenSentinel[cst.BaseSmallStatement]]: 29 | log_stmt = cst.Expr(cst.parse_expression("print('returning')")) 30 | return cst.FlattenSentinel([log_stmt, updated_node]) 31 | 32 | Returning an empty :class:`FlattenSentinel` is equivalent to returning 33 | :attr:`cst.RemovalSentinel.REMOVE` and is subject to its requirements. 34 | """ 35 | 36 | nodes: Sequence[CSTNodeT_co] 37 | 38 | def __init__(self, nodes: Iterable[CSTNodeT_co]) -> None: 39 | self.nodes = tuple(nodes) 40 | 41 | def __getitem__(self, idx: int) -> CSTNodeT_co: 42 | return self.nodes[idx] 43 | 44 | def __len__(self) -> int: 45 | return len(self.nodes) 46 | -------------------------------------------------------------------------------- /libcst/_maybe_sentinel.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from enum import auto, Enum 7 | 8 | 9 | class MaybeSentinel(Enum): 10 | """ 11 | A :class:`MaybeSentinel` value is used as the default value for some attributes to 12 | denote that when generating code (when :attr:`Module.code` is evaluated) we should 13 | optionally include this element in order to generate valid code. 14 | 15 | :class:`MaybeSentinel` is only used for "syntactic trivia" that most users shouldn't 16 | care much about anyways, like commas, semicolons, and whitespace. 17 | 18 | For example, a function call's :attr:`Arg.comma` value defaults to 19 | :attr:`MaybeSentinel.DEFAULT`. A comma is required after every argument, except for 20 | the last one. If a comma is required and :attr:`Arg.comma` is a 21 | :class:`MaybeSentinel`, one is inserted. 22 | 23 | This makes manual node construction easier, but it also means that we safely add 24 | arguments to a preexisting function call without manually fixing the commas: 25 | 26 | >>> import libcst as cst 27 | >>> fn_call = cst.parse_expression("fn(1, 2)") 28 | >>> new_fn_call = fn_call.with_changes( 29 | ... args=[*fn_call.args, cst.Arg(cst.Integer("3"))] 30 | ... ) 31 | >>> dummy_module = cst.parse_module("") # we need to use Module.code_for_node 32 | >>> dummy_module.code_for_node(fn_call) 33 | 'fn(1, 2)' 34 | >>> dummy_module.code_for_node(new_fn_call) 35 | 'fn(1, 2, 3)' 36 | 37 | Notice that a comma was automatically inserted after the second argument. Since the 38 | original second argument had no comma, it was initialized to 39 | :attr:`MaybeSentinel.DEFAULT`. During the code generation of the second argument, a 40 | comma was inserted to ensure that the resulting code is valid. 41 | 42 | .. warning:: 43 | While this sentinel is used in place of nodes, it is not a :class:`CSTNode`, and 44 | will not be visited by a :class:`CSTVisitor`. 45 | 46 | Some other libraries, like `RedBaron`_, take other approaches to this problem. 47 | RedBaron's tree is mutable (LibCST's tree is immutable), and so they're able to 48 | solve this problem with `"proxy lists" 49 | `_. Both approaches come with 50 | different sets of tradeoffs. 51 | 52 | .. _RedBaron: http://redbaron.pycqa.org/en/latest/index.html 53 | """ 54 | 55 | DEFAULT = auto() 56 | 57 | def __repr__(self) -> str: 58 | return str(self) 59 | -------------------------------------------------------------------------------- /libcst/_nodes/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | """ 8 | This package contains CSTNode and all of the subclasses needed to express Python's full 9 | grammar in a whitespace-sensitive fashion, forming a "Concrete" Syntax Tree (CST). 10 | """ 11 | -------------------------------------------------------------------------------- /libcst/_nodes/deep_equals.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | """ 7 | Provides the implementation of `CSTNode.deep_equals`. 8 | """ 9 | 10 | from dataclasses import fields 11 | from typing import Sequence 12 | 13 | from libcst._nodes.base import CSTNode 14 | 15 | 16 | def deep_equals(a: object, b: object) -> bool: 17 | if isinstance(a, CSTNode) and isinstance(b, CSTNode): 18 | return _deep_equals_cst_node(a, b) 19 | elif ( 20 | isinstance(a, Sequence) 21 | and not isinstance(a, (str, bytes)) 22 | and isinstance(b, Sequence) 23 | and not isinstance(b, (str, bytes)) 24 | ): 25 | return _deep_equals_sequence(a, b) 26 | else: 27 | return a == b 28 | 29 | 30 | def _deep_equals_sequence(a: Sequence[object], b: Sequence[object]) -> bool: 31 | """ 32 | A helper function for `CSTNode.deep_equals`. 33 | 34 | Normalizes and compares sequences. Because we only ever expose `Sequence[]` 35 | types, and not `List[]`, `Tuple[]`, or `Iterable[]` values, all sequences should 36 | be treated as equal if they have the same values. 37 | """ 38 | if a is b: # short-circuit 39 | return True 40 | if len(a) != len(b): 41 | return False 42 | return all(deep_equals(a_el, b_el) for (a_el, b_el) in zip(a, b)) 43 | 44 | 45 | def _deep_equals_cst_node(a: "CSTNode", b: "CSTNode") -> bool: 46 | if type(a) is not type(b): 47 | return False 48 | if a is b: # short-circuit 49 | return True 50 | # Ignore metadata and other hidden fields 51 | for field in (f for f in fields(a) if f.compare is True): 52 | a_value = getattr(a, field.name) 53 | b_value = getattr(b, field.name) 54 | if not deep_equals(a_value, b_value): 55 | return False 56 | return True 57 | -------------------------------------------------------------------------------- /libcst/_nodes/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_nodes/tests/test_comment.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Callable 7 | 8 | import libcst as cst 9 | from libcst._nodes.tests.base import CSTNodeTest 10 | from libcst.testing.utils import data_provider 11 | 12 | 13 | class CommentTest(CSTNodeTest): 14 | @data_provider( 15 | ( 16 | (cst.Comment("#"), "#"), 17 | (cst.Comment("#comment text"), "#comment text"), 18 | (cst.Comment("# comment text"), "# comment text"), 19 | ) 20 | ) 21 | def test_valid(self, node: cst.CSTNode, code: str) -> None: 22 | self.validate_node(node, code) 23 | 24 | @data_provider( 25 | ( 26 | (lambda: cst.Comment(" bad input"), "non-comment"), 27 | (lambda: cst.Comment("# newline shouldn't be here\n"), "non-comment"), 28 | (lambda: cst.Comment(" # Leading space is wrong"), "non-comment"), 29 | ) 30 | ) 31 | def test_invalid( 32 | self, get_node: Callable[[], cst.CSTNode], expected_re: str 33 | ) -> None: 34 | self.assert_invalid(get_node, expected_re) 35 | -------------------------------------------------------------------------------- /libcst/_nodes/tests/test_else.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Any 7 | 8 | import libcst as cst 9 | from libcst._nodes.tests.base import CSTNodeTest 10 | from libcst.metadata import CodeRange 11 | from libcst.testing.utils import data_provider 12 | 13 | 14 | class ElseTest(CSTNodeTest): 15 | @data_provider( 16 | ( 17 | { 18 | "node": cst.Else(cst.SimpleStatementSuite((cst.Pass(),))), 19 | "code": "else: pass\n", 20 | "expected_position": CodeRange((1, 0), (1, 10)), 21 | }, 22 | { 23 | "node": cst.Else( 24 | cst.SimpleStatementSuite((cst.Pass(),)), 25 | whitespace_before_colon=cst.SimpleWhitespace(" "), 26 | ), 27 | "code": "else : pass\n", 28 | "expected_position": CodeRange((1, 0), (1, 12)), 29 | }, 30 | ) 31 | ) 32 | def test_valid(self, **kwargs: Any) -> None: 33 | self.validate_node(**kwargs) 34 | -------------------------------------------------------------------------------- /libcst/_nodes/tests/test_empty_line.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import libcst as cst 7 | from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock 8 | from libcst.testing.utils import data_provider 9 | 10 | 11 | class EmptyLineTest(CSTNodeTest): 12 | @data_provider( 13 | ( 14 | (cst.EmptyLine(), "\n"), 15 | (cst.EmptyLine(whitespace=cst.SimpleWhitespace(" ")), " \n"), 16 | (cst.EmptyLine(comment=cst.Comment("# comment")), "# comment\n"), 17 | (cst.EmptyLine(newline=cst.Newline("\r\n")), "\r\n"), 18 | (DummyIndentedBlock(" ", cst.EmptyLine()), " \n"), 19 | (DummyIndentedBlock(" ", cst.EmptyLine(indent=False)), "\n"), 20 | ( 21 | DummyIndentedBlock( 22 | "\t", 23 | cst.EmptyLine( 24 | whitespace=cst.SimpleWhitespace(" "), 25 | comment=cst.Comment("# comment"), 26 | newline=cst.Newline("\r\n"), 27 | ), 28 | ), 29 | "\t # comment\r\n", 30 | ), 31 | ) 32 | ) 33 | def test_valid(self, node: cst.CSTNode, code: str) -> None: 34 | self.validate_node(node, code) 35 | -------------------------------------------------------------------------------- /libcst/_nodes/tests/test_leaf_small_statements.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import libcst as cst 7 | from libcst._nodes.tests.base import CSTNodeTest 8 | from libcst.testing.utils import data_provider 9 | 10 | 11 | class LeafSmallStatementsTest(CSTNodeTest): 12 | @data_provider( 13 | ((cst.Pass(), "pass"), (cst.Break(), "break"), (cst.Continue(), "continue")) 14 | ) 15 | def test_valid(self, node: cst.CSTNode, code: str) -> None: 16 | self.validate_node(node, code) 17 | -------------------------------------------------------------------------------- /libcst/_nodes/tests/test_newline.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Callable 7 | 8 | import libcst as cst 9 | from libcst._nodes.tests.base import CSTNodeTest 10 | from libcst.testing.utils import data_provider 11 | 12 | 13 | class NewlineTest(CSTNodeTest): 14 | @data_provider( 15 | ( 16 | (cst.Newline("\r\n"), "\r\n"), 17 | (cst.Newline("\r"), "\r"), 18 | (cst.Newline("\n"), "\n"), 19 | ) 20 | ) 21 | def test_valid(self, node: cst.CSTNode, code: str) -> None: 22 | self.validate_node(node, code) 23 | 24 | @data_provider( 25 | ( 26 | (lambda: cst.Newline("bad input"), "invalid value"), 27 | (lambda: cst.Newline("\nbad input\n"), "invalid value"), 28 | (lambda: cst.Newline("\n\n"), "invalid value"), 29 | ) 30 | ) 31 | def test_invalid( 32 | self, get_node: Callable[[], cst.CSTNode], expected_re: str 33 | ) -> None: 34 | self.assert_invalid(get_node, expected_re) 35 | -------------------------------------------------------------------------------- /libcst/_nodes/tests/test_simple_string.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import unittest 7 | 8 | import libcst as cst 9 | 10 | 11 | class TestSimpleString(unittest.TestCase): 12 | def test_quote(self) -> None: 13 | test_cases = [ 14 | ('"a"', '"'), 15 | ("'b'", "'"), 16 | ('""', '"'), 17 | ("''", "'"), 18 | ('"""c"""', '"""'), 19 | ("'''d'''", "'''"), 20 | ('""""e"""', '"""'), 21 | ("''''f'''", "'''"), 22 | ('"""""g"""', '"""'), 23 | ("'''''h'''", "'''"), 24 | ('""""""', '"""'), 25 | ("''''''", "'''"), 26 | ] 27 | 28 | for s, expected_quote in test_cases: 29 | simple_string = cst.SimpleString(s) 30 | actual = simple_string.quote 31 | self.assertEqual(expected_quote, actual) 32 | -------------------------------------------------------------------------------- /libcst/_nodes/tests/test_trailing_whitespace.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import libcst as cst 7 | from libcst._nodes.tests.base import CSTNodeTest 8 | from libcst.testing.utils import data_provider 9 | 10 | 11 | class TrailingWhitespaceTest(CSTNodeTest): 12 | @data_provider( 13 | ( 14 | (cst.TrailingWhitespace(), "\n"), 15 | (cst.TrailingWhitespace(whitespace=cst.SimpleWhitespace(" ")), " \n"), 16 | (cst.TrailingWhitespace(comment=cst.Comment("# comment")), "# comment\n"), 17 | (cst.TrailingWhitespace(newline=cst.Newline("\r\n")), "\r\n"), 18 | ( 19 | cst.TrailingWhitespace( 20 | whitespace=cst.SimpleWhitespace(" "), 21 | comment=cst.Comment("# comment"), 22 | newline=cst.Newline("\r\n"), 23 | ), 24 | " # comment\r\n", 25 | ), 26 | ) 27 | ) 28 | def test_valid(self, node: cst.CSTNode, code: str) -> None: 29 | self.validate_node(node, code) 30 | -------------------------------------------------------------------------------- /libcst/_parser/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_parser/conversions/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_parser/conversions/module.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # pyre-unsafe 6 | 7 | from typing import Any, Sequence 8 | 9 | from libcst._nodes.module import Module 10 | from libcst._nodes.whitespace import NEWLINE_RE 11 | from libcst._parser.production_decorator import with_production 12 | from libcst._parser.types.config import ParserConfig 13 | 14 | 15 | @with_production("file_input", "(NEWLINE | stmt)* ENDMARKER") 16 | def convert_file_input(config: ParserConfig, children: Sequence[Any]) -> Any: 17 | *body, footer = children 18 | if len(body) == 0: 19 | # If there's no body, the header and footer are ambiguous. The header is more 20 | # important, and should own the EmptyLine nodes instead of the footer. 21 | header = footer 22 | footer = () 23 | if ( 24 | len(config.lines) == 2 25 | and NEWLINE_RE.fullmatch(config.lines[0]) 26 | and config.lines[1] == "" 27 | ): 28 | # This is an empty file (not even a comment), so special-case this to an 29 | # empty list instead of a single dummy EmptyLine (which is what we'd 30 | # normally parse). 31 | header = () 32 | else: 33 | # Steal the leading lines from the first statement, and move them into the 34 | # header. 35 | first_stmt = body[0] 36 | header = first_stmt.leading_lines 37 | body[0] = first_stmt.with_changes(leading_lines=()) 38 | return Module( 39 | header=header, 40 | body=body, 41 | footer=footer, 42 | encoding=config.encoding, 43 | default_indent=config.default_indent, 44 | default_newline=config.default_newline, 45 | has_trailing_newline=config.has_trailing_newline, 46 | ) 47 | -------------------------------------------------------------------------------- /libcst/_parser/custom_itertools.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from itertools import zip_longest 7 | from typing import Iterable, Iterator, TypeVar 8 | 9 | _T = TypeVar("_T") 10 | 11 | 12 | # https://docs.python.org/3/library/itertools.html#itertools-recipes 13 | def grouper(iterable: Iterable[_T], n: int, fillvalue: _T = None) -> Iterator[_T]: 14 | "Collect data into fixed-length chunks or blocks" 15 | # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx" 16 | args = [iter(iterable)] * n 17 | return zip_longest(*args, fillvalue=fillvalue) 18 | -------------------------------------------------------------------------------- /libcst/_parser/parso/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_parser/parso/pgen2/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_parser/parso/python/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_parser/parso/python/py_token.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | # 4 | # Modifications: 5 | # Copyright David Halter and Contributors 6 | # Modifications are dual-licensed: MIT and PSF. 7 | # 99% of the code is different from pgen2, now. 8 | # 9 | # A fork of `parso.python.token`. 10 | # https://github.com/davidhalter/parso/blob/master/parso/python/token.py 11 | # 12 | # The following changes were made: 13 | # - Explicit TokenType references instead of dynamic creation. 14 | # - Use dataclasses instead of raw classes. 15 | # pyre-unsafe 16 | 17 | from dataclasses import dataclass 18 | 19 | 20 | @dataclass(frozen=True) 21 | class TokenType: 22 | name: str 23 | contains_syntax: bool = False 24 | 25 | def __repr__(self) -> str: 26 | return "%s(%s)" % (self.__class__.__name__, self.name) 27 | 28 | 29 | class PythonTokenTypes: 30 | """ 31 | Basically an enum, but Python 2 doesn't have enums in the standard library. 32 | """ 33 | 34 | STRING: TokenType = TokenType("STRING") 35 | NUMBER: TokenType = TokenType("NUMBER") 36 | NAME: TokenType = TokenType("NAME", contains_syntax=True) 37 | ERRORTOKEN: TokenType = TokenType("ERRORTOKEN") 38 | NEWLINE: TokenType = TokenType("NEWLINE") 39 | INDENT: TokenType = TokenType("INDENT") 40 | DEDENT: TokenType = TokenType("DEDENT") 41 | ERROR_DEDENT: TokenType = TokenType("ERROR_DEDENT") 42 | ASYNC: TokenType = TokenType("ASYNC") 43 | AWAIT: TokenType = TokenType("AWAIT") 44 | FSTRING_STRING: TokenType = TokenType("FSTRING_STRING") 45 | FSTRING_START: TokenType = TokenType("FSTRING_START") 46 | FSTRING_END: TokenType = TokenType("FSTRING_END") 47 | OP: TokenType = TokenType("OP", contains_syntax=True) 48 | ENDMARKER: TokenType = TokenType("ENDMARKER") 49 | -------------------------------------------------------------------------------- /libcst/_parser/parso/python/token.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | try: 7 | from libcst_native import token_type as native_token_type 8 | 9 | TokenType = native_token_type.TokenType 10 | 11 | class PythonTokenTypes: 12 | STRING: TokenType = native_token_type.STRING 13 | NUMBER: TokenType = native_token_type.NUMBER 14 | NAME: TokenType = native_token_type.NAME 15 | NEWLINE: TokenType = native_token_type.NEWLINE 16 | INDENT: TokenType = native_token_type.INDENT 17 | DEDENT: TokenType = native_token_type.DEDENT 18 | ASYNC: TokenType = native_token_type.ASYNC 19 | AWAIT: TokenType = native_token_type.AWAIT 20 | FSTRING_STRING: TokenType = native_token_type.FSTRING_STRING 21 | FSTRING_START: TokenType = native_token_type.FSTRING_START 22 | FSTRING_END: TokenType = native_token_type.FSTRING_END 23 | OP: TokenType = native_token_type.OP 24 | ENDMARKER: TokenType = native_token_type.ENDMARKER 25 | # unused dummy tokens for backwards compat with the parso tokenizer 26 | ERRORTOKEN: TokenType = native_token_type.ERRORTOKEN 27 | ERROR_DEDENT: TokenType = native_token_type.ERROR_DEDENT 28 | 29 | except ImportError: 30 | from libcst._parser.parso.python.py_token import ( # noqa F401 31 | PythonTokenTypes, 32 | TokenType, 33 | ) 34 | -------------------------------------------------------------------------------- /libcst/_parser/parso/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_parser/parso/tests/test_fstring.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | # 4 | # Modifications: 5 | # Copyright David Halter and Contributors 6 | # Modifications are dual-licensed: MIT and PSF. 7 | # 99% of the code is different from pgen2, now. 8 | # 9 | # A fork of Parso's tokenize test 10 | # https://github.com/davidhalter/parso/blob/master/test/test_tokenize.py 11 | # 12 | # The following changes were made: 13 | # - Convert base test to Unittet 14 | # - Remove grammar-specific tests 15 | # pyre-unsafe 16 | from libcst._parser.parso.python.tokenize import tokenize 17 | from libcst._parser.parso.utils import parse_version_string 18 | from libcst.testing.utils import data_provider, UnitTest 19 | 20 | 21 | class ParsoTokenizeTest(UnitTest): 22 | @data_provider( 23 | ( 24 | # 2 times 2, 5 because python expr and endmarker. 25 | ('f"}{"', [(1, 0), (1, 2), (1, 3), (1, 4), (1, 5)]), 26 | ( 27 | 'f" :{ 1 : } "', 28 | [ 29 | (1, 0), 30 | (1, 2), 31 | (1, 4), 32 | (1, 6), 33 | (1, 8), 34 | (1, 9), 35 | (1, 10), 36 | (1, 11), 37 | (1, 12), 38 | (1, 13), 39 | ], 40 | ), 41 | ( 42 | 'f"""\n {\nfoo\n }"""', 43 | [(1, 0), (1, 4), (2, 1), (3, 0), (4, 1), (4, 2), (4, 5)], 44 | ), 45 | ) 46 | ) 47 | def test_tokenize_start_pos(self, code, positions): 48 | tokens = list(tokenize(code, version_info=parse_version_string("3.6"))) 49 | assert positions == [p.start_pos for p in tokens] 50 | -------------------------------------------------------------------------------- /libcst/_parser/parso/tests/test_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2004-2005 Elemental Security, Inc. All Rights Reserved. 2 | # Licensed to PSF under a Contributor Agreement. 3 | # 4 | # Modifications: 5 | # Copyright David Halter and Contributors 6 | # Modifications are dual-licensed: MIT and PSF. 7 | # 99% of the code is different from pgen2, now. 8 | # 9 | # A fork of Parso's tokenize test 10 | # https://github.com/davidhalter/parso/blob/master/test/test_tokenize.py 11 | # 12 | # The following changes were made: 13 | # - Convert base test to Unittet 14 | # - Remove grammar-specific tests 15 | # pyre-unsafe 16 | from libcst._parser.parso.utils import python_bytes_to_unicode, split_lines 17 | from libcst.testing.utils import data_provider, UnitTest 18 | 19 | 20 | class ParsoUtilsTest(UnitTest): 21 | @data_provider( 22 | ( 23 | ("asd\r\n", ["asd", ""], False), 24 | ("asd\r\n", ["asd\r\n", ""], True), 25 | ("asd\r", ["asd", ""], False), 26 | ("asd\r", ["asd\r", ""], True), 27 | ("asd\n", ["asd", ""], False), 28 | ("asd\n", ["asd\n", ""], True), 29 | ("asd\r\n\f", ["asd", "\f"], False), 30 | ("asd\r\n\f", ["asd\r\n", "\f"], True), 31 | ("\fasd\r\n", ["\fasd", ""], False), 32 | ("\fasd\r\n", ["\fasd\r\n", ""], True), 33 | ("", [""], False), 34 | ("", [""], True), 35 | ("\n", ["", ""], False), 36 | ("\n", ["\n", ""], True), 37 | ("\r", ["", ""], False), 38 | ("\r", ["\r", ""], True), 39 | # Invalid line breaks 40 | ("a\vb", ["a\vb"], False), 41 | ("a\vb", ["a\vb"], True), 42 | ("\x1c", ["\x1c"], False), 43 | ("\x1c", ["\x1c"], True), 44 | ) 45 | ) 46 | def test_split_lines(self, string, expected_result, keepends): 47 | assert split_lines(string, keepends=keepends) == expected_result 48 | 49 | def test_python_bytes_to_unicode_unicode_text(self): 50 | source = ( 51 | b"# vim: fileencoding=utf-8\n" 52 | + b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n" 53 | ) 54 | actual = python_bytes_to_unicode(source) 55 | expected = source.decode("utf-8") 56 | assert actual == expected 57 | -------------------------------------------------------------------------------- /libcst/_parser/production_decorator.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Callable, Optional, Sequence, TypeVar 7 | 8 | from libcst._parser.types.conversions import NonterminalConversion 9 | from libcst._parser.types.production import Production 10 | 11 | _NonterminalConversionT = TypeVar( 12 | "_NonterminalConversionT", bound=NonterminalConversion 13 | ) 14 | 15 | 16 | # We could version our grammar at a later point by adding a version metadata kwarg to 17 | # this decorator. 18 | def with_production( 19 | production_name: str, 20 | children: str, 21 | *, 22 | version: Optional[str] = None, 23 | future: Optional[str] = None, 24 | # pyre-fixme[34]: `Variable[_NonterminalConversionT (bound to 25 | # typing.Callable[[libcst_native.parser_config.ParserConfig, 26 | # typing.Sequence[typing.Any]], typing.Any])]` isn't present in the function's 27 | # parameters. 28 | ) -> Callable[[_NonterminalConversionT], _NonterminalConversionT]: 29 | """ 30 | Attaches a bit of grammar to a conversion function. The parser extracts all of these 31 | production strings, and uses it to form the language's full grammar. 32 | 33 | If you need to attach multiple productions to the same conversion function 34 | """ 35 | 36 | def inner(fn: _NonterminalConversionT) -> _NonterminalConversionT: 37 | if not hasattr(fn, "productions"): 38 | fn.productions = [] 39 | # pyre-ignore: Pyre doesn't think that fn has a __name__ attribute 40 | fn_name = fn.__name__ 41 | if not fn_name.startswith("convert_"): 42 | raise Exception( 43 | "A function with a production must be named 'convert_X', not " 44 | + f"'{fn_name}'." 45 | ) 46 | # pyre-ignore: Pyre doesn't know about this magic field we added 47 | fn.productions.append(Production(production_name, children, version, future)) 48 | return fn 49 | 50 | return inner 51 | 52 | 53 | def get_productions(fn: NonterminalConversion) -> Sequence[Production]: 54 | # pyre-ignore Pyre doesn't know about this magic field we added 55 | return fn.productions 56 | -------------------------------------------------------------------------------- /libcst/_parser/python_parser.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # pyre-unsafe 6 | 7 | from typing import Any, Iterable, Mapping, Sequence 8 | 9 | from libcst._parser.base_parser import BaseParser 10 | from libcst._parser.grammar import get_nonterminal_conversions, get_terminal_conversions 11 | from libcst._parser.parso.pgen2.generator import Grammar 12 | from libcst._parser.parso.python.token import TokenType 13 | from libcst._parser.types.config import ParserConfig 14 | from libcst._parser.types.conversions import NonterminalConversion, TerminalConversion 15 | from libcst._parser.types.token import Token 16 | 17 | 18 | class PythonCSTParser(BaseParser[Token, TokenType, Any]): 19 | config: ParserConfig 20 | terminal_conversions: Mapping[str, TerminalConversion] 21 | nonterminal_conversions: Mapping[str, NonterminalConversion] 22 | 23 | def __init__( 24 | self, 25 | *, 26 | tokens: Iterable[Token], 27 | config: ParserConfig, 28 | pgen_grammar: "Grammar[TokenType]", 29 | start_nonterminal: str = "file_input", 30 | ) -> None: 31 | super().__init__( 32 | tokens=tokens, 33 | lines=config.lines, 34 | pgen_grammar=pgen_grammar, 35 | start_nonterminal=start_nonterminal, 36 | ) 37 | self.config = config 38 | self.terminal_conversions = get_terminal_conversions() 39 | self.nonterminal_conversions = get_nonterminal_conversions( 40 | config.version, config.future_imports 41 | ) 42 | 43 | def convert_nonterminal(self, nonterminal: str, children: Sequence[Any]) -> Any: 44 | return self.nonterminal_conversions[nonterminal](self.config, children) 45 | 46 | def convert_terminal(self, token: Token) -> Any: 47 | return self.terminal_conversions[token.type.name](self.config, token) 48 | -------------------------------------------------------------------------------- /libcst/_parser/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_parser/tests/test_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # pyre-strict 7 | from libcst._parser.parso.utils import PythonVersionInfo 8 | from libcst._parser.types.config import _pick_compatible_python_version 9 | from libcst.testing.utils import UnitTest 10 | 11 | 12 | class ConfigTest(UnitTest): 13 | def test_pick_compatible(self) -> None: 14 | self.assertEqual( 15 | PythonVersionInfo(3, 1), _pick_compatible_python_version("3.2") 16 | ) 17 | self.assertEqual( 18 | PythonVersionInfo(3, 1), _pick_compatible_python_version("3.1") 19 | ) 20 | self.assertEqual( 21 | PythonVersionInfo(3, 8), _pick_compatible_python_version("3.9") 22 | ) 23 | self.assertEqual( 24 | PythonVersionInfo(3, 8), _pick_compatible_python_version("3.10") 25 | ) 26 | self.assertEqual( 27 | PythonVersionInfo(3, 8), _pick_compatible_python_version("4.0") 28 | ) 29 | with self.assertRaisesRegex( 30 | ValueError, 31 | ( 32 | r"No version found older than 1\.0 \(PythonVersionInfo\(" 33 | + r"major=1, minor=0\)\) while running on" 34 | ), 35 | ): 36 | _pick_compatible_python_version("1.0") 37 | -------------------------------------------------------------------------------- /libcst/_parser/tests/test_node_identity.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | from collections import Counter 6 | from textwrap import dedent 7 | 8 | import libcst as cst 9 | from libcst.testing.utils import data_provider, UnitTest 10 | 11 | 12 | class DuplicateLeafNodeTest(UnitTest): 13 | @data_provider( 14 | ( 15 | # Simple program 16 | ( 17 | """ 18 | foo = 'toplevel' 19 | fn1(foo) 20 | fn2(foo) 21 | def fn_def(): 22 | foo = 'shadow' 23 | fn3(foo) 24 | """, 25 | ), 26 | ) 27 | ) 28 | def test_tokenize(self, code: str) -> None: 29 | test_case = self 30 | 31 | class CountVisitor(cst.CSTVisitor): 32 | def __init__(self) -> None: 33 | self.count = Counter() 34 | self.nodes = {} 35 | 36 | def on_visit(self, node: cst.CSTNode) -> bool: 37 | self.count[id(node)] += 1 38 | test_case.assertTrue( 39 | self.count[id(node)] == 1, 40 | f"Node duplication detected between {node} and {self.nodes.get(id(node))}", 41 | ) 42 | self.nodes[id(node)] = node 43 | return True 44 | 45 | module = cst.parse_module(dedent(code)) 46 | module.visit(CountVisitor()) 47 | -------------------------------------------------------------------------------- /libcst/_parser/tests/test_version_compare.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from libcst._parser.grammar import _should_include 7 | from libcst._parser.parso.utils import PythonVersionInfo 8 | from libcst.testing.utils import data_provider, UnitTest 9 | 10 | 11 | class VersionCompareTest(UnitTest): 12 | @data_provider( 13 | ( 14 | # Simple equality 15 | ("==3.6", PythonVersionInfo(3, 6), True), 16 | ("!=3.6", PythonVersionInfo(3, 6), False), 17 | # Equal or GT/LT 18 | (">=3.6", PythonVersionInfo(3, 5), False), 19 | (">=3.6", PythonVersionInfo(3, 6), True), 20 | (">=3.6", PythonVersionInfo(3, 7), True), 21 | ("<=3.6", PythonVersionInfo(3, 5), True), 22 | ("<=3.6", PythonVersionInfo(3, 6), True), 23 | ("<=3.6", PythonVersionInfo(3, 7), False), 24 | # GT/LT 25 | (">3.6", PythonVersionInfo(3, 5), False), 26 | (">3.6", PythonVersionInfo(3, 6), False), 27 | (">3.6", PythonVersionInfo(3, 7), True), 28 | ("<3.6", PythonVersionInfo(3, 5), True), 29 | ("<3.6", PythonVersionInfo(3, 6), False), 30 | ("<3.6", PythonVersionInfo(3, 7), False), 31 | # Multiple checks 32 | (">3.6,<3.8", PythonVersionInfo(3, 6), False), 33 | (">3.6,<3.8", PythonVersionInfo(3, 7), True), 34 | (">3.6,<3.8", PythonVersionInfo(3, 8), False), 35 | ) 36 | ) 37 | def test_tokenize( 38 | self, 39 | requested_version: str, 40 | actual_version: PythonVersionInfo, 41 | expected_result: bool, 42 | ) -> None: 43 | self.assertEqual( 44 | _should_include(requested_version, actual_version), expected_result 45 | ) 46 | -------------------------------------------------------------------------------- /libcst/_parser/types/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_parser/types/conversions.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Any, Callable, Sequence 7 | 8 | from libcst._parser.types.config import ParserConfig 9 | from libcst._parser.types.token import Token 10 | 11 | # pyre-fixme[33]: Aliased annotation cannot contain `Any`. 12 | NonterminalConversion = Callable[[ParserConfig, Sequence[Any]], Any] 13 | # pyre-fixme[33]: Aliased annotation cannot contain `Any`. 14 | TerminalConversion = Callable[[ParserConfig, Token], Any] 15 | -------------------------------------------------------------------------------- /libcst/_parser/types/production.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from dataclasses import dataclass 8 | from typing import Optional 9 | 10 | 11 | @dataclass(frozen=True) 12 | class Production: 13 | name: str 14 | children: str 15 | version: Optional[str] 16 | future: Optional[str] 17 | 18 | def __str__(self) -> str: 19 | return f"{self.name}: {self.children}" 20 | -------------------------------------------------------------------------------- /libcst/_parser/types/py_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import abc 7 | from dataclasses import asdict, dataclass 8 | from typing import Any, FrozenSet, Mapping, Sequence 9 | 10 | from libcst._parser.parso.utils import PythonVersionInfo 11 | 12 | 13 | class BaseWhitespaceParserConfig(abc.ABC): 14 | """ 15 | Represents the subset of ParserConfig that the whitespace parser requires. This 16 | makes calling the whitespace parser in tests with a mocked configuration easier. 17 | """ 18 | 19 | lines: Sequence[str] 20 | default_newline: str 21 | 22 | 23 | @dataclass(frozen=True) 24 | class MockWhitespaceParserConfig(BaseWhitespaceParserConfig): 25 | """ 26 | An internal type used by unit tests. 27 | """ 28 | 29 | lines: Sequence[str] 30 | default_newline: str 31 | 32 | 33 | @dataclass(frozen=True) 34 | class ParserConfig(BaseWhitespaceParserConfig): 35 | """ 36 | An internal configuration object that the python parser passes around. These 37 | values are global to the parsed code and should not change during the lifetime 38 | of the parser object. 39 | """ 40 | 41 | lines: Sequence[str] 42 | encoding: str 43 | default_indent: str 44 | default_newline: str 45 | has_trailing_newline: bool 46 | version: PythonVersionInfo 47 | future_imports: FrozenSet[str] 48 | 49 | 50 | def parser_config_asdict(config: ParserConfig) -> Mapping[str, Any]: 51 | """ 52 | An internal helper function used by unit tests to compare configs. 53 | """ 54 | return asdict(config) 55 | -------------------------------------------------------------------------------- /libcst/_parser/types/py_token.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from dataclasses import dataclass 8 | from typing import Optional, Tuple 9 | 10 | from libcst._add_slots import add_slots 11 | from libcst._parser.parso.python.token import TokenType 12 | from libcst._parser.types.whitespace_state import WhitespaceState 13 | 14 | 15 | @add_slots 16 | @dataclass(frozen=True) 17 | class Token: 18 | type: TokenType 19 | string: str 20 | # The start of where `string` is in the source, not including leading whitespace. 21 | start_pos: Tuple[int, int] 22 | # The end of where `string` is in the source, not including trailing whitespace. 23 | end_pos: Tuple[int, int] 24 | whitespace_before: WhitespaceState 25 | whitespace_after: WhitespaceState 26 | # The relative indent this token adds. 27 | relative_indent: Optional[str] 28 | -------------------------------------------------------------------------------- /libcst/_parser/types/py_whitespace_state.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from dataclasses import dataclass 7 | 8 | from libcst._add_slots import add_slots 9 | 10 | 11 | @add_slots 12 | @dataclass(frozen=False) 13 | class WhitespaceState: 14 | """ 15 | A frequently mutated store of the whitespace parser's current state. This object 16 | must be cloned prior to speculative parsing. 17 | 18 | This is in contrast to the `config` object each whitespace parser function takes, 19 | which is frozen and never mutated. 20 | 21 | Whitespace parsing works by mutating this state object. By encapsulating saving, and 22 | re-using state objects inside the top-level python parser, the whitespace parser is 23 | able to be reentrant. One 'convert' function can consume part of the whitespace, and 24 | another 'convert' function can consume the rest, depending on who owns what 25 | whitespace. 26 | 27 | This is similar to the approach you might take to parse nested languages (e.g. 28 | JavaScript inside of HTML). We're treating whitespace as a separate language and 29 | grammar from the rest of Python's grammar. 30 | """ 31 | 32 | line: int # one-indexed (to match parso's behavior) 33 | column: int # zero-indexed (to match parso's behavior) 34 | # What to look for when executing `_parse_indent`. 35 | absolute_indent: str 36 | is_parenthesized: bool 37 | -------------------------------------------------------------------------------- /libcst/_parser/types/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/_parser/types/tests/test_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Callable 7 | 8 | from libcst._parser.types.config import PartialParserConfig 9 | from libcst.testing.utils import data_provider, UnitTest 10 | 11 | 12 | class TestConfig(UnitTest): 13 | @data_provider( 14 | { 15 | "empty": (PartialParserConfig,), 16 | "python_version_a": (lambda: PartialParserConfig(python_version="3.7"),), 17 | "python_version_b": (lambda: PartialParserConfig(python_version="3.7.1"),), 18 | "encoding": (lambda: PartialParserConfig(encoding="latin-1"),), 19 | "default_indent": (lambda: PartialParserConfig(default_indent="\t "),), 20 | "default_newline": (lambda: PartialParserConfig(default_newline="\r\n"),), 21 | } 22 | ) 23 | def test_valid_partial_parser_config( 24 | self, factory: Callable[[], PartialParserConfig] 25 | ) -> None: 26 | self.assertIsInstance(factory(), PartialParserConfig) 27 | 28 | @data_provider( 29 | { 30 | "python_version": ( 31 | lambda: PartialParserConfig(python_version="3.7.1.0"), 32 | "The given version is not in the right format", 33 | ), 34 | "python_version_unsupported": ( 35 | lambda: PartialParserConfig(python_version="3.4"), 36 | "LibCST can only parse code using one of the following versions of Python's grammar", 37 | ), 38 | "encoding": ( 39 | lambda: PartialParserConfig(encoding="utf-42"), 40 | "not a supported encoding", 41 | ), 42 | "default_indent": ( 43 | lambda: PartialParserConfig(default_indent="badinput"), 44 | "invalid value for default_indent", 45 | ), 46 | "default_newline": ( 47 | lambda: PartialParserConfig(default_newline="\n\r"), 48 | "invalid value for default_newline", 49 | ), 50 | } 51 | ) 52 | def test_invalid_partial_parser_config( 53 | self, factory: Callable[[], PartialParserConfig], expected_re: str 54 | ) -> None: 55 | with self.assertRaisesRegex(ValueError, expected_re): 56 | factory() 57 | -------------------------------------------------------------------------------- /libcst/_parser/types/token.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | try: 8 | from libcst_native import tokenize 9 | 10 | Token = tokenize.Token 11 | except ImportError: 12 | from libcst._parser.types.py_token import Token # noqa F401 13 | -------------------------------------------------------------------------------- /libcst/_parser/types/whitespace_state.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | """ 7 | Defines the state object used by the whitespace parser. 8 | """ 9 | 10 | try: 11 | from libcst_native import whitespace_state as mod 12 | except ImportError: 13 | from libcst._parser.types import py_whitespace_state as mod 14 | 15 | WhitespaceState = mod.WhitespaceState 16 | -------------------------------------------------------------------------------- /libcst/_parser/whitespace_parser.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | """ 7 | Parso doesn't attempt to parse (or even emit tokens for) whitespace or comments that 8 | aren't syntatically important. Instead, we're just given the whitespace as a "prefix" of 9 | the token. 10 | 11 | However, in our CST, whitespace is gathered into far more detailed objects than a simple 12 | str. 13 | 14 | Fortunately this isn't hard for us to parse ourselves, so we just use our own 15 | hand-rolled recursive descent parser. 16 | """ 17 | 18 | try: 19 | # It'd be better to do `from libcst_native.whitespace_parser import *`, but we're 20 | # blocked on https://github.com/PyO3/pyo3/issues/759 21 | # (which ultimately seems to be a limitation of how importlib works) 22 | from libcst_native import whitespace_parser as mod 23 | except ImportError: 24 | from libcst._parser import py_whitespace_parser as mod 25 | 26 | parse_simple_whitespace = mod.parse_simple_whitespace 27 | parse_empty_lines = mod.parse_empty_lines 28 | parse_trailing_whitespace = mod.parse_trailing_whitespace 29 | parse_parenthesizable_whitespace = mod.parse_parenthesizable_whitespace 30 | -------------------------------------------------------------------------------- /libcst/_position.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | """ 8 | Data structures used for storing position information. 9 | 10 | These are publicly exported by metadata, but their implementation lives outside of 11 | metadata, because they're used internally by the codegen logic, which computes position 12 | locations. 13 | """ 14 | 15 | from dataclasses import dataclass 16 | from typing import cast, overload, Tuple, Union 17 | 18 | from libcst._add_slots import add_slots 19 | 20 | _CodePositionT = Union[Tuple[int, int], "CodePosition"] 21 | 22 | 23 | @add_slots 24 | @dataclass(frozen=True) 25 | class CodePosition: 26 | #: Line numbers are 1-indexed. 27 | line: int 28 | #: Column numbers are 0-indexed. 29 | column: int 30 | 31 | 32 | @add_slots 33 | @dataclass(frozen=True) 34 | # pyre-fixme[13]: Attribute `end` is never initialized. 35 | # pyre-fixme[13]: Attribute `start` is never initialized. 36 | class CodeRange: 37 | #: Starting position of a node (inclusive). 38 | start: CodePosition 39 | #: Ending position of a node (exclusive). 40 | end: CodePosition 41 | 42 | @overload 43 | def __init__(self, start: CodePosition, end: CodePosition) -> None: ... 44 | 45 | @overload 46 | def __init__(self, start: Tuple[int, int], end: Tuple[int, int]) -> None: ... 47 | 48 | def __init__(self, start: _CodePositionT, end: _CodePositionT) -> None: 49 | if isinstance(start, tuple) and isinstance(end, tuple): 50 | object.__setattr__(self, "start", CodePosition(start[0], start[1])) 51 | object.__setattr__(self, "end", CodePosition(end[0], end[1])) 52 | else: 53 | start = cast(CodePosition, start) 54 | end = cast(CodePosition, end) 55 | object.__setattr__(self, "start", start) 56 | object.__setattr__(self, "end", end) 57 | -------------------------------------------------------------------------------- /libcst/_removal_sentinel.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | """ 7 | Used by visitors. This is hoisted into a separate module to avoid some circular 8 | dependencies in the definition of CSTNode. 9 | """ 10 | 11 | from enum import auto, Enum 12 | 13 | 14 | class RemovalSentinel(Enum): 15 | """ 16 | A :attr:`RemovalSentinel.REMOVE` value should be returned by a 17 | :meth:`CSTTransformer.on_leave` method when we want to remove that child from its 18 | parent. As a convenience, this can be constructed by calling 19 | :func:`libcst.RemoveFromParent`. 20 | 21 | The parent node should make a best-effort to remove the child, but may raise an 22 | exception when removing the child doesn't make sense, or could change the semantics 23 | in an unexpected way. For example, a function definition with no name doesn't make 24 | sense, but removing one of the arguments is valid. 25 | 26 | In we can't automatically remove the child, the developer should instead remove the 27 | child by constructing a new parent in the parent's :meth:`~CSTTransformer.on_leave` 28 | call. 29 | 30 | We use this instead of ``None`` to force developers to be explicit about deletions. 31 | Because ``None`` is the default return value for a function with no return 32 | statement, it would be too easy to accidentally delete nodes from the tree by 33 | forgetting to return a value. 34 | """ 35 | 36 | REMOVE = auto() 37 | 38 | 39 | def RemoveFromParent() -> RemovalSentinel: 40 | """ 41 | A convenience method for requesting that this node be removed by its parent. 42 | Use this in place of returning :class:`RemovalSentinel` directly. 43 | For example, to remove all arguments unconditionally:: 44 | 45 | def leave_Arg( 46 | self, original_node: cst.Arg, updated_node: cst.Arg 47 | ) -> Union[cst.Arg, cst.RemovalSentinel]: 48 | return RemoveFromParent() 49 | """ 50 | return RemovalSentinel.REMOVE 51 | -------------------------------------------------------------------------------- /libcst/_tabs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | def expand_tabs(line: str) -> str: 8 | """ 9 | Tabs are treated as 1-8 spaces according to 10 | https://docs.python.org/3/reference/lexical_analysis.html#indentation 11 | 12 | Given a string with tabs, this removes all tab characters and replaces them with the 13 | appropriate number of spaces. 14 | """ 15 | result_list = [] 16 | total = 0 17 | for ch in line: 18 | if ch == "\t": 19 | prev_total = total 20 | total = ((total + 8) // 8) * 8 21 | result_list.append(" " * (total - prev_total)) 22 | else: 23 | total += 1 24 | result_list.append(ch) 25 | 26 | return "".join(result_list) 27 | -------------------------------------------------------------------------------- /libcst/_typed_visitor_base.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Any, Callable, cast, TypeVar 7 | 8 | 9 | # pyre-fixme[24]: Generic type `Callable` expects 2 type parameters. 10 | F = TypeVar("F", bound=Callable) 11 | 12 | 13 | def mark_no_op(f: F) -> F: 14 | """ 15 | Annotates stubs with a field to indicate they should not be collected 16 | by BatchableCSTVisitor.get_visitors() to reduce function call 17 | overhead when running a batched visitor pass. 18 | """ 19 | 20 | cast(Any, f)._is_no_op = True 21 | return f 22 | -------------------------------------------------------------------------------- /libcst/_types.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from pathlib import PurePath 8 | from typing import TYPE_CHECKING, TypeVar, Union 9 | 10 | if TYPE_CHECKING: 11 | from libcst._nodes.base import CSTNode # noqa: F401 12 | 13 | 14 | CSTNodeT = TypeVar("CSTNodeT", bound="CSTNode") 15 | CSTNodeT_co = TypeVar("CSTNodeT_co", bound="CSTNode", covariant=True) 16 | StrPath = Union[str, PurePath] 17 | -------------------------------------------------------------------------------- /libcst/codegen/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/codegen/gen_type_mapping.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import List 7 | 8 | from libcst.codegen.gather import imports, nodebases, nodeuses 9 | 10 | generated_code: List[str] = [] 11 | generated_code.append("# Copyright (c) Meta Platforms, Inc. and affiliates.") 12 | generated_code.append("#") 13 | generated_code.append( 14 | "# This source code is licensed under the MIT license found in the" 15 | ) 16 | generated_code.append("# LICENSE file in the root directory of this source tree.") 17 | generated_code.append("") 18 | generated_code.append("") 19 | generated_code.append("# This file was generated by libcst.codegen.gen_type_mapping") 20 | generated_code.append("from typing import Dict as TypingDict, Type, Union") 21 | generated_code.append("") 22 | generated_code.append("from libcst._maybe_sentinel import MaybeSentinel") 23 | generated_code.append("from libcst._removal_sentinel import RemovalSentinel") 24 | generated_code.append("from libcst._nodes.base import CSTNode") 25 | 26 | # Import the types we use. These have to be type guarded since it would 27 | # cause an import cycle otherwise. 28 | generated_code.append("") 29 | generated_code.append("") 30 | for module, objects in imports.items(): 31 | generated_code.append(f"from {module} import (") 32 | generated_code.append(f" {', '.join(sorted(objects))}") 33 | generated_code.append(")") 34 | 35 | # Generate the base visit_ methods 36 | generated_code.append("") 37 | generated_code.append("") 38 | generated_code.append( 39 | "TYPED_FUNCTION_RETURN_MAPPING: TypingDict[Type[CSTNode], object] = {" 40 | ) 41 | for node in sorted(nodebases.keys(), key=lambda node: node.__name__): 42 | name = node.__name__ 43 | if name.startswith("Base"): 44 | continue 45 | valid_return_types: List[str] = [nodebases[node].__name__] 46 | node_uses = nodeuses[node] 47 | base_uses = nodeuses[nodebases[node]] 48 | if node_uses.maybe or base_uses.maybe: 49 | valid_return_types.append("MaybeSentinel") 50 | if ( 51 | node_uses.optional 52 | or node_uses.sequence 53 | or base_uses.optional 54 | or base_uses.sequence 55 | ): 56 | valid_return_types.append("RemovalSentinel") 57 | generated_code.append(f' {name}: Union[{", ".join(valid_return_types)}],') 58 | generated_code.append("}") 59 | 60 | if __name__ == "__main__": 61 | # Output the code 62 | print("\n".join(generated_code)) 63 | -------------------------------------------------------------------------------- /libcst/codegen/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/codegen/transforms.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # This holds a series of transforms that help prettify generated code. 7 | # The design is such that any of them could be left out and the code 8 | # in question will still be correct, but possibly uglier to look at. 9 | # Great care should be taken to include only transforms that do not 10 | # affect the behavior of generated code, only the style for readability. 11 | # As a result, since these can be skipped without harm, it is okay to 12 | # use features such as matchers which rely on previously generated 13 | # code to function. 14 | 15 | import ast 16 | 17 | import libcst as cst 18 | import libcst.matchers as m 19 | 20 | 21 | class SimplifyUnionsTransformer(m.MatcherDecoratableTransformer): 22 | @m.leave(m.Subscript(m.Name("Union"))) 23 | def _leave_union( 24 | self, original_node: cst.Subscript, updated_node: cst.Subscript 25 | ) -> cst.BaseExpression: 26 | if len(updated_node.slice) == 1: 27 | # This is a Union[SimpleType,] which is equivalent to just SimpleType 28 | return cst.ensure_type(updated_node.slice[0].slice, cst.Index).value 29 | return updated_node 30 | 31 | 32 | class DoubleQuoteForwardRefsTransformer(m.MatcherDecoratableTransformer): 33 | @m.call_if_inside(m.Annotation()) 34 | def leave_SimpleString( 35 | self, original_node: cst.SimpleString, updated_node: cst.SimpleString 36 | ) -> cst.SimpleString: 37 | # For prettiness, convert all single-quoted forward refs to double-quoted. 38 | if "'" in updated_node.quote: 39 | new_value = f'"{updated_node.value[1:-1]}"' 40 | try: 41 | if updated_node.evaluated_value == ast.literal_eval(new_value): 42 | return updated_node.with_changes(value=new_value) 43 | except SyntaxError: 44 | pass 45 | return updated_node 46 | -------------------------------------------------------------------------------- /libcst/codemod/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from libcst.codemod._cli import ( 7 | diff_code, 8 | exec_transform_with_prettyprint, 9 | gather_files, 10 | parallel_exec_transform_with_prettyprint, 11 | ParallelTransformResult, 12 | ) 13 | from libcst.codemod._codemod import Codemod 14 | from libcst.codemod._command import ( 15 | CodemodCommand, 16 | MagicArgsCodemodCommand, 17 | VisitorBasedCodemodCommand, 18 | ) 19 | from libcst.codemod._context import CodemodContext 20 | from libcst.codemod._runner import ( 21 | SkipFile, 22 | SkipReason, 23 | transform_module, 24 | TransformExit, 25 | TransformFailure, 26 | TransformResult, 27 | TransformSkip, 28 | TransformSuccess, 29 | ) 30 | from libcst.codemod._testing import CodemodTest 31 | from libcst.codemod._visitor import ContextAwareTransformer, ContextAwareVisitor 32 | 33 | __all__ = [ 34 | "Codemod", 35 | "CodemodContext", 36 | "CodemodCommand", 37 | "VisitorBasedCodemodCommand", 38 | "MagicArgsCodemodCommand", 39 | "ContextAwareTransformer", 40 | "ContextAwareVisitor", 41 | "ParallelTransformResult", 42 | "TransformSuccess", 43 | "TransformFailure", 44 | "TransformExit", 45 | "SkipReason", 46 | "TransformSkip", 47 | "SkipFile", 48 | "TransformResult", 49 | "CodemodTest", 50 | "transform_module", 51 | "gather_files", 52 | "exec_transform_with_prettyprint", 53 | "parallel_exec_transform_with_prettyprint", 54 | "diff_code", 55 | ] 56 | -------------------------------------------------------------------------------- /libcst/codemod/_dummy_pool.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import sys 7 | from concurrent.futures import Executor, Future 8 | from types import TracebackType 9 | from typing import Callable, Optional, Type, TypeVar 10 | 11 | if sys.version_info >= (3, 10): 12 | from typing import ParamSpec 13 | else: 14 | from typing_extensions import ParamSpec 15 | 16 | Return = TypeVar("Return") 17 | Params = ParamSpec("Params") 18 | 19 | 20 | class DummyExecutor(Executor): 21 | """ 22 | Synchronous dummy `concurrent.futures.Executor` analogue. 23 | """ 24 | 25 | def submit( 26 | self, 27 | fn: Callable[Params, Return], 28 | /, 29 | *args: Params.args, 30 | **kwargs: Params.kwargs, 31 | ) -> Future[Return]: 32 | future: Future[Return] = Future() 33 | try: 34 | result = fn(*args, **kwargs) 35 | future.set_result(result) 36 | except Exception as exc: 37 | future.set_exception(exc) 38 | return future 39 | 40 | def __enter__(self) -> "DummyExecutor": 41 | return self 42 | 43 | def __exit__( 44 | self, 45 | exc_type: Optional[Type[BaseException]], 46 | exc_val: Optional[BaseException], 47 | exc_tb: Optional[TracebackType], 48 | ) -> None: 49 | pass 50 | -------------------------------------------------------------------------------- /libcst/codemod/commands/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | -------------------------------------------------------------------------------- /libcst/codemod/commands/add_pyre_directive.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | import re 7 | from abc import ABC 8 | from typing import Pattern 9 | 10 | import libcst 11 | from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand 12 | from libcst.helpers import insert_header_comments 13 | 14 | 15 | class AddPyreDirectiveCommand(VisitorBasedCodemodCommand, ABC): 16 | PYRE_TAG: str 17 | 18 | def __init__(self, context: CodemodContext) -> None: 19 | super().__init__(context) 20 | self._regex_pattern: Pattern[str] = re.compile( 21 | rf"^#\s+pyre-{self.PYRE_TAG}\s*$" 22 | ) 23 | self.needs_add = True 24 | 25 | def visit_Comment(self, node: libcst.Comment) -> None: 26 | if self._regex_pattern.search(node.value): 27 | self.needs_add = False 28 | 29 | def leave_Module( 30 | self, original_node: libcst.Module, updated_node: libcst.Module 31 | ) -> libcst.Module: 32 | # If the tag already exists, don't modify the file. 33 | if not self.needs_add: 34 | return updated_node 35 | 36 | return insert_header_comments(updated_node, [f"# pyre-{self.PYRE_TAG}"]) 37 | 38 | 39 | class AddPyreStrictCommand(AddPyreDirectiveCommand): 40 | """ 41 | Given a source file, we'll add the strict tag if the file doesn't already 42 | contain it. 43 | """ 44 | 45 | PYRE_TAG: str = "strict" 46 | 47 | DESCRIPTION: str = "Add the 'pyre-strict' tag to a module." 48 | 49 | 50 | class AddPyreUnsafeCommand(AddPyreDirectiveCommand): 51 | """ 52 | Given a source file, we'll add the unsafe tag if the file doesn't already 53 | contain it. 54 | """ 55 | 56 | PYRE_TAG: str = "unsafe" 57 | 58 | DESCRIPTION: str = "Add the 'pyre-unsafe' tag to a module." 59 | -------------------------------------------------------------------------------- /libcst/codemod/commands/convert_union_to_or.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | # pyre-strict 7 | 8 | import libcst as cst 9 | from libcst.codemod import VisitorBasedCodemodCommand 10 | from libcst.codemod.visitors import RemoveImportsVisitor 11 | from libcst.metadata import QualifiedName, QualifiedNameProvider, QualifiedNameSource 12 | 13 | 14 | class ConvertUnionToOrCommand(VisitorBasedCodemodCommand): 15 | DESCRIPTION: str = "Convert `Union[A, B]` to `A | B` in Python 3.10+" 16 | 17 | METADATA_DEPENDENCIES = (QualifiedNameProvider,) 18 | 19 | def leave_Subscript( 20 | self, original_node: cst.Subscript, updated_node: cst.Subscript 21 | ) -> cst.BaseExpression: 22 | """ 23 | Given a subscript, check if it's a Union - if so, either flatten the members 24 | into a nested BitOr (if multiple members) or unwrap the type (if only one member). 25 | """ 26 | if not QualifiedNameProvider.has_name( 27 | self, 28 | original_node, 29 | QualifiedName(name="typing.Union", source=QualifiedNameSource.IMPORT), 30 | ): 31 | return updated_node 32 | types = [ 33 | cst.ensure_type( 34 | cst.ensure_type(s, cst.SubscriptElement).slice, cst.Index 35 | ).value 36 | for s in updated_node.slice 37 | ] 38 | if len(types) == 1: 39 | return types[0] 40 | else: 41 | replacement = cst.BinaryOperation( 42 | left=types[0], right=types[1], operator=cst.BitOr() 43 | ) 44 | for type_ in types[2:]: 45 | replacement = cst.BinaryOperation( 46 | left=replacement, right=type_, operator=cst.BitOr() 47 | ) 48 | return replacement 49 | 50 | def leave_Module( 51 | self, original_node: cst.Module, updated_node: cst.Module 52 | ) -> cst.Module: 53 | RemoveImportsVisitor.remove_unused_import( 54 | self.context, module="typing", obj="Union" 55 | ) 56 | return updated_node 57 | -------------------------------------------------------------------------------- /libcst/codemod/commands/ensure_import_present.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | import argparse 7 | from typing import Generator, Type 8 | 9 | from libcst.codemod import Codemod, MagicArgsCodemodCommand 10 | from libcst.codemod.visitors import AddImportsVisitor 11 | 12 | 13 | class EnsureImportPresentCommand(MagicArgsCodemodCommand): 14 | DESCRIPTION: str = ( 15 | "Given a module and possibly an entity in that module, add an import " 16 | + "as long as one does not already exist." 17 | ) 18 | 19 | @staticmethod 20 | def add_args(arg_parser: argparse.ArgumentParser) -> None: 21 | arg_parser.add_argument( 22 | "--module", 23 | dest="module", 24 | metavar="MODULE", 25 | help="Module that should be imported.", 26 | type=str, 27 | required=True, 28 | ) 29 | arg_parser.add_argument( 30 | "--entity", 31 | dest="entity", 32 | metavar="ENTITY", 33 | help=( 34 | "Entity that should be imported from module. If left empty, entire " 35 | + " module will be imported." 36 | ), 37 | type=str, 38 | default=None, 39 | ) 40 | arg_parser.add_argument( 41 | "--alias", 42 | dest="alias", 43 | metavar="ALIAS", 44 | help=( 45 | "Alias that will be used for the imported module or entity. If left " 46 | + "empty, no alias will be applied." 47 | ), 48 | type=str, 49 | default=None, 50 | ) 51 | 52 | def get_transforms(self) -> Generator[Type[Codemod], None, None]: 53 | AddImportsVisitor.add_needed_import( 54 | self.context, 55 | self.context.scratch["module"], 56 | self.context.scratch["entity"], 57 | self.context.scratch["alias"], 58 | ) 59 | yield AddImportsVisitor 60 | -------------------------------------------------------------------------------- /libcst/codemod/commands/fix_variadic_callable.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | # pyre-strict 7 | 8 | import libcst as cst 9 | import libcst.matchers as m 10 | from libcst.codemod import VisitorBasedCodemodCommand 11 | from libcst.metadata import QualifiedName, QualifiedNameProvider, QualifiedNameSource 12 | 13 | 14 | class FixVariadicCallableCommmand(VisitorBasedCodemodCommand): 15 | DESCRIPTION: str = ( 16 | "Fix incorrect variadic callable type annotations from `Callable[[...], T]` to `Callable[..., T]``" 17 | ) 18 | 19 | METADATA_DEPENDENCIES = (QualifiedNameProvider,) 20 | 21 | def leave_Subscript( 22 | self, original_node: cst.Subscript, updated_node: cst.Subscript 23 | ) -> cst.BaseExpression: 24 | if QualifiedNameProvider.has_name( 25 | self, 26 | original_node, 27 | QualifiedName(name="typing.Callable", source=QualifiedNameSource.IMPORT), 28 | ): 29 | node_matches = len(updated_node.slice) == 2 and m.matches( 30 | updated_node.slice[0], 31 | m.SubscriptElement( 32 | slice=m.Index(value=m.List(elements=[m.Element(m.Ellipsis())])) 33 | ), 34 | ) 35 | 36 | if node_matches: 37 | slices = list(updated_node.slice) 38 | slices[0] = cst.SubscriptElement(cst.Index(cst.Ellipsis())) 39 | return updated_node.with_changes(slice=slices) 40 | return updated_node 41 | -------------------------------------------------------------------------------- /libcst/codemod/commands/noop.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from libcst import Module 7 | from libcst.codemod import CodemodCommand 8 | 9 | 10 | class NOOPCommand(CodemodCommand): 11 | DESCRIPTION: str = "Does absolutely nothing." 12 | 13 | def transform_module_impl(self, tree: Module) -> Module: 14 | # Return the tree as-is, with absolutely no modification 15 | return tree 16 | -------------------------------------------------------------------------------- /libcst/codemod/commands/remove_pyre_directive.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | import re 7 | from abc import ABC 8 | from typing import Pattern, Union 9 | 10 | import libcst 11 | from libcst.codemod import CodemodContext, VisitorBasedCodemodCommand 12 | 13 | 14 | class RemovePyreDirectiveCommand(VisitorBasedCodemodCommand, ABC): 15 | PYRE_TAG: str 16 | 17 | def __init__(self, context: CodemodContext) -> None: 18 | super().__init__(context) 19 | self._regex_pattern: Pattern[str] = re.compile( 20 | rf"^#\s+pyre-{self.PYRE_TAG}\s*$" 21 | ) 22 | 23 | def leave_EmptyLine( 24 | self, original_node: libcst.EmptyLine, updated_node: libcst.EmptyLine 25 | ) -> Union[libcst.EmptyLine, libcst.RemovalSentinel]: 26 | if updated_node.comment is None or not bool( 27 | self._regex_pattern.search( 28 | libcst.ensure_type(updated_node.comment, libcst.Comment).value 29 | ) 30 | ): 31 | # This is a normal comment 32 | return updated_node 33 | # This is a directive comment matching our tag, so remove it. 34 | return libcst.RemoveFromParent() 35 | 36 | 37 | class RemovePyreStrictCommand(RemovePyreDirectiveCommand): 38 | """ 39 | Given a source file, we'll remove the any strict tag if the file already 40 | contains it. 41 | """ 42 | 43 | DESCRIPTION: str = "Removes the 'pyre-strict' tag from a module." 44 | 45 | PYRE_TAG: str = "strict" 46 | 47 | 48 | class RemovePyreUnsafeCommand(RemovePyreDirectiveCommand): 49 | """ 50 | Given a source file, we'll remove the any unsafe tag if the file already 51 | contains it. 52 | """ 53 | 54 | DESCRIPTION: str = "Removes the 'pyre-unsafe' tag from a module." 55 | 56 | PYRE_TAG: str = "unsafe" 57 | -------------------------------------------------------------------------------- /libcst/codemod/commands/rename_typing_generic_aliases.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | # pyre-strict 7 | from functools import partial 8 | from typing import cast, Generator 9 | 10 | from libcst.codemod import Codemod, MagicArgsCodemodCommand 11 | from libcst.codemod.commands.rename import RenameCommand 12 | 13 | 14 | class RenameTypingGenericAliases(MagicArgsCodemodCommand): 15 | DESCRIPTION: str = ( 16 | "Rename typing module aliases of builtin generics in Python 3.9+, for example: `typing.List` -> `list`" 17 | ) 18 | 19 | MAPPING: dict[str, str] = { 20 | "typing.List": "builtins.list", 21 | "typing.Tuple": "builtins.tuple", 22 | "typing.Dict": "builtins.dict", 23 | "typing.FrozenSet": "builtins.frozenset", 24 | "typing.Set": "builtins.set", 25 | "typing.Type": "builtins.type", 26 | } 27 | 28 | def get_transforms(self) -> Generator[type[Codemod], None, None]: 29 | for from_type, to_type in self.MAPPING.items(): 30 | yield cast( 31 | type[Codemod], 32 | partial( 33 | RenameCommand, 34 | old_name=from_type, 35 | new_name=to_type, 36 | ), 37 | ) 38 | -------------------------------------------------------------------------------- /libcst/codemod/commands/strip_strings_from_types.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from typing import Union 7 | 8 | import libcst 9 | import libcst.matchers as m 10 | from libcst import parse_expression 11 | from libcst.codemod import VisitorBasedCodemodCommand 12 | from libcst.codemod.visitors import AddImportsVisitor 13 | from libcst.metadata import QualifiedNameProvider 14 | 15 | 16 | class StripStringsCommand(VisitorBasedCodemodCommand): 17 | DESCRIPTION: str = ( 18 | "Converts string type annotations to 3.7-compatible forward references." 19 | ) 20 | 21 | METADATA_DEPENDENCIES = (QualifiedNameProvider,) 22 | 23 | # We want to gate the SimpleString visitor below to only SimpleStrings inside 24 | # an Annotation. 25 | @m.call_if_inside(m.Annotation()) 26 | # We also want to gate the SimpleString visitor below to ensure that we don't 27 | # erroneously strip strings from a Literal. 28 | @m.call_if_not_inside( 29 | m.Subscript( 30 | # We could match on value=m.Name("Literal") here, but then we might miss 31 | # instances where people are importing typing_extensions directly, or 32 | # importing Literal as an alias. 33 | value=m.MatchMetadataIfTrue( 34 | QualifiedNameProvider, 35 | lambda qualnames: any( 36 | qualname.name == "typing_extensions.Literal" 37 | for qualname in qualnames 38 | ), 39 | ) 40 | ) 41 | ) 42 | def leave_SimpleString( 43 | self, original_node: libcst.SimpleString, updated_node: libcst.SimpleString 44 | ) -> Union[libcst.SimpleString, libcst.BaseExpression]: 45 | AddImportsVisitor.add_needed_import(self.context, "__future__", "annotations") 46 | evaluated_value = updated_node.evaluated_value 47 | # Just use LibCST to evaluate the expression itself, and insert that as the 48 | # annotation. 49 | if isinstance(evaluated_value, str): 50 | return parse_expression( 51 | evaluated_value, config=self.module.config_for_parsing 52 | ) 53 | else: 54 | return updated_node 55 | -------------------------------------------------------------------------------- /libcst/codemod/commands/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | -------------------------------------------------------------------------------- /libcst/codemod/commands/tests/test_add_trailing_commas.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | 7 | from libcst.codemod import CodemodTest 8 | from libcst.codemod.commands.add_trailing_commas import AddTrailingCommas 9 | 10 | 11 | class AddTrailingCommasTest(CodemodTest): 12 | TRANSFORM = AddTrailingCommas 13 | 14 | def test_transform_defines(self) -> None: 15 | before = """ 16 | def f(x, y): 17 | pass 18 | 19 | """ 20 | after = """ 21 | def f(x, y,): 22 | pass 23 | """ 24 | self.assertCodemod(before, after) 25 | 26 | def test_skip_transforming_defines(self) -> None: 27 | before = """ 28 | # skip defines with no params. 29 | def f0(): 30 | pass 31 | 32 | # skip defines with a single param named `self`. 33 | class Foo: 34 | def __init__(self): 35 | pass 36 | """ 37 | after = before 38 | self.assertCodemod(before, after) 39 | 40 | def test_transform_calls(self) -> None: 41 | before = """ 42 | f(a, b, c) 43 | 44 | g(x=a, y=b, z=c) 45 | """ 46 | after = """ 47 | f(a, b, c,) 48 | 49 | g(x=a, y=b, z=c,) 50 | """ 51 | self.assertCodemod(before, after) 52 | 53 | def test_skip_transforming_calls(self) -> None: 54 | before = """ 55 | # skip empty calls 56 | f() 57 | 58 | # skip calls with one argument 59 | g(a) 60 | g(x=a) 61 | """ 62 | after = before 63 | self.assertCodemod(before, after) 64 | 65 | def test_using_yapf_presets(self) -> None: 66 | before = """ 67 | def f(x): # skip single parameters for yapf 68 | pass 69 | 70 | def g(x, y): 71 | pass 72 | """ 73 | after = """ 74 | def f(x): # skip single parameters for yapf 75 | pass 76 | 77 | def g(x, y,): 78 | pass 79 | """ 80 | self.assertCodemod(before, after, formatter="yapf") 81 | 82 | def test_using_custom_presets(self) -> None: 83 | before = """ 84 | def f(x, y, z): 85 | pass 86 | 87 | f(5, 6, 7) 88 | """ 89 | after = before 90 | self.assertCodemod(before, after, parameter_count=4, argument_count=4) 91 | -------------------------------------------------------------------------------- /libcst/codemod/commands/tests/test_convert_percent_format_to_fstring.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from libcst.codemod import CodemodTest 7 | from libcst.codemod.commands.convert_percent_format_to_fstring import ( 8 | ConvertPercentFormatStringCommand, 9 | ) 10 | 11 | 12 | class ConvertPercentFormatStringCommandTest(CodemodTest): 13 | TRANSFORM = ConvertPercentFormatStringCommand 14 | 15 | def test_simple_cases(self) -> None: 16 | self.assertCodemod('"a name: %s" % name', 'f"a name: {name}"') 17 | self.assertCodemod( 18 | '"an attribute %s ." % obj.attr', 'f"an attribute {obj.attr} ."' 19 | ) 20 | self.assertCodemod('r"raw string value=%s" % val', 'fr"raw string value={val}"') 21 | self.assertCodemod( 22 | '"The type of var: %s" % type(var)', 'f"The type of var: {type(var)}"' 23 | ) 24 | self.assertCodemod( 25 | '"type of var: %s, value of var: %s" % (type(var), var)', 26 | 'f"type of var: {type(var)}, value of var: {var}"', 27 | ) 28 | self.assertCodemod( 29 | '"var1: %s, var2: %s, var3: %s, var4: %s" % (class_object.attribute, dict_lookup["some_key"], some_module.some_function(), var4)', 30 | '''f"var1: {class_object.attribute}, var2: {dict_lookup['some_key']}, var3: {some_module.some_function()}, var4: {var4}"''', 31 | ) 32 | 33 | def test_escaping(self) -> None: 34 | self.assertCodemod('"%s" % "hi"', '''f"{'hi'}"''') # escape quote 35 | self.assertCodemod('"{%s}" % val', 'f"{{{val}}}"') # escape curly bracket 36 | self.assertCodemod('"{%s" % val', 'f"{{{val}"') # escape curly bracket 37 | self.assertCodemod( 38 | "'%s\" double quote is used' % var", "f'{var}\" double quote is used'" 39 | ) # escape quote 40 | self.assertCodemod( 41 | '"a list: %s" % " ".join(var)', '''f"a list: {' '.join(var)}"''' 42 | ) # escape quote 43 | 44 | def test_not_supported_case(self) -> None: 45 | code = '"%s" % obj.this_is_a_very_long_expression(parameter)["a_very_long_key"]' 46 | self.assertCodemod(code, code) 47 | code = 'b"a type %s" % var' 48 | self.assertCodemod(code, code) 49 | -------------------------------------------------------------------------------- /libcst/codemod/commands/tests/test_convert_union_to_or.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | # pyre-strict 7 | 8 | from libcst.codemod import CodemodTest 9 | from libcst.codemod.commands.convert_union_to_or import ConvertUnionToOrCommand 10 | 11 | 12 | class TestConvertUnionToOrCommand(CodemodTest): 13 | TRANSFORM = ConvertUnionToOrCommand 14 | 15 | def test_simple_union(self) -> None: 16 | before = """ 17 | from typing import Union 18 | x: Union[int, str] 19 | """ 20 | after = """ 21 | x: int | str 22 | """ 23 | self.assertCodemod(before, after) 24 | 25 | def test_nested_union(self) -> None: 26 | before = """ 27 | from typing import Union 28 | x: Union[int, Union[str, float]] 29 | """ 30 | after = """ 31 | x: int | str | float 32 | """ 33 | self.assertCodemod(before, after) 34 | 35 | def test_single_type_union(self) -> None: 36 | before = """ 37 | from typing import Union 38 | x: Union[int] 39 | """ 40 | after = """ 41 | x: int 42 | """ 43 | self.assertCodemod(before, after) 44 | 45 | def test_union_with_alias(self) -> None: 46 | before = """ 47 | import typing as t 48 | x: t.Union[int, str] 49 | """ 50 | after = """ 51 | import typing as t 52 | x: int | str 53 | """ 54 | self.assertCodemod(before, after) 55 | 56 | def test_union_with_unused_import(self) -> None: 57 | before = """ 58 | from typing import Union, List 59 | x: Union[int, str] 60 | """ 61 | after = """ 62 | from typing import List 63 | x: int | str 64 | """ 65 | self.assertCodemod(before, after) 66 | 67 | def test_union_no_import(self) -> None: 68 | before = """ 69 | x: Union[int, str] 70 | """ 71 | after = """ 72 | x: Union[int, str] 73 | """ 74 | self.assertCodemod(before, after) 75 | 76 | def test_union_in_function(self) -> None: 77 | before = """ 78 | from typing import Union 79 | def foo(x: Union[int, str]) -> Union[float, None]: 80 | ... 81 | """ 82 | after = """ 83 | def foo(x: int | str) -> float | None: 84 | ... 85 | """ 86 | self.assertCodemod(before, after) 87 | -------------------------------------------------------------------------------- /libcst/codemod/commands/tests/test_ensure_import_present.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from libcst.codemod import CodemodTest 7 | from libcst.codemod.commands.ensure_import_present import EnsureImportPresentCommand 8 | 9 | 10 | class EnsureImportPresentCommandTest(CodemodTest): 11 | TRANSFORM = EnsureImportPresentCommand 12 | 13 | def test_import_module(self) -> None: 14 | before = "" 15 | after = "import a" 16 | self.assertCodemod(before, after, module="a", entity=None, alias=None) 17 | 18 | def test_import_entity(self) -> None: 19 | before = "" 20 | after = "from a import b" 21 | self.assertCodemod(before, after, module="a", entity="b", alias=None) 22 | 23 | def test_import_wildcard(self) -> None: 24 | before = "from a import *" 25 | after = "from a import *" 26 | self.assertCodemod(before, after, module="a", entity="b", alias=None) 27 | 28 | def test_import_module_aliased(self) -> None: 29 | before = "" 30 | after = "import a as c" 31 | self.assertCodemod(before, after, module="a", entity=None, alias="c") 32 | 33 | def test_import_entity_aliased(self) -> None: 34 | before = "" 35 | after = "from a import b as c" 36 | self.assertCodemod(before, after, module="a", entity="b", alias="c") 37 | -------------------------------------------------------------------------------- /libcst/codemod/commands/tests/test_noop.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from libcst.codemod import CodemodTest 7 | from libcst.codemod.commands.noop import NOOPCommand 8 | 9 | 10 | class TestNOOPCodemod(CodemodTest): 11 | TRANSFORM = NOOPCommand 12 | 13 | def test_noop(self) -> None: 14 | before = """ 15 | foo: str = "" 16 | 17 | class Class: 18 | pass 19 | 20 | def foo(a: Class, **kwargs: str) -> Class: 21 | t: Class = Class() # This is a comment 22 | bar = "" 23 | return t 24 | 25 | bar = Class() 26 | foo(bar, baz="bla") 27 | """ 28 | after = """ 29 | foo: str = "" 30 | 31 | class Class: 32 | pass 33 | 34 | def foo(a: Class, **kwargs: str) -> Class: 35 | t: Class = Class() # This is a comment 36 | bar = "" 37 | return t 38 | 39 | bar = Class() 40 | foo(bar, baz="bla") 41 | """ 42 | 43 | self.assertCodemod(before, after) 44 | -------------------------------------------------------------------------------- /libcst/codemod/commands/tests/test_rename_typing_generic_aliases.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | # pyre-strict 7 | 8 | from libcst.codemod import CodemodTest 9 | from libcst.codemod.commands.rename_typing_generic_aliases import ( 10 | RenameTypingGenericAliases, 11 | ) 12 | 13 | 14 | class TestRenameCommand(CodemodTest): 15 | TRANSFORM = RenameTypingGenericAliases 16 | 17 | def test_rename_typing_generic_alias(self) -> None: 18 | before = """ 19 | from typing import List, Set, Dict, FrozenSet, Tuple 20 | x: List[int] = [] 21 | y: Set[int] = set() 22 | z: Dict[str, int] = {} 23 | a: FrozenSet[str] = frozenset() 24 | b: Tuple[int, str] = (1, "hello") 25 | """ 26 | after = """ 27 | x: list[int] = [] 28 | y: set[int] = set() 29 | z: dict[str, int] = {} 30 | a: frozenset[str] = frozenset() 31 | b: tuple[int, str] = (1, "hello") 32 | """ 33 | self.assertCodemod(before, after) 34 | -------------------------------------------------------------------------------- /libcst/codemod/commands/tests/test_unnecessary_format_string.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from typing import Type 7 | 8 | from libcst.codemod import Codemod, CodemodTest 9 | from libcst.codemod.commands.unnecessary_format_string import UnnecessaryFormatString 10 | 11 | 12 | class TestUnnecessaryFormatString(CodemodTest): 13 | TRANSFORM: Type[Codemod] = UnnecessaryFormatString 14 | 15 | def test_replace(self) -> None: 16 | before = r""" 17 | good: str = "good" 18 | good: str = f"with_arg{arg}" 19 | good = "good{arg1}".format(1234) 20 | good = "good".format() 21 | good = "good" % {} 22 | good = "good" % () 23 | good = rf"good\d+{bar}" 24 | good = f"wow i don't have args but don't mess my braces {{ up }}" 25 | 26 | bad: str = f"bad" + "bad" 27 | bad: str = f'bad' 28 | bad: str = rf'bad\d+' 29 | """ 30 | after = r""" 31 | good: str = "good" 32 | good: str = f"with_arg{arg}" 33 | good = "good{arg1}".format(1234) 34 | good = "good".format() 35 | good = "good" % {} 36 | good = "good" % () 37 | good = rf"good\d+{bar}" 38 | good = f"wow i don't have args but don't mess my braces {{ up }}" 39 | 40 | bad: str = "bad" + "bad" 41 | bad: str = 'bad' 42 | bad: str = r'bad\d+' 43 | """ 44 | self.assertCodemod(before, after) 45 | -------------------------------------------------------------------------------- /libcst/codemod/commands/unnecessary_format_string.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | import libcst 7 | import libcst.matchers as m 8 | from libcst.codemod import VisitorBasedCodemodCommand 9 | 10 | 11 | class UnnecessaryFormatString(VisitorBasedCodemodCommand): 12 | DESCRIPTION: str = ( 13 | "Converts f-strings which perform no formatting to regular strings." 14 | ) 15 | 16 | @m.leave(m.FormattedString(parts=(m.FormattedStringText(),))) 17 | def _check_formatted_string( 18 | self, 19 | _original_node: libcst.FormattedString, 20 | updated_node: libcst.FormattedString, 21 | ) -> libcst.BaseExpression: 22 | old_string_inner = libcst.ensure_type( 23 | updated_node.parts[0], libcst.FormattedStringText 24 | ).value 25 | if "{{" in old_string_inner or "}}" in old_string_inner: 26 | # there are only two characters we need to worry about escaping. 27 | return updated_node 28 | 29 | old_string_literal = updated_node.start + old_string_inner + updated_node.end 30 | new_string_literal = ( 31 | updated_node.start.replace("f", "").replace("F", "") 32 | + old_string_inner 33 | + updated_node.end 34 | ) 35 | 36 | old_string_evaled = eval(old_string_literal) # noqa 37 | new_string_evaled = eval(new_string_literal) # noqa 38 | if old_string_evaled != new_string_evaled: 39 | warn_message = ( 40 | f"Attempted to codemod |{old_string_literal}| to " 41 | + f"|{new_string_literal}| but don't eval to the same! First is |{old_string_evaled}| and " 42 | + f"second is |{new_string_evaled}|" 43 | ) 44 | self.warn(warn_message) 45 | return updated_node 46 | 47 | return libcst.SimpleString(new_string_literal) 48 | -------------------------------------------------------------------------------- /libcst/codemod/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | -------------------------------------------------------------------------------- /libcst/codemod/tests/codemod_formatter_error_input.py.txt: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | # pyre-strict 7 | 8 | import subprocess 9 | from contextlib import AsyncExitStack 10 | 11 | 12 | def fun() -> None: 13 | # this is an explicit syntax error to cause formatter error 14 | async with AsyncExitStack() as stack: 15 | stack 16 | -------------------------------------------------------------------------------- /libcst/codemod/tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from textwrap import dedent 7 | 8 | import libcst as cst 9 | from libcst import parse_module 10 | from libcst.codemod import CodemodContext, ContextAwareTransformer, ContextAwareVisitor 11 | from libcst.metadata import PositionProvider 12 | from libcst.testing.utils import UnitTest 13 | 14 | 15 | class TestingCollector(ContextAwareVisitor): 16 | METADATA_DEPENDENCIES = (PositionProvider,) 17 | 18 | def visit_Pass(self, node: cst.Pass) -> None: 19 | position = self.get_metadata(PositionProvider, node) 20 | self.context.scratch["pass"] = (position.start.line, position.start.column) 21 | 22 | 23 | class TestingTransform(ContextAwareTransformer): 24 | METADATA_DEPENDENCIES = (PositionProvider,) 25 | 26 | def visit_FunctionDef(self, node: cst.FunctionDef) -> None: 27 | position = self.get_metadata(PositionProvider, node) 28 | self.context.scratch[node.name.value] = ( 29 | position.start.line, 30 | position.start.column, 31 | ) 32 | node.visit(TestingCollector(self.context)) 33 | 34 | 35 | class TestMetadata(UnitTest): 36 | def test_metadata_works(self) -> None: 37 | code = """ 38 | def foo() -> None: 39 | pass 40 | 41 | def bar() -> int: 42 | return 5 43 | """ 44 | module = parse_module(dedent(code)) 45 | context = CodemodContext() 46 | transform = TestingTransform(context) 47 | transform.transform_module(module) 48 | self.assertEqual( 49 | context.scratch, {"foo": (2, 0), "pass": (3, 4), "bar": (5, 0)} 50 | ) 51 | -------------------------------------------------------------------------------- /libcst/codemod/visitors/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from libcst.codemod.visitors._add_imports import AddImportsVisitor 7 | from libcst.codemod.visitors._apply_type_annotations import ApplyTypeAnnotationsVisitor 8 | from libcst.codemod.visitors._gather_comments import GatherCommentsVisitor 9 | from libcst.codemod.visitors._gather_exports import GatherExportsVisitor 10 | from libcst.codemod.visitors._gather_global_names import GatherGlobalNamesVisitor 11 | from libcst.codemod.visitors._gather_imports import GatherImportsVisitor 12 | from libcst.codemod.visitors._gather_string_annotation_names import ( 13 | GatherNamesFromStringAnnotationsVisitor, 14 | ) 15 | from libcst.codemod.visitors._gather_unused_imports import GatherUnusedImportsVisitor 16 | from libcst.codemod.visitors._imports import ImportItem 17 | from libcst.codemod.visitors._remove_imports import RemoveImportsVisitor 18 | 19 | __all__ = [ 20 | "AddImportsVisitor", 21 | "ApplyTypeAnnotationsVisitor", 22 | "GatherCommentsVisitor", 23 | "GatherExportsVisitor", 24 | "GatherGlobalNamesVisitor", 25 | "GatherImportsVisitor", 26 | "GatherNamesFromStringAnnotationsVisitor", 27 | "GatherUnusedImportsVisitor", 28 | "ImportItem", 29 | "RemoveImportsVisitor", 30 | ] 31 | -------------------------------------------------------------------------------- /libcst/codemod/visitors/_gather_comments.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import re 7 | from typing import Dict, Pattern, Union 8 | 9 | import libcst as cst 10 | from libcst.codemod._context import CodemodContext 11 | from libcst.codemod._visitor import ContextAwareVisitor 12 | from libcst.metadata import PositionProvider 13 | 14 | 15 | class GatherCommentsVisitor(ContextAwareVisitor): 16 | """ 17 | Collects all comments matching a certain regex and their line numbers. 18 | This visitor is useful for capturing special-purpose comments, for example 19 | ``noqa`` style lint suppression annotations. 20 | 21 | Standalone comments are assumed to affect the line following them, and 22 | inline ones are recorded with the line they are on. 23 | 24 | After visiting a CST, matching comments are collected in the ``comments`` 25 | attribute. 26 | """ 27 | 28 | METADATA_DEPENDENCIES = (PositionProvider,) 29 | 30 | def __init__(self, context: CodemodContext, comment_regex: str) -> None: 31 | super().__init__(context) 32 | 33 | #: Dictionary of comments found in the CST. Keys are line numbers, 34 | #: values are comment nodes. 35 | self.comments: Dict[int, cst.Comment] = {} 36 | 37 | self._comment_matcher: Pattern[str] = re.compile(comment_regex) 38 | 39 | def visit_EmptyLine(self, node: cst.EmptyLine) -> bool: 40 | if node.comment is not None: 41 | self.handle_comment(node) 42 | return False 43 | 44 | def visit_TrailingWhitespace(self, node: cst.TrailingWhitespace) -> bool: 45 | if node.comment is not None: 46 | self.handle_comment(node) 47 | return False 48 | 49 | def handle_comment( 50 | self, node: Union[cst.EmptyLine, cst.TrailingWhitespace] 51 | ) -> None: 52 | comment = node.comment 53 | assert comment is not None # ensured by callsites above 54 | if not self._comment_matcher.match(comment.value): 55 | return 56 | line = self.get_metadata(PositionProvider, comment).start.line 57 | if isinstance(node, cst.EmptyLine): 58 | # Standalone comments refer to the next line 59 | line += 1 60 | self.comments[line] = comment 61 | -------------------------------------------------------------------------------- /libcst/codemod/visitors/_imports.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from dataclasses import dataclass, replace 7 | from typing import Optional 8 | 9 | from libcst.helpers import get_absolute_module_from_package 10 | 11 | 12 | @dataclass(frozen=True) 13 | class ImportItem: 14 | """Representation of individual import items for codemods.""" 15 | 16 | module_name: str 17 | obj_name: Optional[str] = None 18 | alias: Optional[str] = None 19 | relative: int = 0 20 | 21 | def __post_init__(self) -> None: 22 | if self.module_name is None: 23 | object.__setattr__(self, "module_name", "") 24 | elif self.module_name.startswith("."): 25 | mod = self.module_name.lstrip(".") 26 | rel = self.relative + len(self.module_name) - len(mod) 27 | object.__setattr__(self, "module_name", mod) 28 | object.__setattr__(self, "relative", rel) 29 | 30 | @property 31 | def module(self) -> str: 32 | return "." * self.relative + self.module_name 33 | 34 | def resolve_relative(self, package_name: Optional[str]) -> "ImportItem": 35 | """Return an ImportItem with an absolute module name if possible.""" 36 | mod = self 37 | # `import ..a` -> `from .. import a` 38 | if mod.relative and mod.obj_name is None: 39 | mod = replace(mod, module_name="", obj_name=mod.module_name) 40 | if package_name is None: 41 | return mod 42 | m = get_absolute_module_from_package( 43 | package_name, mod.module_name or None, self.relative 44 | ) 45 | return mod if m is None else replace(mod, module_name=m, relative=0) 46 | -------------------------------------------------------------------------------- /libcst/codemod/visitors/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | -------------------------------------------------------------------------------- /libcst/codemod/visitors/tests/test_gather_comments.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from libcst import Comment, MetadataWrapper, parse_module 7 | from libcst.codemod import CodemodContext, CodemodTest 8 | from libcst.codemod.visitors import GatherCommentsVisitor 9 | from libcst.testing.utils import UnitTest 10 | 11 | 12 | class TestGatherCommentsVisitor(UnitTest): 13 | def gather_comments(self, code: str) -> GatherCommentsVisitor: 14 | mod = MetadataWrapper(parse_module(CodemodTest.make_fixture_data(code))) 15 | mod.resolve_many(GatherCommentsVisitor.METADATA_DEPENDENCIES) 16 | instance = GatherCommentsVisitor( 17 | CodemodContext(wrapper=mod), r".*\Wnoqa(\W.*)?$" 18 | ) 19 | mod.visit(instance) 20 | return instance 21 | 22 | def test_no_comments(self) -> None: 23 | visitor = self.gather_comments( 24 | """ 25 | def foo() -> None: 26 | pass 27 | """ 28 | ) 29 | self.assertEqual(visitor.comments, {}) 30 | 31 | def test_noqa_comments(self) -> None: 32 | visitor = self.gather_comments( 33 | """ 34 | import a.b.c # noqa 35 | import d # somethingelse 36 | # noqa 37 | def foo() -> None: 38 | pass 39 | 40 | """ 41 | ) 42 | self.assertEqual(visitor.comments.keys(), {1, 4}) 43 | self.assertTrue(isinstance(visitor.comments[1], Comment)) 44 | self.assertEqual(visitor.comments[1].value, "# noqa") 45 | self.assertTrue(isinstance(visitor.comments[4], Comment)) 46 | self.assertEqual(visitor.comments[4].value, "# noqa") 47 | -------------------------------------------------------------------------------- /libcst/codemod/visitors/tests/test_gather_global_names.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from libcst import parse_module 7 | from libcst.codemod import CodemodContext, CodemodTest 8 | from libcst.codemod.visitors import GatherGlobalNamesVisitor 9 | from libcst.testing.utils import UnitTest 10 | 11 | 12 | class TestGatherGlobalNamesVisitor(UnitTest): 13 | def gather_global_names(self, code: str) -> GatherGlobalNamesVisitor: 14 | transform_instance = GatherGlobalNamesVisitor( 15 | CodemodContext(full_module_name="a.b.foobar") 16 | ) 17 | input_tree = parse_module(CodemodTest.make_fixture_data(code)) 18 | input_tree.visit(transform_instance) 19 | return transform_instance 20 | 21 | def test_gather_nothing(self) -> None: 22 | code = """ 23 | from a import b 24 | b() 25 | """ 26 | gatherer = self.gather_global_names(code) 27 | self.assertEqual(gatherer.global_names, set()) 28 | self.assertEqual(gatherer.class_names, set()) 29 | self.assertEqual(gatherer.function_names, set()) 30 | 31 | def test_globals(self) -> None: 32 | code = """ 33 | x = 1 34 | y = 2 35 | def foo(): pass 36 | class Foo: pass 37 | """ 38 | gatherer = self.gather_global_names(code) 39 | self.assertEqual(gatherer.global_names, {"x", "y"}) 40 | self.assertEqual(gatherer.class_names, {"Foo"}) 41 | self.assertEqual(gatherer.function_names, {"foo"}) 42 | 43 | def test_omit_nested(self) -> None: 44 | code = """ 45 | def foo(): 46 | x = 1 47 | 48 | class Foo: 49 | def method(self): pass 50 | """ 51 | gatherer = self.gather_global_names(code) 52 | self.assertEqual(gatherer.global_names, set()) 53 | self.assertEqual(gatherer.class_names, {"Foo"}) 54 | self.assertEqual(gatherer.function_names, {"foo"}) 55 | -------------------------------------------------------------------------------- /libcst/display/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from libcst.display.graphviz import dump_graphviz 7 | from libcst.display.text import dump 8 | 9 | __all__ = [ 10 | "dump", 11 | "dump_graphviz", 12 | ] 13 | -------------------------------------------------------------------------------- /libcst/display/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | 7 | from libcst.helpers._template import ( 8 | parse_template_expression, 9 | parse_template_module, 10 | parse_template_statement, 11 | ) 12 | from libcst.helpers.common import ensure_type 13 | from libcst.helpers.expression import ( 14 | get_full_name_for_node, 15 | get_full_name_for_node_or_raise, 16 | ) 17 | from libcst.helpers.module import ( 18 | calculate_module_and_package, 19 | get_absolute_module, 20 | get_absolute_module_for_import, 21 | get_absolute_module_for_import_or_raise, 22 | get_absolute_module_from_package, 23 | get_absolute_module_from_package_for_import, 24 | get_absolute_module_from_package_for_import_or_raise, 25 | insert_header_comments, 26 | ModuleNameAndPackage, 27 | ) 28 | from libcst.helpers.node_fields import ( 29 | filter_node_fields, 30 | get_field_default_value, 31 | get_node_fields, 32 | is_default_node_field, 33 | is_syntax_node_field, 34 | is_whitespace_node_field, 35 | ) 36 | 37 | __all__ = [ 38 | "calculate_module_and_package", 39 | "get_absolute_module", 40 | "get_absolute_module_for_import", 41 | "get_absolute_module_for_import_or_raise", 42 | "get_absolute_module_from_package", 43 | "get_absolute_module_from_package_for_import", 44 | "get_absolute_module_from_package_for_import_or_raise", 45 | "get_full_name_for_node", 46 | "get_full_name_for_node_or_raise", 47 | "ensure_type", 48 | "insert_header_comments", 49 | "parse_template_module", 50 | "parse_template_statement", 51 | "parse_template_expression", 52 | "ModuleNameAndPackage", 53 | "get_node_fields", 54 | "get_field_default_value", 55 | "is_whitespace_node_field", 56 | "is_syntax_node_field", 57 | "is_default_node_field", 58 | "filter_node_fields", 59 | ] 60 | -------------------------------------------------------------------------------- /libcst/helpers/common.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from typing import Type, TypeVar 7 | 8 | T = TypeVar("T") 9 | 10 | 11 | def ensure_type(node: object, nodetype: Type[T]) -> T: 12 | """ 13 | Takes any python object, and a LibCST :class:`~libcst.CSTNode` subclass and 14 | refines the type of the python object. This is most useful when you already 15 | know that a particular object is a certain type but your type checker is not 16 | convinced. Note that this does an instance check for you and raises an 17 | exception if it is not the right type, so this should be used in situations 18 | where you are sure of the type given previous checks. 19 | """ 20 | 21 | if not isinstance(node, nodetype): 22 | raise Exception( 23 | f"Expected a {nodetype.__name__} but got a {node.__class__.__name__}!" 24 | ) 25 | return node 26 | -------------------------------------------------------------------------------- /libcst/helpers/expression.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | # 6 | from typing import Optional, Union 7 | 8 | import libcst as cst 9 | 10 | 11 | def get_full_name_for_node(node: Union[str, cst.CSTNode]) -> Optional[str]: 12 | """Return a dot concatenated full name for str, :class:`~libcst.Name`, :class:`~libcst.Attribute`. 13 | :class:`~libcst.Call`, :class:`~libcst.Subscript`, :class:`~libcst.FunctionDef`, :class:`~libcst.ClassDef`, 14 | :class:`~libcst.Decorator`. 15 | Return ``None`` for not supported Node. 16 | """ 17 | if isinstance(node, cst.Name): 18 | return node.value 19 | elif isinstance(node, str): 20 | return node 21 | elif isinstance(node, cst.Attribute): 22 | return f"{get_full_name_for_node(node.value)}.{node.attr.value}" 23 | elif isinstance(node, cst.Call): 24 | return get_full_name_for_node(node.func) 25 | elif isinstance(node, cst.Subscript): 26 | return get_full_name_for_node(node.value) 27 | elif isinstance(node, (cst.FunctionDef, cst.ClassDef)): 28 | return get_full_name_for_node(node.name) 29 | elif isinstance(node, cst.Decorator): 30 | return get_full_name_for_node(node.decorator) 31 | return None 32 | 33 | 34 | def get_full_name_for_node_or_raise(node: Union[str, cst.CSTNode]) -> str: 35 | """Return a dot concatenated full name for str, :class:`~libcst.Name`, :class:`~libcst.Attribute`. 36 | :class:`~libcst.Call`, :class:`~libcst.Subscript`, :class:`~libcst.FunctionDef`, :class:`~libcst.ClassDef`. 37 | Raise Exception for not supported Node. 38 | """ 39 | full_name = get_full_name_for_node(node) 40 | if full_name is None: 41 | raise Exception(f"Not able to parse full name for: {node}") 42 | return full_name 43 | -------------------------------------------------------------------------------- /libcst/helpers/paths.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import os 7 | from contextlib import contextmanager 8 | from pathlib import Path 9 | from typing import Generator 10 | 11 | from libcst._types import StrPath 12 | 13 | 14 | @contextmanager 15 | def chdir(path: StrPath) -> Generator[Path, None, None]: 16 | """ 17 | Temporarily chdir to the given path, and then return to the previous path. 18 | """ 19 | try: 20 | path = Path(path).resolve() 21 | cwd = os.getcwd() 22 | os.chdir(path) 23 | yield path 24 | finally: 25 | os.chdir(cwd) 26 | -------------------------------------------------------------------------------- /libcst/helpers/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/helpers/tests/test_paths.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from pathlib import Path 7 | from tempfile import TemporaryDirectory 8 | 9 | from libcst.helpers.paths import chdir 10 | from libcst.testing.utils import UnitTest 11 | 12 | 13 | class PathsTest(UnitTest): 14 | def test_chdir(self) -> None: 15 | with TemporaryDirectory() as td: 16 | tdp = Path(td).resolve() 17 | inner = tdp / "foo" / "bar" 18 | inner.mkdir(parents=True) 19 | 20 | with self.subTest("string paths"): 21 | cwd1 = Path.cwd() 22 | 23 | with chdir(tdp.as_posix()) as path2: 24 | cwd2 = Path.cwd() 25 | self.assertEqual(tdp, cwd2) 26 | self.assertEqual(tdp, path2) 27 | 28 | with chdir(inner.as_posix()) as path3: 29 | cwd3 = Path.cwd() 30 | self.assertEqual(inner, cwd3) 31 | self.assertEqual(inner, path3) 32 | 33 | cwd4 = Path.cwd() 34 | self.assertEqual(tdp, cwd4) 35 | self.assertEqual(cwd2, cwd4) 36 | 37 | cwd5 = Path.cwd() 38 | self.assertEqual(cwd1, cwd5) 39 | 40 | with self.subTest("pathlib objects"): 41 | cwd1 = Path.cwd() 42 | 43 | with chdir(tdp) as path2: 44 | cwd2 = Path.cwd() 45 | self.assertEqual(tdp, cwd2) 46 | self.assertEqual(tdp, path2) 47 | 48 | with chdir(inner) as path3: 49 | cwd3 = Path.cwd() 50 | self.assertEqual(inner, cwd3) 51 | self.assertEqual(inner, path3) 52 | 53 | cwd4 = Path.cwd() 54 | self.assertEqual(tdp, cwd4) 55 | self.assertEqual(cwd2, cwd4) 56 | 57 | cwd5 = Path.cwd() 58 | self.assertEqual(cwd1, cwd5) 59 | -------------------------------------------------------------------------------- /libcst/matchers/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/metadata/accessor_provider.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | import dataclasses 8 | 9 | import libcst as cst 10 | 11 | from libcst.metadata.base_provider import VisitorMetadataProvider 12 | 13 | 14 | class AccessorProvider(VisitorMetadataProvider[str]): 15 | def on_visit(self, node: cst.CSTNode) -> bool: 16 | for f in dataclasses.fields(node): 17 | child = getattr(node, f.name) 18 | self.set_metadata(child, f.name) 19 | return True 20 | -------------------------------------------------------------------------------- /libcst/metadata/file_path_provider.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from pathlib import Path 7 | from typing import Any, List, Mapping, Optional 8 | 9 | import libcst as cst 10 | from libcst.metadata.base_provider import BatchableMetadataProvider 11 | 12 | 13 | class FilePathProvider(BatchableMetadataProvider[Path]): 14 | """ 15 | Provides the path to the current file on disk as metadata for the root 16 | :class:`~libcst.Module` node. Requires a :class:`~libcst.metadata.FullRepoManager`. 17 | The returned path will always be resolved to an absolute path using 18 | :func:`pathlib.Path.resolve`. 19 | 20 | Example usage: 21 | 22 | .. code:: python 23 | 24 | class CustomVisitor(CSTVisitor): 25 | METADATA_DEPENDENCIES = [FilePathProvider] 26 | 27 | path: pathlib.Path 28 | 29 | def visit_Module(self, node: libcst.Module) -> None: 30 | self.path = self.get_metadata(FilePathProvider, node) 31 | 32 | .. code:: 33 | 34 | >>> mgr = FullRepoManager(".", {"libcst/_types.py"}, {FilePathProvider}) 35 | >>> wrapper = mgr.get_metadata_wrapper_for_path("libcst/_types.py") 36 | >>> fqnames = wrapper.resolve(FilePathProvider) 37 | >>> {type(k): v for k, v in wrapper.resolve(FilePathProvider).items()} 38 | {: PosixPath('/home/user/libcst/_types.py')} 39 | 40 | """ 41 | 42 | @classmethod 43 | def gen_cache( 44 | cls, root_path: Path, paths: List[str], **kwargs: Any 45 | ) -> Mapping[str, Path]: 46 | cache = {path: (root_path / path).resolve() for path in paths} 47 | return cache 48 | 49 | def __init__(self, cache: Path) -> None: 50 | super().__init__(cache) 51 | self.path: Path = cache 52 | 53 | def visit_Module(self, node: cst.Module) -> Optional[bool]: 54 | self.set_metadata(node, self.path) 55 | return False 56 | -------------------------------------------------------------------------------- /libcst/metadata/parent_node_provider.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from typing import Optional 8 | 9 | import libcst as cst 10 | from libcst.metadata.base_provider import BatchableMetadataProvider 11 | 12 | 13 | class ParentNodeVisitor(cst.CSTVisitor): 14 | def __init__(self, provider: "ParentNodeProvider") -> None: 15 | self.provider: ParentNodeProvider = provider 16 | super().__init__() 17 | 18 | def on_leave(self, original_node: cst.CSTNode) -> None: 19 | for child in original_node.children: 20 | self.provider.set_metadata(child, original_node) 21 | super().on_leave(original_node) 22 | 23 | 24 | class ParentNodeProvider(BatchableMetadataProvider[cst.CSTNode]): 25 | def visit_Module(self, node: cst.Module) -> Optional[bool]: 26 | node.visit(ParentNodeVisitor(self)) 27 | -------------------------------------------------------------------------------- /libcst/metadata/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/metadata/tests/test_accessor_provider.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import dataclasses 7 | 8 | from textwrap import dedent 9 | 10 | import libcst as cst 11 | from libcst.metadata import AccessorProvider, MetadataWrapper 12 | from libcst.testing.utils import data_provider, UnitTest 13 | 14 | 15 | class DependentVisitor(cst.CSTVisitor): 16 | METADATA_DEPENDENCIES = (AccessorProvider,) 17 | 18 | def __init__(self, *, test: UnitTest) -> None: 19 | self.test = test 20 | 21 | def on_visit(self, node: cst.CSTNode) -> bool: 22 | for f in dataclasses.fields(node): 23 | child = getattr(node, f.name) 24 | if type(child) is cst.CSTNode: 25 | accessor = self.get_metadata(AccessorProvider, child) 26 | self.test.assertEqual(accessor, f.name) 27 | 28 | return True 29 | 30 | 31 | class AccessorProviderTest(UnitTest): 32 | @data_provider( 33 | ( 34 | ( 35 | """ 36 | foo = 'toplevel' 37 | fn1(foo) 38 | fn2(foo) 39 | def fn_def(): 40 | foo = 'shadow' 41 | fn3(foo) 42 | """, 43 | ), 44 | ( 45 | """ 46 | global_var = None 47 | @cls_attr 48 | class Cls(cls_attr, kwarg=cls_attr): 49 | cls_attr = 5 50 | def f(): 51 | pass 52 | """, 53 | ), 54 | ( 55 | """ 56 | iterator = None 57 | condition = None 58 | [elt for target in iterator if condition] 59 | {elt for target in iterator if condition} 60 | {elt: target for target in iterator if condition} 61 | (elt for target in iterator if condition) 62 | """, 63 | ), 64 | ) 65 | ) 66 | def test_accessor_provier(self, code: str) -> None: 67 | wrapper = MetadataWrapper(cst.parse_module(dedent(code))) 68 | wrapper.visit(DependentVisitor(test=self)) 69 | -------------------------------------------------------------------------------- /libcst/metadata/tests/test_parent_node_provider.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from textwrap import dedent 8 | 9 | import libcst as cst 10 | from libcst.metadata import MetadataWrapper, ParentNodeProvider 11 | from libcst.testing.utils import data_provider, UnitTest 12 | 13 | 14 | class DependentVisitor(cst.CSTVisitor): 15 | METADATA_DEPENDENCIES = (ParentNodeProvider,) 16 | 17 | def __init__(self, *, test: UnitTest) -> None: 18 | self.test = test 19 | 20 | def on_visit(self, node: cst.CSTNode) -> bool: 21 | for child in node.children: 22 | parent = self.get_metadata(ParentNodeProvider, child) 23 | self.test.assertEqual(parent, node) 24 | return True 25 | 26 | 27 | class ParentNodeProviderTest(UnitTest): 28 | @data_provider( 29 | ( 30 | ( 31 | """ 32 | foo = 'toplevel' 33 | fn1(foo) 34 | fn2(foo) 35 | def fn_def(): 36 | foo = 'shadow' 37 | fn3(foo) 38 | """, 39 | ), 40 | ( 41 | """ 42 | global_var = None 43 | @cls_attr 44 | class Cls(cls_attr, kwarg=cls_attr): 45 | cls_attr = 5 46 | def f(): 47 | pass 48 | """, 49 | ), 50 | ( 51 | """ 52 | iterator = None 53 | condition = None 54 | [elt for target in iterator if condition] 55 | {elt for target in iterator if condition} 56 | {elt: target for target in iterator if condition} 57 | (elt for target in iterator if condition) 58 | """, 59 | ), 60 | ) 61 | ) 62 | def test_parent_node_provier(self, code: str) -> None: 63 | wrapper = MetadataWrapper(cst.parse_module(dedent(code))) 64 | wrapper.visit(DependentVisitor(test=self)) 65 | -------------------------------------------------------------------------------- /libcst/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Instagram/LibCST/482a2e5f0997273e3fb272346dac275c04e84807/libcst/py.typed -------------------------------------------------------------------------------- /libcst/testing/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | -------------------------------------------------------------------------------- /libcst/tests/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from unittest import main 7 | 8 | from libcst._parser.entrypoints import is_native 9 | 10 | 11 | if __name__ == "__main__": 12 | parser_type = "native" if is_native() else "pure" 13 | print(f"running tests with {parser_type!r} parser") 14 | 15 | main(module=None, verbosity=2) 16 | -------------------------------------------------------------------------------- /libcst/tests/pyre/.pyre_configuration: -------------------------------------------------------------------------------- 1 | { 2 | "source_directories": [ 3 | "." 4 | ], 5 | "search_path": [] 6 | } 7 | -------------------------------------------------------------------------------- /libcst/tests/pyre/simple_class.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | # fmt: off 7 | from typing import Sequence 8 | 9 | 10 | class Item: 11 | def __init__(self, n: int) -> None: 12 | self.number: int = n 13 | 14 | 15 | class ItemCollector: 16 | def get_items(self, n: int) -> Sequence[Item]: 17 | return [Item(i) for i in range(n)] 18 | 19 | 20 | collector = ItemCollector() 21 | items: Sequence[Item] = collector.get_items(3) 22 | for item in items: 23 | item.number 24 | -------------------------------------------------------------------------------- /libcst/tests/test_add_slots.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import pickle 7 | from dataclasses import dataclass 8 | from typing import ClassVar 9 | 10 | from libcst._add_slots import add_slots 11 | 12 | from libcst.testing.utils import UnitTest 13 | 14 | 15 | # this test class needs to be defined at module level to test pickling. 16 | @add_slots 17 | @dataclass(frozen=True) 18 | class A: 19 | x: int 20 | y: str 21 | 22 | Z: ClassVar[int] = 5 23 | 24 | 25 | class AddSlotsTest(UnitTest): 26 | def test_pickle(self) -> None: 27 | a = A(1, "foo") 28 | self.assertEqual(a, pickle.loads(pickle.dumps(a))) 29 | object.__delattr__(a, "y") 30 | self.assertEqual(a.x, pickle.loads(pickle.dumps(a)).x) 31 | 32 | def test_prevents_slots_overlap(self) -> None: 33 | class A: 34 | __slots__ = ("x",) 35 | 36 | class B(A): 37 | __slots__ = ("z",) 38 | 39 | @add_slots 40 | @dataclass 41 | class C(B): 42 | x: int 43 | y: str 44 | z: bool 45 | 46 | self.assertSequenceEqual(C.__slots__, ("y",)) 47 | -------------------------------------------------------------------------------- /libcst/tests/test_deep_clone.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | from textwrap import dedent 6 | from typing import Set 7 | 8 | import libcst as cst 9 | from libcst.testing.utils import data_provider, UnitTest 10 | 11 | 12 | class DeepCloneTest(UnitTest): 13 | @data_provider( 14 | ( 15 | # Simple program 16 | ( 17 | """ 18 | foo = 'toplevel' 19 | fn1(foo) 20 | fn2(foo) 21 | def fn_def(): 22 | foo = 'shadow' 23 | fn3(foo) 24 | """, 25 | ), 26 | ) 27 | ) 28 | def test_deep_clone(self, code: str) -> None: 29 | test_case = self 30 | 31 | class NodeGatherVisitor(cst.CSTVisitor): 32 | def __init__(self) -> None: 33 | self.nodes: Set[int] = set() 34 | 35 | def on_visit(self, node: cst.CSTNode) -> bool: 36 | self.nodes.add(id(node)) 37 | return True 38 | 39 | class NodeVerifyVisitor(cst.CSTVisitor): 40 | def __init__(self, nodes: Set[int]) -> None: 41 | self.nodes = nodes 42 | 43 | def on_visit(self, node: cst.CSTNode) -> bool: 44 | test_case.assertFalse( 45 | id(node) in self.nodes, f"Node {node} was not cloned properly!" 46 | ) 47 | return True 48 | 49 | module = cst.parse_module(dedent(code)) 50 | gatherer = NodeGatherVisitor() 51 | module.visit(gatherer) 52 | new_module = module.deep_clone() 53 | self.assertTrue(module.deep_equals(new_module)) 54 | new_module.visit(NodeVerifyVisitor(gatherer.nodes)) 55 | -------------------------------------------------------------------------------- /libcst/tests/test_roundtrip.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | 7 | from pathlib import Path 8 | from unittest import TestCase 9 | 10 | from libcst import CSTTransformer, parse_module 11 | from libcst._parser.entrypoints import is_native 12 | 13 | fixtures: Path = Path(__file__).parent.parent.parent / "native/libcst/tests/fixtures" 14 | 15 | 16 | class NOOPTransformer(CSTTransformer): 17 | pass 18 | 19 | 20 | class RoundTripTests(TestCase): 21 | def _get_fixtures(self) -> list[Path]: 22 | if not is_native(): 23 | self.skipTest("pure python parser doesn't work with this") 24 | self.assertTrue(fixtures.exists(), f"{fixtures} should exist") 25 | files = list(fixtures.iterdir()) 26 | self.assertGreater(len(files), 0) 27 | return files 28 | 29 | def test_clean_roundtrip(self) -> None: 30 | for file in self._get_fixtures(): 31 | with self.subTest(file=str(file)): 32 | src = file.read_text(encoding="utf-8") 33 | mod = parse_module(src) 34 | self.maxDiff = None 35 | self.assertEqual(mod.code, src) 36 | 37 | def test_transform_roundtrip(self) -> None: 38 | transformer = NOOPTransformer() 39 | self.maxDiff = None 40 | for file in self._get_fixtures(): 41 | with self.subTest(file=str(file)): 42 | src = file.read_text(encoding="utf-8") 43 | mod = parse_module(src) 44 | new_mod = mod.visit(transformer) 45 | self.assertEqual(src, new_mod.code) 46 | -------------------------------------------------------------------------------- /libcst/tests/test_tabs.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from libcst._tabs import expand_tabs 7 | from libcst.testing.utils import data_provider, UnitTest 8 | 9 | 10 | class ExpandTabsTest(UnitTest): 11 | @data_provider( 12 | [ 13 | ("\t", " " * 8), 14 | ("\t\t", " " * 16), 15 | (" \t", " " * 8), 16 | ("\t ", " " * 12), 17 | ("abcd\t", "abcd "), 18 | ("abcdefg\t", "abcdefg "), 19 | ("abcdefgh\t", "abcdefgh "), 20 | ("\tsuffix", " suffix"), 21 | ] 22 | ) 23 | def test_expand_tabs(self, input: str, output: str) -> None: 24 | self.assertEqual(expand_tabs(input), output) 25 | -------------------------------------------------------------------------------- /native/Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | 3 | members = [ 4 | "libcst", 5 | "libcst_derive", 6 | ] 7 | -------------------------------------------------------------------------------- /native/libcst/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | [package] 7 | name = "libcst" 8 | version = "1.8.0" 9 | authors = ["LibCST Developers"] 10 | edition = "2018" 11 | rust-version = "1.70" 12 | description = "A Python parser and Concrete Syntax Tree library." 13 | license = "MIT AND (MIT AND PSF-2.0)" 14 | repository = "https://github.com/Instagram/LibCST" 15 | documentation = "https://libcst.rtfd.org" 16 | keywords = ["python", "cst", "ast"] 17 | categories = ["parser-implementations"] 18 | 19 | [lib] 20 | name = "libcst_native" 21 | crate-type = ["cdylib", "rlib"] 22 | 23 | [[bin]] 24 | name = "parse" 25 | path = "src/bin.rs" 26 | 27 | [features] 28 | # This is a bit of a hack, since `cargo test` doesn't work with `extension-module`. 29 | # To run tests, use `cargo test --no-default-features`. 30 | # 31 | # Once https://github.com/PyO3/pyo3/pull/1123 lands, it may be better to use 32 | # `-Zextra-link-arg` for this instead. 33 | default = ["py"] 34 | py = ["pyo3", "pyo3/extension-module"] 35 | trace = ["peg/trace"] 36 | 37 | [dependencies] 38 | paste = "1.0.15" 39 | pyo3 = { version = "0.25", optional = true } 40 | thiserror = "2.0.12" 41 | peg = "0.8.5" 42 | annotate-snippets = "0.11.5" 43 | regex = "1.11.1" 44 | memchr = "2.7.4" 45 | libcst_derive = { path = "../libcst_derive", version = "1.8.0" } 46 | 47 | [dev-dependencies] 48 | criterion = { version = "0.5.1", features = ["html_reports"] } 49 | difference = "2.0.0" 50 | rayon = "1.10.0" 51 | itertools = "0.13.0" 52 | 53 | [[bench]] 54 | name = "parser_benchmark" 55 | harness = false 56 | -------------------------------------------------------------------------------- /native/libcst/src/bin.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree 5 | 6 | use libcst_native::*; 7 | use std::{ 8 | env, 9 | io::{self, Read}, 10 | process::exit, 11 | }; 12 | 13 | pub fn main() { 14 | let mut str = std::string::String::new(); 15 | io::stdin().read_to_string(&mut str).unwrap(); 16 | match parse_module(str.as_ref(), None) { 17 | Err(e) => { 18 | eprintln!("{}", prettify_error(e, "stdin")); 19 | exit(1); 20 | } 21 | Ok(m) => { 22 | let first_arg = env::args().nth(1).unwrap_or_else(|| "".to_string()); 23 | if first_arg == "-d" { 24 | println!("{:#?}", m); 25 | } 26 | if first_arg != "-n" { 27 | let mut state = Default::default(); 28 | m.codegen(&mut state); 29 | print!("{}", state.to_string()); 30 | } 31 | } 32 | }; 33 | } 34 | -------------------------------------------------------------------------------- /native/libcst/src/nodes/codegen.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | use std::fmt; 7 | #[derive(Debug)] 8 | pub struct CodegenState<'a> { 9 | pub tokens: String, 10 | pub indent_tokens: Vec<&'a str>, 11 | pub default_newline: &'a str, 12 | pub default_indent: &'a str, 13 | } 14 | 15 | impl<'a> CodegenState<'a> { 16 | pub fn indent(&mut self, v: &'a str) { 17 | self.indent_tokens.push(v); 18 | } 19 | pub fn dedent(&mut self) { 20 | self.indent_tokens.pop(); 21 | } 22 | pub fn add_indent(&mut self) { 23 | self.tokens.extend(self.indent_tokens.iter().cloned()); 24 | } 25 | pub fn add_token(&mut self, tok: &'a str) { 26 | self.tokens.push_str(tok); 27 | } 28 | } 29 | 30 | impl<'a> fmt::Display for CodegenState<'a> { 31 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 32 | write!(f, "{}", self.tokens) 33 | } 34 | } 35 | 36 | pub trait Codegen<'a> { 37 | fn codegen(&self, state: &mut CodegenState<'a>); 38 | } 39 | 40 | impl<'a, T> Codegen<'a> for Option 41 | where 42 | T: Codegen<'a>, 43 | { 44 | fn codegen(&self, state: &mut CodegenState<'a>) { 45 | if let Some(s) = &self { 46 | s.codegen(state); 47 | } 48 | } 49 | } 50 | 51 | #[cfg(windows)] 52 | const LINE_ENDING: &str = "\r\n"; 53 | #[cfg(not(windows))] 54 | const LINE_ENDING: &str = "\n"; 55 | 56 | impl<'a> Default for CodegenState<'a> { 57 | fn default() -> Self { 58 | Self { 59 | default_newline: LINE_ENDING, 60 | default_indent: " ", 61 | indent_tokens: Default::default(), 62 | tokens: Default::default(), 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /native/libcst/src/nodes/inflate_helpers.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree 5 | 6 | use crate::{ 7 | nodes::traits::Result, 8 | tokenizer::{ 9 | whitespace_parser::{parse_parenthesizable_whitespace, Config}, 10 | Token, 11 | }, 12 | Param, Parameters, StarArg, 13 | }; 14 | 15 | pub(crate) fn adjust_parameters_trailing_whitespace<'a>( 16 | config: &Config<'a>, 17 | parameters: &mut Parameters<'a>, 18 | next_tok: &Token<'a>, 19 | ) -> Result<()> { 20 | let do_adjust = |param: &mut Param<'a>| -> Result<()> { 21 | let whitespace_after = 22 | parse_parenthesizable_whitespace(config, &mut next_tok.whitespace_before.borrow_mut())?; 23 | if param.comma.is_none() { 24 | param.whitespace_after_param = whitespace_after; 25 | } 26 | Ok(()) 27 | }; 28 | 29 | if let Some(param) = &mut parameters.star_kwarg { 30 | do_adjust(param)?; 31 | } else if let Some(param) = parameters.kwonly_params.last_mut() { 32 | do_adjust(param)?; 33 | } else if let Some(StarArg::Param(param)) = parameters.star_arg.as_mut() { 34 | do_adjust(param)?; 35 | } else if let Some(param) = parameters.params.last_mut() { 36 | do_adjust(param)?; 37 | } 38 | Ok(()) 39 | } 40 | -------------------------------------------------------------------------------- /native/libcst/src/nodes/macros.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | /// Generates a function that lazily imports and caches a module's member. This will hold a 7 | /// permanent reference to the imported member. Python's module cache is rarely purged though, so 8 | /// it typically won't matter. 9 | /// 10 | /// This cache is cheaper than looking up the module in python's module cache inspecting the 11 | /// module's `__dict__` each time you want access to the member. 12 | /// 13 | /// If you have multiple imports from the same module, we'll call `py.import` once for each member 14 | /// of the module. 15 | #[macro_export] 16 | macro_rules! py_import { 17 | ( $module_name:expr, $member_name:expr, $getter_fn:ident ) => { 18 | paste::paste! { 19 | static [] 20 | : pyo3::once_cell::GILOnceCell> 21 | = pyo3::once_cell::GILOnceCell::new(); 22 | 23 | fn $getter_fn<'py>(py: pyo3::Python<'py>) -> pyo3::PyResult<&'py pyo3::PyAny> { 24 | Ok([].get_or_init(py, || { 25 | Ok(py.import($module_name)?.get($member_name)?.to_object(py)) 26 | }) 27 | .as_ref() 28 | .map_err(|err| err.clone_ref(py))? 29 | .as_ref(py)) 30 | } 31 | } 32 | }; 33 | } 34 | -------------------------------------------------------------------------------- /native/libcst/src/nodes/py_cached.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | use pyo3::prelude::*; 7 | use std::convert::AsRef; 8 | use std::ops::Deref; 9 | 10 | /// An immutable wrapper around a rust type T and it's PyObject equivalent. Caches the conversion 11 | /// to and from the PyObject. 12 | pub struct PyCached { 13 | native: T, 14 | py_object: PyObject, 15 | } 16 | 17 | impl PyCached 18 | where 19 | T: ToPyObject, 20 | { 21 | pub fn new(py: Python, native: T) -> Self { 22 | Self { 23 | py_object: native.to_object(py), 24 | native, 25 | } 26 | } 27 | } 28 | 29 | impl<'source, T> FromPyObject<'source> for PyCached 30 | where 31 | T: FromPyObject<'source>, 32 | { 33 | fn extract(ob: &'source PyAny) -> PyResult { 34 | Python::with_gil(|py| { 35 | Ok(PyCached { 36 | native: ob.extract()?, 37 | py_object: ob.to_object(py), 38 | }) 39 | }) 40 | } 41 | } 42 | 43 | impl IntoPy for PyCached { 44 | fn into_py(self, _py: Python) -> PyObject { 45 | self.py_object 46 | } 47 | } 48 | 49 | impl ToPyObject for PyCached { 50 | fn to_object(&self, py: Python) -> PyObject { 51 | self.py_object.clone_ref(py) 52 | } 53 | } 54 | 55 | impl AsRef for PyCached { 56 | fn as_ref(&self) -> &T { 57 | &self.native 58 | } 59 | } 60 | 61 | impl Deref for PyCached { 62 | type Target = T; 63 | 64 | fn deref(&self) -> &Self::Target { 65 | &self.native 66 | } 67 | } 68 | 69 | impl From for PyCached 70 | where 71 | T: ToPyObject, 72 | { 73 | fn from(val: T) -> Self { 74 | Python::with_gil(|py| Self::new(py, val)) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /native/libcst/src/nodes/test_utils.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | use pyo3::prelude::*; 7 | 8 | py_import!("libcst._nodes.deep_equals", "deep_equals", get_deep_equals); 9 | 10 | pub fn repr_or_panic(py: Python, value: T) -> String 11 | where 12 | T: ToPyObject, 13 | { 14 | value 15 | .to_object(py) 16 | .as_ref(py) 17 | .repr() 18 | .expect("failed to call repr") 19 | .extract() 20 | .expect("repr should've returned str") 21 | } 22 | 23 | pub fn py_assert_deep_equals(py: Python, left: L, right: R) 24 | where 25 | L: ToPyObject, 26 | R: ToPyObject, 27 | { 28 | let (left, right) = (left.to_object(py), right.to_object(py)); 29 | let equals = get_deep_equals(py) 30 | .expect("failed to import deep_equals") 31 | .call1((&left, &right)) 32 | .expect("failed to call deep_equals") 33 | .extract::() 34 | .expect("deep_equals should return a bool"); 35 | if !equals { 36 | panic!( 37 | "assertion failed: {} was not deeply equal to {}", 38 | repr_or_panic(py, &left), 39 | repr_or_panic(py, &right), 40 | ); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /native/libcst/src/parser/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree 5 | 6 | mod errors; 7 | mod grammar; 8 | mod numbers; 9 | 10 | pub use errors::ParserError; 11 | pub(crate) use grammar::TokVec; 12 | pub use grammar::{python, Result}; 13 | -------------------------------------------------------------------------------- /native/libcst/src/parser/numbers.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree 5 | 6 | use regex::Regex; 7 | 8 | use crate::nodes::deflated::{Expression, Float, Imaginary, Integer}; 9 | 10 | static HEX: &str = r"0[xX](?:_?[0-9a-fA-F])+"; 11 | static BIN: &str = r"0[bB](?:_?[01])+"; 12 | static OCT: &str = r"0[oO](?:_?[0-7])+"; 13 | static DECIMAL: &str = r"(?:0(?:_?0)*|[1-9](?:_?[0-9])*)"; 14 | 15 | static EXPONENT: &str = r"[eE][-+]?[0-9](?:_?[0-9])*"; 16 | // Note: these don't exactly match the python implementation (exponent is not included) 17 | static POINT_FLOAT: &str = r"([0-9](?:_?[0-9])*\.(?:[0-9](?:_?[0-9])*)?|\.[0-9](?:_?[0-9])*)"; 18 | static EXP_FLOAT: &str = r"[0-9](?:_?[0-9])*"; 19 | 20 | thread_local! { 21 | static INTEGER_RE: Regex = 22 | Regex::new(format!("^({}|{}|{}|{})$", HEX, BIN, OCT, DECIMAL).as_str()).expect("regex"); 23 | static FLOAT_RE: Regex = 24 | Regex::new( 25 | format!( 26 | "^({}({})?|{}{})$", 27 | POINT_FLOAT, EXPONENT, EXP_FLOAT, EXPONENT 28 | ) 29 | .as_str(), 30 | ) 31 | .expect("regex"); 32 | static IMAGINARY_RE: Regex = 33 | Regex::new( 34 | format!( 35 | r"^([0-9](?:_?[0-9])*[jJ]|({}({})?|{}{})[jJ])$", 36 | POINT_FLOAT, EXPONENT, EXP_FLOAT, EXPONENT 37 | ) 38 | .as_str(), 39 | ) 40 | .expect("regex"); 41 | } 42 | 43 | pub(crate) fn parse_number(raw: &str) -> Expression { 44 | if INTEGER_RE.with(|r| r.is_match(raw)) { 45 | Expression::Integer(Box::new(Integer { 46 | value: raw, 47 | lpar: Default::default(), 48 | rpar: Default::default(), 49 | })) 50 | } else if FLOAT_RE.with(|r| r.is_match(raw)) { 51 | Expression::Float(Box::new(Float { 52 | value: raw, 53 | lpar: Default::default(), 54 | rpar: Default::default(), 55 | })) 56 | } else if IMAGINARY_RE.with(|r| r.is_match(raw)) { 57 | Expression::Imaginary(Box::new(Imaginary { 58 | value: raw, 59 | lpar: Default::default(), 60 | rpar: Default::default(), 61 | })) 62 | } else { 63 | Expression::Integer(Box::new(Integer { 64 | value: raw, 65 | lpar: Default::default(), 66 | rpar: Default::default(), 67 | })) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /native/libcst/src/py.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree 5 | 6 | use crate::nodes::traits::py::TryIntoPy; 7 | use pyo3::prelude::*; 8 | 9 | #[pymodule(gil_used = false)] 10 | #[pyo3(name = "native")] 11 | pub fn libcst_native(_py: Python, m: &Bound) -> PyResult<()> { 12 | #[pyfn(m)] 13 | #[pyo3(signature = (source, encoding=None))] 14 | fn parse_module(source: String, encoding: Option<&str>) -> PyResult { 15 | let m = crate::parse_module(source.as_str(), encoding)?; 16 | Python::with_gil(|py| m.try_into_py(py)) 17 | } 18 | 19 | #[pyfn(m)] 20 | fn parse_expression(source: String) -> PyResult { 21 | let expr = crate::parse_expression(source.as_str())?; 22 | Python::with_gil(|py| expr.try_into_py(py)) 23 | } 24 | 25 | #[pyfn(m)] 26 | fn parse_statement(source: String) -> PyResult { 27 | let stm = crate::parse_statement(source.as_str())?; 28 | Python::with_gil(|py| stm.try_into_py(py)) 29 | } 30 | 31 | Ok(()) 32 | } 33 | -------------------------------------------------------------------------------- /native/libcst/src/tokenizer/core/LICENSE: -------------------------------------------------------------------------------- 1 | PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 2 | 3 | 1. This LICENSE AGREEMENT is between the Python Software Foundation 4 | ("PSF"), and the Individual or Organization ("Licensee") accessing and 5 | otherwise using this software ("Python") in source or binary form and 6 | its associated documentation. 7 | 8 | 2. Subject to the terms and conditions of this License Agreement, PSF hereby 9 | grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, 10 | analyze, test, perform and/or display publicly, prepare derivative works, 11 | distribute, and otherwise use Python alone or in any derivative version, 12 | provided, however, that PSF's License Agreement and PSF's notice of copyright, 13 | i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 14 | 2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved" 15 | are retained in Python alone or in any derivative version prepared by Licensee. 16 | 17 | 3. In the event Licensee prepares a derivative work that is based on 18 | or incorporates Python or any part thereof, and wants to make 19 | the derivative work available to others as provided herein, then 20 | Licensee hereby agrees to include in any such work a brief summary of 21 | the changes made to Python. 22 | 23 | 4. PSF is making Python available to Licensee on an "AS IS" 24 | basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR 25 | IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND 26 | DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS 27 | FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT 28 | INFRINGE ANY THIRD PARTY RIGHTS. 29 | 30 | 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 31 | FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS 32 | A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, 33 | OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 34 | 35 | 6. This License Agreement will automatically terminate upon a material 36 | breach of its terms and conditions. 37 | 38 | 7. Nothing in this License Agreement shall be deemed to create any 39 | relationship of agency, partnership, or joint venture between PSF and 40 | Licensee. This License Agreement does not grant permission to use PSF 41 | trademarks or trade name in a trademark sense to endorse or promote 42 | products or services of Licensee, or any third party. 43 | 44 | 8. By copying, installing or otherwise using Python, Licensee 45 | agrees to be bound by the terms and conditions of this License 46 | Agreement. 47 | -------------------------------------------------------------------------------- /native/libcst/src/tokenizer/core/README.md: -------------------------------------------------------------------------------- 1 | Files in this directory are a derivative of CPython's tokenizer, and are 2 | therefore available under the PSF license. 3 | -------------------------------------------------------------------------------- /native/libcst/src/tokenizer/debug_utils.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | use std::fmt; 7 | 8 | /// An empty struct that when writes "..." when using `fmt::Debug`. Useful for omitting fields when 9 | /// using `fmt::Formatter::debug_struct`. 10 | pub struct EllipsisDebug; 11 | 12 | impl fmt::Debug for EllipsisDebug { 13 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 14 | f.write_str("...") 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /native/libcst/src/tokenizer/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree. 5 | 6 | mod core; 7 | mod debug_utils; 8 | mod operators; 9 | mod text_position; 10 | pub mod whitespace_parser; 11 | 12 | pub use self::core::*; 13 | 14 | #[cfg(test)] 15 | mod tests; 16 | -------------------------------------------------------------------------------- /native/libcst/tests/.gitattributes: -------------------------------------------------------------------------------- 1 | fixtures/mixed_newlines.py autocrlf=false -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/big_binary_operator.py: -------------------------------------------------------------------------------- 1 | ( # 350 binary operators lets go 2 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 3 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 4 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 5 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 6 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 7 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 8 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 9 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 10 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 11 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 12 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 13 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 14 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 15 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 16 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 17 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 18 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 19 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 20 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 21 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 22 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 23 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 24 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 25 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 26 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 27 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 28 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 29 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 30 | 'X' + 'Y' + 'Z' + 'Q' + 'T' + 31 | 'X' + 'Y' + 'Z' + 'Q' + 'T' 32 | ) 33 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/class_craziness.py: -------------------------------------------------------------------------------- 1 | class Foo: ... 2 | 3 | class Bar : 4 | ... 5 | 6 | class Old ( ) : 7 | gold : int 8 | 9 | 10 | class OO ( Foo ) : ... 11 | 12 | class OOP ( Foo , Bar, ) : pass 13 | 14 | class OOPS ( 15 | Foo , 16 | 17 | ) : 18 | pass 19 | 20 | class OOPSI ( Foo, * Bar , metaclass = 21 | foo , 22 | ): pass 23 | 24 | class OOPSIE ( list , *args, kw = arg , ** kwargs ) : 25 | what : does_this_even = mean 26 | 27 | def __init__(self) -> None: 28 | self.foo: Bar = Bar() 29 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/comments.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # fmt: on 3 | # Some license here. 4 | # 5 | # Has many lines. Many, many lines. 6 | # Many, many, many lines. 7 | """Module docstring. 8 | 9 | Possibly also many, many lines. 10 | """ 11 | 12 | import os.path 13 | import sys 14 | 15 | import a 16 | from b.c.d.e import X # some noqa comment 17 | 18 | try: 19 | import fast 20 | except ImportError: 21 | import slow as fast 22 | 23 | 24 | # Some comment before a function. 25 | y = 1 26 | ( 27 | # some strings 28 | y # type: ignore 29 | ) 30 | 31 | 32 | def function(default=None): 33 | """Docstring comes first. 34 | 35 | Possibly many lines. 36 | """ 37 | # FIXME: Some comment about why this function is crap but still in production. 38 | import inner_imports 39 | 40 | if inner_imports.are_evil(): 41 | # Explains why we have this if. 42 | # In great detail indeed. 43 | x = X() 44 | return x.method1() # type: ignore 45 | 46 | 47 | # This return is also commented for some reason. 48 | return default 49 | 50 | 51 | # Explains why we use global state. 52 | GLOBAL_STATE = {"a": a(1), "b": a(2), "c": a(3)} 53 | 54 | 55 | # Another comment! 56 | # This time two lines. 57 | 58 | 59 | class Foo: 60 | """Docstring for class Foo. Example from Sphinx docs.""" 61 | 62 | #: Doc comment for class attribute Foo.bar. 63 | #: It can have multiple lines. 64 | bar = 1 65 | 66 | flox = 1.5 #: Doc comment for Foo.flox. One line only. 67 | 68 | baz = 2 69 | """Docstring for class attribute Foo.baz.""" 70 | 71 | def __init__(self): 72 | #: Doc comment for instance attribute qux. 73 | self.qux = 3 74 | 75 | self.spam = 4 76 | """Docstring for instance attribute spam.""" 77 | 78 | 79 | #'

This is pweave!

80 | 81 | 82 | @fast(really=True) 83 | async def wat(): 84 | # This comment, for some reason \ 85 | # contains a trailing backslash. 86 | async with X.open_async() as x: # Some more comments 87 | result = await x.method1() 88 | # Comment after ending a block. 89 | if result: 90 | print("A OK", file=sys.stdout) 91 | # Comment between things. 92 | print() 93 | 94 | 95 | if True: # Hanging comments 96 | # because why not 97 | pass 98 | 99 | # Some closing comments. 100 | # Maybe Vim or Emacs directives for formatting. 101 | # Who knows. 102 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/comparisons.py: -------------------------------------------------------------------------------- 1 | if not 1: pass 2 | if 1 and 1: pass 3 | if 1 or 1: pass 4 | if not not not 1: pass 5 | if not 1 and 1 and 1: pass 6 | if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass 7 | 8 | if 1: pass 9 | #x = (1 == 1) 10 | if 1 == 1: pass 11 | if 1 != 1: pass 12 | if 1 < 1: pass 13 | if 1 > 1: pass 14 | if 1 <= 1: pass 15 | if 1 >= 1: pass 16 | if x is x: pass 17 | #if x is not x: pass 18 | #if 1 in (): pass 19 | #if 1 not in (): pass 20 | if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 in x is x is x: pass 21 | #if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in x is x is not x: pass 22 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/dangling_indent.py: -------------------------------------------------------------------------------- 1 | if 1: 2 | pass 3 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/decorated_function_without_body.py: -------------------------------------------------------------------------------- 1 | @hello 2 | @bello 3 | def f () : ... -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/dysfunctional_del.py: -------------------------------------------------------------------------------- 1 | # dysfunctional_del.py 2 | 3 | del a 4 | 5 | del a[1] 6 | 7 | del a.b.c 8 | del ( a, b , c ) 9 | del [ a, b , c ] 10 | 11 | del a , b, c 12 | 13 | 14 | del a[1] , b [ 2] -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/expr_statement.py: -------------------------------------------------------------------------------- 1 | 1 2 | 1, 2, 3 3 | x = 1 4 | x = 1, 2, 3 5 | x = y = z = 1, 2, 3 6 | x, y, z = 1, 2, 3 7 | abc = a, b, c = x, y, z = xyz = 1, 2, (3, 4) 8 | 9 | ( ( ( ... ) ) ) 10 | 11 | a , = b -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/global_nonlocal.py: -------------------------------------------------------------------------------- 1 | global a 2 | global b , c, d 3 | nonlocal a 4 | nonlocal a , b -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/import.py: -------------------------------------------------------------------------------- 1 | # 'import' dotted_as_names 2 | import sys 3 | import time, sys 4 | # 'from' dotted_name 'import' ('*' | '(' import_as_names ')' | import_as_names) 5 | from time import time 6 | from time import (time) 7 | from sys import path, argv 8 | from sys import (path, argv) 9 | from sys import (path, argv,) 10 | from sys import * 11 | 12 | 13 | from a import (b, ) 14 | from . import a 15 | from .a import b 16 | from ... import a 17 | from ...a import b 18 | from .... import a 19 | from ...... import a -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/indents_but_no_eol_before_eof.py: -------------------------------------------------------------------------------- 1 | if 1: 2 | if 2: 3 | if 3: 4 | pass -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/just_a_comment_without_nl.py: -------------------------------------------------------------------------------- 1 | # just a comment without a newline -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/malicious_match.py: -------------------------------------------------------------------------------- 1 | 2 | # foo 3 | 4 | match ( foo ) : #comment 5 | 6 | # more comments 7 | case False : # comment 8 | 9 | ... 10 | case ( True ) : ... 11 | case _ : ... 12 | case ( _ ) : ... # foo 13 | 14 | # bar 15 | 16 | match x: 17 | case "StringMatchValue" : pass 18 | case [1, 2] : pass 19 | case [ 1 , * foo , * _ , ]: pass 20 | case [ [ _, ] , *_ ]: pass 21 | case {1: _, 2: _}: pass 22 | case { "foo" : bar , ** rest } : pass 23 | case { 1 : {**rest} , } : pass 24 | case Point2D(): pass 25 | case Cls ( 0 , ) : pass 26 | case Cls ( x=0, y = 2) :pass 27 | case Cls ( 0 , 1 , x = 0 , y = 2 ) : pass 28 | case [x] as y: pass 29 | case [x] as y : pass 30 | case (True)as x:pass 31 | case Foo:pass 32 | case (Foo):pass 33 | case ( Foo ) : pass 34 | case [ ( Foo ) , ]: pass 35 | case Foo|Bar|Baz : pass 36 | case Foo | Bar | ( Baz): pass 37 | case x,y , * more :pass 38 | case y.z: pass 39 | case 1, 2: pass 40 | case ( Foo ( ) ) : pass 41 | case (lol) if ( True , ) :pass 42 | 43 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/mixed_newlines.py: -------------------------------------------------------------------------------- 1 | "" % { 2 | 'test1': '', 'test2': '', 3 | } 4 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/pep646.py: -------------------------------------------------------------------------------- 1 | # see https://github.com/python/cpython/pull/31018/files#diff-3f516b60719dd445d33225e4f316b36e85c9c51a843a0147349d11a005c55937 2 | 3 | A[*b] 4 | A[ * b ] 5 | A[ * b , ] 6 | A[*b] = 1 7 | del A[*b] 8 | 9 | A[* b , * b] 10 | A[ b, *b] 11 | A[* b, b] 12 | A[ * b,b, b] 13 | A[b, *b, b] 14 | 15 | A[*A[b, *b, b], b] 16 | A[b, ...] 17 | A[*A[b, ...]] 18 | 19 | A[ * ( 1,2,3)] 20 | A[ * [ 1,2,3]] 21 | 22 | A[1:2, *t] 23 | A[1:, *t, 1:2] 24 | A[:, *t, :] 25 | A[*t, :, *t] 26 | 27 | A[* returns_list()] 28 | A[*returns_list(), * returns_list(), b] 29 | 30 | def f1(*args: *b): pass 31 | def f2(*args: *b, arg1): pass 32 | def f3(*args: *b, arg1: int): pass 33 | def f4(*args: *b, arg1: int = 1): pass 34 | 35 | def f(*args: *tuple[int, ...]): pass 36 | def f(*args: *tuple[int, *Ts]): pass 37 | def f() -> tuple[int, *tuple[int, ...]]: pass -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/raise.py: -------------------------------------------------------------------------------- 1 | raise 2 | raise foo 3 | raise foo from bar 4 | raise lol() from f() + 1 -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/smol_statements.py: -------------------------------------------------------------------------------- 1 | def f(): 2 | pass ; break ; continue ; return ; return foo 3 | 4 | assert foo , bar ; a += 2 -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/spacious_spaces.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/starry_tries.py: -------------------------------------------------------------------------------- 1 | #foo. 2 | 3 | try : 4 | pass 5 | 6 | # foo 7 | 8 | except * lol as LOL : 9 | 10 | pass 11 | 12 | except * f: 13 | 14 | # foo 15 | 16 | pass 17 | 18 | else : 19 | 20 | pass 21 | 22 | finally : 23 | 24 | foo 25 | 26 | try: 27 | pass 28 | except*f: 29 | pass 30 | finally: 31 | pass 32 | 33 | 34 | try: 35 | 36 | # 1 37 | 38 | try: 39 | 40 | # 2 41 | 42 | pass 43 | 44 | # 3 45 | 46 | # 4 47 | 48 | finally: 49 | 50 | # 5 51 | 52 | pass 53 | 54 | # 6 55 | 56 | # 7 57 | 58 | except *foo: 59 | 60 | #8 61 | 62 | pass 63 | 64 | #9 65 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/suicidal_slices.py: -------------------------------------------------------------------------------- 1 | slice[0] 2 | slice[0:1] 3 | slice[0:1:2] 4 | slice[:] 5 | slice[:-1] 6 | slice[1:] 7 | slice[::-1] 8 | slice[d :: d + 1] 9 | slice[:c, c - 1] 10 | numpy[:, 0:1] 11 | numpy[:, :-1] 12 | numpy[0, :] 13 | numpy[:, i] 14 | numpy[0, :2] 15 | numpy[:N, 0] 16 | numpy[:2, :4] 17 | numpy[2:4, 1:5] 18 | numpy[4:, 2:] 19 | numpy[:, (0, 1, 2, 5)] 20 | numpy[0, [0]] 21 | numpy[:, [i]] 22 | numpy[1 : c + 1, c] 23 | numpy[-(c + 1) :, d] 24 | numpy[:, l[-2]] 25 | numpy[:, ::-1] 26 | numpy[np.newaxis, :] 27 | 28 | ( spaces [:: , a : , a : a : a , ] ) -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/super_strings.py: -------------------------------------------------------------------------------- 1 | _ = "" 2 | _ = '' 3 | _ = """""" 4 | _ = '''''' 5 | 6 | _ = 'a' "string" 'that' r"is" 'concatenated ' 7 | 8 | b"string " 9 | b"and non f" rb'string' 10 | 11 | ( 12 | "parenthesized" 13 | "concatenated" 14 | """triple 15 | quoted 16 | """ 17 | 18 | ) 19 | 20 | _ = f"string" 21 | 22 | f"string" "bonanza" f'starts' r"""here""" 23 | 24 | _ = f"something {{**not** an expression}} {but(this._is)} {{and this isn't.}} end" 25 | 26 | _(f"ok { expr = !r: aosidjhoi } end") 27 | 28 | print(f"{self.ERASE_CURRENT_LINE}{self._human_seconds(elapsed_time)} {percent:.{self.pretty_precision}f}% complete, {self.estimate_completion(elapsed_time, finished, left)} estimated for {left} files to go...") 29 | 30 | f"{"\n".join()}" 31 | 32 | f"___{ 33 | x 34 | }___" 35 | 36 | f"___{( 37 | x 38 | )}___" 39 | 40 | f'\{{\}}' 41 | f"regexp_like(path, '.*\{file_type}$')" 42 | f"\lfoo" 43 | 44 | f"{_:{_:}{a}}" 45 | 46 | f"foo {f"bar {x}"} baz" 47 | f'some words {a+b:.3f} more words {c+d=} final words' 48 | f"{'':*^{1:{1}}}" 49 | f"{'':*^{1:{1:{1}}}}" 50 | f"{f"{f"{f"{f"{f"{1+1}"}"}"}"}"}" 51 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/terrible_tries.py: -------------------------------------------------------------------------------- 1 | #foo. 2 | 3 | try : 4 | bar() 5 | 6 | finally : 7 | pass 8 | 9 | 10 | try : 11 | pass 12 | 13 | # foo 14 | 15 | except lol as LOL : 16 | 17 | pass 18 | 19 | except : 20 | 21 | # foo 22 | 23 | pass 24 | 25 | else : 26 | 27 | pass 28 | 29 | finally : 30 | 31 | foo 32 | 33 | try: 34 | pass 35 | except: 36 | pass 37 | finally: 38 | pass 39 | 40 | 41 | try: 42 | 43 | # 1 44 | 45 | try: 46 | 47 | # 2 48 | 49 | pass 50 | 51 | # 3 52 | 53 | # 4 54 | 55 | finally: 56 | 57 | # 5 58 | 59 | pass 60 | 61 | # 6 62 | 63 | # 7 64 | 65 | except foo: 66 | 67 | #8 68 | 69 | pass 70 | 71 | #9 72 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/trailing_comment_without_nl.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # hehehe >:) -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/trailing_whitespace.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | x = 42 4 | print(x) 5 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/tuple_shenanigans.py: -------------------------------------------------------------------------------- 1 | (1, 2) 2 | (1, 2, 3) 3 | 4 | # alright here we go. 5 | 6 | () 7 | (()) 8 | (((())), ()) 9 | ( # evil >:) 10 | # evil >:( 11 | ) # ... 12 | (1,) 13 | ( * 1 , * 2 ,) 14 | *_ = (l,) 15 | () = x 16 | ( ) = ( x, ) 17 | (x) = (x) 18 | ( x , ) = x 19 | ( x , *y , * z , ) = l 20 | ( x , *y , * z , ) = ( x , *y , * z , ) = ( x , *y , * z , x ) 21 | ( 22 | x , # :) 23 | bar, * 24 | baz 25 | , 26 | ) =\ 27 | ( 28 | (let, *s, ( ) ) , 29 | nest , them , ( * t , * u , * p , l , * e , s , ) 30 | ) -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/type_parameters.py: -------------------------------------------------------------------------------- 1 | # fmt: off 2 | 3 | type TA = int 4 | 5 | type TA1[A] = lambda A: A 6 | 7 | class Outer[A]: 8 | type TA1[A] = None 9 | 10 | type TA1[A, B] = dict[A, B] 11 | 12 | class Outer[A]: 13 | def inner[B](self): 14 | type TA1[C] = TA1[A, B] | int 15 | return TA1 16 | 17 | def more_generic[T, *Ts, **P](): 18 | type TA[T2, *Ts2, **P2] = tuple[Callable[P, tuple[T, *Ts]], Callable[P2, tuple[T2, *Ts2]]] 19 | return TA 20 | 21 | type Recursive = Recursive 22 | 23 | def func[A](A): return A 24 | 25 | class ClassA: 26 | def func[__A](self, __A): return __A 27 | 28 | class ClassA[A, B](dict[A, B]): 29 | ... 30 | 31 | class ClassA[A]: 32 | def funcB[B](self): 33 | class ClassC[C]: 34 | def funcD[D](self): 35 | return lambda: (A, B, C, D) 36 | return ClassC 37 | 38 | class Child[T](Base[lambda: (int, outer_var, T)]): ... 39 | 40 | type Alias[T: ([T for T in (T, [1])[1]], T)] = [T for T in T.__name__] 41 | type Alias[T: [lambda: T for T in (T, [1])[1]]] = [lambda: T for T in T.__name__] 42 | 43 | class Foo[T: Foo, U: (Foo, Foo)]: 44 | pass 45 | 46 | def func[T](a: T = "a", *, b: T = "b"): 47 | return (a, b) 48 | 49 | def func1[A: str, B: str | int, C: (int, str)](): 50 | return (A, B, C) 51 | 52 | type A [ T , * V ] =foo;type B=A 53 | 54 | def AAAAAAAAAAAAAAAAAA [ T : int ,*Ts , ** TT ] ():pass 55 | class AAAAAAAAAAAAAAAAAA [ T : int ,*Ts , ** TT ] :pass 56 | 57 | def yikes[A:int,*B,**C](*d:*tuple[A,*B,...])->A:pass 58 | 59 | def func[T=int, **U=float, *V=None](): pass 60 | 61 | class C[T=int, **U=float, *V=None]: pass 62 | 63 | type Alias[T = int, **U = float, *V = None] = int 64 | 65 | default = tuple[int, str] 66 | type Alias[*Ts = *default] = Ts 67 | type Foo[ * T = * default ] = int 68 | type Foo[*T=*default ]=int 69 | type Foo [ * T = * default ] = int -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/vast_emptiness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Instagram/LibCST/482a2e5f0997273e3fb272346dac275c04e84807/native/libcst/tests/fixtures/vast_emptiness.py -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/with_wickedness.py: -------------------------------------------------------------------------------- 1 | # with_wickedness 2 | 3 | with foo : 4 | pass 5 | 6 | with foo, bar: 7 | pass 8 | 9 | with (foo, bar): 10 | pass 11 | 12 | with (foo, bar,): 13 | pass 14 | 15 | with foo, bar as bar: 16 | pass 17 | 18 | with (foo, bar as bar): 19 | pass 20 | 21 | with (foo, bar as bar,): 22 | pass 23 | 24 | async def f(): 25 | async with foo: 26 | 27 | with bar: 28 | pass 29 | 30 | async with foo : 31 | pass 32 | 33 | async with foo, bar: 34 | pass 35 | 36 | async with (foo, bar): 37 | pass 38 | 39 | async with (foo, bar,): 40 | pass 41 | 42 | async with foo, bar as bar: 43 | pass 44 | 45 | async with (foo, bar as bar): 46 | pass 47 | 48 | async with (foo, bar as bar,): 49 | pass 50 | 51 | async with foo(1+1) as bar , 1 as (a, b, ) , 2 as [a, b] , 3 as a[b] : 52 | pass 53 | -------------------------------------------------------------------------------- /native/libcst/tests/fixtures/wonky_walrus.py: -------------------------------------------------------------------------------- 1 | ( foo := 5 ) 2 | 3 | any((lastNum := num) == 1 for num in [1, 2, 3]) 4 | 5 | [(lastNum := num) == 1 for num in [1, 2, 3]] 6 | 7 | while f := x(): 8 | pass 9 | 10 | if f := x(): pass 11 | 12 | f(y:=1) 13 | f(x, y := 1 ) 14 | 15 | _[_:=10] -------------------------------------------------------------------------------- /native/libcst/tests/parser_roundtrip.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree 5 | 6 | use difference::assert_diff; 7 | use itertools::Itertools; 8 | use libcst_native::{parse_module, prettify_error, Codegen}; 9 | use std::{ 10 | iter::once, 11 | path::{Component, PathBuf}, 12 | }; 13 | 14 | fn all_fixtures() -> impl Iterator { 15 | let mut path = PathBuf::from(file!()); 16 | path.pop(); 17 | path = path 18 | .components() 19 | .skip(1) 20 | .chain(once(Component::Normal("fixtures".as_ref()))) 21 | .collect(); 22 | 23 | path.read_dir().expect("read_dir").into_iter().map(|file| { 24 | let path = file.unwrap().path(); 25 | let contents = std::fs::read_to_string(&path).expect("reading file"); 26 | (path, contents) 27 | }) 28 | } 29 | 30 | #[test] 31 | fn roundtrip_fixtures() { 32 | for (path, input) in all_fixtures() { 33 | let input = if let Some(stripped) = input.strip_prefix('\u{feff}') { 34 | stripped 35 | } else { 36 | &input 37 | }; 38 | let m = match parse_module(input, None) { 39 | Ok(m) => m, 40 | Err(e) => panic!("{}", prettify_error(e, format!("{:#?}", path).as_ref())), 41 | }; 42 | let mut state = Default::default(); 43 | m.codegen(&mut state); 44 | let generated = state.to_string(); 45 | if generated != input { 46 | let got = visualize(&generated); 47 | let expected = visualize(input); 48 | assert_diff!(expected.as_ref(), got.as_ref(), "", 0); 49 | } 50 | } 51 | } 52 | 53 | fn visualize(s: &str) -> String { 54 | s.replace(' ', "▩").lines().join("↩\n") 55 | } 56 | -------------------------------------------------------------------------------- /native/libcst_derive/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "libcst_derive" 3 | version = "1.8.0" 4 | edition = "2018" 5 | description = "Proc macro helpers for libcst." 6 | license = "MIT" 7 | repository = "https://github.com/Instagram/LibCST" 8 | documentation = "https://libcst.rtfd.org" 9 | keywords = ["macros", "python"] 10 | 11 | [lib] 12 | proc-macro = true 13 | 14 | [dependencies] 15 | syn = "2.0" 16 | quote = "1.0" 17 | 18 | [dev-dependencies] 19 | trybuild = "1.0" 20 | -------------------------------------------------------------------------------- /native/libcst_derive/src/codegen.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree 5 | 6 | use proc_macro::TokenStream; 7 | use quote::{quote, quote_spanned}; 8 | use syn::{self, spanned::Spanned, Data, DataEnum, DeriveInput, Fields, FieldsUnnamed}; 9 | 10 | pub(crate) fn impl_codegen(ast: &DeriveInput) -> TokenStream { 11 | match &ast.data { 12 | Data::Enum(e) => impl_enum(ast, e), 13 | Data::Struct(s) => quote_spanned! { 14 | s.struct_token.span() => 15 | compile_error!("Struct type is not supported") 16 | } 17 | .into(), 18 | Data::Union(u) => quote_spanned! { 19 | u.union_token.span() => 20 | compile_error!("Union type is not supported") 21 | } 22 | .into(), 23 | } 24 | } 25 | 26 | fn impl_enum(ast: &DeriveInput, e: &DataEnum) -> TokenStream { 27 | let mut varnames = vec![]; 28 | for var in e.variants.iter() { 29 | match &var.fields { 30 | Fields::Named(n) => { 31 | return quote_spanned! { 32 | n.span() => 33 | compile_error!("Named enum fields not supported") 34 | } 35 | .into() 36 | } 37 | f @ Fields::Unit => { 38 | return quote_spanned! { 39 | f.span() => 40 | compile_error!("Empty enum variants not supported") 41 | } 42 | .into() 43 | } 44 | Fields::Unnamed(FieldsUnnamed { unnamed, .. }) => { 45 | if unnamed.len() > 1 { 46 | return quote_spanned! { 47 | unnamed.span() => 48 | compile_error!("Multiple unnamed fields not supported") 49 | } 50 | .into(); 51 | } 52 | varnames.push(&var.ident); 53 | } 54 | } 55 | } 56 | let ident = &ast.ident; 57 | let generics = &ast.generics; 58 | let gen = quote! { 59 | impl<'a> Codegen<'a> for #ident #generics { 60 | fn codegen(&self, state: &mut CodegenState<'a>) { 61 | match self { 62 | #(Self::#varnames(x) => x.codegen(state),)* 63 | } 64 | } 65 | } 66 | }; 67 | gen.into() 68 | } 69 | -------------------------------------------------------------------------------- /native/libcst_derive/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree 5 | 6 | mod inflate; 7 | use inflate::impl_inflate; 8 | mod parenthesized_node; 9 | use parenthesized_node::impl_parenthesized_node; 10 | mod codegen; 11 | use codegen::impl_codegen; 12 | mod into_py; 13 | use into_py::impl_into_py; 14 | mod cstnode; 15 | use cstnode::{impl_cst_node, CSTNodeParams}; 16 | 17 | use proc_macro::TokenStream; 18 | use syn::{parse_macro_input, DeriveInput}; 19 | 20 | #[proc_macro_derive(Inflate)] 21 | pub fn inflate_derive(input: TokenStream) -> TokenStream { 22 | let ast = syn::parse(input).unwrap(); 23 | impl_inflate(&ast) 24 | } 25 | 26 | #[proc_macro_derive(ParenthesizedNode)] 27 | pub fn parenthesized_node_derive(input: TokenStream) -> TokenStream { 28 | impl_parenthesized_node(&syn::parse(input).unwrap(), false) 29 | } 30 | 31 | #[proc_macro_derive(ParenthesizedDeflatedNode)] 32 | pub fn parenthesized_deflated_node_derive(input: TokenStream) -> TokenStream { 33 | impl_parenthesized_node(&syn::parse(input).unwrap(), true) 34 | } 35 | 36 | #[proc_macro_derive(Codegen)] 37 | pub fn codegen_derive(input: TokenStream) -> TokenStream { 38 | impl_codegen(&syn::parse(input).unwrap()) 39 | } 40 | 41 | #[proc_macro_derive(TryIntoPy, attributes(skip_py, no_py_default))] 42 | pub fn into_py(input: TokenStream) -> TokenStream { 43 | impl_into_py(&syn::parse(input).unwrap()) 44 | } 45 | 46 | #[proc_macro_attribute] 47 | pub fn cst_node(args: TokenStream, input: TokenStream) -> TokenStream { 48 | let args = parse_macro_input!(args as CSTNodeParams); 49 | impl_cst_node(parse_macro_input!(input as DeriveInput), args) 50 | } 51 | -------------------------------------------------------------------------------- /native/libcst_derive/tests/pass/simple.rs: -------------------------------------------------------------------------------- 1 | // Copyright (c) Meta Platforms, Inc. and affiliates. 2 | // 3 | // This source code is licensed under the MIT license found in the 4 | // LICENSE file in the root directory of this source tree 5 | 6 | use libcst_derive::cst_node; 7 | 8 | #[derive(Debug, PartialEq, Eq, Clone)] 9 | pub struct WS<'a>(&'a str); 10 | 11 | type TokenRef<'r, 'a> = &'r &'a str; 12 | 13 | #[cst_node] 14 | pub enum Foo<'a> { 15 | One(One<'a>), 16 | Two(Box>), 17 | } 18 | 19 | #[cst_node] 20 | pub struct One<'a> { 21 | pub two: Box>, 22 | pub header: WS<'a>, 23 | 24 | pub(crate) newline_tok: TokenRef<'a>, 25 | } 26 | 27 | #[cst_node] 28 | pub struct Two<'a> { 29 | pub whitespace_before: WS<'a>, 30 | pub(crate) tok: TokenRef<'a>, 31 | } 32 | 33 | #[cst_node] 34 | struct Thin<'a> { 35 | pub whitespace: WS<'a>, 36 | } 37 | 38 | #[cst_node] 39 | struct Value<'a> { 40 | pub value: &'a str, 41 | } 42 | 43 | #[cst_node] 44 | struct Empty {} 45 | 46 | #[cst_node] 47 | enum Smol<'a> { 48 | #[allow(dead_code)] 49 | Thin(Thin<'a>), 50 | #[allow(dead_code)] 51 | Empty(Empty), 52 | } 53 | 54 | fn main() {} 55 | -------------------------------------------------------------------------------- /native/roundtrip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) Meta Platforms, Inc. and affiliates. 4 | # 5 | # This source code is licensed under the MIT license found in the 6 | # LICENSE file in the root directory of this source tree. 7 | 8 | PARSE=$(dirname $0)/target/release/parse 9 | 10 | exec diff -u "$1" <($PARSE < "$1") 11 | -------------------------------------------------------------------------------- /scripts/check_copyright.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | import re 7 | import sys 8 | from pathlib import Path 9 | from subprocess import run 10 | from typing import Iterable, List, Pattern 11 | 12 | # Use the copyright header from this file as the benchmark for all files 13 | EXPECTED_HEADER: str = "\n".join( 14 | line for line in Path(__file__).read_text().splitlines()[:4] 15 | ) 16 | 17 | EXCEPTION_PATTERNS: List[Pattern[str]] = [ 18 | re.compile(pattern) 19 | for pattern in ( 20 | r"^native/libcst/tests/fixtures/", 21 | r"^libcst/_add_slots\.py$", 22 | r"^libcst/tests/test_(e2e|fuzz)\.py$", 23 | r"^libcst/_parser/base_parser\.py$", 24 | r"^libcst/_parser/parso/utils\.py$", 25 | r"^libcst/_parser/parso/pgen2/(generator|grammar_parser)\.py$", 26 | r"^libcst/_parser/parso/python/(py_token|tokenize)\.py$", 27 | r"^libcst/_parser/parso/tests/test_(fstring|tokenize|utils)\.py$", 28 | ) 29 | ] 30 | 31 | 32 | def tracked_files() -> Iterable[Path]: 33 | proc = run( 34 | ["git", "ls-tree", "-r", "--name-only", "HEAD"], 35 | check=True, 36 | capture_output=True, 37 | encoding="utf-8", 38 | ) 39 | yield from ( 40 | path 41 | for line in proc.stdout.splitlines() 42 | if not any(pattern.search(line) for pattern in EXCEPTION_PATTERNS) 43 | if (path := Path(line)) and path.is_file() and path.suffix in (".py", ".sh") 44 | ) 45 | 46 | 47 | def main() -> None: 48 | error = False 49 | for path in tracked_files(): 50 | content = path.read_text("utf-8") 51 | if EXPECTED_HEADER not in content: 52 | print(f"Missing or incomplete copyright in {path}") 53 | error = True 54 | sys.exit(1 if error else 0) 55 | 56 | 57 | if __name__ == "__main__": 58 | main() 59 | -------------------------------------------------------------------------------- /scripts/regenerate-fixtures.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | """ 7 | Regenerate test fixtures, eg. after upgrading Pyre 8 | """ 9 | 10 | import json 11 | import os 12 | from pathlib import Path 13 | from subprocess import run 14 | 15 | from libcst.metadata import TypeInferenceProvider 16 | 17 | 18 | def main() -> None: 19 | CWD = Path.cwd() 20 | repo_root = Path(__file__).parent.parent 21 | test_root = repo_root / "libcst" / "tests" / "pyre" 22 | 23 | try: 24 | os.chdir(test_root) 25 | run(["pyre", "-n", "start", "--no-watchman"], check=True) 26 | 27 | for file_path in test_root.glob("*.py"): 28 | json_path = file_path.with_suffix(".json") 29 | print(f"generating {file_path} -> {json_path}") 30 | 31 | path_str = file_path.as_posix() 32 | cache = TypeInferenceProvider.gen_cache(test_root, [path_str], timeout=None) 33 | result = cache[path_str] 34 | json_path.write_text(json.dumps(result, sort_keys=True, indent=2)) 35 | 36 | finally: 37 | run(["pyre", "-n", "stop"], check=True) 38 | os.chdir(CWD) 39 | 40 | 41 | if __name__ == "__main__": 42 | main() 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from os import environ 7 | 8 | import setuptools 9 | from setuptools_rust import Binding, RustExtension 10 | 11 | 12 | def no_local_scheme(version: str) -> str: 13 | return "" 14 | 15 | 16 | setuptools.setup( 17 | setup_requires=["setuptools-rust", "setuptools_scm"], 18 | use_scm_version={ 19 | "write_to": "libcst/_version.py", 20 | **( 21 | {"local_scheme": no_local_scheme} 22 | if "LIBCST_NO_LOCAL_SCHEME" in environ 23 | else {} 24 | ), 25 | }, 26 | packages=setuptools.find_packages(), 27 | package_data={ 28 | "libcst": ["py.typed"], 29 | "libcst.tests.pyre": ["*"], 30 | "libcst.codemod.tests": ["*"], 31 | }, 32 | test_suite="libcst", 33 | rust_extensions=[ 34 | RustExtension( 35 | "libcst.native", 36 | path="native/libcst/Cargo.toml", 37 | binding=Binding.PyO3, 38 | ) 39 | ], 40 | zip_safe=False, # for mypy compatibility https://mypy.readthedocs.io/en/latest/installed_packages.html 41 | ) 42 | -------------------------------------------------------------------------------- /stubs/hypothesis.pyi: -------------------------------------------------------------------------------- 1 | # pyre-unsafe 2 | 3 | from typing import Any 4 | 5 | def __getattr__(name: str) -> Any: ... 6 | -------------------------------------------------------------------------------- /stubs/hypothesmith.pyi: -------------------------------------------------------------------------------- 1 | # pyre-unsafe 2 | 3 | from typing import Any 4 | 5 | def __getattr__(name: str) -> Any: ... 6 | -------------------------------------------------------------------------------- /stubs/libcst/native.pyi: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Optional 7 | import libcst 8 | 9 | def parse_module(source: str, encoding: Optional[str]) -> libcst.Module: ... 10 | def parse_expression(source: str) -> libcst.BaseExpression: ... 11 | def parse_statement(source: str) -> libcst.BaseStatement: ... 12 | -------------------------------------------------------------------------------- /stubs/libcst_native/parser_config.pyi: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Any, FrozenSet, Mapping, Sequence 7 | 8 | from libcst._parser.parso.utils import PythonVersionInfo 9 | 10 | class BaseWhitespaceParserConfig: 11 | def __new__( 12 | cls, 13 | *, 14 | lines: Sequence[str], 15 | default_newline: str, 16 | ) -> BaseWhitespaceParserConfig: ... 17 | lines: Sequence[str] 18 | default_newline: str 19 | 20 | class ParserConfig(BaseWhitespaceParserConfig): 21 | def __new__( 22 | cls, 23 | *, 24 | lines: Sequence[str], 25 | encoding: str, 26 | default_indent: str, 27 | default_newline: str, 28 | has_trailing_newline: bool, 29 | version: PythonVersionInfo, 30 | future_imports: FrozenSet[str], 31 | ) -> BaseWhitespaceParserConfig: ... 32 | # lines is inherited 33 | encoding: str 34 | default_indent: str 35 | # default_newline is inherited 36 | has_trailing_newline: bool 37 | version: PythonVersionInfo 38 | future_imports: FrozenSet[str] 39 | 40 | def parser_config_asdict(config: ParserConfig) -> Mapping[str, Any]: ... 41 | -------------------------------------------------------------------------------- /stubs/libcst_native/token_type.pyi: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | class TokenType: 7 | name: str 8 | contains_syntax: bool 9 | 10 | STRING: TokenType = ... 11 | NAME: TokenType = ... 12 | NUMBER: TokenType = ... 13 | OP: TokenType = ... 14 | NEWLINE: TokenType = ... 15 | INDENT: TokenType = ... 16 | DEDENT: TokenType = ... 17 | ASYNC: TokenType = ... 18 | AWAIT: TokenType = ... 19 | FSTRING_START: TokenType = ... 20 | FSTRING_STRING: TokenType = ... 21 | FSTRING_END: TokenType = ... 22 | ENDMARKER: TokenType = ... 23 | # unused dummy tokens for backwards compat with the parso tokenizer 24 | ERRORTOKEN: TokenType = ... 25 | ERROR_DEDENT: TokenType = ... 26 | -------------------------------------------------------------------------------- /stubs/libcst_native/tokenize.pyi: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Iterator, Optional, Tuple 7 | 8 | from libcst_native import token_type, whitespace_state 9 | 10 | class Token: 11 | def __new__( 12 | cls, 13 | type: token_type.TokenType, 14 | string: str, 15 | start_pos: Tuple[int, int], 16 | end_pos: Tuple[int, int], 17 | whitespace_before: whitespace_state.WhitespaceState, 18 | whitespace_after: whitespace_state.WhitespaceState, 19 | relative_indent: Optional[str], 20 | ) -> Token: ... 21 | type: token_type.TokenType 22 | string: str 23 | start_pos: Tuple[int, int] 24 | end_pos: Tuple[int, int] 25 | whitespace_before: whitespace_state.WhitespaceState 26 | whitespace_after: whitespace_state.WhitespaceState 27 | relative_indent: Optional[str] 28 | 29 | def tokenize(text: str) -> Iterator[Token]: ... 30 | -------------------------------------------------------------------------------- /stubs/libcst_native/whitespace_parser.pyi: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | from typing import Optional, Sequence, Union 7 | 8 | from libcst._nodes.whitespace import ( 9 | EmptyLine, 10 | Newline, 11 | ParenthesizedWhitespace, 12 | SimpleWhitespace, 13 | TrailingWhitespace, 14 | ) 15 | from libcst._parser.types.config import BaseWhitespaceParserConfig as Config 16 | from libcst._parser.types.whitespace_state import WhitespaceState as State 17 | 18 | def parse_simple_whitespace(config: Config, state: State) -> SimpleWhitespace: ... 19 | def parse_empty_lines( 20 | config: Config, 21 | state: State, 22 | *, 23 | override_absolute_indent: Optional[str] = None, 24 | ) -> Sequence[EmptyLine]: ... 25 | def parse_trailing_whitespace(config: Config, state: State) -> TrailingWhitespace: ... 26 | def parse_parenthesizable_whitespace( 27 | config: Config, state: State 28 | ) -> Union[SimpleWhitespace, ParenthesizedWhitespace]: ... 29 | -------------------------------------------------------------------------------- /stubs/libcst_native/whitespace_state.pyi: -------------------------------------------------------------------------------- 1 | # Copyright (c) Meta Platforms, Inc. and affiliates. 2 | # 3 | # This source code is licensed under the MIT license found in the 4 | # LICENSE file in the root directory of this source tree. 5 | 6 | class WhitespaceState: 7 | def __new__( 8 | cls, line: int, column: int, absolute_indent: str, is_parenthesized: bool 9 | ) -> WhitespaceState: ... 10 | line: int # one-indexed (to match parso's behavior) 11 | column: int # zero-indexed (to match parso's behavior) 12 | # What to look for when executing `_parse_indent`. 13 | absolute_indent: str 14 | is_parenthesized: bool 15 | -------------------------------------------------------------------------------- /stubs/setuptools.pyi: -------------------------------------------------------------------------------- 1 | # pyre-unsafe 2 | 3 | from typing import Any 4 | 5 | def __getattr__(name: str) -> Any: ... 6 | -------------------------------------------------------------------------------- /stubs/tokenize.pyi: -------------------------------------------------------------------------------- 1 | from token import ( 2 | AMPER, 3 | AMPEREQUAL, 4 | AT, 5 | ATEQUAL, 6 | CIRCUMFLEX, 7 | CIRCUMFLEXEQUAL, 8 | COLON, 9 | COLONEQUAL, 10 | COMMA, 11 | COMMENT, 12 | DEDENT, 13 | DOT, 14 | DOUBLESLASH, 15 | DOUBLESLASHEQUAL, 16 | DOUBLESTAR, 17 | DOUBLESTAREQUAL, 18 | ELLIPSIS, 19 | ENCODING, 20 | ENDMARKER, 21 | EQEQUAL, 22 | EQUAL, 23 | ERRORTOKEN, 24 | EXACT_TOKEN_TYPES, 25 | GREATER, 26 | GREATEREQUAL, 27 | INDENT, 28 | LBRACE, 29 | LEFTSHIFT, 30 | LEFTSHIFTEQUAL, 31 | LESS, 32 | LESSEQUAL, 33 | LPAR, 34 | LSQB, 35 | MINEQUAL, 36 | MINUS, 37 | N_TOKENS, 38 | NAME, 39 | NEWLINE, 40 | NL, 41 | NOTEQUAL, 42 | NT_OFFSET, 43 | NUMBER, 44 | OP, 45 | PERCENT, 46 | PERCENTEQUAL, 47 | PLUS, 48 | PLUSEQUAL, 49 | RARROW, 50 | RBRACE, 51 | RIGHTSHIFT, 52 | RIGHTSHIFTEQUAL, 53 | RPAR, 54 | RSQB, 55 | SEMI, 56 | SLASH, 57 | SLASHEQUAL, 58 | STAR, 59 | STAREQUAL, 60 | STRING, 61 | TILDE, 62 | TYPE_COMMENT, 63 | TYPE_IGNORE, 64 | VBAR, 65 | VBAREQUAL, 66 | ) 67 | from typing import Callable, Generator, Sequence, Tuple 68 | 69 | Hexnumber: str = ... 70 | Binnumber: str = ... 71 | Octnumber: str = ... 72 | Decnumber: str = ... 73 | Intnumber: str = ... 74 | Exponent: str = ... 75 | Pointfloat: str = ... 76 | Expfloat: str = ... 77 | Floatnumber: str = ... 78 | Imagnumber: str = ... 79 | Number: str = ... 80 | Whitespace: str = ... 81 | Comment: str = ... 82 | Ignore: str = ... 83 | Name: str = ... 84 | 85 | class TokenInfo(Tuple[int, str, Tuple[int, int], Tuple[int, int], int]): 86 | exact_type: int = ... 87 | type: int = ... 88 | string: str = ... 89 | start: Tuple[int, int] = ... 90 | end: Tuple[int, int] = ... 91 | line: int = ... 92 | def __repr__(self) -> str: ... 93 | 94 | def detect_encoding(readline: Callable[[], bytes]) -> Tuple[str, Sequence[bytes]]: ... 95 | def tokenize(readline: Callable[[], bytes]) -> Generator[TokenInfo, None, None]: ... 96 | -------------------------------------------------------------------------------- /stubs/typing_inspect.pyi: -------------------------------------------------------------------------------- 1 | # pyre-unsafe 2 | 3 | from typing import Any 4 | 5 | def __getattr__(name: str) -> Any: ... 6 | -------------------------------------------------------------------------------- /zizmor.yml: -------------------------------------------------------------------------------- 1 | rules: 2 | unpinned-uses: 3 | config: 4 | policies: 5 | "*": ref-pin --------------------------------------------------------------------------------