├── bindings ├── python │ ├── tree_sitter_elixir │ │ ├── py.typed │ │ ├── __init__.pyi │ │ ├── __init__.py │ │ └── binding.c │ └── tests │ │ └── test_binding.py ├── node │ ├── index.js │ ├── binding_test.js │ ├── index.d.ts │ └── binding.cc ├── c │ ├── tree-sitter-elixir.pc.in │ ├── tree-sitter.h.in │ └── tree-sitter.pc.in ├── swift │ ├── TreeSitterElixir │ │ └── elixir.h │ └── TreeSitterElixirTests │ │ └── TreeSitterElixirTests.swift ├── go │ ├── binding.go │ └── binding_test.go └── rust │ ├── build.rs │ └── lib.rs ├── .gitattributes ├── test ├── corpus │ ├── term │ │ ├── nil.txt │ │ ├── boolean.txt │ │ ├── char.txt │ │ ├── float.txt │ │ ├── integer.txt │ │ ├── tuple.txt │ │ ├── alias.txt │ │ ├── list.txt │ │ ├── atom.txt │ │ ├── map.txt │ │ ├── bitstring.txt │ │ ├── charlist.txt │ │ ├── keyword_list.txt │ │ ├── struct.txt │ │ └── string.txt │ ├── semicolon.txt │ ├── variable.txt │ ├── comment.txt │ ├── unicode.txt │ ├── edge_syntax.txt │ ├── expression │ │ ├── capture.txt │ │ ├── block.txt │ │ ├── sigil.txt │ │ └── anonymous_function.txt │ └── integration │ │ ├── kernel.txt │ │ ├── module_definition.txt │ │ ├── spec.txt │ │ └── function_definition.txt ├── highlight │ ├── identifiers.ex │ ├── operators.ex │ ├── anonymous.ex │ ├── calls.ex │ ├── literals.ex │ ├── kernel.ex │ ├── data_structures.ex │ └── module.ex └── tags │ ├── protocol.ex │ └── module.ex ├── go.mod ├── .clang-format ├── .gitignore ├── .github ├── ISSUE_TEMPLATE.md └── workflows │ ├── publish.yml │ ├── test.yml │ └── generate.yml ├── README.md ├── binding.gyp ├── scripts ├── integration_test.sh └── parse_repo.sh ├── .editorconfig ├── Cargo.toml ├── pyproject.toml ├── docs ├── index.md ├── highlighting.md └── parser.md ├── tree-sitter.json ├── src └── tree_sitter │ ├── alloc.h │ ├── parser.h │ └── array.h ├── Package.swift ├── package.json ├── queries ├── injections.scm ├── tags.scm └── highlights.scm ├── setup.py ├── CHANGELOG.md ├── CMakeLists.txt ├── NOTICE ├── Makefile ├── go.sum └── LICENSE /bindings/python/tree_sitter_elixir/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bindings/python/tree_sitter_elixir/__init__.pyi: -------------------------------------------------------------------------------- 1 | def language() -> int: ... 2 | -------------------------------------------------------------------------------- /bindings/python/tree_sitter_elixir/__init__.py: -------------------------------------------------------------------------------- 1 | "Elixir grammar for tree-sitter" 2 | 3 | from ._binding import language 4 | 5 | __all__ = ["language"] 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | /src/**/* linguist-generated=true 2 | /src/scanner.c linguist-generated=false 3 | # Exclude test files from language stats 4 | /test/**/*.ex linguist-documentation=true 5 | -------------------------------------------------------------------------------- /test/corpus/term/nil.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | simple literal 3 | ===================================== 4 | 5 | nil 6 | 7 | --- 8 | 9 | (source 10 | (nil)) 11 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tree-sitter/tree-sitter-elixir 2 | 3 | go 1.23 4 | 5 | require github.com/tree-sitter/go-tree-sitter v0.23.1 6 | 7 | require github.com/mattn/go-pointer v0.0.1 // indirect 8 | -------------------------------------------------------------------------------- /test/corpus/term/boolean.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | simple literal 3 | ===================================== 4 | 5 | true 6 | false 7 | 8 | --- 9 | 10 | (source 11 | (boolean) 12 | (boolean)) 13 | -------------------------------------------------------------------------------- /bindings/node/index.js: -------------------------------------------------------------------------------- 1 | const root = require("path").join(__dirname, "..", ".."); 2 | 3 | module.exports = require("node-gyp-build")(root); 4 | 5 | try { 6 | module.exports.nodeTypeInfo = require("../../src/node-types.json"); 7 | } catch (_) {} 8 | -------------------------------------------------------------------------------- /test/highlight/identifiers.ex: -------------------------------------------------------------------------------- 1 | abc_123 2 | # ^ variable 3 | 4 | _018OP 5 | # ^ comment.unused 6 | 7 | A__0 8 | # ^ module 9 | 10 | __MODULE__ ; __STACKTRACE__ 11 | # ^ constant.builtin 12 | # ^ constant.builtin 13 | 14 | __OTHER__ 15 | # ^ comment.unused 16 | -------------------------------------------------------------------------------- /bindings/node/binding_test.js: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | const assert = require("node:assert"); 4 | const { test } = require("node:test"); 5 | 6 | test("can load grammar", () => { 7 | const parser = new (require("tree-sitter"))(); 8 | assert.doesNotThrow(() => parser.setLanguage(require("."))); 9 | }); 10 | -------------------------------------------------------------------------------- /bindings/c/tree-sitter-elixir.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@PREFIX@ 2 | libdir=@LIBDIR@ 3 | includedir=@INCLUDEDIR@ 4 | 5 | Name: tree-sitter-elixir 6 | Description: Elixir grammar for tree-sitter 7 | URL: @URL@ 8 | Version: @VERSION@ 9 | Requires: @REQUIRES@ 10 | Libs: -L${libdir} @ADDITIONAL_LIBS@ -ltree-sitter-elixir 11 | Cflags: -I${includedir} 12 | -------------------------------------------------------------------------------- /bindings/swift/TreeSitterElixir/elixir.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ELIXIR_H_ 2 | #define TREE_SITTER_ELIXIR_H_ 3 | 4 | typedef struct TSLanguage TSLanguage; 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | extern TSLanguage *tree_sitter_elixir(); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | 16 | #endif // TREE_SITTER_ELIXIR_H_ 17 | -------------------------------------------------------------------------------- /bindings/go/binding.go: -------------------------------------------------------------------------------- 1 | package tree_sitter_elixir 2 | 3 | // #cgo CFLAGS: -std=c11 -fPIC 4 | // #include "../../src/parser.c" 5 | // #include "../../src/scanner.c" 6 | import "C" 7 | 8 | import "unsafe" 9 | 10 | // Get the tree-sitter Language for this grammar. 11 | func Language() unsafe.Pointer { 12 | return unsafe.Pointer(C.tree_sitter_elixir()) 13 | } 14 | -------------------------------------------------------------------------------- /bindings/c/tree-sitter.h.in: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_@UPPER_PARSERNAME@_H_ 2 | #define TREE_SITTER_@UPPER_PARSERNAME@_H_ 3 | 4 | #include 5 | 6 | #ifdef __cplusplus 7 | extern "C" { 8 | #endif 9 | 10 | extern TSLanguage *tree_sitter_@PARSERNAME@(); 11 | 12 | #ifdef __cplusplus 13 | } 14 | #endif 15 | 16 | #endif // TREE_SITTER_@UPPER_PARSERNAME@_H_ 17 | -------------------------------------------------------------------------------- /bindings/python/tests/test_binding.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | import tree_sitter, tree_sitter_elixir 4 | 5 | 6 | class TestLanguage(TestCase): 7 | def test_can_load_grammar(self): 8 | try: 9 | tree_sitter.Language(tree_sitter_elixir.language()) 10 | except Exception: 11 | self.fail("Error loading Elixir grammar") 12 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | AlignArrayOfStructures: Left 2 | BasedOnStyle: LLVM 3 | IndentCaseLabels: true 4 | IndentGotoLabels: true 5 | IndentPPDirectives: AfterHash 6 | IndentWidth: 2 7 | KeepEmptyLinesAtTheStartOfBlocks: false 8 | SeparateDefinitionBlocks: Always 9 | SortIncludes: CaseInsensitive 10 | SpaceAfterCStyleCast: false 11 | SpaceAfterLogicalNot: false 12 | SpaceBeforeCaseColon: false 13 | -------------------------------------------------------------------------------- /bindings/c/tree-sitter.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@PREFIX@ 2 | libdir=@LIBDIR@ 3 | includedir=@INCLUDEDIR@ 4 | additionallibs=@ADDITIONALLIBS@ 5 | 6 | Name: tree-sitter-@PARSERNAME@ 7 | Description: A tree-sitter grammar for the @PARSERNAME@ programming language. 8 | URL: @PARSERURL@ 9 | Version: @VERSION@ 10 | Libs: -L${libdir} ${additionallibs} -ltree-sitter-@PARSERNAME@ 11 | Cflags: -I${includedir} 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Deps 2 | /node_modules/ 3 | 4 | # Temporary files 5 | /tmp/ 6 | 7 | # Temporary files generated by Tree-sitter 8 | /build/ 9 | log.html 10 | tree-sitter-elixir.wasm 11 | 12 | # Files generated by Cargo 13 | /target/ 14 | Cargo.lock 15 | 16 | # C bindings/binaries 17 | /*.a 18 | /*.dylib 19 | /*.so* 20 | *.o 21 | /bindings/c/*.h 22 | /bindings/c/tree-sitter-*.pc 23 | 24 | # Files generated by Swift 25 | /.build/ 26 | -------------------------------------------------------------------------------- /bindings/go/binding_test.go: -------------------------------------------------------------------------------- 1 | package tree_sitter_elixir_test 2 | 3 | import ( 4 | "testing" 5 | 6 | tree_sitter "github.com/tree-sitter/go-tree-sitter" 7 | tree_sitter_elixir "github.com/tree-sitter/tree-sitter-elixir/bindings/go" 8 | ) 9 | 10 | func TestCanLoadGrammar(t *testing.T) { 11 | language := tree_sitter.NewLanguage(tree_sitter_elixir.Language()) 12 | if language == nil { 13 | t.Errorf("Error loading Elixir grammar") 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 11 | -------------------------------------------------------------------------------- /bindings/swift/TreeSitterElixirTests/TreeSitterElixirTests.swift: -------------------------------------------------------------------------------- 1 | import XCTest 2 | import SwiftTreeSitter 3 | import TreeSitterElixir 4 | 5 | final class TreeSitterElixirTests: XCTestCase { 6 | func testCanLoadGrammar() throws { 7 | let parser = Parser() 8 | let language = Language(language: tree_sitter_elixir()) 9 | XCTAssertNoThrow(try parser.setLanguage(language), 10 | "Error loading Elixir grammar") 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # tree-sitter-elixir 2 | 3 | [![Test](https://github.com/elixir-lang/tree-sitter-elixir/actions/workflows/test.yml/badge.svg)](https://github.com/elixir-lang/tree-sitter-elixir/actions/workflows/test.yml) 4 | 5 | Elixir grammar for [tree-sitter](https://github.com/tree-sitter/tree-sitter). 6 | 7 | Ready for production. Currently used by GitHub itself for source code highlighting and code navigation. 8 | 9 | ## Development 10 | 11 | See [the docs](./docs/index.md) for more details. 12 | -------------------------------------------------------------------------------- /binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | "targets": [ 3 | { 4 | "target_name": "tree_sitter_elixir_binding", 5 | "dependencies": [ 6 | " 2 | 3 | typedef struct TSLanguage TSLanguage; 4 | 5 | extern "C" TSLanguage *tree_sitter_elixir(); 6 | 7 | // "tree-sitter", "language" hashed with BLAKE2 8 | const napi_type_tag LANGUAGE_TYPE_TAG = { 9 | 0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16 10 | }; 11 | 12 | Napi::Object Init(Napi::Env env, Napi::Object exports) { 13 | exports["name"] = Napi::String::New(env, "elixir"); 14 | auto language = Napi::External::New(env, tree_sitter_elixir()); 15 | language.TypeTag(&LANGUAGE_TYPE_TAG); 16 | exports["language"] = language; 17 | return exports; 18 | } 19 | 20 | NODE_API_MODULE(tree_sitter_elixir_binding, Init) 21 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | trim_trailing_whitespace = true 8 | 9 | [*.{json,toml,yml,gyp}] 10 | indent_style = space 11 | indent_size = 2 12 | 13 | [*.js] 14 | indent_style = space 15 | indent_size = 2 16 | 17 | [*.rs] 18 | indent_style = space 19 | indent_size = 4 20 | 21 | [*.{c,cc,h}] 22 | indent_style = space 23 | indent_size = 4 24 | 25 | [*.{py,pyi}] 26 | indent_style = space 27 | indent_size = 4 28 | 29 | [*.swift] 30 | indent_style = space 31 | indent_size = 4 32 | 33 | [*.go] 34 | indent_style = tab 35 | indent_size = 8 36 | 37 | [Makefile] 38 | indent_style = tab 39 | indent_size = 8 40 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "tree-sitter-elixir" 3 | description = "Elixir grammar for the tree-sitter parsing library" 4 | version = "0.3.4" 5 | keywords = ["incremental", "parsing", "elixir"] 6 | categories = ["parsing", "text-editors"] 7 | repository = "https://github.com/elixir-lang/tree-sitter-elixir" 8 | edition = "2018" 9 | license = "Apache-2.0" 10 | 11 | build = "bindings/rust/build.rs" 12 | include = [ 13 | "LICENSE", 14 | "bindings/rust/*", 15 | "grammar.js", 16 | "queries/*", 17 | "src/*", 18 | ] 19 | 20 | [lib] 21 | path = "bindings/rust/lib.rs" 22 | 23 | [dependencies] 24 | tree-sitter-language = "0.1.0" 25 | 26 | [dev-dependencies] 27 | tree-sitter = "0.23.0" 28 | 29 | [build-dependencies] 30 | cc = "1.0" 31 | -------------------------------------------------------------------------------- /scripts/parse_repo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cd "$(dirname "$0")/.." 6 | 7 | print_usage_and_exit() { 8 | echo "Usage: $0 " 9 | echo "" 10 | echo "Clones the given repository and runs the parser against all Elixir files" 11 | echo "" 12 | echo "## Examples" 13 | echo "" 14 | echo " $0 elixir-lang/elixir" 15 | echo "" 16 | exit 1 17 | } 18 | 19 | if [ $# -ne 1 ]; then 20 | print_usage_and_exit 21 | fi 22 | 23 | gh_repo="$1" 24 | 25 | dir="tmp/gh/${gh_repo//[\/-]/_}" 26 | 27 | if [[ ! -d "$dir" ]]; then 28 | mkdir -p "$(dirname "$dir")" 29 | git clone --depth 1 "https://github.com/$gh_repo.git" "$dir" 30 | fi 31 | 32 | echo "Running parser against $gh_repo" 33 | 34 | npx tree-sitter parse --quiet --stat "$dir/**/*.ex*" 35 | -------------------------------------------------------------------------------- /test/corpus/term/char.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | regular character 3 | ===================================== 4 | 5 | ?a 6 | ?Z 7 | ?0 8 | ?9 9 | ?_ 10 | ?? 11 | 12 | --- 13 | 14 | (source 15 | (char) 16 | (char) 17 | (char) 18 | (char) 19 | (char) 20 | (char)) 21 | 22 | ===================================== 23 | escaped character 24 | ===================================== 25 | 26 | ?\n 27 | ?\t 28 | ?\s 29 | ?\\ 30 | ?\a 31 | ?\b 32 | 33 | --- 34 | 35 | (source 36 | (char) 37 | (char) 38 | (char) 39 | (char) 40 | (char) 41 | (char)) 42 | 43 | ===================================== 44 | list of char literals 45 | ===================================== 46 | 47 | [?a, ?b, ?c] 48 | 49 | --- 50 | 51 | (source 52 | (list 53 | (char) 54 | (char) 55 | (char))) 56 | -------------------------------------------------------------------------------- /bindings/rust/build.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | let src_dir = std::path::Path::new("src"); 3 | 4 | let mut c_config = cc::Build::new(); 5 | c_config.include(src_dir); 6 | c_config 7 | .flag_if_supported("-Wno-unused-parameter") 8 | .flag_if_supported("-Wno-unused-but-set-variable") 9 | .flag_if_supported("-Wno-trigraphs"); 10 | #[cfg(target_env = "msvc")] 11 | c_config.flag("-utf-8"); 12 | 13 | let parser_path = src_dir.join("parser.c"); 14 | c_config.file(&parser_path); 15 | println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap()); 16 | 17 | let scanner_path = src_dir.join("scanner.c"); 18 | c_config.file(&scanner_path); 19 | println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap()); 20 | 21 | c_config.compile("parser-scanner"); 22 | } 23 | -------------------------------------------------------------------------------- /bindings/python/tree_sitter_elixir/binding.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | typedef struct TSLanguage TSLanguage; 4 | 5 | TSLanguage *tree_sitter_elixir(void); 6 | 7 | static PyObject* _binding_language(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) { 8 | return PyCapsule_New(tree_sitter_elixir(), "tree_sitter.Language", NULL); 9 | } 10 | 11 | static PyMethodDef methods[] = { 12 | {"language", _binding_language, METH_NOARGS, 13 | "Get the tree-sitter language for this grammar."}, 14 | {NULL, NULL, 0, NULL} 15 | }; 16 | 17 | static struct PyModuleDef module = { 18 | .m_base = PyModuleDef_HEAD_INIT, 19 | .m_name = "_binding", 20 | .m_doc = NULL, 21 | .m_size = -1, 22 | .m_methods = methods 23 | }; 24 | 25 | PyMODINIT_FUNC PyInit__binding(void) { 26 | return PyModule_Create(&module); 27 | } 28 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish packages 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*.*.*" 7 | 8 | permissions: 9 | contents: write 10 | id-token: write 11 | attestations: write 12 | 13 | jobs: 14 | github: 15 | uses: tree-sitter/workflows/.github/workflows/release.yml@main 16 | with: 17 | attestations: true 18 | npm: 19 | uses: tree-sitter/workflows/.github/workflows/package-npm.yml@main 20 | secrets: 21 | NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN}} 22 | crates: 23 | uses: tree-sitter/workflows/.github/workflows/package-crates.yml@main 24 | secrets: 25 | CARGO_REGISTRY_TOKEN: ${{secrets.CARGO_REGISTRY_TOKEN}} 26 | pypi: 27 | uses: tree-sitter/workflows/.github/workflows/package-pypi.yml@main 28 | secrets: 29 | PYPI_API_TOKEN: ${{secrets.PYPI_API_TOKEN}} 30 | -------------------------------------------------------------------------------- /test/tags/module.ex: -------------------------------------------------------------------------------- 1 | defmodule Foo.Bar.Baz do 2 | # ^ definition.module 3 | # ^ definition.module 4 | # ^ definition.module 5 | 6 | def init(arg) do 7 | # ^ definition.function 8 | state = 9 | arg 10 | |> map(&(&1 * 2)) 11 | # ^ reference.call 12 | |> map(&(&1 + 1)) 13 | # ^ reference.call 14 | 15 | {:ok, arg} 16 | end 17 | 18 | def map(list, fun, acc \\ []) 19 | # ^ definition.function 20 | 21 | def map([head | rest], fun, acc) do 22 | # ^ definition.function 23 | map(rest, fun, [fun.(head) | acc]) 24 | # <- reference.call 25 | end 26 | 27 | def map([], _fun, acc), do: Enum.reverse(acc) 28 | # ^ definition.function 29 | # ^ reference.module 30 | # ^ reference.call 31 | end 32 | -------------------------------------------------------------------------------- /test/corpus/term/float.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | simple literal 3 | ===================================== 4 | 5 | 1234567890.1234567890 6 | -1234567890.1234567890 7 | -1_234_567_890.123_456_789_0 8 | 9 | --- 10 | 11 | (source 12 | (float) 13 | (unary_operator 14 | (float)) 15 | (unary_operator 16 | (float))) 17 | 18 | ===================================== 19 | scientific notation 20 | ===================================== 21 | 22 | 1.0e6 23 | 1.0e+6 24 | 1.0e-6 25 | -1.0e6 26 | -1.0e+6 27 | -1.0e-6 28 | 1.0E6 29 | 1.0E+6 30 | 1.0E-6 31 | 1_234_567_890.123_456_789_0e1_234_567_890 32 | 33 | --- 34 | 35 | (source 36 | (float) 37 | (float) 38 | (float) 39 | (unary_operator 40 | (float)) 41 | (unary_operator 42 | (float)) 43 | (unary_operator 44 | (float)) 45 | (float) 46 | (float) 47 | (float) 48 | (float)) 49 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=42", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "tree-sitter-elixir" 7 | description = "Elixir grammar for tree-sitter" 8 | version = "0.3.4" 9 | keywords = ["incremental", "parsing", "tree-sitter", "elixir"] 10 | classifiers = [ 11 | "Intended Audience :: Developers", 12 | "License :: OSI Approved :: MIT License", 13 | "Topic :: Software Development :: Compilers", 14 | "Topic :: Text Processing :: Linguistic", 15 | "Typing :: Typed" 16 | ] 17 | requires-python = ">=3.9" 18 | license.text = "Apache-2.0" 19 | readme = "README.md" 20 | 21 | [project.urls] 22 | Homepage = "https://github.com/elixir-lang/tree-sitter-elixir" 23 | 24 | [project.optional-dependencies] 25 | core = ["tree-sitter~=0.23"] 26 | 27 | [tool.cibuildwheel] 28 | build = "cp39-*" 29 | build-frontend = "build" 30 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | # Development notes 2 | 3 | This documentation covers rationale behind some of the design and implementation decisions, 4 | as well as basic Tree-sitter tips that are relevant. 5 | 6 | ## Pages 7 | 8 | * [Parser](./parser.md) 9 | * [Highlighting](./highlighting.md) 10 | 11 | ## Acknowledgements 12 | 13 | While this parser is written from scratch, there were previous efforts that made 14 | for a helpful reference: 15 | 16 | * [tree-sitter-elixir](https://github.com/ananthakumaran/tree-sitter-elixir) developed 17 | by [@ananthakumaran](https://github.com/ananthakumaran) 18 | * [tree-sitter-elixir](https://github.com/wingyplus/tree-sitter-elixir) developed 19 | by [@wingyplus](https://github.com/wingyplus) and [@Tuxified](https://github.com/Tuxified) 20 | 21 | In particular, some test cases were sourced from [ananthakumaran/tree-sitter-elixir](https://github.com/ananthakumaran/tree-sitter-elixir). 22 | -------------------------------------------------------------------------------- /test/corpus/variable.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | regular 3 | ===================================== 4 | 5 | snake_case 6 | camelCase 7 | az_AZ_19 8 | bang! 9 | question? 10 | doctest 11 | not1 12 | notfalse 13 | 14 | --- 15 | 16 | (source 17 | (identifier) 18 | (identifier) 19 | (identifier) 20 | (identifier) 21 | (identifier) 22 | (identifier) 23 | (identifier) 24 | (identifier)) 25 | 26 | ===================================== 27 | unused 28 | ===================================== 29 | 30 | _ 31 | _number 32 | __TEST__ 33 | 34 | --- 35 | 36 | (source 37 | (identifier) 38 | (identifier) 39 | (identifier)) 40 | 41 | ===================================== 42 | three dots identifier 43 | ===================================== 44 | 45 | ... 46 | 47 | --- 48 | 49 | (source 50 | (identifier)) 51 | 52 | ===================================== 53 | special identifier 54 | ===================================== 55 | 56 | __MODULE__ 57 | __DIR__ 58 | 59 | --- 60 | 61 | (source 62 | (identifier) 63 | (identifier)) 64 | -------------------------------------------------------------------------------- /tree-sitter.json: -------------------------------------------------------------------------------- 1 | { 2 | "grammars": [ 3 | { 4 | "name": "elixir", 5 | "camelcase": "Elixir", 6 | "scope": "source.elixir", 7 | "path": ".", 8 | "file-types": ["ex", "exs"], 9 | "highlights": "queries/highlights.scm", 10 | "tags": "queries/tags.scm", 11 | "injection-regex": "^(ex|elixir)$" 12 | } 13 | ], 14 | "metadata": { 15 | "version": "0.3.4", 16 | "license": "Apache-2.0", 17 | "description": "Elixir grammar for the tree-sitter parsing library", 18 | "authors": [ 19 | { 20 | "name": "Jonatan Kłosko", 21 | "email": "jonatanklosko@gmail.com" 22 | }, 23 | { 24 | "name": "Michael Davis", 25 | "email": "mcarsondavis@gmail.com" 26 | } 27 | ], 28 | "links": { 29 | "repository": "https://github.com/elixir-lang/tree-sitter-elixir" 30 | } 31 | }, 32 | "bindings": { 33 | "c": true, 34 | "go": true, 35 | "node": true, 36 | "python": true, 37 | "rust": true, 38 | "swift": true 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /test/corpus/term/integer.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | decimal 3 | ===================================== 4 | 5 | 1234567890 6 | -1234567890 7 | 1_234_567_890 8 | 019 9 | 10 | --- 11 | 12 | (source 13 | (integer) 14 | (unary_operator 15 | (integer)) 16 | (integer) 17 | (integer)) 18 | 19 | ===================================== 20 | binary 21 | ===================================== 22 | 23 | 0b0101011 24 | -0b0101011 25 | 0b0_10_10_11 26 | 27 | --- 28 | 29 | (source 30 | (integer) 31 | (unary_operator 32 | (integer)) 33 | (integer)) 34 | 35 | ===================================== 36 | octal 37 | ===================================== 38 | 39 | 0o1234567 40 | -0o1234567 41 | 0o1_23_45_67 42 | 43 | --- 44 | 45 | (source 46 | (integer) 47 | (unary_operator 48 | (integer)) 49 | (integer)) 50 | 51 | ===================================== 52 | hexadecimal 53 | ===================================== 54 | 55 | 0x123456789abcdefABCDEF 56 | -0x123456789abcdefABCDEF 57 | 0x123456789_abcdef_ABCDEF 58 | 59 | --- 60 | 61 | (source 62 | (integer) 63 | (unary_operator 64 | (integer)) 65 | (integer)) 66 | -------------------------------------------------------------------------------- /src/tree_sitter/alloc.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ALLOC_H_ 2 | #define TREE_SITTER_ALLOC_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | // Allow clients to override allocation functions 13 | #ifdef TREE_SITTER_REUSE_ALLOCATOR 14 | 15 | extern void *(*ts_current_malloc)(size_t size); 16 | extern void *(*ts_current_calloc)(size_t count, size_t size); 17 | extern void *(*ts_current_realloc)(void *ptr, size_t size); 18 | extern void (*ts_current_free)(void *ptr); 19 | 20 | #ifndef ts_malloc 21 | #define ts_malloc ts_current_malloc 22 | #endif 23 | #ifndef ts_calloc 24 | #define ts_calloc ts_current_calloc 25 | #endif 26 | #ifndef ts_realloc 27 | #define ts_realloc ts_current_realloc 28 | #endif 29 | #ifndef ts_free 30 | #define ts_free ts_current_free 31 | #endif 32 | 33 | #else 34 | 35 | #ifndef ts_malloc 36 | #define ts_malloc malloc 37 | #endif 38 | #ifndef ts_calloc 39 | #define ts_calloc calloc 40 | #endif 41 | #ifndef ts_realloc 42 | #define ts_realloc realloc 43 | #endif 44 | #ifndef ts_free 45 | #define ts_free free 46 | #endif 47 | 48 | #endif 49 | 50 | #ifdef __cplusplus 51 | } 52 | #endif 53 | 54 | #endif // TREE_SITTER_ALLOC_H_ 55 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | on: 3 | pull_request: 4 | push: 5 | branches: 6 | - main 7 | schedule: 8 | - cron: "0 0 * * *" 9 | jobs: 10 | main: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v4 14 | # Workaround for https://github.com/nodejs/node-gyp/issues/2219#issuecomment-1359162118 15 | - uses: actions/setup-python@v5 16 | with: 17 | python-version: "3.10" 18 | - name: Install Node 19 | uses: actions/setup-node@v2 20 | with: 21 | node-version: "14.x" 22 | - name: Cache npm dependencies 23 | uses: actions/cache@v4 24 | with: 25 | path: ~/.npm 26 | key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} 27 | restore-keys: | 28 | ${{ runner.os }}-node- 29 | - name: Install npm dependencies 30 | run: npm ci 31 | - name: Check formatting 32 | run: npm run format-check 33 | # Ensure the generated parser is up to date 34 | - run: npx tree-sitter generate 35 | - name: Run Tree-sitter tests 36 | run: npx tree-sitter test 37 | - name: Run integration tests 38 | run: scripts/integration_test.sh 39 | -------------------------------------------------------------------------------- /test/corpus/term/tuple.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | simple literal 3 | ===================================== 4 | 5 | {} 6 | {1} 7 | {1, 2} 8 | {1,2} 9 | { 1 , 2 } 10 | 11 | --- 12 | 13 | (source 14 | (tuple) 15 | (tuple 16 | (integer)) 17 | (tuple 18 | (integer) 19 | (integer)) 20 | (tuple 21 | (integer) 22 | (integer)) 23 | (tuple 24 | (integer) 25 | (integer))) 26 | 27 | ===================================== 28 | nested 29 | ===================================== 30 | 31 | {{1}, 1} 32 | 33 | --- 34 | 35 | (source 36 | (tuple 37 | (tuple 38 | (integer)) 39 | (integer))) 40 | 41 | ===================================== 42 | trailing separator 43 | ===================================== 44 | 45 | {1,} 46 | {1,2,} 47 | 48 | --- 49 | 50 | (source 51 | (tuple 52 | (integer)) 53 | (tuple 54 | (integer) 55 | (integer))) 56 | 57 | ===================================== 58 | [error] missing element 59 | ===================================== 60 | 61 | {, 1} 62 | 63 | --- 64 | 65 | (source 66 | (tuple 67 | (ERROR) 68 | (integer))) 69 | 70 | ===================================== 71 | [error] missing separator 72 | ===================================== 73 | 74 | {1 2} 75 | 76 | --- 77 | 78 | (source 79 | (tuple 80 | (integer) 81 | (ERROR 82 | (integer)))) 83 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.3 2 | 3 | import PackageDescription 4 | 5 | let package = Package( 6 | name: "TreeSitterElixir", 7 | platforms: [.macOS(.v10_13), .iOS(.v11)], 8 | products: [ 9 | .library(name: "TreeSitterElixir", targets: ["TreeSitterElixir"]), 10 | ], 11 | dependencies: [], 12 | targets: [ 13 | .target(name: "TreeSitterElixir", 14 | path: ".", 15 | exclude: [ 16 | "binding.gyp", 17 | "bindings", 18 | "Cargo.toml", 19 | "docs", 20 | "grammar.js", 21 | "LICENSE", 22 | "Makefile", 23 | "NOTICE", 24 | "package.json", 25 | "README.md", 26 | "scripts", 27 | "src/grammar.json", 28 | "src/node-types.json", 29 | "test", 30 | ], 31 | sources: [ 32 | "src/parser.c", 33 | "src/scanner.c", 34 | ], 35 | resources: [ 36 | .copy("queries") 37 | ], 38 | publicHeadersPath: "bindings/swift", 39 | cSettings: [.headerSearchPath("src")]) 40 | ] 41 | ) 42 | -------------------------------------------------------------------------------- /test/corpus/term/alias.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | single part 3 | ===================================== 4 | 5 | Mod 6 | AZ_az_19_ 7 | 8 | --- 9 | 10 | (source 11 | (alias) 12 | (alias)) 13 | 14 | ===================================== 15 | multiple parts 16 | ===================================== 17 | 18 | Mod.Child 19 | Mod.Child.Child 20 | 21 | --- 22 | 23 | (source 24 | (alias) 25 | (alias)) 26 | 27 | ===================================== 28 | spacing 29 | ===================================== 30 | 31 | Mod . Child 32 | 33 | Mod 34 | . 35 | Child 36 | 37 | --- 38 | 39 | (source 40 | (alias) 41 | (alias)) 42 | 43 | ===================================== 44 | qualified tuples 45 | ===================================== 46 | 47 | Mod.{Child1, Child2} 48 | 49 | --- 50 | 51 | (source 52 | (dot 53 | (alias) 54 | (tuple 55 | (alias) 56 | (alias)))) 57 | 58 | ===================================== 59 | dot on identifier 60 | ===================================== 61 | 62 | name.Mod 63 | name.Mod.Child 64 | 65 | --- 66 | 67 | (source 68 | (dot 69 | (identifier) 70 | (alias)) 71 | (dot 72 | (identifier) 73 | (alias))) 74 | 75 | ===================================== 76 | dot on special identifier 77 | ===================================== 78 | 79 | __MODULE__.Child 80 | 81 | (source 82 | (dot 83 | (identifier) 84 | (alias))) 85 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tree-sitter-elixir", 3 | "version": "0.3.4", 4 | "description": "Elixir grammar for the tree-sitter parsing library", 5 | "main": "bindings/node", 6 | "types": "bindings/node", 7 | "keywords": [ 8 | "parser", 9 | "lexer", 10 | "elixir", 11 | "tree-sitter" 12 | ], 13 | "files": [ 14 | "grammar.js", 15 | "binding.gyp", 16 | "prebuilds/**", 17 | "bindings/node/*", 18 | "queries/*", 19 | "src/**" 20 | ], 21 | "license": "Apache-2.0", 22 | "repository": { 23 | "type": "git", 24 | "url": "https://github.com/elixir-lang/tree-sitter-elixir.git" 25 | }, 26 | "scripts": { 27 | "build": "tree-sitter generate", 28 | "test": "tree-sitter test", 29 | "format": "prettier --write grammar.js && clang-format -i src/scanner.c", 30 | "format-check": "prettier --check grammar.js && cat src/scanner.c | clang-format src/scanner.c | diff src/scanner.c -", 31 | "install": "node-gyp-build", 32 | "prestart": "tree-sitter build --wasm", 33 | "start": "tree-sitter playground" 34 | }, 35 | "dependencies": { 36 | "node-addon-api": "^7.1.0", 37 | "node-gyp-build": "^4.8.0" 38 | }, 39 | "devDependencies": { 40 | "clang-format": "^1.8.0", 41 | "prettier": "^3.4.2", 42 | "tree-sitter-cli": "^0.24.0", 43 | "prebuildify": "^6.0.0" 44 | }, 45 | "peerDependencies": { 46 | "tree-sitter": "^0.21.0" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /test/corpus/term/list.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | simple literal 3 | ===================================== 4 | 5 | [] 6 | [a] 7 | [A] 8 | [1] 9 | [1, 2] 10 | [1,2] 11 | [ 1 , 2 ] 12 | 13 | --- 14 | 15 | (source 16 | (list) 17 | (list 18 | (identifier)) 19 | (list 20 | (alias)) 21 | (list 22 | (integer)) 23 | (list 24 | (integer) 25 | (integer)) 26 | (list 27 | (integer) 28 | (integer)) 29 | (list 30 | (integer) 31 | (integer))) 32 | 33 | ===================================== 34 | nested 35 | ===================================== 36 | 37 | [[1], 1] 38 | 39 | --- 40 | 41 | (source 42 | (list 43 | (list 44 | (integer)) 45 | (integer))) 46 | 47 | ===================================== 48 | trailing separator 49 | ===================================== 50 | 51 | [1,] 52 | [1,2,] 53 | 54 | --- 55 | 56 | (source 57 | (list 58 | (integer)) 59 | (list 60 | (integer) 61 | (integer))) 62 | 63 | ===================================== 64 | [error] missing element 65 | ===================================== 66 | 67 | [, 1] 68 | 69 | --- 70 | 71 | (source 72 | (list 73 | (ERROR) 74 | (integer))) 75 | 76 | ===================================== 77 | [error] missing separator 78 | ===================================== 79 | 80 | [1 2] 81 | 82 | --- 83 | 84 | (source 85 | (list 86 | (integer) 87 | (ERROR 88 | (integer)))) 89 | -------------------------------------------------------------------------------- /queries/injections.scm: -------------------------------------------------------------------------------- 1 | ; Phoenix HTML template 2 | ((sigil 3 | (sigil_name) @_sigil_name 4 | (quoted_content) @injection.content) 5 | (#any-of? @_sigil_name "H" "LVN" "HOLO") 6 | (#set! injection.language "heex") 7 | (#set! injection.combined)) 8 | 9 | ; Regex 10 | ((sigil 11 | (sigil_name) @_sigil_name 12 | (quoted_content) @injection.content) 13 | (#any-of? @_sigil_name "r" "R") 14 | (#set! injection.language "regex") 15 | (#set! injection.combined)) 16 | 17 | ; SQL injection 18 | ((sigil 19 | (sigil_name) @_sigil_name 20 | (quoted_content) @injection.content) 21 | (#eq? @_sigil_name "SQL") 22 | (#set! injection.language "sql") 23 | (#set! injection.combined)) 24 | 25 | ; Markdown 26 | ((sigil 27 | (sigil_name) @_sigil_name 28 | (quoted_content) @injection.content) 29 | (#eq? @_sigil_name "MD") 30 | (#set! injection.language "markdown") 31 | (#set! injection.combined)) 32 | 33 | ; Python 34 | ((sigil 35 | (sigil_name) @_sigil_name 36 | (quoted_content) @injection.content) 37 | (#eq? @_sigil_name "PY") 38 | (#set! injection.language "python") 39 | (#set! injection.combined)) 40 | 41 | ; JavaScript 42 | ((sigil 43 | (sigil_name) @_sigil_name 44 | (quoted_content) @injection.content) 45 | (#eq? @_sigil_name "JS") 46 | (#set! injection.language "javascript") 47 | (#set! injection.combined)) 48 | 49 | ; Vue 50 | ((sigil 51 | (sigil_name) @_sigil_name 52 | (quoted_content) @injection.content) 53 | (#eq? @_sigil_name "VUE") 54 | (#set! injection.language "vue") 55 | (#set! injection.combined)) 56 | -------------------------------------------------------------------------------- /.github/workflows/generate.yml: -------------------------------------------------------------------------------- 1 | name: Generate 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | main: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: actions/checkout@v4 11 | # Workaround for https://github.com/nodejs/node-gyp/issues/2219#issuecomment-1359162118 12 | - uses: actions/setup-python@v5 13 | with: 14 | python-version: "3.10" 15 | - name: Install Node 16 | uses: actions/setup-node@v2 17 | with: 18 | node-version: "14.x" 19 | - name: Cache npm dependencies 20 | uses: actions/cache@v4 21 | with: 22 | path: ~/.npm 23 | key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }} 24 | restore-keys: | 25 | ${{ runner.os }}-node- 26 | - name: Install npm dependencies 27 | run: npm ci 28 | - name: Generate parser 29 | run: | 30 | npx tree-sitter generate 31 | npx tree-sitter build --wasm -o tree-sitter-elixir.wasm 32 | - name: Update parser files 33 | uses: stefanzweifel/git-auto-commit-action@v4 34 | with: 35 | commit_message: Generate parser 36 | file_pattern: src 37 | - name: Checkout gh-pages branch to ./gh-pages 38 | uses: actions/checkout@v4 39 | with: 40 | ref: gh-pages 41 | path: ./gh-pages 42 | - run: mv *.wasm ./gh-pages 43 | - name: Update WASM file on gh-pages branch 44 | uses: stefanzweifel/git-auto-commit-action@v4 45 | with: 46 | commit_message: Generate WASM 47 | file_pattern: "*.wasm" 48 | repository: ./gh-pages 49 | -------------------------------------------------------------------------------- /test/corpus/comment.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | empty 3 | ===================================== 4 | 5 | # 6 | 7 | --- 8 | 9 | (source 10 | (comment)) 11 | 12 | ===================================== 13 | single line 14 | ===================================== 15 | 16 | # single comment 17 | 18 | --- 19 | 20 | (source 21 | (comment)) 22 | 23 | ===================================== 24 | multiple start symbols 25 | ===================================== 26 | 27 | ### multiple "#" 28 | 29 | --- 30 | 31 | (source 32 | (comment)) 33 | 34 | ===================================== 35 | many consecutive lines 36 | ===================================== 37 | 38 | # many 39 | # consecutive 40 | 1 41 | # lines 42 | 43 | --- 44 | 45 | (source 46 | (comment) 47 | (comment) 48 | (integer) 49 | (comment)) 50 | 51 | ===================================== 52 | in the same line as regular code 53 | ===================================== 54 | 55 | 1 # comment 56 | 57 | --- 58 | 59 | (source 60 | (integer) 61 | (comment)) 62 | 63 | ===================================== 64 | matches inside a nested structure 65 | ===================================== 66 | 67 | [ 1 ## inside a list 68 | , { 2 # and a tuple, too! 69 | , 3 } 70 | ] 71 | 72 | --- 73 | 74 | (source 75 | (list 76 | (integer) 77 | (comment) 78 | (tuple 79 | (integer) 80 | (comment) 81 | (integer)))) 82 | 83 | ===================================== 84 | does not match inside a string 85 | ===================================== 86 | 87 | "# string" 88 | "this is #{interpolation}" 89 | 90 | --- 91 | 92 | (source 93 | (string 94 | (quoted_content)) 95 | (string 96 | (quoted_content) 97 | (interpolation (identifier)))) 98 | -------------------------------------------------------------------------------- /queries/tags.scm: -------------------------------------------------------------------------------- 1 | ; Definitions 2 | 3 | ; * modules and protocols 4 | (call 5 | target: (identifier) @ignore 6 | (arguments (alias) @name) 7 | (#any-of? @ignore "defmodule" "defprotocol")) @definition.module 8 | 9 | ; * functions/macros 10 | (call 11 | target: (identifier) @ignore 12 | (arguments 13 | [ 14 | ; zero-arity functions with no parentheses 15 | (identifier) @name 16 | ; regular function clause 17 | (call target: (identifier) @name) 18 | ; function clause with a guard clause 19 | (binary_operator 20 | left: (call target: (identifier) @name) 21 | operator: "when") 22 | ]) 23 | (#any-of? @ignore "def" "defp" "defdelegate" "defguard" "defguardp" "defmacro" "defmacrop" "defn" "defnp")) @definition.function 24 | 25 | ; References 26 | 27 | ; ignore calls to kernel/special-forms keywords 28 | (call 29 | target: (identifier) @ignore 30 | (#any-of? @ignore "def" "defp" "defdelegate" "defguard" "defguardp" "defmacro" "defmacrop" "defn" "defnp" "defmodule" "defprotocol" "defimpl" "defstruct" "defexception" "defoverridable" "alias" "case" "cond" "else" "for" "if" "import" "quote" "raise" "receive" "require" "reraise" "super" "throw" "try" "unless" "unquote" "unquote_splicing" "use" "with")) 31 | 32 | ; ignore module attributes 33 | (unary_operator 34 | operator: "@" 35 | operand: (call 36 | target: (identifier) @ignore)) 37 | 38 | ; * function call 39 | (call 40 | target: [ 41 | ; local 42 | (identifier) @name 43 | ; remote 44 | (dot 45 | right: (identifier) @name) 46 | ]) @reference.call 47 | 48 | ; * pipe into function call 49 | (binary_operator 50 | operator: "|>" 51 | right: (identifier) @name) @reference.call 52 | 53 | ; * modules 54 | (alias) @name @reference.module 55 | -------------------------------------------------------------------------------- /docs/highlighting.md: -------------------------------------------------------------------------------- 1 | # Syntax highlighting 2 | 3 | For detailed introduction see the official guide on [Syntax highlighting](https://tree-sitter.github.io/tree-sitter/syntax-highlighting). 4 | 5 | Briefly speaking, Tree-sitter uses the rules in `queries/highlights.scm` to annotate nodes 6 | with specific tokens, then it maps those tokens to formatting style according to user-defined 7 | theme. 8 | 9 | To test highlighting using the CLI, you need to create local configuration. 10 | 11 | ```shell 12 | # Create the config file 13 | npx tree-sitter init-config 14 | ``` 15 | 16 | The above command should print out the config location, so that you can further configure it. 17 | Open the file and modify `"parser-directories"` to include the parent directory of `tree-sitter-elixir`. 18 | Also, you can optionally customize the theme, here's a tiny subset of the One Dark theme: 19 | 20 | ```json 21 | { 22 | "number": { 23 | "color": "#61afef", 24 | "bold": true 25 | }, 26 | "string": "#98c379", 27 | "string.escape": "#56b6c2", 28 | "string.special": "#61afef", 29 | "string.regexp": "#e06c75", 30 | "type": "#e06c75", 31 | "comment": { 32 | "color": "#5c6370", 33 | "italic": true 34 | }, 35 | "punctuation": "#abb2bf", 36 | "punctuation.special": "#be5046", 37 | "operator": { 38 | "color": "#d19a66", 39 | "bold": true 40 | }, 41 | "variable": "#abb2bf", 42 | "function": "#61afef", 43 | "constant": "#61afef", 44 | "constant.builtin": { 45 | "color": "#e06c75", 46 | "bold": true 47 | }, 48 | "keyword": "#c678dd", 49 | "attribute": "#e06c75", 50 | "embedded": null 51 | } 52 | ``` 53 | 54 | With this setup you can test highlighting on files using the Tree-sitter CLI. 55 | 56 | ```shell 57 | npx tree-sitter highlight tmp/test.ex 58 | 59 | npx tree-sitter highlight test/highlight/**/*.ex 60 | ``` 61 | -------------------------------------------------------------------------------- /test/corpus/unicode.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | atom 3 | ===================================== 4 | 5 | :time_μs 6 | :"£" 7 | :'£' 8 | :こんにちは世界 9 | :Ólá 10 | :olá 11 | :Olá 12 | 13 | --- 14 | 15 | (source 16 | (atom) 17 | (quoted_atom 18 | (quoted_content)) 19 | (quoted_atom 20 | (quoted_content)) 21 | (atom) 22 | (atom) 23 | (atom) 24 | (atom)) 25 | 26 | ===================================== 27 | string 28 | ===================================== 29 | 30 | "time_μs" 31 | "£" 32 | "こんにちは世界" 33 | "Ólá" 34 | "olá" 35 | "Olá" 36 | 37 | --- 38 | 39 | (source 40 | (string 41 | (quoted_content)) 42 | (string 43 | (quoted_content)) 44 | (string 45 | (quoted_content)) 46 | (string 47 | (quoted_content)) 48 | (string 49 | (quoted_content)) 50 | (string 51 | (quoted_content))) 52 | 53 | ===================================== 54 | charlist 55 | ===================================== 56 | 57 | 'time_μs' 58 | '£' 59 | 'こんにちは世界' 60 | 'Ólá' 61 | 'olá' 62 | 'Olá' 63 | --- 64 | 65 | (source 66 | (charlist 67 | (quoted_content)) 68 | (charlist 69 | (quoted_content)) 70 | (charlist 71 | (quoted_content)) 72 | (charlist 73 | (quoted_content)) 74 | (charlist 75 | (quoted_content)) 76 | (charlist 77 | (quoted_content))) 78 | 79 | ===================================== 80 | char 81 | ===================================== 82 | 83 | ?ł 84 | ?μ 85 | ?£ 86 | ?こ 87 | 88 | --- 89 | 90 | (source 91 | (char) 92 | (char) 93 | (char) 94 | (char)) 95 | 96 | ===================================== 97 | variable 98 | ===================================== 99 | 100 | time_μs 101 | こんにちは世界 102 | olá 103 | 104 | --- 105 | 106 | (source 107 | (identifier) 108 | (identifier) 109 | (identifier)) 110 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from os.path import isdir, join 2 | from platform import system 3 | 4 | from setuptools import Extension, find_packages, setup 5 | from setuptools.command.build import build 6 | from wheel.bdist_wheel import bdist_wheel 7 | 8 | 9 | class Build(build): 10 | def run(self): 11 | if isdir("queries"): 12 | dest = join(self.build_lib, "tree_sitter_elixir", "queries") 13 | self.copy_tree("queries", dest) 14 | super().run() 15 | 16 | 17 | class BdistWheel(bdist_wheel): 18 | def get_tag(self): 19 | python, abi, platform = super().get_tag() 20 | if python.startswith("cp"): 21 | python, abi = "cp39", "abi3" 22 | return python, abi, platform 23 | 24 | 25 | setup( 26 | packages=find_packages("bindings/python"), 27 | package_dir={"": "bindings/python"}, 28 | package_data={ 29 | "tree_sitter_elixir": ["*.pyi", "py.typed"], 30 | "tree_sitter_elixir.queries": ["*.scm"], 31 | }, 32 | ext_package="tree_sitter_elixir", 33 | ext_modules=[ 34 | Extension( 35 | name="_binding", 36 | sources=[ 37 | "bindings/python/tree_sitter_elixir/binding.c", 38 | "src/parser.c", 39 | "src/scanner.c", 40 | ], 41 | extra_compile_args=[ 42 | "-std=c11", 43 | "-fvisibility=hidden", 44 | ] if system() != "Windows" else [ 45 | "/std:c11", 46 | "/utf-8", 47 | ], 48 | define_macros=[ 49 | ("Py_LIMITED_API", "0x03090000"), 50 | ("PY_SSIZE_T_CLEAN", None), 51 | ("TREE_SITTER_HIDE_SYMBOLS", None), 52 | ], 53 | include_dirs=["src"], 54 | py_limited_api=True, 55 | ) 56 | ], 57 | cmdclass={ 58 | "build": Build, 59 | "bdist_wheel": BdistWheel 60 | }, 61 | zip_safe=False 62 | ) 63 | -------------------------------------------------------------------------------- /test/highlight/operators.ex: -------------------------------------------------------------------------------- 1 | a in b 2 | # <- variable 3 | # ^ keyword 4 | # ^ variable 5 | 6 | a not in b 7 | # <- variable 8 | # ^ keyword 9 | # ^ keyword 10 | # ^ variable 11 | 12 | a not in b 13 | # <- variable 14 | # ^ keyword 15 | # ^ keyword 16 | # ^ variable 17 | 18 | a ~>> b = bind(a, b) 19 | # <- variable 20 | # ^ operator 21 | # ^ variable 22 | # ^ operator 23 | # ^ function 24 | # ^ punctuation.bracket 25 | # ^ variable 26 | # ^ punctuation.delimiter 27 | # ^ variable 28 | # ^ punctuation.bracket 29 | 30 | a ~> b 31 | # ^ operator 32 | 33 | a + b 34 | # ^ operator 35 | 36 | ... == !x && y || z 37 | # <- variable 38 | # ^ operator 39 | # ^ operator 40 | # ^ variable 41 | # ^ operator 42 | # ^ variable 43 | # ^ operator 44 | # ^ variable 45 | 46 | x = 1 + 2.0 * 3 47 | # <- variable 48 | # ^ operator 49 | # ^ number 50 | # ^ operator 51 | # ^ number 52 | # ^ operator 53 | # ^ number 54 | 55 | y = true and false 56 | # <- variable 57 | # ^ operator 58 | # ^ constant 59 | # ^ keyword 60 | # ^ constant 61 | 62 | { ^z, a } = {true, x} 63 | # <- punctuation.bracket 64 | # ^ operator 65 | # ^ variable 66 | # ^ punctuation.delimiter 67 | # ^ variable 68 | # ^ operator 69 | # ^ punctuation.bracket 70 | # ^ constant 71 | # ^ punctuation.delimiter 72 | # ^ variable 73 | # ^ punctuation.bracket 74 | 75 | "hello" |> String.upcase |> String.downcase() 76 | # ^ string 77 | # ^ operator 78 | # ^ module 79 | # ^ operator 80 | # ^ function 81 | # ^ operator 82 | # ^ module 83 | # ^ operator 84 | # ^ function 85 | 86 | range = .. 87 | # <- variable 88 | # ^ operator 89 | # ^ operator 90 | -------------------------------------------------------------------------------- /test/corpus/term/atom.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | simple literal 3 | ===================================== 4 | 5 | :atom 6 | :_az_AZ_19_ 7 | :nonode@nohost 8 | :bang! 9 | :question? 10 | 11 | --- 12 | 13 | (source 14 | (atom) 15 | (atom) 16 | (atom) 17 | (atom) 18 | (atom)) 19 | 20 | ===================================== 21 | operators 22 | ===================================== 23 | 24 | [:~~~, :~>>, :~>, :|||, :||, :|>, :|, :>>>, :>=, :>, :=~, :===, :==, :=, :<~>, :<~, :<|>, :<>, :<=, :<<~, :<<<, :<-, :<, :+++, :++, :+, :^^^, :^, :&&&, :&&, :&, :\\, :/, :**, :*, :@, :.., :., :!==, :!=, :!, :::, :->, :---, :--, :-] 25 | 26 | --- 27 | 28 | (source 29 | (list (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom) (atom))) 30 | 31 | ===================================== 32 | special operator-like atoms 33 | ===================================== 34 | 35 | [:..., :%{}, :{}, :%, :<<>>, :..//] 36 | 37 | --- 38 | 39 | (source 40 | (list 41 | (atom) 42 | (atom) 43 | (atom) 44 | (atom) 45 | (atom) 46 | (atom))) 47 | 48 | ===================================== 49 | quoted atom 50 | ===================================== 51 | 52 | :"atom ?? !! ' \n" 53 | :'atom ?? !! " \n' 54 | 55 | --- 56 | 57 | (source 58 | (quoted_atom 59 | (quoted_content) 60 | (escape_sequence)) 61 | (quoted_atom 62 | (quoted_content) 63 | (escape_sequence))) 64 | 65 | ===================================== 66 | interpolation 67 | ===================================== 68 | 69 | :"hey #{name}!" 70 | :'hey #{name}!' 71 | 72 | --- 73 | 74 | (source 75 | (quoted_atom 76 | (quoted_content) 77 | (interpolation 78 | (identifier)) 79 | (quoted_content)) 80 | (quoted_atom 81 | (quoted_content) 82 | (interpolation 83 | (identifier)) 84 | (quoted_content))) 85 | -------------------------------------------------------------------------------- /test/highlight/anonymous.ex: -------------------------------------------------------------------------------- 1 | fn x, y, z -> 2 | # <- keyword 3 | # ^ variable 4 | # ^ punctuation.delimiter 5 | # ^ variable 6 | # ^ punctuation.delimiter 7 | # ^ variable 8 | # ^ operator 9 | fn(a, b, c) -> 10 | # <- keyword 11 | # ^ punctuation.bracket 12 | # ^ variable 13 | # ^ punctuation.delimiter 14 | # ^ variable 15 | # ^ punctuation.delimiter 16 | # ^ variable 17 | # ^ punctuation.bracket 18 | # ^ operator 19 | &(x + y - z * a / &1 + b + div(&2, c)) 20 | #<- operator 21 | #^ punctuation.bracket 22 | # ^ variable 23 | # ^ operator 24 | # ^ variable 25 | # ^ operator 26 | # ^ variable 27 | # ^ operator 28 | # ^ variable 29 | # ^ operator 30 | # ^ operator 31 | # ^ operator 32 | # ^ variable 33 | # ^ operator 34 | # ^ function 35 | # ^ punctuation.bracket 36 | # ^ operator 37 | # ^ punctuation.delimiter 38 | # ^ variable 39 | # ^ punctuation.bracket 40 | # ^ punctuation.bracket 41 | end 42 | end 43 | 44 | fn -> 45 | # <- keyword 46 | # ^ operator 47 | end 48 | # <- keyword 49 | 50 | &Set.put(&1, &2) 51 | # <- operator 52 | # ^ module 53 | # ^ operator 54 | # ^ function 55 | # ^ punctuation.bracket 56 | # ^ operator 57 | # ^ punctuation.delimiter 58 | # ^ operator 59 | # ^ punctuation.bracket 60 | 61 | &( Set.put(&1, &1) ) 62 | #<- operator 63 | #^ punctuation.bracket 64 | # ^ module 65 | # ^ operator 66 | # ^ function 67 | # ^ punctuation.bracket 68 | # ^ operator 69 | # ^ punctuation.delimiter 70 | # ^ operator 71 | # ^ punctuation.bracket 72 | # ^ punctuation.bracket 73 | -------------------------------------------------------------------------------- /bindings/rust/lib.rs: -------------------------------------------------------------------------------- 1 | //! This crate provides Elixir language support for the [tree-sitter][] parsing library. 2 | //! 3 | //! Typically, you will use the [language][language func] function to add this language to a 4 | //! tree-sitter [Parser][], and then use the parser to parse some code: 5 | //! 6 | //! ``` 7 | //! let code = r#" 8 | //! "#; 9 | //! let mut parser = tree_sitter::Parser::new(); 10 | //! let language = tree_sitter_elixir::LANGUAGE; 11 | //! parser 12 | //! .set_language(&language.into()) 13 | //! .expect("Error loading Elixir parser"); 14 | //! let tree = parser.parse(code, None).unwrap(); 15 | //! assert!(!tree.root_node().has_error()); 16 | //! ``` 17 | //! 18 | //! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html 19 | //! [language func]: fn.language.html 20 | //! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html 21 | //! [tree-sitter]: https://tree-sitter.github.io/ 22 | 23 | use tree_sitter_language::LanguageFn; 24 | 25 | extern "C" { 26 | fn tree_sitter_elixir() -> *const (); 27 | } 28 | 29 | /// The tree-sitter [`LanguageFn`] for this grammar. 30 | pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_elixir) }; 31 | 32 | /// The content of the [`node-types.json`][] file for this grammar. 33 | /// 34 | /// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types 35 | pub const NODE_TYPES: &str = include_str!("../../src/node-types.json"); 36 | 37 | // NOTE: uncomment these to include any queries that this grammar contains: 38 | 39 | pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm"); 40 | pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm"); 41 | // pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm"); 42 | pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm"); 43 | 44 | #[cfg(test)] 45 | mod tests { 46 | #[test] 47 | fn test_can_load_grammar() { 48 | let mut parser = tree_sitter::Parser::new(); 49 | parser 50 | .set_language(&super::LANGUAGE.into()) 51 | .expect("Error loading Elixir parser"); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /test/corpus/edge_syntax.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | operator with arity (valid and supported by IEx.Helpers.h) 3 | ===================================== 4 | 5 | ::/2 6 | @ / 1 7 | & / 1 8 | not / 1 9 | not in / 2 10 | * / 2 11 | h +/2 12 | 13 | --- 14 | 15 | (source 16 | (binary_operator 17 | (operator_identifier) 18 | (integer)) 19 | (binary_operator 20 | (operator_identifier) 21 | (integer)) 22 | (binary_operator 23 | (operator_identifier) 24 | (integer)) 25 | (binary_operator 26 | (operator_identifier) 27 | (integer)) 28 | (binary_operator 29 | (operator_identifier) 30 | (integer)) 31 | (binary_operator 32 | (operator_identifier) 33 | (integer)) 34 | (call 35 | (identifier) 36 | (arguments 37 | (binary_operator 38 | (operator_identifier) 39 | (integer))))) 40 | 41 | ===================================== 42 | stab and slash ambiguity 43 | ===================================== 44 | 45 | (-> / 2) 46 | (-> / / 2) 47 | 48 | --- 49 | 50 | (source 51 | (block 52 | (binary_operator 53 | (operator_identifier) 54 | (integer))) 55 | (block 56 | (stab_clause 57 | (body 58 | (binary_operator 59 | (operator_identifier) 60 | (integer)))))) 61 | 62 | ===================================== 63 | unary operator and slash ambiguity 64 | ===================================== 65 | 66 | & / 2 67 | & / / 2 68 | --- 69 | 70 | (source 71 | (binary_operator 72 | (operator_identifier) 73 | (integer)) 74 | (unary_operator 75 | (binary_operator 76 | (operator_identifier) 77 | (integer)))) 78 | 79 | ===================================== 80 | map with identifiers 81 | ===================================== 82 | 83 | %{a} 84 | %{a, b} 85 | 86 | --- 87 | 88 | (source 89 | (map 90 | (map_content 91 | (identifier))) 92 | (map 93 | (map_content 94 | (identifier) 95 | (identifier)))) 96 | 97 | ===================================== 98 | def with remote call 99 | ===================================== 100 | 101 | def Mod.fun(x), do: 1 102 | 103 | --- 104 | 105 | (source 106 | (call 107 | (identifier) 108 | (arguments 109 | (call 110 | (dot 111 | (alias) 112 | (identifier)) 113 | (arguments 114 | (identifier))) 115 | (keywords 116 | (pair 117 | (keyword) 118 | (integer)))))) 119 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). 6 | 7 | ## [Unreleased](https://github.com/elixir-lang/tree-sitter-elixir/tree/main) 8 | 9 | ### Added 10 | 11 | * Support for ~SQL sigils in the built-in injections ([#84](https://github.com/elixir-lang/tree-sitter-elixir/pull/84)) 12 | 13 | ## [v0.3.4](https://github.com/elixir-lang/tree-sitter-elixir/tree/v0.3.4) (2025-02-06) 14 | 15 | ### Changed 16 | 17 | * Changed the built-in queries to use `#any-of?` predicate over `#match?` ([#80](https://github.com/elixir-lang/tree-sitter-elixir/pull/80)) 18 | 19 | ## [v0.3.3](https://github.com/elixir-lang/tree-sitter-elixir/tree/v0.3.3) (2024-12-09) 20 | 21 | ### Changed 22 | 23 | * The parser now accepts anonymous functions with no clauses ([#78](https://github.com/elixir-lang/tree-sitter-elixir/pull/78)) 24 | * Moved parser information to tree-sitter.json ([#79](https://github.com/elixir-lang/tree-sitter-elixir/pull/79)) 25 | 26 | ## [v0.3.2](https://github.com/elixir-lang/tree-sitter-elixir/tree/v0.3.2) (2024-12-02) 27 | 28 | ### Added 29 | 30 | * Support for ~LVN sigils (LiveView Native templates) in the built-in injections ([#75](https://github.com/elixir-lang/tree-sitter-elixir/pull/75)) 31 | 32 | ## [v0.3.1](https://github.com/elixir-lang/tree-sitter-elixir/tree/v0.3.1) (2024-09-28) 33 | 34 | ### Changed 35 | 36 | * Changed highlight queries to distinguish field access from calls ([#73](https://github.com/elixir-lang/tree-sitter-elixir/pull/73)) 37 | 38 | ## [v0.3.0](https://github.com/elixir-lang/tree-sitter-elixir/tree/v0.3.0) (2024-09-05) 39 | 40 | ### Changed 41 | 42 | * The Rust crate to depend on tree-sitter-language rather than tree-sitter ([#70](https://github.com/elixir-lang/tree-sitter-elixir/pull/70)) 43 | 44 | ## [v0.2.0](https://github.com/elixir-lang/tree-sitter-elixir/tree/v0.2.0) (2024-04-08) 45 | 46 | ### Changed 47 | 48 | * Required tree-sitter version to 0.21+ ([#66](https://github.com/elixir-lang/tree-sitter-elixir/pull/66)) 49 | 50 | ## [v0.1.1](https://github.com/elixir-lang/tree-sitter-elixir/tree/v0.1.1) (2023-12-04) 51 | 52 | ### Changed 53 | 54 | * Rewritten the custom scanner code in C ([#56](https://github.com/elixir-lang/tree-sitter-elixir/pull/56)) 55 | 56 | ### Fixed 57 | 58 | * Parsing empty interpolation ([#55](https://github.com/elixir-lang/tree-sitter-elixir/pull/55)) 59 | * Fixed the repository URL in the Rust crate ([#57](https://github.com/elixir-lang/tree-sitter-elixir/pull/57)) 60 | 61 | ## [v0.1.0](https://github.com/elixir-lang/tree-sitter-elixir/tree/v0.1.0) (2023-03-14) 62 | 63 | Initial release. 64 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.13) 2 | 3 | project(tree-sitter-elixir 4 | VERSION "0.3.4" 5 | DESCRIPTION "Elixir grammar for the tree-sitter parsing library" 6 | HOMEPAGE_URL "https://github.com/elixir-lang/tree-sitter-elixir" 7 | LANGUAGES C) 8 | 9 | option(BUILD_SHARED_LIBS "Build using shared libraries" ON) 10 | option(TREE_SITTER_REUSE_ALLOCATOR "Reuse the library allocator" OFF) 11 | 12 | set(TREE_SITTER_ABI_VERSION 14 CACHE STRING "Tree-sitter ABI version") 13 | if(NOT ${TREE_SITTER_ABI_VERSION} MATCHES "^[0-9]+$") 14 | unset(TREE_SITTER_ABI_VERSION CACHE) 15 | message(FATAL_ERROR "TREE_SITTER_ABI_VERSION must be an integer") 16 | endif() 17 | 18 | find_program(TREE_SITTER_CLI tree-sitter DOC "Tree-sitter CLI") 19 | 20 | add_custom_command(OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/src/parser.c" 21 | DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/grammar.json" 22 | COMMAND "${TREE_SITTER_CLI}" generate src/grammar.json 23 | --abi=${TREE_SITTER_ABI_VERSION} 24 | WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" 25 | COMMENT "Generating parser.c") 26 | 27 | add_library(tree-sitter-elixir src/parser.c) 28 | if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/src/scanner.c) 29 | target_sources(tree-sitter-elixir PRIVATE src/scanner.c) 30 | endif() 31 | target_include_directories(tree-sitter-elixir PRIVATE src) 32 | 33 | target_compile_definitions(tree-sitter-elixir PRIVATE 34 | $<$:TREE_SITTER_REUSE_ALLOCATOR> 35 | $<$:TREE_SITTER_DEBUG>) 36 | 37 | set_target_properties(tree-sitter-elixir 38 | PROPERTIES 39 | C_STANDARD 11 40 | POSITION_INDEPENDENT_CODE ON 41 | SOVERSION "${TREE_SITTER_ABI_VERSION}.${PROJECT_VERSION_MAJOR}" 42 | DEFINE_SYMBOL "") 43 | 44 | configure_file(bindings/c/tree-sitter-elixir.pc.in 45 | "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-elixir.pc" @ONLY) 46 | 47 | include(GNUInstallDirs) 48 | 49 | install(FILES bindings/c/tree-sitter-elixir.h 50 | DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tree_sitter") 51 | install(FILES "${CMAKE_CURRENT_BINARY_DIR}/tree-sitter-elixir.pc" 52 | DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig") 53 | install(TARGETS tree-sitter-elixir 54 | LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}") 55 | 56 | add_custom_target(ts-test "${TREE_SITTER_CLI}" test 57 | WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" 58 | COMMENT "tree-sitter test") 59 | -------------------------------------------------------------------------------- /test/corpus/expression/capture.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | anonymous function 3 | ===================================== 4 | 5 | & &1 + &2 6 | &(&1 + &2) 7 | &foo(&1, a, &2) 8 | 9 | --- 10 | 11 | (source 12 | (unary_operator 13 | (binary_operator 14 | (unary_operator 15 | (integer)) 16 | (unary_operator 17 | (integer)))) 18 | (unary_operator 19 | (binary_operator 20 | (unary_operator 21 | (integer)) 22 | (unary_operator 23 | (integer)))) 24 | (unary_operator 25 | (call 26 | (identifier) 27 | (arguments 28 | (unary_operator 29 | (integer)) 30 | (identifier) 31 | (unary_operator 32 | (integer)))))) 33 | 34 | ===================================== 35 | argument call 36 | ===================================== 37 | 38 | & &1.some_fun 39 | &(&1.some_fun) 40 | & &1.(&2) 41 | 42 | --- 43 | 44 | (source 45 | (unary_operator 46 | (call 47 | (dot 48 | (unary_operator 49 | (integer)) 50 | (identifier)))) 51 | (unary_operator 52 | (call 53 | (dot 54 | (unary_operator 55 | (integer)) 56 | (identifier)))) 57 | (unary_operator 58 | (call 59 | (dot 60 | (unary_operator 61 | (integer))) 62 | (arguments 63 | (unary_operator 64 | (integer)))))) 65 | 66 | ===================================== 67 | remote MFA 68 | ===================================== 69 | 70 | &Mod.fun/1 71 | 72 | --- 73 | 74 | (source 75 | (unary_operator 76 | (binary_operator 77 | (call 78 | (dot 79 | (alias) 80 | (identifier))) 81 | (integer)))) 82 | 83 | ===================================== 84 | remote operator MFA 85 | ===================================== 86 | 87 | &Kernel.>=/2 88 | 89 | --- 90 | 91 | (source 92 | (unary_operator 93 | (binary_operator 94 | (call 95 | (dot 96 | (alias) 97 | (operator_identifier))) 98 | (integer)))) 99 | 100 | ===================================== 101 | local MFA 102 | ===================================== 103 | 104 | &fun/1 105 | 106 | --- 107 | 108 | (source 109 | (unary_operator 110 | (binary_operator 111 | (identifier) 112 | (integer)))) 113 | 114 | ===================================== 115 | local operator MFA 116 | ===================================== 117 | 118 | &>=/2 119 | &//2 120 | 121 | --- 122 | 123 | (source 124 | (unary_operator 125 | (binary_operator 126 | (operator_identifier) 127 | (integer))) 128 | (unary_operator 129 | (binary_operator 130 | (operator_identifier) 131 | (integer)))) 132 | -------------------------------------------------------------------------------- /test/highlight/calls.ex: -------------------------------------------------------------------------------- 1 | Path.expand("..", __DIR__) 2 | # ^ module 3 | # ^ operator 4 | # ^ function 5 | # ^ punctuation.bracket 6 | # ^ string 7 | # ^ punctuation.delimiter 8 | # ^ constant.builtin 9 | # ^ punctuation.bracket 10 | 11 | func.(1) 12 | # ^ variable 13 | # ^ operator 14 | # ^ punctuation.bracket 15 | # ^ number 16 | # ^ punctuation.bracket 17 | 18 | arg |> func 19 | # ^ variable 20 | # ^ operator 21 | # ^ function 22 | 23 | func 1 24 | # ^ function 25 | # ^ number 26 | 27 | hd([1,2]) 28 | # <- function 29 | # ^ punctuation.bracket 30 | # ^ punctuation.bracket 31 | # ^ number 32 | # ^ punctuation.delimiter 33 | # ^ number 34 | # ^ punctuation.bracket 35 | # ^ punctuation.bracket 36 | 37 | Kernel.spawn(fn -> :ok end) 38 | # ^ module 39 | # ^ operator 40 | # ^ function 41 | # ^ punctuation.bracket 42 | # ^ keyword 43 | # ^ operator 44 | # ^ string.special.symbol 45 | # ^ keyword 46 | # ^ punctuation.bracket 47 | 48 | IO.ANSI.black 49 | # ^ module 50 | # ^ operator 51 | # ^ function 52 | 53 | Kernel.-(number) 54 | # ^ module 55 | # ^ operator 56 | # ^ operator 57 | # ^ punctuation.bracket 58 | # ^ variable 59 | # ^ punctuation.bracket 60 | 61 | Enum.map([1, 2], fn x -> 62 | # ^ module 63 | # ^ operator 64 | # ^ function 65 | # ^ punctuation.bracket 66 | # ^ punctuation.bracket 67 | # ^ number 68 | # ^ punctuation.delimiter 69 | # ^ number 70 | # ^ punctuation.bracket 71 | # ^ punctuation.delimiter 72 | # ^ keyword 73 | # ^ variable 74 | # ^ operator 75 | x * 2 76 | # <- variable 77 | # ^ operator 78 | # ^ number 79 | end) 80 | # <- keyword 81 | # ^ punctuation.bracket 82 | 83 | :erlang.abs(-1) 84 | # ^ module 85 | # ^ operator 86 | # ^ function 87 | # ^ punctuation.bracket 88 | # ^ operator 89 | # ^ number 90 | # ^ punctuation.bracket 91 | 92 | map.key1 93 | # ^ variable 94 | # ^ property 95 | 96 | map.key1.key2 97 | # ^ variable 98 | # ^ property 99 | # ^ property 100 | 101 | DateTime.utc_now.day 102 | # ^ module 103 | # ^ function 104 | # ^ property 105 | 106 | arg |> mod.func 107 | # ^ variable 108 | # ^ operator 109 | # ^ variable 110 | # ^ function 111 | 112 | Mod.fun do 113 | # ^ module 114 | # ^ function 115 | # ^ keyword 116 | end 117 | # ^ keyword 118 | 119 | mod.fun do 120 | # ^ variable 121 | # ^ function 122 | # ^ keyword 123 | end 124 | # ^ keyword 125 | -------------------------------------------------------------------------------- /test/corpus/integration/kernel.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | for / enumerable 3 | ===================================== 4 | 5 | for n <- [1, 2], do: n * 2 6 | 7 | --- 8 | 9 | (source 10 | (call 11 | (identifier) 12 | (arguments 13 | (binary_operator 14 | (identifier) 15 | (list 16 | (integer) 17 | (integer))) 18 | (keywords 19 | (pair 20 | (keyword) 21 | (binary_operator 22 | (identifier) 23 | (integer))))))) 24 | 25 | ===================================== 26 | for / enumerable / with options and block 27 | ===================================== 28 | 29 | for line <- IO.stream(), into: IO.stream() do 30 | String.upcase(line) 31 | end 32 | 33 | --- 34 | 35 | (source 36 | (call 37 | (identifier) 38 | (arguments 39 | (binary_operator 40 | (identifier) 41 | (call 42 | (dot 43 | (alias) 44 | (identifier)) 45 | (arguments))) 46 | (keywords 47 | (pair 48 | (keyword) 49 | (call 50 | (dot 51 | (alias) 52 | (identifier)) 53 | (arguments))))) 54 | (do_block 55 | (call 56 | (dot 57 | (alias) 58 | (identifier)) 59 | (arguments 60 | (identifier)))))) 61 | 62 | ===================================== 63 | for / binary 64 | ===================================== 65 | 66 | for <>, c != ?\s, into: "", do: <> 67 | 68 | --- 69 | 70 | (source 71 | (call 72 | (identifier) 73 | (arguments 74 | (bitstring 75 | (binary_operator 76 | (identifier) 77 | (string 78 | (quoted_content)))) 79 | (binary_operator 80 | (identifier) 81 | (char)) 82 | (keywords 83 | (pair 84 | (keyword) 85 | (string)) 86 | (pair 87 | (keyword) 88 | (bitstring 89 | (identifier))))))) 90 | 91 | ===================================== 92 | for / reduce 93 | ===================================== 94 | 95 | for x <- [1, 2, 1], reduce: %{} do 96 | acc -> Map.update(acc, x, 1, & &1 + 1) 97 | end 98 | 99 | --- 100 | 101 | (source 102 | (call 103 | (identifier) 104 | (arguments 105 | (binary_operator 106 | (identifier) 107 | (list 108 | (integer) 109 | (integer) 110 | (integer))) 111 | (keywords 112 | (pair 113 | (keyword) 114 | (map)))) 115 | (do_block 116 | (stab_clause 117 | (arguments 118 | (identifier)) 119 | (body 120 | (call 121 | (dot 122 | (alias) 123 | (identifier)) 124 | (arguments 125 | (identifier) 126 | (identifier) 127 | (integer) 128 | (unary_operator 129 | (binary_operator 130 | (unary_operator 131 | (integer)) 132 | (integer)))))))))) 133 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | LEGAL NOTICE INFORMATION 2 | ------------------------ 3 | 4 | All the files in this distribution are copyright to the terms below. 5 | 6 | == All files in src/ except scanner.cc (generated by tree-sitter-cli) 7 | 8 | Copyright (c) 2018-2021 Max Brunsfeld 9 | 10 | The MIT License (MIT) 11 | 12 | Permission is hereby granted, free of charge, to any person obtaining a copy 13 | of this software and associated documentation files (the "Software"), to deal 14 | in the Software without restriction, including without limitation the rights 15 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 16 | copies of the Software, and to permit persons to whom the Software is 17 | furnished to do so, subject to the following conditions: 18 | 19 | The above copyright notice and this permission notice shall be included in all 20 | copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 25 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 26 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 27 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 28 | SOFTWARE. 29 | 30 | == Some file fragments in test/corpus/ 31 | 32 | Copyright (c) 2021 Anantha Kumaran 33 | 34 | MIT License 35 | 36 | Permission is hereby granted, free of charge, to any person obtaining a copy 37 | of this software and associated documentation files (the "Software"), to deal 38 | in the Software without restriction, including without limitation the rights 39 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 40 | copies of the Software, and to permit persons to whom the Software is 41 | furnished to do so, subject to the following conditions: 42 | 43 | The above copyright notice and this permission notice shall be included in all 44 | copies or substantial portions of the Software. 45 | 46 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 47 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 48 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 49 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 50 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 51 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 52 | SOFTWARE. 53 | 54 | == All other files 55 | 56 | Copyright 2021 The Elixir Team 57 | 58 | Licensed under the Apache License, Version 2.0 (the "License"); 59 | you may not use this file except in compliance with the License. 60 | You may obtain a copy of the License at 61 | 62 | https://www.apache.org/licenses/LICENSE-2.0 63 | 64 | Unless required by applicable law or agreed to in writing, software 65 | distributed under the License is distributed on an "AS IS" BASIS, 66 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 67 | See the License for the specific language governing permissions and 68 | limitations under the License. 69 | -------------------------------------------------------------------------------- /test/corpus/integration/module_definition.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | empty module definition 3 | ===================================== 4 | 5 | defmodule Mod do 6 | end 7 | 8 | defmodule Mod.Child do 9 | end 10 | 11 | --- 12 | 13 | (source 14 | (call 15 | (identifier) 16 | (arguments 17 | (alias)) 18 | (do_block)) 19 | (call 20 | (identifier) 21 | (arguments 22 | (alias)) 23 | (do_block))) 24 | 25 | ===================================== 26 | module definition with atom literal 27 | ===================================== 28 | 29 | defmodule :mod do 30 | end 31 | 32 | --- 33 | 34 | (source 35 | (call 36 | (identifier) 37 | (arguments 38 | (atom)) 39 | (do_block))) 40 | 41 | ===================================== 42 | full module definition 43 | ===================================== 44 | 45 | defmodule Mod do 46 | @moduledoc """ 47 | Example module 48 | """ 49 | 50 | use UseMod 51 | 52 | @attribute 1 53 | 54 | @doc """ 55 | Example function 56 | """ 57 | @spec func(integer) :: integer 58 | def func(x) when is_integer(x) do 59 | priv(x) + priv(x) 60 | end 61 | 62 | defp priv(x), do: x * x 63 | end 64 | 65 | --- 66 | 67 | (source 68 | (call 69 | (identifier) 70 | (arguments 71 | (alias)) 72 | (do_block 73 | (unary_operator 74 | (call 75 | (identifier) 76 | (arguments 77 | (string 78 | (quoted_content))))) 79 | (call 80 | (identifier) 81 | (arguments 82 | (alias))) 83 | (unary_operator 84 | (call 85 | (identifier) 86 | (arguments 87 | (integer)))) 88 | (unary_operator 89 | (call 90 | (identifier) 91 | (arguments 92 | (string 93 | (quoted_content))))) 94 | (unary_operator 95 | (call 96 | (identifier) 97 | (arguments 98 | (binary_operator 99 | (call 100 | (identifier) 101 | (arguments 102 | (identifier))) 103 | (identifier))))) 104 | (call 105 | (identifier) 106 | (arguments 107 | (binary_operator 108 | (call 109 | (identifier) 110 | (arguments 111 | (identifier))) 112 | (call 113 | (identifier) 114 | (arguments 115 | (identifier))))) 116 | (do_block 117 | (binary_operator 118 | (call 119 | (identifier) 120 | (arguments 121 | (identifier))) 122 | (call 123 | (identifier) 124 | (arguments 125 | (identifier)))))) 126 | (call 127 | (identifier) 128 | (arguments 129 | (call 130 | (identifier) 131 | (arguments 132 | (identifier))) 133 | (keywords 134 | (pair 135 | (keyword) 136 | (binary_operator 137 | (identifier) 138 | (identifier))))))))) 139 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ifeq ($(OS),Windows_NT) 2 | $(error Windows is not supported) 3 | endif 4 | 5 | LANGUAGE_NAME := tree-sitter-elixir 6 | HOMEPAGE_URL := https://github.com/elixir-lang/tree-sitter-elixir 7 | VERSION := 0.3.4 8 | 9 | # repository 10 | SRC_DIR := src 11 | 12 | TS ?= tree-sitter 13 | 14 | # install directory layout 15 | PREFIX ?= /usr/local 16 | INCLUDEDIR ?= $(PREFIX)/include 17 | LIBDIR ?= $(PREFIX)/lib 18 | PCLIBDIR ?= $(LIBDIR)/pkgconfig 19 | 20 | # source/object files 21 | PARSER := $(SRC_DIR)/parser.c 22 | EXTRAS := $(filter-out $(PARSER),$(wildcard $(SRC_DIR)/*.c)) 23 | OBJS := $(patsubst %.c,%.o,$(PARSER) $(EXTRAS)) 24 | 25 | # flags 26 | ARFLAGS ?= rcs 27 | override CFLAGS += -I$(SRC_DIR) -std=c11 -fPIC 28 | 29 | # ABI versioning 30 | SONAME_MAJOR = $(shell sed -n 's/\#define LANGUAGE_VERSION //p' $(PARSER)) 31 | SONAME_MINOR = $(word 1,$(subst ., ,$(VERSION))) 32 | 33 | # OS-specific bits 34 | ifeq ($(shell uname),Darwin) 35 | SOEXT = dylib 36 | SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT) 37 | SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT) 38 | LINKSHARED = -dynamiclib -Wl,-install_name,$(LIBDIR)/lib$(LANGUAGE_NAME).$(SOEXTVER),-rpath,@executable_path/../Frameworks 39 | else 40 | SOEXT = so 41 | SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR) 42 | SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR) 43 | LINKSHARED = -shared -Wl,-soname,lib$(LANGUAGE_NAME).$(SOEXTVER) 44 | endif 45 | ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),) 46 | PCLIBDIR := $(PREFIX)/libdata/pkgconfig 47 | endif 48 | 49 | all: lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) $(LANGUAGE_NAME).pc 50 | 51 | lib$(LANGUAGE_NAME).a: $(OBJS) 52 | $(AR) $(ARFLAGS) $@ $^ 53 | 54 | lib$(LANGUAGE_NAME).$(SOEXT): $(OBJS) 55 | $(CC) $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@ 56 | ifneq ($(STRIP),) 57 | $(STRIP) $@ 58 | endif 59 | 60 | $(LANGUAGE_NAME).pc: bindings/c/$(LANGUAGE_NAME).pc.in 61 | sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \ 62 | -e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \ 63 | -e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \ 64 | -e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \ 65 | -e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \ 66 | -e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@ 67 | 68 | $(PARSER): $(SRC_DIR)/grammar.json 69 | $(TS) generate $^ 70 | 71 | install: all 72 | install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)' 73 | install -m644 bindings/c/$(LANGUAGE_NAME).h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h 74 | install -m644 $(LANGUAGE_NAME).pc '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc 75 | install -m644 lib$(LANGUAGE_NAME).a '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a 76 | install -m755 lib$(LANGUAGE_NAME).$(SOEXT) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) 77 | ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) 78 | ln -sf lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) 79 | 80 | uninstall: 81 | $(RM) '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).a \ 82 | '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER) \ 83 | '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXTVER_MAJOR) \ 84 | '$(DESTDIR)$(LIBDIR)'/lib$(LANGUAGE_NAME).$(SOEXT) \ 85 | '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/$(LANGUAGE_NAME).h \ 86 | '$(DESTDIR)$(PCLIBDIR)'/$(LANGUAGE_NAME).pc 87 | 88 | clean: 89 | $(RM) $(OBJS) $(LANGUAGE_NAME).pc lib$(LANGUAGE_NAME).a lib$(LANGUAGE_NAME).$(SOEXT) 90 | 91 | test: 92 | $(TS) test 93 | 94 | .PHONY: all install uninstall clean test 95 | -------------------------------------------------------------------------------- /test/corpus/term/map.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | empty 3 | ===================================== 4 | 5 | %{} 6 | 7 | --- 8 | 9 | (source 10 | (map)) 11 | 12 | ===================================== 13 | from keywords 14 | ===================================== 15 | 16 | %{a: 1, b: 2} 17 | 18 | --- 19 | 20 | (source 21 | (map 22 | (map_content 23 | (keywords 24 | (pair 25 | (keyword) 26 | (integer)) 27 | (pair 28 | (keyword) 29 | (integer)))))) 30 | 31 | ===================================== 32 | from arrow entries 33 | ===================================== 34 | 35 | %{:a => 1, "b" => 2, c => 3} 36 | 37 | --- 38 | 39 | (source 40 | (map 41 | (map_content 42 | (binary_operator 43 | (atom) 44 | (integer)) 45 | (binary_operator 46 | (string 47 | (quoted_content)) 48 | (integer)) 49 | (binary_operator 50 | (identifier) 51 | (integer))))) 52 | 53 | ===================================== 54 | from both arrow entries and keywords 55 | ===================================== 56 | 57 | %{"a" => 1, b: 2, c: 3} 58 | 59 | --- 60 | 61 | (source 62 | (map 63 | (map_content 64 | (binary_operator 65 | (string 66 | (quoted_content)) 67 | (integer)) 68 | (keywords 69 | (pair 70 | (keyword) 71 | (integer)) 72 | (pair 73 | (keyword) 74 | (integer)))))) 75 | 76 | ===================================== 77 | trailing separator 78 | ===================================== 79 | 80 | %{"a" => 1,} 81 | 82 | --- 83 | 84 | (source 85 | (map 86 | (map_content 87 | (binary_operator 88 | (string 89 | (quoted_content)) 90 | (integer))))) 91 | 92 | ===================================== 93 | update syntax 94 | ===================================== 95 | 96 | %{user | name: "Jane", email: "jane@example.com"} 97 | %{user | "name" => "Jane"} 98 | 99 | --- 100 | 101 | (source 102 | (map 103 | (map_content 104 | (binary_operator 105 | (identifier) 106 | (keywords 107 | (pair 108 | (keyword) 109 | (string 110 | (quoted_content))) 111 | (pair 112 | (keyword) 113 | (string 114 | (quoted_content))))))) 115 | (map 116 | (map_content 117 | (binary_operator 118 | (identifier) 119 | (binary_operator 120 | (string 121 | (quoted_content)) 122 | (string 123 | (quoted_content))))))) 124 | 125 | ===================================== 126 | [error] ordering 127 | ===================================== 128 | 129 | %{b: 2, c: 3, 1 => 1} 130 | 131 | --- 132 | 133 | (source 134 | (map 135 | (ERROR 136 | (keywords 137 | (pair 138 | (keyword) 139 | (integer)) 140 | (pair 141 | (keyword) 142 | (integer)))) 143 | (map_content 144 | (binary_operator 145 | (integer) 146 | (integer))))) 147 | 148 | ===================================== 149 | [error] missing separator 150 | ===================================== 151 | 152 | %{"a" => 1 "b" => 2} 153 | 154 | --- 155 | 156 | (source 157 | (map 158 | (map_content 159 | (binary_operator 160 | (string 161 | (quoted_content)) 162 | (ERROR (integer)) 163 | (binary_operator 164 | (string 165 | (quoted_content)) 166 | (integer)))))) 167 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o0= 4 | github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= 5 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 6 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 7 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 8 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 9 | github.com/tree-sitter/go-tree-sitter v0.23.1 h1:HCfaE19sKfG7q190xfM1loUZf6wEHa4TDqDEW46s9Lg= 10 | github.com/tree-sitter/go-tree-sitter v0.23.1/go.mod h1:EvIVhMvvPNvhu9x+ddSPxSnUEU5AnsSwi1LMqXIVE3A= 11 | github.com/tree-sitter/tree-sitter-c v0.21.5-0.20240818205408-927da1f210eb h1:A8425heRM8mylnv4H58FPUiH+aYivyitre0PzxrfmWs= 12 | github.com/tree-sitter/tree-sitter-c v0.21.5-0.20240818205408-927da1f210eb/go.mod h1:dOF6gtQiF9UwNh995T5OphYmtIypkjsp3ap7r9AN/iA= 13 | github.com/tree-sitter/tree-sitter-cpp v0.22.4-0.20240818224355-b1a4e2b25148 h1:AfFPZwtwGN01BW1jDdqBVqscTwetvMpydqYZz57RSlc= 14 | github.com/tree-sitter/tree-sitter-cpp v0.22.4-0.20240818224355-b1a4e2b25148/go.mod h1:Bh6U3viD57rFXRYIQ+kmiYtr+1Bx0AceypDLJJSyi9s= 15 | github.com/tree-sitter/tree-sitter-embedded-template v0.21.1-0.20240819044651-ffbf64942c33 h1:TwqSV3qLp3tKSqirGLRHnjFk9Tc2oy57LIl+FQ4GjI4= 16 | github.com/tree-sitter/tree-sitter-embedded-template v0.21.1-0.20240819044651-ffbf64942c33/go.mod h1:CvCKCt3v04Ufos1zZnNCelBDeCGRpPucaN8QczoUsN4= 17 | github.com/tree-sitter/tree-sitter-go v0.21.3-0.20240818010209-8c0f0e7a6012 h1:Xvxck3tE5FW7F7bTS97iNM2ADMyCMJztVqn5HYKdJGo= 18 | github.com/tree-sitter/tree-sitter-go v0.21.3-0.20240818010209-8c0f0e7a6012/go.mod h1:T40D0O1cPvUU/+AmiXVXy1cncYQT6wem4Z0g4SfAYvY= 19 | github.com/tree-sitter/tree-sitter-html v0.20.5-0.20240818004741-d11201a263d0 h1:c46K6uh5Dz00zJeU9BfjXdb8I+E4RkUdfnWJpQADXFo= 20 | github.com/tree-sitter/tree-sitter-html v0.20.5-0.20240818004741-d11201a263d0/go.mod h1:hcNt/kOJHcIcuMvouE7LJcYdeFUFbVpBJ6d4wmOA+tU= 21 | github.com/tree-sitter/tree-sitter-java v0.21.1-0.20240824015150-576d8097e495 h1:jrt4qbJVEFs4H93/ITxygHc6u0TGqAkkate7TQ4wFSA= 22 | github.com/tree-sitter/tree-sitter-java v0.21.1-0.20240824015150-576d8097e495/go.mod h1:oyaR7fLnRV0hT9z6qwE9GkaeTom/hTDwK3H2idcOJFc= 23 | github.com/tree-sitter/tree-sitter-javascript v0.21.5-0.20240818005344-15887341e5b5 h1:om4X9AVg3asL8gxNJDcz4e/Wp+VpQj1PY3uJXKr6EOg= 24 | github.com/tree-sitter/tree-sitter-javascript v0.21.5-0.20240818005344-15887341e5b5/go.mod h1:nNqgPoV/h9uYWk6kYEFdEAhNVOacpfpRW5SFmdaP4tU= 25 | github.com/tree-sitter/tree-sitter-json v0.21.1-0.20240818005659-bdd69eb8c8a5 h1:pfV3G3k7NCKqKk8THBmyuh2zA33lgYHS3GVrzRR8ry4= 26 | github.com/tree-sitter/tree-sitter-json v0.21.1-0.20240818005659-bdd69eb8c8a5/go.mod h1:GbMKRjLfk0H+PI7nLi1Sx5lHf5wCpLz9al8tQYSxpEk= 27 | github.com/tree-sitter/tree-sitter-php v0.22.9-0.20240819002312-a552625b56c1 h1:ZXZMDwE+IhUtGug4Brv6NjJWUU3rfkZBKpemf6RY8/g= 28 | github.com/tree-sitter/tree-sitter-php v0.22.9-0.20240819002312-a552625b56c1/go.mod h1:UKCLuYnJ312Mei+3cyTmGOHzn0YAnaPRECgJmHtzrqs= 29 | github.com/tree-sitter/tree-sitter-python v0.21.1-0.20240818005537-55a9b8a4fbfb h1:EXEM82lFM7JjJb6qiKZXkpIDaCcbV2obNn82ghwj9lw= 30 | github.com/tree-sitter/tree-sitter-python v0.21.1-0.20240818005537-55a9b8a4fbfb/go.mod h1:lXCF1nGG5Dr4J3BTS0ObN4xJCCICiSu/b+Xe/VqMV7g= 31 | github.com/tree-sitter/tree-sitter-ruby v0.21.1-0.20240818211811-7dbc1e2d0e2d h1:fcYCvoXdcP1uRQYXqJHRy6Hec+uKScQdKVtMwK9JeCI= 32 | github.com/tree-sitter/tree-sitter-ruby v0.21.1-0.20240818211811-7dbc1e2d0e2d/go.mod h1:T1nShQ4v5AJtozZ8YyAS4uzUtDAJj/iv4YfwXSbUHzg= 33 | github.com/tree-sitter/tree-sitter-rust v0.21.3-0.20240818005432-2b43eafe6447 h1:o9alBu1J/WjrcTKEthYtXmdkDc5OVXD+PqlvnEZ0Lzc= 34 | github.com/tree-sitter/tree-sitter-rust v0.21.3-0.20240818005432-2b43eafe6447/go.mod h1:1Oh95COkkTn6Ezp0vcMbvfhRP5gLeqqljR0BYnBzWvc= 35 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 36 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 37 | -------------------------------------------------------------------------------- /test/corpus/term/bitstring.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | single item 3 | ===================================== 4 | 5 | <<>> 6 | <<10>> 7 | <<10.0>> 8 | <<"string">> 9 | 10 | --- 11 | 12 | (source 13 | (bitstring) 14 | (bitstring 15 | (integer)) 16 | (bitstring 17 | (float)) 18 | (bitstring 19 | (string 20 | (quoted_content)))) 21 | 22 | ===================================== 23 | multiple items 24 | ===================================== 25 | 26 | << 27 | 10, 28 | 10.0, 29 | "string" 30 | >> 31 | 32 | --- 33 | 34 | (source 35 | (bitstring 36 | (integer) 37 | (float) 38 | (string 39 | (quoted_content)))) 40 | 41 | ===================================== 42 | size modifiers 43 | ===================================== 44 | 45 | <<10::4>> 46 | <<10::size(4)>> 47 | 48 | --- 49 | 50 | (source 51 | (bitstring 52 | (binary_operator 53 | (integer) 54 | (integer))) 55 | (bitstring 56 | (binary_operator 57 | (integer) 58 | (call 59 | (identifier) 60 | (arguments 61 | (integer)))))) 62 | 63 | ===================================== 64 | multiple modifiers 65 | ===================================== 66 | 67 | <<"string"::utf8-big>> 68 | <<"string"::utf16-big>> 69 | <<"string"::utf32-big>> 70 | <<10::32-little-unsigned>> 71 | <<10::integer-signed-big>> 72 | <<10.10::float-signed-native>> 73 | 74 | --- 75 | 76 | (source 77 | (bitstring 78 | (binary_operator 79 | (string 80 | (quoted_content)) 81 | (binary_operator 82 | (identifier) 83 | (identifier)))) 84 | (bitstring 85 | (binary_operator 86 | (string 87 | (quoted_content)) 88 | (binary_operator 89 | (identifier) 90 | (identifier)))) 91 | (bitstring 92 | (binary_operator 93 | (string 94 | (quoted_content)) 95 | (binary_operator 96 | (identifier) 97 | (identifier)))) 98 | (bitstring 99 | (binary_operator 100 | (integer) 101 | (binary_operator 102 | (binary_operator 103 | (integer) 104 | (identifier)) 105 | (identifier)))) 106 | (bitstring 107 | (binary_operator 108 | (integer) 109 | (binary_operator 110 | (binary_operator 111 | (identifier) 112 | (identifier)) 113 | (identifier)))) 114 | (bitstring 115 | (binary_operator 116 | (float) 117 | (binary_operator 118 | (binary_operator 119 | (identifier) 120 | (identifier)) 121 | (identifier))))) 122 | 123 | ===================================== 124 | multiple components with modifiers 125 | ===================================== 126 | 127 | <<10::8-native, "string", 3.14::float, a::8, b::binary-size(known_size)>> 128 | 129 | --- 130 | 131 | (source 132 | (bitstring 133 | (binary_operator 134 | (integer) 135 | (binary_operator 136 | (integer) 137 | (identifier))) 138 | (string 139 | (quoted_content)) 140 | (binary_operator 141 | (float) 142 | (identifier)) 143 | (binary_operator 144 | (identifier) 145 | (integer)) 146 | (binary_operator 147 | (identifier) 148 | (binary_operator 149 | (identifier) 150 | (call 151 | (identifier) 152 | (arguments 153 | (identifier))))))) 154 | 155 | ===================================== 156 | spacing 157 | ===================================== 158 | 159 | << 160 | 10 :: 8-native, 161 | b :: binary - size(known_size) 162 | >> 163 | 164 | --- 165 | 166 | (source 167 | (bitstring 168 | (binary_operator 169 | (integer) 170 | (binary_operator 171 | (integer) 172 | (identifier))) 173 | (binary_operator 174 | (identifier) 175 | (binary_operator 176 | (identifier) 177 | (call 178 | (identifier) 179 | (arguments 180 | (identifier))))))) 181 | 182 | ===================================== 183 | trailing separator 184 | ===================================== 185 | 186 | <<1,>> 187 | 188 | --- 189 | 190 | (source 191 | (bitstring 192 | (integer))) 193 | -------------------------------------------------------------------------------- /test/corpus/term/charlist.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | single line 3 | ===================================== 4 | 5 | 'Hello, 123!' 6 | 7 | --- 8 | 9 | (source 10 | (charlist 11 | (quoted_content))) 12 | 13 | ===================================== 14 | multiple lines 15 | ===================================== 16 | 17 | 'line 1 18 | line 2' 19 | 20 | --- 21 | 22 | (source 23 | (charlist 24 | (quoted_content))) 25 | 26 | ===================================== 27 | interpolation 28 | ===================================== 29 | 30 | 'hey #{name}!' 31 | 'hey #{ 32 | name 33 | }!' 34 | '##{name}#' 35 | 36 | --- 37 | 38 | (source 39 | (charlist 40 | (quoted_content) 41 | (interpolation 42 | (identifier)) 43 | (quoted_content)) 44 | (charlist 45 | (quoted_content) 46 | (interpolation 47 | (identifier)) 48 | (quoted_content)) 49 | (charlist 50 | (quoted_content) 51 | (interpolation 52 | (identifier)) 53 | (quoted_content))) 54 | 55 | ===================================== 56 | nested interpolation 57 | ===================================== 58 | 59 | 'this is #{'number #{1}'}!' 60 | 61 | --- 62 | 63 | (source 64 | (charlist 65 | (quoted_content) 66 | (interpolation 67 | (charlist 68 | (quoted_content) 69 | (interpolation 70 | (integer)))) 71 | (quoted_content))) 72 | 73 | ===================================== 74 | escape sequence 75 | ===================================== 76 | 77 | '_\'_\n_\t_\r_\e_\\_\1_\x3f_\u0065\u0301_' 78 | 79 | --- 80 | 81 | (source 82 | (charlist 83 | (quoted_content) 84 | (escape_sequence) 85 | (quoted_content) 86 | (escape_sequence) 87 | (quoted_content) 88 | (escape_sequence) 89 | (quoted_content) 90 | (escape_sequence) 91 | (quoted_content) 92 | (escape_sequence) 93 | (quoted_content) 94 | (escape_sequence) 95 | (quoted_content) 96 | (escape_sequence) 97 | (quoted_content) 98 | (escape_sequence) 99 | (quoted_content) 100 | (escape_sequence) 101 | (escape_sequence) 102 | (quoted_content))) 103 | 104 | ===================================== 105 | escaped interpolation 106 | ===================================== 107 | 108 | '\#{1}' 109 | 110 | --- 111 | 112 | (source 113 | (charlist 114 | (escape_sequence) 115 | (quoted_content))) 116 | 117 | ===================================== 118 | heredoc / charlist 119 | ===================================== 120 | 121 | ''' 122 | text 123 | with 'quotes' 124 | ''' 125 | 126 | --- 127 | 128 | (source 129 | (charlist 130 | (quoted_content))) 131 | 132 | ===================================== 133 | heredoc / interpolation 134 | ===================================== 135 | 136 | ''' 137 | hey #{name}! 138 | ''' 139 | 140 | --- 141 | 142 | (source 143 | (charlist 144 | (quoted_content) 145 | (interpolation 146 | (identifier)) 147 | (quoted_content))) 148 | 149 | ===================================== 150 | heredoc / nested interpolation 151 | ===================================== 152 | 153 | ''' 154 | this is #{ 155 | ''' 156 | number #{1} 157 | ''' 158 | }! 159 | ''' 160 | 161 | --- 162 | 163 | (source 164 | (charlist 165 | (quoted_content) 166 | (interpolation 167 | (charlist 168 | (quoted_content) 169 | (interpolation 170 | (integer)) 171 | (quoted_content))) 172 | (quoted_content))) 173 | 174 | ===================================== 175 | heredoc / escaped delimiter 176 | ===================================== 177 | 178 | ''' 179 | \''' 180 | ''' 181 | 182 | ''' 183 | \'\'\' 184 | ''' 185 | 186 | --- 187 | 188 | (source 189 | (charlist 190 | (quoted_content) 191 | (escape_sequence) 192 | (quoted_content)) 193 | (charlist 194 | (quoted_content) 195 | (escape_sequence) 196 | (escape_sequence) 197 | (escape_sequence) 198 | (quoted_content))) 199 | 200 | ===================================== 201 | heredoc / escaped interpolation 202 | ===================================== 203 | 204 | ''' 205 | \#{1} 206 | ''' 207 | 208 | --- 209 | 210 | (source 211 | (charlist 212 | (quoted_content) 213 | (escape_sequence) 214 | (quoted_content))) 215 | -------------------------------------------------------------------------------- /test/corpus/expression/block.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | empty 3 | ===================================== 4 | 5 | () 6 | 7 | --- 8 | 9 | (source 10 | (block)) 11 | 12 | ===================================== 13 | single expression 14 | ===================================== 15 | 16 | (1) 17 | 18 | --- 19 | 20 | (source 21 | (block 22 | (integer))) 23 | 24 | ===================================== 25 | multiple expressions separated by newline 26 | ===================================== 27 | 28 | ( 29 | 1 30 | 2 31 | ) 32 | 33 | --- 34 | 35 | (source 36 | (block 37 | (integer) 38 | (integer))) 39 | 40 | ===================================== 41 | multiple expressions separated by semicolon 42 | ===================================== 43 | 44 | (1;2) 45 | 46 | --- 47 | 48 | (source 49 | (block 50 | (integer) 51 | (integer))) 52 | 53 | ===================================== 54 | multiple expressions separated by mixed separators 55 | ===================================== 56 | 57 | ( 58 | 1 59 | 60 | ; 61 | 62 | 2 63 | ) 64 | 65 | --- 66 | 67 | (source 68 | (block 69 | (integer) 70 | (integer))) 71 | 72 | ===================================== 73 | leading semicolon 74 | ===================================== 75 | 76 | (;1;2) 77 | 78 | --- 79 | 80 | (source 81 | (block 82 | (integer) 83 | (integer))) 84 | 85 | ===================================== 86 | trailing semicolon 87 | ===================================== 88 | 89 | (1;2;) 90 | 91 | --- 92 | 93 | (source 94 | (block 95 | (integer) 96 | (integer))) 97 | 98 | ===================================== 99 | stab clause / multiple clauses 100 | ===================================== 101 | 102 | (x -> x; y -> y 103 | z -> z) 104 | 105 | --- 106 | 107 | (source 108 | (block 109 | (stab_clause 110 | (arguments 111 | (identifier)) 112 | (body 113 | (identifier))) 114 | (stab_clause 115 | (arguments 116 | (identifier)) 117 | (body 118 | (identifier))) 119 | (stab_clause 120 | (arguments 121 | (identifier)) 122 | (body 123 | (identifier))))) 124 | 125 | ===================================== 126 | stab clause / multiple arguments 127 | ===================================== 128 | 129 | (x, y, z -> x) 130 | ((x, y, z) -> x) 131 | 132 | --- 133 | 134 | (source 135 | (block 136 | (stab_clause 137 | (arguments 138 | (identifier) 139 | (identifier) 140 | (identifier)) 141 | (body 142 | (identifier)))) 143 | (block 144 | (stab_clause 145 | (arguments 146 | (identifier) 147 | (identifier) 148 | (identifier)) 149 | (body 150 | (identifier))))) 151 | 152 | ===================================== 153 | stab clause / guard 154 | ===================================== 155 | 156 | (x, y when x == y -> 1) 157 | ((x, y when x == y -> 1)) 158 | ((x, y when x == y) -> 1) 159 | (x, y when x, z -> 1) 160 | ((x, y when x, z -> 1)) 161 | ((x, y when x, z) -> 1) 162 | 163 | --- 164 | 165 | (source 166 | (block 167 | (stab_clause 168 | (binary_operator 169 | (arguments 170 | (identifier) 171 | (identifier)) 172 | (binary_operator 173 | (identifier) 174 | (identifier))) 175 | (body 176 | (integer)))) 177 | (block 178 | (block 179 | (stab_clause 180 | (binary_operator 181 | (arguments 182 | (identifier) 183 | (identifier)) 184 | (binary_operator 185 | (identifier) 186 | (identifier))) 187 | (body 188 | (integer))))) 189 | (block 190 | (stab_clause 191 | (arguments 192 | (identifier) 193 | (binary_operator 194 | (identifier) 195 | (binary_operator 196 | (identifier) 197 | (identifier)))) 198 | (body 199 | (integer)))) 200 | (block 201 | (stab_clause 202 | (arguments 203 | (identifier) 204 | (binary_operator 205 | (identifier) 206 | (identifier)) 207 | (identifier)) 208 | (body 209 | (integer)))) 210 | (block 211 | (block 212 | (stab_clause 213 | (arguments 214 | (identifier) 215 | (binary_operator 216 | (identifier) 217 | (identifier)) 218 | (identifier)) 219 | (body 220 | (integer))))) 221 | (block 222 | (stab_clause 223 | (arguments 224 | (identifier) 225 | (binary_operator 226 | (identifier) 227 | (identifier)) 228 | (identifier)) 229 | (body 230 | (integer))))) 231 | -------------------------------------------------------------------------------- /test/corpus/term/keyword_list.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | simple literal 3 | ===================================== 4 | 5 | [a: 1, a_b@12?: 2, A_B@12!: 3, Mod: 4, __struct__: 5] 6 | 7 | --- 8 | 9 | (source 10 | (list 11 | (keywords 12 | (pair 13 | (keyword) 14 | (integer)) 15 | (pair 16 | (keyword) 17 | (integer)) 18 | (pair 19 | (keyword) 20 | (integer)) 21 | (pair 22 | (keyword) 23 | (integer)) 24 | (pair 25 | (keyword) 26 | (integer))))) 27 | 28 | ===================================== 29 | trailing separator 30 | ===================================== 31 | 32 | [a: 1,] 33 | 34 | --- 35 | 36 | (source 37 | (list 38 | (keywords 39 | (pair 40 | (keyword) 41 | (integer))))) 42 | 43 | ===================================== 44 | with leading items 45 | ===================================== 46 | 47 | [1, {:c, 1}, a: 1, b: 2] 48 | 49 | --- 50 | 51 | (source 52 | (list 53 | (integer) 54 | (tuple 55 | (atom) 56 | (integer)) 57 | (keywords 58 | (pair 59 | (keyword) 60 | (integer)) 61 | (pair 62 | (keyword) 63 | (integer))))) 64 | 65 | ===================================== 66 | operator key 67 | ===================================== 68 | 69 | [~~~: 1, ==: 2, >: 3] 70 | 71 | --- 72 | 73 | (source 74 | (list 75 | (keywords 76 | (pair 77 | (keyword) 78 | (integer)) 79 | (pair 80 | (keyword) 81 | (integer)) 82 | (pair 83 | (keyword) 84 | (integer))))) 85 | 86 | ===================================== 87 | special atom key 88 | ===================================== 89 | 90 | [...: 1, %{}: 2, {}: 3, %: 4, <<>>: 5, ..//: 6] 91 | 92 | --- 93 | 94 | (source 95 | (list 96 | (keywords 97 | (pair 98 | (keyword) 99 | (integer)) 100 | (pair 101 | (keyword) 102 | (integer)) 103 | (pair 104 | (keyword) 105 | (integer)) 106 | (pair 107 | (keyword) 108 | (integer)) 109 | (pair 110 | (keyword) 111 | (integer)) 112 | (pair 113 | (keyword) 114 | (integer))))) 115 | 116 | ===================================== 117 | reserved token key 118 | ===================================== 119 | 120 | [not: 1, and: 2] 121 | [nil: 1, true: 2] 122 | 123 | --- 124 | 125 | (source 126 | (list 127 | (keywords 128 | (pair 129 | (keyword) 130 | (integer)) 131 | (pair 132 | (keyword) 133 | (integer)))) 134 | (list 135 | (keywords 136 | (pair 137 | (keyword) 138 | (integer)) 139 | (pair 140 | (keyword) 141 | (integer))))) 142 | 143 | ===================================== 144 | quoted key 145 | ===================================== 146 | 147 | [ 148 | "key1 ?? !! ' \n": 1, 149 | 'key2 ?? !! " \n': 2 150 | ] 151 | 152 | --- 153 | 154 | (source 155 | (list 156 | (keywords 157 | (pair 158 | (quoted_keyword 159 | (quoted_content) 160 | (escape_sequence)) 161 | (integer)) 162 | (pair 163 | (quoted_keyword 164 | (quoted_content) 165 | (escape_sequence)) 166 | (integer))))) 167 | 168 | ===================================== 169 | key interpolation 170 | ===================================== 171 | 172 | [ 173 | "hey #{name}!": 1, 174 | 'hey #{name}!': 2 175 | ] 176 | 177 | --- 178 | 179 | (source 180 | (list 181 | (keywords 182 | (pair 183 | (quoted_keyword 184 | (quoted_content) 185 | (interpolation 186 | (identifier)) 187 | (quoted_content)) 188 | (integer)) 189 | (pair 190 | (quoted_keyword 191 | (quoted_content) 192 | (interpolation 193 | (identifier)) 194 | (quoted_content)) 195 | (integer))))) 196 | 197 | ===================================== 198 | [field names] 199 | ===================================== 200 | 201 | [a: 1, b: 2] 202 | --- 203 | 204 | (source 205 | (list 206 | (keywords 207 | (pair 208 | key: (keyword) 209 | value: (integer)) 210 | (pair 211 | key: (keyword) 212 | value: (integer))))) 213 | 214 | ===================================== 215 | [error] with trailing items 216 | ===================================== 217 | 218 | [a: 1, b: 2, 1 => 1] 219 | 220 | --- 221 | 222 | (source 223 | (list 224 | (ERROR 225 | (keywords 226 | (pair 227 | (keyword) 228 | (integer)) 229 | (pair 230 | (keyword) 231 | (integer)))) 232 | (binary_operator 233 | (integer) 234 | (integer)))) 235 | -------------------------------------------------------------------------------- /test/corpus/term/struct.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | empty 3 | ===================================== 4 | 5 | %User{} 6 | 7 | --- 8 | 9 | (source 10 | (map 11 | (struct 12 | (alias)))) 13 | 14 | ===================================== 15 | from keywords 16 | ===================================== 17 | 18 | %User{a: 1, b: 2} 19 | 20 | --- 21 | 22 | (source 23 | (map 24 | (struct 25 | (alias)) 26 | (map_content 27 | (keywords 28 | (pair 29 | (keyword) 30 | (integer)) 31 | (pair 32 | (keyword) 33 | (integer)))))) 34 | 35 | ===================================== 36 | from arrow entries 37 | ===================================== 38 | 39 | %User{:a => 1, "b" => 2, c => 3} 40 | 41 | --- 42 | 43 | (source 44 | (map 45 | (struct 46 | (alias)) 47 | (map_content 48 | (binary_operator 49 | (atom) 50 | (integer)) 51 | (binary_operator 52 | (string 53 | (quoted_content)) 54 | (integer)) 55 | (binary_operator 56 | (identifier) 57 | (integer))))) 58 | 59 | ===================================== 60 | from both arrow entries and keywords 61 | ===================================== 62 | 63 | %User{"a" => 1, b: 2, c: 3} 64 | 65 | --- 66 | 67 | (source 68 | (map 69 | (struct 70 | (alias)) 71 | (map_content 72 | (binary_operator 73 | (string 74 | (quoted_content)) 75 | (integer)) 76 | (keywords 77 | (pair 78 | (keyword) 79 | (integer)) 80 | (pair 81 | (keyword) 82 | (integer)))))) 83 | 84 | ===================================== 85 | trailing separator 86 | ===================================== 87 | 88 | %User{"a" => 1,} 89 | 90 | --- 91 | 92 | (source 93 | (map 94 | (struct 95 | (alias)) 96 | (map_content 97 | (binary_operator 98 | (string 99 | (quoted_content)) 100 | (integer))))) 101 | 102 | ===================================== 103 | update syntax 104 | ===================================== 105 | 106 | %User{user | name: "Jane", email: "jane@example.com"} 107 | %User{user | "name" => "Jane"} 108 | 109 | --- 110 | 111 | (source 112 | (map 113 | (struct (alias)) 114 | (map_content 115 | (binary_operator 116 | (identifier) 117 | (keywords 118 | (pair 119 | (keyword) 120 | (string 121 | (quoted_content))) 122 | (pair 123 | (keyword) 124 | (string 125 | (quoted_content))))))) 126 | (map 127 | (struct 128 | (alias)) 129 | (map_content 130 | (binary_operator 131 | (identifier) 132 | (binary_operator 133 | (string 134 | (quoted_content)) 135 | (string 136 | (quoted_content))))))) 137 | 138 | ===================================== 139 | unused struct identifier 140 | ===================================== 141 | 142 | %_{} 143 | 144 | --- 145 | 146 | (source 147 | (map 148 | (struct 149 | (identifier)))) 150 | 151 | ===================================== 152 | matching struct identifier 153 | ===================================== 154 | 155 | %name{} 156 | 157 | --- 158 | 159 | (source 160 | (map 161 | (struct 162 | (identifier)))) 163 | 164 | ===================================== 165 | pinned struct identifier 166 | ===================================== 167 | 168 | %^name{} 169 | 170 | --- 171 | 172 | (source 173 | (map 174 | (struct 175 | (unary_operator 176 | (identifier))))) 177 | 178 | ===================================== 179 | with special identifier 180 | ===================================== 181 | 182 | %__MODULE__{} 183 | %__MODULE__.Child{} 184 | 185 | --- 186 | 187 | (source 188 | (map 189 | (struct 190 | (identifier))) 191 | (map 192 | (struct 193 | (dot 194 | (identifier) 195 | (alias))))) 196 | 197 | ===================================== 198 | with atom 199 | ===================================== 200 | 201 | %:"Elixir.Mod"{} 202 | 203 | --- 204 | 205 | (source 206 | (map 207 | (struct 208 | (quoted_atom 209 | (quoted_content))))) 210 | 211 | ===================================== 212 | with call 213 | ===================================== 214 | 215 | %fun(){} 216 | %Mod.fun(){} 217 | %fun.(){} 218 | 219 | --- 220 | 221 | (source 222 | (map 223 | (struct 224 | (call 225 | (identifier) 226 | (arguments)))) 227 | (map 228 | (struct 229 | (call 230 | (dot 231 | (alias) 232 | (identifier)) 233 | (arguments)))) 234 | (map 235 | (struct 236 | (call 237 | (dot 238 | (identifier)) 239 | (arguments))))) 240 | -------------------------------------------------------------------------------- /test/highlight/literals.ex: -------------------------------------------------------------------------------- 1 | 1234 ; 01234 2 | # ^ number 3 | # ^ punctuation.delimiter 4 | # ^ number 5 | 6 | 1.23 ; 1.23e10 ; 1.23e-10 7 | # ^ number 8 | # ^ number 9 | # ^ number 10 | 11 | 0xab ; 0o171 ; 0b01001 12 | # ^ number 13 | # ^ number 14 | # ^ number 15 | 16 | 100_000_000 ; 0b1111_0000 17 | # ^ number 18 | # ^ number 19 | 20 | ?a ; ?1 ; ?\n ; ?\s ; ?\c ; ? ; ?, 21 | # <- constant 22 | # ^ constant 23 | # ^ constant 24 | # ^ constant 25 | # ^ constant 26 | # ^ constant 27 | # ^ constant 28 | # ^ constant 29 | 30 | true ; false ; nil 31 | # ^ constant 32 | # ^ constant 33 | # ^ constant 34 | 35 | :this ; :that 36 | # ^ string.special.symbol 37 | # ^ string.special.symbol 38 | 39 | :'complex atom' 40 | # ^ string.special.symbol 41 | 42 | :"with' \" 'quotes \u0301" 43 | # ^ string.special.symbol 44 | # ^ string.escape 45 | # ^ string.escape 46 | # ^ string.special.symbol 47 | # ^ string.escape 48 | # ^ string.escape 49 | 50 | :"with #{1 + 1} interpol" 51 | # ^ punctuation.special 52 | # ^ number 53 | # ^ operator 54 | # ^ number 55 | # ^ punctuation.special 56 | 57 | :... ; :<<>> ; :%{} ; :% ; :{} 58 | # ^ string.special.symbol 59 | # ^ string.special.symbol 60 | # ^ string.special.symbol 61 | # ^ string.special.symbol 62 | # ^ string.special.symbol 63 | 64 | :++ ; :-- ; :* ; :~~~ ; ::: 65 | # ^ string.special.symbol 66 | # ^ string.special.symbol 67 | # ^ string.special.symbol 68 | # ^ string.special.symbol 69 | # ^ string.special.symbol 70 | 71 | :% ; :. ; :<- 72 | # <- string.special.symbol 73 | # ^ string.special.symbol 74 | # ^ string.special.symbol 75 | 76 | "simple string" 77 | # ^ string 78 | # ^ string 79 | 80 | "with \x{ff} code" 81 | # ^ string 82 | # ^ string.escape 83 | # ^ string.escape 84 | # ^ string 85 | 86 | "with \7 \016 \t \\s" 87 | # ^ string 88 | # ^ string.escape 89 | # ^ string.escape 90 | # ^ string 91 | # ^ string.escape 92 | # ^ string 93 | # ^ string.escape 94 | # ^ string 95 | 96 | "with #{inspect "inner"} interpolation" 97 | # ^ string 98 | # ^ punctuation.special 99 | # ^ function 100 | # ^ string 101 | # ^ punctuation.special 102 | # ^ string 103 | 104 | "Multiline 105 | string" 106 | # <- string 107 | # ^ string 108 | 109 | """ 110 | Heredoc 111 | """ 112 | # ^ string 113 | 114 | 'charlist' 115 | # ^ string 116 | 117 | ~s(string sigil #{1}) 118 | # <- string 119 | # ^ string 120 | # ^ string 121 | # ^ number 122 | 123 | ~S/string #{1}/ 124 | # ^ string 125 | 126 | ~r/x?/iu 127 | # <- string.regex 128 | # ^ string.regex 129 | # ^ string.regex 130 | 131 | ~R{noescapes\}[a-z]*} 132 | # ^ string.regex 133 | # ^ string.escape 134 | # ^ string.regex 135 | 136 | ~w(hello #{ ["has" <> "123", '\c\d', "\123 interpol" | []] } world)s 137 | #<- string.special 138 | # ^ string.special 139 | # ^ punctuation.special 140 | # ^ punctuation.bracket 141 | # ^ string 142 | # ^ operator 143 | # ^ string 144 | # ^ punctuation.delimiter 145 | # ^ string 146 | # ^ string.escape 147 | # ^ string.escape 148 | # ^ string 149 | # ^ punctuation.delimiter 150 | # ^ string 151 | # ^ string.escape 152 | # ^ string 153 | # ^ operator 154 | # ^ punctuation.bracket 155 | # ^ punctuation.bracket 156 | # ^ punctuation.bracket 157 | # ^ punctuation.special 158 | # ^ string.special 159 | 160 | ~w/yoo \\ \/ =)/ 161 | # ^ string.special 162 | # ^ string.escape 163 | # ^ string.escape 164 | # ^ string.special 165 | 166 | ~W/yoo \\ \/ =)/ 167 | # ^ string.special 168 | # ^ string.special 169 | # ^ string.escape 170 | # ^ string.special 171 | 172 | ~D/2020-01-01/d 173 | # ^ string.special 174 | -------------------------------------------------------------------------------- /test/corpus/term/string.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | empty 3 | ===================================== 4 | 5 | "" 6 | 7 | --- 8 | 9 | (source 10 | (string)) 11 | 12 | ===================================== 13 | single line 14 | ===================================== 15 | 16 | "Hello, 123!" 17 | 18 | --- 19 | 20 | (source 21 | (string 22 | (quoted_content))) 23 | 24 | ===================================== 25 | multiple lines 26 | ===================================== 27 | 28 | "line 1 29 | line 2" 30 | 31 | --- 32 | 33 | (source 34 | (string 35 | (quoted_content))) 36 | 37 | ===================================== 38 | interpolation 39 | ===================================== 40 | 41 | "hey #{name}!" 42 | "hey #{ 43 | name 44 | }!" 45 | "##{name}#" 46 | 47 | --- 48 | 49 | (source 50 | (string 51 | (quoted_content) 52 | (interpolation 53 | (identifier)) 54 | (quoted_content)) 55 | (string 56 | (quoted_content) 57 | (interpolation 58 | (identifier)) 59 | (quoted_content)) 60 | (string 61 | (quoted_content) 62 | (interpolation 63 | (identifier)) 64 | (quoted_content))) 65 | 66 | ===================================== 67 | nested interpolation 68 | ===================================== 69 | 70 | "this is #{"number #{1}"}!" 71 | 72 | --- 73 | 74 | (source 75 | (string 76 | (quoted_content) 77 | (interpolation 78 | (string 79 | (quoted_content) 80 | (interpolation 81 | (integer)))) 82 | (quoted_content))) 83 | 84 | ===================================== 85 | empty interpolation 86 | ===================================== 87 | 88 | "#{}" 89 | 90 | --- 91 | 92 | (source 93 | (string 94 | (interpolation))) 95 | 96 | ===================================== 97 | escape sequence 98 | ===================================== 99 | 100 | "_\"_\n_\t_\r_\e_\\_\1_\x3f_\u0065\u0301_" 101 | 102 | --- 103 | 104 | (source 105 | (string 106 | (quoted_content) 107 | (escape_sequence) 108 | (quoted_content) 109 | (escape_sequence) 110 | (quoted_content) 111 | (escape_sequence) 112 | (quoted_content) 113 | (escape_sequence) 114 | (quoted_content) 115 | (escape_sequence) 116 | (quoted_content) 117 | (escape_sequence) 118 | (quoted_content) 119 | (escape_sequence) 120 | (quoted_content) 121 | (escape_sequence) 122 | (quoted_content) 123 | (escape_sequence) 124 | (escape_sequence) 125 | (quoted_content))) 126 | 127 | ===================================== 128 | escaped interpolation 129 | ===================================== 130 | 131 | "\#{1}" 132 | 133 | --- 134 | 135 | (source 136 | (string 137 | (escape_sequence) 138 | (quoted_content))) 139 | 140 | ===================================== 141 | heredoc / string 142 | ===================================== 143 | 144 | """ 145 | text 146 | with "quotes" 147 | """ 148 | 149 | --- 150 | 151 | (source 152 | (string 153 | (quoted_content))) 154 | 155 | ===================================== 156 | heredoc / interpolation 157 | ===================================== 158 | 159 | """ 160 | hey #{name}! 161 | """ 162 | 163 | --- 164 | 165 | (source 166 | (string 167 | (quoted_content) 168 | (interpolation 169 | (identifier)) 170 | (quoted_content))) 171 | 172 | ===================================== 173 | heredoc / nested interpolation 174 | ===================================== 175 | 176 | """ 177 | this is #{ 178 | """ 179 | number #{1} 180 | """ 181 | }! 182 | """ 183 | 184 | --- 185 | 186 | (source 187 | (string 188 | (quoted_content) 189 | (interpolation 190 | (string 191 | (quoted_content) 192 | (interpolation 193 | (integer)) 194 | (quoted_content))) 195 | (quoted_content))) 196 | 197 | ===================================== 198 | heredoc / delimiter in the middle 199 | ===================================== 200 | 201 | """ 202 | hey """ 203 | """ 204 | 205 | --- 206 | 207 | (source 208 | (string 209 | (quoted_content))) 210 | 211 | ===================================== 212 | heredoc / escaped newline (ignored) 213 | ===================================== 214 | 215 | """ 216 | hey \ 217 | """ 218 | 219 | """ 220 | hey \ 221 | """ 222 | 223 | """ 224 | hey \ 225 | there 226 | """ 227 | 228 | --- 229 | 230 | (source 231 | (string 232 | (quoted_content)) 233 | (string 234 | (quoted_content)) 235 | (string 236 | (quoted_content))) 237 | 238 | ===================================== 239 | heredoc / escaped delimiter 240 | ===================================== 241 | 242 | """ 243 | \""" 244 | """ 245 | 246 | """ 247 | \"\"\" 248 | """ 249 | 250 | --- 251 | 252 | (source 253 | (string 254 | (quoted_content) 255 | (escape_sequence) 256 | (quoted_content)) 257 | (string 258 | (quoted_content) 259 | (escape_sequence) 260 | (escape_sequence) 261 | (escape_sequence) 262 | (quoted_content))) 263 | 264 | ===================================== 265 | heredoc / escaped interpolation 266 | ===================================== 267 | 268 | """ 269 | \#{1} 270 | """ 271 | 272 | --- 273 | 274 | (source 275 | (string 276 | (quoted_content) 277 | (escape_sequence) 278 | (quoted_content))) 279 | -------------------------------------------------------------------------------- /queries/highlights.scm: -------------------------------------------------------------------------------- 1 | ; Punctuation 2 | 3 | [ 4 | "%" 5 | ] @punctuation 6 | 7 | [ 8 | "," 9 | ";" 10 | ] @punctuation.delimiter 11 | 12 | [ 13 | "(" 14 | ")" 15 | "[" 16 | "]" 17 | "{" 18 | "}" 19 | "<<" 20 | ">>" 21 | ] @punctuation.bracket 22 | 23 | ; Literals 24 | 25 | [ 26 | (boolean) 27 | (nil) 28 | ] @constant 29 | 30 | [ 31 | (integer) 32 | (float) 33 | ] @number 34 | 35 | (char) @constant 36 | 37 | ; Identifiers 38 | 39 | ; * regular 40 | (identifier) @variable 41 | 42 | ; * unused 43 | ( 44 | (identifier) @comment.unused 45 | (#match? @comment.unused "^_") 46 | ) 47 | 48 | ; * special 49 | ( 50 | (identifier) @constant.builtin 51 | (#any-of? @constant.builtin "__MODULE__" "__DIR__" "__ENV__" "__CALLER__" "__STACKTRACE__") 52 | ) 53 | 54 | ; Comment 55 | 56 | (comment) @comment 57 | 58 | ; Quoted content 59 | 60 | (interpolation "#{" @punctuation.special "}" @punctuation.special) @embedded 61 | 62 | (escape_sequence) @string.escape 63 | 64 | [ 65 | (string) 66 | (charlist) 67 | ] @string 68 | 69 | [ 70 | (atom) 71 | (quoted_atom) 72 | (keyword) 73 | (quoted_keyword) 74 | ] @string.special.symbol 75 | 76 | ; Note that we explicitly target sigil quoted start/end, so they are not overridden by delimiters 77 | 78 | (sigil 79 | (sigil_name) @__name__ 80 | quoted_start: _ @string.special 81 | quoted_end: _ @string.special) @string.special 82 | 83 | (sigil 84 | (sigil_name) @__name__ 85 | quoted_start: _ @string 86 | quoted_end: _ @string 87 | (#match? @__name__ "^[sS]$")) @string 88 | 89 | (sigil 90 | (sigil_name) @__name__ 91 | quoted_start: _ @string.regex 92 | quoted_end: _ @string.regex 93 | (#match? @__name__ "^[rR]$")) @string.regex 94 | 95 | ; Calls 96 | 97 | ; * local function call 98 | (call 99 | target: (identifier) @function) 100 | 101 | ; * remote function call 102 | (call 103 | target: (dot 104 | right: (identifier) @function)) 105 | 106 | ; * field without parentheses or block 107 | (call 108 | target: (dot 109 | right: (identifier) @property) 110 | .) 111 | 112 | ; * remote call without parentheses or block (overrides above) 113 | (call 114 | target: (dot 115 | left: [ 116 | (alias) 117 | (atom) 118 | ] 119 | right: (identifier) @function) 120 | .) 121 | 122 | ; * definition keyword 123 | (call 124 | target: (identifier) @keyword 125 | (#any-of? @keyword "def" "defdelegate" "defexception" "defguard" "defguardp" "defimpl" "defmacro" "defmacrop" "defmodule" "defn" "defnp" "defoverridable" "defp" "defprotocol" "defstruct")) 126 | 127 | ; * kernel or special forms keyword 128 | (call 129 | target: (identifier) @keyword 130 | (#any-of? @keyword "alias" "case" "cond" "for" "if" "import" "quote" "raise" "receive" "require" "reraise" "super" "throw" "try" "unless" "unquote" "unquote_splicing" "use" "with")) 131 | 132 | ; * just identifier in function definition 133 | (call 134 | target: (identifier) @keyword 135 | (arguments 136 | [ 137 | (identifier) @function 138 | (binary_operator 139 | left: (identifier) @function 140 | operator: "when") 141 | ]) 142 | (#any-of? @keyword "def" "defdelegate" "defguard" "defguardp" "defmacro" "defmacrop" "defn" "defnp" "defp")) 143 | 144 | ; * pipe into identifier (function call) 145 | (binary_operator 146 | operator: "|>" 147 | right: (identifier) @function) 148 | 149 | ; * pipe into identifier (definition) 150 | (call 151 | target: (identifier) @keyword 152 | (arguments 153 | (binary_operator 154 | operator: "|>" 155 | right: (identifier) @variable)) 156 | (#any-of? @keyword "def" "defdelegate" "defguard" "defguardp" "defmacro" "defmacrop" "defn" "defnp" "defp")) 157 | 158 | ; * pipe into field without parentheses (function call) 159 | (binary_operator 160 | operator: "|>" 161 | right: (call 162 | target: (dot 163 | right: (identifier) @function))) 164 | 165 | ; Operators 166 | 167 | ; * capture operand 168 | (unary_operator 169 | operator: "&" 170 | operand: (integer) @operator) 171 | 172 | (operator_identifier) @operator 173 | 174 | (unary_operator 175 | operator: _ @operator) 176 | 177 | (binary_operator 178 | operator: _ @operator) 179 | 180 | (dot 181 | operator: _ @operator) 182 | 183 | (stab_clause 184 | operator: _ @operator) 185 | 186 | ; * module attribute 187 | (unary_operator 188 | operator: "@" @attribute 189 | operand: [ 190 | (identifier) @attribute 191 | (call 192 | target: (identifier) @attribute) 193 | (boolean) @attribute 194 | (nil) @attribute 195 | ]) 196 | 197 | ; * doc string 198 | (unary_operator 199 | operator: "@" @comment.doc 200 | operand: (call 201 | target: (identifier) @comment.doc.__attribute__ 202 | (arguments 203 | [ 204 | (string) @comment.doc 205 | (charlist) @comment.doc 206 | (sigil 207 | quoted_start: _ @comment.doc 208 | quoted_end: _ @comment.doc) @comment.doc 209 | (boolean) @comment.doc 210 | ])) 211 | (#any-of? @comment.doc.__attribute__ "moduledoc" "typedoc" "doc")) 212 | 213 | ; Module 214 | 215 | (alias) @module 216 | 217 | (call 218 | target: (dot 219 | left: (atom) @module)) 220 | 221 | ; Reserved keywords 222 | 223 | ["when" "and" "or" "not" "in" "not in" "fn" "do" "end" "catch" "rescue" "after" "else"] @keyword 224 | -------------------------------------------------------------------------------- /test/highlight/kernel.ex: -------------------------------------------------------------------------------- 1 | for x <- 1..10, x < 5, do: {x, x} 2 | # <- keyword 3 | # ^ variable 4 | # ^ operator 5 | # ^ number 6 | # ^ operator 7 | # ^ number 8 | # ^ punctuation.delimiter 9 | # ^ variable 10 | # ^ operator 11 | # ^ number 12 | # ^ punctuation.delimiter 13 | # ^ string.special.symbol 14 | # ^ punctuation.bracket 15 | # ^ variable 16 | # ^ punctuation.delimiter 17 | # ^ variable 18 | # ^ punctuation.bracket 19 | 20 | for << <> <- pixels >> do 21 | # <- keyword 22 | # ^ punctuation.bracket 23 | # ^ punctuation.bracket 24 | # ^ variable 25 | # ^ operator 26 | # ^ number 27 | # ^ punctuation.delimiter 28 | # ^ variable 29 | # ^ operator 30 | # ^ number 31 | # ^ punctuation.delimiter 32 | # ^ variable 33 | # ^ operator 34 | # ^ number 35 | # ^ punctuation.delimiter 36 | # ^ variable 37 | # ^ operator 38 | # ^ function 39 | # ^ punctuation.bracket 40 | # ^ number 41 | # ^ punctuation.bracket 42 | # ^ punctuation.bracket 43 | # ^ operator 44 | # ^ variable 45 | # ^ punctuation.bracket 46 | # ^ keyword 47 | end 48 | # <- keyword 49 | 50 | if :this do 51 | # <- keyword 52 | # ^ string.special.symbol 53 | # ^ keyword 54 | :that 55 | # ^ string.special.symbol 56 | else 57 | # <- keyword 58 | :otherwise 59 | # ^ string.special.symbol 60 | end 61 | # <- keyword 62 | 63 | receive do 64 | # ^ keyword 65 | # ^ keyword 66 | {:EXIT, _} -> :done 67 | # <- punctuation.bracket 68 | # ^ string.special.symbol 69 | # ^ punctuation.delimiter 70 | # ^ comment.unused 71 | # ^ punctuation.bracket 72 | # ^ operator 73 | # ^ string.special.symbol 74 | { ^pid, :_ } -> nil 75 | # <- punctuation.bracket 76 | # ^ operator 77 | # ^ variable 78 | # ^ punctuation.delimiter 79 | # ^ string.special.symbol 80 | # ^ punctuation.bracket 81 | # ^ operator 82 | # ^ constant 83 | after 100 -> :no_luck 84 | # ^ keyword 85 | # ^ number 86 | # ^ operator 87 | # ^ string.special.symbol 88 | end 89 | # <- keyword 90 | 91 | case __ENV__.line do 92 | # ^ keyword 93 | # ^ constant.builtin 94 | # ^ operator 95 | # ^ property 96 | # ^ keyword 97 | x when is_integer(x) -> x 98 | # <- variable 99 | # ^ keyword 100 | # ^ function 101 | # ^ punctuation.bracket 102 | # ^ variable 103 | # ^ punctuation.bracket 104 | # ^ operator 105 | # ^ variable 106 | x when x in 1..12 -> -x 107 | # <- variable 108 | # ^ keyword 109 | # ^ variable 110 | # ^ keyword 111 | # ^ number 112 | # ^ operator 113 | # ^ number 114 | # ^ operator 115 | # ^ operator 116 | # ^ variable 117 | end 118 | # <- keyword 119 | 120 | cond do 121 | # <- keyword 122 | # ^ keyword 123 | false -> "too bad" 124 | # ^ constant 125 | # ^ operator 126 | # ^ string 127 | 4 > 5 -> "oops" 128 | # <- number 129 | # ^ operator 130 | # ^ number 131 | # ^ operator 132 | # ^ string 133 | true -> nil 134 | # ^ constant 135 | # ^ operator 136 | # ^ constant 137 | end 138 | # <- keyword 139 | 140 | raise RuntimeError, message: "This is not an error" 141 | # ^ keyword 142 | # ^ module 143 | # ^ punctuation.delimiter 144 | # ^ string.special.symbol 145 | # ^ string 146 | 147 | import Kernel, except: [spawn: 1, +: 2, /: 2, Unless: 2] 148 | # ^ keyword 149 | # ^ module 150 | # ^ punctuation.delimiter 151 | # ^ string.special.symbol 152 | # ^ punctuation.bracket 153 | # ^ string.special.symbol 154 | # ^ number 155 | # ^ punctuation.delimiter 156 | # ^ string.special.symbol 157 | # ^ number 158 | # ^ punctuation.delimiter 159 | # ^ string.special.symbol 160 | # ^ number 161 | # ^ punctuation.delimiter 162 | # ^ string.special.symbol 163 | # ^ number 164 | # ^ punctuation.bracket 165 | 166 | alias Long.Module.Name, as: N0men123_and4 167 | # ^ keyword 168 | # ^ module 169 | # ^ punctuation.delimiter 170 | # ^ string.special.symbol 171 | # ^ module 172 | 173 | use Bitwise 174 | # ^ keyword 175 | # ^ module 176 | -------------------------------------------------------------------------------- /test/corpus/expression/sigil.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | simple literal 3 | ===================================== 4 | 5 | ~s(content) 6 | ~r{content} 7 | ~w[content] 8 | ~a 9 | ~b"content" 10 | ~c'content' 11 | ~d|content| 12 | ~e/content/ 13 | 14 | --- 15 | 16 | (source 17 | (sigil (sigil_name) (quoted_content)) 18 | (sigil (sigil_name) (quoted_content)) 19 | (sigil (sigil_name) (quoted_content)) 20 | (sigil (sigil_name) (quoted_content)) 21 | (sigil (sigil_name) (quoted_content)) 22 | (sigil (sigil_name) (quoted_content)) 23 | (sigil (sigil_name) (quoted_content)) 24 | (sigil (sigil_name) (quoted_content))) 25 | 26 | 27 | ===================================== 28 | multiple lines 29 | ===================================== 30 | 31 | ~s"line 1 32 | line 2" 33 | 34 | --- 35 | 36 | (source 37 | (sigil 38 | (sigil_name) 39 | (quoted_content))) 40 | 41 | ===================================== 42 | interpolation 43 | ===================================== 44 | 45 | ~s"hey #{name}!" 46 | ~r/hey #{ 47 | name 48 | }!/ 49 | ~w{##{name}#} 50 | 51 | --- 52 | 53 | (source 54 | (sigil 55 | (sigil_name) 56 | (quoted_content) 57 | (interpolation 58 | (identifier)) 59 | (quoted_content)) 60 | (sigil 61 | (sigil_name) 62 | (quoted_content) 63 | (interpolation 64 | (identifier)) 65 | (quoted_content)) 66 | (sigil 67 | (sigil_name) 68 | (quoted_content) 69 | (interpolation 70 | (identifier)) 71 | (quoted_content))) 72 | 73 | ===================================== 74 | nested interpolation 75 | ===================================== 76 | 77 | ~s{this is #{~s{number #{1}}}!} 78 | 79 | --- 80 | 81 | (source 82 | (sigil 83 | (sigil_name) 84 | (quoted_content) 85 | (interpolation 86 | (sigil 87 | (sigil_name) 88 | (quoted_content) 89 | (interpolation 90 | (integer)))) 91 | (quoted_content))) 92 | 93 | ===================================== 94 | escape sequence 95 | ===================================== 96 | 97 | ~s{_\}_\n_\t_\r_\e_\\_\1_\x3f_\u0065\u0301_} 98 | 99 | --- 100 | 101 | (source 102 | (sigil 103 | (sigil_name) 104 | (quoted_content) 105 | (escape_sequence) 106 | (quoted_content) 107 | (escape_sequence) 108 | (quoted_content) 109 | (escape_sequence) 110 | (quoted_content) 111 | (escape_sequence) 112 | (quoted_content) 113 | (escape_sequence) 114 | (quoted_content) 115 | (escape_sequence) 116 | (quoted_content) 117 | (escape_sequence) 118 | (quoted_content) 119 | (escape_sequence) 120 | (quoted_content) 121 | (escape_sequence) 122 | (escape_sequence) 123 | (quoted_content))) 124 | 125 | ===================================== 126 | escaped interpolation 127 | ===================================== 128 | 129 | ~s/\#{1}/ 130 | 131 | --- 132 | 133 | (source 134 | (sigil 135 | (sigil_name) 136 | (escape_sequence) 137 | (quoted_content))) 138 | 139 | ===================================== 140 | upper sigil / no interpolation 141 | ===================================== 142 | 143 | ~S"hey #{name}!" 144 | 145 | --- 146 | 147 | (source 148 | (sigil 149 | (sigil_name) 150 | (quoted_content))) 151 | 152 | ===================================== 153 | upper sigil / no escape sequence 154 | ===================================== 155 | 156 | ~S"\n" 157 | 158 | --- 159 | 160 | (source 161 | (sigil 162 | (sigil_name) 163 | (quoted_content))) 164 | 165 | ===================================== 166 | upper sigil / escape terminator 167 | ===================================== 168 | 169 | ~S"content \" content" 170 | ~S{content \} content} 171 | ~S/content \/ content/ 172 | 173 | --- 174 | 175 | (source 176 | (sigil 177 | (sigil_name) 178 | (quoted_content) 179 | (escape_sequence) 180 | (quoted_content)) 181 | (sigil 182 | (sigil_name) 183 | (quoted_content) 184 | (escape_sequence) 185 | (quoted_content)) 186 | (sigil 187 | (sigil_name) 188 | (quoted_content) 189 | (escape_sequence) 190 | (quoted_content))) 191 | 192 | ===================================== 193 | upper sigil / multiple characters 194 | ===================================== 195 | 196 | ~MAT"1 2" 197 | ~I18N"text" 198 | ~A1B2"text" 199 | 200 | --- 201 | 202 | (source 203 | (sigil 204 | (sigil_name) 205 | (quoted_content)) 206 | (sigil 207 | (sigil_name) 208 | (quoted_content)) 209 | (sigil 210 | (sigil_name) 211 | (quoted_content))) 212 | 213 | ===================================== 214 | heredoc delimiter 215 | ===================================== 216 | 217 | ~s""" 218 | text 219 | with "quotes" 220 | """ 221 | 222 | ~s''' 223 | text 224 | with 'quotes' 225 | ''' 226 | 227 | --- 228 | 229 | (source 230 | (sigil 231 | (sigil_name) 232 | (quoted_content)) 233 | (sigil 234 | (sigil_name) 235 | (quoted_content))) 236 | 237 | ===================================== 238 | modifiers 239 | ===================================== 240 | 241 | ~r/left|right/i 242 | ~r/left|right/iUx 243 | ~r/left|right/0 244 | ~r/left|right/u8 245 | 246 | --- 247 | 248 | (source 249 | (sigil 250 | (sigil_name) 251 | (quoted_content) 252 | (sigil_modifiers)) 253 | (sigil 254 | (sigil_name) 255 | (quoted_content) 256 | (sigil_modifiers)) 257 | (sigil 258 | (sigil_name) 259 | (quoted_content) 260 | (sigil_modifiers)) 261 | (sigil 262 | (sigil_name) 263 | (quoted_content) 264 | (sigil_modifiers))) 265 | 266 | ===================================== 267 | [error] accepts only a single character 268 | ===================================== 269 | 270 | ~mysigil"content" 271 | 272 | --- 273 | 274 | (source 275 | (sigil 276 | (sigil_name) 277 | (ERROR) 278 | (quoted_content))) 279 | -------------------------------------------------------------------------------- /test/corpus/integration/spec.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | without type parentheses 3 | ===================================== 4 | 5 | @spec fun(atom, integer, keyword) :: string 6 | 7 | --- 8 | 9 | (source 10 | (unary_operator 11 | (call 12 | (identifier) 13 | (arguments 14 | (binary_operator 15 | (call 16 | (identifier) 17 | (arguments 18 | (identifier) 19 | (identifier) 20 | (identifier))) 21 | (identifier)))))) 22 | 23 | ===================================== 24 | with type parentheses 25 | ===================================== 26 | 27 | @spec fun(atom(), integer(), keyword()) :: string() 28 | 29 | --- 30 | 31 | (source 32 | (unary_operator 33 | (call 34 | (identifier) 35 | (arguments 36 | (binary_operator 37 | (call 38 | (identifier) 39 | (arguments 40 | (call 41 | (identifier) 42 | (arguments)) 43 | (call 44 | (identifier) 45 | (arguments)) 46 | (call 47 | (identifier) 48 | (arguments)))) 49 | (call 50 | (identifier) 51 | (arguments))))))) 52 | 53 | ===================================== 54 | with literals 55 | ===================================== 56 | 57 | @spec fun(%{key: atom}) :: {:ok, atom} | {:error, binary} 58 | 59 | --- 60 | 61 | (source 62 | (unary_operator 63 | (call 64 | (identifier) 65 | (arguments 66 | (binary_operator 67 | (call 68 | (identifier) 69 | (arguments 70 | (map 71 | (map_content 72 | (keywords 73 | (pair 74 | (keyword) 75 | (identifier))))))) 76 | (binary_operator 77 | (tuple 78 | (atom) 79 | (identifier)) 80 | (tuple 81 | (atom) 82 | (identifier)))))))) 83 | 84 | ===================================== 85 | with function reference 86 | ===================================== 87 | 88 | @spec fun((-> atom), (atom -> integer)) :: integer 89 | 90 | --- 91 | 92 | (source 93 | (unary_operator 94 | (call 95 | (identifier) 96 | (arguments 97 | (binary_operator 98 | (call 99 | (identifier) 100 | (arguments 101 | (block 102 | (stab_clause 103 | (body 104 | (identifier)))) 105 | (block 106 | (stab_clause 107 | (arguments 108 | (identifier)) 109 | (body 110 | (identifier)))))) 111 | (identifier)))))) 112 | 113 | ===================================== 114 | with remote type 115 | ===================================== 116 | 117 | @spec fun(Keyword.t()) :: String.t() 118 | 119 | --- 120 | 121 | (source 122 | (unary_operator 123 | (call 124 | (identifier) 125 | (arguments 126 | (binary_operator 127 | (call 128 | (identifier) 129 | (arguments 130 | (call 131 | (dot 132 | (alias) 133 | (identifier)) 134 | (arguments)))) 135 | (call 136 | (dot 137 | (alias) 138 | (identifier)) 139 | (arguments))))))) 140 | 141 | ===================================== 142 | with type guard 143 | ===================================== 144 | 145 | @spec fun(arg1, arg2) :: {arg1, arg2} when arg1: atom, arg2: integer 146 | 147 | --- 148 | 149 | (source 150 | (unary_operator 151 | (call 152 | (identifier) 153 | (arguments 154 | (binary_operator 155 | (binary_operator 156 | (call 157 | (identifier) 158 | (arguments 159 | (identifier) 160 | (identifier))) 161 | (tuple 162 | (identifier) 163 | (identifier))) 164 | (keywords 165 | (pair 166 | (keyword) 167 | (identifier)) 168 | (pair 169 | (keyword) 170 | (identifier)))))))) 171 | 172 | ===================================== 173 | with named arguments 174 | ===================================== 175 | 176 | @spec days_since_epoch(year :: integer, month :: integer, day :: integer) :: integer 177 | 178 | --- 179 | 180 | (source 181 | (unary_operator 182 | (call 183 | (identifier) 184 | (arguments 185 | (binary_operator 186 | (call 187 | (identifier) 188 | (arguments 189 | (binary_operator 190 | (identifier) 191 | (identifier)) 192 | (binary_operator 193 | (identifier) 194 | (identifier)) 195 | (binary_operator 196 | (identifier) 197 | (identifier)))) 198 | (identifier)))))) 199 | 200 | ===================================== 201 | nonempty list 202 | ===================================== 203 | 204 | @spec fun() :: [integer, ...] 205 | 206 | --- 207 | 208 | (source 209 | (unary_operator 210 | (call 211 | (identifier) 212 | (arguments 213 | (binary_operator 214 | (call 215 | (identifier) 216 | (arguments)) 217 | (list 218 | (identifier) 219 | (identifier))))))) 220 | 221 | ===================================== 222 | [error] type guard cannot end with keyword separator 223 | ===================================== 224 | 225 | @spec fun(arg) :: arg when arg: atom, 226 | 227 | --- 228 | 229 | (source 230 | (unary_operator 231 | (call 232 | (identifier) 233 | (arguments 234 | (binary_operator 235 | (binary_operator 236 | (call 237 | (identifier) 238 | (arguments 239 | (identifier))) 240 | (identifier)) 241 | (keywords 242 | (pair 243 | (keyword) 244 | (identifier))))))) 245 | (ERROR)) 246 | -------------------------------------------------------------------------------- /test/corpus/expression/anonymous_function.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | no arguments 3 | ===================================== 4 | 5 | fn() -> 1 end 6 | fn () -> 1 end 7 | 8 | --- 9 | 10 | (source 11 | (anonymous_function 12 | (stab_clause 13 | (arguments) 14 | (body 15 | (integer)))) 16 | (anonymous_function 17 | (stab_clause 18 | (arguments) 19 | (body 20 | (integer))))) 21 | 22 | ===================================== 23 | no arguments without parentheses 24 | ===================================== 25 | 26 | fn -> 1 end 27 | 28 | --- 29 | 30 | (source 31 | (anonymous_function 32 | (stab_clause 33 | (body 34 | (integer))))) 35 | 36 | ===================================== 37 | one argument 38 | ===================================== 39 | 40 | fn(x) -> x end 41 | 42 | --- 43 | 44 | (source 45 | (anonymous_function 46 | (stab_clause 47 | (arguments 48 | (identifier)) 49 | (body 50 | (identifier))))) 51 | 52 | ===================================== 53 | one argument without parentheses 54 | ===================================== 55 | 56 | fn x -> x end 57 | 58 | --- 59 | 60 | (source 61 | (anonymous_function 62 | (stab_clause 63 | (arguments 64 | (identifier)) 65 | (body 66 | (identifier))))) 67 | 68 | ===================================== 69 | many arguments 70 | ===================================== 71 | 72 | fn(x, y, z) -> x + y end 73 | 74 | --- 75 | 76 | (source 77 | (anonymous_function 78 | (stab_clause 79 | (arguments 80 | (identifier) 81 | (identifier) 82 | (identifier)) 83 | (body 84 | (binary_operator 85 | (identifier) 86 | (identifier)))))) 87 | 88 | ===================================== 89 | many arguments without parentheses 90 | ===================================== 91 | 92 | fn x, y -> x + y end 93 | 94 | --- 95 | 96 | (source 97 | (anonymous_function 98 | (stab_clause 99 | (arguments 100 | (identifier) 101 | (identifier)) 102 | (body 103 | (binary_operator 104 | (identifier) 105 | (identifier)))))) 106 | 107 | ===================================== 108 | multiline body 109 | ===================================== 110 | 111 | fn x, y -> 112 | y 113 | x 114 | end 115 | 116 | --- 117 | 118 | (source 119 | (anonymous_function 120 | (stab_clause 121 | (arguments 122 | (identifier) 123 | (identifier)) 124 | (body 125 | (identifier) 126 | (identifier))))) 127 | 128 | ===================================== 129 | multiline body with extra newlines 130 | ===================================== 131 | 132 | fn x, y -> 133 | y 134 | 135 | x 136 | 137 | end 138 | 139 | --- 140 | 141 | (source 142 | (anonymous_function 143 | (stab_clause 144 | (arguments 145 | (identifier) 146 | (identifier)) 147 | (body 148 | (identifier) 149 | (identifier))))) 150 | 151 | ===================================== 152 | many clauses 153 | ===================================== 154 | 155 | fn 156 | 1 -> :yes 157 | 2 -> :no 158 | other -> :maybe 159 | end 160 | 161 | --- 162 | 163 | (source 164 | (anonymous_function 165 | (stab_clause 166 | (arguments 167 | (integer)) 168 | (body 169 | (atom))) 170 | (stab_clause 171 | (arguments 172 | (integer)) 173 | (body 174 | (atom))) 175 | (stab_clause 176 | (arguments 177 | (identifier)) 178 | (body 179 | (atom))))) 180 | 181 | ===================================== 182 | no clauses 183 | ===================================== 184 | 185 | fn 186 | end 187 | 188 | fn end 189 | 190 | --- 191 | 192 | (source 193 | (anonymous_function) 194 | (anonymous_function)) 195 | 196 | ===================================== 197 | with guard / no arguments 198 | ===================================== 199 | 200 | fn 201 | () when node() == :nonode@nohost -> true 202 | end 203 | 204 | --- 205 | 206 | (source 207 | (anonymous_function 208 | (stab_clause 209 | (binary_operator 210 | (arguments) 211 | (binary_operator 212 | (call 213 | (identifier) 214 | (arguments)) 215 | (atom))) 216 | (body 217 | (boolean))))) 218 | 219 | ===================================== 220 | with guard / one argument 221 | ===================================== 222 | 223 | fn 224 | x when x == [] -> x 225 | end 226 | 227 | --- 228 | 229 | (source 230 | (anonymous_function 231 | (stab_clause 232 | (binary_operator 233 | (arguments 234 | (identifier)) 235 | (binary_operator 236 | (identifier) 237 | (list))) 238 | (body 239 | (identifier))))) 240 | 241 | ===================================== 242 | with guard / multiple arguments 243 | ===================================== 244 | 245 | fn 246 | x, y when x == [] -> x 247 | end 248 | 249 | --- 250 | 251 | (source 252 | (anonymous_function 253 | (stab_clause 254 | (binary_operator 255 | (arguments 256 | (identifier) 257 | (identifier)) 258 | (binary_operator 259 | (identifier) 260 | (list))) 261 | (body 262 | (identifier))))) 263 | 264 | ===================================== 265 | with guard / arguments in parentheses 266 | ===================================== 267 | 268 | fn 269 | (x, y) when y == [] -> y 270 | end 271 | 272 | --- 273 | 274 | (source 275 | (anonymous_function 276 | (stab_clause 277 | (binary_operator 278 | (arguments 279 | (identifier) 280 | (identifier)) 281 | (binary_operator 282 | (identifier) 283 | (list))) 284 | (body 285 | (identifier))))) 286 | 287 | ===================================== 288 | with guard / multiple guards 289 | ===================================== 290 | 291 | fn 292 | x when x > 10 when x < 5 -> x 293 | end 294 | 295 | --- 296 | 297 | (source 298 | (anonymous_function 299 | (stab_clause 300 | (binary_operator 301 | (arguments 302 | (identifier)) 303 | (binary_operator 304 | (binary_operator 305 | (identifier) 306 | (integer)) 307 | (binary_operator 308 | (identifier) 309 | (integer)))) 310 | (body 311 | (identifier))))) 312 | 313 | ===================================== 314 | pattern matching 315 | ===================================== 316 | 317 | fn 318 | [h | tail] -> {h, tail} 319 | %{x: x} when x == 1 -> 1 320 | end 321 | 322 | --- 323 | 324 | (source 325 | (anonymous_function 326 | (stab_clause 327 | (arguments 328 | (list 329 | (binary_operator 330 | (identifier) 331 | (identifier)))) 332 | (body 333 | (tuple 334 | (identifier) 335 | (identifier)))) 336 | (stab_clause 337 | (binary_operator 338 | (arguments 339 | (map 340 | (map_content 341 | (keywords 342 | (pair 343 | (keyword) 344 | (identifier)))))) 345 | (binary_operator 346 | (identifier) 347 | (integer))) 348 | (body 349 | (integer))))) 350 | -------------------------------------------------------------------------------- /test/highlight/data_structures.ex: -------------------------------------------------------------------------------- 1 | <<1, 2, 3>> 2 | # <- punctuation.bracket 3 | # ^ number 4 | # ^ punctuation.delimiter 5 | # ^ number 6 | # ^ punctuation.delimiter 7 | # ^ number 8 | # ^ punctuation.bracket 9 | 10 | << header :: size(8), data :: binary >> 11 | # <- punctuation.bracket 12 | # ^ variable 13 | # ^ operator 14 | # ^ function 15 | # ^ punctuation.bracket 16 | # ^ number 17 | # ^ punctuation.bracket 18 | # ^ punctuation.delimiter 19 | # ^ variable 20 | # ^ operator 21 | # ^ variable 22 | # ^ punctuation.bracket 23 | 24 | <<"hello"::binary, c :: utf8, x::[4, unit(2)]>> = "hello™1" 25 | # <- punctuation.bracket 26 | # ^ string 27 | # ^ operator 28 | # ^ variable 29 | # ^ punctuation.delimiter 30 | # ^ variable 31 | # ^ operator 32 | # ^ variable 33 | # ^ punctuation.delimiter 34 | # ^ variable 35 | # ^ operator 36 | # ^ punctuation.bracket 37 | # ^ number 38 | # ^ punctuation.delimiter 39 | # ^ function 40 | # ^ punctuation.bracket 41 | # ^ number 42 | # ^ punctuation.bracket 43 | # ^ punctuation.bracket 44 | # ^ punctuation.bracket 45 | # ^ operator 46 | # ^ string 47 | 48 | [1, :a, 'hello'] ++ [2, 3] 49 | # <- punctuation.bracket 50 | # ^ punctuation.delimiter 51 | # ^ string.special.symbol 52 | # ^ punctuation.delimiter 53 | # ^ string 54 | # ^ punctuation.bracket 55 | # ^ operator 56 | # ^ punctuation.bracket 57 | # ^ number 58 | # ^ punctuation.delimiter 59 | # ^ number 60 | # ^ punctuation.bracket 61 | 62 | [:head | [?t, ?a]] 63 | # <- punctuation.bracket 64 | # ^ string.special.symbol 65 | # ^ operator 66 | # ^ punctuation.bracket 67 | # ^ constant 68 | # ^ punctuation.delimiter 69 | # ^ constant 70 | # ^ punctuation.bracket 71 | # ^ punctuation.bracket 72 | 73 | {:one, 2.0, "three"} 74 | # <- punctuation.bracket 75 | # ^ string.special.symbol 76 | # ^ punctuation.delimiter 77 | # ^ number 78 | # ^ punctuation.delimiter 79 | # ^ string 80 | # ^ punctuation.bracket 81 | 82 | [option: "value", key: :word] 83 | # <- punctuation.bracket 84 | # ^ string.special.symbol 85 | # ^ string 86 | # ^ punctuation.delimiter 87 | # ^ string.special.symbol 88 | # ^ string.special.symbol 89 | # ^ punctuation.bracket 90 | 91 | [++: "operator", ~~~: :&&&] 92 | # <- punctuation.bracket 93 | # ^ string.special.symbol 94 | # ^ string 95 | # ^ punctuation.delimiter 96 | # ^ string.special.symbol 97 | # ^ string.special.symbol 98 | # ^ punctuation.bracket 99 | 100 | [...: 1, <<>>: 2, %{}: 3, %: 4, {}: 5] 101 | # <- punctuation.bracket 102 | # ^ string.special.symbol 103 | # ^ number 104 | # ^ punctuation.delimiter 105 | # ^ string.special.symbol 106 | # ^ number 107 | # ^ punctuation.delimiter 108 | # ^ string.special.symbol 109 | # ^ number 110 | # ^ punctuation.delimiter 111 | # ^ string.special.symbol 112 | # ^ number 113 | # ^ punctuation.delimiter 114 | # ^ string.special.symbol 115 | # ^ number 116 | # ^ punctuation.bracket 117 | 118 | ["this is an atom too": 1, "so is #{1} this": 2] 119 | # <- punctuation.bracket 120 | # ^ string.special.symbol 121 | # ^ number 122 | # ^ punctuation.delimiter 123 | # ^ string.special.symbol 124 | # ^ punctuation.special 125 | # ^ number 126 | # ^ punctuation.special 127 | # ^ number 128 | # ^ punctuation.bracket 129 | 130 | %{shortcut: "syntax"} 131 | #<- punctuation 132 | #^ punctuation.bracket 133 | # ^ string.special.symbol 134 | # ^ string 135 | # ^ punctuation.bracket 136 | 137 | %{map | "update" => "me"} 138 | #<- punctuation 139 | #^ punctuation.bracket 140 | # ^ variable 141 | # ^ operator 142 | # ^ string 143 | # ^ operator 144 | # ^ string 145 | # ^ punctuation.bracket 146 | 147 | %{ 12 => 13, :weird => ['thing'] } 148 | #<- punctuation 149 | #^ punctuation.bracket 150 | # ^ number 151 | # ^ operator 152 | # ^ number 153 | # ^ punctuation.delimiter 154 | # ^ string.special.symbol 155 | # ^ operator 156 | # ^ punctuation.bracket 157 | # ^ string 158 | # ^ punctuation.bracket 159 | # ^ punctuation.bracket 160 | 161 | %Long.Module.Name{name: "Silly"} 162 | # <- punctuation 163 | # ^ module 164 | # ^ module 165 | # ^ punctuation.bracket 166 | # ^ string.special.symbol 167 | # ^ string 168 | # ^ punctuation.bracket 169 | 170 | %Long.Module.Name{s | height: {192, :cm}} 171 | # <- punctuation 172 | # ^ module 173 | # ^ punctuation.bracket 174 | # ^ variable 175 | # ^ operator 176 | # ^ string.special.symbol 177 | # ^ punctuation.bracket 178 | # ^ number 179 | # ^ punctuation.delimiter 180 | # ^ string.special.symbol 181 | # ^ punctuation.bracket 182 | # ^ punctuation.bracket 183 | 184 | ".. #{%Long.Module.Name{s | height: {192, :cm}}} .." 185 | # ^ string 186 | # ^ punctuation.special 187 | # ^ punctuation 188 | # ^ module 189 | # ^ punctuation.bracket 190 | # ^ variable 191 | # ^ operator 192 | # ^ string.special.symbol 193 | # ^ punctuation.bracket 194 | # ^ number 195 | # ^ punctuation.delimiter 196 | # ^ string.special.symbol 197 | # ^ punctuation.bracket 198 | # ^ punctuation.bracket 199 | # ^ punctuation.special 200 | # ^ string 201 | -------------------------------------------------------------------------------- /src/tree_sitter/parser.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_PARSER_H_ 2 | #define TREE_SITTER_PARSER_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #define ts_builtin_sym_error ((TSSymbol)-1) 13 | #define ts_builtin_sym_end 0 14 | #define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024 15 | 16 | #ifndef TREE_SITTER_API_H_ 17 | typedef uint16_t TSStateId; 18 | typedef uint16_t TSSymbol; 19 | typedef uint16_t TSFieldId; 20 | typedef struct TSLanguage TSLanguage; 21 | #endif 22 | 23 | typedef struct { 24 | TSFieldId field_id; 25 | uint8_t child_index; 26 | bool inherited; 27 | } TSFieldMapEntry; 28 | 29 | typedef struct { 30 | uint16_t index; 31 | uint16_t length; 32 | } TSFieldMapSlice; 33 | 34 | typedef struct { 35 | bool visible; 36 | bool named; 37 | bool supertype; 38 | } TSSymbolMetadata; 39 | 40 | typedef struct TSLexer TSLexer; 41 | 42 | struct TSLexer { 43 | int32_t lookahead; 44 | TSSymbol result_symbol; 45 | void (*advance)(TSLexer *, bool); 46 | void (*mark_end)(TSLexer *); 47 | uint32_t (*get_column)(TSLexer *); 48 | bool (*is_at_included_range_start)(const TSLexer *); 49 | bool (*eof)(const TSLexer *); 50 | void (*log)(const TSLexer *, const char *, ...); 51 | }; 52 | 53 | typedef enum { 54 | TSParseActionTypeShift, 55 | TSParseActionTypeReduce, 56 | TSParseActionTypeAccept, 57 | TSParseActionTypeRecover, 58 | } TSParseActionType; 59 | 60 | typedef union { 61 | struct { 62 | uint8_t type; 63 | TSStateId state; 64 | bool extra; 65 | bool repetition; 66 | } shift; 67 | struct { 68 | uint8_t type; 69 | uint8_t child_count; 70 | TSSymbol symbol; 71 | int16_t dynamic_precedence; 72 | uint16_t production_id; 73 | } reduce; 74 | uint8_t type; 75 | } TSParseAction; 76 | 77 | typedef struct { 78 | uint16_t lex_state; 79 | uint16_t external_lex_state; 80 | } TSLexMode; 81 | 82 | typedef union { 83 | TSParseAction action; 84 | struct { 85 | uint8_t count; 86 | bool reusable; 87 | } entry; 88 | } TSParseActionEntry; 89 | 90 | typedef struct { 91 | int32_t start; 92 | int32_t end; 93 | } TSCharacterRange; 94 | 95 | struct TSLanguage { 96 | uint32_t version; 97 | uint32_t symbol_count; 98 | uint32_t alias_count; 99 | uint32_t token_count; 100 | uint32_t external_token_count; 101 | uint32_t state_count; 102 | uint32_t large_state_count; 103 | uint32_t production_id_count; 104 | uint32_t field_count; 105 | uint16_t max_alias_sequence_length; 106 | const uint16_t *parse_table; 107 | const uint16_t *small_parse_table; 108 | const uint32_t *small_parse_table_map; 109 | const TSParseActionEntry *parse_actions; 110 | const char * const *symbol_names; 111 | const char * const *field_names; 112 | const TSFieldMapSlice *field_map_slices; 113 | const TSFieldMapEntry *field_map_entries; 114 | const TSSymbolMetadata *symbol_metadata; 115 | const TSSymbol *public_symbol_map; 116 | const uint16_t *alias_map; 117 | const TSSymbol *alias_sequences; 118 | const TSLexMode *lex_modes; 119 | bool (*lex_fn)(TSLexer *, TSStateId); 120 | bool (*keyword_lex_fn)(TSLexer *, TSStateId); 121 | TSSymbol keyword_capture_token; 122 | struct { 123 | const bool *states; 124 | const TSSymbol *symbol_map; 125 | void *(*create)(void); 126 | void (*destroy)(void *); 127 | bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist); 128 | unsigned (*serialize)(void *, char *); 129 | void (*deserialize)(void *, const char *, unsigned); 130 | } external_scanner; 131 | const TSStateId *primary_state_ids; 132 | }; 133 | 134 | static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) { 135 | uint32_t index = 0; 136 | uint32_t size = len - index; 137 | while (size > 1) { 138 | uint32_t half_size = size / 2; 139 | uint32_t mid_index = index + half_size; 140 | TSCharacterRange *range = &ranges[mid_index]; 141 | if (lookahead >= range->start && lookahead <= range->end) { 142 | return true; 143 | } else if (lookahead > range->end) { 144 | index = mid_index; 145 | } 146 | size -= half_size; 147 | } 148 | TSCharacterRange *range = &ranges[index]; 149 | return (lookahead >= range->start && lookahead <= range->end); 150 | } 151 | 152 | /* 153 | * Lexer Macros 154 | */ 155 | 156 | #ifdef _MSC_VER 157 | #define UNUSED __pragma(warning(suppress : 4101)) 158 | #else 159 | #define UNUSED __attribute__((unused)) 160 | #endif 161 | 162 | #define START_LEXER() \ 163 | bool result = false; \ 164 | bool skip = false; \ 165 | UNUSED \ 166 | bool eof = false; \ 167 | int32_t lookahead; \ 168 | goto start; \ 169 | next_state: \ 170 | lexer->advance(lexer, skip); \ 171 | start: \ 172 | skip = false; \ 173 | lookahead = lexer->lookahead; 174 | 175 | #define ADVANCE(state_value) \ 176 | { \ 177 | state = state_value; \ 178 | goto next_state; \ 179 | } 180 | 181 | #define ADVANCE_MAP(...) \ 182 | { \ 183 | static const uint16_t map[] = { __VA_ARGS__ }; \ 184 | for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \ 185 | if (map[i] == lookahead) { \ 186 | state = map[i + 1]; \ 187 | goto next_state; \ 188 | } \ 189 | } \ 190 | } 191 | 192 | #define SKIP(state_value) \ 193 | { \ 194 | skip = true; \ 195 | state = state_value; \ 196 | goto next_state; \ 197 | } 198 | 199 | #define ACCEPT_TOKEN(symbol_value) \ 200 | result = true; \ 201 | lexer->result_symbol = symbol_value; \ 202 | lexer->mark_end(lexer); 203 | 204 | #define END_STATE() return result; 205 | 206 | /* 207 | * Parse Table Macros 208 | */ 209 | 210 | #define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT) 211 | 212 | #define STATE(id) id 213 | 214 | #define ACTIONS(id) id 215 | 216 | #define SHIFT(state_value) \ 217 | {{ \ 218 | .shift = { \ 219 | .type = TSParseActionTypeShift, \ 220 | .state = (state_value) \ 221 | } \ 222 | }} 223 | 224 | #define SHIFT_REPEAT(state_value) \ 225 | {{ \ 226 | .shift = { \ 227 | .type = TSParseActionTypeShift, \ 228 | .state = (state_value), \ 229 | .repetition = true \ 230 | } \ 231 | }} 232 | 233 | #define SHIFT_EXTRA() \ 234 | {{ \ 235 | .shift = { \ 236 | .type = TSParseActionTypeShift, \ 237 | .extra = true \ 238 | } \ 239 | }} 240 | 241 | #define REDUCE(symbol_name, children, precedence, prod_id) \ 242 | {{ \ 243 | .reduce = { \ 244 | .type = TSParseActionTypeReduce, \ 245 | .symbol = symbol_name, \ 246 | .child_count = children, \ 247 | .dynamic_precedence = precedence, \ 248 | .production_id = prod_id \ 249 | }, \ 250 | }} 251 | 252 | #define RECOVER() \ 253 | {{ \ 254 | .type = TSParseActionTypeRecover \ 255 | }} 256 | 257 | #define ACCEPT_INPUT() \ 258 | {{ \ 259 | .type = TSParseActionTypeAccept \ 260 | }} 261 | 262 | #ifdef __cplusplus 263 | } 264 | #endif 265 | 266 | #endif // TREE_SITTER_PARSER_H_ 267 | -------------------------------------------------------------------------------- /test/corpus/integration/function_definition.txt: -------------------------------------------------------------------------------- 1 | ===================================== 2 | def / no arguments 3 | ===================================== 4 | 5 | def fun() do 6 | end 7 | 8 | --- 9 | 10 | (source 11 | (call 12 | (identifier) 13 | (arguments 14 | (call 15 | (identifier) 16 | (arguments))) 17 | (do_block))) 18 | 19 | ===================================== 20 | def / no arguments without parentheses 21 | ===================================== 22 | 23 | def fun do 24 | end 25 | 26 | --- 27 | 28 | (source 29 | (call 30 | (identifier) 31 | (arguments 32 | (identifier)) 33 | (do_block))) 34 | 35 | ===================================== 36 | def / one argument 37 | ===================================== 38 | 39 | def fun(x) do 40 | x 41 | end 42 | 43 | --- 44 | 45 | (source 46 | (call 47 | (identifier) 48 | (arguments 49 | (call 50 | (identifier) 51 | (arguments 52 | (identifier)))) 53 | (do_block 54 | (identifier)))) 55 | 56 | ===================================== 57 | def / one argument without parentheses 58 | ===================================== 59 | 60 | def fun x do 61 | x 62 | end 63 | 64 | --- 65 | 66 | (source 67 | (call 68 | (identifier) 69 | (arguments 70 | (call 71 | (identifier) 72 | (arguments 73 | (identifier)))) 74 | (do_block 75 | (identifier)))) 76 | 77 | ===================================== 78 | def / many arguments 79 | ===================================== 80 | 81 | def fun(x, y) do 82 | x + y 83 | end 84 | 85 | --- 86 | 87 | (source 88 | (call 89 | (identifier) 90 | (arguments 91 | (call 92 | (identifier) 93 | (arguments 94 | (identifier) 95 | (identifier)))) 96 | (do_block 97 | (binary_operator 98 | (identifier) 99 | (identifier))))) 100 | 101 | ===================================== 102 | def / many arguments without parentheses 103 | ===================================== 104 | 105 | def fun x, y do 106 | x + y 107 | end 108 | 109 | --- 110 | 111 | (source 112 | (call 113 | (identifier) 114 | (arguments 115 | (call 116 | (identifier) 117 | (arguments 118 | (identifier) 119 | (identifier)))) 120 | (do_block 121 | (binary_operator 122 | (identifier) 123 | (identifier))))) 124 | 125 | ===================================== 126 | def / default arguments 127 | ===================================== 128 | 129 | def fun x, y \\ 1 do 130 | x + y 131 | end 132 | 133 | def fun(x, y \\ 1) do 134 | x + y 135 | end 136 | 137 | --- 138 | 139 | (source 140 | (call 141 | (identifier) 142 | (arguments 143 | (call 144 | (identifier) 145 | (arguments 146 | (identifier) 147 | (binary_operator 148 | (identifier) 149 | (integer))))) 150 | (do_block 151 | (binary_operator 152 | (identifier) 153 | (identifier)))) 154 | (call 155 | (identifier) 156 | (arguments 157 | (call 158 | (identifier) 159 | (arguments 160 | (identifier) 161 | (binary_operator 162 | (identifier) 163 | (integer))))) 164 | (do_block 165 | (binary_operator 166 | (identifier) 167 | (identifier))))) 168 | 169 | ===================================== 170 | def / keyword do block 171 | ===================================== 172 | 173 | def fun(), do: 1 174 | def fun(x), do: x 175 | 176 | --- 177 | 178 | (source 179 | (call 180 | (identifier) 181 | (arguments 182 | (call 183 | (identifier) 184 | (arguments)) 185 | (keywords 186 | (pair 187 | (keyword) 188 | (integer))))) 189 | (call 190 | (identifier) 191 | (arguments 192 | (call 193 | (identifier) 194 | (arguments 195 | (identifier))) 196 | (keywords 197 | (pair 198 | (keyword) 199 | (identifier)))))) 200 | 201 | ===================================== 202 | def / pattern matching 203 | ===================================== 204 | 205 | def fun([{x, y} | tail]) do 206 | x + y 207 | end 208 | 209 | --- 210 | 211 | (source 212 | (call 213 | (identifier) 214 | (arguments 215 | (call 216 | (identifier) 217 | (arguments 218 | (list 219 | (binary_operator 220 | (tuple 221 | (identifier) 222 | (identifier)) 223 | (identifier)))))) 224 | (do_block 225 | (binary_operator 226 | (identifier) 227 | (identifier))))) 228 | 229 | ===================================== 230 | def / with guard 231 | ===================================== 232 | 233 | def fun(x) when x == 1 do 234 | x 235 | end 236 | 237 | --- 238 | 239 | (source 240 | (call 241 | (identifier) 242 | (arguments 243 | (binary_operator 244 | (call 245 | (identifier) 246 | (arguments 247 | (identifier))) 248 | (binary_operator 249 | (identifier) 250 | (integer)))) 251 | (do_block 252 | (identifier)))) 253 | 254 | ===================================== 255 | def / with guard / multiple guards 256 | ===================================== 257 | 258 | def fun(x) when x > 10 when x < 5 do 259 | x 260 | end 261 | 262 | --- 263 | 264 | (source 265 | (call 266 | (identifier) 267 | (arguments 268 | (binary_operator 269 | (call 270 | (identifier) 271 | (arguments 272 | (identifier))) 273 | (binary_operator 274 | (binary_operator 275 | (identifier) 276 | (integer)) 277 | (binary_operator 278 | (identifier) 279 | (integer))))) 280 | (do_block 281 | (identifier)))) 282 | 283 | ===================================== 284 | defp 285 | ===================================== 286 | 287 | defp fun(x) do 288 | x 289 | end 290 | 291 | --- 292 | 293 | (source 294 | (call 295 | (identifier) 296 | (arguments 297 | (call 298 | (identifier) 299 | (arguments 300 | (identifier)))) 301 | (do_block 302 | (identifier)))) 303 | 304 | ===================================== 305 | defmacro 306 | ===================================== 307 | 308 | defmacro fun(x) do 309 | quote do 310 | [unquote(x)] 311 | end 312 | end 313 | 314 | --- 315 | 316 | (source 317 | (call 318 | (identifier) 319 | (arguments 320 | (call 321 | (identifier) 322 | (arguments 323 | (identifier)))) 324 | (do_block 325 | (call 326 | (identifier) 327 | (do_block 328 | (list 329 | (call 330 | (identifier) 331 | (arguments 332 | (identifier))))))))) 333 | 334 | ===================================== 335 | defguard 336 | ===================================== 337 | 338 | defguard is_even(term) when is_integer(term) and rem(term, 2) == 0 339 | 340 | --- 341 | 342 | (source 343 | (call 344 | (identifier) 345 | (arguments 346 | (binary_operator 347 | (call 348 | (identifier) 349 | (arguments 350 | (identifier))) 351 | (binary_operator 352 | (call 353 | (identifier) 354 | (arguments 355 | (identifier))) 356 | (binary_operator 357 | (call 358 | (identifier) 359 | (arguments 360 | (identifier) 361 | (integer))) 362 | (integer))))))) 363 | 364 | ===================================== 365 | def in macro 366 | ===================================== 367 | 368 | def unquote(name)(unquote_splicing(args)) do 369 | unquote(compiled) 370 | end 371 | 372 | --- 373 | 374 | (source 375 | (call 376 | (identifier) 377 | (arguments 378 | (call 379 | (call 380 | (identifier) 381 | (arguments 382 | (identifier))) 383 | (arguments 384 | (call 385 | (identifier) 386 | (arguments 387 | (identifier)))))) 388 | (do_block 389 | (call 390 | (identifier) 391 | (arguments 392 | (identifier)))))) 393 | -------------------------------------------------------------------------------- /test/highlight/module.ex: -------------------------------------------------------------------------------- 1 | defmodule Long.Module.Name do 2 | # ^ keyword 3 | # ^ module 4 | # ^ module 5 | # ^ keyword 6 | 7 | @moduledoc "Simple doc" 8 | # <- comment.doc 9 | # ^ comment.doc.__attribute__ 10 | # ^ comment.doc 11 | 12 | @moduledoc false 13 | # <- comment.doc 14 | # ^ comment.doc.__attribute__ 15 | # ^ comment.doc 16 | 17 | @moduledoc """ 18 | Heredoc doc 19 | """ 20 | # ^ comment.doc 21 | 22 | @moduledoc ~S''' 23 | Sigil doc 24 | ''' 25 | # ^ comment.doc 26 | 27 | @moduledoc "With #{1} interpolation" 28 | # ^ punctuation.special 29 | # ^ number 30 | # ^ punctuation.special 31 | 32 | @typedoc "Type doc" 33 | # <- comment.doc 34 | 35 | @doc "Type doc" 36 | # <- comment.doc 37 | 38 | @doc """ 39 | Multiline docstring 40 | "with quotes" 41 | and #{ inspect %{"interpolation" => "in" <> "action"} } 42 | now with #{ {:a, 'tuple'} } 43 | and #{ inspect { 44 | :tuple, 45 | %{ with: "nested #{ inspect %{ :interpolation => %{} } }" } 46 | } } 47 | """ 48 | # <- comment.doc 49 | 50 | @spec func(type, integer()) :: :ok | :fail 51 | # <- attribute 52 | # ^ attribute 53 | # ^ function 54 | # ^ punctuation.bracket 55 | # ^ variable 56 | # ^ punctuation.delimiter 57 | # ^ function 58 | # ^ punctuation.bracket 59 | # ^ punctuation.bracket 60 | # ^ punctuation.bracket 61 | # ^ operator 62 | # ^ string.special.symbol 63 | # ^ operator 64 | # ^ string.special.symbol 65 | 66 | defstruct items: [] 67 | # ^ keyword 68 | # ^ string.special.symbol 69 | # ^ punctuation.bracket 70 | # ^ punctuation.bracket 71 | 72 | defexception [:message] 73 | # ^ keyword 74 | # ^ punctuation.bracket 75 | # ^ string.special.symbol 76 | # ^ punctuation.bracket 77 | 78 | @type item :: String.t() 79 | # <- attribute 80 | # ^ attribute 81 | # ^ variable 82 | # ^ operator 83 | # ^ module 84 | # ^ operator 85 | # ^ function 86 | # ^ punctuation.bracket 87 | # ^ punctuation.bracket 88 | 89 | @attr "value" 90 | # <- attribute 91 | # ^ attribute 92 | # ^ string 93 | 94 | @true 95 | # ^ attribute 96 | 97 | @nil 98 | # ^ attribute 99 | 100 | def f, do: nil 101 | # ^ keyword 102 | # ^ function 103 | # ^ punctuation.delimiter 104 | # ^ string.special.symbol 105 | # ^ constant 106 | 107 | def f(x), do: x 108 | # ^ keyword 109 | # ^ function 110 | # ^ punctuation.bracket 111 | # ^ variable 112 | # ^ punctuation.bracket 113 | # ^ punctuation.delimiter 114 | # ^ string.special.symbol 115 | # ^ variable 116 | 117 | def f(10), do: nil 118 | # ^ keyword 119 | # ^ function 120 | # ^ punctuation.bracket 121 | # ^ punctuation.bracket 122 | # ^ punctuation.delimiter 123 | # ^ string.special.symbol 124 | # ^ constant 125 | 126 | def f(a, b \\ []), do: nil 127 | # <- keyword 128 | # ^ function 129 | # ^ punctuation.bracket 130 | # ^ variable 131 | # ^ punctuation.delimiter 132 | # ^ variable 133 | # ^ operator 134 | # ^ punctuation.bracket 135 | # ^ punctuation.bracket 136 | # ^ punctuation.delimiter 137 | # ^ string.special.symbol 138 | # ^ constant 139 | 140 | def __before_compile__(_) do 141 | # <- keyword 142 | # ^ function 143 | # ^ punctuation.bracket 144 | # ^ comment.unused 145 | # ^ punctuation.bracket 146 | # ^ keyword 147 | end 148 | # <- keyword 149 | 150 | def with_guard(x) when x == 1, do: nil 151 | # <- keyword 152 | # ^ function 153 | # ^ variable 154 | # ^ keyword 155 | # ^ variable 156 | # ^ operator 157 | # ^ number 158 | # ^ string.special.symbol 159 | # <- keyword 160 | 161 | def with_guard when is_integer(1), do: nil 162 | # <- keyword 163 | # ^ function 164 | # ^ keyword 165 | # ^ function 166 | # ^ punctuation.bracket 167 | # ^ number 168 | # ^ punctuation.bracket 169 | # ^ punctuation.delimiter 170 | # ^ string.special.symbol 171 | # ^ constant 172 | 173 | def x + y, do: nil 174 | # ^ keyword 175 | # ^ variable 176 | # ^ operator 177 | # ^ variable 178 | # ^ punctuation.delimiter 179 | # ^ string.special.symbol 180 | # ^ constant 181 | 182 | def x |> y, do: nil 183 | # ^ keyword 184 | # ^ variable 185 | # ^ operator 186 | # ^ variable 187 | # ^ punctuation.delimiter 188 | # ^ string.special.symbol 189 | # ^ constant 190 | 191 | def x and y, do: nil 192 | # ^ keyword 193 | # ^ variable 194 | # ^ keyword 195 | # ^ variable 196 | # ^ punctuation.delimiter 197 | # ^ string.special.symbol 198 | # ^ constant 199 | 200 | def unquote(f)(x), do: nil 201 | # ^ keyword 202 | # ^ keyword 203 | # ^ punctuation.bracket 204 | # ^ variable 205 | # ^ punctuation.bracket 206 | # ^ punctuation.bracket 207 | # ^ variable 208 | # ^ punctuation.bracket 209 | # ^ punctuation.delimiter 210 | # ^ string.special.symbol 211 | # ^ constant 212 | 213 | def unquote(name)(unquote_splicing(args)), do: nil 214 | # ^ keyword 215 | # ^ keyword 216 | # ^ punctuation.bracket 217 | # ^ variable 218 | # ^ punctuation.bracket 219 | # ^ punctuation.bracket 220 | # ^ keyword 221 | # ^ punctuation.bracket 222 | # ^ variable 223 | # ^ punctuation.bracket 224 | # ^ punctuation.bracket 225 | # ^ string.special.symbol 226 | # ^ constant 227 | 228 | def(test(x), do: x) 229 | # ^ keyword 230 | # ^ punctuation.bracket 231 | # ^ punctuation.bracket 232 | # ^ variable 233 | # ^ punctuation.bracket 234 | # ^ punctuation.delimiter 235 | # ^ string.special.symbol 236 | # ^ variable 237 | # ^ punctuation.bracket 238 | 239 | defguard is_something(one), do: one != nil 240 | # ^ keyword 241 | # ^ function 242 | # ^ punctuation.bracket 243 | # ^ variable 244 | # ^ punctuation.bracket 245 | # ^ punctuation.delimiter 246 | # ^ string.special.symbol 247 | # ^ variable 248 | # ^ operator 249 | # ^ constant 250 | 251 | defdelegate empty?(list), to: Enum 252 | # ^ keyword 253 | # ^ function 254 | # ^ punctuation.bracket 255 | # ^ variable 256 | # ^ punctuation.bracket 257 | # ^ punctuation.delimiter 258 | # ^ string.special.symbol 259 | # ^ module 260 | 261 | defmacro meta_function(name) do 262 | # ^ keyword 263 | # ^ function 264 | # ^ punctuation.bracket 265 | # ^ variable 266 | # ^ punctuation.bracket 267 | # ^ keyword 268 | quote do 269 | # ^ keyword 270 | # ^ keyword 271 | def unquote(:"#{name}_foo")() do 272 | # ^ keyword 273 | # ^ keyword 274 | # ^ punctuation.bracket 275 | # ^ string.special.symbol 276 | # ^ punctuation.special 277 | # ^ variable 278 | # ^ punctuation.special 279 | # ^ string.special.symbol 280 | # ^ punctuation.bracket 281 | # ^ punctuation.bracket 282 | # ^ punctuation.bracket 283 | # ^ keyword 284 | unquote("yessir") 285 | # ^ keyword 286 | # ^ punctuation.bracket 287 | # ^ string 288 | # ^ punctuation.bracket 289 | end 290 | # <- keyword 291 | end 292 | # <- keyword 293 | end 294 | # <- keyword 295 | end 296 | # <- keyword 297 | 298 | defprotocol Useless do 299 | # ^ keyword 300 | # ^ module 301 | # ^ keyword 302 | def func(this) 303 | # ^ keyword 304 | # ^ function 305 | # ^ punctuation.bracket 306 | # ^ variable 307 | # ^ punctuation.bracket 308 | end 309 | # <- keyword 310 | 311 | defimpl Useless, for: Atom do 312 | # ^ keyword 313 | # ^ module 314 | # ^ punctuation.delimiter 315 | # ^ string.special.symbol 316 | # ^ module 317 | # ^ keyword 318 | end 319 | # <- keyword 320 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | -------------------------------------------------------------------------------- /src/tree_sitter/array.h: -------------------------------------------------------------------------------- 1 | #ifndef TREE_SITTER_ARRAY_H_ 2 | #define TREE_SITTER_ARRAY_H_ 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #include "./alloc.h" 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #ifdef _MSC_VER 17 | #pragma warning(disable : 4101) 18 | #elif defined(__GNUC__) || defined(__clang__) 19 | #pragma GCC diagnostic push 20 | #pragma GCC diagnostic ignored "-Wunused-variable" 21 | #endif 22 | 23 | #define Array(T) \ 24 | struct { \ 25 | T *contents; \ 26 | uint32_t size; \ 27 | uint32_t capacity; \ 28 | } 29 | 30 | /// Initialize an array. 31 | #define array_init(self) \ 32 | ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL) 33 | 34 | /// Create an empty array. 35 | #define array_new() \ 36 | { NULL, 0, 0 } 37 | 38 | /// Get a pointer to the element at a given `index` in the array. 39 | #define array_get(self, _index) \ 40 | (assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index]) 41 | 42 | /// Get a pointer to the first element in the array. 43 | #define array_front(self) array_get(self, 0) 44 | 45 | /// Get a pointer to the last element in the array. 46 | #define array_back(self) array_get(self, (self)->size - 1) 47 | 48 | /// Clear the array, setting its size to zero. Note that this does not free any 49 | /// memory allocated for the array's contents. 50 | #define array_clear(self) ((self)->size = 0) 51 | 52 | /// Reserve `new_capacity` elements of space in the array. If `new_capacity` is 53 | /// less than the array's current capacity, this function has no effect. 54 | #define array_reserve(self, new_capacity) \ 55 | _array__reserve((Array *)(self), array_elem_size(self), new_capacity) 56 | 57 | /// Free any memory allocated for this array. Note that this does not free any 58 | /// memory allocated for the array's contents. 59 | #define array_delete(self) _array__delete((Array *)(self)) 60 | 61 | /// Push a new `element` onto the end of the array. 62 | #define array_push(self, element) \ 63 | (_array__grow((Array *)(self), 1, array_elem_size(self)), \ 64 | (self)->contents[(self)->size++] = (element)) 65 | 66 | /// Increase the array's size by `count` elements. 67 | /// New elements are zero-initialized. 68 | #define array_grow_by(self, count) \ 69 | do { \ 70 | if ((count) == 0) break; \ 71 | _array__grow((Array *)(self), count, array_elem_size(self)); \ 72 | memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \ 73 | (self)->size += (count); \ 74 | } while (0) 75 | 76 | /// Append all elements from one array to the end of another. 77 | #define array_push_all(self, other) \ 78 | array_extend((self), (other)->size, (other)->contents) 79 | 80 | /// Append `count` elements to the end of the array, reading their values from the 81 | /// `contents` pointer. 82 | #define array_extend(self, count, contents) \ 83 | _array__splice( \ 84 | (Array *)(self), array_elem_size(self), (self)->size, \ 85 | 0, count, contents \ 86 | ) 87 | 88 | /// Remove `old_count` elements from the array starting at the given `index`. At 89 | /// the same index, insert `new_count` new elements, reading their values from the 90 | /// `new_contents` pointer. 91 | #define array_splice(self, _index, old_count, new_count, new_contents) \ 92 | _array__splice( \ 93 | (Array *)(self), array_elem_size(self), _index, \ 94 | old_count, new_count, new_contents \ 95 | ) 96 | 97 | /// Insert one `element` into the array at the given `index`. 98 | #define array_insert(self, _index, element) \ 99 | _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element)) 100 | 101 | /// Remove one element from the array at the given `index`. 102 | #define array_erase(self, _index) \ 103 | _array__erase((Array *)(self), array_elem_size(self), _index) 104 | 105 | /// Pop the last element off the array, returning the element by value. 106 | #define array_pop(self) ((self)->contents[--(self)->size]) 107 | 108 | /// Assign the contents of one array to another, reallocating if necessary. 109 | #define array_assign(self, other) \ 110 | _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self)) 111 | 112 | /// Swap one array with another 113 | #define array_swap(self, other) \ 114 | _array__swap((Array *)(self), (Array *)(other)) 115 | 116 | /// Get the size of the array contents 117 | #define array_elem_size(self) (sizeof *(self)->contents) 118 | 119 | /// Search a sorted array for a given `needle` value, using the given `compare` 120 | /// callback to determine the order. 121 | /// 122 | /// If an existing element is found to be equal to `needle`, then the `index` 123 | /// out-parameter is set to the existing value's index, and the `exists` 124 | /// out-parameter is set to true. Otherwise, `index` is set to an index where 125 | /// `needle` should be inserted in order to preserve the sorting, and `exists` 126 | /// is set to false. 127 | #define array_search_sorted_with(self, compare, needle, _index, _exists) \ 128 | _array__search_sorted(self, 0, compare, , needle, _index, _exists) 129 | 130 | /// Search a sorted array for a given `needle` value, using integer comparisons 131 | /// of a given struct field (specified with a leading dot) to determine the order. 132 | /// 133 | /// See also `array_search_sorted_with`. 134 | #define array_search_sorted_by(self, field, needle, _index, _exists) \ 135 | _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists) 136 | 137 | /// Insert a given `value` into a sorted array, using the given `compare` 138 | /// callback to determine the order. 139 | #define array_insert_sorted_with(self, compare, value) \ 140 | do { \ 141 | unsigned _index, _exists; \ 142 | array_search_sorted_with(self, compare, &(value), &_index, &_exists); \ 143 | if (!_exists) array_insert(self, _index, value); \ 144 | } while (0) 145 | 146 | /// Insert a given `value` into a sorted array, using integer comparisons of 147 | /// a given struct field (specified with a leading dot) to determine the order. 148 | /// 149 | /// See also `array_search_sorted_by`. 150 | #define array_insert_sorted_by(self, field, value) \ 151 | do { \ 152 | unsigned _index, _exists; \ 153 | array_search_sorted_by(self, field, (value) field, &_index, &_exists); \ 154 | if (!_exists) array_insert(self, _index, value); \ 155 | } while (0) 156 | 157 | // Private 158 | 159 | typedef Array(void) Array; 160 | 161 | /// This is not what you're looking for, see `array_delete`. 162 | static inline void _array__delete(Array *self) { 163 | if (self->contents) { 164 | ts_free(self->contents); 165 | self->contents = NULL; 166 | self->size = 0; 167 | self->capacity = 0; 168 | } 169 | } 170 | 171 | /// This is not what you're looking for, see `array_erase`. 172 | static inline void _array__erase(Array *self, size_t element_size, 173 | uint32_t index) { 174 | assert(index < self->size); 175 | char *contents = (char *)self->contents; 176 | memmove(contents + index * element_size, contents + (index + 1) * element_size, 177 | (self->size - index - 1) * element_size); 178 | self->size--; 179 | } 180 | 181 | /// This is not what you're looking for, see `array_reserve`. 182 | static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) { 183 | if (new_capacity > self->capacity) { 184 | if (self->contents) { 185 | self->contents = ts_realloc(self->contents, new_capacity * element_size); 186 | } else { 187 | self->contents = ts_malloc(new_capacity * element_size); 188 | } 189 | self->capacity = new_capacity; 190 | } 191 | } 192 | 193 | /// This is not what you're looking for, see `array_assign`. 194 | static inline void _array__assign(Array *self, const Array *other, size_t element_size) { 195 | _array__reserve(self, element_size, other->size); 196 | self->size = other->size; 197 | memcpy(self->contents, other->contents, self->size * element_size); 198 | } 199 | 200 | /// This is not what you're looking for, see `array_swap`. 201 | static inline void _array__swap(Array *self, Array *other) { 202 | Array swap = *other; 203 | *other = *self; 204 | *self = swap; 205 | } 206 | 207 | /// This is not what you're looking for, see `array_push` or `array_grow_by`. 208 | static inline void _array__grow(Array *self, uint32_t count, size_t element_size) { 209 | uint32_t new_size = self->size + count; 210 | if (new_size > self->capacity) { 211 | uint32_t new_capacity = self->capacity * 2; 212 | if (new_capacity < 8) new_capacity = 8; 213 | if (new_capacity < new_size) new_capacity = new_size; 214 | _array__reserve(self, element_size, new_capacity); 215 | } 216 | } 217 | 218 | /// This is not what you're looking for, see `array_splice`. 219 | static inline void _array__splice(Array *self, size_t element_size, 220 | uint32_t index, uint32_t old_count, 221 | uint32_t new_count, const void *elements) { 222 | uint32_t new_size = self->size + new_count - old_count; 223 | uint32_t old_end = index + old_count; 224 | uint32_t new_end = index + new_count; 225 | assert(old_end <= self->size); 226 | 227 | _array__reserve(self, element_size, new_size); 228 | 229 | char *contents = (char *)self->contents; 230 | if (self->size > old_end) { 231 | memmove( 232 | contents + new_end * element_size, 233 | contents + old_end * element_size, 234 | (self->size - old_end) * element_size 235 | ); 236 | } 237 | if (new_count > 0) { 238 | if (elements) { 239 | memcpy( 240 | (contents + index * element_size), 241 | elements, 242 | new_count * element_size 243 | ); 244 | } else { 245 | memset( 246 | (contents + index * element_size), 247 | 0, 248 | new_count * element_size 249 | ); 250 | } 251 | } 252 | self->size += new_count - old_count; 253 | } 254 | 255 | /// A binary search routine, based on Rust's `std::slice::binary_search_by`. 256 | /// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`. 257 | #define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \ 258 | do { \ 259 | *(_index) = start; \ 260 | *(_exists) = false; \ 261 | uint32_t size = (self)->size - *(_index); \ 262 | if (size == 0) break; \ 263 | int comparison; \ 264 | while (size > 1) { \ 265 | uint32_t half_size = size / 2; \ 266 | uint32_t mid_index = *(_index) + half_size; \ 267 | comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \ 268 | if (comparison <= 0) *(_index) = mid_index; \ 269 | size -= half_size; \ 270 | } \ 271 | comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \ 272 | if (comparison == 0) *(_exists) = true; \ 273 | else if (comparison < 0) *(_index) += 1; \ 274 | } while (0) 275 | 276 | /// Helper macro for the `_sorted_by` routines below. This takes the left (existing) 277 | /// parameter by reference in order to work with the generic sorting function above. 278 | #define _compare_int(a, b) ((int)*(a) - (int)(b)) 279 | 280 | #ifdef _MSC_VER 281 | #pragma warning(default : 4101) 282 | #elif defined(__GNUC__) || defined(__clang__) 283 | #pragma GCC diagnostic pop 284 | #endif 285 | 286 | #ifdef __cplusplus 287 | } 288 | #endif 289 | 290 | #endif // TREE_SITTER_ARRAY_H_ 291 | -------------------------------------------------------------------------------- /docs/parser.md: -------------------------------------------------------------------------------- 1 | # Parser 2 | 3 | ## The AST 4 | 5 | When it comes to AST Elixir is a rather specific language due to its macro system. 6 | From the perspective of our parser, the important implication is that a seemingly 7 | invalid code can be a valid syntax when used in a macro (or just put in the `quote` 8 | expression). For example: 9 | 10 | ```elixir 11 | quote do 12 | def Bar.foo(x), definitely_not_do: 1 13 | %{a} 14 | */2 15 | end 16 | ``` 17 | 18 | As opposed to other languages, core constructs like `def`, `if` and `for` are not 19 | particularly special either, since they are itself regular functions (or macros rather). 20 | As a result, these constructs can be used "improperly" in a quoted expression, as shown above. 21 | 22 | Consequently, to correctly parse all Elixir code, we need the AST to closely match 23 | the Elixir AST. See [Elixir / Syntax reference](https://hexdocs.pm/elixir/syntax-reference.html) 24 | for more details. 25 | 26 | Whenever possible, we try using a more specific nodes (like binary/unary operator), but only 27 | to the extent that doesn't lose on generality. To get a sense of what the AST looks like, have 28 | a look at the tests in `test/corpus/`. 29 | 30 | ## Getting started with Tree-sitter 31 | 32 | For detailed introduction see the official guide on [Creating parsers](https://tree-sitter.github.io/tree-sitter/creating-parsers). 33 | 34 | Essentially, we define relevant language rules in `grammar.js`, based on which 35 | Tree-sitter generates parser code (under `src/`). In some cases, we want to write 36 | custom C code for tokenizing specific character sequences (in `src/scanner.c`). 37 | 38 | The grammar rules may often conflict with each other, meaning that the given 39 | sequence of tokens has multiple valid interpretations given one _token_ of lookahead. 40 | In many conflicts we always want to pick one interpretation over the other and we can 41 | do this by assigning different precedence and associativity to relevant rules, which 42 | tells the parser which way to go. 43 | 44 | For example given `expression1 * expression2 • *` the next token we _see_ ahead is `*`. 45 | The parser needs to decide whether `expression1 * expression2` is a complete binary operator 46 | node, or if it should await the next expression and interpret it as `expression1 * (expression2 * expression3)`. 47 | Since the `*` operator is left-associative we can use `prec.left` on the corresponding 48 | grammar rule, to inform the parser how to resolve this conflict. 49 | 50 | However, in some cases looking at one token ahead isn't enough, in which case we can add 51 | the conflicting rules to the `conflicts` list in the grammar. Whenever the parser stumbles 52 | upon this conflict it uses its GLR algorithm, basically considering both interpretations 53 | until one leads to parsing error. If both paths parse correctly (there's a genuine ambiguity) 54 | we can use dynamic precedence (`prec.dynamic`) to decide on the preferred path. 55 | 56 | ## Using the CLI 57 | 58 | ### tree-sitter 59 | 60 | ```shell 61 | # See CLI usage 62 | npx tree-sitter -h 63 | 64 | # Generate the the parser code based on grammar.js 65 | npx tree-sitter generate 66 | 67 | # Run tests 68 | npx tree-sitter test 69 | npx tree-sitter test --filter "access syntax" 70 | 71 | # Parse a specific file 72 | npx tree-sitter parse tmp/test.ex 73 | npx tree-sitter parse -x tmp/test.ex 74 | 75 | # Parse codebase to verify syntax coverage 76 | npx tree-sitter parse --quiet --stat 'tmp/elixir/**/*.ex*' 77 | ``` 78 | 79 | Whenever you make a change to `grammar.js` remember to run `generate`, 80 | before verifying the result. To test custom code, create an Elixir file 81 | like `tmp/test.ex` and use `parse` on it. The `-x` flag prints out the 82 | source grouped into AST nodes as XML. 83 | 84 | ### Additional scripts 85 | 86 | ```shell 87 | # Format the grammar.js file 88 | npm run format 89 | 90 | # Run parser against the given repository 91 | scripts/parse_repo.sh elixir-lang/elixir 92 | 93 | # Run parser against a predefined list of popular repositories 94 | scripts/integration_test.sh 95 | ``` 96 | 97 | ## Implementation notes 98 | 99 | This section covers some of the implementation decisions that have a more 100 | elaborated rationale. The individual subsections are referenced in the code. 101 | 102 | ### Ref 1. External scanner for quoted content 103 | 104 | We want to scan quoted content as a single token, but it requires lookahead. 105 | Specifically the `#` character may no longer be quoted content if followed by `{`. 106 | Also, inside heredoc string tokenizing `"` (or `'`) requires lookahead to know 107 | if it's already part of the end delimiter or not. 108 | 109 | Since we need to use external scanner, we need to know the delimiter type. 110 | One way to achieve this is using external scanner to scan the start delimiter 111 | and then storing its type on the parser stack. This approach requires the parser 112 | to allocate enough memory upfront and implement serialization/deserialization, 113 | which ideally would be avoided. To avoid this, we use a different approach! 114 | Instead of having a single `quoted_content` token, we have specific tokens for 115 | each quoted content type, such as `_quoted_content_i_single`, `_quoted_content_i_double`. 116 | Once the start delimiter is tokenized, we know which quoted content should be 117 | tokenized next, and from the token we can infer the end delimiter and whether 118 | it supports interpolation. In other words, we extract the information from the 119 | parsing state, rather than maintaining custom parser state. 120 | 121 | ### Ref 2. External scanner for newlines 122 | 123 | Generally newlines may appear in the middle of expressions and we ignore them 124 | as long as the expression is valid, that's why we list newline under extras. 125 | 126 | When a newline follows a complete expression, most of the time it should be 127 | treated as terminator. However, there are specific cases where the newline is 128 | non-breaking and treated as if it was just a space. This cases are: 129 | 130 | * call followed by newline and a `do end` block 131 | * expression followed by newline and a binary operator 132 | 133 | In both cases we want to tokenize the newline as non-breaking, so we use external 134 | scanner for lookahead. 135 | 136 | Note that the relevant rules already specify left/right associativity, so if we 137 | simply added `optional("\n")` the conflicts would be resolved immediately rather 138 | without using GLR. 139 | 140 | Additionally, since comments may appear anywhere and don't change the context, 141 | we also tokenize newlines before comments as non-breaking. 142 | 143 | ### Ref 3. External scanner for unary + and - 144 | 145 | Plus and minus are either binary or unary operators, depending on the context. 146 | Consider the following variants 147 | 148 | ``` 149 | a + b 150 | a+b 151 | a+ b 152 | a +b 153 | ``` 154 | 155 | In the first three expressions `+` is a binary operator, while in the last one 156 | `+` is an unary operator referring to local call argument. 157 | 158 | To correctly tokenize all cases we use external scanner to tokenize a special empty 159 | token (`_before_unary_operator`) when the spacing matches `a +b`, which forces the 160 | parser to pick the unary operator path. 161 | 162 | ### Ref 4. External scanner for `not in` 163 | 164 | The `not in` operator may have an arbitrary inline whitespace between `not` and `in`. 165 | 166 | We cannot use a regular expression like `/not[ \t]+in/`, because it would also match 167 | in expressions like `a not inn` as the longest matching token. 168 | 169 | A possible solution could be `seq("not", "in")` with dynamic conflict resolution, but 170 | then we tokenize two separate tokens. Also to properly handle `a not inn`, we would need 171 | keyword extraction, which causes problems in our case (https://github.com/tree-sitter/tree-sitter/issues/1404). 172 | 173 | In the end it's easiest to use external scanner, so that we can skip inline whitespace 174 | and ensure token ends after `in`. 175 | 176 | ### Ref 5. External scanner for quoted atom start 177 | 178 | For parsing quoted atom `:` we could make the `"` (or `'`) token immediate, however this 179 | would require adding immediate rules for single/double quoted content and listing them 180 | in relevant places. We could definitely do that, but using external scanner is actually 181 | simpler. 182 | 183 | ### Ref 6. Identifier pattern 184 | 185 | See [Elixir / Unicode Syntax](https://hexdocs.pm/elixir/unicode-syntax.html) for official 186 | notes. 187 | 188 | Tree-sitter already supports unicode properties in regular expressions, however character 189 | class subtraction is not supported. 190 | 191 | For the base `` and `` we can use `[\p{ID_Start}]` and `[\p{ID_Continue}]` 192 | respectively, since both are supported and according to the 193 | [Unicode Annex #31](https://unicode.org/reports/tr31/#Table_Lexical_Classes_for_Identifiers) 194 | they match the ranges listed in the Elixir docs. 195 | 196 | For atoms this translates to a clean regular expression. 197 | 198 | For variables however, we want to exclude uppercase (`\p{Lu}`) and titlecase (`\p{Lt}`) 199 | categories from `\p{ID_Start}`. As already mentioned, we cannot use group subtraction 200 | in the regular expression, so instead we need to create a suitable group of characters 201 | on our own. 202 | 203 | After removing the uppercase/titlecase categories from `[\p{ID_Start}]`, we obtain the 204 | following group: 205 | 206 | `[\p{Ll}\p{Lm}\p{Lo}\p{Nl}\p{Other_ID_Start}-\p{Pattern_Syntax}-\p{Pattern_White_Space}]` 207 | 208 | At the time of writing the subtracted groups actually only remove a single character: 209 | 210 | ```elixir 211 | Mix.install([{:unicode_set, "~> 1.1"}]) 212 | 213 | Unicode.Set.to_utf8_char( 214 | "[[[:Ll:][:Lm:][:Lo:][:Nl:][:Other_ID_Start:]] & [[:Pattern_Syntax:][:Pattern_White_Space:]]]" 215 | ) 216 | #=> {:ok, [11823]} 217 | ``` 218 | 219 | Consequently, by removing the subtraction we allow just one additional (not common) character, 220 | which is perfectly acceptable. 221 | 222 | It's important to note that JavaScript regular expressions don't support the `\p{Other_ID_Start}` 223 | unicode category. Fortunately this category is a small set of characters introduces for 224 | [backward compatibility](https://unicode.org/reports/tr31/#Backward_Compatibility), so we can 225 | enumerate it manually: 226 | 227 | ```elixir 228 | Mix.install([{:unicode_set, "~> 1.1"}]) 229 | 230 | Unicode.Set.to_utf8_char("[[[:Other_ID_Start:]] - [[:Pattern_Syntax:][:Pattern_White_Space:]]]") 231 | |> elem(1) 232 | |> Enum.flat_map(fn 233 | n when is_number(n) -> [n] 234 | range -> range 235 | end) 236 | |> Enum.map(&Integer.to_string(&1, 16)) 237 | #=> ["1885", "1886", "2118", "212E", "309B", "309C"] 238 | ``` 239 | 240 | Finally, we obtain this regular expression group for variable ``: 241 | 242 | `[\p{Ll}\p{Lm}\p{Lo}\p{Nl}\u1885\u1886\u2118\u212E\u309B\u309C]` 243 | 244 | ### Ref 7. Keyword token 245 | 246 | We tokenize the whole keyword sequence like `do: ` as a single token. 247 | Ideally we wouldn't include the whitespace, but since we use `token` 248 | it gets include. However, this is an intentionally accepted tradeoff, 249 | because using `token` significantly simplifies the grammar and avoids 250 | conflicts. 251 | 252 | The alternative approach would be to define keyword as `seq(alias(choice(...), $._keyword_literal), $._keyword_end)`, 253 | where we list all other tokens that make for for valid keyword literal 254 | and use custom scanner for `_keyword_end` to look ahead without tokenizing 255 | the whitespace. However, this approach generates a number of conflicts 256 | because `:` is tokenized separately and phrases like `fun fun • do` or 257 | `fun • {}` are ambiguous (interpretation depends on whether `:` comes next). 258 | Resolving some of these conflicts (for instance special keywords like `{}` or `%{}`) 259 | requires the use of external scanner. Given the complexities this approach 260 | brings to the grammar, and consequently the parser, we stick to the simpler 261 | approach. 262 | 263 | ### Ref 8. Empty anonymous function 264 | 265 | As opposed to the Elixir parser, we successfully parse anonymous functions with 266 | no stab clauses, so this is valid: 267 | 268 | ``` 269 | x = fn 270 | 271 | end 272 | ``` 273 | 274 | This code may appear if an editor extension automatically inserts `end` after 275 | `fn`. We want it to parse as anonymous function node, so that functionality such 276 | as indentation works as expected. 277 | 278 | If we require at least one stab clause, the above would be parsed with an error, 279 | where `fn` and `end` are both identifiers. That is not useful. Note that both 280 | `fn` and `end` are reserved keywords, so there is no case where they would 281 | actually be identifiers, hence no ambiguity. 282 | 283 | Ideally, this would parse it as an anonymous function node with an error, however 284 | that does not seem straightforward to achieve. 285 | --------------------------------------------------------------------------------