├── .codeclimate.yml ├── .editorconfig ├── .github └── workflows │ └── tokentree.yml ├── .gitignore ├── .haxerc ├── CHANGELOG.md ├── LICENSE ├── README.md ├── buildToken.hxml ├── checkstyle.json ├── haxe_libraries ├── exception.hxml ├── haxeparser.hxml ├── hxjsonast.hxml ├── hxparse.hxml ├── instrument.hxml ├── json2object.hxml ├── safety.hxml ├── test-adapter.hxml └── utest.hxml ├── haxelib.json ├── hxformat.json ├── makeReleaseZip.sh ├── package-lock.json ├── package.json ├── src ├── import.hx └── tokentree │ ├── ToTokenTreeDef.hx │ ├── TokenStream.hx │ ├── TokenStreamProgress.hx │ ├── TokenTree.hx │ ├── TokenTreeAccessHelper.hx │ ├── TokenTreeBuilder.hx │ ├── TokenTreeDef.hx │ ├── TokenTreeDefPrinter.hx │ ├── utils │ ├── FieldUtils.hx │ └── TokenTreeCheckUtils.hx │ └── walk │ ├── WalkAbstract.hx │ ├── WalkArrayAccess.hx │ ├── WalkAt.hx │ ├── WalkBinopSub.hx │ ├── WalkBlock.hx │ ├── WalkClass.hx │ ├── WalkComment.hx │ ├── WalkDoWhile.hx │ ├── WalkEnum.hx │ ├── WalkExtends.hx │ ├── WalkFieldDef.hx │ ├── WalkFile.hx │ ├── WalkFinal.hx │ ├── WalkFor.hx │ ├── WalkFunction.hx │ ├── WalkIf.hx │ ├── WalkImplements.hx │ ├── WalkInterface.hx │ ├── WalkLtGt.hx │ ├── WalkNew.hx │ ├── WalkPOpen.hx │ ├── WalkPackageImport.hx │ ├── WalkQuestion.hx │ ├── WalkSharp.hx │ ├── WalkStatement.hx │ ├── WalkSwitch.hx │ ├── WalkTry.hx │ ├── WalkType.hx │ ├── WalkTypeNameDef.hx │ ├── WalkTypedef.hx │ ├── WalkTypedefBody.hx │ ├── WalkVar.hx │ └── WalkWhile.hx ├── test.hxml ├── test ├── TestMain.hx ├── TokenTestMain.hx ├── import.hx └── tokentree │ ├── TestTokenTreeBuilder.hx │ ├── TokenStreamTest.hx │ ├── TokenTreeBuilderParsingTest.hx │ ├── TokenTreeBuilderTest.hx │ ├── utils │ ├── FieldUtilsTest.hx │ └── TokenTreeCheckUtilsTest.hx │ ├── verify │ ├── IVerifyTokenTree.hx │ ├── VerifyTokenTree.hx │ ├── VerifyTokenTreeBase.hx │ ├── VerifyTokenTreeList.hx │ └── VerifyTokenTreeTest.hx │ └── walk │ ├── WalkFileTest.hx │ ├── WalkIfTest.hx │ └── WalkWhileTest.hx └── testWithoutCoverage.hxml /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | haxe-checkstyle: 3 | enabled: true 4 | markdownlint: 5 | enabled: true 6 | checks: 7 | MD002: 8 | enabled: false 9 | MD013: 10 | enabled: false 11 | MD033: 12 | enabled: false 13 | 14 | exclude_paths: 15 | - resources/**/* 16 | 17 | ratings: 18 | paths: 19 | - "**.hx" 20 | - "**.md" -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is for unifying the coding style for different editors and IDEs. 2 | # More information at http://EditorConfig.org 3 | root = true 4 | 5 | [*] 6 | indent_style = tab 7 | indent_size = 4 8 | insert_final_newline = false 9 | trim_trailing_whitespace = true 10 | 11 | [*.hx] 12 | end_of_line = lf 13 | charset = utf-8 14 | 15 | [*.md] 16 | indent_style = space 17 | indent_size = 2 18 | 19 | [*.yml] 20 | indent_style = space 21 | indent_size = 2 -------------------------------------------------------------------------------- /.github/workflows/tokentree.yml: -------------------------------------------------------------------------------- 1 | name: Haxe-Tokentree 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | schedule: 11 | - cron: '15 23 * * *' 12 | 13 | jobs: 14 | build: 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | haxe-version: ['4.3.6', 'nightly'] 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Use Node.js 18 22 | uses: actions/setup-node@v4 23 | with: 24 | node-version: 18 25 | - name: Run npm install 26 | run: npm ci 27 | - name: Cache Lix repository state 28 | uses: actions/cache@v3 29 | with: 30 | path: ~/haxe 31 | key: ${{ runner.os }}-${{ matrix.haxe-version }}-${{ hashFiles('haxe_libraries/*.hxml') }} 32 | - name: Install Haxe version ${{ matrix.haxe-version }} 33 | run: | 34 | npx lix download haxe ${{ matrix.haxe-version }} 35 | npx lix use haxe ${{ matrix.haxe-version }} 36 | - name: Run lix download 37 | run: npx lix download 38 | - name: Print versions 39 | run: | 40 | npx haxe -version 41 | npx neko -version 42 | npx haxelib list 43 | - name: Clone checkstyle 44 | run: git clone -b dev https://github.com/HaxeCheckstyle/haxe-checkstyle.git --depth=1 checkstyle 45 | - name: Run npm install for checkstyle 46 | run: | 47 | (cd checkstyle; npm install) 48 | - name: Install Haxe version ${{ matrix.haxe-version }} for checkstyle 49 | run: | 50 | (cd checkstyle; npx lix download haxe ${{ matrix.haxe-version }}) 51 | (cd checkstyle; npx lix use haxe ${{ matrix.haxe-version }}) 52 | - name: Run lix download for checkstyle 53 | run: (cd checkstyle; npx lix download) 54 | - name: Setup tokentree dev for checkstyle 55 | run: | 56 | (cd checkstyle/haxe_libraries; echo "-cp ../src" > tokentree.hxml) 57 | - name: Run eval tests (with coverage) 58 | run: npx haxe test.hxml 59 | - name: Build and test checkstyle 60 | if: matrix.haxe-version != 'nightly' 61 | run: (cd checkstyle && npx haxe buildAll.hxml && rm -f coverage.json) 62 | - name: Run Node version with --check 63 | if: matrix.haxe-version != 'nightly' 64 | run: npx neko checkstyle/run.n -s src -s test -r xml 65 | - name: Coverage report 66 | if: success() && (matrix.haxe-version != 'nightly') 67 | uses: livewing/lcov-job-summary@v1.1.0 68 | with: 69 | lcov: lcov.info 70 | - name: Checkstyle report 71 | if: success() && (matrix.haxe-version != 'nightly') 72 | uses: jwgmeligmeyling/checkstyle-github-action@master 73 | with: 74 | path: '**/check-style-report.xml' -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .idea 3 | .vscode 4 | .temp 5 | *.iws 6 | *.log 7 | *.cache 8 | .DS_Store 9 | *.n 10 | *.zip 11 | .haxelib.*/ 12 | out/ 13 | node_modules 14 | .DS_Store 15 | log/ 16 | target/ 17 | .unittest/ 18 | dump/ 19 | haxe3/ 20 | checkstyle-report.json 21 | checkstyle-report.xml 22 | coverage.json 23 | lcov.info 24 | display.hxml 25 | codecov.json 26 | flame.json 27 | tokenmain* -------------------------------------------------------------------------------- /.haxerc: -------------------------------------------------------------------------------- 1 | { 2 | "version": "4.3.6", 3 | "resolveLibs": "scoped" 4 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Version History 2 | 3 | ## dev branch / next version (1.x.x) 4 | 5 | ## version 1.2.18 (2024-11-01) 6 | 7 | - Fixed TokenTreeCheckUtils.getBrOpenType detection of pattern extraction block 8 | 9 | ## version 1.2.17 (2024-10-04) 10 | 11 | - Fixed nested conditionals 12 | 13 | ## version 1.2.16 (2024-10-03) 14 | 15 | - Fixed conditionals after PClose 16 | 17 | ## version 1.2.15 (2024-10-02) 18 | 19 | - Fixed operator keyword used in Haxe 3 code 20 | 21 | ## version 1.2.14 (2024-09-18) 22 | 23 | - Fixed getArrowType running into a null pointer exception with conditionals 24 | 25 | ## version 1.2.13 (2024-08-02) 26 | 27 | - Fixed metadata for parens 28 | 29 | ## version 1.2.12 (2024-07-30) 30 | 31 | - Fixed parsing a single `false`, `true` or `null` 32 | 33 | ## version 1.2.11 (2024-02-10) 34 | 35 | - Added getBkOpenType utils function 36 | - Fixed empty switch cases with comments 37 | 38 | ## version 1.2.10 (2023-05-21) 39 | 40 | - Fixed extends handling in interfaces 41 | 42 | ## version 1.2.9 (2023-05-21) 43 | 44 | - Fixed unicode in regexp, fixes [#219](https://github.com/HaxeCheckstyle/tokentree/issues/219) 45 | 46 | ## version 1.2.8 (2023-02-22) 47 | 48 | - Fixed inline call() and inline new handling ([#217](https://github.com/HaxeCheckstyle/tokentree/issues/217)) 49 | 50 | ## version 1.2.7 (2023-02-15) 51 | 52 | - Fixed type parameter detection ([#216](https://github.com/HaxeCheckstyle/tokentree/issues/216)) 53 | 54 | ## version 1.2.6 (2023-02-06) 55 | 56 | - Fixed pos info of Root nodes, fixes [#215](https://github.com/HaxeCheckstyle/tokentree/issues/215) 57 | 58 | ## version 1.2.5 (2022-12-03) 59 | 60 | - Fixed ternary detection ([#214](https://github.com/HaxeCheckstyle/tokentree/issues/214)) 61 | 62 | ## version 1.2.4 (2022-12-01) 63 | 64 | - Added findLastBinop utils function ([#213](https://github.com/HaxeCheckstyle/tokentree/issues/213)) 65 | - Fixed macro type hint ([#213](https://github.com/HaxeCheckstyle/tokentree/issues/213)) 66 | - Fixed enum type parameter ([#213](https://github.com/HaxeCheckstyle/tokentree/issues/213)) 67 | 68 | ## version 1.2.3 (2022-09-14) 69 | 70 | - Fixed suffixes for negative numeric literals 71 | 72 | ## version 1.2.2 (2022-09-14) 73 | 74 | - Updated haxeparser to support latest Haxe nightly syntax 75 | 76 | ## version 1.2.1 (2022-09-07) 77 | 78 | - Moved inline markup code to haxeparser ([#212](https://github.com/HaxeCheckstyle/tokentree/issues/212)) 79 | 80 | ## version 1.2.0 (2022-08-27) 81 | 82 | - Added support for inline markup ([#211](https://github.com/HaxeCheckstyle/tokentree/issues/211)) 83 | 84 | ## version 1.1.2 (2022-04-23) 85 | 86 | - Added support for default type parameter ([#207](https://github.com/HaxeCheckstyle/tokentree/issues/207)) 87 | - Added support for static locals ([#208](https://github.com/HaxeCheckstyle/tokentree/issues/208) + [#209](https://github.com/HaxeCheckstyle/tokentree/issues/209)) 88 | - Fixed Dot handling for new Class() instances ([#210](https://github.com/HaxeCheckstyle/tokentree/issues/210)) 89 | 90 | ## version 1.1.1 (2021-01-22) 91 | 92 | - Fixed POpen after block curlies ([#206](https://github.com/HaxeCheckstyle/tokentree/issues/206)) 93 | 94 | ## version 1.1.0 (2020-12-23) 95 | 96 | - Retired Haxe 3.4.7 compile support ([#202](https://github.com/HaxeCheckstyle/tokentree/issues/202)) 97 | - Upgraded to latest haxeparser version ([#202](https://github.com/HaxeCheckstyle/tokentree/issues/202) + [#204](https://github.com/HaxeCheckstyle/tokentree/issues/204)) 98 | - Added testcases for ExpressionLevel code ([#202](https://github.com/HaxeCheckstyle/tokentree/issues/202)) 99 | - Added stream end detection to POpen and array access parsing ([#202](https://github.com/HaxeCheckstyle/tokentree/issues/202)) 100 | - Added overload support ([#204](https://github.com/HaxeCheckstyle/tokentree/issues/204)) 101 | - Fixed function type parameter ([#203](https://github.com/HaxeCheckstyle/tokentree/issues/203)) 102 | 103 | ## version 1.0.31 (2020-11-04) 104 | 105 | - Fixed type parameter with array ([#201](https://github.com/HaxeCheckstyle/tokentree/issues/201)) 106 | 107 | ## version 1.0.30 (2020-11-02) 108 | 109 | - Fixed arrow functions ([#200](https://github.com/HaxeCheckstyle/tokentree/issues/200)) 110 | - Fixed type parameter detection ([#200](https://github.com/HaxeCheckstyle/tokentree/issues/200)) 111 | - Fixed Dollar handling ([#200](https://github.com/HaxeCheckstyle/tokentree/issues/200)) 112 | 113 | ## version 1.0.29 (2020-11-01) 114 | 115 | - Added TokenTreeDef ([#191](https://github.com/HaxeCheckstyle/tokentree/issues/191)) 116 | - Added support for abstract classes / interfaces ([#195](https://github.com/HaxeCheckstyle/tokentree/issues/195)) 117 | - Added hasTempStore function ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 118 | - Fixed negative const / Binop(OpSub) detection ([#189](https://github.com/HaxeCheckstyle/tokentree/issues/189)) 119 | - Fixed final with multiple vars ([#191](https://github.com/HaxeCheckstyle/tokentree/issues/191)) 120 | - Fixed is operator ([#194](https://github.com/HaxeCheckstyle/tokentree/issues/194)) 121 | - Fixed semicolon with multiple binops ([#197](https://github.com/HaxeCheckstyle/tokentree/issues/197) + [#198](https://github.com/HaxeCheckstyle/tokentree/issues/198)) 122 | - Fixed Dollar handling ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 123 | - Fixed expression handling after BkClose ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 124 | - Fixed expression handling after BrClose ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 125 | - Fixed POpen type detection ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 126 | - Fixed new handling ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 127 | - Fixed sharp handling for function type hint ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 128 | - Fixed macro complextype ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 129 | - Fixed array access on new ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 130 | - Fixed arrow functions ([#199](https://github.com/HaxeCheckstyle/tokentree/issues/199)) 131 | - Refactored enums to use CamelCase ([#191](https://github.com/HaxeCheckstyle/tokentree/issues/191)) 132 | - Refactored to reduce usage of Type.enumEq ([#195](https://github.com/HaxeCheckstyle/tokentree/issues/195) + [#196](https://github.com/HaxeCheckstyle/tokentree/issues/196)) 133 | - Removed `is` operator ([#195](https://github.com/HaxeCheckstyle/tokentree/issues/195)) 134 | 135 | ## version 1.0.28 (2020-06-07) 136 | 137 | - Expanded `POpen` types in getPOpenType to distinguish between different condition expressions ([#185](https://github.com/HaxeCheckstyle/tokentree/issues/185)) 138 | - Fixed type hint detection for anon types ([#186](https://github.com/HaxeCheckstyle/tokentree/issues/186)) 139 | - Fixed final position for Haxe 3 ([#186](https://github.com/HaxeCheckstyle/tokentree/issues/186)) 140 | - Fixed @in handling ([#187](https://github.com/HaxeCheckstyle/tokentree/issues/187)) 141 | 142 | ## version 1.0.27 (2020-04-18) 143 | 144 | - Fixed null pointer exception when pasrsing `private final class`, fixes [#183](https://github.com/HaxeCheckstyle/tokentree/issues/183) ([#184](https://github.com/HaxeCheckstyle/tokentree/issues/184)) 145 | 146 | ## version 1.0.26 (2020-04-12) 147 | 148 | - Fixed `Binop(OpOr)` position for non `case` places ([#182](https://github.com/HaxeCheckstyle/tokentree/issues/182)) 149 | - Fixed doc comments and metadata ([#182](https://github.com/HaxeCheckstyle/tokentree/issues/182)) 150 | 151 | ## version 1.0.25 (2020-04-11) 152 | 153 | - Added support for module level statics [#175](https://github.com/HaxeCheckstyle/tokentree/issues/175) 154 | - Added final handling for interfaces [#176](https://github.com/HaxeCheckstyle/tokentree/issues/176) 155 | - Fixed unary `OpSub` handling [#174](https://github.com/HaxeCheckstyle/tokentree/issues/174) 156 | - Fixed arrow function detection [#175](https://github.com/HaxeCheckstyle/tokentree/issues/175) + [#177](https://github.com/HaxeCheckstyle/tokentree/issues/177) 157 | - Fixed `POpen` type detection [#177](https://github.com/HaxeCheckstyle/tokentree/issues/177) + [#180](https://github.com/HaxeCheckstyle/tokentree/issues/180) 158 | - Fixed `BrOpen` type detection [#178](https://github.com/HaxeCheckstyle/tokentree/issues/178) 159 | - Fixed `Binop(OpOr)` position [#178](https://github.com/HaxeCheckstyle/tokentree/issues/178) 160 | - Fixed expression metadata position [#179](https://github.com/HaxeCheckstyle/tokentree/issues/179) 161 | - Fixed `Binop(OpBoolOr)` and `Binop(OpBoolAnd)` positions [#180](https://github.com/HaxeCheckstyle/tokentree/issues/180) 162 | 163 | ## version 1.0.24 (2019-12-01) 164 | 165 | - Added comment handling after `PClose` [#172](https://github.com/HaxeCheckstyle/tokentree/issues/172) 166 | - Added entry point TYPE_HINT_LEVEL, fixes [#147](https://github.com/HaxeCheckstyle/tokentree/issues/147) 167 | - Fixed semicolon handling after null, true or false [#171](https://github.com/HaxeCheckstyle/tokentree/issues/171) 168 | - Refactored build system to use lix [#173](https://github.com/HaxeCheckstyle/tokentree/issues/173) 169 | 170 | ## version 1.0.23 (2019-09-10) 171 | 172 | - Fixed postfix exclamation mark [#165](https://github.com/HaxeCheckstyle/tokentree/issues/165) 173 | - Fixed C# exception handling [#166](https://github.com/HaxeCheckstyle/tokentree/issues/166) 174 | - Fixed null pointer issues in TokenTreeCheckUtils [#167](https://github.com/HaxeCheckstyle/tokentree/issues/167) 175 | - Fixed endless loop in for loops with macro reification, fixes [#168](https://github.com/HaxeCheckstyle/tokentree/issues/168) ([#169](https://github.com/HaxeCheckstyle/tokentree/issues/169)) 176 | 177 | ## version 1.0.22 (2019-06-13) 178 | 179 | - Fixed type hint detection in enums and overload metas [#161](https://github.com/HaxeCheckstyle/tokentree/issues/161) 180 | - Fixed comments in OpAdd chains [#162](https://github.com/HaxeCheckstyle/tokentree/issues/162) 181 | - Fixed imports with conditionals [#163](https://github.com/HaxeCheckstyle/tokentree/issues/163) 182 | 183 | ## version 1.0.21 (2019-06-01) 184 | 185 | - Fixed conditionals using dot idents without parens [#159](https://github.com/HaxeCheckstyle/tokentree/issues/159) 186 | 187 | ## version 1.0.20 (2019-05-17) 188 | 189 | - Fixed potential null pointer in `TokenTreeAccessHelper.findParent` [#157](https://github.com/HaxeCheckstyle/tokentree/issues/157) 190 | 191 | ## version 1.0.19 (2019-05-17) 192 | 193 | - Added `TokenTreeAccessHelper.findParent` [#150](https://github.com/HaxeCheckstyle/tokentree/issues/150) 194 | - Added a cache for getArrowType, getBrOpenType, getColonType and getPOpenType results [#155](https://github.com/HaxeCheckstyle/tokentree/issues/155) 195 | - Fixed `TokenTreeCheckUtils.getColonType` detection of type checks [#149](https://github.com/HaxeCheckstyle/tokentree/issues/149) 196 | - Fixed `OpBool` position in case body [#151](https://github.com/HaxeCheckstyle/tokentree/issues/151) 197 | - Fixed object literal detection when using expression entry point [#152](https://github.com/HaxeCheckstyle/tokentree/issues/152) + [#153](https://github.com/HaxeCheckstyle/tokentree/issues/153) 198 | - Fixed `TokenTreeCheckUtils.getColonType` in if body [#154](https://github.com/HaxeCheckstyle/tokentree/issues/154) 199 | - Fixed `TokenTreeCheckUtils.getPOpenType` in if body [#154](https://github.com/HaxeCheckstyle/tokentree/issues/154) 200 | 201 | ## version 1.0.18 (2019-03-22) 202 | 203 | - Added `TokenTreeEntryPoint` to `TokenTreeBuilder` [#146](https://github.com/HaxeCheckstyle/tokentree/issues/146) 204 | 205 | ## version 1.0.17 (2019-03-06) 206 | 207 | - Fixed wrong position of elements preceeding inserted dummy tokens in relax mode [#145](https://github.com/HaxeCheckstyle/tokentree/issues/145) 208 | 209 | ## version 1.0.16 (2019-03-04) 210 | 211 | - Added `TokenTreeCheckUtils.isMetadata` [#143](https://github.com/HaxeCheckstyle/tokentree/issues/143) 212 | - Fixed handling of OpBool after `null` [#140](https://github.com/HaxeCheckstyle/tokentree/issues/140) 213 | - Fixed comments before `catch` [#140](https://github.com/HaxeCheckstyle/tokentree/issues/140) 214 | - Fixed BrOpen detection for type parameter with structure type [#140](https://github.com/HaxeCheckstyle/tokentree/issues/140) 215 | - Fixed colon type detection when using conditionals [#140](https://github.com/HaxeCheckstyle/tokentree/issues/140) 216 | - Fixed colon type detection when using OpBool [#140](https://github.com/HaxeCheckstyle/tokentree/issues/140) 217 | - Fixed handling of condition bodies (if, while, for) [#141](https://github.com/HaxeCheckstyle/tokentree/issues/141) 218 | - Fixed POpen type detection for conditionals [#141](https://github.com/HaxeCheckstyle/tokentree/issues/141) 219 | - Fixed comments in abstracts `from` and `to` [#142](https://github.com/HaxeCheckstyle/tokentree/issues/142) 220 | - Fixed arrow type detection with comments [#142](https://github.com/HaxeCheckstyle/tokentree/issues/142) 221 | - Fixed position of `OpAdd` / `OpSub` [#142](https://github.com/HaxeCheckstyle/tokentree/issues/142) 222 | - Fixed detection of "unary" `OpSub` [#142](https://github.com/HaxeCheckstyle/tokentree/issues/142) 223 | 224 | ## version 1.0.15 (2019-02-07) 225 | 226 | - Fixed `TokenTreeCheckUtils.getColonType` detection of type check in array comprehension [#136](https://github.com/HaxeCheckstyle/tokentree/issues/136) 227 | - Fixed handling of multiple `implements` [#137](https://github.com/HaxeCheckstyle/tokentree/issues/137) 228 | - Fixed comments after typedefs without semicolon [#138](https://github.com/HaxeCheckstyle/tokentree/issues/138) 229 | 230 | ## version 1.0.14 (2018-12-05) 231 | 232 | - Fixed properties in anon types [#130](https://github.com/HaxeCheckstyle/tokentree/issues/130) 233 | - Fixed `return` in ternary [#132](https://github.com/HaxeCheckstyle/tokentree/issues/132) 234 | - Fixed handling of `;` [#133](https://github.com/HaxeCheckstyle/tokentree/issues/133) + [#134](https://github.com/HaxeCheckstyle/tokentree/issues/134) 235 | - Refactored coverage report generation [#131](https://github.com/HaxeCheckstyle/tokentree/issues/131) 236 | 237 | ## version 1.0.13 (2018-11-11) 238 | 239 | - Added testcase for multiline string interpolation [#126](https://github.com/HaxeCheckstyle/tokentree/issues/126) 240 | - Fixed handling of OpBool (`||` and `&&`) chains [#124](https://github.com/HaxeCheckstyle/tokentree/issues/124) 241 | - Fixed handling of Comma after `#end` [#124](https://github.com/HaxeCheckstyle/tokentree/issues/124) 242 | - Fixed conditionals in interface definition [#125](https://github.com/HaxeCheckstyle/tokentree/issues/125) 243 | - Fixed handling of Arrow in arrow functions [#127](https://github.com/HaxeCheckstyle/tokentree/issues/127) 244 | - Fixed POpen detection in arrow functions [#127](https://github.com/HaxeCheckstyle/tokentree/issues/127) 245 | - Fixed OpBool chain handling [#128](https://github.com/HaxeCheckstyle/tokentree/issues/128) 246 | 247 | ## version 1.0.12 (2018-10-15) 248 | 249 | - Added support for final class / interface, fixes [#118](https://github.com/HaxeCheckstyle/tokentree/issues/118) ([#119](https://github.com/HaxeCheckstyle/tokentree/issues/119)) 250 | - Added support for key-value iterators [#120](https://github.com/HaxeCheckstyle/tokentree/issues/120) 251 | - Fixed handling of metadata with parameter [#121](https://github.com/HaxeCheckstyle/tokentree/issues/121) 252 | - Fixed comments in method chains [#122](https://github.com/HaxeCheckstyle/tokentree/issues/122) 253 | 254 | ## version 1.0.11 (2018-09-03) 255 | 256 | - Fixed position of semicolon [#113](https://github.com/HaxeCheckstyle/tokentree/issues/113) 257 | - Fixed handling of unary expressions [#113](https://github.com/HaxeCheckstyle/tokentree/issues/113) 258 | - Fixed detection of type check on numbers [#114](https://github.com/HaxeCheckstyle/tokentree/issues/114) 259 | - Fixed detection of type parameters with anon types [#115](https://github.com/HaxeCheckstyle/tokentree/issues/115) 260 | - Fixed handling of typedef without assign [#116](https://github.com/HaxeCheckstyle/tokentree/issues/116) 261 | 262 | ## version 1.0.10 (2018-08-20) 263 | 264 | - Added `TokenTreeCheckUtils.getColonType()` [#99](https://github.com/HaxeCheckstyle/tokentree/issues/99) + [#100](https://github.com/HaxeCheckstyle/tokentree/issues/100) + [#101](https://github.com/HaxeCheckstyle/tokentree/issues/101) 265 | - Fixed implements / extends handling [#94](https://github.com/HaxeCheckstyle/tokentree/issues/94) 266 | - Fixed endless loop with macro patterns [#94](https://github.com/HaxeCheckstyle/tokentree/issues/94) 267 | - Fixed macro class $name [#94](https://github.com/HaxeCheckstyle/tokentree/issues/94) 268 | - Fixed conditionals in new [#95](https://github.com/HaxeCheckstyle/tokentree/issues/95) 269 | - Fixed object literal detection [#96](https://github.com/HaxeCheckstyle/tokentree/issues/96) 270 | - Fixed `getBrOpenType` detection of blocks [#96](https://github.com/HaxeCheckstyle/tokentree/issues/96) + [#99](https://github.com/HaxeCheckstyle/tokentree/issues/99) + [#107](https://github.com/HaxeCheckstyle/tokentree/issues/107) + [#110](https://github.com/HaxeCheckstyle/tokentree/issues/110) 271 | - Fixed `getArrowType` detection [#97](https://github.com/HaxeCheckstyle/tokentree/issues/97) + [#110](https://github.com/HaxeCheckstyle/tokentree/issues/110) + [#111](https://github.com/HaxeCheckstyle/tokentree/issues/111) 272 | - Fixed position of trailing comments in switch/cases [#98](https://github.com/HaxeCheckstyle/tokentree/issues/98) 273 | - Fixed position of type checks on object literals [#99](https://github.com/HaxeCheckstyle/tokentree/issues/99) 274 | - Fixed `getPOpenType` parameter detection [#100](https://github.com/HaxeCheckstyle/tokentree/issues/100) + [#101](https://github.com/HaxeCheckstyle/tokentree/issues/101) + [#104](https://github.com/HaxeCheckstyle/tokentree/issues/104) 275 | - Fixed ternary position [#102](https://github.com/HaxeCheckstyle/tokentree/issues/102) + [#105](https://github.com/HaxeCheckstyle/tokentree/issues/105) + [#108](https://github.com/HaxeCheckstyle/tokentree/issues/108) + [#109](https://github.com/HaxeCheckstyle/tokentree/issues/109) 276 | - Fixed structure extension position of `&` [#103](https://github.com/HaxeCheckstyle/tokentree/issues/103) 277 | - Fixed `do…while` handling [#105](https://github.com/HaxeCheckstyle/tokentree/issues/105) 278 | - Fixed "interesting" use of conditionals [#106](https://github.com/HaxeCheckstyle/tokentree/issues/106) 279 | - Fixed arrow position in typedef to function type with structure argument [#108](https://github.com/HaxeCheckstyle/tokentree/issues/108) 280 | - Fixed nested while loop detection [#108](https://github.com/HaxeCheckstyle/tokentree/issues/108) 281 | - Fixed handling of typed parameters in structure extensions [#109](https://github.com/HaxeCheckstyle/tokentree/issues/109) 282 | 283 | ## version 1.0.9 (2018-08-13) 284 | 285 | - Fixed endless loop with case and conditionals [#93](https://github.com/HaxeCheckstyle/tokentree/issues/93) 286 | 287 | ## version 1.0.8 (2018-08-13) 288 | 289 | - Fixed null poiner and detection in `getPOpenType` [#73](https://github.com/HaxeCheckstyle/tokentree/issues/73) + [#88](https://github.com/HaxeCheckstyle/tokentree/issues/88) + [#91](https://github.com/HaxeCheckstyle/tokentree/issues/91) 290 | - Fixed property modifiers [#74](https://github.com/HaxeCheckstyle/tokentree/issues/74) 291 | - Fixed function type detection [#74](https://github.com/HaxeCheckstyle/tokentree/issues/74) 292 | - Fixed `getBrOpenType` detection of blocks [#75](https://github.com/HaxeCheckstyle/tokentree/issues/75) + [#77](https://github.com/HaxeCheckstyle/tokentree/issues/77) + [#79](https://github.com/HaxeCheckstyle/tokentree/issues/79) + [#81](https://github.com/HaxeCheckstyle/tokentree/issues/81) + [#83](https://github.com/HaxeCheckstyle/tokentree/issues/83) + [#86](https://github.com/HaxeCheckstyle/tokentree/issues/86) + [#87](https://github.com/HaxeCheckstyle/tokentree/issues/87) + [#88](https://github.com/HaxeCheckstyle/tokentree/issues/88) 293 | - Fixed using multiple metadata with popen [#76](https://github.com/HaxeCheckstyle/tokentree/issues/76) 294 | - Fixed support for @new [#78](https://github.com/HaxeCheckstyle/tokentree/issues/78) 295 | - Fixed `isTernary` [#80](https://github.com/HaxeCheckstyle/tokentree/issues/80) + [#89](https://github.com/HaxeCheckstyle/tokentree/issues/89) 296 | - Fixed `getArrowType` [#80](https://github.com/HaxeCheckstyle/tokentree/issues/80) + [#85](https://github.com/HaxeCheckstyle/tokentree/issues/85) 297 | - Fixed comment detection in type hints [#82](https://github.com/HaxeCheckstyle/tokentree/issues/82) 298 | - Fixed property handling in `getFieldType` [#84](https://github.com/HaxeCheckstyle/tokentree/issues/84) 299 | - Fixed handling of typedef fields (@:overload, etc.) [#89](https://github.com/HaxeCheckstyle/tokentree/issues/89) 300 | - Fixed ternary parsing [#90](https://github.com/HaxeCheckstyle/tokentree/issues/90) 301 | - Fixed endless loop with switch and conditionals [#92](https://github.com/HaxeCheckstyle/tokentree/issues/92) 302 | 303 | ## version 1.0.7 (2018-07-26) 304 | 305 | - Added `TokenTreeCheckUtils.isTypeStructure()` [#36](https://github.com/HaxeCheckstyle/tokentree/issues/36) + [#50](https://github.com/HaxeCheckstyle/tokentree/issues/50) 306 | - Added `TokenTreeCheckUtils.isTypeEnum()` [#40](https://github.com/HaxeCheckstyle/tokentree/issues/40) 307 | - Added `TokenTreeCheckUtils.isMacroClass()` [#42](https://github.com/HaxeCheckstyle/tokentree/issues/42) 308 | - Added `TokenTreeCheckUtils.isOperatorFunction` [#44](https://github.com/HaxeCheckstyle/tokentree/issues/44) 309 | - Added `TokenTreeCheckUtils.getMetadata()` and `isModifier()` [#44](https://github.com/HaxeCheckstyle/tokentree/issues/44) 310 | - Added `TokenTreeCheckUtils.getBrOpenType()` and `getPOpenType()` [#51](https://github.com/HaxeCheckstyle/tokentree/issues/51) 311 | - Added `TokenTreeCheckUtils.isDeprecated()` [#67](https://github.com/HaxeCheckstyle/tokentree/issues/67) 312 | - Added `TokenTreeCheckUtils.getName()` [#68](https://github.com/HaxeCheckstyle/tokentree/issues/68) 313 | - Added `TokenTreeCheckUtils.getNameToken()` [#68](https://github.com/HaxeCheckstyle/tokentree/issues/68) 314 | - Added `TokenTreeCheckUtils.isNameToken()` [#68](https://github.com/HaxeCheckstyle/tokentree/issues/68) 315 | - Added `TokenTreeCheckUtils.getDocComment()` [#69](https://github.com/HaxeCheckstyle/tokentree/issues/69) 316 | - Added support for structural extension [#49](https://github.com/HaxeCheckstyle/tokentree/issues/49) 317 | - Added support for `final` [#55](https://github.com/HaxeCheckstyle/tokentree/issues/55) + [#56](https://github.com/HaxeCheckstyle/tokentree/issues/56) + [#61](https://github.com/HaxeCheckstyle/tokentree/issues/61) 318 | - Added `TokenTreeCheckUtils.getArrowType()` [#70](https://github.com/HaxeCheckstyle/tokentree/issues/70) + [#71](https://github.com/HaxeCheckstyle/tokentree/issues/71) 319 | - Fixed running out of tokens during package and imports [#28](https://github.com/HaxeCheckstyle/tokentree/issues/28) 320 | - Fixed `function` in typedef body [#30](https://github.com/HaxeCheckstyle/tokentree/issues/30) 321 | - Fixed handling of `switch this {` [#34](https://github.com/HaxeCheckstyle/tokentree/issues/34) 322 | - Fixed handling of `macro class` [#37](https://github.com/HaxeCheckstyle/tokentree/issues/37) 323 | - Fixed `isTernary` [#38](https://github.com/HaxeCheckstyle/tokentree/issues/38) 324 | - Fixed handling `return if (cond) -1 else 0` [#46](https://github.com/HaxeCheckstyle/tokentree/issues/46) 325 | - Fixed `Comma` in object declarations [#52](https://github.com/HaxeCheckstyle/tokentree/issues/52) 326 | - Fixed `@:final` handling in Haxe 4 [#53](https://github.com/HaxeCheckstyle/tokentree/issues/53) 327 | - Fixed `Binop(OpSub)` detection [#54](https://github.com/HaxeCheckstyle/tokentree/issues/54) 328 | - Fixed `final` handling in `FieldUtils` when compiling with Haxe 3 [#56](https://github.com/HaxeCheckstyle/tokentree/issues/56) 329 | - Fixed handling of const type parameters [#57](https://github.com/HaxeCheckstyle/tokentree/issues/57) 330 | - Fixed `case var` handling [#58](https://github.com/HaxeCheckstyle/tokentree/issues/58) 331 | - Fixed `case Pattern(var foo, var bar)` handling [#59](https://github.com/HaxeCheckstyle/tokentree/issues/59) 332 | - Fixed comments in if…else [#60](https://github.com/HaxeCheckstyle/tokentree/issues/60) +[#72](https://github.com/HaxeCheckstyle/tokentree/issues/72) 333 | - Fixed comments in typedefs [#62](https://github.com/HaxeCheckstyle/tokentree/issues/62) 334 | - Fixed handling of array items [#62](https://github.com/HaxeCheckstyle/tokentree/issues/62) 335 | - Fixed handling of `null`, `true` and `false` as body of if [#63](https://github.com/HaxeCheckstyle/tokentree/issues/63) 336 | - Changed `getPos` to calculate full position of all childs [#29](https://github.com/HaxeCheckstyle/tokentree/issues/29) 337 | - Changed position of `enum` in `enum abstract` [#65](https://github.com/HaxeCheckstyle/tokentree/issues/65) 338 | - Refactored `tempStore` handling and moved it to `TokenStream` childs [#28](https://github.com/HaxeCheckstyle/tokentree/issues/28) + [#30](https://github.com/HaxeCheckstyle/tokentree/issues/30) 339 | - Refactored `TokenTreeAccessHelper` and made it an abstract [#31](https://github.com/HaxeCheckstyle/tokentree/issues/31) + [#32](https://github.com/HaxeCheckstyle/tokentree/issues/32) + [#41](https://github.com/HaxeCheckstyle/tokentree/issues/41) 340 | - Refactored `isTypeEnumAbstract` [#33](https://github.com/HaxeCheckstyle/tokentree/issues/33) 341 | - Refactored to apply formatting [#63](https://github.com/HaxeCheckstyle/tokentree/issues/63) + [#65](https://github.com/HaxeCheckstyle/tokentree/issues/65) 342 | - Refactored `filterCallback` to remove `Array.concat` calls [#66](https://github.com/HaxeCheckstyle/tokentree/issues/66) 343 | 344 | ## version 1.0.6 (2018-07-16) 345 | 346 | - Added `TokenTreeCheckUtils.isTypeEnumAbstract` [#16](https://github.com/HaxeCheckstyle/tokentree/issues/16) 347 | - Added FieldUtils [#17](https://github.com/HaxeCheckstyle/tokentree/issues/17) 348 | - Added `isComment` and `isCIdent` to `TokentTreeAccessHelper` [#23](https://github.com/HaxeCheckstyle/tokentree/issues/23) 349 | - Added `TokenTreeCheckUtils.isOpGtTypedefExtension` [#24](https://github.com/HaxeCheckstyle/tokentree/issues/24) 350 | - Added `TokenTreeCheckUtils.isBrOpenAnonTypeOrTypedef` [#26](https://github.com/HaxeCheckstyle/tokentree/issues/26) 351 | - Added `tempStore` to `TokenStream` [#27](https://github.com/HaxeCheckstyle/tokentree/issues/27) 352 | - Fixed `@:default` [#18](https://github.com/HaxeCheckstyle/tokentree/issues/18) 353 | - Fixed position of `cast` children [#19](https://github.com/HaxeCheckstyle/tokentree/issues/19) 354 | - Fixed position of comments in `case`/`default` [#20](https://github.com/HaxeCheckstyle/tokentree/issues/20) 355 | - Fixed error handling to work with `--no-inline` [#21](https://github.com/HaxeCheckstyle/tokentree/issues/21) 356 | - Fixed position of BinOp() in anon objects [#22](https://github.com/HaxeCheckstyle/tokentree/issues/22) 357 | - Fixed ternary handling [#25](https://github.com/HaxeCheckstyle/tokentree/issues/25) 358 | - Fixed missing modifiers in conditionals [#27](https://github.com/HaxeCheckstyle/tokentree/issues/27) 359 | - Changed `isImport` to also incluide `using` [#20](https://github.com/HaxeCheckstyle/tokentree/issues/20) 360 | 361 | ## version 1.0.5 (2018-07-08) 362 | 363 | - Added support for `enum abstract` [#14](https://github.com/HaxeCheckstyle/tokentree/issues/14) 364 | - Added support for `@:a.b.c` [#14](https://github.com/HaxeCheckstyle/tokentree/issues/14) 365 | - Added support for `var ?x:Int` [#14](https://github.com/HaxeCheckstyle/tokentree/issues/14) 366 | - Added support for `extern` fields [#14](https://github.com/HaxeCheckstyle/tokentree/issues/14) 367 | - Changed position of Dot in method chains [#15](https://github.com/HaxeCheckstyle/tokentree/issues/15) 368 | 369 | ## version 1.0.4 (2018-06-30) 370 | 371 | - Added parent and sibling access methods to TokenTreeAccessHelper [#10](https://github.com/HaxeCheckstyle/tokentree/issues/10) 372 | - Added support for haxeparser's whitespace (`-D keep_whitespace`) [#12](https://github.com/HaxeCheckstyle/tokentree/issues/12) 373 | - Fixed handling of `if`, `while` and `do`…`while` conditions [#11](https://github.com/HaxeCheckstyle/tokentree/issues/11) 374 | - Fixed handling of ternary expressions [#11](https://github.com/HaxeCheckstyle/tokentree/issues/11) 375 | - Fixed tree position of `Comma` [#13](https://github.com/HaxeCheckstyle/tokentree/issues/13) 376 | 377 | ## version 1.0.3 (2018-06-24) 378 | 379 | - Fixed position of comments in abstracts and interfaces [#7](https://github.com/HaxeCheckstyle/tokentree/issues/7) 380 | - Fixed unittest and coverage reporting for Haxe 4 [#8](https://github.com/HaxeCheckstyle/tokentree/issues/8) 381 | 382 | ## version 1.0.2 (2018-06-22) 383 | 384 | - Fixed thread safety of SharpIf handling [#4](https://github.com/HaxeCheckstyle/tokentree/issues/4) 385 | - Fixed handling of Semicolon in typedefs [#4](https://github.com/HaxeCheckstyle/tokentree/issues/4) 386 | - Fixed WalkComment at end of stream [#4](https://github.com/HaxeCheckstyle/tokentree/issues/4) 387 | - Fixed handling of switch in object declaration [#5](https://github.com/HaxeCheckstyle/tokentree/issues/5) 388 | - Changed position of top level comments [#4](https://github.com/HaxeCheckstyle/tokentree/issues/4) 389 | 390 | ## version 1.0.1 (2018-06-14) 391 | 392 | - Added `RELAX` mode to make parser more fault tolerant [#2](https://github.com/HaxeCheckstyle/tokentree/issues/2) 393 | - Fixed handling of `cast -1` [#1](https://github.com/HaxeCheckstyle/tokentree/issues/1) 394 | 395 | ## version 1.0.0 (2018-06-09) 396 | 397 | - initial move to separate repository 398 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Haxe Checkstyle 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TokenTree 2 | [![Release](https://img.shields.io/github/release/HaxeCheckstyle/tokentree.svg)](http://lib.haxe.org/p/tokentree/) 3 | [![Haxe-Formatter](https://github.com/HaxeCheckstyle/tokentree/workflows/Haxe-Tokentree/badge.svg)](https://github.com/HaxeCheckstyle/tokentree/actions) 4 | [![codecov](https://codecov.io/gh/HaxeCheckstyle/tokentree/branch/master/graph/badge.svg)](https://codecov.io/gh/HaxeCheckstyle/tokentree) 5 | 6 | TokenTree library used by haxe-checkstyle, haxe-formatter and haxe-languageserver is an alternative to AST. 7 | -------------------------------------------------------------------------------- /buildToken.hxml: -------------------------------------------------------------------------------- 1 | -cp src 2 | -cp test 3 | -lib hxparse 4 | -lib haxeparser 5 | -lib safety 6 | -lib utest 7 | 8 | -debug 9 | 10 | -D keep_whitespace 11 | -D absolute-path 12 | -D checkstyle 13 | -D debugTokenTree 14 | 15 | --run TokenTestMain 16 | --macro nullSafety('tokentree') 17 | -------------------------------------------------------------------------------- /checkstyle.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultSeverity": "INFO", 3 | "checks": [{ 4 | "type": "Anonymous" 5 | }, 6 | { 7 | "type": "ArrayAccess" 8 | }, 9 | { 10 | "type": "ArrayLiteral" 11 | }, 12 | { 13 | "type": "AvoidStarImport" 14 | }, 15 | { 16 | "type": "BlockBreakingConditional" 17 | }, 18 | { 19 | "props": { 20 | "format": "^(e)$" 21 | }, 22 | "type": "CatchParameterName" 23 | }, 24 | { 25 | "type": "CodeSimilarity" 26 | }, 27 | { 28 | "props": { 29 | "ignoreExtern": true, 30 | "format": "^[A-Z][A-Z0-9]*(_[A-Z0-9_]+)*$" 31 | }, 32 | "type": "ConstantName" 33 | }, 34 | { 35 | "props": { 36 | "thresholds": [{ 37 | "complexity": 20, 38 | "severity": "WARNING" 39 | }, 40 | { 41 | "complexity": 25, 42 | "severity": "ERROR" 43 | } 44 | ] 45 | }, 46 | "type": "CyclomaticComplexity" 47 | }, 48 | { 49 | "type": "DefaultComesLast" 50 | }, 51 | { 52 | "type": "Dynamic" 53 | }, 54 | { 55 | "type": "ERegLiteral" 56 | }, 57 | { 58 | "props": { 59 | "tokens": [ 60 | "CLASS_DEF", 61 | "ENUM_DEF", 62 | "ABSTRACT_DEF", 63 | "TYPEDEF_DEF", 64 | "INTERFACE_DEF", 65 | "OBJECT_DECL", 66 | "FUNCTION", 67 | "FOR", 68 | "IF", 69 | "WHILE", 70 | "SWITCH", 71 | "TRY", 72 | "CATCH" 73 | ], 74 | "option": "empty" 75 | }, 76 | "type": "EmptyBlock" 77 | }, 78 | { 79 | "type": "EmptyPackage" 80 | }, 81 | { 82 | "props": { 83 | "max": 2000, 84 | "ignoreEmptyLines": true 85 | }, 86 | "type": "FileLength" 87 | }, 88 | { 89 | "props": { 90 | "option": "upperCase" 91 | }, 92 | "type": "HexadecimalLiteral" 93 | }, 94 | { 95 | "props": { 96 | "ignoreSetter": true, 97 | "ignoreFormat": "^(main|run)$", 98 | "ignoreConstructorParameter": true 99 | }, 100 | "type": "HiddenField" 101 | }, 102 | { 103 | "props": { 104 | "character": "tab" 105 | }, 106 | "type": "IndentationCharacter" 107 | }, 108 | { 109 | "type": "InnerAssignment" 110 | }, 111 | { 112 | "props": { 113 | "allowMarkerInterfaces": true, 114 | "allowProperties": false 115 | }, 116 | "type": "Interface" 117 | }, 118 | { 119 | "props": { 120 | "tokens": [ 121 | "CLASS_DEF", 122 | "ENUM_DEF", 123 | "ABSTRACT_DEF", 124 | "TYPEDEF_DEF", 125 | "INTERFACE_DEF", 126 | "FUNCTION", 127 | "FOR", 128 | "IF", 129 | "WHILE", 130 | "SWITCH", 131 | "TRY", 132 | "CATCH" 133 | ], 134 | "ignoreEmptySingleline": true, 135 | "option": "eol" 136 | }, 137 | "type": "LeftCurly" 138 | }, 139 | { 140 | "props": { 141 | "max": 160, 142 | "ignorePattern": "(^@desc)|(LONG LINE TEST)" 143 | }, 144 | "type": "LineLength" 145 | }, 146 | { 147 | "props": { 148 | "listeners": [ 149 | "addEventListener", 150 | "addListener", 151 | "on", 152 | "once" 153 | ], 154 | "format": "^_?[a-z][a-zA-Z0-9]*$" 155 | }, 156 | "type": "ListenerName" 157 | }, 158 | { 159 | "props": { 160 | "ignoreExtern": true, 161 | "format": "^[a-z][a-zA-Z0-9]*$" 162 | }, 163 | "type": "LocalVariableName" 164 | }, 165 | { 166 | "props": { 167 | "ignoreExtern": true, 168 | "format": "^[a-z][a-zA-Z0-9]*$", 169 | "tokens": [ 170 | "CLASS", 171 | "PUBLIC", 172 | "PRIVATE", 173 | "TYPEDEF" 174 | ] 175 | }, 176 | "type": "MemberName" 177 | }, 178 | { 179 | "props": { 180 | "ignoreExtern": true, 181 | "format": "^[A-Z0-9_][a-zA-Z0-9]*$", 182 | "tokens": [ 183 | "ENUM" 184 | ] 185 | }, 186 | "type": "MemberName" 187 | }, 188 | { 189 | "props": { 190 | "maxPrivate": 50, 191 | "maxPublic": 50, 192 | "maxTotal": 50 193 | }, 194 | "type": "MethodCount" 195 | }, 196 | { 197 | "props": { 198 | "max": 50, 199 | "ignoreEmptyLines": true 200 | }, 201 | "type": "MethodLength" 202 | }, 203 | { 204 | "props": { 205 | "ignoreExtern": true, 206 | "format": "^[a-z][a-zA-Z0-9]*$" 207 | }, 208 | "type": "MethodName" 209 | }, 210 | { 211 | "props": { 212 | "modifiers": [ 213 | "MACRO", 214 | "OVERRIDE", 215 | "PUBLIC_PRIVATE", 216 | "STATIC", 217 | "INLINE", 218 | "DYNAMIC" 219 | ] 220 | }, 221 | "type": "ModifierOrder" 222 | }, 223 | { 224 | "props": { 225 | "minLength": 2, 226 | "ignore": "^\\s+$", 227 | "allowDuplicates": 2 228 | }, 229 | "type": "MultipleStringLiterals" 230 | }, 231 | { 232 | "type": "MultipleVariableDeclarations" 233 | }, 234 | { 235 | "props": { 236 | "allowSingleLineStatement": true, 237 | "tokens": [ 238 | "FOR", 239 | "IF", 240 | "ELSE_IF", 241 | "WHILE", 242 | "DO_WHILE" 243 | ] 244 | }, 245 | "type": "NeedBraces" 246 | }, 247 | { 248 | "type": "NestedForDepth" 249 | }, 250 | { 251 | "type": "NestedIfDepth" 252 | }, 253 | { 254 | "type": "NestedTryDepth" 255 | }, 256 | { 257 | "props": { 258 | "assignOpPolicy": "around", 259 | "unaryOpPolicy": "none", 260 | "ternaryOpPolicy": "around", 261 | "arithmeticOpPolicy": "around", 262 | "compareOpPolicy": "around", 263 | "bitwiseOpPolicy": "around", 264 | "boolOpPolicy": "around", 265 | "intervalOpPolicy": "none", 266 | "arrowPolicy": "around", 267 | "arrowFunctionPolicy": "around", 268 | "newFunctionTypePolicy": "around", 269 | "oldFunctionTypePolicy": "none" 270 | }, 271 | "type": "OperatorWhitespace" 272 | }, 273 | { 274 | "props": { 275 | "tokens": [ 276 | "=", 277 | "+", 278 | "-", 279 | "*", 280 | "/", 281 | "%", 282 | ">", 283 | "<", 284 | ">=", 285 | "<=", 286 | "==", 287 | "!=", 288 | "&", 289 | "|", 290 | "^", 291 | "&&", 292 | "||", 293 | "<<", 294 | ">>", 295 | ">>>", 296 | "+=", 297 | "-=", 298 | "*=", 299 | "/=", 300 | "%=", 301 | "<<=", 302 | ">>=", 303 | ">>>=", 304 | "|=", 305 | "&=", 306 | "^=", 307 | "...", 308 | "=>", 309 | "++", 310 | "--" 311 | ], 312 | "option": "eol" 313 | }, 314 | "type": "OperatorWrap" 315 | }, 316 | { 317 | "props": { 318 | "ignoreExtern": true, 319 | "format": "^(_|[a-z][a-zA-Z0-9]*$)" 320 | }, 321 | "type": "ParameterName" 322 | }, 323 | { 324 | "props": { 325 | "max": 7, 326 | "ignoreOverriddenMethods": false 327 | }, 328 | "type": "ParameterNumber" 329 | }, 330 | { 331 | "type": "RedundantModifier" 332 | }, 333 | { 334 | "props": { 335 | "allowEmptyReturn": true, 336 | "enforceReturnType": false 337 | }, 338 | "type": "Return" 339 | }, 340 | { 341 | "props": { 342 | "tokens": [ 343 | "CLASS_DEF", 344 | "ENUM_DEF", 345 | "ABSTRACT_DEF", 346 | "TYPEDEF_DEF", 347 | "INTERFACE_DEF", 348 | "OBJECT_DECL", 349 | "FUNCTION", 350 | "FOR", 351 | "IF", 352 | "WHILE", 353 | "SWITCH", 354 | "TRY", 355 | "CATCH" 356 | ], 357 | "option": "aloneorsingle" 358 | }, 359 | "type": "RightCurly" 360 | }, 361 | { 362 | "props": { 363 | "dotPolicy": "none", 364 | "commaPolicy": "after", 365 | "semicolonPolicy": "after" 366 | }, 367 | "type": "SeparatorWhitespace" 368 | }, 369 | { 370 | "props": { 371 | "tokens": [ 372 | "," 373 | ], 374 | "option": "eol" 375 | }, 376 | "type": "SeparatorWrap" 377 | }, 378 | { 379 | "type": "SimplifyBooleanExpression" 380 | }, 381 | { 382 | "type": "SimplifyBooleanReturn" 383 | }, 384 | { 385 | "props": { 386 | "spaceIfCondition": "should", 387 | "spaceAroundBinop": true, 388 | "spaceForLoop": "should", 389 | "ignoreRangeOperator": true, 390 | "spaceWhileLoop": "should", 391 | "spaceCatch": "should", 392 | "spaceSwitchCase": "should", 393 | "noSpaceAroundUnop": true 394 | }, 395 | "type": "Spacing" 396 | }, 397 | { 398 | "props": { 399 | "allowException": true, 400 | "policy": "doubleAndInterpolation" 401 | }, 402 | "type": "StringLiteral" 403 | }, 404 | { 405 | "type": "TODOComment" 406 | }, 407 | { 408 | "type": "TabForAligning" 409 | }, 410 | { 411 | "type": "Trace" 412 | }, 413 | { 414 | "type": "TrailingWhitespace" 415 | }, 416 | { 417 | "props": { 418 | "ignoreEnumAbstractValues": true 419 | }, 420 | "type": "Type" 421 | }, 422 | { 423 | "props": { 424 | "ignoreExtern": true, 425 | "format": "^[A-Z]+[a-zA-Z0-9]*$" 426 | }, 427 | "type": "TypeName" 428 | }, 429 | { 430 | "type": "UnnecessaryConstructor" 431 | }, 432 | { 433 | "props": { 434 | "ignoreModules": [ 435 | "haxeparser.Data", 436 | "haxeparser.Data.Token", 437 | "haxeparser.Data.TokenDef", 438 | "haxeparser.Data.TypeDecl", 439 | "haxe.macro.Expr", 440 | "haxe.macro.Expr.Access", 441 | "haxe.macro.Expr.Field", 442 | "haxe.macro.Expr.Function" 443 | ], 444 | "moduleTypeMap": { 445 | "checkstyle.Config": [ 446 | "CheckConfig" 447 | ], 448 | "checkstyle.ChecksInfo": [ 449 | "CheckInfo" 450 | ] 451 | } 452 | }, 453 | "type": "UnusedImport" 454 | }, 455 | { 456 | "type": "UnusedLocalVar" 457 | }, 458 | { 459 | "type": "VariableInitialisation" 460 | }, 461 | { 462 | "props": { 463 | "tokens": [ 464 | ",", 465 | ";" 466 | ] 467 | }, 468 | "type": "WhitespaceAfter" 469 | }, 470 | { 471 | "props": { 472 | "tokens": [ 473 | "=", 474 | "+", 475 | "-", 476 | "*", 477 | "/", 478 | "%", 479 | ">", 480 | "<", 481 | ">=", 482 | "<=", 483 | "==", 484 | "!=", 485 | "&", 486 | "|", 487 | "^", 488 | "&&", 489 | "||", 490 | "<<", 491 | ">>", 492 | ">>>", 493 | "+=", 494 | "-=", 495 | "*=", 496 | "/=", 497 | "%=", 498 | "<<=", 499 | ">>=", 500 | ">>>=", 501 | "|=", 502 | "&=", 503 | "^=", 504 | "=>" 505 | ] 506 | }, 507 | "type": "WhitespaceAround" 508 | } 509 | ], 510 | "exclude": { 511 | "path": "RELATIVE_TO_PROJECT", 512 | "all": [], 513 | "Dynamic": [ 514 | "TestMain", 515 | "tokentree/TokenTreeBuilderParsingTest" 516 | ], 517 | "MultipleStringLiterals": [ 518 | "token" 519 | ], 520 | "CyclomaticComplexity": [ 521 | "tokentree/walk/WalkQuestion:isTernary", 522 | "tokentree/walk/WalkStatement", 523 | "tokentree/walk/WalkFile", 524 | "tokentree/utils/TokenTreeCheckUtils:filterOpSub", 525 | "tokentree/utils/TokenTreeCheckUtils:determineBrOpenType", 526 | "tokentree/utils/TokenTreeCheckUtils:determineColonType", 527 | "tokentree/utils/TokenTreeCheckUtils:determinePOpenType", 528 | "tokentree/utils/TokenTreeCheckUtils:determinBrChildren", 529 | "tokentree/utils/TokenTreeCheckUtils:findColonParent", 530 | "tokentree/ToTokenTreeDef", 531 | "tokentree/TokenStream:isTypedParam", 532 | "tokentree/TokenStream:consumeInlineMarkup", 533 | "tokentree/TokenTreeDefPrinter" 534 | ], 535 | "MethodLength": [ 536 | "tokentree/walk/WalkStatement", 537 | "tokentree/walk/WalkFinal", 538 | "tokentree/utils/TokenTreeCheckUtils", 539 | "tokentree/TokenStream:consumeInlineMarkup" 540 | ], 541 | "NestedForDepth": [ 542 | "TestMain" 543 | ] 544 | } 545 | } -------------------------------------------------------------------------------- /haxe_libraries/exception.hxml: -------------------------------------------------------------------------------- 1 | -D exception=1.1.0 2 | # @install: lix --silent download "haxelib:/exception#1.1.0" into exception/1.1.0/haxelib 3 | -cp ${HAXE_LIBCACHE}/exception/1.1.0/haxelib/src 4 | -------------------------------------------------------------------------------- /haxe_libraries/haxeparser.hxml: -------------------------------------------------------------------------------- 1 | # @install: lix --silent download "gh://github.com/HaxeCheckstyle/haxeparser#a5fce2ecf5fb3bdfebfd7efd8b05329d456ec0d2" into haxeparser/4.3.0-rc.1/github/a5fce2ecf5fb3bdfebfd7efd8b05329d456ec0d2 2 | -lib hxparse 3 | -cp ${HAXE_LIBCACHE}/haxeparser/4.3.0-rc.1/github/a5fce2ecf5fb3bdfebfd7efd8b05329d456ec0d2/src 4 | -D haxeparser=4.3.0-rc.1 -------------------------------------------------------------------------------- /haxe_libraries/hxjsonast.hxml: -------------------------------------------------------------------------------- 1 | # @install: lix --silent download "haxelib:/hxjsonast#1.1.0" into hxjsonast/1.1.0/haxelib 2 | -cp ${HAXE_LIBCACHE}/hxjsonast/1.1.0/haxelib/src 3 | -D hxjsonast=1.1.0 -------------------------------------------------------------------------------- /haxe_libraries/hxparse.hxml: -------------------------------------------------------------------------------- 1 | # @install: lix --silent download "gh://github.com/simn/hxparse#876070ec62a4869de60081f87763e23457a3bda8" into hxparse/4.3.0/github/876070ec62a4869de60081f87763e23457a3bda8 2 | -cp ${HAXE_LIBCACHE}/hxparse/4.3.0/github/876070ec62a4869de60081f87763e23457a3bda8/src 3 | -D hxparse=4.3.0 -------------------------------------------------------------------------------- /haxe_libraries/instrument.hxml: -------------------------------------------------------------------------------- 1 | # @install: lix --silent download "haxelib:/instrument#1.2.0" into instrument/1.2.0/haxelib 2 | -lib safety 3 | -cp ${HAXE_LIBCACHE}/instrument/1.2.0/haxelib/src 4 | -D instrument=1.2.0 -------------------------------------------------------------------------------- /haxe_libraries/json2object.hxml: -------------------------------------------------------------------------------- 1 | # @install: lix --silent download "gh://github.com/elnabo/json2object#a75859de1e966c09e73591b6c9186086c143fe60" into json2object/3.11.0/github/a75859de1e966c09e73591b6c9186086c143fe60 2 | -lib hxjsonast 3 | -cp ${HAXE_LIBCACHE}/json2object/3.11.0/github/a75859de1e966c09e73591b6c9186086c143fe60/src 4 | -D json2object=3.11.0 -------------------------------------------------------------------------------- /haxe_libraries/safety.hxml: -------------------------------------------------------------------------------- 1 | # @install: lix --silent download "haxelib:/safety#1.1.2" into safety/1.1.2/haxelib 2 | -cp ${HAXE_LIBCACHE}/safety/1.1.2/haxelib/src 3 | -D safety=1.1.2 -------------------------------------------------------------------------------- /haxe_libraries/test-adapter.hxml: -------------------------------------------------------------------------------- 1 | # @install: lix --silent download "haxelib:/test-adapter#2.0.7" into test-adapter/2.0.7/haxelib 2 | -lib json2object 3 | -cp ${HAXE_LIBCACHE}/test-adapter/2.0.7/haxelib/ 4 | -D test-adapter=2.0.7 5 | --macro _testadapter.Macro.init() -------------------------------------------------------------------------------- /haxe_libraries/utest.hxml: -------------------------------------------------------------------------------- 1 | # @install: lix --silent download "gh://github.com/haxe-utest/utest#bdb5fec4b8e77d9a3c079d9cfb108f29f153721a" into utest/2.0.0-alpha/github/bdb5fec4b8e77d9a3c079d9cfb108f29f153721a 2 | -cp ${HAXE_LIBCACHE}/utest/2.0.0-alpha/github/bdb5fec4b8e77d9a3c079d9cfb108f29f153721a/src 3 | -D utest=2.0.0-alpha 4 | --macro utest.utils.Macro.importEnvSettings() 5 | -------------------------------------------------------------------------------- /haxelib.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tokentree", 3 | "classPath": "src", 4 | "license": "MIT", 5 | "tags": [], 6 | "description": "TokenTree library used by haxe-checkstyle, haxe-formatter, haxe-languageserver, etc.", 7 | "contributors": [ 8 | "AlexHaxe" 9 | ], 10 | "releasenote": "fixed TokenTreeCheckUtils.getBrOpenType detection of pattern extraction block - see CHANGELOG for details", 11 | "version": "1.2.18", 12 | "url": "https://github.com/HaxeCheckstyle/tokentree", 13 | "dependencies": {} 14 | } -------------------------------------------------------------------------------- /hxformat.json: -------------------------------------------------------------------------------- 1 | { 2 | "sameLine":{ 3 | "ifBody": "same", 4 | "elseBody": "same", 5 | "ifElse": "next", 6 | "forBody": "same", 7 | "whileBody": "same", 8 | "doWhile": "next", 9 | "tryCatch": "next", 10 | "tryBody": "next", 11 | "catchBody": "next", 12 | "caseBody": "next", 13 | "expressionCase": "same" 14 | }, 15 | "emptyLines":{ 16 | "finalNewline": false, 17 | "enumAbstractEmptyLines":{ 18 | "betweenVars": 1 19 | } 20 | }, 21 | "wrapping": { 22 | "methodChain": { 23 | "defaultWrap": "fillLine", 24 | "rules": [] 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /makeReleaseZip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | npm install 4 | npx lix download 5 | npx lix use haxe 4.3.6 6 | 7 | haxe test.hxml 8 | 9 | rm -f tokentree.zip 10 | zip -9 -r -q tokentree.zip src haxelib.json hxformat.json package.json README.md CHANGELOG.md LICENSE 11 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tokentree", 3 | "version": "1.2.18", 4 | "lockfileVersion": 2, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "tokentree", 9 | "version": "1.2.18", 10 | "license": "MIT", 11 | "devDependencies": { 12 | "lix": "^15.12.0" 13 | } 14 | }, 15 | "node_modules/lix": { 16 | "version": "15.12.0", 17 | "resolved": "https://registry.npmjs.org/lix/-/lix-15.12.0.tgz", 18 | "integrity": "sha512-FA36oCl+M+3Of8L4eErXw7tAHGOjqEC4IgEvH6oPDsiYd4yN6XpzZGcbLuIyu4PiztjOrr1TKJnpwi32qb2ddw==", 19 | "dev": true, 20 | "hasInstallScript": true, 21 | "bin": { 22 | "haxe": "bin/haxeshim.js", 23 | "haxelib": "bin/haxelibshim.js", 24 | "lix": "bin/lix.js", 25 | "neko": "bin/nekoshim.js" 26 | } 27 | } 28 | }, 29 | "dependencies": { 30 | "lix": { 31 | "version": "15.12.0", 32 | "resolved": "https://registry.npmjs.org/lix/-/lix-15.12.0.tgz", 33 | "integrity": "sha512-FA36oCl+M+3Of8L4eErXw7tAHGOjqEC4IgEvH6oPDsiYd4yN6XpzZGcbLuIyu4PiztjOrr1TKJnpwi32qb2ddw==", 34 | "dev": true 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tokentree", 3 | "version": "1.2.18", 4 | "description": "TokenTree library used by haxe-checkstyle, haxe-formatter and haxe-languageserver", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/HaxeCheckstyle/tokentree" 8 | }, 9 | "keywords": [ 10 | "checkstyle", 11 | "haxe", 12 | "token" 13 | ], 14 | "author": { 15 | "name": "Alexander Blum", 16 | "email": "Alexander.Blum@gmail.com" 17 | }, 18 | "devDependencies": { 19 | "lix": "^15.12.0" 20 | }, 21 | "bugs": "https://github.com/HaxeCheckstyle/tokentree/issues", 22 | "license": "MIT" 23 | } 24 | -------------------------------------------------------------------------------- /src/import.hx: -------------------------------------------------------------------------------- 1 | import haxe.macro.Expr; 2 | import tokentree.TokenStream; 3 | import tokentree.TokenStreamProgress; 4 | import tokentree.TokenTreeAccessHelper; 5 | import tokentree.TokenTreeDef; 6 | 7 | using tokentree.TokenTreeAccessHelper; -------------------------------------------------------------------------------- /src/tokentree/ToTokenTreeDef.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | import haxeparser.Data; 4 | 5 | abstract ToTokenTreeDef(TokenTreeDef) { 6 | function new(tok:TokenTreeDef) { 7 | this = tok; 8 | } 9 | 10 | @:to 11 | public function toTokenTreeDef():TokenTreeDef { 12 | return this; 13 | } 14 | 15 | @:from 16 | public static function fromTokenDef(tok:TokenDef):ToTokenTreeDef { 17 | return new ToTokenTreeDef(switch (tok) { 18 | case Kwd(k): Kwd(k); 19 | case Const(c): switch (c) { 20 | #if (haxe >= version("4.3.0-rc.1")) 21 | case CInt(v, s): Const(CInt(v, s)); 22 | case CFloat(f, s): Const(CFloat(f, s)); 23 | #else 24 | case CInt(v): Const(CInt(v)); 25 | case CFloat(f): Const(CFloat(f)); 26 | #end 27 | case CString(s, kind): Const(CString(s, kind)); 28 | case CIdent(s): Const(CIdent(s)); 29 | case CRegexp(r, opt): Const(CRegexp(r, opt)); 30 | case CMarkup(s): Const(CMarkup(s)); 31 | } 32 | case Sharp(s): Sharp(s); 33 | case Dollar(s): Dollar(s); 34 | case Unop(op): Unop(op); 35 | case Binop(op): Binop(op); 36 | case Comment(s): Comment(s); 37 | case CommentLine(s): CommentLine(s); 38 | case IntInterval(s): IntInterval(s); 39 | case Semicolon: Semicolon; 40 | case Dot: Dot; 41 | case DblDot: DblDot; 42 | case QuestionDot: QuestionDot; 43 | case Arrow: Arrow; 44 | case Comma: Comma; 45 | case BkOpen: BkOpen; 46 | case BkClose: BkClose; 47 | case BrOpen: BrOpen; 48 | case BrClose: BrClose; 49 | case POpen: POpen; 50 | case PClose: PClose; 51 | case Question: Question; 52 | case At: At; 53 | case Eof: Eof; 54 | case Spread: Spread; 55 | }); 56 | } 57 | } -------------------------------------------------------------------------------- /src/tokentree/TokenStream.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | import byte.ByteData; 4 | import haxeparser.Data; 5 | 6 | class TokenStream { 7 | public static inline var NO_MORE_TOKENS:String = "no more tokens"; 8 | public static var MODE:TokenStreamMode = Strict; 9 | 10 | var tokens:Array; 11 | var current:Int; 12 | var bytes:ByteData; 13 | var sharpIfStack:Array; 14 | var tempStore:Array; 15 | 16 | public function new(tokens:Array, bytes:ByteData) { 17 | this.tokens = tokens; 18 | this.bytes = bytes; 19 | sharpIfStack = []; 20 | tempStore = []; 21 | current = 0; 22 | } 23 | 24 | public function hasMore():Bool { 25 | return current < tokens.length; 26 | } 27 | 28 | public function consumeToken():TokenTree { 29 | if ((current < 0) || (current >= tokens.length)) { 30 | switch (MODE) { 31 | case Relaxed: 32 | return createDummyToken(CommentLine("auto insert")); 33 | case Strict: 34 | throw NO_MORE_TOKENS; 35 | } 36 | } 37 | var token:Token = tokens[current]; 38 | current++; 39 | #if debugTokenTree 40 | Sys.println(token); 41 | #end 42 | var space:String = ""; 43 | #if keep_whitespace 44 | space = token.space; 45 | #end 46 | return new TokenTree(ToTokenTreeDef.fromTokenDef(token.tok), space, token.pos, current - 1); 47 | } 48 | 49 | public function consumeConstIdent():TokenTree { 50 | switch (token()) { 51 | case Dollar(_): 52 | return consumeToken(); 53 | case Const(CIdent(_)): 54 | return consumeToken(); 55 | case Kwd(KwdOperator): 56 | // pre 4.0.0 code might still use `operator` as an identifier 57 | return consumeToken(); 58 | default: 59 | switch (MODE) { 60 | case Relaxed: return createDummyToken(Const(CIdent("autoInsert"))); 61 | case Strict: error('bad token ${token()} != Const(CIdent(_))'); 62 | } 63 | } 64 | } 65 | 66 | public function consumeConst():TokenTree { 67 | switch (token()) { 68 | case Const(_): 69 | return consumeToken(); 70 | default: 71 | switch (MODE) { 72 | case Relaxed: return createDummyToken(Const(CString("autoInsert"))); 73 | case Strict: error('bad token ${token()} != Const(_)'); 74 | } 75 | } 76 | } 77 | 78 | public function consumeTokenDef(tokenDef:TokenTreeDef):TokenTree { 79 | if (matches(tokenDef)) return consumeToken(); 80 | switch (MODE) { 81 | case Relaxed: 82 | return createDummyToken(tokenDef); 83 | case Strict: 84 | error('bad token ${token()} != $tokenDef'); 85 | } 86 | } 87 | 88 | public function consumeToTempStore() { 89 | tempStore.push(consumeToken()); 90 | } 91 | 92 | public function addToTempStore(token:TokenTree) { 93 | tempStore.push(token); 94 | } 95 | 96 | public function applyTempStore(parent:TokenTree) { 97 | while (tempStore.length > 0) { 98 | parent.addChild(tempStore.shift()); 99 | } 100 | } 101 | 102 | public function hasTempStore():Bool { 103 | return tempStore.length > 0; 104 | } 105 | 106 | public function getTempStore():Array { 107 | return tempStore; 108 | } 109 | 110 | public function clearTempStore() { 111 | tempStore = []; 112 | } 113 | 114 | public inline function error(s:String) { 115 | throw formatCurrentPos() + ": " + s; 116 | } 117 | 118 | function formatCurrentPos():String { 119 | var pos = tokens[current].pos; 120 | return new hxparse.Position(pos.file, pos.min, pos.max).format(bytes); 121 | } 122 | 123 | public function matches(tokenDef:TokenTreeDef):Bool { 124 | if ((current < 0) || (current >= tokens.length)) return false; 125 | var token:Token = tokens[current]; 126 | return Type.enumEq(tokenDef, ToTokenTreeDef.fromTokenDef(token.tok)); 127 | } 128 | 129 | public function isSharp():Bool { 130 | if ((current < 0) || (current >= tokens.length)) return false; 131 | var token:Token = tokens[current]; 132 | return switch (token.tok) { 133 | case Sharp(_): true; 134 | default: false; 135 | } 136 | } 137 | 138 | public function isTypedParam():Bool { 139 | if ((current < 0) || (current >= tokens.length)) return false; 140 | var index:Int = current + 1; 141 | var token:Token = tokens[current]; 142 | switch (token.tok) { 143 | case Binop(OpLt): 144 | default: 145 | return false; 146 | } 147 | var depth:Int = 1; 148 | var brDepth:Int = 0; 149 | var bkDepth:Int = 0; 150 | var pDepth:Int = 0; 151 | while (true) { 152 | token = tokens[index++]; 153 | switch (token.tok) { 154 | case Dot | DblDot | Comma | Arrow | Const(_) | Dollar(_): 155 | case POpen: 156 | pDepth++; 157 | case PClose: 158 | if (pDepth <= 0) { 159 | return false; 160 | } 161 | pDepth--; 162 | case BrOpen: 163 | brDepth++; 164 | case BrClose: 165 | if (brDepth <= 0) { 166 | return false; 167 | } 168 | brDepth--; 169 | case BkOpen: 170 | bkDepth++; 171 | case BkClose: 172 | if (bkDepth <= 0) { 173 | return false; 174 | } 175 | bkDepth--; 176 | case Binop(OpLt): 177 | if ((pDepth > 0) || (bkDepth > 0) || (brDepth > 0)) { 178 | continue; 179 | } 180 | depth++; 181 | case Binop(OpGt): 182 | if ((pDepth > 0) || (bkDepth > 0) || (brDepth > 0)) { 183 | continue; 184 | } 185 | depth--; 186 | if (depth <= 0) { 187 | return true; 188 | } 189 | default: 190 | return false; 191 | } 192 | if (index >= tokens.length) { 193 | return false; 194 | } 195 | } 196 | return false; 197 | } 198 | 199 | public function token():TokenTreeDef { 200 | if ((current < 0) || (current >= tokens.length)) { 201 | switch (MODE) { 202 | case Relaxed: 203 | return CommentLine("auto insert"); 204 | case Strict: 205 | throw NO_MORE_TOKENS; 206 | } 207 | } 208 | return ToTokenTreeDef.fromTokenDef(tokens[current].tok); 209 | } 210 | 211 | public function tokenForMatch():TokenTreeDef { 212 | if ((current < 0) || (current >= tokens.length)) { 213 | return Root; 214 | } 215 | return ToTokenTreeDef.fromTokenDef(tokens[current].tok); 216 | } 217 | 218 | public function peekNonCommentToken():Null { 219 | if ((current < 0) || (current >= tokens.length)) { 220 | switch (MODE) { 221 | case Relaxed: 222 | return Const(CString("auto insert")); 223 | case Strict: 224 | throw NO_MORE_TOKENS; 225 | } 226 | } 227 | var index:Int = current; 228 | while (index < tokens.length) { 229 | var token:Token = tokens[index++]; 230 | if (Token == null) { 231 | continue; 232 | } 233 | switch (token.tok) { 234 | case Comment(_), CommentLine(_): 235 | default: 236 | return ToTokenTreeDef.fromTokenDef(token.tok); 237 | } 238 | } 239 | return Root; 240 | } 241 | 242 | public function getTokenPos():Null { 243 | if ((current < 0) || (current >= tokens.length)) { 244 | return null; 245 | } 246 | return tokens[current].pos; 247 | } 248 | 249 | public function rewind() { 250 | if (current <= 0) return; 251 | current--; 252 | } 253 | 254 | public function getStreamIndex():Int { 255 | return current; 256 | } 257 | 258 | public function rewindTo(pos:Int) { 259 | current = pos; 260 | } 261 | 262 | /** 263 | * HaxeLexer does not handle '>=', '>>', '>>=' and '>>>=' it produces an 264 | * individual token for each character. 265 | * This function provides a workaround, which scans the tokens following a 266 | * Binop(OpGt) and if it is followed by Binop(OpAssign) or Binop(OpGt), 267 | * it returns the correct token: 268 | * '>' -> Binop(OpGt) 269 | * '>=' -> Binop(OpGte) 270 | * '>>' -> Binop(OpShr) 271 | * '>>=' -> Binop(OpAssignOp(OpShr)) 272 | * '>>>=' -> Binop(OpAssignOp(OpUShr)) 273 | * 274 | */ 275 | public function consumeOpGt():TokenTree { 276 | var tok:TokenTree = consumeTokenDef(Binop(OpGt)); 277 | switch (token()) { 278 | case Binop(OpGt): 279 | return consumeOpShr(tok); 280 | case Binop(OpAssign): 281 | var assignTok:TokenTree = consumeToken(); 282 | return new TokenTree(Binop(OpGte), tok.space + assignTok.space, {file: tok.pos.file, min: tok.pos.min, max: assignTok.pos.max}, tok.index); 283 | default: 284 | return tok; 285 | } 286 | } 287 | 288 | function consumeOpShr(parent:TokenTree):TokenTree { 289 | var tok:TokenTree = consumeTokenDef(Binop(OpGt)); 290 | switch (token()) { 291 | case Binop(OpGt): 292 | var innerGt:TokenTree = consumeToken(); 293 | switch (token()) { 294 | case Binop(OpAssign): 295 | var assignTok:TokenTree = consumeToken(); 296 | return new TokenTree(Binop(OpAssignOp(OpUShr)), assignTok.space, { 297 | file: parent.pos.file, 298 | min: parent.pos.min, 299 | max: assignTok.pos.max 300 | }, parent.index); 301 | default: 302 | } 303 | return new TokenTree(Binop(OpUShr), innerGt.space, {file: parent.pos.file, min: parent.pos.min, max: innerGt.pos.max}, parent.index); 304 | case Binop(OpAssign): 305 | var assignTok:TokenTree = consumeToken(); 306 | return new TokenTree(Binop(OpAssignOp(OpShr)), assignTok.space, { 307 | file: parent.pos.file, 308 | min: parent.pos.min, 309 | max: assignTok.pos.max 310 | }, parent.index); 311 | default: 312 | return new TokenTree(Binop(OpShr), tok.space, {file: parent.pos.file, min: parent.pos.min, max: tok.pos.max}, parent.index); 313 | } 314 | } 315 | 316 | /** 317 | * HaxeLexer does not detect negative Const(CInt(_)) or Const(CFloat(_)) 318 | * This function provides a workaround, which scans the tokens around 319 | * Binop(OpSub) to see if the token stream should contain a negative const 320 | * value and returns a proper Const(CInt(-x)) or Const(CFloat(-x)) token 321 | */ 322 | public function consumeOpSub(parent:TokenTree):TokenTree { 323 | var tok:TokenTree = consumeTokenDef(Binop(OpSub)); 324 | switch (token()) { 325 | case Const(CInt(_)) | Const(CFloat(_)): 326 | default: 327 | return new TokenTree(tok.tok, tok.space, tok.pos, tok.index); 328 | } 329 | var previous:Int = current - 2; 330 | if (previous < 0) throw NO_MORE_TOKENS; 331 | var prevTok:Token = tokens[previous]; 332 | switch (prevTok.tok) { 333 | case Binop(_) | Unop(_) | BrOpen | BkOpen | POpen | Comma | DblDot | IntInterval(_) | Question | Semicolon: 334 | case Kwd(KwdReturn) | Kwd(KwdIf) | Kwd(KwdElse) | Kwd(KwdWhile) | Kwd(KwdDo) | Kwd(KwdFor) | Kwd(KwdCase) | Kwd(KwdCast): 335 | case PClose: 336 | if ((parent == null) || (parent.tok == Root)) { 337 | return new TokenTree(tok.tok, tok.space, tok.pos, tok.index); 338 | } 339 | switch (parent.tok) { 340 | case Kwd(KwdIf) | Kwd(KwdElse) | Kwd(KwdWhile) | Kwd(KwdDo) | Kwd(KwdFor) | Kwd(KwdCatch): 341 | default: return new TokenTree(tok.tok, tok.space, tok.pos, tok.index); 342 | } 343 | default: 344 | return new TokenTree(tok.tok, tok.space, tok.pos, tok.index); 345 | } 346 | switch (token()) { 347 | #if (haxe >= version("4.3.0-rc.1")) 348 | case Const(CInt(v, s)): 349 | var const:TokenTree = consumeConst(); 350 | return new TokenTree(Const(CInt('-$v', s)), const.space, {file: tok.pos.file, min: tok.pos.min, max: const.pos.max}, tok.index); 351 | case Const(CFloat(v, s)): 352 | var const:TokenTree = consumeConst(); 353 | return new TokenTree(Const(CFloat('-$v', s)), const.space, {file: tok.pos.file, min: tok.pos.min, max: const.pos.max}, tok.index); 354 | #else 355 | case Const(CInt(n)): 356 | var const:TokenTree = consumeConst(); 357 | return new TokenTree(Const(CInt('-$n')), const.space, {file: tok.pos.file, min: tok.pos.min, max: const.pos.max}, tok.index); 358 | case Const(CFloat(n)): 359 | var const:TokenTree = consumeConst(); 360 | return new TokenTree(Const(CFloat('-$n')), const.space, {file: tok.pos.file, min: tok.pos.min, max: const.pos.max}, tok.index); 361 | #end 362 | default: 363 | throw NO_MORE_TOKENS; 364 | } 365 | } 366 | 367 | public function pushSharpIf(token:TokenTree) { 368 | sharpIfStack.push(token); 369 | } 370 | 371 | public function popSharpIf():TokenTree { 372 | var token:Null = sharpIfStack.pop(); 373 | if (token == null) { 374 | switch (MODE) { 375 | case Relaxed: 376 | return createDummyToken(CommentLine("dummy token")); 377 | case Strict: 378 | throw NO_MORE_TOKENS; 379 | } 380 | } 381 | return @:nullSafety(Off) token; 382 | } 383 | 384 | public function peekSharpIf():TokenTree { 385 | if (sharpIfStack.length <= 0) { 386 | switch (MODE) { 387 | case Relaxed: 388 | return createDummyToken(CommentLine("dummy token")); 389 | case Strict: 390 | throw NO_MORE_TOKENS; 391 | } 392 | } 393 | return sharpIfStack[sharpIfStack.length - 1]; 394 | } 395 | 396 | public function createDummyToken(tokDef:TokenTreeDef):TokenTree { 397 | var pos:Position; 398 | if (tokens.length <= 0) { 399 | return new TokenTree(tokDef, "", {file: "", min: 0, max: 0}, -1, true); 400 | } 401 | if ((current < 0) || (current >= tokens.length)) { 402 | var prevPos:Position = tokens[tokens.length - 1].pos; 403 | pos = { 404 | min: prevPos.max, 405 | max: prevPos.max, 406 | file: prevPos.file 407 | } 408 | } 409 | else { 410 | var prevPos:Position = tokens[current].pos; 411 | pos = { 412 | min: prevPos.min, 413 | max: prevPos.min, 414 | file: prevPos.file 415 | } 416 | } 417 | return new TokenTree(tokDef, "", pos, -1, true); 418 | } 419 | } 420 | 421 | enum TokenStreamMode { 422 | Strict; 423 | Relaxed; 424 | } -------------------------------------------------------------------------------- /src/tokentree/TokenStreamProgress.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | class TokenStreamProgress { 4 | var stream:TokenStream; 5 | var pos:Int; 6 | 7 | public function new(stream:TokenStream) { 8 | this.stream = stream; 9 | pos = -1; 10 | } 11 | 12 | public function streamHasChanged():Bool { 13 | if (pos == -1) { 14 | pos = stream.getStreamIndex(); 15 | return true; 16 | } 17 | var oldPos:Int = pos; 18 | pos = stream.getStreamIndex(); 19 | return (pos != oldPos); 20 | } 21 | } -------------------------------------------------------------------------------- /src/tokentree/TokenTree.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | import tokentree.utils.TokenTreeCheckUtils.ArrowType; 4 | import tokentree.utils.TokenTreeCheckUtils.BkOpenType; 5 | import tokentree.utils.TokenTreeCheckUtils.BrOpenType; 6 | import tokentree.utils.TokenTreeCheckUtils.ColonType; 7 | import tokentree.utils.TokenTreeCheckUtils.POpenType; 8 | 9 | class TokenTree { 10 | static inline var MAX_LEVEL:Int = 9999; 11 | 12 | public var tok:TokenTreeDef; 13 | public var pos:Position; 14 | public var parent:Null; 15 | public var previousSibling:Null; 16 | public var nextSibling:Null; 17 | public var children:Null>; 18 | public var index:Int; 19 | public var inserted:Bool; 20 | public var space:String; 21 | 22 | @:allow(tokentree.utils.TokenTreeCheckUtils) 23 | var tokenTypeCache:TokenTypeCache; 24 | 25 | public function new(tok:TokenTreeDef, space:String, pos:Position, index:Int, inserted:Bool = false) { 26 | this.tok = tok; 27 | this.pos = pos; 28 | this.index = index; 29 | this.inserted = inserted; 30 | this.space = space; 31 | tokenTypeCache = {}; 32 | } 33 | 34 | public function matches(tokenDef:TokenTreeDef):Bool { 35 | return Type.enumEq(tokenDef, tok); 36 | } 37 | 38 | public function isComment():Bool { 39 | return switch (tok) { 40 | case Comment(_), CommentLine(_): true; 41 | default: false; 42 | } 43 | } 44 | 45 | public function isCIdent():Bool { 46 | return switch (tok) { 47 | case Const(CIdent(_)): true; 48 | default: false; 49 | } 50 | } 51 | 52 | public function isCIdentOrCString():Bool { 53 | return switch (tok) { 54 | case Const(CIdent(_)): true; 55 | case Const(CString(_)): true; 56 | default: false; 57 | } 58 | } 59 | 60 | public function addChild(child:Null) { 61 | if (child == null) return; 62 | if (children == null) children = []; 63 | if (children.length > 0) { 64 | child.previousSibling = children[children.length - 1]; 65 | children[children.length - 1].nextSibling = child; 66 | } 67 | children.push(child); 68 | child.parent = this; 69 | } 70 | 71 | public function hasChildren():Bool { 72 | if (children == null) return false; 73 | return children.length > 0; 74 | } 75 | 76 | public function getFirstChild():Null { 77 | if (!hasChildren()) return null; 78 | return @:nullSafety(Off) children[0]; 79 | } 80 | 81 | public function getLastChild():Null { 82 | if (!hasChildren()) return null; 83 | return @:nullSafety(Off) children[children.length - 1]; 84 | } 85 | 86 | public function getPos():Position { 87 | if ((children == null) || (children.length <= 0)) return pos; 88 | var fullPos:Position = {file: pos.file, min: pos.min, max: pos.max}; 89 | var childPos:Position; 90 | for (child in children) { 91 | childPos = child.getPos(); 92 | if (childPos.min < fullPos.min) fullPos.min = childPos.min; 93 | if (childPos.max > fullPos.max) fullPos.max = childPos.max; 94 | } 95 | return fullPos; 96 | } 97 | 98 | public function filter(searchFor:Array, mode:TokenFilterMode, maxLevel:Int = MAX_LEVEL):Array { 99 | return filterCallback(function(token:TokenTree, depth:Int):FilterResult { 100 | if (depth > maxLevel) return SkipSubtree; 101 | if (token.matchesAny(searchFor)) { 102 | if (mode == All) return FoundGoDeeper; 103 | return FoundSkipSubtree; 104 | } 105 | else return GoDeeper; 106 | }); 107 | } 108 | 109 | public function filterCallback(callback:FilterCallback):Array { 110 | var results:Array = []; 111 | internalFilterCallback(callback, results, 0); 112 | return results; 113 | } 114 | 115 | function internalFilterCallback(callback:FilterCallback, results:Array, depth:Int = 0) { 116 | switch (tok) { 117 | case Root: 118 | default: 119 | switch (callback(this, depth)) { 120 | case FoundGoDeeper: results.push(this); 121 | case FoundSkipSubtree: 122 | results.push(this); 123 | return; 124 | case GoDeeper: 125 | case SkipSubtree: return; 126 | } 127 | } 128 | if (children == null) return; 129 | for (child in children) { 130 | switch (child.tok) { 131 | case Sharp(_): 132 | child.internalFilterCallback(callback, results, depth); 133 | default: 134 | child.internalFilterCallback(callback, results, depth + 1); 135 | } 136 | } 137 | } 138 | 139 | function matchesAny(searchFor:Array):Bool { 140 | if (searchFor == null || tok == Root) return false; 141 | for (search in searchFor) { 142 | if (Type.enumEq(tok, search)) return true; 143 | } 144 | return false; 145 | } 146 | 147 | public function printTokenTree(prefix:String = ""):String { 148 | var buf:StringBuf = new StringBuf(); 149 | var tokString:String = '$tok'; 150 | if (inserted) tokString = '*** $tokString ***'; 151 | if (tok != Root) buf.add('$prefix$tokString\t\t\t\t${getPos()}'); 152 | if (children == null) return buf.toString(); 153 | for (child in children) buf.add('\n$prefix${child.printTokenTree(prefix + " ")}'); 154 | return buf.toString(); 155 | } 156 | 157 | public function toString():String { 158 | return TokenTreeDefPrinter.toString(tok); 159 | } 160 | } 161 | 162 | enum TokenFilterMode { 163 | All; 164 | First; 165 | } 166 | 167 | typedef FilterCallback = TokenTree->Int->FilterResult; 168 | 169 | enum FilterResult { 170 | FoundSkipSubtree; 171 | FoundGoDeeper; 172 | SkipSubtree; 173 | GoDeeper; 174 | } 175 | 176 | typedef TokenTypeCache = { 177 | @:optional var brOpenType:Null; 178 | @:optional var bkOpenType:Null; 179 | @:optional var pOpenType:Null; 180 | @:optional var colonType:Null; 181 | @:optional var arrowType:Null; 182 | } -------------------------------------------------------------------------------- /src/tokentree/TokenTreeAccessHelper.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | abstract TokenTreeAccessHelper(Null) from TokenTree { 4 | public var token(get, never):TokenTree; 5 | 6 | inline function get_token():TokenTree { 7 | return this; 8 | } 9 | 10 | public static inline function access(tok:TokenTree):TokenTreeAccessHelper { 11 | return tok; 12 | } 13 | 14 | public function parent():Null { 15 | return if (exists()) this.parent else null; 16 | } 17 | 18 | public function findParent(predicate:TokenTreeAccessHelper->Bool):Null { 19 | var parent:Null = parent(); 20 | while (parent.exists() && parent.token.tok != Root) { 21 | if (predicate(parent)) { 22 | return parent; 23 | } 24 | parent = parent.parent(); 25 | } 26 | return null; 27 | } 28 | 29 | public function previousSibling():Null { 30 | return if (exists()) this.previousSibling else null; 31 | } 32 | 33 | public function nextSibling():Null { 34 | return if (exists()) this.nextSibling else null; 35 | } 36 | 37 | public function firstChild():Null { 38 | return if (exists()) this.getFirstChild() else null; 39 | } 40 | 41 | public function lastChild():Null { 42 | return if (exists()) this.getLastChild() else null; 43 | } 44 | 45 | public function firstOf(token:TokenTreeDef):Null { 46 | if (!exists() || this.children == null) return null; 47 | for (tok in this.children) { 48 | if (Type.enumEq(tok.tok, token)) return tok; 49 | } 50 | return null; 51 | } 52 | 53 | public function lastOf(token:TokenTreeDef):Null { 54 | if (!exists() || this.children == null) return null; 55 | var found:Null = null; 56 | for (tok in this.children) { 57 | if (Type.enumEq(tok.tok, token)) found = tok; 58 | } 59 | return found; 60 | } 61 | 62 | public function child(index:Int):Null { 63 | return if (exists() && this.children != null) this.children[index] else null; 64 | } 65 | 66 | public function matches(tok:TokenTreeDef):Null { 67 | return if (exists() && Type.enumEq(this.tok, tok)) this else null; 68 | } 69 | 70 | public function isComment():Null { 71 | return if (exists() && this.isComment()) this else null; 72 | } 73 | 74 | public function isCIdent():Null { 75 | return if (exists() && this.isCIdent()) this else null; 76 | } 77 | 78 | public function or(other:TokenTree):TokenTree { 79 | return if (exists()) this else other; 80 | } 81 | 82 | public inline function exists():Bool { 83 | return this != null; 84 | } 85 | } 86 | 87 | typedef TokenTreeDefMatcher = TokenTreeDef->Bool; -------------------------------------------------------------------------------- /src/tokentree/TokenTreeBuilder.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | import byte.ByteData; 4 | import haxeparser.Data; 5 | import tokentree.walk.WalkClass; 6 | import tokentree.walk.WalkFile; 7 | import tokentree.walk.WalkStatement; 8 | import tokentree.walk.WalkTypeNameDef; 9 | 10 | class TokenTreeBuilder { 11 | public static function buildTokenTree(tokens:Array, bytes:ByteData, entryPoint:Null = null):TokenTree { 12 | if (entryPoint == null) { 13 | entryPoint = TypeLevel; 14 | } 15 | return buildTokenTreeFromStream(new TokenStream(tokens, bytes), entryPoint); 16 | } 17 | 18 | static function buildTokenTreeFromStream(stream:TokenStream, entryPoint:TokenTreeEntryPoint):TokenTree { 19 | var root:TokenTree = stream.createDummyToken(Root); 20 | switch (entryPoint) { 21 | case TypeLevel: 22 | WalkFile.walkFile(stream, root); 23 | case FieldLevel: 24 | WalkClass.walkClassBody(stream, root); 25 | case ExpressionLevel: 26 | WalkStatement.walkStatement(stream, root); 27 | case TypeHintLevel: 28 | WalkTypeNameDef.walkTypeNameDef(stream, root); 29 | } 30 | if (stream.hasMore()) { 31 | // stream is not empty! 32 | switch (TokenStream.MODE) { 33 | case Relaxed: 34 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 35 | while (progress.streamHasChanged()) { 36 | WalkStatement.walkStatement(stream, root); 37 | } 38 | if (stream.hasMore()) { 39 | throw "invalid token tree structure - found:" + stream.token(); 40 | } 41 | case Strict: 42 | throw "invalid token tree structure - found:" + stream.token(); 43 | } 44 | } 45 | var tempStore:Array = stream.getTempStore(); 46 | switch (TokenStream.MODE) { 47 | case Strict: 48 | if (tempStore.length != 0) { 49 | throw "invalid token tree structure - tokens in temp store:" + tempStore.join(", "); 50 | } 51 | case Relaxed: 52 | for (stored in tempStore) { 53 | root.addChild(stored); 54 | } 55 | } 56 | return root; 57 | } 58 | } 59 | 60 | enum TokenTreeEntryPoint { 61 | TypeLevel; 62 | FieldLevel; 63 | ExpressionLevel; 64 | TypeHintLevel; 65 | } -------------------------------------------------------------------------------- /src/tokentree/TokenTreeDef.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | import haxe.macro.Expr; 4 | import haxeparser.Data; 5 | 6 | enum TokenTreeDef { 7 | Root; 8 | Kwd(k:Keyword); 9 | Const(c:haxeparser.Data.Constant); 10 | Sharp(s:String); 11 | Dollar(s:String); 12 | Unop(op:haxe.macro.Expr.Unop); 13 | Binop(op:haxe.macro.Expr.Binop); 14 | Comment(s:String); 15 | CommentLine(s:String); 16 | IntInterval(s:String); 17 | Semicolon; 18 | Dot; 19 | DblDot; 20 | QuestionDot; 21 | Arrow; 22 | Comma; 23 | BkOpen; 24 | BkClose; 25 | BrOpen; 26 | BrClose; 27 | POpen; 28 | PClose; 29 | Question; 30 | At; 31 | Eof; 32 | Spread; 33 | } -------------------------------------------------------------------------------- /src/tokentree/TokenTreeDefPrinter.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | class TokenTreeDefPrinter { 4 | public static function toString(def:TokenTreeDef):String { 5 | return switch (def) { 6 | case Root: ""; 7 | case Kwd(k): k.getName().substr(3).toLowerCase(); 8 | #if (haxe >= version("4.3.0-rc.1")) 9 | case Const(CInt(v, null) | CFloat(v, null)): v; 10 | case Const(CInt(v, s) | CFloat(v, s)): '$v$s'; 11 | #else 12 | case Const(CInt(s) | CFloat(s)): s; 13 | #end 14 | case Const(CIdent(s)): s; 15 | case Const(CString(s)): '"$s"'; 16 | case Const(CRegexp(r, opt)): '~/$r/$opt'; 17 | case Const(CMarkup(s)): '$s'; 18 | case Sharp(s): '#$s'; 19 | case Dollar(s): '$$$s'; 20 | case Unop(op): new haxe.macro.Printer("").printUnop(op); 21 | case Binop(op): new haxe.macro.Printer("").printBinop(op); 22 | case Comment(s): '/*$s*/'; 23 | case CommentLine(s): '//$s'; 24 | case IntInterval(s): '$s...'; 25 | case Semicolon: ";"; 26 | case Dot: "."; 27 | case DblDot: ":"; 28 | case QuestionDot: "?."; 29 | case Arrow: "->"; 30 | case Comma: ","; 31 | case BkOpen: "["; 32 | case BkClose: "]"; 33 | case BrOpen: "{"; 34 | case BrClose: "}"; 35 | case POpen: "("; 36 | case PClose: ")"; 37 | case Question: "?"; 38 | case At: "@"; 39 | case Eof: ""; 40 | case Spread: "..."; 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /src/tokentree/utils/FieldUtils.hx: -------------------------------------------------------------------------------- 1 | package tokentree.utils; 2 | 3 | using Lambda; 4 | using tokentree.utils.TokenTreeCheckUtils; 5 | 6 | class FieldUtils { 7 | public static function getFieldType(field:Null, defaultVisibility:TokenFieldVisibility):TokenFieldType { 8 | if (field == null) { 9 | return Unknown; 10 | } 11 | switch (field.tok) { 12 | case Kwd(KwdFunction): 13 | return getFunctionFieldType(field, defaultVisibility); 14 | case Kwd(KwdVar), Kwd(KwdFinal): 15 | return getVarFieldType(field, defaultVisibility); 16 | default: 17 | } 18 | return Unknown; 19 | } 20 | 21 | static function getFunctionFieldType(field:TokenTree, defaultVisibility:TokenFieldVisibility):TokenFieldType { 22 | var access:TokenTreeAccessHelper = TokenTreeAccessHelper.access(field).firstChild(); 23 | if (access.token == null) { 24 | return Unknown; 25 | } 26 | var name = field.getNameToken().getName(); 27 | var visibility:TokenFieldVisibility = defaultVisibility; 28 | var isStatic:Bool = false; 29 | var isInline:Bool = false; 30 | var isOverride:Bool = false; 31 | var isFinal:Bool = false; 32 | var isExtern:Bool = false; 33 | if (access.token.children != null) { 34 | for (child in access.token.children) { 35 | switch (child.tok) { 36 | case Kwd(KwdPublic): 37 | visibility = Public; 38 | case Kwd(KwdPrivate): 39 | visibility = Private; 40 | case Kwd(KwdStatic): 41 | isStatic = true; 42 | case Kwd(KwdInline): 43 | isInline = true; 44 | case Kwd(KwdOverride): 45 | isOverride = true; 46 | case Kwd(KwdExtern): 47 | isExtern = true; 48 | case Kwd(KwdFinal): 49 | isFinal = true; 50 | case POpen, BrOpen: 51 | break; 52 | default: 53 | } 54 | } 55 | } 56 | return Function(name, visibility, isStatic, isInline, isOverride, isFinal, isExtern); 57 | } 58 | 59 | static function getVarFieldType(field:TokenTree, defaultVisibility:TokenFieldVisibility):TokenFieldType { 60 | var access:TokenTreeAccessHelper = TokenTreeAccessHelper.access(field).firstChild(); 61 | if (access.token == null) { 62 | return Unknown; 63 | } 64 | var name = field.getNameToken().getName(); 65 | var visibility:TokenFieldVisibility = defaultVisibility; 66 | var isStatic:Bool = false; 67 | var isInline:Bool = false; 68 | var isFinal:Bool = field.tok.match(Kwd(KwdFinal)); 69 | var isExtern:Bool = false; 70 | if (access.token.children != null) { 71 | for (child in access.token.children) { 72 | switch (child.tok) { 73 | case Kwd(KwdPublic): 74 | visibility = Public; 75 | case Kwd(KwdPrivate): 76 | visibility = Private; 77 | case Kwd(KwdStatic): 78 | isStatic = true; 79 | case Kwd(KwdInline): 80 | isInline = true; 81 | case Kwd(KwdExtern): 82 | isExtern = true; 83 | default: 84 | } 85 | } 86 | } 87 | access = access.firstOf(POpen); 88 | if (isFinal || access.token == null) { 89 | return Var(name, visibility, isStatic, isInline, isFinal, isExtern); 90 | } 91 | var getterAccess:TokenPropertyAccess = makePropertyAccess(access.firstChild().token); 92 | var setterAccess:TokenPropertyAccess = makePropertyAccess(access.child(1).token); 93 | return Prop(name, visibility, isStatic, getterAccess, setterAccess); 94 | } 95 | 96 | static function makePropertyAccess(accessToken:TokenTree):TokenPropertyAccess { 97 | if (accessToken == null) { 98 | return Default; 99 | } 100 | return switch (accessToken.tok) { 101 | case Kwd(KwdDefault): Default; 102 | case Kwd(KwdNull): NullAccess; 103 | case Kwd(KwdDynamic): DynamicAccess; 104 | case Const(CIdent("never")): Never; 105 | case Const(CIdent("get")): Get; 106 | case Const(CIdent("set")): Set; 107 | default: Default; 108 | } 109 | } 110 | 111 | public static function isOperatorFunction(functionToken:TokenTree):Bool { 112 | return functionToken.getMetadata().exists(function(meta) { 113 | return switch (meta.tok) { 114 | case Const(CIdent("op")): true; 115 | case Const(CIdent("arrayAccess")): true; 116 | case Const(CIdent("resolve")): true; 117 | case _: false; 118 | } 119 | }); 120 | } 121 | } 122 | 123 | enum TokenFieldType { 124 | Function(name:String, visibility:TokenFieldVisibility, isStatic:Bool, isInline:Bool, isOverride:Bool, isFinal:Bool, isExtern:Bool); 125 | Var(name:String, visibility:TokenFieldVisibility, isStatic:Bool, isInline:Bool, isFinal:Bool, isExtern:Bool); 126 | Prop(name:String, visibility:TokenFieldVisibility, isStatic:Bool, getter:TokenPropertyAccess, setter:TokenPropertyAccess); 127 | Unknown; 128 | } 129 | 130 | enum TokenFieldVisibility { 131 | Public; 132 | Private; 133 | } 134 | 135 | enum TokenPropertyAccess { 136 | Default; 137 | NullAccess; 138 | Get; 139 | Set; 140 | DynamicAccess; 141 | Never; 142 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkAbstract.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkAbstract { 4 | public static function walkAbstract(stream:TokenStream, parent:TokenTree) { 5 | var typeTok:TokenTree = stream.consumeToken(); 6 | switch (stream.token()) { 7 | case Kwd(KwdClass) | Kwd(KwdInterface): 8 | stream.addToTempStore(typeTok); 9 | WalkType.walkType(stream, parent); 10 | return; 11 | default: 12 | } 13 | parent.addChild(typeTok); 14 | var name:TokenTree = WalkTypeNameDef.walkTypeNameDef(stream, typeTok); 15 | // add all comments, annotations 16 | stream.applyTempStore(name); 17 | if (stream.tokenForMatch().match(POpen)) WalkPOpen.walkPOpen(stream, name); 18 | var typeParent:TokenTree = name; 19 | var typeChild:TokenTree; 20 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 21 | while (progress.streamHasChanged()) { 22 | switch (stream.token()) { 23 | case BrOpen: 24 | break; 25 | case Comment(_), CommentLine(_): 26 | name.addChild(stream.consumeToken()); 27 | case Const(CIdent("from")), Const(CIdent("to")): 28 | var fromToken:TokenTree = stream.consumeToken(); 29 | name.addChild(fromToken); 30 | WalkTypeNameDef.walkTypeNameDef(stream, fromToken); 31 | default: 32 | typeChild = stream.consumeToken(); 33 | typeParent.addChild(typeChild); 34 | typeParent = typeChild; 35 | } 36 | } 37 | var block:TokenTree = stream.consumeTokenDef(BrOpen); 38 | name.addChild(block); 39 | WalkAbstract.walkAbstractBody(stream, block); 40 | block.addChild(stream.consumeTokenDef(BrClose)); 41 | } 42 | 43 | public static function walkAbstractBody(stream:TokenStream, parent:TokenTree) { 44 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 45 | while (progress.streamHasChanged()) { 46 | switch (stream.token()) { 47 | case Kwd(KwdVar): 48 | WalkVar.walkVar(stream, parent); 49 | case Kwd(KwdFunction): 50 | WalkFunction.walkFunction(stream, parent); 51 | case Kwd(KwdFinal): 52 | WalkFinal.walkFinal(stream, parent); 53 | case Sharp(_): 54 | WalkSharp.walkSharp(stream, parent, WalkAbstract.walkAbstractBody); 55 | case At: 56 | stream.addToTempStore(WalkAt.walkAt(stream)); 57 | case BrClose: 58 | break; 59 | case Semicolon: 60 | parent.addChild(stream.consumeToken()); 61 | case Comment(_), CommentLine(_): 62 | parent.addChild(stream.consumeToken()); 63 | default: 64 | stream.consumeToTempStore(); 65 | } 66 | } 67 | stream.applyTempStore(parent); 68 | } 69 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkArrayAccess.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkArrayAccess { 4 | public static function walkArrayAccess(stream:TokenStream, parent:TokenTree) { 5 | var bkOpen:TokenTree = stream.consumeTokenDef(BkOpen); 6 | parent.addChild(bkOpen); 7 | stream.applyTempStore(bkOpen); 8 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 9 | while (progress.streamHasChanged()) { 10 | switch (stream.token()) { 11 | case Kwd(KwdFor): 12 | stream.applyTempStore(bkOpen); 13 | WalkFor.walkFor(stream, bkOpen); 14 | case Kwd(KwdWhile): 15 | stream.applyTempStore(bkOpen); 16 | WalkWhile.walkWhile(stream, bkOpen); 17 | case POpen: 18 | stream.applyTempStore(bkOpen); 19 | WalkPOpen.walkPOpen(stream, bkOpen); 20 | case BrOpen: 21 | stream.applyTempStore(bkOpen); 22 | WalkBlock.walkBlock(stream, bkOpen); 23 | case BkOpen: 24 | WalkArrayAccess.walkArrayAccess(stream, bkOpen); 25 | case BkClose: 26 | break; 27 | case At: 28 | stream.addToTempStore(WalkAt.walkAt(stream)); 29 | case Kwd(KwdFunction): 30 | WalkFunction.walkFunction(stream, bkOpen); 31 | case Comma: 32 | var comma:TokenTree = stream.consumeToken(); 33 | var child:Null = bkOpen.getLastChild(); 34 | if (child == null) child = bkOpen; 35 | child.addChild(comma); 36 | case Binop(OpArrow): 37 | var child:Null = bkOpen.getLastChild(); 38 | if (child == null) child = bkOpen; 39 | WalkStatement.walkStatement(stream, child); 40 | default: 41 | stream.applyTempStore(bkOpen); 42 | WalkStatement.walkStatement(stream, bkOpen); 43 | } 44 | } 45 | bkOpen.addChild(stream.consumeTokenDef(BkClose)); 46 | if (stream.hasMore()) { 47 | switch (stream.token()) { 48 | case BkOpen | Dot | Binop(_) | Const(CIdent("is")): 49 | WalkStatement.walkStatementContinue(stream, bkOpen); 50 | default: 51 | } 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkAt.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkAt { 4 | /** 5 | * At 6 | * |- DblDot 7 | * |- Const(CIdent) 8 | * |- POpen 9 | * |- expression 10 | * |- PClose 11 | * 12 | * At 13 | * |- DblDot 14 | * |- Const(CIdent) 15 | * 16 | * At 17 | * |- Const(CIdent) 18 | * |- POpen 19 | * |- expression 20 | * |- PClose 21 | * 22 | * At 23 | * |- Const(CIdent) 24 | * 25 | */ 26 | public static function walkAt(stream:TokenStream):TokenTree { 27 | var atTok:TokenTree = stream.consumeTokenDef(At); 28 | var parent:TokenTree = atTok; 29 | if (stream.tokenForMatch().match(DblDot)) { 30 | var dblDot:TokenTree = stream.consumeToken(); 31 | atTok.addChild(dblDot); 32 | parent = dblDot; 33 | } 34 | walkIdent(stream, parent); 35 | return atTok; 36 | } 37 | 38 | static function walkIdent(stream:TokenStream, parent:TokenTree) { 39 | var ident:TokenTree; 40 | switch (stream.token()) { 41 | case Const(CIdent(_)): 42 | ident = stream.consumeConstIdent(); 43 | case Kwd(_): 44 | ident = stream.consumeToken(); 45 | case Binop(OpIn): 46 | ident = stream.consumeToken(); 47 | default: 48 | return; 49 | } 50 | parent.addChild(ident); 51 | switch (stream.token()) { 52 | case Dot: 53 | var child:TokenTree = stream.consumeToken(); 54 | ident.addChild(child); 55 | walkIdent(stream, child); 56 | case POpen: 57 | var pOpenPos:Position = stream.getTokenPos(); 58 | if (ident.pos.max == pOpenPos.min) { 59 | var tempStore:Array = stream.getTempStore(); 60 | stream.clearTempStore(); 61 | WalkPOpen.walkPOpen(stream, ident, false); 62 | for (temp in tempStore) { 63 | stream.addToTempStore(temp); 64 | } 65 | } 66 | default: 67 | } 68 | } 69 | 70 | public static function walkAts(stream:TokenStream) { 71 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 72 | while (progress.streamHasChanged()) { 73 | switch (stream.token()) { 74 | case At: 75 | stream.addToTempStore(WalkAt.walkAt(stream)); 76 | default: 77 | } 78 | } 79 | } 80 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkBinopSub.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkBinopSub { 4 | public static function walkBinopSub(stream:TokenStream, parent:TokenTree) { 5 | var sub:TokenTree = stream.consumeOpSub(parent); 6 | parent.addChild(sub); 7 | switch (sub.tok) { 8 | case Const(_): 9 | WalkStatement.walkStatementContinue(stream, sub); 10 | default: 11 | WalkStatement.walkStatement(stream, sub); 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkBlock.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkBlock { 4 | /** 5 | * BrOpen 6 | * |- statement 7 | * |- statement 8 | * |- BrClose 9 | * 10 | */ 11 | public static function walkBlock(stream:TokenStream, parent:TokenTree) { 12 | while (stream.tokenForMatch().match(At)) stream.addToTempStore(WalkAt.walkAt(stream)); 13 | if (stream.tokenForMatch().match(BrOpen)) { 14 | var openTok:TokenTree = stream.consumeTokenDef(BrOpen); 15 | parent.addChild(openTok); 16 | stream.applyTempStore(openTok); 17 | walkBlockContinue(stream, openTok); 18 | stream.applyTempStore(openTok); 19 | } 20 | else { 21 | WalkStatement.walkStatement(stream, parent); 22 | } 23 | } 24 | 25 | public static function walkBlockContinue(stream:TokenStream, parent:TokenTree) { 26 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 27 | while (progress.streamHasChanged()) { 28 | switch (stream.token()) { 29 | case BrClose: 30 | break; 31 | case Comma: 32 | var child:TokenTree = stream.consumeToken(); 33 | var lastChild:Null = parent.getLastChild(); 34 | if (lastChild == null) { 35 | parent.addChild(child); 36 | } 37 | else { 38 | lastChild.addChild(child); 39 | } 40 | case BkClose, PClose: 41 | var child:TokenTree = stream.consumeToken(); 42 | parent.addChild(child); 43 | case Kwd(KwdCase), Kwd(KwdDefault): 44 | WalkSwitch.walkSwitchCases(stream, parent); 45 | default: 46 | WalkStatement.walkStatement(stream, parent); 47 | } 48 | } 49 | walkBlockEnd(stream, parent); 50 | } 51 | 52 | public static function walkBlockEnd(stream:TokenStream, parent:TokenTree) { 53 | parent.addChild(stream.consumeTokenDef(BrClose)); 54 | if (stream.hasMore()) { 55 | switch (stream.token()) { 56 | case Binop(OpGt): 57 | return; 58 | default: 59 | } 60 | walkAfterBlock(stream, parent); 61 | if (stream.hasMore()) { 62 | switch (stream.token()) { 63 | case Semicolon: 64 | var semicolon:TokenTree = stream.consumeToken(); 65 | parent.addChild(semicolon); 66 | default: 67 | } 68 | } 69 | } 70 | } 71 | 72 | @:access(tokentree.walk.WalkStatement) 73 | static function walkAfterBlock(stream:TokenStream, parent:TokenTree) { 74 | if (!stream.hasMore()) return; 75 | switch (stream.token()) { 76 | case Dot: 77 | WalkStatement.walkStatementWithoutSemicolon(stream, parent); 78 | case DblDot: 79 | WalkStatement.walkDblDot(stream, parent); 80 | case Semicolon: 81 | return; 82 | case Arrow: 83 | WalkStatement.walkStatementWithoutSemicolon(stream, parent); 84 | case Binop(_): 85 | WalkStatement.walkStatementWithoutSemicolon(stream, parent); 86 | case Const(CIdent("is")): 87 | WalkStatement.walkStatementWithoutSemicolon(stream, parent); 88 | case Unop(_): 89 | if (parent.isCIdentOrCString()) { 90 | WalkStatement.walkStatementWithoutSemicolon(stream, parent); 91 | } 92 | case Question: 93 | WalkQuestion.walkQuestion(stream, parent); 94 | case POpen: 95 | switch (parent.parent.tok) { 96 | case Dollar(_): WalkStatement.walkStatementWithoutSemicolon(stream, parent); 97 | default: 98 | } 99 | case CommentLine(_), Comment(_): 100 | var nextTokDef:Null = stream.peekNonCommentToken(); 101 | if (nextTokDef == null) { 102 | return; 103 | } 104 | switch (nextTokDef) { 105 | case Dot, DblDot, Binop(_), Unop(_), Question: 106 | WalkComment.walkComment(stream, parent); 107 | WalkStatement.walkStatementContinue(stream, parent); 108 | default: 109 | } 110 | default: 111 | } 112 | } 113 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkClass.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | import tokentree.TokenTreeAccessHelper; 4 | 5 | class WalkClass { 6 | public static function walkClass(stream:TokenStream, parent:TokenTree) { 7 | var typeTok:TokenTree = stream.consumeToken(); 8 | parent.addChild(typeTok); 9 | WalkComment.walkComment(stream, parent); 10 | var name:TokenTree = typeTok; 11 | switch (stream.token()) { 12 | case Const(CIdent(_)): 13 | name = WalkTypeNameDef.walkTypeNameDef(stream, typeTok); 14 | // add all comments, annotations 15 | stream.applyTempStore(name); 16 | case Dollar(_): 17 | name = WalkTypeNameDef.walkTypeNameDef(stream, typeTok); 18 | stream.applyTempStore(name); 19 | default: 20 | } 21 | WalkClass.walkClassExtends(stream, name); 22 | var block:TokenTree = stream.consumeTokenDef(BrOpen); 23 | name.addChild(block); 24 | WalkClass.walkClassBody(stream, block); 25 | block.addChild(stream.consumeTokenDef(BrClose)); 26 | } 27 | 28 | public static function walkClassExtends(stream:TokenStream, name:TokenTree) { 29 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 30 | while (progress.streamHasChanged()) { 31 | WalkExtends.walkExtends(stream, name); 32 | WalkImplements.walkImplements(stream, name); 33 | if (stream.isSharp()) WalkSharp.walkSharp(stream, name, WalkClass.walkClassExtends); 34 | WalkComment.walkComment(stream, name); 35 | } 36 | } 37 | 38 | public static function walkClassBody(stream:TokenStream, parent:TokenTree) { 39 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 40 | while (stream.hasMore() && progress.streamHasChanged()) { 41 | switch (stream.token()) { 42 | case Kwd(KwdVar): 43 | WalkVar.walkVar(stream, parent); 44 | case Kwd(KwdFunction): 45 | WalkFunction.walkFunction(stream, parent); 46 | case Sharp(_): 47 | WalkSharp.walkSharp(stream, parent, WalkClass.walkClassBody); 48 | walkClassContinueAfterSharp(stream, parent); 49 | case At: 50 | stream.addToTempStore(WalkAt.walkAt(stream)); 51 | case BrClose: 52 | break; 53 | case Semicolon: 54 | parent.addChild(stream.consumeToken()); 55 | case Kwd(KwdPublic) | Kwd(KwdPrivate) | Kwd(KwdStatic) | Kwd(KwdInline) | Kwd(KwdMacro) | Kwd(KwdOverride) | Kwd(KwdDynamic) | 56 | Kwd(KwdExtern) | Kwd(KwdAbstract) | Kwd(KwdOverload): 57 | stream.consumeToTempStore(); 58 | case Kwd(KwdFinal): 59 | WalkFinal.walkFinal(stream, parent); 60 | case Comment(_) | CommentLine(_): 61 | if (stream.hasTempStore()) { 62 | stream.consumeToTempStore(); 63 | } 64 | else { 65 | parent.addChild(stream.consumeToken()); 66 | } 67 | default: 68 | switch (TokenStream.MODE) { 69 | case Relaxed: WalkStatement.walkStatement(stream, parent); 70 | case Strict: throw "invalid token tree structure - found:" + '${stream.token()}'; 71 | } 72 | } 73 | } 74 | var tempStore:Array = stream.getTempStore(); 75 | if (tempStore.length > 0) { 76 | switch (TokenStream.MODE) { 77 | case Relaxed: 78 | stream.applyTempStore(parent); 79 | case Strict: 80 | throw "invalid token tree structure - found:" + '$tempStore'; 81 | } 82 | } 83 | } 84 | 85 | static function walkClassContinueAfterSharp(stream:TokenStream, parent:TokenTree) { 86 | @:nullSafety(Off) 87 | var brOpen:TokenTreeAccessHelper = TokenTreeAccessHelper.access(parent).lastChild().matches(Sharp("if")).lastOf(Kwd(KwdFunction)).firstChild() 88 | .lastChild().matches(BrOpen); 89 | 90 | if (brOpen.token == null) return; 91 | 92 | @:nullSafety(Off) 93 | if (brOpen.lastChild().matches(BrClose).token != null) return; 94 | 95 | WalkBlock.walkBlockContinue(stream, parent); 96 | } 97 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkComment.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkComment { 4 | public static function walkComment(stream:TokenStream, parent:TokenTree) { 5 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 6 | while (stream.hasMore() && progress.streamHasChanged()) { 7 | switch (stream.token()) { 8 | case Comment(_), CommentLine(_): 9 | var comment:TokenTree = stream.consumeToken(); 10 | parent.addChild(comment); 11 | default: 12 | return; 13 | } 14 | } 15 | } 16 | 17 | public static function tryWalkComment(stream:TokenStream, parent:TokenTree, expect:TokenTreeDef) { 18 | var currentPos:Int = stream.getStreamIndex(); 19 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 20 | var comments:Array = []; 21 | while (stream.hasMore() && progress.streamHasChanged()) { 22 | switch (stream.token()) { 23 | case Comment(_), CommentLine(_): 24 | comments.push(stream.consumeToken()); 25 | default: 26 | if (stream.matches(expect)) { 27 | for (comment in comments) { 28 | parent.addChild(comment); 29 | } 30 | return; 31 | } 32 | stream.rewindTo(currentPos); 33 | return; 34 | } 35 | } 36 | } 37 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkDoWhile.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkDoWhile { 4 | /** 5 | * Kwd(KwdDo) 6 | * |- BrOpen 7 | * | |- statement 8 | * | |- statement 9 | * | |- BrClose 10 | * |- Kwd(KwdWhile) 11 | * |- POpen 12 | * | |- expression 13 | * | |- PClose 14 | * |- Semicolon 15 | * 16 | */ 17 | public static function walkDoWhile(stream:TokenStream, parent:TokenTree) { 18 | var doTok:TokenTree = stream.consumeTokenDef(Kwd(KwdDo)); 19 | parent.addChild(doTok); 20 | stream.applyTempStore(doTok); 21 | WalkComment.walkComment(stream, doTok); 22 | WalkBlock.walkBlock(stream, doTok); 23 | var whileTok:TokenTree = stream.consumeTokenDef(Kwd(KwdWhile)); 24 | doTok.addChild(whileTok); 25 | WalkStatement.walkStatement(stream, whileTok); 26 | WalkComment.walkComment(stream, whileTok); 27 | if (stream.tokenForMatch().match(Semicolon)) whileTok.addChild(stream.consumeToken()); 28 | } 29 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkEnum.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkEnum { 4 | public static function walkEnum(stream:TokenStream, parent:TokenTree) { 5 | var typeTok:TokenTree = stream.consumeToken(); 6 | if (stream.tokenForMatch().match(Kwd(KwdAbstract))) { 7 | stream.addToTempStore(typeTok); 8 | WalkAbstract.walkAbstract(stream, parent); 9 | return; 10 | } 11 | parent.addChild(typeTok); 12 | var name:TokenTree = WalkTypeNameDef.walkTypeNameDef(stream, typeTok); 13 | // add all comments, annotations 14 | stream.applyTempStore(name); 15 | WalkBlock.walkBlock(stream, name); 16 | } 17 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkExtends.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkExtends { 4 | public static function walkExtends(stream:TokenStream, parent:TokenTree) { 5 | if (!stream.tokenForMatch().match(Kwd(KwdExtends))) return; 6 | var parentType:TokenTree = stream.consumeTokenDef(Kwd(KwdExtends)); 7 | parent.addChild(parentType); 8 | WalkComment.walkComment(stream, parent); 9 | WalkTypeNameDef.walkTypeNameDef(stream, parentType); 10 | WalkComment.walkComment(stream, parent); 11 | WalkExtends.walkExtends(stream, parent); 12 | WalkComment.walkComment(stream, parent); 13 | } 14 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkFieldDef.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkFieldDef { 4 | public static function walkFieldDef(stream:TokenStream, parent:TokenTree) { 5 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 6 | while (progress.streamHasChanged()) { 7 | switch (stream.token()) { 8 | case Kwd(KwdVar) | Kwd(KwdFunction) | Kwd(KwdFinal): 9 | var tok:TokenTree = stream.consumeToken(); 10 | parent.addChild(tok); 11 | parent = tok; 12 | case At: 13 | stream.addToTempStore(WalkAt.walkAt(stream)); 14 | case Comment(_), CommentLine(_): 15 | WalkComment.walkComment(stream, parent); 16 | default: 17 | break; 18 | } 19 | } 20 | 21 | var name:TokenTree = WalkTypeNameDef.walkTypeNameDef(stream, parent); 22 | stream.applyTempStore(name); 23 | if (stream.tokenForMatch().match(DblDot)) { 24 | var dblDot:TokenTree = stream.consumeTokenDef(DblDot); 25 | name.addChild(dblDot); 26 | WalkTypedefBody.walkTypedefBody(stream, dblDot); 27 | } 28 | if (stream.tokenForMatch().match(Binop(OpAssign))) { 29 | WalkStatement.walkStatement(stream, name); 30 | } 31 | switch (stream.token()) { 32 | case Comma: 33 | name.addChild(stream.consumeTokenDef(Comma)); 34 | case Semicolon: 35 | name.addChild(stream.consumeTokenDef(Semicolon)); 36 | default: 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkFile.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkFile { 4 | public static function walkFile(stream:TokenStream, parent:TokenTree) { 5 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 6 | while (stream.hasMore() && progress.streamHasChanged()) { 7 | switch (stream.token()) { 8 | case Kwd(KwdPackage) | Kwd(KwdImport) | Kwd(KwdUsing): 9 | stream.applyTempStore(parent); 10 | WalkPackageImport.walkPackageImport(stream, parent); 11 | case Sharp(_): 12 | WalkSharp.walkSharp(stream, parent, WalkFile.walkFile); 13 | if (!stream.hasMore()) return; 14 | switch (stream.token()) { 15 | case BrOpen: WalkBlock.walkBlock(stream, parent.children[parent.children.length - 1]); 16 | default: 17 | } 18 | case At: 19 | stream.addToTempStore(WalkAt.walkAt(stream)); 20 | case Comment(_) | CommentLine(_): 21 | if (stream.hasTempStore()) { 22 | stream.consumeToTempStore(); 23 | } 24 | else { 25 | WalkComment.walkComment(stream, parent); 26 | } 27 | case Kwd(KwdClass) | Kwd(KwdInterface) | Kwd(KwdEnum) | Kwd(KwdTypedef) | Kwd(KwdAbstract): 28 | WalkType.walkType(stream, parent); 29 | case PClose, BrClose, BkClose, Semicolon, Comma: 30 | parent.addChild(stream.consumeToken()); 31 | case Kwd(KwdPublic) | Kwd(KwdPrivate) | Kwd(KwdStatic) | Kwd(KwdInline) | Kwd(KwdMacro) | Kwd(KwdDynamic) | Kwd(KwdExtern) | Kwd(KwdOverload): 32 | stream.consumeToTempStore(); 33 | case Kwd(KwdFinal): 34 | WalkFinal.walkFinal(stream, parent); 35 | case Kwd(KwdVar): 36 | WalkVar.walkVar(stream, parent); 37 | case Kwd(KwdFunction): 38 | WalkFunction.walkFunction(stream, parent); 39 | default: 40 | WalkBlock.walkBlock(stream, parent); 41 | } 42 | } 43 | var tempStore:Array = stream.getTempStore(); 44 | for (stored in tempStore) { 45 | switch (stored.tok) { 46 | case Kwd(KwdExtern) | Kwd(KwdPrivate) | Kwd(KwdPublic) | At: 47 | switch (TokenStream.MODE) { 48 | case Relaxed: parent.addChild(stored); 49 | case Strict: throw "invalid token tree structure - found:" + stored; 50 | } 51 | default: 52 | parent.addChild(stored); 53 | } 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkFinal.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkFinal { 4 | public static function walkFinal(stream:TokenStream, parent:TokenTree) { 5 | var name:Null = null; 6 | var finalTok:TokenTree = stream.consumeToken(); 7 | stream.addToTempStore(finalTok); 8 | WalkComment.walkComment(stream, parent); 9 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 10 | while (progress.streamHasChanged()) { 11 | switch (stream.token()) { 12 | case Const(CIdent(_)): 13 | break; 14 | case Kwd(KwdPublic) | Kwd(KwdPrivate) | Kwd(KwdStatic) | Kwd(KwdInline) | Kwd(KwdMacro) | Kwd(KwdOverride) | Kwd(KwdDynamic) | Kwd(KwdExtern): 15 | stream.consumeToTempStore(); 16 | case Comment(_) | CommentLine(_): 17 | stream.consumeToTempStore(); 18 | case Kwd(KwdFunction): 19 | return; 20 | case Kwd(KwdClass) | Kwd(KwdInterface): 21 | return; 22 | default: 23 | } 24 | } 25 | parent.addChild(finalTok); 26 | 27 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 28 | while (progress.streamHasChanged()) { 29 | WalkComment.walkComment(stream, parent); 30 | switch (stream.token()) { 31 | case At: 32 | WalkAt.walkAts(stream); 33 | default: 34 | } 35 | WalkComment.walkComment(stream, parent); 36 | var nameParent:TokenTree = finalTok; 37 | if (stream.tokenForMatch().match(Question)) { 38 | nameParent = stream.consumeToken(); 39 | finalTok.addChild(nameParent); 40 | } 41 | name = stream.consumeConstIdent(); 42 | nameParent.addChild(name); 43 | var tempStore:Array = stream.getTempStore(); 44 | for (stored in tempStore) { 45 | switch (stored.tok) { 46 | case Kwd(KwdFinal): 47 | default: 48 | name.addChild(stored); 49 | } 50 | } 51 | stream.clearTempStore(); 52 | WalkComment.walkComment(stream, name); 53 | if (stream.tokenForMatch().match(POpen)) { 54 | WalkPOpen.walkPOpen(stream, name); 55 | } 56 | if (stream.tokenForMatch().match(DblDot)) { 57 | var dblDot:TokenTree = stream.consumeToken(); 58 | name.addChild(dblDot); 59 | WalkTypedefBody.walkTypedefAlias(stream, dblDot); 60 | } 61 | if (stream.tokenForMatch().match(Binop(OpAssign))) { 62 | WalkStatement.walkStatement(stream, name); 63 | } 64 | if (stream.tokenForMatch().match(Comma)) { 65 | var comma:TokenTree = stream.consumeToken(); 66 | name.addChild(comma); 67 | continue; 68 | } 69 | break; 70 | } 71 | 72 | if (stream.tokenForMatch().match(Semicolon)) { 73 | name.addChild(stream.consumeToken()); 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkFor.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkFor { 4 | /** 5 | * Kwd(KwdFor) 6 | * |- POpen 7 | * | |- Const(CIdent(_)) 8 | * | | |- Kwd(KwdIn) 9 | * | | |- Const(CIdent(_) 10 | * | |- PClose 11 | * |- BrOpen 12 | * |- statement 13 | * |- statement 14 | * |- BrClose 15 | * 16 | * Kwd(KwdFor) 17 | * |- POpen 18 | * | |- Const(CIdent(_)) 19 | * | | |- Kwd(KwdIn) 20 | * | | |- IntInterval(_) 21 | * | | |- Const(CInt(_)) 22 | * | |- PClose 23 | * |- BrOpen 24 | * |- statement 25 | * |- statement 26 | * |- BrClose 27 | * 28 | */ 29 | public static function walkFor(stream:TokenStream, parent:TokenTree) { 30 | var forTok:TokenTree = stream.consumeTokenDef(Kwd(KwdFor)); 31 | parent.addChild(forTok); 32 | stream.applyTempStore(forTok); 33 | WalkComment.walkComment(stream, forTok); 34 | WalkFor.walkForPOpen(stream, forTok); 35 | WalkComment.walkComment(stream, forTok); 36 | WalkBlock.walkBlock(stream, forTok); 37 | } 38 | 39 | /** 40 | * POpen 41 | * |- Const(CIdent(_)) 42 | * | |- Kwd(KwdIn) 43 | * | |- Const(CIdent(_) 44 | * |- PClose 45 | * 46 | * POpen 47 | * |- Const(CIdent(_)) 48 | * | |- Kwd(KwdIn) 49 | * | |- IntInterval(_) 50 | * | |- Const(CInt(_)) 51 | * |- PClose 52 | * 53 | */ 54 | static function walkForPOpen(stream:TokenStream, parent:TokenTree) { 55 | var pOpen:TokenTree = stream.consumeTokenDef(POpen); 56 | parent.addChild(pOpen); 57 | WalkComment.walkComment(stream, pOpen); 58 | var identifier:Null = null; 59 | switch (stream.token()) { 60 | case Dollar(_): 61 | WalkStatement.walkStatement(stream, pOpen); 62 | identifier = pOpen.getLastChild(); 63 | default: 64 | identifier = stream.consumeConstIdent(); 65 | pOpen.addChild(identifier); 66 | } 67 | WalkComment.walkComment(stream, identifier); 68 | if (stream.tokenForMatch().match(Binop(OpArrow))) { 69 | var arrowTok:TokenTree = stream.consumeToken(); 70 | identifier.addChild(arrowTok); 71 | switch (stream.token()) { 72 | case Dollar(_): 73 | WalkStatement.walkStatement(stream, arrowTok); 74 | default: 75 | arrowTok.addChild(stream.consumeConstIdent()); 76 | } 77 | } 78 | var inTok:Null = null; 79 | switch (stream.token()) { 80 | case Kwd(KwdIn) | Binop(OpIn): 81 | inTok = stream.consumeToken(); 82 | identifier.addChild(inTok); 83 | WalkComment.walkComment(stream, inTok); 84 | WalkStatement.walkStatement(stream, inTok); 85 | WalkComment.walkComment(stream, pOpen); 86 | pOpen.addChild(stream.consumeTokenDef(PClose)); 87 | WalkComment.walkComment(stream, parent); 88 | case PClose: 89 | pOpen.addChild(stream.consumeToken()); 90 | WalkComment.walkComment(stream, parent); 91 | return; 92 | default: 93 | } 94 | } 95 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkFunction.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkFunction { 4 | public static function walkFunction(stream:TokenStream, parent:TokenTree) { 5 | var funcTok:Null = stream.consumeTokenDef(Kwd(KwdFunction)); 6 | parent.addChild(funcTok); 7 | WalkComment.walkComment(stream, funcTok); 8 | 9 | var name:TokenTree = funcTok; 10 | switch (stream.token()) { 11 | case Kwd(KwdNew): 12 | name = WalkTypeNameDef.walkTypeNameDef(stream, funcTok); 13 | case POpen: 14 | case Binop(OpLt): 15 | WalkLtGt.walkLtGt(stream, funcTok); 16 | name = funcTok.getLastChild(); 17 | default: 18 | name = WalkTypeNameDef.walkTypeNameDef(stream, funcTok); 19 | } 20 | stream.applyTempStore(name); 21 | WalkComment.walkComment(stream, name); 22 | WalkFunction.walkFunctionParameters(stream, name); 23 | WalkComment.walkComment(stream, name); 24 | 25 | switch (stream.token()) { 26 | case Sharp(_): 27 | WalkSharp.walkSharp(stream, name, WalkStatement.walkStatement); 28 | switch (stream.token()) { 29 | case DblDot | BrOpen: 30 | default: return; 31 | } 32 | default: 33 | } 34 | 35 | if (stream.tokenForMatch().match(DblDot)) { 36 | var dblDot:Null = stream.consumeToken(); 37 | name.addChild(dblDot); 38 | WalkTypeNameDef.walkTypeNameDef(stream, dblDot); 39 | } 40 | WalkBlock.walkBlock(stream, name); 41 | } 42 | 43 | static function walkFunctionParameters(stream:TokenStream, parent:TokenTree) { 44 | var pOpen:TokenTree = stream.consumeTokenDef(POpen); 45 | parent.addChild(pOpen); 46 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 47 | while (progress.streamHasChanged()) { 48 | WalkComment.walkComment(stream, pOpen); 49 | if (stream.tokenForMatch().match(PClose)) break; 50 | WalkFieldDef.walkFieldDef(stream, pOpen); 51 | } 52 | pOpen.addChild(stream.consumeTokenDef(PClose)); 53 | } 54 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkIf.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkIf { 4 | /** 5 | * Kwd(KwdIf) 6 | * |- POpen 7 | * | |- expression 8 | * | |- PClose 9 | * |- BrOpen 10 | * | |- statement 11 | * | |- statement 12 | * | |- BrClose 13 | * |- Kwd(KwdElse) 14 | * |- BrOpen 15 | * |- statement 16 | * |- statement 17 | * |- BrClose 18 | * 19 | */ 20 | public static function walkIf(stream:TokenStream, parent:TokenTree) { 21 | var ifTok:TokenTree = stream.consumeTokenDef(Kwd(KwdIf)); 22 | parent.addChild(ifTok); 23 | // condition 24 | stream.applyTempStore(ifTok); 25 | WalkStatement.walkStatement(stream, ifTok); 26 | if (stream.tokenForMatch().match(DblDot)) return; 27 | // if-expr 28 | WalkBlock.walkBlock(stream, ifTok); 29 | WalkComment.tryWalkComment(stream, ifTok, Kwd(KwdElse)); 30 | if (stream.tokenForMatch().match(Kwd(KwdElse))) { 31 | var elseTok:TokenTree = stream.consumeTokenDef(Kwd(KwdElse)); 32 | ifTok.addChild(elseTok); 33 | // else-expr 34 | WalkBlock.walkBlock(stream, elseTok); 35 | } 36 | } 37 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkImplements.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkImplements { 4 | public static function walkImplements(stream:TokenStream, parent:TokenTree) { 5 | if (!stream.tokenForMatch().match(Kwd(KwdImplements))) return; 6 | var interfacePart:TokenTree = stream.consumeTokenDef(Kwd(KwdImplements)); 7 | parent.addChild(interfacePart); 8 | WalkComment.walkComment(stream, parent); 9 | WalkTypeNameDef.walkTypeNameDef(stream, interfacePart); 10 | WalkComment.walkComment(stream, parent); 11 | WalkImplements.walkImplements(stream, parent); 12 | WalkComment.walkComment(stream, parent); 13 | } 14 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkInterface.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkInterface { 4 | public static function walkInterface(stream:TokenStream, parent:TokenTree) { 5 | var typeTok:TokenTree = stream.consumeToken(); 6 | parent.addChild(typeTok); 7 | // add name 8 | var name:TokenTree = WalkTypeNameDef.walkTypeNameDef(stream, typeTok); 9 | // add all comments, annotations 10 | stream.applyTempStore(name); 11 | WalkClass.walkClassExtends(stream, name); 12 | var block:TokenTree = stream.consumeTokenDef(BrOpen); 13 | name.addChild(block); 14 | WalkInterface.walkInterfaceBody(stream, block); 15 | block.addChild(stream.consumeTokenDef(BrClose)); 16 | } 17 | 18 | public static function walkInterfaceBody(stream:TokenStream, parent:TokenTree) { 19 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 20 | while (progress.streamHasChanged()) { 21 | switch (stream.token()) { 22 | case Kwd(KwdVar): 23 | WalkVar.walkVar(stream, parent); 24 | case Kwd(KwdFunction): 25 | WalkFunction.walkFunction(stream, parent); 26 | case Sharp(_): 27 | WalkSharp.walkSharp(stream, parent, WalkInterface.walkInterfaceBody); 28 | case At: 29 | stream.addToTempStore(WalkAt.walkAt(stream)); 30 | case BrClose: 31 | break; 32 | case Semicolon: 33 | parent.addChild(stream.consumeToken()); 34 | case Kwd(KwdPublic), Kwd(KwdPrivate), Kwd(KwdStatic), Kwd(KwdInline), Kwd(KwdMacro), Kwd(KwdOverride), Kwd(KwdDynamic), Kwd(KwdExtern): 35 | stream.consumeToTempStore(); 36 | case Kwd(KwdFinal): 37 | WalkFinal.walkFinal(stream, parent); 38 | case Comment(_), CommentLine(_): 39 | if (stream.hasTempStore()) { 40 | stream.consumeToTempStore(); 41 | } 42 | else { 43 | parent.addChild(stream.consumeToken()); 44 | } 45 | default: 46 | stream.consumeToTempStore(); 47 | } 48 | } 49 | stream.applyTempStore(parent); 50 | } 51 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkLtGt.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkLtGt { 4 | public static function walkLtGt(stream:TokenStream, parent:TokenTree) { 5 | var ltTok:TokenTree = stream.consumeTokenDef(Binop(OpLt)); 6 | parent.addChild(ltTok); 7 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 8 | while (progress.streamHasChanged()) { 9 | switch (stream.token()) { 10 | case Comma: 11 | var comma:TokenTree = stream.consumeToken(); 12 | ltTok.addChild(comma); 13 | WalkTypeNameDef.walkTypeNameDef(stream, ltTok); 14 | case Binop(OpAssign): 15 | var equals:TokenTree = stream.consumeToken(); 16 | ltTok.getLastChild().addChild(equals); 17 | WalkTypeNameDef.walkTypeNameDef(stream, equals); 18 | case Binop(OpGt): 19 | break; 20 | case DblDot: 21 | var dblDot:TokenTree = stream.consumeToken(); 22 | ltTok.addChild(dblDot); 23 | WalkTypeNameDef.walkTypeNameDef(stream, dblDot); 24 | case POpen: 25 | WalkPOpen.walkPOpen(stream, ltTok); 26 | default: 27 | WalkTypeNameDef.walkTypeNameDef(stream, ltTok); 28 | } 29 | } 30 | ltTok.addChild(stream.consumeTokenDef(Binop(OpGt))); 31 | } 32 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkNew.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkNew { 4 | public static function walkNew(stream:TokenStream, parent:TokenTree) { 5 | var newTok:TokenTree = stream.consumeTokenDef(Kwd(KwdNew)); 6 | parent.addChild(newTok); 7 | var name:TokenTree = WalkTypeNameDef.walkTypeNameDef(stream, newTok); 8 | var pOpen:Null = null; 9 | 10 | WalkComment.walkComment(stream, name); 11 | switch (stream.token()) { 12 | case POpen: 13 | pOpen = WalkPOpen.walkPOpen(stream, name); 14 | case Sharp(_): 15 | WalkSharp.walkSharp(stream, parent, WalkStatement.walkStatement); 16 | default: 17 | } 18 | WalkComment.walkComment(stream, name); 19 | 20 | switch (stream.token()) { 21 | case Dot | Binop(_) | Const(CIdent("is")) | BkOpen: 22 | if (pOpen != null) { 23 | WalkStatement.walkStatement(stream, pOpen); 24 | } 25 | else { 26 | WalkStatement.walkStatement(stream, name); 27 | } 28 | default: 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkPOpen.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | import tokentree.walk.WalkSharp.WalkSharpConsts; 4 | 5 | class WalkPOpen { 6 | public static function walkPOpen(stream:TokenStream, parent:TokenTree, walkTrailingComments:Bool = true):TokenTree { 7 | var pOpen:TokenTree = stream.consumeTokenDef(POpen); 8 | parent.addChild(pOpen); 9 | stream.applyTempStore(pOpen); 10 | WalkPOpen.walkPOpenParts(stream, pOpen); 11 | pOpen.addChild(stream.consumeTokenDef(PClose)); 12 | if (walkTrailingComments) { 13 | WalkComment.walkComment(stream, parent); 14 | } 15 | if (stream.hasMore()) { 16 | switch (stream.token()) { 17 | case Arrow: 18 | var arrow:TokenTree = stream.consumeToken(); 19 | pOpen.addChild(arrow); 20 | WalkBlock.walkBlock(stream, arrow); 21 | case Sharp(WalkSharpConsts.IF): 22 | switch (parent.tok) { 23 | case Sharp(WalkSharpConsts.IF): 24 | default: WalkSharp.walkSharp(stream, parent, WalkStatement.walkStatement); 25 | } 26 | default: 27 | } 28 | } 29 | return pOpen; 30 | } 31 | 32 | public static function walkPOpenParts(stream:TokenStream, parent:TokenTree) { 33 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 34 | while (progress.streamHasChanged()) { 35 | switch (stream.token()) { 36 | case BrOpen: 37 | WalkBlock.walkBlock(stream, parent); 38 | case BkOpen: 39 | WalkArrayAccess.walkArrayAccess(stream, parent); 40 | case PClose: 41 | break; 42 | case Sharp(_): 43 | WalkSharp.walkSharp(stream, parent, WalkPOpen.walkPOpenParts); 44 | case Comma: 45 | var comma:TokenTree = stream.consumeToken(); 46 | var child:Null = parent.getLastChild(); 47 | if (child == null) child = parent; 48 | child.addChild(comma); 49 | default: 50 | WalkStatement.walkStatement(stream, parent); 51 | } 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkPackageImport.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkPackageImport { 4 | /** 5 | * Kwd(KwdPackage) 6 | * |- Const(CIdent(_)) 7 | * |- Dot 8 | * |- Const(CIdent(_)) 9 | * |- Semicolon 10 | * 11 | * Kwd(KwdImport) 12 | * |- Const(CIdent(_)) 13 | * |- Dot 14 | * |- Const(CIdent(_)) 15 | * |- Semicolon 16 | * 17 | */ 18 | public static function walkPackageImport(stream:TokenStream, parent:TokenTree) { 19 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 20 | while (stream.hasMore() && progress.streamHasChanged()) { 21 | switch (stream.token()) { 22 | case Sharp(_): 23 | WalkSharp.walkSharp(stream, parent, walkPackageImport); 24 | case Semicolon: 25 | var newChild:TokenTree = stream.consumeToken(); 26 | parent.addChild(newChild); 27 | return; 28 | default: 29 | var newChild:TokenTree = stream.consumeToken(); 30 | parent.addChild(newChild); 31 | parent = newChild; 32 | } 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkQuestion.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkQuestion { 4 | public static function walkQuestion(stream:TokenStream, parent:TokenTree) { 5 | var ternary:Bool = isTernary(stream, parent); 6 | if (!ternary) { 7 | WalkFieldDef.walkFieldDef(stream, parent); 8 | return; 9 | } 10 | var question:TokenTree = stream.consumeTokenDef(Question); 11 | parent.addChild(question); 12 | WalkComment.walkComment(stream, question); 13 | WalkStatement.walkStatement(stream, question); 14 | WalkComment.walkComment(stream, question); 15 | if (!stream.tokenForMatch().match(DblDot)) return; 16 | var dblDotTok:TokenTree = stream.consumeTokenDef(DblDot); 17 | question.addChild(dblDotTok); 18 | WalkStatement.walkStatement(stream, dblDotTok); 19 | } 20 | 21 | public static function isTernary(stream:TokenStream, parent:TokenTree):Bool { 22 | var lastChild:Null = parent.getLastChild(); 23 | if (lastChild == null) { 24 | return switch (parent.tok) { 25 | case Const(_): true; 26 | case Kwd(KwdTrue), Kwd(KwdFalse), Kwd(KwdNull): true; 27 | case Kwd(KwdThis): true; 28 | case Dollar(_): true; 29 | case PClose: true; 30 | default: false; 31 | } 32 | } 33 | return switch (lastChild.tok) { 34 | case Const(_): true; 35 | case BkOpen: true; 36 | case BrOpen: true; 37 | case Binop(OpAdd), Binop(OpSub): true; 38 | case Unop(_): true; 39 | case Kwd(KwdCast): true; 40 | case Kwd(KwdNew): true; 41 | case Kwd(KwdTrue), Kwd(KwdFalse), Kwd(KwdNull): true; 42 | case Kwd(KwdMacro): (lastChild.index + 1 != stream.getStreamIndex()); 43 | case Kwd(KwdThis), Kwd(KwdUntyped): true; 44 | case Kwd(KwdFunction): true; 45 | case Dollar(_): true; 46 | case POpen: true; 47 | case PClose: true; 48 | case DblDot: true; 49 | default: false; 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkSharp.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkSharp { 4 | /** 5 | * Sharp("if") | Sharp("elseif") 6 | * |- POpen 7 | * |- expression 8 | * |- PClose 9 | * 10 | * Sharp("if") | Sharp("elseif") 11 | * |- expression 12 | * 13 | * Sharp("end") 14 | * 15 | * Sharp("else") 16 | * 17 | * Sharp(_) 18 | * 19 | */ 20 | public static function walkSharp(stream:TokenStream, parent:TokenTree, walker:WalkCB) { 21 | switch (stream.token()) { 22 | case Sharp(IF): 23 | WalkSharp.walkSharpIf(stream, parent, walker); 24 | case Sharp(ERROR): 25 | var errorToken:TokenTree = stream.consumeToken(); 26 | parent.addChild(errorToken); 27 | switch (stream.token()) { 28 | case Const(CString(_)): errorToken.addChild(stream.consumeToken()); 29 | default: 30 | } 31 | case Sharp(ELSEIF): 32 | WalkSharp.walkSharpElseIf(stream, parent); 33 | case Sharp(ELSE): 34 | WalkSharp.walkSharpElse(stream, parent); 35 | case Sharp(END): 36 | WalkSharp.walkSharpEnd(stream, parent); 37 | case Sharp(_): 38 | parent.addChild(stream.consumeToken()); 39 | default: 40 | } 41 | } 42 | 43 | static function walkSharpIf(stream:TokenStream, parent:TokenTree, walker:WalkCB) { 44 | var ifToken:TokenTree = stream.consumeToken(); 45 | parent.addChild(ifToken); 46 | WalkSharp.walkSharpIfExpr(stream, ifToken); 47 | stream.pushSharpIf(ifToken); 48 | var newParent:TokenTree = ifToken; 49 | 50 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 51 | while (progress.streamHasChanged()) { 52 | try { 53 | #if cs 54 | csharpWalkerGuard(stream, newParent, walker); 55 | #else 56 | walker(stream, newParent); 57 | #end 58 | switch (stream.token()) { 59 | case BrClose, PClose, Comma: 60 | var newChild:TokenTree = stream.consumeToken(); 61 | newParent.addChild(newChild); 62 | default: 63 | } 64 | } 65 | catch (e:SharpElseException) { 66 | // continue; 67 | newParent = e.token; 68 | } 69 | catch (e:SharpEndException) { 70 | stream.popSharpIf(); 71 | return; 72 | } 73 | } 74 | } 75 | 76 | #if cs 77 | static function csharpWalkerGuard(stream:TokenStream, parent:TokenTree, walker:WalkCB) { 78 | try { 79 | walker(stream, parent); 80 | } 81 | catch (e:SharpElseException) { 82 | throw e; 83 | } 84 | catch (e:SharpEndException) { 85 | throw e; 86 | } 87 | catch (e:Any) { 88 | var exception:cs.system.Exception = cast(e, cs.system.Exception); 89 | throw exception.InnerException; 90 | } 91 | } 92 | #end 93 | 94 | static function walkSharpElse(stream:TokenStream, parent:TokenTree) { 95 | var sharpIfParent:TokenTree = stream.peekSharpIf(); 96 | var ifToken:TokenTree = stream.consumeToken(); 97 | sharpIfParent.addChild(ifToken); 98 | stream.applyTempStore(sharpIfParent); 99 | throw new SharpElseException(ifToken); 100 | } 101 | 102 | static function walkSharpElseIf(stream:TokenStream, parent:TokenTree) { 103 | var sharpIfParent:TokenTree = stream.peekSharpIf(); 104 | var ifToken:TokenTree = stream.consumeToken(); 105 | sharpIfParent.addChild(ifToken); 106 | stream.applyTempStore(sharpIfParent); 107 | WalkSharp.walkSharpIfExpr(stream, ifToken); 108 | throw new SharpElseException(ifToken); 109 | } 110 | 111 | static function walkSharpEnd(stream:TokenStream, parent:TokenTree) { 112 | var sharpIfParent:TokenTree = stream.peekSharpIf(); 113 | var endToken:TokenTree = stream.consumeToken(); 114 | stream.applyTempStore(sharpIfParent); 115 | sharpIfParent.addChild(endToken); 116 | throw new SharpEndException(); 117 | } 118 | 119 | static function walkSharpIfExpr(stream:TokenStream, parent:TokenTree) { 120 | var childToken:TokenTree; 121 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 122 | while (progress.streamHasChanged()) { 123 | switch (stream.token()) { 124 | case Unop(OpNot): 125 | childToken = stream.consumeToken(); 126 | parent.addChild(childToken); 127 | WalkSharp.walkSharpIfExpr(stream, childToken); 128 | return; 129 | case POpen: 130 | WalkPOpen.walkPOpen(stream, parent); 131 | return; 132 | case Kwd(_), Const(CIdent(_)): 133 | childToken = stream.consumeToken(); 134 | parent.addChild(childToken); 135 | if (!stream.hasMore()) return; 136 | switch (stream.token()) { 137 | case Dot: 138 | default: return; 139 | } 140 | var pos:Null = stream.getTokenPos(); 141 | if (pos == null) return; 142 | if (pos.min == childToken.pos.max + 1) continue; 143 | 144 | var dot:TokenTree = stream.consumeToken(); 145 | childToken.addChild(dot); 146 | WalkSharp.walkSharpIfExpr(stream, dot); 147 | return; 148 | default: 149 | return; 150 | } 151 | } 152 | } 153 | } 154 | 155 | typedef WalkCB = TokenStream->TokenTree->Void; 156 | 157 | class SharpElseException { 158 | public var token:TokenTree; 159 | 160 | public function new(token:TokenTree) { 161 | this.token = token; 162 | } 163 | } 164 | 165 | class SharpEndException { 166 | public function new() {} 167 | } 168 | 169 | enum abstract WalkSharpConsts(String) to String { 170 | var IF = "if"; 171 | 172 | var ELSEIF = "elseif"; 173 | 174 | var ELSE = "else"; 175 | 176 | var END = "end"; 177 | 178 | var ERROR = "error"; 179 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkStatement.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | import tokentree.utils.TokenTreeCheckUtils; 4 | 5 | class WalkStatement { 6 | public static function walkStatement(stream:TokenStream, parent:TokenTree) { 7 | walkStatementWithoutSemicolon(stream, parent); 8 | if (stream.tokenForMatch().match(Semicolon)) { 9 | var semicolon:TokenTree = stream.consumeToken(); 10 | 11 | var lastChild:Null = switch (parent.tok) { 12 | case Binop(OpAdd) | Binop(OpSub) | Binop(OpBoolAnd) | Binop(OpBoolOr): parent.parent.getLastChild(); 13 | default: parent.getLastChild(); 14 | } 15 | 16 | if (lastChild == null) { 17 | lastChild = parent; 18 | } 19 | switch (lastChild.tok) { 20 | case BrClose, BkClose, PClose: 21 | lastChild = parent; 22 | default: 23 | } 24 | lastChild.addChild(semicolon); 25 | } 26 | } 27 | 28 | static function walkStatementWithoutSemicolon(stream:TokenStream, parent:TokenTree) { 29 | WalkComment.walkComment(stream, parent); 30 | 31 | var wantMore:Bool = true; 32 | 33 | WalkAt.walkAts(stream); 34 | switch (stream.token()) { 35 | case Binop(OpSub): 36 | WalkBinopSub.walkBinopSub(stream, parent); 37 | return; 38 | case Binop(OpLt): 39 | if (stream.isTypedParam()) { 40 | WalkLtGt.walkLtGt(stream, parent); 41 | if (stream.tokenForMatch().match(Arrow)) { 42 | walkStatementWithoutSemicolon(stream, parent); 43 | } 44 | if (stream.tokenForMatch().match(POpen)) { 45 | walkStatementWithoutSemicolon(stream, parent); 46 | } 47 | return; 48 | } 49 | wantMore = true; 50 | case Binop(OpGt): 51 | var gtTok:TokenTree = stream.consumeOpGt(); 52 | parent.addChild(gtTok); 53 | walkStatementWithoutSemicolon(stream, gtTok); 54 | return; 55 | case Binop(OpOr): 56 | if ((parent.parent != null) && (parent.parent.tok != Root)) { 57 | switch (parent.parent.tok) { 58 | case Kwd(KwdCase): 59 | var orTok:TokenTree = stream.consumeToken(); 60 | parent.addChild(orTok); 61 | walkStatementWithoutSemicolon(stream, parent.parent); 62 | return; 63 | default: 64 | } 65 | } 66 | wantMore = true; 67 | case Binop(_): 68 | wantMore = true; 69 | case Const(CIdent("is")): 70 | wantMore = true; 71 | case Unop(_): 72 | if (parent.isCIdentOrCString()) { 73 | var newChild:TokenTree = stream.consumeToken(); 74 | parent.addChild(newChild); 75 | if (!stream.hasMore()) return; 76 | switch (stream.token()) { 77 | case Dot | Binop(_) | Const(CIdent("is")): walkStatementWithoutSemicolon(stream, newChild); 78 | default: 79 | } 80 | return; 81 | } 82 | case IntInterval(_) | Spread: 83 | wantMore = true; 84 | case Const(CIdent("final")) | Kwd(_): 85 | if (walkKeyword(stream, parent)) wantMore = true; 86 | else return; 87 | case Arrow: 88 | wantMore = true; 89 | case BrOpen: 90 | WalkBlock.walkBlock(stream, parent); 91 | return; 92 | case BkOpen: 93 | WalkArrayAccess.walkArrayAccess(stream, parent); 94 | walkStatementContinue(stream, parent); 95 | return; 96 | case Dollar(name): 97 | walkDollarStatement(stream, parent); 98 | return; 99 | case POpen: 100 | walkPOpen(stream, parent); 101 | return; 102 | case Question: 103 | WalkQuestion.walkQuestion(stream, parent); 104 | return; 105 | case PClose, BrClose, BkClose: 106 | return; 107 | case Comma: 108 | return; 109 | case Semicolon: 110 | return; 111 | case Sharp(_): 112 | WalkSharp.walkSharp(stream, parent, walkStatement); 113 | walkStatementContinueAfterSharp(stream, parent); 114 | return; 115 | case Dot | QuestionDot: 116 | wantMore = true; 117 | case DblDot: 118 | switch (parent.tok) { 119 | case Dot: return; 120 | case Kwd(KwdMacro): 121 | walkDblDot(stream, parent); 122 | return; 123 | default: 124 | } 125 | if (parent.tok.match(Dot)) { 126 | return; 127 | } 128 | if (WalkQuestion.isTernary(stream, parent)) { 129 | walkStatementContinue(stream, parent); 130 | return; 131 | } 132 | wantMore = true; 133 | default: 134 | wantMore = false; 135 | } 136 | var newChild:TokenTree = stream.consumeToken(); 137 | parent.addChild(newChild); 138 | stream.applyTempStore(newChild); 139 | walkTrailingComment(stream, newChild); 140 | 141 | if (wantMore) walkStatementWithoutSemicolon(stream, newChild); 142 | walkStatementContinue(stream, newChild); 143 | walkTrailingComment(stream, newChild); 144 | } 145 | 146 | public static function walkTrailingComment(stream:TokenStream, parent:TokenTree) { 147 | if (!stream.hasMore()) { 148 | return; 149 | } 150 | switch (stream.token()) { 151 | case CommentLine(_): 152 | var currentPos:Int = stream.getStreamIndex(); 153 | var commentTok:TokenTree = stream.consumeToken(); 154 | if (!stream.tokenForMatch().match(Kwd(KwdElse))) { 155 | stream.rewindTo(currentPos); 156 | return; 157 | } 158 | parent.addChild(commentTok); 159 | default: 160 | } 161 | } 162 | 163 | public static function walkStatementContinue(stream:TokenStream, parent:TokenTree) { 164 | if (!stream.hasMore()) return; 165 | switch (stream.token()) { 166 | case Dot | QuestionDot: 167 | walkStatementWithoutSemicolon(stream, parent); 168 | case DblDot: 169 | walkDblDot(stream, parent); 170 | case Semicolon: 171 | return; 172 | case Arrow: 173 | walkStatementWithoutSemicolon(stream, parent); 174 | case Binop(OpBoolAnd), Binop(OpBoolOr): 175 | walkOpBool(stream, parent); 176 | case Binop(OpAdd), Binop(OpSub): 177 | walkOpAdd(stream, parent); 178 | case Binop(OpGt): 179 | var ltParent:TokenTree = parent; 180 | while (true) { 181 | switch (ltParent.tok) { 182 | case Root: break; 183 | case Dot | DblDot | Comma | Arrow | POpen | Const(_) | Dollar(_) | BkOpen | BrOpen | Binop(OpGt): ltParent = ltParent.parent; 184 | case Binop(OpLt): return; 185 | default: break; 186 | } 187 | } 188 | walkStatementWithoutSemicolon(stream, parent); 189 | case Binop(_): 190 | walkStatementWithoutSemicolon(stream, parent); 191 | case Const(CIdent("is")): 192 | walkStatementWithoutSemicolon(stream, parent); 193 | case Unop(_): 194 | if (parent.isCIdentOrCString()) { 195 | walkStatementWithoutSemicolon(stream, parent); 196 | } 197 | case Question: 198 | WalkQuestion.walkQuestion(stream, parent); 199 | case BkOpen: 200 | walkStatementWithoutSemicolon(stream, parent); 201 | case POpen: 202 | walkStatementWithoutSemicolon(stream, parent); 203 | case CommentLine(_), Comment(_): 204 | var nextTokDef:Null = stream.peekNonCommentToken(); 205 | if (nextTokDef == null) { 206 | return; 207 | } 208 | switch (nextTokDef) { 209 | case Dot, DblDot, Binop(_), Unop(_), Question: 210 | WalkComment.walkComment(stream, parent); 211 | walkStatementContinue(stream, parent); 212 | default: 213 | } 214 | case Spread: 215 | walkStatementWithoutSemicolon(stream, parent); 216 | default: 217 | } 218 | } 219 | 220 | static function walkKeyword(stream:TokenStream, parent:TokenTree):Bool { 221 | switch (stream.token()) { 222 | case Kwd(KwdVar): 223 | WalkVar.walkVar(stream, parent); 224 | case Kwd(KwdFinal): 225 | WalkFinal.walkFinal(stream, parent); 226 | case Kwd(KwdInline): 227 | return walkInline(stream, parent); 228 | case Kwd(KwdStatic) | Kwd(KwdPublic) | Kwd(KwdPrivate): 229 | stream.addToTempStore(stream.consumeToken()); 230 | return false; 231 | case Kwd(KwdNew): 232 | if (parent.tok.match(Dot)) { 233 | var newChild:TokenTree = stream.consumeToken(); 234 | parent.addChild(newChild); 235 | walkStatementContinue(stream, newChild); 236 | } 237 | else { 238 | WalkNew.walkNew(stream, parent); 239 | } 240 | case Kwd(KwdFor): 241 | WalkFor.walkFor(stream, parent); 242 | case Kwd(KwdFunction): 243 | WalkFunction.walkFunction(stream, parent); 244 | case Kwd(KwdClass): 245 | WalkClass.walkClass(stream, parent); 246 | case Kwd(KwdMacro), Kwd(KwdReturn): 247 | return true; 248 | case Kwd(KwdSwitch): 249 | WalkSwitch.walkSwitch(stream, parent); 250 | case Kwd(KwdCase): 251 | return false; 252 | case Kwd(KwdDefault): 253 | // switch or property 254 | if (parent.tok.match(BrOpen)) return false; 255 | return true; 256 | case Kwd(KwdIf): 257 | WalkIf.walkIf(stream, parent); 258 | case Kwd(KwdTry): 259 | WalkTry.walkTry(stream, parent); 260 | case Kwd(KwdDo): 261 | WalkDoWhile.walkDoWhile(stream, parent); 262 | case Kwd(KwdWhile): 263 | if (!parent.tok.match(BrOpen) && parent.parent.tok.match(Kwd(KwdDo))) { 264 | return false; 265 | } 266 | WalkWhile.walkWhile(stream, parent); 267 | case Kwd(KwdNull), Kwd(KwdTrue), Kwd(KwdFalse): 268 | var newChild:TokenTree = stream.consumeToken(); 269 | parent.addChild(newChild); 270 | if (!stream.hasMore()) { 271 | return false; 272 | } 273 | switch (stream.token()) { 274 | case Binop(OpBoolAnd), Binop(OpBoolOr): walkOpBool(stream, newChild); 275 | case Question: WalkQuestion.walkQuestion(stream, newChild); 276 | case Binop(_): walkStatementWithoutSemicolon(stream, newChild); 277 | default: 278 | } 279 | return false; 280 | case Kwd(KwdCast): 281 | var newChild:TokenTree = stream.consumeToken(); 282 | parent.addChild(newChild); 283 | walkStatementWithoutSemicolon(stream, newChild); 284 | return false; 285 | case Kwd(KwdThis): 286 | var newChild:TokenTree = stream.consumeToken(); 287 | parent.addChild(newChild); 288 | walkStatementContinue(stream, newChild); 289 | return false; 290 | default: 291 | return true; 292 | } 293 | return false; 294 | } 295 | 296 | static function walkInline(stream:TokenStream, parent:TokenTree):Bool { 297 | stream.addToTempStore(stream.consumeToken()); 298 | return switch (stream.token()) { 299 | case Const(_) | Kwd(KwdNew): true; 300 | case Kwd(KwdFunction): 301 | WalkFunction.walkFunction(stream, parent); 302 | false; 303 | default: false; 304 | } 305 | } 306 | 307 | public static function walkDblDot(stream:TokenStream, parent:TokenTree) { 308 | var question:Null = findQuestionParent(stream, parent); 309 | if (question != null) { 310 | return; 311 | } 312 | var dblDotTok:TokenTree = stream.consumeToken(); 313 | parent.addChild(dblDotTok); 314 | if (parent.isCIdentOrCString() && parent.parent.tok.match(BrOpen)) { 315 | walkStatementWithoutSemicolon(stream, dblDotTok); 316 | return; 317 | } 318 | if (stream.tokenForMatch().match(Kwd(KwdNew))) { 319 | WalkNew.walkNew(stream, dblDotTok); 320 | return; 321 | } 322 | if (!walkKeyword(stream, dblDotTok)) return; 323 | WalkTypeNameDef.walkTypeNameDef(stream, dblDotTok); 324 | if (stream.tokenForMatch().match(Binop(OpAssign))) { 325 | walkStatementWithoutSemicolon(stream, parent); 326 | } 327 | if (stream.tokenForMatch().match(Arrow)) { 328 | walkStatementWithoutSemicolon(stream, parent); 329 | } 330 | } 331 | 332 | static function walkPOpen(stream:TokenStream, parent:TokenTree) { 333 | var pOpen:TokenTree = WalkPOpen.walkPOpen(stream, parent); 334 | if (parent.tok == Root) { 335 | return; 336 | } 337 | if (parent.isCIdent()) { 338 | walkStatementContinue(stream, parent); 339 | } 340 | else { 341 | switch (parent.tok) { 342 | case Kwd(KwdIf) | Kwd(KwdSwitch) | Kwd(KwdFor) | Kwd(KwdWhile): 343 | switch (stream.token()) { 344 | case Binop(OpSub): return; 345 | case Binop(_): 346 | case Dot: 347 | default: return; 348 | } 349 | default: 350 | } 351 | walkStatementContinue(stream, pOpen); 352 | } 353 | } 354 | 355 | static function findQuestionParent(stream:TokenStream, token:TokenTree):Null { 356 | var parent:Null = token; 357 | while (parent != null && parent.tok != Root) { 358 | switch (parent.tok) { 359 | case Question: 360 | if (WalkQuestion.isTernary(stream, parent)) return parent; 361 | return null; 362 | case Comma: 363 | return null; 364 | case BrOpen: 365 | if (!TokenTreeAccessHelper.access(parent).firstOf(BrClose).exists()) { 366 | return null; 367 | } 368 | case POpen: 369 | if (!TokenTreeAccessHelper.access(parent).firstOf(PClose).exists()) { 370 | return null; 371 | } 372 | case Kwd(KwdReturn): 373 | return parent; 374 | case Kwd(KwdCase): 375 | return parent; 376 | case Kwd(KwdMacro): 377 | if (parent.index + 1 == stream.getStreamIndex()) return null; 378 | parent = findQuestionParent(stream, parent.parent); 379 | if (parent == null) { 380 | return null; 381 | } 382 | switch (parent.tok) { 383 | case Kwd(KwdCase) | Kwd(KwdDefault) | Question: return parent; 384 | default: return null; 385 | } 386 | return null; 387 | case Kwd(KwdDefault): 388 | return parent; 389 | case Binop(_): 390 | return parent; 391 | case DblDot: 392 | var type:ColonType = TokenTreeCheckUtils.determineColonType(parent); 393 | switch (type) { 394 | case SwitchCase | At: return null; 395 | case TypeHint | TypeCheck | Ternary | ObjectLiteral | Unknown: 396 | } 397 | default: 398 | } 399 | parent = parent.parent; 400 | } 401 | return null; 402 | } 403 | 404 | static function walkStatementContinueAfterSharp(stream:TokenStream, parent:TokenTree) { 405 | switch (stream.token()) { 406 | case Kwd(KwdCase), Kwd(KwdDefault): 407 | var lastChild:Null = parent.getLastChild(); 408 | if (lastChild == null) { 409 | lastChild = parent; 410 | } 411 | WalkSwitch.walkSwitchCases(stream, lastChild); 412 | default: 413 | } 414 | } 415 | 416 | static function walkOpBool(stream:TokenStream, token:TokenTree) { 417 | var parent = token.parent; 418 | while (parent.tok != Root) { 419 | switch (parent.tok) { 420 | case Binop(OpAssign), Binop(OpAssignOp(_)): 421 | break; 422 | case Binop(OpBoolAnd), Binop(OpBoolOr): 423 | token = parent.parent; 424 | break; 425 | case POpen: 426 | if (token.tok.match(POpen)) { 427 | token = parent; 428 | } 429 | break; 430 | case Kwd(KwdReturn), Kwd(KwdUntyped), Kwd(KwdIf), Kwd(KwdWhile), Kwd(KwdThrow): 431 | break; 432 | case Kwd(KwdFunction), Arrow, Question: 433 | break; 434 | case Kwd(KwdSwitch), Kwd(KwdCase), Kwd(KwdDefault): 435 | break; 436 | case DblDot: 437 | token = parent; 438 | break; 439 | default: 440 | token = parent; 441 | parent = parent.parent; 442 | } 443 | } 444 | walkStatementWithoutSemicolon(stream, token); 445 | } 446 | 447 | static function walkOpAdd(stream:TokenStream, token:TokenTree) { 448 | var parent = token.parent; 449 | while (parent.tok != Root) { 450 | switch (parent.tok) { 451 | case Binop(OpAssign), Binop(OpAssignOp(_)): 452 | break; 453 | case IntInterval(_), BkOpen, BrOpen: 454 | break; 455 | case Binop(OpMult), Binop(OpDiv): 456 | token = parent; 457 | parent = parent.parent; 458 | case Binop(OpAdd), Binop(OpSub): 459 | token = parent.parent; 460 | break; 461 | case Binop(_): 462 | break; 463 | case POpen: 464 | var pClose:Null = parent.access().firstOf(PClose).token; 465 | if (pClose == null) { 466 | token = parent; 467 | break; 468 | } 469 | token = parent; 470 | parent = parent.parent; 471 | case Kwd(KwdReturn), Kwd(KwdUntyped), Kwd(KwdIf), Kwd(KwdWhile), Kwd(KwdThrow): 472 | break; 473 | case Kwd(KwdFunction), Arrow, Question: 474 | break; 475 | case DblDot: 476 | break; 477 | default: 478 | token = parent; 479 | parent = parent.parent; 480 | } 481 | } 482 | walkStatementWithoutSemicolon(stream, token); 483 | } 484 | 485 | static function walkDollarStatement(stream:TokenStream, parent:TokenTree) { 486 | var dollarTok:TokenTree = stream.consumeToken(); 487 | parent.addChild(dollarTok); 488 | switch (stream.token()) { 489 | case POpen | BrOpen | BkOpen | Dot | Binop(_) | Const(CIdent("is")): 490 | WalkBlock.walkBlock(stream, dollarTok); 491 | default: 492 | } 493 | } 494 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkSwitch.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkSwitch { 4 | /** 5 | * Kwd(KwdSwitch) 6 | * |- POpen 7 | * | |- expression 8 | * | |- PClose 9 | * |- BrOpen 10 | * |- Kwd(KwdCase) 11 | * | |- expression 12 | * | |- DblDot 13 | * | |- statement 14 | * | |- statement 15 | * |- Kwd(KwdCase) 16 | * | |- expression 17 | * | |- DblDot 18 | * | |- BrOpen 19 | * | |- statement 20 | * | |- BrClose 21 | * |- Kwd(KwdDefault) 22 | * |- BrClose 23 | * 24 | */ 25 | public static function walkSwitch(stream:TokenStream, parent:TokenTree) { 26 | var switchTok:TokenTree = stream.consumeTokenDef(Kwd(KwdSwitch)); 27 | parent.addChild(switchTok); 28 | stream.applyTempStore(switchTok); 29 | WalkComment.walkComment(stream, switchTok); 30 | WalkStatement.walkStatement(stream, switchTok); 31 | WalkComment.walkComment(stream, switchTok); 32 | switch (stream.token()) { 33 | case Sharp(_): 34 | WalkSharp.walkSharp(stream, parent, WalkSwitch.walkSwitchCases); 35 | default: 36 | } 37 | if (stream.tokenForMatch().match(BrOpen)) { 38 | var brOpen:TokenTree = stream.consumeToken(); 39 | switchTok.addChild(brOpen); 40 | WalkSwitch.walkSwitchCases(stream, brOpen); 41 | brOpen.addChild(stream.consumeTokenDef(BrClose)); 42 | } 43 | } 44 | 45 | public static function walkSwitchCases(stream:TokenStream, parent:TokenTree) { 46 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 47 | while (progress.streamHasChanged()) { 48 | switch (stream.token()) { 49 | case BrClose: 50 | break; 51 | case Kwd(KwdCase), Kwd(KwdDefault): 52 | WalkSwitch.walkCase(stream, parent); 53 | case Sharp(_): 54 | WalkSharp.walkSharp(stream, parent, WalkSwitch.walkSwitchCases); 55 | case Comment(_), CommentLine(_): 56 | WalkComment.walkComment(stream, parent); 57 | default: 58 | WalkStatement.walkStatement(stream, parent); 59 | } 60 | } 61 | } 62 | 63 | /** 64 | * Kwd(KwdCase) | Kwd(KwdDefault) 65 | * |- expression 66 | * |- DblDot 67 | * |- statement 68 | * |- statement 69 | * 70 | * Kwd(KwdCase) | Kwd(KwdDefault) 71 | * |- expression 72 | * |- DblDot 73 | * |- BrOpen 74 | * |- statement 75 | * |- BrClose 76 | * 77 | */ 78 | public static function walkCase(stream:TokenStream, parent:TokenTree) { 79 | WalkComment.walkComment(stream, parent); 80 | var caseTok:TokenTree = stream.consumeToken(); 81 | parent.addChild(caseTok); 82 | WalkSwitch.walkCaseExpr(stream, caseTok); 83 | var dblDot:TokenTree = stream.consumeTokenDef(DblDot); 84 | caseTok.addChild(dblDot); 85 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 86 | while (progress.streamHasChanged()) { 87 | switch (stream.token()) { 88 | case Kwd(KwdCase), Kwd(KwdDefault), BrClose: 89 | return; 90 | case BrOpen: 91 | WalkBlock.walkBlock(stream, dblDot); 92 | case Comment(_), CommentLine(_): 93 | switch (stream.peekNonCommentToken()) { 94 | case Kwd(KwdCase) | Kwd(KwdDefault) if (stream.getStreamIndex() > dblDot.index + 1): return; 95 | default: 96 | } 97 | WalkComment.walkComment(stream, dblDot); 98 | case Sharp(_): 99 | WalkSharp.walkSharp(stream, parent, WalkSwitch.walkSwitchCases); 100 | relocateSharpTree(parent, dblDot); 101 | default: 102 | WalkStatement.walkStatement(stream, dblDot); 103 | } 104 | } 105 | } 106 | 107 | static function relocateSharpTree(parent:TokenTree, dblDot:TokenTree) { 108 | /* 109 | * relocate sharp subtree from: 110 | * |- BrOpen 111 | * |- Kwd(KwdCase) 112 | * | |- expression 113 | * | |- DblDot 114 | * | |- statement 115 | * |- Sharp(If) 116 | * | |- condition 117 | * | |- statement (if not a new case) 118 | * to: 119 | * |- Kwd(KwdCase) 120 | * | |- expression 121 | * | |- DblDot 122 | * | |- statement 123 | * | |- Sharp(If) 124 | * | |- condition 125 | * | |- statement 126 | */ 127 | var sharp:TokenTree = parent.getLastChild(); 128 | if (sharp.children.length < 2) return; 129 | var body:TokenTree = sharp.children[1]; 130 | if (body.tok.match(Kwd(KwdCase))) return; 131 | parent.children.pop(); 132 | dblDot.addChild(sharp); 133 | } 134 | 135 | static function walkCaseExpr(stream:TokenStream, parent:TokenTree) { 136 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 137 | while (progress.streamHasChanged()) { 138 | switch (stream.token()) { 139 | case Comma: 140 | var comma:TokenTree = stream.consumeToken(); 141 | var child:TokenTree = parent.getLastChild(); 142 | if (child == null) child = parent; 143 | child.addChild(comma); 144 | case Semicolon, BrClose, BkClose, PClose, DblDot: 145 | return; 146 | case Kwd(KwdVar): 147 | var varTok:TokenTree = stream.consumeToken(); 148 | parent.addChild(varTok); 149 | WalkStatement.walkStatement(stream, varTok); 150 | default: 151 | WalkStatement.walkStatement(stream, parent); 152 | } 153 | } 154 | } 155 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkTry.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkTry { 4 | /** 5 | * Kwd(KwdTry) 6 | * |- BrOpen 7 | * | |- statement 8 | * | |- statement 9 | * | |- BrClose 10 | * |- Kwd(KwdCatch) 11 | * | |- BrOpen 12 | * | |- statement 13 | * | |- statement 14 | * | |- BrClose 15 | * |- Kwd(KwdCatch) 16 | * |- BrOpen 17 | * |- statement 18 | * |- statement 19 | * |- BrClose 20 | * 21 | */ 22 | public static function walkTry(stream:TokenStream, parent:TokenTree) { 23 | var tryTok:TokenTree = stream.consumeTokenDef(Kwd(KwdTry)); 24 | parent.addChild(tryTok); 25 | stream.applyTempStore(tryTok); 26 | WalkBlock.walkBlock(stream, tryTok); 27 | 28 | var currentPos:Int = stream.getStreamIndex(); 29 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 30 | var comments:Array = []; 31 | while (stream.hasMore() && progress.streamHasChanged()) { 32 | switch (stream.token()) { 33 | case Comment(_), CommentLine(_): 34 | comments.push(stream.consumeToken()); 35 | case Kwd(KwdCatch): 36 | for (comment in comments) { 37 | tryTok.addChild(comment); 38 | } 39 | comments = []; 40 | WalkTry.walkCatch(stream, tryTok); 41 | currentPos = stream.getStreamIndex(); 42 | default: 43 | stream.rewindTo(currentPos); 44 | return; 45 | } 46 | } 47 | } 48 | 49 | /** 50 | * Kwd(KwdCatch) 51 | * |- BrOpen 52 | * |- statement 53 | * |- statement 54 | * |- BrClose 55 | * 56 | */ 57 | static function walkCatch(stream:TokenStream, parent:TokenTree) { 58 | var catchTok:TokenTree = stream.consumeTokenDef(Kwd(KwdCatch)); 59 | parent.addChild(catchTok); 60 | WalkPOpen.walkPOpen(stream, catchTok); 61 | WalkComment.walkComment(stream, catchTok); 62 | WalkBlock.walkBlock(stream, catchTok); 63 | } 64 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkType.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkType { 4 | public static function walkType(stream:TokenStream, parent:TokenTree) { 5 | switch (stream.token()) { 6 | case Kwd(KwdClass): 7 | WalkClass.walkClass(stream, parent); 8 | case Kwd(KwdInterface): 9 | WalkInterface.walkInterface(stream, parent); 10 | case Kwd(KwdAbstract): 11 | WalkAbstract.walkAbstract(stream, parent); 12 | case Kwd(KwdTypedef): 13 | WalkTypedef.walkTypedef(stream, parent); 14 | case Kwd(KwdEnum): 15 | WalkEnum.walkEnum(stream, parent); 16 | default: 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkTypeNameDef.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkTypeNameDef { 4 | public static function walkTypeNameDef(stream:TokenStream, parent:TokenTree):TokenTree { 5 | WalkComment.walkComment(stream, parent); 6 | WalkAt.walkAts(stream); 7 | if (stream.tokenForMatch().match(Question)) { 8 | var questTok:TokenTree = stream.consumeTokenDef(Question); 9 | parent.addChild(questTok); 10 | parent = questTok; 11 | WalkComment.walkComment(stream, parent); 12 | } 13 | var name:Null; 14 | var bAdd:Bool = true; 15 | switch (stream.token()) { 16 | case BrOpen: 17 | WalkTypedefBody.walkTypedefBody(stream, parent); 18 | return parent.getFirstChild(); 19 | case BkOpen: 20 | WalkArrayAccess.walkArrayAccess(stream, parent); 21 | return parent.getFirstChild(); 22 | case Kwd(KwdMacro), Kwd(KwdExtern), Kwd(KwdNew): 23 | name = stream.consumeToken(); 24 | case Const(_): 25 | name = stream.consumeConst(); 26 | case Dollar(_): 27 | name = stream.consumeToken(); 28 | case POpen: 29 | name = WalkPOpen.walkPOpen(stream, parent); 30 | if (stream.tokenForMatch().match(Question)) { 31 | WalkQuestion.walkQuestion(stream, name); 32 | } 33 | bAdd = false; 34 | case Sharp(_): 35 | WalkSharp.walkSharp(stream, parent, WalkStatement.walkStatement); 36 | if (!stream.hasMore()) return parent.getFirstChild(); 37 | switch (stream.token()) { 38 | case Const(_): name = stream.consumeConst(); 39 | default: return parent.getFirstChild(); 40 | } 41 | default: 42 | name = stream.consumeToken(); 43 | } 44 | stream.applyTempStore(name); 45 | if (bAdd) parent.addChild(name); 46 | walkTypeNameDefContinue(stream, name); 47 | return name; 48 | } 49 | 50 | static function walkTypeNameDefContinue(stream:TokenStream, parent:TokenTree) { 51 | walkTypeNameDefComment(stream, parent); 52 | if (stream.tokenForMatch().match(Dot)) { 53 | var dot:TokenTree = stream.consumeToken(); 54 | parent.addChild(dot); 55 | WalkTypeNameDef.walkTypeNameDef(stream, dot); 56 | return; 57 | } 58 | if (stream.tokenForMatch().match(Binop(OpLt))) WalkLtGt.walkLtGt(stream, parent); 59 | if (stream.tokenForMatch().match(Arrow)) { 60 | var arrow:TokenTree = stream.consumeToken(); 61 | parent.addChild(arrow); 62 | WalkTypeNameDef.walkTypeNameDef(stream, arrow); 63 | return; 64 | } 65 | if (stream.tokenForMatch().match(BkOpen)) WalkArrayAccess.walkArrayAccess(stream, parent); 66 | walkTypeNameDefComment(stream, parent); 67 | } 68 | 69 | static function walkTypeNameDefComment(stream:TokenStream, parent:TokenTree) { 70 | var currentPos:Int = stream.getStreamIndex(); 71 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 72 | var comments:Array = []; 73 | while (stream.hasMore() && progress.streamHasChanged()) { 74 | switch (stream.token()) { 75 | case Comment(_), CommentLine(_): 76 | comments.push(stream.consumeToken()); 77 | case DblDot, Comma, Semicolon, Dot, BrOpen, BkOpen, POpen, Binop(_): 78 | for (comment in comments) { 79 | parent.addChild(comment); 80 | } 81 | return; 82 | default: 83 | stream.rewindTo(currentPos); 84 | return; 85 | } 86 | } 87 | } 88 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkTypedef.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkTypedef { 4 | public static function walkTypedef(stream:TokenStream, parent:TokenTree) { 5 | var typeTok:TokenTree = stream.consumeToken(); 6 | parent.addChild(typeTok); 7 | var name:TokenTree = WalkTypeNameDef.walkTypeNameDef(stream, typeTok); 8 | // add all comments, annotations 9 | stream.applyTempStore(name); 10 | if (stream.tokenForMatch().match(Binop(OpAssign))) { 11 | var assign:TokenTree = stream.consumeTokenDef(Binop(OpAssign)); 12 | name.addChild(assign); 13 | name = assign; 14 | } 15 | WalkTypedefBody.walkTypedefBody(stream, name); 16 | } 17 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkTypedefBody.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkTypedefBody { 4 | public static function walkTypedefBody(stream:TokenStream, parent:TokenTree) { 5 | if (stream.tokenForMatch().match(BrOpen)) { 6 | var openTok:TokenTree = stream.consumeToken(); 7 | parent.addChild(openTok); 8 | walkTypedefCurlyBody(stream, openTok); 9 | openTok.addChild(stream.consumeTokenDef(BrClose)); 10 | } 11 | else { 12 | walkTypedefAlias(stream, parent); 13 | } 14 | if (stream.tokenForMatch().match(Binop(OpAnd))) { 15 | var and:TokenTree = stream.consumeTokenDef(Binop(OpAnd)); 16 | parent.getLastChild().addChild(and); 17 | walkTypedefBody(stream, parent); 18 | } 19 | if (stream.tokenForMatch().match(Arrow)) { 20 | WalkStatement.walkStatement(stream, parent); 21 | } 22 | } 23 | 24 | public static function walkTypedefCurlyBody(stream:TokenStream, openTok:TokenTree) { 25 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 26 | while (progress.streamHasChanged()) { 27 | switch (stream.token()) { 28 | case At: 29 | stream.addToTempStore(WalkAt.walkAt(stream)); 30 | case BrClose: 31 | break; 32 | case Sharp(_): 33 | WalkSharp.walkSharp(stream, openTok, WalkTypedefBody.walkTypedefCurlyBody); 34 | case Binop(OpGt): 35 | walkStructureExtension(stream, openTok); 36 | case Comment(_), CommentLine(_): 37 | if (stream.hasTempStore()) { 38 | stream.consumeToTempStore(); 39 | } 40 | else { 41 | WalkComment.walkComment(stream, openTok); 42 | } 43 | case Kwd(KwdFunction): 44 | WalkFunction.walkFunction(stream, openTok); 45 | case Kwd(KwdVar): 46 | WalkVar.walkVar(stream, openTok); 47 | case Kwd(KwdPublic), Kwd(KwdPrivate), Kwd(KwdStatic), Kwd(KwdInline), Kwd(KwdMacro), Kwd(KwdOverride), Kwd(KwdDynamic), Kwd(KwdExtern): 48 | stream.consumeToTempStore(); 49 | case Kwd(KwdFinal): 50 | WalkFinal.walkFinal(stream, openTok); 51 | default: 52 | WalkFieldDef.walkFieldDef(stream, openTok); 53 | } 54 | } 55 | var tempStore:Array = stream.getTempStore(); 56 | if (tempStore.length > 0) { 57 | switch (TokenStream.MODE) { 58 | case Relaxed: 59 | stream.applyTempStore(openTok); 60 | case Strict: 61 | throw "invalid token tree structure - found:" + '$tempStore'; 62 | } 63 | } 64 | } 65 | 66 | public static function walkTypedefAlias(stream:TokenStream, parent:TokenTree) { 67 | var newParent:TokenTree; 68 | if (stream.tokenForMatch().match(POpen)) { 69 | newParent = WalkPOpen.walkPOpen(stream, parent); 70 | } 71 | else { 72 | newParent = WalkTypeNameDef.walkTypeNameDef(stream, parent); 73 | } 74 | if (stream.tokenForMatch().match(Arrow)) { 75 | var arrowTok:TokenTree = stream.consumeToken(); 76 | newParent.addChild(arrowTok); 77 | walkTypedefAlias(stream, arrowTok); 78 | } 79 | if (stream.tokenForMatch().match(Semicolon)) { 80 | newParent.addChild(stream.consumeToken()); 81 | } 82 | } 83 | 84 | static function walkStructureExtension(stream:TokenStream, parent:TokenTree) { 85 | var gt:TokenTree = stream.consumeTokenDef(Binop(OpGt)); 86 | parent.addChild(gt); 87 | var name:TokenTree = WalkTypeNameDef.walkTypeNameDef(stream, parent); 88 | gt.addChild(name); 89 | if (stream.tokenForMatch().match(Comma)) { 90 | name.addChild(stream.consumeToken()); 91 | } 92 | } 93 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkVar.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkVar { 4 | public static function walkVar(stream:TokenStream, parent:TokenTree) { 5 | var name:Null = null; 6 | var varTok:Null = stream.consumeTokenDef(Kwd(KwdVar)); 7 | parent.addChild(varTok); 8 | WalkComment.walkComment(stream, parent); 9 | var progress:TokenStreamProgress = new TokenStreamProgress(stream); 10 | while (progress.streamHasChanged()) { 11 | if (stream.tokenForMatch().match(Kwd(KwdVar))) { 12 | return; 13 | } 14 | WalkComment.walkComment(stream, parent); 15 | switch (stream.token()) { 16 | case At: 17 | WalkAt.walkAts(stream); 18 | default: 19 | } 20 | WalkComment.walkComment(stream, parent); 21 | var nameParent:TokenTree = varTok; 22 | if (stream.tokenForMatch().match(Question)) { 23 | nameParent = stream.consumeToken(); 24 | varTok.addChild(nameParent); 25 | } 26 | name = stream.consumeConstIdent(); 27 | nameParent.addChild(name); 28 | stream.applyTempStore(name); 29 | WalkComment.walkComment(stream, name); 30 | if (stream.tokenForMatch().match(POpen)) { 31 | WalkPOpen.walkPOpen(stream, name); 32 | } 33 | if (stream.tokenForMatch().match(DblDot)) { 34 | var dblDot:TokenTree = stream.consumeToken(); 35 | name.addChild(dblDot); 36 | WalkTypedefBody.walkTypedefAlias(stream, dblDot); 37 | } 38 | if (stream.tokenForMatch().match(Binop(OpAssign))) { 39 | WalkStatement.walkStatement(stream, name); 40 | } 41 | if (stream.tokenForMatch().match(Comma)) { 42 | var comma:TokenTree = stream.consumeToken(); 43 | name.addChild(comma); 44 | continue; 45 | } 46 | break; 47 | } 48 | if (stream.tokenForMatch().match(Semicolon)) { 49 | name.addChild(stream.consumeToken()); 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /src/tokentree/walk/WalkWhile.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | class WalkWhile { 4 | /** 5 | * Kwd(KwdWhile) 6 | * |- POpen 7 | * | |- expression 8 | * | |- PClose 9 | * |- BrOpen 10 | * |- statement 11 | * |- statement 12 | * |- BrClose 13 | * 14 | */ 15 | public static function walkWhile(stream:TokenStream, parent:TokenTree) { 16 | var whileTok:TokenTree = stream.consumeTokenDef(Kwd(KwdWhile)); 17 | parent.addChild(whileTok); 18 | stream.applyTempStore(whileTok); 19 | WalkComment.walkComment(stream, whileTok); 20 | WalkStatement.walkStatement(stream, whileTok); 21 | WalkComment.walkComment(stream, whileTok); 22 | WalkBlock.walkBlock(stream, whileTok); 23 | } 24 | } -------------------------------------------------------------------------------- /test.hxml: -------------------------------------------------------------------------------- 1 | -cp src 2 | -cp test 3 | 4 | -lib haxeparser 5 | -lib utest 6 | -lib instrument 7 | 8 | -D unittest 9 | -x TestMain 10 | 11 | -D message-reporting=pretty 12 | 13 | -D coverage-console-summary-reporter 14 | #-D coverage-console-file-summary-reporter 15 | -D coverage-console-package-summary-reporter 16 | -D coverage-lcov-reporter 17 | -D coverage-codecov-reporter 18 | 19 | --macro instrument.Instrumentation.coverage(['tokentree'], ['src'], []) 20 | # --macro instrument.Instrumentation.profiling(['tokentree'], ['src'], []) 21 | # -D profiler-console-detail-reporter 22 | # -D debug_instrumentation 23 | 24 | # --macro nullSafety('tokentree') 25 | -------------------------------------------------------------------------------- /test/TestMain.hx: -------------------------------------------------------------------------------- 1 | import tokentree.TokenStreamTest; 2 | import tokentree.TokenTreeBuilderParsingTest; 3 | import tokentree.TokenTreeBuilderTest; 4 | import tokentree.utils.FieldUtilsTest; 5 | import tokentree.utils.TokenTreeCheckUtilsTest; 6 | import tokentree.verify.VerifyTokenTreeTest; 7 | import tokentree.walk.WalkFileTest; 8 | import tokentree.walk.WalkIfTest; 9 | import tokentree.walk.WalkWhileTest; 10 | import utest.Runner; 11 | import utest.ui.Report; 12 | 13 | using StringTools; 14 | 15 | class TestMain { 16 | static function main():Void { 17 | var tests:Array = [ 18 | new FieldUtilsTest(), 19 | new TokenTreeCheckUtilsTest(), 20 | new TokenStreamTest(), 21 | new TokenTreeBuilderTest(), 22 | new TokenTreeBuilderParsingTest(), 23 | new VerifyTokenTreeTest(), 24 | new WalkIfTest(), 25 | new WalkWhileTest(), 26 | new WalkFileTest() 27 | ]; 28 | var runner:Runner = new Runner(); 29 | 30 | #if instrument 31 | runner.onComplete.add(_ -> { 32 | instrument.coverage.Coverage.endCoverage(); 33 | }); 34 | #end 35 | 36 | Report.create(runner); 37 | for (test in tests) runner.addCase(test); 38 | runner.run(); 39 | } 40 | } -------------------------------------------------------------------------------- /test/TokenTestMain.hx: -------------------------------------------------------------------------------- 1 | import byte.ByteData; 2 | // import sys.io.File; 3 | import haxeparser.HaxeLexer; 4 | import haxeparser.Data.Token; 5 | import tokentree.TokenTree; 6 | import tokentree.TokenTreeBuilder; 7 | import tokentree.TokenStream; 8 | 9 | class TokenTestMain { 10 | public static inline var TOKENTREE_BUILDER_TEST:String = " 11 | 12 | abstract class Abstract { 13 | abstract dynamic function abstractFunction():Void; 14 | } 15 | @:structInit abstract class Abstract {} 16 | abstract interface I {} 17 | "; 18 | 19 | public static function main() { 20 | // var code = File.getContent("../../haxe-languageserver/cases/foldingRange/Input.hx"); 21 | var code = TOKENTREE_BUILDER_TEST; 22 | var tokens:Array = []; 23 | var bytes:ByteData = ByteData.ofString(code); 24 | var lexer = new HaxeLexer(bytes, "TokenStream"); 25 | var t:Token = lexer.token(HaxeLexer.tok); 26 | 27 | while (t.tok != Eof) { 28 | tokens.push(t); 29 | t = lexer.token(haxeparser.HaxeLexer.tok); 30 | } 31 | 32 | TokenStream.MODE = Relaxed; 33 | 34 | for (tok in tokens) Sys.println('${tok.tok} ${tok.pos}'); 35 | var root:TokenTree = TokenTreeBuilder.buildTokenTree(tokens, bytes); 36 | Sys.print(root.printTokenTree()); 37 | } 38 | } -------------------------------------------------------------------------------- /test/import.hx: -------------------------------------------------------------------------------- 1 | import utest.Assert; 2 | import utest.ITest; -------------------------------------------------------------------------------- /test/tokentree/TestTokenTreeBuilder.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | import byte.ByteData; 4 | import haxeparser.Data.Token; 5 | import haxeparser.HaxeLexer; 6 | import tokentree.TokenTreeBuilder.TokenTreeEntryPoint; 7 | import tokentree.walk.WalkFile; 8 | import tokentree.walk.WalkStatement; 9 | 10 | class TestTokenTreeBuilder extends TokenTreeBuilder { 11 | public static function parseCode(code:String):TestTokenTreeBuilder { 12 | var builder:TestTokenTreeBuilder = new TestTokenTreeBuilder(code, new TokenTree(Root, "", null, -1)); 13 | WalkFile.walkFile(builder.stream, builder.root); 14 | return builder; 15 | } 16 | 17 | public static function parseExpressionCode(code:String):TestTokenTreeBuilder { 18 | var builder:TestTokenTreeBuilder = new TestTokenTreeBuilder(code, new TokenTree(Root, "", null, -1)); 19 | WalkStatement.walkStatement(builder.stream, builder.root); 20 | return builder; 21 | } 22 | 23 | public static function makeTokenStream(code:String):TokenStream { 24 | var tokens:Array = []; 25 | var lexer = new HaxeLexer(byte.ByteData.ofString(code), "TokenStream"); 26 | var t:Token = lexer.token(HaxeLexer.tok); 27 | while (t.tok != Eof) { 28 | tokens.push(t); 29 | t = lexer.token(haxeparser.HaxeLexer.tok); 30 | } 31 | return new TokenStream(tokens, ByteData.ofString(code)); 32 | } 33 | 34 | var stream:TokenStream; 35 | 36 | public var root:TokenTree; 37 | 38 | public function new(code:String, root:TokenTree) { 39 | stream = makeTokenStream(code); 40 | this.root = root; 41 | } 42 | 43 | public function setTokenStream(newStream:TokenStream) { 44 | this.stream = newStream; 45 | } 46 | 47 | public function getTokenStream():TokenStream { 48 | return stream; 49 | } 50 | 51 | public function isStreamEmpty():Bool { 52 | return !stream.hasMore(); 53 | } 54 | 55 | public function buildTokenTree(level:TokenTreeEntryPoint) { 56 | TokenTreeBuilder.buildTokenTreeFromStream(stream, level); 57 | } 58 | } -------------------------------------------------------------------------------- /test/tokentree/TokenStreamTest.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | import haxe.PosInfos; 4 | 5 | class TokenStreamTest implements ITest { 6 | public function new() {} 7 | 8 | @Test 9 | public function testIsTypedParam() { 10 | function codeIsTyeParam(code:String):Bool { 11 | var stream:TokenStream = TestTokenTreeBuilder.makeTokenStream(code); 12 | return stream.isTypedParam(); 13 | } 14 | Assert.isFalse(codeIsTyeParam("")); 15 | Assert.isFalse(codeIsTyeParam("1")); 16 | Assert.isFalse(codeIsTyeParam("< 1")); 17 | Assert.isFalse(codeIsTyeParam("< (y >> 1);")); 18 | Assert.isFalse(codeIsTyeParam("< arr[1];")); 19 | Assert.isFalse(codeIsTyeParam("< arr[i << 1];")); 20 | Assert.isFalse(codeIsTyeParam("< arr[i]];")); 21 | Assert.isFalse(codeIsTyeParam("< arr[i]};")); 22 | 23 | Assert.isTrue(codeIsTyeParam("")); 24 | Assert.isTrue(codeIsTyeParam("")); 25 | Assert.isTrue(codeIsTyeParam(">")); 26 | Assert.isTrue(codeIsTyeParam(">")); 27 | } 28 | } -------------------------------------------------------------------------------- /test/tokentree/TokenTreeBuilderTest.hx: -------------------------------------------------------------------------------- 1 | package tokentree; 2 | 3 | import haxe.PosInfos; 4 | import tokentree.TokenTreeBuilder.TokenTreeEntryPoint; 5 | import tokentree.walk.WalkAt; 6 | import tokentree.walk.WalkIf; 7 | import tokentree.walk.WalkPackageImport; 8 | 9 | class TokenTreeBuilderTest implements ITest { 10 | public function new() {} 11 | 12 | function assertTokenEquals(testCase:TokenTreeBuilderTests, actual:String, ?pos:PosInfos) { 13 | Assert.equals((testCase : String), actual, pos); 14 | } 15 | 16 | @Test 17 | public function testImports() { 18 | var root:TokenTree = new TokenTree(null, "", null, -1); 19 | var builder:TestTokenTreeBuilder = new TestTokenTreeBuilder(TokenTreeBuilderTests.IMPORT, root); 20 | var stream:TokenStream = builder.getTokenStream(); 21 | WalkPackageImport.walkPackageImport(stream, root); 22 | WalkPackageImport.walkPackageImport(stream, root); 23 | WalkPackageImport.walkPackageImport(stream, root); 24 | WalkPackageImport.walkPackageImport(stream, root); 25 | WalkPackageImport.walkPackageImport(stream, root); 26 | checkStreamEmpty(builder); 27 | 28 | assertTokenEquals(IMPORT_GOLD, treeToString(root)); 29 | } 30 | 31 | @Test 32 | public function testAt() { 33 | var root:TokenTree = new TokenTree(null, "", null, -1); 34 | var builder:TestTokenTreeBuilder = new TestTokenTreeBuilder(TokenTreeBuilderTests.AT_ANNOTATION, root); 35 | var stream:TokenStream = builder.getTokenStream(); 36 | root.addChild(WalkAt.walkAt(stream)); 37 | root.addChild(WalkAt.walkAt(stream)); 38 | root.addChild(WalkAt.walkAt(stream)); 39 | root.addChild(WalkAt.walkAt(stream)); 40 | builder.getTokenStream().consumeToken(); // remove comment line 41 | checkStreamEmpty(builder); 42 | 43 | assertTokenEquals(AT_ANNOTATION_GOLD, treeToString(root)); 44 | } 45 | 46 | @Test 47 | public function testIf() { 48 | var root:TokenTree = new TokenTree(null, "", null, -1); 49 | var builder:TestTokenTreeBuilder = new TestTokenTreeBuilder(TokenTreeBuilderTests.IF, root); 50 | var stream:TokenStream = builder.getTokenStream(); 51 | WalkIf.walkIf(stream, root); 52 | WalkIf.walkIf(stream, root); 53 | WalkIf.walkIf(stream, root); 54 | WalkIf.walkIf(stream, root); 55 | checkStreamEmpty(builder); 56 | 57 | assertTokenEquals(IF_GOLD, treeToString(root)); 58 | } 59 | 60 | // public static function buildTokenTree(tokens:Array, bytes:ByteData, entryPoint:TokenTreeEntryPoint):TokenTree { 61 | 62 | @Test 63 | public function testEntryPoint() { 64 | parseNoException(ENTRY_POINT_FILE, TypeLevel); 65 | parseNoException(ENTRY_POINT_FUNCTION, FieldLevel); 66 | parseNoException(ENTRY_POINT_FUNCTION_WITH_BODY, FieldLevel); 67 | parseNoException(ENTRY_POINT_EXPRESSION, ExpressionLevel); 68 | parseNoException(ENTRY_POINT_EXPRESSION_SWITCH, ExpressionLevel); 69 | 70 | parseWithException(ENTRY_POINT_FILE, ExpressionLevel); 71 | parseWithException(ENTRY_POINT_EXPRESSION_SWITCH, FieldLevel); 72 | 73 | parseNoException(ENTRY_POINT_TYPEDEF, TypeHintLevel); 74 | parseNoException(ENTRY_POINT_MAP, TypeHintLevel); 75 | parseWithException(ENTRY_POINT_FILE, TypeHintLevel); 76 | } 77 | 78 | function parseWithException(code:String, level:TokenTreeEntryPoint, ?pos:PosInfos) { 79 | try { 80 | parseNoException(code, level); 81 | } 82 | catch (e:Any) { 83 | Assert.isTrue(true, pos); 84 | return; 85 | } 86 | Assert.fail("should throw an exception!", pos); 87 | } 88 | 89 | function parseNoException(code:String, level:TokenTreeEntryPoint) { 90 | TokenStream.MODE = Strict; 91 | var root:TokenTree = new TokenTree(Root, "", null, -1); 92 | var testBuilder:TestTokenTreeBuilder = new TestTokenTreeBuilder(code, root); 93 | testBuilder.buildTokenTree(level); 94 | } 95 | 96 | function checkStreamEmpty(builder:TestTokenTreeBuilder) { 97 | Assert.isTrue(builder.isStreamEmpty()); 98 | } 99 | 100 | function treeToString(token:TokenTree, prefix:String = ""):String { 101 | var buf:StringBuf = new StringBuf(); 102 | var tokDef:TokenTreeDef = token.tok; 103 | if (tokDef != null) buf.add('$prefix${tokDef}\n'); 104 | if (token.hasChildren()) { 105 | @:nullSafety(Off) 106 | for (child in token.children) { 107 | buf.add(treeToString(child, prefix + " ")); 108 | } 109 | } 110 | return buf.toString(); 111 | } 112 | } 113 | 114 | // @formatter:off 115 | enum abstract TokenTreeBuilderTests(String) to String { 116 | var IMPORT = " 117 | package checkstyle.checks; 118 | 119 | import haxeparser.*; 120 | import checkstyle.TokenTree; 121 | import checkstyle.TokenStream; 122 | import checkstyle.TokenTreeBuilder; 123 | "; 124 | var IMPORT_GOLD = 125 | " Kwd(KwdPackage)\n" + 126 | " Const(CIdent(checkstyle))\n" + 127 | " Dot\n" + 128 | " Const(CIdent(checks))\n" + 129 | " Semicolon\n" + 130 | " Kwd(KwdImport)\n" + 131 | " Const(CIdent(haxeparser))\n" + 132 | " Dot\n" + 133 | " Binop(OpMult)\n" + 134 | " Semicolon\n" + 135 | " Kwd(KwdImport)\n" + 136 | " Const(CIdent(checkstyle))\n" + 137 | " Dot\n" + 138 | " Const(CIdent(TokenTree))\n" + 139 | " Semicolon\n" + 140 | " Kwd(KwdImport)\n" + 141 | " Const(CIdent(checkstyle))\n" + 142 | " Dot\n" + 143 | " Const(CIdent(TokenStream))\n" + 144 | " Semicolon\n" + 145 | " Kwd(KwdImport)\n" + 146 | " Const(CIdent(checkstyle))\n" + 147 | " Dot\n" + 148 | " Const(CIdent(TokenTreeBuilder))\n" + 149 | " Semicolon\n"; 150 | 151 | var AT_ANNOTATION = ' 152 | @SuppressWarnings("checkstyle:MagicNumber") 153 | @SuppressWarnings(["checkstyle:MagicNumber", "checkstyle:AvoidStarImport"]) 154 | @:from 155 | @Before 156 | // EOF 157 | '; 158 | var AT_ANNOTATION_GOLD = 159 | " At\n" + 160 | " Const(CIdent(SuppressWarnings))\n" + 161 | " POpen\n" + 162 | " Const(CString(checkstyle:MagicNumber,DoubleQuotes))\n" + 163 | " PClose\n" + 164 | " At\n" + 165 | " Const(CIdent(SuppressWarnings))\n" + 166 | " POpen\n" + 167 | " BkOpen\n" + 168 | " Const(CString(checkstyle:MagicNumber,DoubleQuotes))\n" + 169 | " Comma\n" + 170 | " Const(CString(checkstyle:AvoidStarImport,DoubleQuotes))\n" + 171 | " BkClose\n" + 172 | " PClose\n" + 173 | " At\n" + 174 | " DblDot\n" + 175 | " Const(CIdent(from))\n" + 176 | " At\n" + 177 | " Const(CIdent(Before))\n"; 178 | 179 | var IF = ' 180 | if (tokDef != null) return; 181 | if (tokDef != null) 182 | return; 183 | else 184 | throw "error"; 185 | if (token.hasChildren()) { 186 | return token.children; 187 | } 188 | if (token.hasChildren()) { 189 | return token.children; 190 | } 191 | else { 192 | return []; 193 | } 194 | '; 195 | var IF_GOLD = 196 | " Kwd(KwdIf)\n" + 197 | " POpen\n" + 198 | " Const(CIdent(tokDef))\n" + 199 | " Binop(OpNotEq)\n" + 200 | " Kwd(KwdNull)\n" + 201 | " PClose\n" + 202 | " Kwd(KwdReturn)\n" + 203 | " Semicolon\n" + 204 | " Kwd(KwdIf)\n" + 205 | " POpen\n" + 206 | " Const(CIdent(tokDef))\n" + 207 | " Binop(OpNotEq)\n" + 208 | " Kwd(KwdNull)\n" + 209 | " PClose\n" + 210 | " Kwd(KwdReturn)\n" + 211 | " Semicolon\n" + 212 | " Kwd(KwdElse)\n" + 213 | " Kwd(KwdThrow)\n" + 214 | " Const(CString(error,DoubleQuotes))\n" + 215 | " Semicolon\n" + 216 | " Kwd(KwdIf)\n" + 217 | " POpen\n" + 218 | " Const(CIdent(token))\n" + 219 | " Dot\n" + 220 | " Const(CIdent(hasChildren))\n" + 221 | " POpen\n" + 222 | " PClose\n" + 223 | " PClose\n" + 224 | " BrOpen\n" + 225 | " Kwd(KwdReturn)\n" + 226 | " Const(CIdent(token))\n" + 227 | " Dot\n" + 228 | " Const(CIdent(children))\n" + 229 | " Semicolon\n" + 230 | " BrClose\n" + 231 | " Kwd(KwdIf)\n" + 232 | " POpen\n" + 233 | " Const(CIdent(token))\n" + 234 | " Dot\n" + 235 | " Const(CIdent(hasChildren))\n" + 236 | " POpen\n" + 237 | " PClose\n" + 238 | " PClose\n" + 239 | " BrOpen\n" + 240 | " Kwd(KwdReturn)\n" + 241 | " Const(CIdent(token))\n" + 242 | " Dot\n" + 243 | " Const(CIdent(children))\n" + 244 | " Semicolon\n" + 245 | " BrClose\n" + 246 | " Kwd(KwdElse)\n" + 247 | " BrOpen\n" + 248 | " Kwd(KwdReturn)\n" + 249 | " BkOpen\n" + 250 | " BkClose\n" + 251 | " Semicolon\n" + 252 | " BrClose\n"; 253 | 254 | var ENTRY_POINT_FILE = " 255 | package checkstyle.checks; 256 | 257 | import haxeparser.*; 258 | import checkstyle.TokenTree; 259 | import checkstyle.TokenStream; 260 | import checkstyle.TokenTreeBuilder; 261 | 262 | @:access(haxe.Json) 263 | class Main { 264 | @:access(test.Main) 265 | public function main() { 266 | } 267 | } 268 | "; 269 | 270 | var ENTRY_POINT_FUNCTION = " 271 | @:access(test.Main) 272 | public static inline function main (param:Int, param2:String):T; 273 | "; 274 | 275 | var ENTRY_POINT_FUNCTION_WITH_BODY = " 276 | @:access(test.Main) 277 | public static inline function main (param:Int, param2:String):T { 278 | trace (param2); 279 | } 280 | "; 281 | 282 | var ENTRY_POINT_EXPRESSION = " 283 | trace (param2); 284 | "; 285 | 286 | var ENTRY_POINT_EXPRESSION_SWITCH = " 287 | switch (entryPoint) { 288 | case FileLevel: 289 | WalkFile.walkFile(stream, root); 290 | case FunctionLevel: 291 | WalkClass.walkClassBody(stream, root); 292 | case ExpressionLevel: 293 | WalkStatement.walkStatement(stream, root); 294 | } 295 | "; 296 | 297 | var ENTRY_POINT_TYPEDEF = " 298 | { 299 | var conditions:Array; 300 | var type:WrappingType; 301 | var location:WrappingLocation; 302 | var additionalIndent:Int; 303 | } 304 | "; 305 | 306 | var ENTRY_POINT_MAP = " 307 | Map 308 | "; 309 | } -------------------------------------------------------------------------------- /test/tokentree/utils/FieldUtilsTest.hx: -------------------------------------------------------------------------------- 1 | package tokentree.utils; 2 | 3 | import haxe.PosInfos; 4 | import tokentree.TokenTree.FilterResult; 5 | import tokentree.utils.FieldUtils.TokenFieldType; 6 | 7 | class FieldUtilsTest implements ITest { 8 | public function new() {} 9 | 10 | @Test 11 | @:nullSafety(Off) 12 | public function testProperties() { 13 | var root:Null = assertCodeParses(FieldUtilsTests.PROPERTIES); 14 | Assert.isFalse(root.inserted); 15 | 16 | var allBr:Array = root.filterCallback(function(token:TokenTree, depth:Int):FilterResult { 17 | return switch (token.tok) { 18 | case BrOpen: FoundGoDeeper; 19 | default: GoDeeper; 20 | } 21 | }); 22 | Assert.equals(1, allBr.length); 23 | Assert.equals(7 + 1, allBr[0].children.length); 24 | 25 | checkFieldType(Var("_haxelibRepo", Private, true, false, false, false), FieldUtils.getFieldType(allBr[0].children[0], Private)); 26 | checkFieldType(Var("_haxelibRepo", Public, true, false, false, false), FieldUtils.getFieldType(allBr[0].children[0], Public)); 27 | 28 | checkFieldType(Prop("haxelibRepo", Private, true, Get, Never), FieldUtils.getFieldType(allBr[0].children[1], Private)); 29 | checkFieldType(Prop("haxelibRepo", Public, true, Get, Never), FieldUtils.getFieldType(allBr[0].children[1], Public)); 30 | 31 | checkFieldType(Var("_haxelibRepo", Public, false, false, false, false), FieldUtils.getFieldType(allBr[0].children[2], Private)); 32 | checkFieldType(Var("_haxelibRepo", Public, false, false, false, false), FieldUtils.getFieldType(allBr[0].children[2], Public)); 33 | 34 | checkFieldType(Prop("haxelibRepo", Public, false, Get, Never), FieldUtils.getFieldType(allBr[0].children[3], Private)); 35 | checkFieldType(Prop("haxelibRepo", Public, false, Get, Never), FieldUtils.getFieldType(allBr[0].children[3], Public)); 36 | 37 | checkFieldType(Var("_haxelibRepo", Public, true, true, false, false), FieldUtils.getFieldType(allBr[0].children[4], Private)); 38 | checkFieldType(Var("_haxelibRepo", Public, true, true, false, false), FieldUtils.getFieldType(allBr[0].children[4], Public)); 39 | 40 | checkFieldType(Function("main", Public, true, false, false, false, false), FieldUtils.getFieldType(allBr[0].children[5], Private)); 41 | checkFieldType(Function("main", Public, true, false, false, false, false), FieldUtils.getFieldType(allBr[0].children[5], Public)); 42 | 43 | checkFieldType(Function("main", Private, false, false, true, false, false), FieldUtils.getFieldType(allBr[0].children[6], Private)); 44 | checkFieldType(Function("main", Public, false, false, true, false, false), FieldUtils.getFieldType(allBr[0].children[6], Public)); 45 | } 46 | 47 | function checkFieldType(expected:TokenFieldType, actual:TokenFieldType, ?pos:PosInfos) { 48 | Assert.isTrue(Type.enumEq(expected, actual), pos); 49 | } 50 | 51 | public function assertCodeParses(code:String, ?pos:PosInfos):TokenTree { 52 | var builder:Null = null; 53 | try { 54 | builder = TestTokenTreeBuilder.parseCode(code); 55 | Assert.isTrue(builder.isStreamEmpty(), pos); 56 | return builder.root; 57 | } 58 | catch (e:Any) { 59 | Assert.fail("code should not throw execption " + e, pos); 60 | } 61 | return new TokenTree(null, "", null, 0, true); 62 | } 63 | } 64 | 65 | enum abstract FieldUtilsTests(String) to String { 66 | var PROPERTIES = " 67 | class Main { 68 | static var _haxelibRepo:Null; 69 | static var haxelibRepo(get, never):Null; 70 | public var _haxelibRepo:Null; 71 | public var haxelibRepo(get, never):Null; 72 | public static inline var _haxelibRepo:Null; 73 | 74 | public static function main(); 75 | override function main(); 76 | } 77 | "; 78 | } -------------------------------------------------------------------------------- /test/tokentree/verify/IVerifyTokenTree.hx: -------------------------------------------------------------------------------- 1 | package tokentree.verify; 2 | 3 | import haxe.PosInfos; 4 | import haxeparser.Data; 5 | 6 | interface IVerifyTokenTree { 7 | function filter(tok:TokenTreeDef, ?pos:PosInfos):IVerifyTokenTree; 8 | function childs(?pos:PosInfos):IVerifyTokenTree; 9 | function first(?pos:PosInfos):IVerifyTokenTree; 10 | function last(?pos:PosInfos):IVerifyTokenTree; 11 | function at(index:Int, ?pos:PosInfos):IVerifyTokenTree; 12 | function count(count:Int, ?pos:PosInfos):IVerifyTokenTree; 13 | function noChilds(?pos:PosInfos):IVerifyTokenTree; 14 | function oneChild(?pos:PosInfos):IVerifyTokenTree; 15 | function childFirst(?pos:PosInfos):IVerifyTokenTree; 16 | function childLast(?pos:PosInfos):IVerifyTokenTree; 17 | function childAt(index:Int, ?pos:PosInfos):IVerifyTokenTree; 18 | function childCount(num:Int, ?pos:PosInfos):IVerifyTokenTree; 19 | function childCountAtLeast(count:Int, ?pos:PosInfos):IVerifyTokenTree; 20 | function matches(tok:TokenTreeDef, ?pos:PosInfos):IVerifyTokenTree; 21 | function isComment(?pos:PosInfos):IVerifyTokenTree; 22 | function isEmpty(?pos:PosInfos):Bool; 23 | } -------------------------------------------------------------------------------- /test/tokentree/verify/VerifyTokenTree.hx: -------------------------------------------------------------------------------- 1 | package tokentree.verify; 2 | 3 | import haxe.PosInfos; 4 | import haxeparser.Data; 5 | import tokentree.TokenTree; 6 | 7 | class VerifyTokenTree implements IVerifyTokenTree { 8 | var token:TokenTree; 9 | 10 | public function new(token:Null) { 11 | Assert.notNull(token); 12 | this.token = @:nullSafety(Off) token; 13 | } 14 | 15 | public function filter(tok:TokenTreeDef, ?pos:PosInfos):IVerifyTokenTree { 16 | var list:Array = []; 17 | if (token.children != null) { 18 | for (child in token.children) { 19 | if (child.matches(tok)) list.push(new VerifyTokenTree(child)); 20 | } 21 | } 22 | return new VerifyTokenTreeList(list); 23 | } 24 | 25 | public function childs(?pos:PosInfos):IVerifyTokenTree { 26 | var list:Array = []; 27 | if (token.children != null) { 28 | for (child in token.children) list.push(new VerifyTokenTree(child)); 29 | } 30 | return new VerifyTokenTreeList(list); 31 | } 32 | 33 | public function first(?pos:PosInfos):IVerifyTokenTree { 34 | return childFirst(pos); 35 | } 36 | 37 | public function last(?pos:PosInfos):IVerifyTokenTree { 38 | return childLast(pos); 39 | } 40 | 41 | public function at(index:Int, ?pos:PosInfos):IVerifyTokenTree { 42 | return childAt(index, pos); 43 | } 44 | 45 | public function count(num:Int, ?pos:PosInfos):IVerifyTokenTree { 46 | Assert.equals(num, 1, pos); 47 | return this; 48 | } 49 | 50 | public function noChilds(?pos:PosInfos):IVerifyTokenTree { 51 | if (token.children == null) return this; 52 | Assert.equals(0, token.children.length, pos); 53 | return this; 54 | } 55 | 56 | public function oneChild(?pos:PosInfos):IVerifyTokenTree { 57 | if (token.children == null) Assert.fail("no childs", pos); 58 | @:nullSafety(Off) 59 | Assert.equals(1, token.children.length, pos); 60 | return this; 61 | } 62 | 63 | public function childFirst(?pos:PosInfos):IVerifyTokenTree { 64 | childCountAtLeast(1, pos); 65 | return new VerifyTokenTree(token.getFirstChild()); 66 | } 67 | 68 | public function childLast(?pos:PosInfos):IVerifyTokenTree { 69 | childCountAtLeast(1, pos); 70 | return new VerifyTokenTree(token.getLastChild()); 71 | } 72 | 73 | public function childAt(index:Int, ?pos:PosInfos):IVerifyTokenTree { 74 | childCountAtLeast(index + 1, pos); 75 | return new VerifyTokenTree(@:nullSafety(Off) token.children[index]); 76 | } 77 | 78 | public function childCount(count:Int, ?pos:PosInfos):IVerifyTokenTree { 79 | if (token.children == null) { 80 | if (count != 0) Assert.fail('${token.tok} has no childs', pos); 81 | else return this; 82 | } 83 | @:nullSafety(Off) 84 | Assert.equals(count, token.children.length, '[${token.tok}] child count [${token.children.length}] was not equal to expected [$count]', pos); 85 | return this; 86 | } 87 | 88 | public function childCountAtLeast(count:Int, ?pos:PosInfos):IVerifyTokenTree { 89 | if ((token.children == null) && (count != 0)) Assert.fail('${token.tok} has no childs', pos); 90 | @:nullSafety(Off) 91 | Assert.isTrue(token.children.length >= count, pos); 92 | return this; 93 | } 94 | 95 | public function matches(tok:TokenTreeDef, ?pos:PosInfos):IVerifyTokenTree { 96 | Assert.isTrue(token.matches(tok), '$tok != ${token.tok}', pos); 97 | return this; 98 | } 99 | 100 | public function isComment(?pos:PosInfos):IVerifyTokenTree { 101 | switch (token.tok) { 102 | case Comment(_), CommentLine(_): 103 | return this; 104 | default: 105 | Assert.fail('${token.tok} is not a comment', pos); 106 | } 107 | return this; 108 | } 109 | 110 | public function isEmpty(?pos:PosInfos):Bool { 111 | return token == null; 112 | } 113 | } -------------------------------------------------------------------------------- /test/tokentree/verify/VerifyTokenTreeBase.hx: -------------------------------------------------------------------------------- 1 | package tokentree.verify; 2 | 3 | import haxe.PosInfos; 4 | import tokentree.TokenTree; 5 | import tokentree.walk.WalkFile; 6 | 7 | class VerifyTokenTreeBase { 8 | function buildTokenTree(content:String, ?pos:PosInfos):IVerifyTokenTree { 9 | var root:TokenTree = new TokenTree(null, "", null, -1); 10 | var builder:TestTokenTreeBuilder = new TestTokenTreeBuilder(content, root); 11 | walkStream(builder.getTokenStream(), root); 12 | Assert.isTrue(builder.isStreamEmpty(), "there are still unused tokens in stream!!", pos); 13 | return new VerifyTokenTree(root); 14 | } 15 | 16 | function walkStream(stream:TokenStream, parent:TokenTree) { 17 | WalkFile.walkFile(stream, parent); 18 | } 19 | } -------------------------------------------------------------------------------- /test/tokentree/verify/VerifyTokenTreeList.hx: -------------------------------------------------------------------------------- 1 | package tokentree.verify; 2 | 3 | import haxe.PosInfos; 4 | import haxeparser.Data; 5 | 6 | class VerifyTokenTreeList implements IVerifyTokenTree { 7 | var tokens:Array; 8 | 9 | public function new(tokens:Array) { 10 | Assert.notNull(tokens); 11 | this.tokens = tokens; 12 | } 13 | 14 | public function filter(tok:TokenTreeDef, ?pos:PosInfos):IVerifyTokenTree { 15 | var list:Array = []; 16 | for (token in tokens) { 17 | var child:IVerifyTokenTree = token.filter(tok, pos); 18 | if (!child.isEmpty()) list.push(child); 19 | } 20 | return new VerifyTokenTreeList(list); 21 | } 22 | 23 | public function childs(?pos:PosInfos):IVerifyTokenTree { 24 | var list:Array = []; 25 | for (token in tokens) { 26 | var childList:VerifyTokenTreeList = cast token.childs(pos); 27 | for (i in 0...childList.tokens.length) { 28 | list.push(childList.tokens[i]); 29 | } 30 | } 31 | return new VerifyTokenTreeList(list); 32 | } 33 | 34 | public function first(?pos:PosInfos):IVerifyTokenTree { 35 | Assert.isTrue(tokens.length >= 1, pos); 36 | return tokens[0]; 37 | } 38 | 39 | public function last(?pos:PosInfos):IVerifyTokenTree { 40 | Assert.isTrue(tokens.length >= 2, pos); 41 | return tokens[tokens.length - 1]; 42 | } 43 | 44 | public function at(index:Int, ?pos:PosInfos):IVerifyTokenTree { 45 | Assert.isTrue(tokens.length > index, pos); 46 | return tokens[index]; 47 | } 48 | 49 | public function count(num:Int, ?pos:PosInfos):IVerifyTokenTree { 50 | Assert.equals(num, tokens.length, pos); 51 | return this; 52 | } 53 | 54 | public function noChilds(?pos:PosInfos):IVerifyTokenTree { 55 | var list:Array = []; 56 | for (token in tokens) list.push(token.noChilds(pos)); 57 | return new VerifyTokenTreeList(list); 58 | } 59 | 60 | public function oneChild(?pos:PosInfos):IVerifyTokenTree { 61 | var list:Array = []; 62 | for (token in tokens) list.push(token.oneChild()); 63 | return new VerifyTokenTreeList(list); 64 | } 65 | 66 | public function childFirst(?pos:PosInfos):IVerifyTokenTree { 67 | var list:Array = []; 68 | for (token in tokens) list.push(token.childFirst(pos)); 69 | return new VerifyTokenTreeList(list); 70 | } 71 | 72 | public function childLast(?pos:PosInfos):IVerifyTokenTree { 73 | var list:Array = []; 74 | for (token in tokens) list.push(token.childLast(pos)); 75 | return new VerifyTokenTreeList(list); 76 | } 77 | 78 | public function childAt(index:Int, ?pos:PosInfos):IVerifyTokenTree { 79 | var list:Array = []; 80 | for (token in tokens) list.push(token.childAt(index, pos)); 81 | return new VerifyTokenTreeList(list); 82 | } 83 | 84 | public function childCount(count:Int, ?pos:PosInfos):IVerifyTokenTree { 85 | var list:Array = []; 86 | for (token in tokens) list.push(token.childCount(count, pos)); 87 | return new VerifyTokenTreeList(list); 88 | } 89 | 90 | public function childCountAtLeast(count:Int, ?pos:PosInfos):IVerifyTokenTree { 91 | var list:Array = []; 92 | for (token in tokens) list.push(token.childCountAtLeast(count, pos)); 93 | return new VerifyTokenTreeList(list); 94 | } 95 | 96 | public function matches(tok:TokenTreeDef, ?pos:PosInfos):IVerifyTokenTree { 97 | var list:Array = []; 98 | for (token in tokens) list.push(token.matches(tok, pos)); 99 | return new VerifyTokenTreeList(list); 100 | } 101 | 102 | public function isComment(?pos:PosInfos):IVerifyTokenTree { 103 | var list:Array = []; 104 | for (token in tokens) list.push(token.isComment(pos)); 105 | return new VerifyTokenTreeList(list); 106 | } 107 | 108 | public function isEmpty(?pos:PosInfos):Bool { 109 | return ((tokens == null) || (tokens.length <= 0)); 110 | } 111 | } -------------------------------------------------------------------------------- /test/tokentree/verify/VerifyTokenTreeTest.hx: -------------------------------------------------------------------------------- 1 | package tokentree.verify; 2 | 3 | import haxeparser.Data; 4 | import tokentree.TokenTreeBuilderParsingTest.TokenTreeBuilderParsingTests; 5 | 6 | class VerifyTokenTreeTest extends VerifyTokenTreeBase implements ITest { 7 | public function new() {} 8 | 9 | @Test 10 | public function testImport() { 11 | var root:IVerifyTokenTree = buildTokenTree(VerifyTokenTreeTests.IMPORT); 12 | 13 | root.first().matches(Kwd(KwdPackage)).count(1).filter(Const(CIdent("checkstyle"))).count(1).childs().matches(Dot).childs().childs().matches(Semicolon); 14 | 15 | root.filter(Kwd(KwdImport)).count(4).filter(Const(CIdent("checkstyle"))).count(3).childs().matches(Dot).childs().childs().count(3).matches(Semicolon); 16 | root.filter(Kwd(KwdImport)).count(4).filter(Const(CIdent("haxeparser"))).count(1).childs().matches(Dot).childs().childs().count(1).matches(Semicolon); 17 | 18 | root.last().matches(Kwd(KwdUsing)).filter(Const(CIdent("checkstyle"))).count(1).childs().matches(Dot).childs().childs().count(1).matches(Semicolon); 19 | } 20 | 21 | @Test 22 | public function testObjectDecl() { 23 | var root:IVerifyTokenTree = buildTokenTree(TokenTreeBuilderParsingTests.BLOCK_OBJECT_DECL_SAMPLES_ISSUE_396_1); 24 | 25 | // class 26 | var block:IVerifyTokenTree = root.childFirst().matches(Kwd(KwdClass)).count(1).childFirst().matches(Const(CIdent("Test"))).childFirst().matches(BrOpen) 27 | .childCount(2); 28 | block.childLast().matches(BrClose).noChilds(); 29 | 30 | // function test() 31 | var func:IVerifyTokenTree = block.childFirst().matches(Kwd(KwdFunction)).childFirst().matches(Const(CIdent("test"))).childCount(2); 32 | func.childFirst().matches(POpen).childFirst().matches(PClose).noChilds(); 33 | 34 | // function body 35 | block = func.last().matches(BrOpen).childCount(3); 36 | block.childFirst().matches(CommentLine("fails with: bad token Comma != BrClose")).noChilds(); 37 | block.last().matches(BrClose).noChilds(); 38 | 39 | // var test = switch a 40 | var eq:IVerifyTokenTree = block.childAt(1).matches(Kwd(KwdVar)).childs().count(1).matches(Const(CIdent("test"))).childs().count(1) 41 | .matches(Binop(OpAssign)); 42 | var sw:IVerifyTokenTree = eq.childs().count(1).matches(Kwd(KwdSwitch)).childCount(2); 43 | sw.childFirst().matches(Const(CIdent("a"))).noChilds(); 44 | 45 | // switch body 46 | block = sw.childLast().matches(BrOpen).childCount(3); 47 | 48 | // case 3 49 | var cas:IVerifyTokenTree = block.childFirst().matches(Kwd(KwdCase)).childCount(2); 50 | cas.childFirst().matches(Const(CInt("3"))).noChilds(); 51 | cas = cas.childLast().matches(DblDot).childs().count(1); 52 | cas = cas.first().matches(BrOpen).childCount(4).childs(); 53 | cas.first().matches(Const(CIdent("a"))).childs().count(2).first().matches(DblDot).childs().count(1).matches(Const(CInt("1"))).noChilds(); 54 | cas.first().matches(Const(CIdent("a"))).childs().count(2).last().matches(Comma).noChilds(); 55 | cas.at(1).matches(Const(CIdent("b"))).childs().count(1).matches(DblDot).childs().count(1).matches(Const(CInt("2"))).noChilds(); 56 | cas.at(2).matches(BrClose).noChilds(); 57 | cas.last().matches(Semicolon).noChilds(); 58 | 59 | // default 60 | cas = block.childAt(1).matches(Kwd(KwdDefault)).childs().matches(DblDot).childs().count(1); 61 | cas = cas.first().matches(BrOpen).childCount(4).childs(); 62 | cas.first().matches(Const(CIdent("a"))).childs().count(2).first().matches(DblDot).childs().count(1).matches(Const(CInt("0"))).noChilds(); 63 | cas.first().matches(Const(CIdent("a"))).childs().count(2).last().matches(Comma).noChilds(); 64 | cas.at(1).matches(Const(CIdent("b"))).childs().count(1).matches(DblDot).childs().count(1).matches(Const(CInt("2"))).noChilds(); 65 | cas.at(2).matches(BrClose).noChilds(); 66 | cas.last().matches(Semicolon).noChilds(); 67 | 68 | block.childLast().matches(BrClose).noChilds(); 69 | } 70 | 71 | @Test 72 | public function testTypedefComments() { 73 | var root:IVerifyTokenTree = buildTokenTree(TokenTreeBuilderParsingTests.TYPEDEF_COMMENTS); 74 | 75 | // typedef CheckFile 76 | var type:IVerifyTokenTree = root.childFirst().matches(Kwd(KwdTypedef)).oneChild().childFirst().matches(Const(CIdent("CheckFile"))); 77 | var brOpen:IVerifyTokenTree = type.childFirst().matches(Binop(OpAssign)).oneChild().childFirst().matches(BrOpen).childCount(8); 78 | brOpen.childAt(0).matches(CommentLine(" °")); 79 | 80 | // var name:String; 81 | var v:IVerifyTokenTree = brOpen.childAt(1).matches(Kwd(KwdVar)).oneChild().childFirst().matches(Const(CIdent("name"))).oneChild(); 82 | v.childFirst().matches(DblDot).oneChild().childFirst().matches(Const(CIdent("String"))).oneChild().childFirst().matches(Semicolon).noChilds(); 83 | 84 | brOpen.childAt(2).matches(CommentLine(" öäü")).noChilds(); 85 | 86 | // var content:String; 87 | v = brOpen.childAt(3).matches(Kwd(KwdVar)).oneChild().childFirst().matches(Const(CIdent("content"))).oneChild(); 88 | v.childFirst().matches(DblDot).oneChild().childFirst().matches(Const(CIdent("String"))).oneChild().childFirst().matches(Semicolon).noChilds(); 89 | 90 | brOpen.childAt(4).matches(CommentLine(" €łµ")).noChilds(); 91 | 92 | // var index:Int; 93 | v = brOpen.childAt(5).matches(Kwd(KwdVar)).oneChild().childFirst().matches(Const(CIdent("index"))).oneChild(); 94 | v.childFirst().matches(DblDot).oneChild().childFirst().matches(Const(CIdent("Int"))).oneChild().childFirst().matches(Semicolon).noChilds(); 95 | 96 | brOpen.childAt(6).matches(CommentLine(" æ@ð")).noChilds(); 97 | brOpen.childLast().matches(BrClose).noChilds(); 98 | } 99 | 100 | @Test 101 | public function testTypedefComments2() { 102 | var root:IVerifyTokenTree = buildTokenTree(TokenTreeBuilderParsingTests.TYPEDEF_COMMENTS_2); 103 | 104 | // typedef CheckFile 105 | var type:IVerifyTokenTree = root.childFirst().matches(Kwd(KwdTypedef)).oneChild().childFirst().matches(Const(CIdent("CheckFile"))); 106 | var brOpen:IVerifyTokenTree = type.childFirst().matches(Binop(OpAssign)).oneChild().childFirst().matches(BrOpen).childCount(6); 107 | brOpen.childAt(0).matches(CommentLine(" °")); 108 | 109 | // var name:String; 110 | var v:IVerifyTokenTree = brOpen.childAt(1).matches(Kwd(KwdVar)).oneChild().childFirst().matches(Const(CIdent("name"))).oneChild(); 111 | v.childFirst().matches(DblDot).oneChild().childFirst().matches(Const(CIdent("String"))).oneChild().childFirst().matches(Semicolon).noChilds(); 112 | 113 | // var content:String; 114 | v = brOpen.childAt(2).matches(Kwd(KwdVar)).oneChild().childFirst().matches(Const(CIdent("content"))).oneChild(); 115 | v.childFirst().matches(DblDot).oneChild().childFirst().matches(Const(CIdent("String"))).oneChild().childFirst().matches(Semicolon).noChilds(); 116 | 117 | brOpen.childAt(3).matches(CommentLine(" €łµ")).noChilds(); 118 | 119 | // var index:Int; 120 | v = brOpen.childAt(4).matches(Kwd(KwdVar)).oneChild().childFirst().matches(Const(CIdent("index"))).oneChild(); 121 | v.childFirst().matches(DblDot).oneChild().childFirst().matches(Const(CIdent("Int"))).oneChild().childFirst().matches(Semicolon).noChilds(); 122 | 123 | brOpen.childLast().matches(BrClose).noChilds(); 124 | } 125 | } 126 | 127 | enum abstract VerifyTokenTreeTests(String) to String { 128 | var IMPORT = " 129 | package checkstyle.checks; 130 | 131 | import haxeparser.*; 132 | import checkstyle.TokenTree;, 133 | import checkstyle.TokenStream; 134 | import checkstyle.TokenTreeBuilder; 135 | using checkstyle.TokenTree; 136 | "; 137 | } -------------------------------------------------------------------------------- /test/tokentree/walk/WalkFileTest.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | import tokentree.TokenStream; 4 | import tokentree.TokenTree; 5 | import tokentree.verify.IVerifyTokenTree; 6 | import tokentree.verify.VerifyTokenTreeBase; 7 | 8 | class WalkFileTest extends VerifyTokenTreeBase implements ITest { 9 | public function new() {} 10 | 11 | @Test 12 | public function testEmpty() { 13 | var root:IVerifyTokenTree = buildTokenTree(""); 14 | root.childCount(0); 15 | } 16 | 17 | @Test 18 | public function testImportCompletion() { 19 | var root:IVerifyTokenTree = buildTokenTree("import MyModule."); 20 | root.oneChild().first().matches(Kwd(KwdImport)).oneChild().first().matches(Const(CIdent("MyModule"))).oneChild().first().matches(Dot).noChilds(); 21 | } 22 | 23 | override function walkStream(stream:TokenStream, parent:TokenTree) { 24 | WalkFile.walkFile(stream, parent); 25 | } 26 | } -------------------------------------------------------------------------------- /test/tokentree/walk/WalkIfTest.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | import tokentree.TokenStream; 4 | import tokentree.TokenTree; 5 | import tokentree.verify.IVerifyTokenTree; 6 | import tokentree.verify.VerifyTokenTreeBase; 7 | 8 | class WalkIfTest extends VerifyTokenTreeBase implements ITest { 9 | public function new() {} 10 | 11 | @Test 12 | public function testIfExpr() { 13 | var root:IVerifyTokenTree = buildTokenTree("if (test) {doSomething();}"); 14 | 15 | var ifTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdIf)).childCount(2); 16 | var ifExpr:IVerifyTokenTree = ifTok.first().matches(POpen).childCount(2); 17 | ifExpr.first().matches(Const(CIdent("test"))).noChilds(); 18 | ifExpr.last().matches(PClose).noChilds(); 19 | ifBodyTest(ifTok); 20 | } 21 | 22 | @Test 23 | public function testIfUnOpSub() { 24 | var root:IVerifyTokenTree = buildTokenTree("if (test) -1;"); 25 | 26 | var ifTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdIf)).childCount(2); 27 | var ifExpr:IVerifyTokenTree = ifTok.first().matches(POpen).childCount(2); 28 | ifExpr.first().matches(Const(CIdent("test"))).noChilds(); 29 | ifExpr.last().matches(PClose).noChilds(); 30 | ifTok.last().matches(Const(CInt("-1"))).oneChild().first().matches(Semicolon).noChilds(); 31 | } 32 | 33 | @Test 34 | public function testIfNull() { 35 | var root:IVerifyTokenTree = buildTokenTree("if (test) null;"); 36 | 37 | var ifTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdIf)).childCount(2); 38 | var ifExpr:IVerifyTokenTree = ifTok.first().matches(POpen).childCount(2); 39 | ifExpr.first().matches(Const(CIdent("test"))).noChilds(); 40 | ifExpr.last().matches(PClose).noChilds(); 41 | ifTok.last().matches(Kwd(KwdNull)).oneChild().first().matches(Semicolon).noChilds(); 42 | } 43 | 44 | @Test 45 | public function testIfExprAndExpr() { 46 | var root:IVerifyTokenTree = buildTokenTree("if (test && test2) {doSomething();}"); 47 | 48 | var ifTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdIf)).childCount(2); 49 | var ifExpr:IVerifyTokenTree = ifTok.first().matches(POpen).childCount(2); 50 | ifExpr.first().matches(Const(CIdent("test"))).oneChild().first().matches(Binop(OpBoolAnd)).oneChild().first().matches(Const(CIdent("test2"))) 51 | .noChilds(); 52 | ifExpr.last().matches(PClose).noChilds(); 53 | ifBodyTest(ifTok); 54 | } 55 | 56 | @Test 57 | public function testIfExprAndExpr2() { 58 | var root:IVerifyTokenTree = buildTokenTree("if (test) && (test2) {doSomething();}"); 59 | 60 | var ifTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdIf)).childCount(2); 61 | var ifExpr:IVerifyTokenTree = ifTok.first().matches(POpen).childCount(3); 62 | ifExpr.first().matches(Const(CIdent("test"))).noChilds(); 63 | ifExpr.childAt(1).matches(PClose).noChilds(); 64 | ifExpr = ifExpr.last().matches(Binop(OpBoolAnd)).oneChild().first().matches(POpen).childCount(2); 65 | ifExpr.first().matches(Const(CIdent("test2"))).noChilds(); 66 | ifExpr.last().matches(PClose).noChilds(); 67 | ifBodyTest(ifTok); 68 | } 69 | 70 | @Test 71 | public function testIfExprAndExpr3() { 72 | var root:IVerifyTokenTree = buildTokenTree("if test && test2 {doSomething();}"); 73 | 74 | var ifTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdIf)).childCount(2); 75 | ifTok.first().matches(Const(CIdent("test"))).oneChild().first().matches(Binop(OpBoolAnd)).oneChild().first().matches(Const(CIdent("test2"))).noChilds(); 76 | ifBodyTest(ifTok); 77 | } 78 | 79 | @Test 80 | public function testIfExprAndExpr4() { 81 | var root:IVerifyTokenTree = buildTokenTree("if test && test2 doSomething();"); 82 | 83 | var ifTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdIf)).childCount(2); 84 | ifTok.first().matches(Const(CIdent("test"))).oneChild().first().matches(Binop(OpBoolAnd)).oneChild().first().matches(Const(CIdent("test2"))).noChilds(); 85 | 86 | var exprCall:IVerifyTokenTree = ifTok.last().matches(Const(CIdent("doSomething"))).childCount(2); 87 | exprCall.first().matches(POpen).oneChild().first().matches(PClose).noChilds(); 88 | exprCall.last().matches(Semicolon).noChilds(); 89 | } 90 | 91 | function ifBodyTest(ifTok:IVerifyTokenTree) { 92 | var ifBody:IVerifyTokenTree = ifTok.last().matches(BrOpen).childCount(2); 93 | var exprCall:IVerifyTokenTree = ifBody.first().matches(Const(CIdent("doSomething"))).childCount(2); 94 | exprCall.first().matches(POpen).oneChild().first().matches(PClose).noChilds(); 95 | exprCall.last().matches(Semicolon).noChilds(); 96 | ifBody.last().matches(BrClose).noChilds(); 97 | } 98 | 99 | override function walkStream(stream:TokenStream, parent:TokenTree) { 100 | WalkIf.walkIf(stream, parent); 101 | } 102 | } -------------------------------------------------------------------------------- /test/tokentree/walk/WalkWhileTest.hx: -------------------------------------------------------------------------------- 1 | package tokentree.walk; 2 | 3 | import tokentree.TokenStream; 4 | import tokentree.TokenTree; 5 | import tokentree.verify.IVerifyTokenTree; 6 | import tokentree.verify.VerifyTokenTreeBase; 7 | 8 | class WalkWhileTest extends VerifyTokenTreeBase implements ITest { 9 | public function new() {} 10 | 11 | @Test 12 | public function testWhile() { 13 | var root:IVerifyTokenTree = buildTokenTree("while (test) {doSomething();}"); 14 | 15 | var whileTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdWhile)).childCount(2); 16 | var ifExpr:IVerifyTokenTree = whileTok.first().matches(POpen).childCount(2); 17 | ifExpr.first().matches(Const(CIdent("test"))).noChilds(); 18 | ifExpr.last().matches(PClose).noChilds(); 19 | whileBodyTest(whileTok); 20 | } 21 | 22 | @Test 23 | public function testWhileExprAndExpr() { 24 | var root:IVerifyTokenTree = buildTokenTree("while (test && test2) {doSomething();}"); 25 | 26 | var whileTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdWhile)).childCount(2); 27 | var ifExpr:IVerifyTokenTree = whileTok.first().matches(POpen).childCount(2); 28 | ifExpr.first().matches(Const(CIdent("test"))).oneChild().first().matches(Binop(OpBoolAnd)).oneChild().first().matches(Const(CIdent("test2"))) 29 | .noChilds(); 30 | ifExpr.last().matches(PClose).noChilds(); 31 | whileBodyTest(whileTok); 32 | } 33 | 34 | @Test 35 | public function testIfExprAndExpr2() { 36 | var root:IVerifyTokenTree = buildTokenTree("while (test) && (test2) {doSomething();}"); 37 | 38 | var whileTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdWhile)).childCount(2); 39 | var ifExpr:IVerifyTokenTree = whileTok.first().matches(POpen).childCount(3); 40 | ifExpr.first().matches(Const(CIdent("test"))).noChilds(); 41 | ifExpr.childAt(1).matches(PClose).noChilds(); 42 | ifExpr = ifExpr.last().matches(Binop(OpBoolAnd)).oneChild().first().matches(POpen).childCount(2); 43 | ifExpr.first().matches(Const(CIdent("test2"))).noChilds(); 44 | ifExpr.last().matches(PClose).noChilds(); 45 | whileBodyTest(whileTok); 46 | } 47 | 48 | @Test 49 | public function testIfExprAndExpr3() { 50 | var root:IVerifyTokenTree = buildTokenTree("while test && test2 {doSomething();}"); 51 | 52 | var whileTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdWhile)).childCount(2); 53 | whileTok.first().matches(Const(CIdent("test"))).oneChild().first().matches(Binop(OpBoolAnd)).oneChild().first().matches(Const(CIdent("test2"))) 54 | .noChilds(); 55 | whileBodyTest(whileTok); 56 | } 57 | 58 | @Test 59 | public function testIfExprAndExpr4() { 60 | var root:IVerifyTokenTree = buildTokenTree("while test && test2 doSomething();"); 61 | 62 | var whileTok:IVerifyTokenTree = root.oneChild().first().matches(Kwd(KwdWhile)).childCount(2); 63 | whileTok.first().matches(Const(CIdent("test"))).oneChild().first().matches(Binop(OpBoolAnd)).oneChild().first().matches(Const(CIdent("test2"))) 64 | .noChilds(); 65 | 66 | var exprCall:IVerifyTokenTree = whileTok.last().matches(Const(CIdent("doSomething"))).childCount(2); 67 | exprCall.first().matches(POpen).childCount(1).first().matches(PClose).noChilds(); 68 | exprCall.last().matches(Semicolon).noChilds(); 69 | } 70 | 71 | function whileBodyTest(whileTok:IVerifyTokenTree) { 72 | var ifBody:IVerifyTokenTree = whileTok.last().matches(BrOpen).childCount(2); 73 | var exprCall:IVerifyTokenTree = ifBody.first().matches(Const(CIdent("doSomething"))).childCount(2); 74 | exprCall.first().matches(POpen).childCount(1).first().matches(PClose).noChilds(); 75 | exprCall.last().matches(Semicolon).noChilds(); 76 | ifBody.last().matches(BrClose).noChilds(); 77 | } 78 | 79 | override function walkStream(stream:TokenStream, parent:TokenTree) { 80 | WalkWhile.walkWhile(stream, parent); 81 | } 82 | } -------------------------------------------------------------------------------- /testWithoutCoverage.hxml: -------------------------------------------------------------------------------- 1 | -cp src 2 | -cp test 3 | 4 | -lib haxeparser 5 | -lib utest 6 | 7 | -D unittest 8 | -x TestMain --------------------------------------------------------------------------------