├── CODE_OF_CONDUCT.md ├── LICENSE ├── README.md ├── build.gradle.kts ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── src └── main └── kotlin └── org └── jetbrains └── kotlin └── spec └── grammar └── tools ├── KotlinGrammarTools.kt ├── loader └── GrammarLoader.kt └── parsing └── Parser.kt /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | This project and the corresponding community is governed by the [JetBrains Open Source and Community Code of Conduct](https://github.com/JetBrains#code-of-conduct). 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2000-2020 JetBrains s.r.o. and Kotlin Programming Language contributors. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kotlin-grammar-tools 2 | 3 | [![JetBrains team project](https://jb.gg/badges/team.svg)](https://github.com/JetBrains#jetbrains-on-github) 4 | [![TeamCity (simple build status)](https://img.shields.io/teamcity/https/teamcity.jetbrains.com/e/Kotlin_Spec_GrammarMaster.svg?style=flat)](https://teamcity.jetbrains.com/viewType.html?buildTypeId=Kotlin_Spec_GrammarMaster&branch_Kotlin_dev=%3Cdefault%3E&tab=buildTypeStatusDiv) 5 | [![GitHub license](https://img.shields.io/badge/license-Apache%20License%202.0-blue.svg?style=flat)](https://www.apache.org/licenses/LICENSE-2.0) 6 | 7 | ## Description 8 | 9 | This library allows tokenize and parse Kotlin code in your program using the Kotlin grammar. 10 | 11 | Simple example: 12 | ```kotlin 13 | fun main() { 14 | val tokens = tokenizeKotlinCode("val x = foo() + 10;") 15 | val parseTree = parseKotlinCode(tokens) 16 | // or just `val parseTree = parseKotlinCode("val x = foo() + 10;")` 17 | 18 | println(parseTree) 19 | } 20 | ``` 21 | 22 | Tokens or parse tree can be used for various Kotlin code analysis. 23 | 24 | Note that the parse tree may not match exactly to PSI (parse tree generated by Kotlin compiler). 25 | This is due to the fact that some errors for the user convenience are not generated at the parser level, but later; the grammar, in turn, takes into account such cases and may not allow the code that could be parsed by the Kotlin compiler parser. 26 | 27 | ### Kotlin grammar 28 | 29 | The grammar is located in the [Kotlin specification repository](https://github.com/JetBrains/kotlin-spec/tree/master/grammar). 30 | 31 | ## Status 32 | 33 | The library is developed **only for internal purposes** of the Kotlin team, and actual state of the library isn't guaranteed. 34 | 35 | ## Using 36 | 37 | To use the library, you need to perform the following steps. 38 | 39 | 1. (Prerequisite) Get the [Kotlin specification repository](https://github.com/Kotlin/kotlin-spec) and run its `:grammar:publishToMavenLocal` gradle task. 40 | This will build and install the `kotlin-grammar-parser` dependency. 41 | 1. Run the `publishToMavenLocal` gradle task. 42 | This will build and install the `kotlin-grammar-tools` library. 43 | 1. Add `mavenLocal` to repositories in your project, and then add the dependency for this library. 44 | For example (gradle): `implementation("org.jetbrains.kotlin.spec.grammar.tools:kotlin-grammar-tools:0.1")`. 45 | 46 | As an alternative for steps 1 and 2, you can just download the jars from [Releases](https://github.com/Kotlin/kotlin-grammar-tools/releases) or TeamCity (both `kotlin-grammar-parser` and `kotlin-grammar-tools` artifacts can be found on the [TeamCity Kotlin grammar build page](https://teamcity.jetbrains.com/viewType.html?buildTypeId=Kotlin_Spec_GrammarMaster)). 47 | 48 | ## Exceptions 49 | 50 | Lexer and parser are throwing exceptions if it has been inputted the invalid code (in terms of lexer or parser): `KotlinLexerException` and `KotlinParserException`. 51 | 52 | Example of handling this exceptions: 53 | ```kotlin 54 | fun foo(): ParseTree? { 55 | val tokens = try { 56 | tokenizeKotlinCode("val x = foo() + 10;") 57 | } catch (e: KotlinLexerException) { 58 | println("Tokenization the code fails") 59 | return null 60 | } 61 | val parseTree = try { 62 | parseKotlinCode(tokens) 63 | } catch (e: KotlinParserException) { 64 | println("Parsing the code fails") 65 | return null 66 | } 67 | 68 | return parseTree 69 | } 70 | ``` 71 | -------------------------------------------------------------------------------- /build.gradle.kts: -------------------------------------------------------------------------------- 1 | import org.jetbrains.kotlin.gradle.tasks.KotlinCompile 2 | 3 | plugins { 4 | kotlin("jvm") version "1.7.0" 5 | `maven-publish` 6 | } 7 | 8 | group = "org.jetbrains.kotlin.spec.grammar.tools" 9 | version = "0.1" 10 | 11 | val archivePrefix = "kotlin-grammar-tools" 12 | 13 | repositories { 14 | mavenLocal() 15 | mavenCentral() 16 | } 17 | 18 | dependencies { 19 | compileOnly(kotlin("stdlib-jdk8")) 20 | implementation("org.jetbrains.kotlin.spec.grammar:kotlin-grammar-parser:0.1") 21 | } 22 | 23 | publishing { 24 | publications { 25 | create("maven") { 26 | groupId = group as String 27 | artifactId = archivePrefix 28 | version = version as String 29 | 30 | from(components["java"]) 31 | } 32 | } 33 | } 34 | 35 | tasks.withType { 36 | kotlinOptions.jvmTarget = "11" 37 | } 38 | 39 | tasks.withType { 40 | archiveFileName.set("$archivePrefix-${archiveVersion.get()}.jar") 41 | 42 | manifest { 43 | attributes( 44 | mapOf( 45 | "Class-Path" to configurations.runtimeClasspath.get().files.joinToString(" ") { it.name } 46 | ) 47 | ) 48 | } 49 | 50 | from(configurations.runtimeClasspath.get().files.map { if (it.isDirectory) it else zipTree(it) }) 51 | 52 | duplicatesStrategy = DuplicatesStrategy.INCLUDE 53 | } 54 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | kotlin.code.style=official -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kotlin/grammar-tools/77e80a1ae681b782ad75a5309ac80d52c2fb6448/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.2-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'kotlin-grammar-tools' 2 | 3 | -------------------------------------------------------------------------------- /src/main/kotlin/org/jetbrains/kotlin/spec/grammar/tools/KotlinGrammarTools.kt: -------------------------------------------------------------------------------- 1 | package org.jetbrains.kotlin.spec.grammar.tools 2 | 3 | import org.jetbrains.kotlin.spec.grammar.tools.parsing.Parser 4 | 5 | class KotlinToken( 6 | val type: String, 7 | val text: String, 8 | val channel: Int 9 | ) { 10 | override fun toString() = "$type(\"${text.replace(System.lineSeparator(), "\\n")}\")" 11 | } 12 | 13 | class KotlinTokensList(list: List): ArrayList(list) { 14 | override fun toString() = joinToString(System.lineSeparator()) 15 | } 16 | 17 | enum class KotlinParseTreeNodeType { 18 | RULE, TERMINAL 19 | } 20 | 21 | class KotlinLexerException(val lexerMessage: String, val position: Pair) : Throwable() { 22 | init { 23 | System.err.println("Lexer error: $lexerMessage, position: $position") 24 | } 25 | } 26 | 27 | class KotlinParserException(val parserMessage: String, val position: Pair) : Throwable() { 28 | init { 29 | System.err.println("Parser error: $parserMessage, position: $position") 30 | } 31 | } 32 | 33 | class KotlinParseTree( 34 | val type: KotlinParseTreeNodeType, 35 | val name: String, 36 | val text: String? = null, 37 | val children: MutableList = mutableListOf() 38 | ) { 39 | companion object { 40 | private val ls = System.lineSeparator() 41 | } 42 | 43 | private fun stringifyTree(builder: StringBuilder, node: KotlinParseTree, depth: Int = 0): StringBuilder = 44 | builder.apply { 45 | node.children.forEach { child -> 46 | when (child.type) { 47 | KotlinParseTreeNodeType.RULE -> append(" ".repeat(depth) + child.name + ls) 48 | KotlinParseTreeNodeType.TERMINAL -> append(" ".repeat(depth) + "${child.name}(\"${child.text!!.replace(ls, "\\n")}\")" + ls) 49 | } 50 | stringifyTree(builder, child, depth + 1) 51 | } 52 | } 53 | 54 | override fun toString() = stringifyTree(StringBuilder(), this).toString() 55 | } 56 | 57 | fun parseKotlinCode(tokens: List) = Parser.parse(tokens) 58 | 59 | fun parseKotlinCode(sourceCode: String) = parseKotlinCode(tokenizeKotlinCode(sourceCode)) 60 | 61 | fun tokenizeKotlinCode(sourceCode: String) = Parser.tokenize(sourceCode) 62 | -------------------------------------------------------------------------------- /src/main/kotlin/org/jetbrains/kotlin/spec/grammar/tools/loader/GrammarLoader.kt: -------------------------------------------------------------------------------- 1 | package org.jetbrains.kotlin.spec.grammar.tools.loader 2 | 3 | import java.io.File 4 | 5 | internal object GrammarLoader { 6 | private const val GRAMMAR_DIR = "src/org.jetbrains.kotlin.grammar.validate.main/antlr" 7 | private const val LEXER_FILENAME = "KotlinLexer.g4" 8 | private const val PARSER_FILENAME = "KotlinParser.g4" 9 | private const val UNICODE_CLASSES_FILENAME = "UnicodeClasses.g4" 10 | 11 | private fun getGrammarFileText(grammarName: String) = File("$GRAMMAR_DIR/$grammarName").readText() 12 | 13 | fun getLexerGrammarAsText() = getGrammarFileText(LEXER_FILENAME) 14 | 15 | fun getParserGrammarAsText() = getGrammarFileText(PARSER_FILENAME) 16 | 17 | fun getUnicodeClassesGrammarAsText() = getGrammarFileText(UNICODE_CLASSES_FILENAME) 18 | } 19 | -------------------------------------------------------------------------------- /src/main/kotlin/org/jetbrains/kotlin/spec/grammar/tools/parsing/Parser.kt: -------------------------------------------------------------------------------- 1 | package org.jetbrains.kotlin.spec.grammar.tools.parsing 2 | 3 | import org.antlr.v4.runtime.* 4 | import org.antlr.v4.runtime.tree.ParseTree 5 | import java.io.ByteArrayInputStream 6 | import org.antlr.v4.runtime.misc.Pair 7 | import java.nio.charset.StandardCharsets 8 | import org.antlr.v4.runtime.Token.DEFAULT_CHANNEL 9 | import org.antlr.v4.runtime.tree.TerminalNodeImpl 10 | import org.jetbrains.kotlin.spec.grammar.tools.* 11 | import org.jetbrains.kotlin.spec.grammar.KotlinLexer 12 | import org.jetbrains.kotlin.spec.grammar.KotlinParser 13 | 14 | internal object Parser { 15 | private val errorLexerListener = object : BaseErrorListener() { 16 | override fun syntaxError( 17 | recognizer: Recognizer<*, *>?, 18 | offendingSymbol: Any?, 19 | line: Int, 20 | charPositionInLine: Int, 21 | message: String, 22 | e: RecognitionException? 23 | ) = throw KotlinLexerException(message, kotlin.Pair(line, charPositionInLine)) 24 | } 25 | 26 | private val errorParserListener = object : BaseErrorListener() { 27 | override fun syntaxError( 28 | recognizer: Recognizer<*, *>?, 29 | offendingSymbol: Any?, 30 | line: Int, 31 | charPositionInLine: Int, 32 | message: String, 33 | e: RecognitionException? 34 | ) = throw KotlinParserException(message, kotlin.Pair(line, charPositionInLine)) 35 | } 36 | 37 | private fun getCharsStream(str: String) = 38 | CharStreams.fromStream(ByteArrayInputStream(str.toByteArray()), StandardCharsets.UTF_8) 39 | 40 | private fun getAntlrTokenByKotlinToken(token: KotlinToken, tokenTypeMap: Map): CommonToken { 41 | val tokenNumber = tokenTypeMap[token.type]!! 42 | 43 | return if (token.channel == DEFAULT_CHANNEL) 44 | CommonToken(tokenNumber, token.text) 45 | else { 46 | CommonToken(Pair(null, getCharsStream(token.text)), tokenNumber, token.channel, 0, 0) 47 | } 48 | } 49 | 50 | fun tokenize(sourceCode: String): KotlinTokensList { 51 | val lexer = KotlinLexer(getCharsStream(sourceCode)).apply { 52 | removeErrorListeners() 53 | addErrorListener(errorLexerListener) 54 | } 55 | 56 | return KotlinTokensList( 57 | lexer.allTokens.map { 58 | KotlinToken( 59 | lexer.vocabulary.getSymbolicName(it.type), 60 | it.text, 61 | it.channel 62 | ) 63 | } 64 | ) 65 | } 66 | 67 | private fun buildTree( 68 | parser: KotlinParser, 69 | tokenTypeMap: Map, 70 | antlrParseTree: ParseTree, 71 | kotlinParseTree: KotlinParseTree 72 | ): KotlinParseTree { 73 | for (i in 0..antlrParseTree.childCount) { 74 | val antlrParseTreeNode = antlrParseTree.getChild(i) ?: continue 75 | val kotlinParseTreeNode = when (antlrParseTreeNode) { 76 | is TerminalNodeImpl -> 77 | KotlinParseTree( 78 | KotlinParseTreeNodeType.TERMINAL, 79 | KotlinLexer.VOCABULARY.getSymbolicName(antlrParseTreeNode.symbol.type), 80 | antlrParseTreeNode.symbol.text 81 | ) 82 | else -> 83 | KotlinParseTree( 84 | KotlinParseTreeNodeType.RULE, 85 | parser.ruleNames[(antlrParseTreeNode as RuleContext).ruleIndex] 86 | ) 87 | } 88 | 89 | kotlinParseTree.children.add(kotlinParseTreeNode) 90 | buildTree(parser, tokenTypeMap, antlrParseTreeNode, kotlinParseTreeNode) 91 | } 92 | 93 | return kotlinParseTree 94 | } 95 | 96 | fun parse(tokens: List): KotlinParseTree { 97 | val tokenTypeMap = KotlinLexer(null).tokenTypeMap 98 | val tokensList = ListTokenSource(tokens.map { getAntlrTokenByKotlinToken(it, tokenTypeMap) }) 99 | val parser = KotlinParser(CommonTokenStream(tokensList)).apply { 100 | removeErrorListeners() 101 | addErrorListener(errorParserListener) 102 | } 103 | val tree = parser.kotlinFile() 104 | 105 | return buildTree( 106 | parser, 107 | tokenTypeMap, 108 | tree, 109 | KotlinParseTree( 110 | KotlinParseTreeNodeType.RULE, 111 | parser.ruleNames[parser.ruleIndexMap["kotlinFile"]!!] 112 | ) 113 | ) 114 | } 115 | } 116 | --------------------------------------------------------------------------------