├── .github ├── FUNDING.yml └── workflows │ └── actions.yml ├── .gitignore ├── LICENSE ├── README.md ├── build.mill ├── build.sbt ├── cssparse ├── src │ └── cssparse │ │ ├── Ast.scala │ │ ├── CssParser.scala │ │ └── PrettyPrinter.scala └── test │ ├── src-jvm │ └── cssparse │ │ ├── ProjectTests.scala │ │ └── TestUtil.scala │ └── src │ └── cssparse │ ├── CssTests.scala │ └── PrettyPrinterTests.scala ├── demo └── src │ └── demo │ └── DemoMain.scala ├── fastparse ├── src-2.12 │ └── fastparse │ │ └── internal │ │ └── NoWarn.scala ├── src-2.13 │ └── fastparse │ │ └── internal │ │ └── NoWarn.scala ├── src-2 │ └── fastparse │ │ ├── internal │ │ ├── MacroImpls.scala │ │ ├── MacroRepImpls.scala │ │ └── RepImpls.scala │ │ └── package.scala ├── src-3 │ └── fastparse │ │ ├── internal │ │ ├── MacroInlineImpls.scala │ │ ├── MacroRepImpls.scala │ │ └── NoWarn.scala │ │ └── package.scala ├── src-js │ └── fastparse │ │ └── CharPredicates.scala ├── src-jvm │ └── fastparse │ │ └── CharPredicates.scala ├── src-native │ └── fastparse │ │ └── CharPredicates.scala ├── src │ └── fastparse │ │ ├── Implicits.scala │ │ ├── Parsed.scala │ │ ├── ParserInput.scala │ │ ├── ParsingRun.scala │ │ ├── SharedPackageDefs.scala │ │ ├── Whitespace.scala │ │ └── internal │ │ ├── UberBuffer.scala │ │ └── Util.scala └── test │ ├── resources │ └── fastparse │ │ └── test.json │ ├── src-2.12+ │ └── fastparse │ │ ├── CustomWhitespaceMathTests.scala │ │ └── IteratorTests.scala │ └── src │ └── fastparse │ ├── ExampleTests.scala │ ├── FailureTests.scala │ ├── GnipSubSyntaxTest.scala │ ├── IndentationTests.scala │ ├── JsonTests.scala │ ├── Main.scala │ ├── MathTests.scala │ ├── ParsingTests.scala │ ├── TypeTests.scala │ ├── UtilTests.scala │ ├── WhitespaceMathTests.scala │ └── WhitespaceTests.scala ├── mill ├── perftests ├── bench1 │ └── src │ │ └── perftests │ │ ├── CssParse.scala │ │ ├── Expr.scala │ │ ├── FastParseParser.scala │ │ ├── JsonParse.scala │ │ ├── JsonnetParse.scala │ │ ├── PythonParse.scala │ │ ├── ScalaParse.scala │ │ └── Utils.scala ├── bench2 │ └── src │ │ └── perftests │ │ ├── CssParse.scala │ │ ├── Expr.scala │ │ ├── FasterParserParser.scala │ │ ├── JsonParse.scala │ │ ├── JsonnetParse.scala │ │ ├── PythonParse.scala │ │ ├── ScalaParse.scala │ │ └── Utils.scala ├── compare │ └── src │ │ └── compare │ │ ├── JsonBench.scala │ │ ├── PythonBench.scala │ │ └── ScalaBench.scala └── resources │ ├── GenJSCode.scala │ ├── bootstrap.css │ ├── consul-alerts.jsonnet │ ├── consul-dashboards.jsonnet │ ├── cross_validation.py │ ├── grafana.jsonnet │ ├── ksonnet-util.jsonnet │ ├── oauth2-proxy.jsonnet │ ├── prometheus-alertmanager.jsonnet │ ├── prometheus-config-kops.jsonnet │ ├── prometheus-config.jsonnet │ ├── prometheus-grafana.jsonnet │ ├── prometheus-kube-state-metrics.jsonnet │ ├── prometheus-nginx.jsonnet │ ├── prometheus-node-exporter.jsonnet │ └── prometheus.jsonnet ├── project ├── build.properties └── build.sbt ├── pythonparse ├── src │ └── pythonparse │ │ ├── Ast.scala │ │ ├── Expressions.scala │ │ ├── Lexical.scala │ │ └── Statements.scala └── test │ ├── resources │ ├── bench.py │ └── parse.py │ ├── src-jvm │ └── pythonparse │ │ └── ProjectTests.scala │ └── src │ └── pythonparse │ ├── RegressionTests.scala │ ├── TestUtils.scala │ └── UnitTests.scala ├── readme ├── ApiHighlights.scalatex ├── Changelog.scalatex ├── Comparisons.scalatex ├── DebuggingParsers.scalatex ├── ErrorReportingInternals.scalatex ├── ExampleParsers.scalatex ├── FastParseInternals.scalatex ├── GettingStarted.scalatex ├── Performance.scalatex ├── Readme.scalatex ├── StreamingParsing.scalatex ├── WritingParsers.scalatex └── resources │ ├── JProfiler.png │ └── favicon.png └── scalaparse ├── src └── scalaparse │ ├── Core.scala │ ├── Exprs.scala │ ├── Scala.scala │ ├── Types.scala │ ├── Xml.scala │ └── syntax │ ├── Basic.scala │ ├── Identifiers.scala │ └── Literals.scala └── test ├── resources └── scalaparse │ └── Test.scala ├── src-2-jvm └── scalaparse │ ├── ProjectTests.scala │ └── TestMain.scala ├── src-2.12-jvm └── scalaparse │ └── ScalacParser.scala ├── src-2.13-jvm └── scalaparse │ └── ScalacParser.scala └── src └── scalaparse ├── TestUtil.scala └── unit ├── FailureTests.scala ├── SuccessTests.scala └── TrailingCommasTests.scala /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: lihaoyi 4 | -------------------------------------------------------------------------------- /.github/workflows/actions.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | tags: 8 | - '*' 9 | pull_request: 10 | branches: 11 | - master 12 | 13 | jobs: 14 | test-js: 15 | runs-on: ubuntu-22.04 16 | steps: 17 | - uses: actions/checkout@v3 18 | - uses: coursier/cache-action@v6 19 | - uses: actions/setup-java@v3 20 | with: 21 | java-version: "11" 22 | distribution: "temurin" 23 | - name: Run tests 24 | run: ./mill -i __.js.__.test 25 | test-native: 26 | runs-on: ubuntu-22.04 27 | steps: 28 | - uses: actions/checkout@v3 29 | - uses: coursier/cache-action@v6 30 | - uses: actions/setup-java@v3 31 | with: 32 | java-version: "11" 33 | distribution: "temurin" 34 | - name: Run tests 35 | run: ./mill -i __.native.__.test 36 | test-jvm: 37 | runs-on: ubuntu-22.04 38 | steps: 39 | - uses: actions/checkout@v3 40 | - uses: coursier/cache-action@v6 41 | - uses: actions/setup-java@v3 42 | with: 43 | java-version: "11" 44 | distribution: "temurin" 45 | - name: Run tests 46 | run: ./mill -i __.jvm.__.test 47 | check-binary-compatibility: 48 | runs-on: ubuntu-22.04 49 | steps: 50 | - uses: actions/checkout@v3 51 | with: 52 | fetch-depth: 0 53 | - uses: coursier/cache-action@v6 54 | - uses: actions/setup-java@v3 55 | with: 56 | java-version: "11" 57 | distribution: "temurin" 58 | - name: Check Binary Compatibility 59 | run: ./mill -i __.mimaReportBinaryIssues 60 | 61 | publish-sonatype: 62 | if: github.repository == 'com-lihaoyi/fastparse' && contains(github.ref, 'refs/tags/') 63 | needs: 64 | - test-jvm 65 | - test-js 66 | - test-native 67 | runs-on: ubuntu-latest 68 | env: 69 | MILL_SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} 70 | MILL_SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} 71 | MILL_PGP_SECRET_BASE64: ${{ secrets.SONATYPE_PGP_PRIVATE_KEY }} 72 | MILL_PGP_PASSPHRASE: ${{ secrets.SONATYPE_PGP_PRIVATE_KEY_PASSWORD }} 73 | LANG: "en_US.UTF-8" 74 | LC_MESSAGES: "en_US.UTF-8" 75 | LC_ALL: "en_US.UTF-8" 76 | 77 | steps: 78 | - uses: actions/checkout@v3 79 | - uses: actions/setup-java@v3 80 | with: 81 | distribution: 'temurin' 82 | java-version: 11 83 | - name: Publish to Maven Central 84 | run: ./mill -i mill.scalalib.SonatypeCentralPublishModule/ 85 | 86 | - name: Create GitHub Release 87 | id: create_gh_release 88 | uses: actions/create-release@v1.1.4 89 | env: 90 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token 91 | with: 92 | tag_name: ${{ github.ref }} 93 | release_name: ${{ github.ref }} 94 | draft: false 95 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /*/target/ 2 | target/ 3 | out/ 4 | .DS_STORE 5 | *.iml 6 | .idea 7 | .coursier 8 | .bloop/ 9 | .metals/ 10 | project/metals.sbt 11 | .vscode/ 12 | .bsp/ 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Li Haoyi (haoyi.sg@gmail.com) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | FastParse [![Build Status](https://travis-ci.org/lihaoyi/fastparse.svg?branch=master)](https://travis-ci.org/lihaoyi/fastparse) [![Join the chat at https://gitter.im/lihaoyi/Ammonite](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/lihaoyi/fastparse?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) 2 | ========= 3 | 4 | This is where the code for the FastParse parsing library lives! If you want 5 | to use Fastparse, you probably will want to check out the documentation: 6 | 7 | - [Documentation](https://com-lihaoyi.github.io/fastparse) 8 | 9 | If you use FastParse and like it, you will probably enjoy the following book by the Author: 10 | 11 | - [*Hands-on Scala Programming*](https://www.handsonscala.com/) 12 | 13 | *Hands-on Scala* has uses FastParse extensively throughout the book, with 14 | the entirety of *Chapter 19: Parsing Structured Text* dedicated to 15 | the library and *Chapter 20: Implementing a Programming Language* making heavy 16 | use of it. *Hands-on Scala* is a great way to level up your skills in Scala 17 | in general and FastParse in particular. 18 | 19 | For a good hands-on tutorial working through the basics of how to use this 20 | library, check out the following blog post: 21 | 22 | - [Easy Parsing with Parser Combinators](http://www.lihaoyi.com/post/EasyParsingwithParserCombinators.html) 23 | 24 | The following post gives a good overview of the design of FastParse: 25 | 26 | - [Fastparse 2: Even Faster Scala Parser Combinators](https://www.lihaoyi.com/post/Fastparse2EvenFasterScalaParserCombinators.html) 27 | 28 | This readme contains some developer docs, if you intend on working on the 29 | fastparse repo, not just using it as a library. 30 | 31 | Developer Docs 32 | ============== 33 | 34 | The core of FastParse lives in the `fastparse/` folder. It is cross-built 35 | ScalaJVM/Scala.js codebase, with almost everything shared between the two 36 | platforms in the `fastparse/src/` and minor differences in `fastparse/src-js/` 37 | and `fastparse/src-jvm/`. 38 | 39 | The three subprojects `scalaparse/`, `pythonparse/` and `cssparse/` are 40 | FastParse parsers for those respective languages. These are both usable as 41 | standalone libraries, and also serve as extensive test-suites and use-cases for 42 | FastParse itself. Each of those projects clones & parses large quantities of 43 | code from Github as part of *their* own test suites. 44 | 45 | `perftests/` constains performance tests for main projects in the library 46 | including `ScalaParse`, `PythonParse`, `CssParse`, `readme/` contains the 47 | documentation site, which contains several live demos of FastParse parsers 48 | compiled to Scala.js. These all live in `demo/`. 49 | 50 | Common Commands 51 | --------------- 52 | 53 | Note: you should use mill 0.11 or later. 54 | 55 | - `mill -w "fastparse.jvm[2.12.10].test"` runs the main testsuite. If you're 56 | hacking on FastParse, this is often where you want to go 57 | 58 | - You can run the other suites via `fastparse.js`, `scalaparse.jvm`, etc. if you 59 | wish, but I typically don't and leave that to CI unless I'm actively working 60 | on the sub-project 61 | 62 | - You can use `mill -w "fastparse.jvm[_].test"` to run it under different Scala 63 | versions, but again I usually don't bother 64 | 65 | - `mill __.test.test` is the aggregate test-all command, but is pretty slow. You 66 | can use `mill "__.jvm[2.12.17].test"` to run all tests only under JVM/Scala-2.12, 67 | which is much faster and catches most issues 68 | 69 | - `mill demo.fullOpt && sbt readme/run` builds the documentation site, which can 70 | then be found at `readme/target/scalatex/index.html` 71 | 72 | Contribution Guidelines 73 | ----------------------- 74 | 75 | - **If you're not sure if something is a bug or not, ask on Gitter first =)** 76 | - **All code PRs should come with**: a meaningful description, inline comments for important things, unit tests, and a green build 77 | - **Non-trivial changes, including bug fixes, should appear in the changelog**. Feel free to add your name and link to your github profile! 78 | - **New features should be added to the relevant parts of the documentation** 79 | - **To a large extent, FastParse is designed so that you can extend it in your own code** without needing to modify the core. If you want to add features, be prepared to argue why it should be built-in and not just part of your own code. 80 | - **It's entirely possible your changes won't be merged**, or will get ripped out later. This is also the case for my changes, as the Author! 81 | - **Even a rejected/reverted PR is valuable**! It helps explore the solution space, and know what works and what doesn't. For every line in the repo, at least three lines were tried, committed, and reverted/refactored, and more than 10 were tried without committing. 82 | - **Feel free to send Proof-Of-Concept PRs** that you don't intend to get merged. 83 | - **No binary or source compatibility is guaranteed between any releases**. FastParse keeps compatibility following the SemVer rule, upgrading is usually trivial, and I don't expect existing functionality to go away 84 | 85 | License 86 | ======= 87 | 88 | The MIT License (MIT) 89 | 90 | Copyright (c) 2014 Li Haoyi (haoyi.sg@gmail.com) 91 | 92 | Permission is hereby granted, free of charge, to any person obtaining a copy 93 | of this software and associated documentation files (the "Software"), to deal 94 | in the Software without restriction, including without limitation the rights 95 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 96 | copies of the Software, and to permit persons to whom the Software is 97 | furnished to do so, subject to the following conditions: 98 | 99 | The above copyright notice and this permission notice shall be included in 100 | all copies or substantial portions of the Software. 101 | 102 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 103 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 104 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 105 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 106 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 107 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 108 | THE SOFTWARE. 109 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | lazy val readme = scalatex.ScalatexReadme( 2 | projectId = "readme", 3 | wd = file(""), 4 | url = "https://github.com/lihaoyi/fastparse/tree/master", 5 | source = "Readme", 6 | autoResources = List("out.js", "JProfiler.png") 7 | ).settings( 8 | (Compile / resources) += baseDirectory.value/".."/"out"/"demo"/"fullOpt.dest"/"out.js", 9 | scalaVersion := "2.12.19" 10 | ) 11 | 12 | -------------------------------------------------------------------------------- /cssparse/src/cssparse/Ast.scala: -------------------------------------------------------------------------------- 1 | package cssparse 2 | 3 | // According to https://www.w3.org/TR/css-syntax-3/ 4 | 5 | object Ast { 6 | 7 | sealed abstract class ComponentValue 8 | 9 | sealed abstract class SimpleToken extends ComponentValue 10 | 11 | sealed case class IdentToken(name: String) extends SimpleToken 12 | sealed case class FunctionToken(name: String) 13 | sealed case class AtWordToken(name: String) extends SimpleToken 14 | sealed case class HashWordToken(name: String) extends SimpleToken 15 | sealed case class StringToken(string: String) extends SimpleToken 16 | sealed case class UrlToken(url: String) extends SimpleToken 17 | sealed case class NumberToken(number: String) extends SimpleToken 18 | sealed case class DimensionToken(number: String, dimensionName: String) extends SimpleToken 19 | sealed case class PercentageToken(number: String) extends SimpleToken 20 | sealed case class UnicodeRangeToken(left: String, right: String) extends SimpleToken 21 | 22 | sealed case class IncludeMatchToken() extends SimpleToken 23 | sealed case class DashMatchToken() extends SimpleToken 24 | sealed case class PrefixMatchToken() extends SimpleToken 25 | sealed case class SuffixMatchToken() extends SimpleToken 26 | sealed case class SubstringMatchToken() extends SimpleToken 27 | sealed case class ColumnToken() extends SimpleToken 28 | 29 | sealed abstract class CToken extends SimpleToken 30 | sealed case class CdoToken() extends CToken 31 | sealed case class CdcToken() extends CToken 32 | 33 | sealed case class DelimToken(delimeter: String) extends SimpleToken 34 | 35 | sealed class Block(val leftBracket: String, val rightBracket: String, val values: Seq[ComponentValue]) 36 | extends ComponentValue 37 | sealed case class BracketsBlock(override val values: Seq[ComponentValue]) extends Block("(", ")", values) 38 | sealed case class CurlyBracketsBlock(override val values: Seq[ComponentValue]) extends Block("{", "}", values) 39 | sealed case class SquareBracketsBlock(override val values: Seq[ComponentValue]) extends Block("[", "]", values) 40 | 41 | sealed case class FunctionBlock(name: String, bracketsBlock: BracketsBlock) extends ComponentValue 42 | 43 | sealed abstract class Selector 44 | sealed abstract class SingleSelector extends Selector 45 | sealed abstract class PartSelector extends SingleSelector 46 | 47 | sealed case class AllSelector() extends PartSelector 48 | sealed case class ElementSelector(name: String) extends PartSelector 49 | sealed case class IdSelector(id: String) extends SingleSelector 50 | sealed case class AttributeSelector(name: Option[String], 51 | attrs: Seq[(String, Option[String], Option[String])]) extends PartSelector 52 | 53 | sealed abstract class ComplexSelectorPart 54 | 55 | sealed case class ClassSelectorPart(part: PartSelector) extends ComplexSelectorPart 56 | sealed case class PseudoSelectorPart(pseudoClass: String, param: Seq[ComponentValue]) extends ComplexSelectorPart 57 | sealed case class ComplexSelector(firstPart: Option[PartSelector], 58 | parts: Seq[ComplexSelectorPart]) extends SingleSelector 59 | 60 | sealed case class MultipleSelector(firstSelector: SingleSelector, 61 | selectors: Seq[(String, SingleSelector)]) extends Selector 62 | 63 | sealed abstract class Rule 64 | 65 | sealed case class QualifiedRule(selector: Either[Selector, Seq[ComponentValue]], 66 | block: DeclarationList) extends Rule 67 | sealed case class AtRule(name: String, options: Seq[ComponentValue], 68 | block: Option[Either[DeclarationList, RuleList]]) extends Rule 69 | sealed case class RuleList(rules: Seq[Rule]) 70 | 71 | sealed case class Stylesheet(rules: Seq[Either[Rule, CToken]]) 72 | 73 | sealed case class Declaration(name: String, value: Seq[ComponentValue], isImportant: Boolean) 74 | sealed case class DeclarationList(declarations: Seq[Either[Declaration, AtRule]]) 75 | } 76 | -------------------------------------------------------------------------------- /cssparse/src/cssparse/PrettyPrinter.scala: -------------------------------------------------------------------------------- 1 | package cssparse 2 | 3 | import cssparse.Ast._ 4 | 5 | object PrettyPrinter { 6 | 7 | val indentSize = 2 8 | def indentation(indent: Int, isIndentation: Boolean): String = 9 | if (isIndentation) "\n" + " " * indentSize * indent else " " 10 | 11 | def printToken(token: SimpleToken): String = { 12 | token match { 13 | case IdentToken("and") => "and " 14 | case IdentToken("or") => " or " 15 | case DelimToken(",") => ", " 16 | case DelimToken(".") => "." 17 | case DelimToken(":") => ":" 18 | case DelimToken("::") => "::" 19 | case DelimToken("=") => "=" 20 | case DelimToken(";") => ";" 21 | case DelimToken(delim) => s" $delim " 22 | 23 | case IdentToken(name) => name 24 | case AtWordToken(name) => s"@$name" 25 | case HashWordToken(name) => s"#$name" 26 | case StringToken(string) => s"'$string'" 27 | case UrlToken(url) => s"url($url)" 28 | case NumberToken(number) => number 29 | case DimensionToken(number, dimensionName) => number + dimensionName 30 | case PercentageToken(number) => s"$number%" 31 | case UnicodeRangeToken(left, right) if left == right => s"U+$left" 32 | case UnicodeRangeToken(left, right) => s"U+$left-$right" 33 | case IncludeMatchToken() => "~=" 34 | case DashMatchToken() => "|=" 35 | case PrefixMatchToken() => "^=" 36 | case SuffixMatchToken() => "$=" 37 | case SubstringMatchToken() => "*=" 38 | case ColumnToken() => "||" 39 | case CdcToken() => "" 41 | } 42 | } 43 | 44 | def printComponentValues(values: Seq[ComponentValue]): String = { 45 | if (values.nonEmpty) { 46 | def printBlock(block: Block): String = 47 | block.leftBracket + printComponentValues(block.values) + block.rightBracket 48 | 49 | values.dropRight(1).zip(values.drop(1)).map(p => { 50 | val (first, last) = p 51 | first match { 52 | case DelimToken(";") => "; " 53 | case st: SimpleToken => 54 | val isTokenFirst = first.isInstanceOf[Ast.SimpleToken] && !first.isInstanceOf[DelimToken] 55 | val isTokenLast = last.isInstanceOf[Ast.SimpleToken] && !last.isInstanceOf[DelimToken] 56 | printToken(st) + (if (isTokenFirst && isTokenLast) " " else "") 57 | case block: Block => printBlock(block) 58 | case FunctionBlock(name, block) => name + " " + printBlock(block) 59 | } 60 | }).mkString + 61 | (values.last match { 62 | case st: SimpleToken => printToken(st) 63 | case block: Block => printBlock(block) 64 | case FunctionBlock(name, block) => name + " " + printBlock(block) 65 | }) 66 | } else { 67 | "" 68 | } 69 | } 70 | 71 | def printSelector(selector: Selector): String = { 72 | 73 | def printPart(part: ComplexSelectorPart): String = { 74 | part match { 75 | case ClassSelectorPart(part) => "." + printSelector(part) 76 | case PseudoSelectorPart(pseudoClass, param) => pseudoClass + 77 | (if (param.nonEmpty) "(" + printComponentValues(param) + ")" else "") 78 | } 79 | } 80 | 81 | selector match { 82 | case AllSelector() => "*" 83 | case ElementSelector(name) => name 84 | case IdSelector(id) => "#" + id 85 | case AttributeSelector(optName, attrs) => 86 | optName.getOrElse("") + attrs.map({ 87 | case (attr, optToken, optValue) => 88 | "[" + attr + optToken.getOrElse("") + optValue.map("\"" + _ + "\"").getOrElse("") + "]" 89 | }).mkString 90 | case ComplexSelector(firstPart, parts) => 91 | firstPart.map(printSelector).getOrElse("") + parts.map(printPart).mkString 92 | case MultipleSelector(firstSelector, selectors) => 93 | printSelector(firstSelector) + 94 | selectors.map({ 95 | case (sep, selector) => 96 | (sep match { 97 | case " " => " " 98 | case "," => ", " 99 | case s => " " + s + " " 100 | }) + printSelector(selector) 101 | }).mkString 102 | } 103 | } 104 | 105 | def printDeclarationList(list: DeclarationList, indent: Int = 0, isIndentation: Boolean = true): String = { 106 | val indentPart = indentation(indent, isIndentation) 107 | list.declarations.map({ 108 | case Left(Declaration(name, values, isImportant)) => 109 | indentPart + name + ": " + printComponentValues(values) + {if (isImportant) " ! important" else ""} + ";" 110 | case Right(atRule) => printRule(atRule) 111 | }).mkString 112 | } 113 | 114 | def printRule(rule: Rule, indent: Int = 0, isIndentation: Boolean = true): String = { 115 | val indentPart = indentation(indent, isIndentation) 116 | def indentBlock[T](block: T, printBlock: (T, Int, Boolean) => String) = 117 | " {" + printBlock(block, indent + 1, isIndentation) + indentPart + "}" 118 | 119 | rule match { 120 | case QualifiedRule(Left(selector), block) => 121 | printSelector(selector) + indentBlock(block, printDeclarationList) 122 | case QualifiedRule(Right(values), block) => 123 | printComponentValues(values) + indentBlock(block, printDeclarationList) 124 | case AtRule(name, options, None) => 125 | indentPart + "@" + name + " " + printComponentValues(options) + ";" 126 | case AtRule(name, options, Some(Left(declartions))) => 127 | indentPart + "@" + name + " " + printComponentValues(options) + indentBlock(declartions, printDeclarationList) 128 | case AtRule(name, options, Some(Right(rules))) => 129 | indentPart + "@" + name + " " + printComponentValues(options) + indentBlock(rules, printRuleList) 130 | } 131 | } 132 | 133 | def printRuleList(ruleList: RuleList, indent: Int = 0, isIndentation: Boolean = true): String = { 134 | val indentPart = indentation(indent, isIndentation) 135 | ruleList.rules.map(rule => indentPart + printRule(rule, indent, isIndentation = isIndentation)).mkString("\n") 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /cssparse/test/src-jvm/cssparse/ProjectTests.scala: -------------------------------------------------------------------------------- 1 | package cssparse 2 | 3 | import java.nio.file.{Files, Paths} 4 | 5 | import utest._ 6 | 7 | import scala.sys.process._ 8 | 9 | object ProjectTests extends TestSuite { 10 | 11 | def checkCss()(implicit testPath: utest.framework.TestPath) = { 12 | val url = "https://github.com/" + testPath.value.last 13 | val name = url.split("/").last 14 | 15 | println(Paths.get("target", "files", name)) 16 | if (!Files.exists(Paths.get("out", "repos", name))){ 17 | println("DOWNLOADING") 18 | Seq("wget", url, "-O", "out/repos/" + name).! 19 | } 20 | val css = new String( 21 | java.nio.file.Files.readAllBytes( 22 | java.nio.file.Paths.get("out", "repos", name) 23 | ) 24 | ) 25 | 26 | TestUtil.checkParsing(css, tag = name) 27 | TestUtil.checkPrinting(css, tag = name) 28 | } 29 | 30 | val tests = Tests { 31 | 32 | 33 | test("twbs/bootstrap/raw/2c2ac3356425e192f7537227508c809a14aa5850/dist/css/bootstrap.css") - { 34 | Seq("mkdir", "-p", "out/repos").! 35 | checkCss() 36 | } 37 | test("twbs/bootstrap/raw/2c2ac3356425e192f7537227508c809a14aa5850/dist/css/bootstrap.min.css") - { 38 | Seq("mkdir", "-p", "out/repos").! 39 | checkCss() 40 | } 41 | // "primer/primer/raw/2c2ac3356425e192f7537227508c809a14aa5850/css/primer.css" - checkCss() 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /cssparse/test/src-jvm/cssparse/TestUtil.scala: -------------------------------------------------------------------------------- 1 | package cssparse 2 | 3 | import fastparse._ 4 | 5 | object TestUtil { 6 | 7 | def checkParsing(input: String, tag: String = "") = { 8 | 9 | def checkParsed(input: String, res: Parsed[Ast.RuleList]) = { 10 | res match { 11 | case f: Parsed.Failure => 12 | throw new Exception(tag + "\n" + input + "\n" + f.trace().longAggregateMsg) 13 | case s: Parsed.Success[Ast.RuleList] => 14 | val inputLength = input.length 15 | val index = s.index 16 | utest.assert(index == inputLength) 17 | } 18 | } 19 | 20 | val res = parse(input, CssRulesParser.ruleList(_)) 21 | checkParsed(input, res) 22 | 23 | val parsedInput = PrettyPrinter.printRuleList(res.get.value) 24 | val res2 = parse(parsedInput, CssRulesParser.ruleList(_)) 25 | checkParsed(parsedInput, res2) 26 | } 27 | 28 | 29 | def checkPrinting(input: String, tag: String = "") = { 30 | import java.io.StringReader 31 | import org.w3c.css.sac.InputSource 32 | import org.w3c.css.sac._ 33 | import com.steadystate.css.parser._ 34 | import scala.collection.mutable.ArrayBuffer 35 | 36 | def getErrors(css: String): Seq[String] = { 37 | val errors = ArrayBuffer[String]() 38 | val source = new InputSource(new StringReader(css)) 39 | val parser = new CSSOMParser(new SACParserCSS3()) 40 | parser.setErrorHandler(new ErrorHandler{ 41 | def error(ex: CSSParseException) = { 42 | errors += ex.toString 43 | println("ERROR " + ex + " Line: " + ex.getLineNumber + " Column:" + ex.getColumnNumber) 44 | } 45 | def fatalError(ex: CSSParseException) = { 46 | errors += ex.toString 47 | println("FATAL ERROR " + ex) 48 | } 49 | def warning(ex: CSSParseException) = println("WARNING " + ex) 50 | }) 51 | val sheet = parser.parseStyleSheet(source, null, null) 52 | assert(sheet != null) 53 | errors.toSeq 54 | } 55 | 56 | val parsedInput = PrettyPrinter.printRuleList(parse(input, CssRulesParser.ruleList(_)).get.value) 57 | val initialErrors = getErrors(input) 58 | val parsingErrors = getErrors(parsedInput) 59 | 60 | assert(initialErrors.sorted == parsingErrors.sorted) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /demo/src/demo/DemoMain.scala: -------------------------------------------------------------------------------- 1 | package demo 2 | 3 | import cssparse.PrettyPrinter 4 | import org.scalajs.dom 5 | import org.scalajs.dom.{Event, UIEvent, html} 6 | import fastparse._ 7 | import fastparse.internal.Util 8 | 9 | import scala.scalajs.js 10 | import scala.scalajs.js.annotation.{JSExport, JSExportTopLevel} 11 | import scala.scalajs.js.typedarray.{ArrayBuffer, Uint8Array} 12 | 13 | @JSExportTopLevel("DemoMain") 14 | object DemoMain { 15 | @JSExport 16 | def scalaparser(container: html.Div) = { 17 | val example = 18 | """ 19 | |package scalaparser 20 | | 21 | |/** 22 | | * Created by haoyi on 4/26/15. 23 | | */ 24 | |object Main { 25 | | def main(args: Array[String]): Unit = { 26 | | println("Hello World!") 27 | | } 28 | |}""".stripMargin 29 | helper(container, scalaparse.Scala.CompilationUnit(_), example) 30 | } 31 | @JSExport 32 | def math(container: html.Div) = { 33 | helper(container, test.fastparse.MathTests.expr(_), "((1+1*2)+(3*4*5))/3") 34 | } 35 | @JSExport 36 | def whitespaceMath(container: html.Div) = { 37 | helper(container, test.fastparse.WhitespaceMathTests.expr(_), " ( ( 1+1 * 2 ) +( 3* 4 *5 ) )/3") 38 | } 39 | @JSExport 40 | def indentation(container: html.Div) = { 41 | helper( 42 | container, 43 | test.fastparse.IndentationTests.expr(_), 44 | """+ 45 | | + 46 | | 1 47 | | * 48 | | 1 49 | | 2 50 | | * 51 | | 3 52 | | 4 53 | | 5""".stripMargin 54 | ) 55 | } 56 | @JSExport 57 | def json(container: html.Div) = { 58 | helper(container, test.fastparse.Json.jsonExpr(_), 59 | """{ 60 | | "firstName": "John", 61 | | "lastName": "Smith", 62 | | "age": 25, 63 | | "address": { 64 | | "streetAddress": "21 2nd Street", 65 | | "city": "New York", 66 | | "state": "NY", 67 | | "postalCode": 10021 68 | | }, 69 | | "phoneNumbers": [ 70 | | { 71 | | "type": "home", 72 | | "number": "212 555-1234" 73 | | }, 74 | | { 75 | | "type": "fax", 76 | | "number": "646 555-4567" 77 | | } 78 | | ] 79 | |}""".stripMargin) 80 | } 81 | @JSExport 82 | def css(container: html.Div) = { 83 | helper(container, cssparse.CssRulesParser.ruleList(_).map(PrettyPrinter.printRuleList(_)), 84 | """b, 85 | |strong { 86 | | font-weight: bold; 87 | |} 88 | |dfn 89 | |{ 90 | | font-style: italic; 91 | |} 92 | |h1 { 93 | | margin: .67em 0; 94 | | 95 | | 96 | | font-size: 2em 97 | |}""".stripMargin) 98 | } 99 | @JSExport 100 | def python(container: html.Div) = { 101 | helper(container, pythonparse.Statements.file_input(_), 102 | """def foo(x, y): 103 | | return x - y 104 | |""".stripMargin) 105 | } 106 | 107 | def helper(container: html.Div, parser: P[_] => P[Any], default: String) = { 108 | import scalatags.JsDom.all._ 109 | val inputBox = textarea( 110 | width := "45%", 111 | float.left, 112 | fontFamily := "monospace", 113 | fontSize := 16, 114 | default 115 | ).render 116 | 117 | val outputBox = div(width:="45%", float.right, overflowX.scroll).render 118 | 119 | def recalc() = { 120 | inputBox.rows = inputBox.value.linesIterator.length 121 | val details = parse(inputBox.value, parser) match{ 122 | case s: fastparse.Parsed.Success[_] => 123 | table( 124 | width := "100%", 125 | tr(td("Success!")), 126 | tr(td("value:"), td(pre(s.value.toString))) 127 | ) 128 | 129 | case fastparse.Parsed.Failure(stack, index, extra) => 130 | val pretty = Util.literalize( extra.input.slice( index, index + 15)).toString 131 | table( 132 | width := "100%", 133 | tr(td("Failure!")), 134 | tr(td("at index:"), td(code(index))), 135 | tr(td("found:"), td("...", code(pretty))), 136 | tr(td("expected:"), td(code(extra.trace().label))) 137 | ) 138 | } 139 | outputBox.innerHTML = "" 140 | outputBox.appendChild(details.render) 141 | } 142 | recalc() 143 | inputBox.onkeyup = (e: dom.Event) => recalc() 144 | 145 | container.appendChild(div(inputBox, outputBox, div(clear.both)).render) 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /fastparse/src-2.12/fastparse/internal/NoWarn.scala: -------------------------------------------------------------------------------- 1 | package fastparse.internal 2 | 3 | object NoWarn { 4 | @deprecated("Use scala.annotation.nowarn instead", "3.1.1") 5 | class nowarn(msg: String = "") 6 | } 7 | -------------------------------------------------------------------------------- /fastparse/src-2.13/fastparse/internal/NoWarn.scala: -------------------------------------------------------------------------------- 1 | package fastparse.internal 2 | 3 | object NoWarn{ 4 | @deprecated("Use scala.annotation.nowarn instead", "3.1.1") 5 | type nowarn = scala.annotation.nowarn 6 | } -------------------------------------------------------------------------------- /fastparse/src-2/fastparse/internal/MacroRepImpls.scala: -------------------------------------------------------------------------------- 1 | package fastparse.internal 2 | 3 | import scala.reflect.macros.blackbox.Context 4 | 5 | /** 6 | * Implementations of the various `.rep`/`.repX` overloads. The most common 7 | * and simple overloads are implemented as macros for performance, while the 8 | * more complex/general cases are left as normal methods to avoid code bloat 9 | * and allow the use of default/named arguments (which don't work in macros 10 | * due to https://github.com/scala/bug/issues/5920). 11 | * 12 | * Even the normal method overloads are manually-specialized to some extent 13 | * for various sorts of inputs as a best-effort attempt ot minimize branching 14 | * in the hot paths. 15 | */ 16 | object MacroRepImpls{ 17 | def repXMacro0[T: c.WeakTypeTag, V: c.WeakTypeTag](c: Context) 18 | (whitespace: Option[c.Tree], min: Option[c.Tree]) 19 | (repeater: c.Tree, 20 | ctx: c.Tree): c.Tree = { 21 | import c.universe._ 22 | val repeater1 = TermName(c.freshName("repeater")) 23 | val ctx1 = TermName(c.freshName("repeater")) 24 | val acc = TermName(c.freshName("acc")) 25 | val startIndex = TermName(c.freshName("startIndex")) 26 | val count = TermName(c.freshName("count")) 27 | val beforeSepIndex = TermName(c.freshName("beforeSepIndex")) 28 | val rec = TermName(c.freshName("rec")) 29 | val originalCut = TermName(c.freshName("originalCut")) 30 | val parsedMsg = TermName(c.freshName("parsedMsg")) 31 | val lastAgg = TermName(c.freshName("lastAgg")) 32 | val parsedAgg = TermName(c.freshName("parsedAgg")) 33 | val ((endSnippet, _), minCut) = min match{ 34 | case None => 35 | q""" 36 | $ctx1.freshSuccess($repeater1.result($acc), $startIndex, $originalCut) 37 | """ -> 38 | q""" "" """ -> 39 | q"""false""" 40 | case Some(min1) => 41 | q""" 42 | if ($count < $min1) $ctx1.augmentFailure($startIndex, $originalCut) 43 | else $ctx1.freshSuccess($repeater1.result($acc), $startIndex, $originalCut) 44 | """ -> 45 | q"""if($min1 == 0) "" else "(" + $min1 + ")"""" -> 46 | q"""$originalCut && ($count < $min1)""" 47 | } 48 | 49 | val rhsSnippet = 50 | q""" 51 | if (!$ctx1.isSuccess && $ctx1.cut) $ctx1.asInstanceOf[_root_.fastparse.ParsingRun[scala.Nothing]] 52 | else { 53 | $ctx1.cut = false 54 | $rec($beforeSepIndex, $count + 1, $parsedAgg) 55 | } 56 | """ 57 | 58 | val wsSnippet = whitespace match{ 59 | case None => q"$rec($beforeSepIndex, $count + 1, $parsedAgg)" 60 | case Some(ws) => 61 | if (ws.tpe =:= typeOf[fastparse.NoWhitespace.noWhitespaceImplicit.type]) 62 | rhsSnippet 63 | else 64 | q""" 65 | _root_.fastparse.internal.Util.consumeWhitespace($ws, $ctx1) 66 | $rhsSnippet 67 | """ 68 | } 69 | 70 | q""" 71 | $ctx match{ case $ctx1 => 72 | $repeater match {case $repeater1 => 73 | var $originalCut = $ctx1.cut 74 | val $acc = $repeater1.initial 75 | @_root_.scala.annotation.tailrec 76 | def $rec($startIndex: _root_.scala.Int, 77 | $count: _root_.scala.Int, 78 | $lastAgg: _root_.fastparse.internal.Msgs): _root_.fastparse.P[${c.weakTypeOf[V]}] = { 79 | $ctx1.cut = $minCut 80 | ${c.prefix}.parse0() 81 | 82 | val $parsedMsg = $ctx1.shortMsg 83 | val $parsedAgg = $ctx1.aggregateMsgs 84 | $originalCut |= $ctx1.cut 85 | if (!$ctx1.isSuccess) { 86 | val res = 87 | if ($ctx1.cut) $ctx1.asInstanceOf[_root_.fastparse.P[${c.weakTypeOf[V]}]] 88 | else $endSnippet 89 | 90 | if ($ctx1.verboseFailures) _root_.fastparse.internal.Util.reportParseMsgInRep( 91 | $startIndex, 92 | ${min.getOrElse(q"0")}, 93 | $ctx1, 94 | _root_.fastparse.internal.Msgs.empty, 95 | $parsedMsg, 96 | $lastAgg, 97 | true 98 | ) 99 | 100 | res 101 | }else { 102 | val $beforeSepIndex = $ctx1.index 103 | $repeater1.accumulate($ctx1.successValue.asInstanceOf[${c.weakTypeOf[T]}], $acc) 104 | $ctx1.cut = false 105 | $wsSnippet 106 | } 107 | } 108 | $rec($ctx1.index, 0, null) 109 | } 110 | } 111 | """ 112 | } 113 | 114 | def repXMacro1[T: c.WeakTypeTag, V: c.WeakTypeTag](c: Context) 115 | (repeater: c.Tree, 116 | ctx: c.Tree): c.Tree = { 117 | MacroRepImpls.repXMacro0[T, V](c)(None, None)(repeater, ctx) 118 | } 119 | 120 | def repXMacro2[T: c.WeakTypeTag, V: c.WeakTypeTag](c: Context) 121 | (min: c.Tree) 122 | (repeater: c.Tree, 123 | ctx: c.Tree): c.Tree = { 124 | MacroRepImpls.repXMacro0[T, V](c)(None, Some(min))(repeater, ctx) 125 | } 126 | 127 | def repXMacro1ws[T: c.WeakTypeTag, V: c.WeakTypeTag](c: Context) 128 | (repeater: c.Tree, 129 | whitespace: c.Tree, 130 | ctx: c.Tree): c.Tree = { 131 | MacroRepImpls.repXMacro0[T, V](c)(Some(whitespace), None)(repeater, ctx) 132 | } 133 | 134 | def repXMacro2ws[T: c.WeakTypeTag, V: c.WeakTypeTag](c: Context) 135 | (min: c.Tree) 136 | (repeater: c.Tree, 137 | whitespace: c.Tree, 138 | ctx: c.Tree): c.Tree = { 139 | MacroRepImpls.repXMacro0[T, V](c)(Some(whitespace), Some(min))(repeater, ctx) 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /fastparse/src-3/fastparse/internal/NoWarn.scala: -------------------------------------------------------------------------------- 1 | package fastparse.internal 2 | 3 | object NoWarn{ 4 | @deprecated("Use scala.annotation.nowarn instead", "3.1.1") 5 | type nowarn = scala.annotation.nowarn 6 | } 7 | -------------------------------------------------------------------------------- /fastparse/src-js/fastparse/CharPredicates.scala: -------------------------------------------------------------------------------- 1 | package fastparse 2 | 3 | /** 4 | * Fast, pre-computed character predicates for charactes from 0 to 65535 5 | * 6 | * Useful because FastParse does it's parsing character by character, so 7 | * although this doesn't have the full range of the java 8 | * `Character.getType(c: Int)` functions, it still is good enough for 9 | * a wide range of use cases 10 | */ 11 | object CharPredicates{ 12 | 13 | def isMathSymbol(c: Char) = Character.getType(c) == Character.MATH_SYMBOL 14 | def isOtherSymbol(c: Char) = Character.getType(c) == Character.OTHER_SYMBOL 15 | def isLetter(c: Char) = { 16 | ((((1 << Character.UPPERCASE_LETTER) | 17 | (1 << Character.LOWERCASE_LETTER) | 18 | (1 << Character.TITLECASE_LETTER) | 19 | (1 << Character.MODIFIER_LETTER) | 20 | (1 << Character.OTHER_LETTER)) >> Character.getType(c)) & 1) != 0 21 | } 22 | 23 | def isPrintableChar(c: Char) = { 24 | // Don't bother checking for Unicode SPECIAL block characters 25 | // in Scala.js because Scala.js doesn't really support it 26 | !java.lang.Character.isISOControl(c) && 27 | !java.lang.Character.isSurrogate(c) 28 | } 29 | def isDigit(c: Char) = Character.isDigit(c) 30 | def isLower(c: Char) = Character.isLowerCase((c)) 31 | def isUpper(c: Char) = Character.isUpperCase(c) 32 | } -------------------------------------------------------------------------------- /fastparse/src-jvm/fastparse/CharPredicates.scala: -------------------------------------------------------------------------------- 1 | package fastparse 2 | 3 | 4 | /** 5 | * Fast, pre-computed character predicates for charactes from 0 to 65535 6 | * 7 | * Useful because FastParse does it's parsing character by character, so 8 | * although this doesn't have the full range of the java 9 | * `Character.getType(c: Int)` functions, it still is good enough for 10 | * a wide range of use cases 11 | */ 12 | object CharPredicates{ 13 | 14 | def isMathSymbol(c: Char) = Character.getType(c) == Character.MATH_SYMBOL 15 | def isOtherSymbol(c: Char) = Character.getType(c) == Character.OTHER_SYMBOL 16 | def isLetter(c: Char) = { 17 | ((((1 << Character.UPPERCASE_LETTER) | 18 | (1 << Character.LOWERCASE_LETTER) | 19 | (1 << Character.TITLECASE_LETTER) | 20 | (1 << Character.MODIFIER_LETTER) | 21 | (1 << Character.OTHER_LETTER)) >> Character.getType(c)) & 1) != 0 22 | } 23 | 24 | def isPrintableChar(c: Char) = { 25 | val block = java.lang.Character.UnicodeBlock.of(c) 26 | !java.lang.Character.isISOControl(c) && 27 | !java.lang.Character.isSurrogate(c) && 28 | block != null && 29 | block != java.lang.Character.UnicodeBlock.SPECIALS 30 | } 31 | def isDigit(c: Char) = Character.isDigit(c) 32 | def isLower(c: Char) = Character.isLowerCase((c)) 33 | def isUpper(c: Char) = Character.isUpperCase(c) 34 | } -------------------------------------------------------------------------------- /fastparse/src-native/fastparse/CharPredicates.scala: -------------------------------------------------------------------------------- 1 | package fastparse 2 | 3 | /** 4 | * Fast, pre-computed character predicates for charactes from 0 to 65535 5 | * 6 | * Useful because FastParse does it's parsing character by character, so 7 | * although this doesn't have the full range of the java 8 | * `Character.getType(c: Int)` functions, it still is good enough for 9 | * a wide range of use cases 10 | */ 11 | object CharPredicates{ 12 | 13 | def isMathSymbol(c: Char) = Character.getType(c) == Character.MATH_SYMBOL 14 | def isOtherSymbol(c: Char) = Character.getType(c) == Character.OTHER_SYMBOL 15 | def isLetter(c: Char) = { 16 | ((((1 << Character.UPPERCASE_LETTER) | 17 | (1 << Character.LOWERCASE_LETTER) | 18 | (1 << Character.TITLECASE_LETTER) | 19 | (1 << Character.MODIFIER_LETTER) | 20 | (1 << Character.OTHER_LETTER)) >> Character.getType(c)) & 1) != 0 21 | } 22 | 23 | def isPrintableChar(c: Char) = { 24 | // Don't bother checking for Unicode SPECIAL block characters 25 | // in Scala.js because Scala.js doesn't really support it 26 | !java.lang.Character.isISOControl(c) && 27 | !java.lang.Character.isSurrogate(c) 28 | } 29 | def isDigit(c: Char) = Character.isDigit(c) 30 | def isLower(c: Char) = Character.isLowerCase((c)) 31 | def isUpper(c: Char) = Character.isUpperCase(c) 32 | } -------------------------------------------------------------------------------- /fastparse/src/fastparse/Implicits.scala: -------------------------------------------------------------------------------- 1 | package fastparse 2 | 3 | import scala.collection.mutable 4 | 5 | /** 6 | * Container for all the type-level logic around appending things 7 | * to tuples or flattening `Seq[Unit]`s into `Unit`s. 8 | * 9 | * Some of these implicits make liberal use of mutable state, so as 10 | * to minimize allocations while parsing. 11 | */ 12 | object Implicits { 13 | trait Sequencer[-T, V, R]{ 14 | def apply(t: T, v: V): R 15 | } 16 | object Sequencer extends LowPriSequencer{ 17 | class NarySequencer[T, V, R](f: (T, V) => R) extends Sequencer[T, V, R]{ 18 | def apply(t: T, v: V): R = f(t, v) 19 | } 20 | implicit def SingleSequencer[T]: Sequencer[Unit, T, T] = SingleSequencer0.asInstanceOf[Sequencer[Unit, T, T]] 21 | object SingleSequencer0 extends Sequencer[Unit, Any, Any]{ 22 | def apply(t: Unit, v: Any): Any = v 23 | } 24 | } 25 | trait LowPriSequencer extends LowerPriSequencer{ 26 | object UnitSequencer0 extends Sequencer[Any, Unit, Any]{ 27 | def apply(t: Any, v: Unit): Any = t 28 | } 29 | implicit def UnitSequencer[T]: Sequencer[T, Unit, T] = UnitSequencer0.asInstanceOf[Sequencer[T, Unit, T]] 30 | } 31 | trait LowerPriSequencer extends SequencerGen[Sequencer]{ 32 | protected[this] def Sequencer0[A, B, C](f: (A, B) => C) = new Sequencer.NarySequencer(f) 33 | } 34 | trait Repeater[-T, R]{ 35 | type Acc 36 | def initial: Acc 37 | def accumulate(t: T, acc: Acc): Unit 38 | def result(acc: Acc): R 39 | } 40 | object Repeater extends LowPriRepeater{ 41 | implicit object UnitRepeater extends Repeater[Unit, Unit]{ 42 | type Acc = Unit 43 | def initial = () 44 | def accumulate(t: Unit, acc: Unit) = acc 45 | def result(acc: Unit) = () 46 | } 47 | } 48 | trait LowPriRepeater{ 49 | implicit def GenericRepeaterImplicit[T]: Repeater[T, Seq[T]] = GenericRepeatedImplicit0.asInstanceOf[Repeater[T, Seq[T]]] 50 | object GenericRepeatedImplicit0 extends Repeater[Any, Seq[Any]]{ 51 | type Acc = mutable.Buffer[Any] 52 | def initial = mutable.Buffer.empty[Any] 53 | def accumulate(t: Any, acc: mutable.Buffer[Any]) = acc += t 54 | def result(acc: mutable.Buffer[Any]) = acc.toSeq 55 | } 56 | } 57 | 58 | trait Optioner[-T, R]{ 59 | def none: R 60 | def some(value: T): R 61 | } 62 | 63 | object Optioner extends LowPriOptioner{ 64 | implicit object UnitOptioner extends Optioner[Unit, Unit]{ 65 | def none = () 66 | def some(value: Unit) = () 67 | } 68 | } 69 | trait LowPriOptioner{ 70 | implicit def GenericOptionerImplicit[T]: Optioner[T, Option[T]] = GenericOptionerImplicit0.asInstanceOf[Optioner[T, Option[T]]] 71 | object GenericOptionerImplicit0 extends Optioner[Any, Option[Any]]{ 72 | def none = None 73 | def some(value: Any) = Some(value) 74 | } 75 | } 76 | } -------------------------------------------------------------------------------- /fastparse/src/fastparse/internal/UberBuffer.scala: -------------------------------------------------------------------------------- 1 | package fastparse.internal 2 | 3 | 4 | /** 5 | * A very fast circular, growable read-write byte buffer. 6 | */ 7 | class UberBuffer(initSize: Int = 32){ self => 8 | private[this] var data = new Array[Char](initSize) 9 | private[this] var readPos = 0 10 | private[this] var writePos = 0 11 | 12 | def capacity = data.length 13 | def apply(index: Int) = { 14 | if (index < 0 || index >= length) throw new IndexOutOfBoundsException( 15 | s"UberBuffer index $index must be between 0 and $length" 16 | ) 17 | data((index + readPos) % capacity) 18 | } 19 | def length = { 20 | if (writePos == readPos){ 21 | 0 22 | }else if(writePos > readPos){ 23 | // 1 2 3 24 | // - - - - - 25 | // r W 26 | // R 27 | writePos - readPos 28 | } else { 29 | // 3 4 1 2 30 | // - - - - - 31 | // W r 32 | // Rs 33 | data.length - readPos + writePos 34 | } 35 | } 36 | private[this] def writeAvailable = { 37 | if (writePos == readPos){ 38 | data.length - 1 39 | }else if (writePos > readPos){ 40 | // 1 2 3 4 41 | // - - - - - 42 | // W R w 43 | data.length - writePos - 1 + readPos 44 | }else{ 45 | // 1 46 | // - - - - - 47 | // w W R 48 | readPos - writePos - 1 49 | } 50 | } 51 | 52 | private[this] def expand() = { 53 | val newData = new Array[Char](data.length * 2) 54 | 55 | if (readPos <= writePos){ 56 | System.arraycopy(data, readPos, newData, 0, writePos - readPos) 57 | writePos = writePos - readPos 58 | }else{ 59 | System.arraycopy(data, readPos, newData, 0, data.length - readPos) 60 | System.arraycopy(data, 0, newData, data.length - readPos, writePos) 61 | writePos = writePos + data.length - readPos 62 | } 63 | readPos = 0 64 | 65 | data = newData 66 | } 67 | 68 | def write(in: Array[Char], length0: Int) = { 69 | while (writeAvailable < length0) expand() 70 | 71 | if (writePos + length0 <= data.length){ 72 | System.arraycopy(in, 0, data, writePos, length0) 73 | }else{ 74 | val firstHalfLength = data.length - writePos 75 | System.arraycopy(in, 0, data, writePos, firstHalfLength) 76 | System.arraycopy(in, firstHalfLength, data, 0, length0 - firstHalfLength) 77 | } 78 | 79 | writePos = incr(writePos, length0) 80 | } 81 | 82 | 83 | def slice(start: Int, end: Int) = { 84 | assert(end >= start, s"end:$end must be >= start:$start") 85 | val startClamped = math.max(0, start) 86 | val endClamped = math.min(length, end) 87 | val actualStart = (readPos + startClamped) % data.length 88 | val actualEnd = (readPos + endClamped) % data.length 89 | val output = new Array[Char](endClamped - startClamped) 90 | if (actualEnd >= actualStart){ 91 | System.arraycopy(data, actualStart, output, 0, actualEnd - actualStart) 92 | }else{ 93 | System.arraycopy(data, actualStart, output, 0, data.length - actualStart) 94 | System.arraycopy(data, 0, output, data.length - actualStart, actualEnd) 95 | } 96 | output 97 | } 98 | def drop(n: Int) = { 99 | readPos = incr(readPos, n) 100 | } 101 | 102 | private[this] def incr(n: Int, d: Long) = { 103 | ((n + d) % data.length).toInt 104 | } 105 | } -------------------------------------------------------------------------------- /fastparse/src/fastparse/internal/Util.scala: -------------------------------------------------------------------------------- 1 | package fastparse.internal 2 | 3 | import fastparse.{ParserInput, ParsingRun} 4 | 5 | import scala.annotation.{switch, tailrec} 6 | import scala.collection.mutable.ArrayBuffer 7 | 8 | object Util { 9 | def parenthize(fs: List[Lazy[String]]) = fs.reverseIterator.map(_()).toSeq.distinct match{ 10 | case Seq(x) => x 11 | case xs => xs.mkString("(", " | ", ")") 12 | } 13 | def joinBinOp(lhs: Msgs, rhs: Msgs): Msgs = { 14 | if (lhs.value.isEmpty) rhs 15 | else if (rhs.value.isEmpty) lhs 16 | else Msgs.fromFunction(() => lhs.render + " ~ " + rhs.render) 17 | } 18 | 19 | def consumeWhitespace[V](whitespace: fastparse.Whitespace, ctx: ParsingRun[Any]) = { 20 | val oldCapturing = ctx.noDropBuffer // completely disallow dropBuffer 21 | ctx.noDropBuffer = true 22 | whitespace(ctx) 23 | ctx.noDropBuffer = oldCapturing 24 | } 25 | 26 | def startsWith(src: ParserInput, prefix: String, offset: Int) = { 27 | @tailrec def rec(i: Int): Boolean = { 28 | if (i >= prefix.length) true 29 | else if (!src.isReachable(i + offset)) false 30 | else if (src(i + offset) != prefix.charAt(i)) false 31 | else rec(i + 1) 32 | } 33 | rec(0) 34 | } 35 | 36 | def startsWithIgnoreCase(src: ParserInput, prefix: IndexedSeq[Char], offset: Int) = { 37 | @tailrec def rec(i: Int): Boolean = { 38 | if (i >= prefix.length) true 39 | else if(!src.isReachable(i + offset)) false 40 | else { 41 | val c1: Char = src(i + offset) 42 | val c2: Char = prefix(i) 43 | if (c1 != c2 && c1.toLower != c2.toLower) false 44 | else rec(i + 1) 45 | } 46 | } 47 | rec(0) 48 | } 49 | def lineNumberLookup(data: String): Array[Int] = { 50 | val lineStarts = ArrayBuffer[Int](0) 51 | var i = 0 52 | var col = 1 53 | // Stores the previous char we saw, or -1 if we just saw a \r\n or \n\r pair 54 | var state: Int = 0 55 | while (i < data.length){ 56 | val char = data(i) 57 | if (char == '\r' && state == '\n' || char == '\n' && state == '\r'){ 58 | col += 1 59 | state = -1 60 | } else if (state == '\r' || state == '\n' || state == -1) { 61 | lineStarts.append(i) 62 | col = 1 63 | state = char 64 | }else{ 65 | col += 1 66 | state = char 67 | } 68 | 69 | i += 1 70 | } 71 | 72 | if (state == '\r' || state == '\n' || state == -1) { 73 | lineStarts.append(i) 74 | } 75 | 76 | lineStarts.toArray 77 | } 78 | 79 | def literalize(s: IndexedSeq[Char], unicode: Boolean = false) = { 80 | val sb = new StringBuilder 81 | sb.append('"') 82 | var i = 0 83 | val len = s.length 84 | while (i < len) { 85 | (s(i): @switch) match { 86 | case '"' => sb.append("\\\"") 87 | case '\\' => sb.append("\\\\") 88 | case '\b' => sb.append("\\b") 89 | case '\f' => sb.append("\\f") 90 | case '\n' => sb.append("\\n") 91 | case '\r' => sb.append("\\r") 92 | case '\t' => sb.append("\\t") 93 | case c => 94 | if (c < ' ' || (c > '~' && unicode)) sb.append("\\u%04x" format c.toInt) 95 | else sb.append(c) 96 | } 97 | i += 1 98 | } 99 | sb.append('"') 100 | 101 | sb.result() 102 | } 103 | 104 | 105 | def reportParseMsgPostSep(startIndex: Int, 106 | min: Int, 107 | ctx: ParsingRun[Any], 108 | parsedMsg: Msgs, 109 | lastAgg: Msgs) = { 110 | reportParseMsgInRep(startIndex, min, ctx, null, parsedMsg, lastAgg, true) 111 | } 112 | 113 | def reportParseMsgInRep(startIndex: Int, 114 | min: Int, 115 | ctx: ParsingRun[Any], 116 | sepMsg: Msgs, 117 | parsedMsg: Msgs, 118 | lastAgg: Msgs, 119 | precut: Boolean) = { 120 | 121 | // When we fail on a rep body, we collect both the concatenated 122 | // sep and failure aggregate of the rep body that we tried (because 123 | // we backtrack past the sep on failure) as well as the failure 124 | // aggregate of the previous rep, which we could have continued 125 | val newAgg = 126 | if (sepMsg == null || precut) ctx.aggregateMsgs 127 | else Util.joinBinOp(sepMsg, parsedMsg) 128 | 129 | ctx.reportAggregateMsg( 130 | () => parsedMsg.render + ".rep" + (if (min == 0) "" else s"(${min})"), 131 | if (lastAgg == null) newAgg 132 | else newAgg ::: lastAgg 133 | ) 134 | } 135 | } 136 | 137 | class Lazy[T](calc0: () => T){ 138 | lazy val force = calc0() 139 | def apply(): T = force 140 | } 141 | 142 | case class Logger(f: String => Unit) 143 | object Logger { 144 | implicit val stdout: Logger = Logger(println) 145 | } 146 | 147 | trait Instrument{ 148 | def beforeParse(parser: String, index: Int): Unit 149 | def afterParse(parser: String, index: Int, success: Boolean): Unit 150 | } 151 | 152 | 153 | final class TrieNode(strings: Seq[String], ignoreCase: Boolean = false) { 154 | 155 | val ignoreCaseStrings = if (ignoreCase) strings.map(_.map(_.toLower)) else strings 156 | val children = ignoreCaseStrings.filter(!_.isEmpty) 157 | .groupBy(_(0)) 158 | .map { case (k,ss) => k -> new TrieNode(ss.map(_.tail), ignoreCase) } 159 | 160 | val rawSize = children.values.map(_.size).sum + children.size 161 | 162 | val break = rawSize >= 8 163 | val size: Int = if (break) 1 else rawSize 164 | val word: Boolean = strings.exists(_.isEmpty) || children.isEmpty 165 | } 166 | final class CompactTrieNode(source: TrieNode){ 167 | val children: Map[Char, (String, CompactTrieNode)] = { 168 | val iter = for(char <- source.children.keys) yield char -> { 169 | val string = new StringBuilder 170 | var child = source.children(char) 171 | while(!child.word && child.children.size == 1){ 172 | string.append(child.children.keys.head) 173 | child = child.children.values.head 174 | } 175 | (string.toString(), new CompactTrieNode(child)) 176 | } 177 | iter.toMap 178 | } 179 | 180 | 181 | val word = source.word 182 | } 183 | object Msgs{ 184 | val empty = Msgs(Nil) 185 | implicit def fromFunction(msgToSet: () => String): Msgs = { 186 | Msgs(new Lazy(() => msgToSet()):: Nil) 187 | } 188 | implicit def fromStrings(msgsToSet: List[String]): Msgs = { 189 | Msgs(msgsToSet.map(s => new Lazy(() => s))) 190 | } 191 | } 192 | 193 | case class Msgs(value: List[Lazy[String]]){ 194 | def :::(other: Msgs) = Msgs(other.value ::: value) 195 | def ::(other: Lazy[String]) = Msgs(other :: value) 196 | override def toString = render 197 | def render = Util.parenthize(value) 198 | } -------------------------------------------------------------------------------- /fastparse/test/src-2.12+/fastparse/CustomWhitespaceMathTests.scala: -------------------------------------------------------------------------------- 1 | package test.fastparse 2 | 3 | import fastparse._ 4 | import utest._ 5 | 6 | /** 7 | * Same as MathTests, but demonstrating the use of whitespace 8 | */ 9 | object CustomWhitespaceMathTests extends TestSuite{ 10 | implicit object whitespace extends Whitespace{ 11 | def apply(ctx: fastparse.ParsingRun[_]): P[Unit] = { 12 | implicit val ctx0 = ctx 13 | CharsWhileIn(" \t", 0) 14 | } 15 | } 16 | def eval(tree: (Int, Seq[(String, Int)])): Int = { 17 | val (base, ops) = tree 18 | ops.foldLeft(base){ case (left, (op, right)) => op match{ 19 | case "+" => left + right case "-" => left - right 20 | case "*" => left * right case "/" => left / right 21 | }} 22 | } 23 | def number[$: P]: P[Int] = P( CharIn("0-9").rep(1).!.map(_.toInt) ) 24 | def parens[$: P]: P[Int] = P( "(" ~/ addSub ~ ")" ) 25 | def factor[$: P]: P[Int] = P( number | parens ) 26 | 27 | def divMul[$: P]: P[Int] = P( factor ~ (CharIn("*/").! ~/ factor).rep ).map(eval) 28 | def addSub[$: P]: P[Int] = P( divMul ~ (CharIn("+\\-").! ~/ divMul).rep ).map(eval) 29 | def expr[$: P]: P[Int] = P( " ".rep ~ addSub ~ " ".rep ~ End ) 30 | 31 | val tests = Tests { 32 | test("pass"){ 33 | def check(str: String, num: Int) = { 34 | val Parsed.Success(value, _) = parse(str, expr(_)) 35 | assert(value == num) 36 | } 37 | 38 | test - check("1+1", 2) 39 | test - check("1+ 1* 2", 3) 40 | test - check("(1+ 1 * 2)+( 3*4*5)", 63) 41 | test - check("15/3", 5) 42 | test - check("63 /3", 21) 43 | test - check("(1+ 1*2)+(3 *4*5)/20", 6) 44 | test - check("((1+ 1*2)+(3*4*5))/3", 21) 45 | } 46 | test("fail"){ 47 | def check(input: String, expectedTrace: String) = { 48 | val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] 49 | val actualTrace = failure.trace().longAggregateMsg 50 | assert(expectedTrace.trim == actualTrace.trim) 51 | } 52 | test - check( 53 | "( + )", 54 | """ Expected expr:1:1 / addSub:1:1 / divMul:1:1 / factor:1:1 / parens:1:1 / addSub:1:4 / divMul:1:4 / factor:1:4 / (number | parens):1:4, found "+ )" """ 55 | ) 56 | test - check( 57 | "1 + - ", 58 | """ Expected expr:1:1 / addSub:1:1 / divMul:1:7 / factor:1:7 / (number | parens):1:7, found "- " """ 59 | ) 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /fastparse/test/src/fastparse/GnipSubSyntaxTest.scala: -------------------------------------------------------------------------------- 1 | package test.fastparse 2 | 3 | import utest._ 4 | import fastparse._ 5 | 6 | import scala.language.postfixOps 7 | 8 | /** 9 | * This is a regression test for the pathological behavior (#77) where 10 | * `traceParsers` grows exponentially in cases with lots of backtracking. 11 | * 12 | * Without that issue being fixed, the this test runs forever and never 13 | * terminates. With that issue fixed, it should complete trivially quickly. 14 | */ 15 | object GnipSubSyntaxTest extends TestSuite { 16 | object GnipRuleParser { 17 | import SingleLineWhitespace._ 18 | 19 | def keyword[$: P] = P(CharIn("a-z")!) 20 | def maybeNegatedKeyword[$: P] = P((("-"?) ~~ keyword)!) 21 | 22 | def keywordGroupWithoutOrClause[$: P] = P((maybeNegatedKeyword | (("-"?) ~~ keywordsInParentheses))!) 23 | def keywordGroup[$: P] = P(orClause | keywordGroupWithoutOrClause) 24 | 25 | def keywordsInParentheses[$: P] = P("(" ~ gnipKeywordPhrase ~ ")") 26 | def orClause[$: P] = P(!(("-" ~~ keywordGroupWithoutOrClause.rep(1)) ~ "OR") ~ keywordGroupWithoutOrClause ~ ("OR"!) ~ gnipKeywordPhrase) 27 | def gnipKeywordPhrase[$: P]: P[String] = P(keywordGroup.rep(1))! 28 | 29 | def parse[$: P] = P(Start ~ gnipKeywordPhrase ~ End) 30 | } 31 | 32 | object GnipRuleValidator { 33 | 34 | def apply(rule: String) = parse(rule, GnipRuleParser.parse(_)) 35 | } 36 | 37 | val tests = Tests { 38 | test("fail"){ 39 | val res = GnipRuleValidator("( ab ( cd ( ef ( gh ( ij ( ( hello ( world ) bla ) lol ) hehe ) ) ) xz )") 40 | assert(!res.isSuccess) 41 | } 42 | 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /fastparse/test/src/fastparse/IndentationTests.scala: -------------------------------------------------------------------------------- 1 | package test.fastparse 2 | 3 | import utest._ 4 | import fastparse._ 5 | 6 | /** 7 | * Same as MathTests, but demonstrating the use of whitespace 8 | */ 9 | object IndentationTests extends TestSuite{ 10 | import fastparse.NoWhitespace._ 11 | def eval(tree: (String, Seq[Int])) = tree match{ 12 | case ("+", nums) => nums.reduceLeft(_+_) 13 | case ("-", nums) => nums.reduceLeft(_-_) 14 | case ("*", nums) => nums.reduceLeft(_*_) 15 | case ("/", nums) => nums.reduceLeft(_/_) 16 | } 17 | 18 | /** 19 | * Parser for an indentation-based math syntax. Parens are no longer 20 | * necessary, and the whole parser is parametrized with the current 21 | * depth of indentation 22 | */ 23 | class Parser(indent: Int){ 24 | def number[$: P]: P[Int] = P( CharIn("0-9").rep(1).!.map(_.toInt) ) 25 | 26 | def deeper[$: P]: P[Int] = P( " ".rep(indent + 1).!.map(_.length) ) 27 | def blockBody[$: P]: P[Seq[Int]] = "\n" ~ deeper.flatMapX(i => 28 | new Parser(indent = i).factor.rep(1, sep = ("\n" + " " * i)./) 29 | ) 30 | def block[$: P]: P[Int] = P( CharIn("+\\-*/").! ~/ blockBody).map(eval) 31 | 32 | def factor[$: P]: P[Int] = P( number | block ) 33 | 34 | def expr[$: P]: P[Int] = P( block ~ End ) 35 | } 36 | def expr[$: P] = new Parser(indent = 0).expr 37 | 38 | val tests = Tests { 39 | test("pass"){ 40 | def check(str: String, num: Int) = { 41 | val Parsed.Success(value, _) = parse(str, expr(_)) 42 | assert(value == num) 43 | } 44 | 45 | check( 46 | """+ 47 | | 1 48 | | 1 49 | """.stripMargin.trim, 50 | 2 51 | ) 52 | 53 | check( 54 | """+ 55 | | 1 56 | | * 57 | | 1 58 | | 2 59 | """.stripMargin.trim, 60 | 3 61 | ) 62 | 63 | check( 64 | """+ 65 | | + 66 | | 1 67 | | * 68 | | 1 69 | | 2 70 | | * 71 | | 3 72 | | 4 73 | | 5 74 | | 75 | """.stripMargin.trim, 76 | 63 77 | ) 78 | check( 79 | """/ 80 | | 15 81 | | 3 82 | """.stripMargin.trim, 83 | 5 84 | ) 85 | check( 86 | """/ 87 | | 63 88 | | 3 89 | """.stripMargin.trim, 90 | 21 91 | ) 92 | check( 93 | """+ 94 | | + 95 | | 1 96 | | * 97 | | 1 98 | | 2 99 | | / 100 | | * 101 | | 3 102 | | 4 103 | | 5 104 | | 20 105 | """.stripMargin.trim, 106 | 6 107 | ) 108 | check( 109 | """/ 110 | | + 111 | | + 112 | | 1 113 | | * 114 | | 1 115 | | 2 116 | | * 117 | | 3 118 | | 4 119 | | 5 120 | | 3 121 | """.stripMargin.trim, 122 | 21 123 | ) 124 | } 125 | test("fail"){ 126 | def check(input: String, expectedTrace: String): Unit = { 127 | val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] 128 | val actualTrace = failure.trace(enableLogging = true).longAggregateMsg 129 | assert(expectedTrace.trim == actualTrace.trim) 130 | } 131 | test - check( 132 | "+", 133 | """ Expected expr:1:1 / block:1:1 / "\n":1:2, found "" """ 134 | ) 135 | test - check( 136 | """+ 137 | | 1 138 | |1 139 | """.stripMargin.trim, 140 | """ Expected expr:1:1 / ([0-9] | "\n " | end-of-input):2:4, found "\n1" """ 141 | ) 142 | test - check( 143 | """+ 144 | | 1 145 | | 1 146 | """.stripMargin.trim, 147 | """ Expected expr:1:1 / block:1:1 / factor:3:3 / (number | block):3:3, found " 1" """ 148 | ) 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /fastparse/test/src/fastparse/Main.scala: -------------------------------------------------------------------------------- 1 | package test.fastparse 2 | 3 | object Main { 4 | def main(args: Array[String]): Unit = { 5 | import fastparse._, NoWhitespace._ 6 | def iam[$: P] = P( "i am" ) 7 | def hello[$: P] = P( "hello" ) 8 | def combined[$: P] = P( (iam | hello).? ~ ("cow" | "world") ) 9 | val Parsed.Failure(_, _, extra) = parse("lol", combined(_)) 10 | println(extra.trace().longAggregateMsg) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /fastparse/test/src/fastparse/MathTests.scala: -------------------------------------------------------------------------------- 1 | package test.fastparse 2 | 3 | import fastparse.internal.Instrument 4 | import utest._ 5 | 6 | import scala.collection.mutable 7 | 8 | /** 9 | * Demonstrates simultaneously parsing and 10 | * evaluating simple arithmetic expressions 11 | */ 12 | object MathTests extends TestSuite{ 13 | 14 | def eval(tree: (Int, Seq[(String, Int)])) = { 15 | val (base, ops) = tree 16 | ops.foldLeft(base){ case (left, (op, right)) => op match{ 17 | case "+" => left + right case "-" => left - right 18 | case "*" => left * right case "/" => left / right 19 | }} 20 | } 21 | import fastparse._, NoWhitespace._ 22 | def number[$: P]: P[Int] = P( CharIn("0-9").rep(1).!.map(_.toInt) ) 23 | def parens[$: P]: P[Int] = P( "(" ~/ addSub ~ ")" ) 24 | def factor[$: P]: P[Int] = P( number | parens ) 25 | 26 | def divMul[$: P]: P[Int] = P( factor ~ (CharIn("*/").! ~/ factor).rep ).map(eval) 27 | def addSub[$: P]: P[Int] = P( divMul ~ (CharIn("+\\-").! ~/ divMul).rep ).map(eval) 28 | def expr[$: P]: P[Int] = P( addSub ~ End ) 29 | 30 | val tests = Tests { 31 | test("pass"){ 32 | val Parsed.Success(2, _) = parse("1+1", expr(_)) 33 | val Parsed.Success(15, _) = parse("(1+1*2)+3*4", expr(_)) 34 | val Parsed.Success(21, _) = parse("((1+1*2)+(3*4*5))/3", expr(_)) 35 | val Parsed.Failure(expected, failIndex, extra) = parse("1+1*", expr(_)) 36 | val longAggMsg = extra.trace().longAggregateMsg 37 | assert( 38 | failIndex == 4, 39 | longAggMsg == 40 | """Expected expr:1:1 / addSub:1:1 / divMul:1:3 / factor:1:5 / (number | parens):1:5, found """"" 41 | ) 42 | } 43 | test("fail"){ 44 | def check(input: String, 45 | expectedTrace: String, 46 | expectedShortTrace: String, 47 | expectedTerminalTrace: String) = { 48 | val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] 49 | val trace = failure.trace() 50 | val index = failure.index 51 | 52 | assert( 53 | expectedTrace.trim == trace.longAggregateMsg.trim, 54 | expectedTerminalTrace.trim == trace.longTerminalsMsg.trim, 55 | expectedShortTrace.trim == failure.msg 56 | ) 57 | 58 | // Check iterator parsing results in a failure in the right place. Note 59 | // that we aren't checking the `.traced.trace` because that requires a 60 | // second parse which doesn't work with iterators (which get exhausted) 61 | for(chunkSize <- Seq(1, 4, 16, 64, 256, 1024)) { 62 | val failure2 = parse(input.grouped(chunkSize), expr(_)).asInstanceOf[Parsed.Failure] 63 | assert(failure2.index == index) 64 | } 65 | } 66 | check( 67 | "(+)", 68 | """Expected expr:1:1 / addSub:1:1 / divMul:1:1 / factor:1:1 / parens:1:1 / addSub:1:2 / divMul:1:2 / factor:1:2 / (number | parens):1:2, found "+)"""", 69 | """Position 1:2, found "+)"""", 70 | """Expected expr:1:1 / addSub:1:1 / divMul:1:1 / factor:1:1 / parens:1:1 / addSub:1:2 / divMul:1:2 / factor:1:2 / ([0-9] | "("):1:2, found "+)"""" 71 | ) 72 | check( 73 | "1+-", 74 | """Expected expr:1:1 / addSub:1:1 / divMul:1:3 / factor:1:3 / (number | parens):1:3, found "-"""", 75 | """Position 1:3, found "-"""", 76 | """Expected expr:1:1 / addSub:1:1 / divMul:1:3 / factor:1:3 / ([0-9] | "("):1:3, found "-"""" 77 | ) 78 | check( 79 | "(2+3x)", 80 | """Expected expr:1:1 / addSub:1:1 / divMul:1:1 / factor:1:1 / parens:1:1 / ([0-9] | [*/] | [+\\-] | ")"):1:5, found "x)"""", 81 | """Position 1:5, found "x)"""", 82 | """Expected expr:1:1 / addSub:1:1 / divMul:1:1 / factor:1:1 / parens:1:1 / ([0-9] | [*/] | [+\\-] | ")"):1:5, found "x)"""" 83 | ) 84 | check( 85 | "(1+(2+3x))+4", 86 | """Expected expr:1:1 / addSub:1:1 / divMul:1:1 / factor:1:1 / parens:1:1 / addSub:1:2 / divMul:1:4 / factor:1:4 / parens:1:4 / ([0-9] | [*/] | [+\\-] | ")"):1:8, found "x))+4"""", 87 | """Position 1:8, found "x))+4"""", 88 | """Expected expr:1:1 / addSub:1:1 / divMul:1:1 / factor:1:1 / parens:1:1 / addSub:1:2 / divMul:1:4 / factor:1:4 / parens:1:4 / ([0-9] | [*/] | [+\\-] | ")"):1:8, found "x))+4"""" 89 | ) 90 | } 91 | 92 | test("instrument"){ 93 | test("simple"){ 94 | val callCount = mutable.Map.empty[String, Int] 95 | 96 | 97 | val instrument = new Instrument { 98 | def beforeParse(parser: String, index: Int): Unit = { 99 | callCount(parser) = callCount.getOrElse(parser, 0) + 1 100 | } 101 | def afterParse(parser: String, index: Int, success: Boolean): Unit = () 102 | } 103 | 104 | parse("((1+1*2)+(3*4*5))/3", expr(_), instrument = instrument) 105 | 106 | val expectedCallCount = Map( 107 | "expr" -> 1, 108 | "addSub" -> 4, 109 | "divMul" -> 6, 110 | "factor" -> 10, 111 | "number" -> 10, 112 | "parens" -> 3 113 | ) 114 | assert(callCount == expectedCallCount) 115 | } 116 | test("continuation"){ 117 | val resultCount = mutable.Map.empty[(String, Boolean), Int] 118 | val instrument = new Instrument { 119 | def beforeParse(parser: String, index: Int): Unit = () 120 | def afterParse(parser: String, index: Int, success: Boolean): Unit = { 121 | val resultKey = (parser, success) 122 | resultCount(resultKey) = resultCount.getOrElse(resultKey, 0) + 1 123 | } 124 | } 125 | 126 | // Good Parse 127 | parse("((1+1*2)+(3*4*5))/3", expr(_), instrument = instrument) 128 | 129 | val expectedResultCount = Map( 130 | ("expr", true) -> 1, 131 | ("addSub", true) -> 4, 132 | ("divMul", true) -> 6, 133 | ("factor", true) -> 10, 134 | ("number", true) -> 7, 135 | ("number", false) -> 3, 136 | ("parens", true) -> 3 137 | ) 138 | assert(resultCount == expectedResultCount) 139 | 140 | // Bad Parse 141 | resultCount.clear() 142 | parse("((1+1*2)+(3*4*))/3", expr(_), instrument = instrument) 143 | 144 | val expectedResultCount2 = Map( 145 | ("expr", false) -> 1, 146 | 147 | ("addSub", true) -> 1, 148 | ("addSub", false) -> 3, 149 | 150 | ("divMul", true) -> 3, 151 | ("divMul", false) -> 3, 152 | 153 | ("factor", true) -> 6, 154 | ("factor", false) -> 3, 155 | 156 | ("number", true) -> 5, 157 | ("number", false) -> 4, 158 | 159 | ("parens", true) -> 1, 160 | ("parens", false) -> 3 161 | ) 162 | assert(resultCount == expectedResultCount2) 163 | } 164 | 165 | } 166 | 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /fastparse/test/src/fastparse/TypeTests.scala: -------------------------------------------------------------------------------- 1 | 2 | package test.fastparse 3 | 4 | import fastparse.Implicits 5 | 6 | import scala.annotation.nowarn 7 | 8 | /** 9 | * Make sure the type-level logic does the right thing. Doesn't 10 | * actually need to execute; compiling is enough! 11 | */ 12 | object TypeTests { 13 | class P[T]{ 14 | def ~[V, R](other: P[V])(implicit @nowarn("msg=never used") sum: Implicits.Sequencer[T, V, R]): P[R] = new P[R] 15 | def rep[R](implicit @nowarn("msg=never used") rep: Implicits.Repeater[T, R]): P[R] = new P[R] 16 | def ?[R](implicit @nowarn("msg=never used") rep: Implicits.Optioner[T, R]): P[R] = new P[R] 17 | } 18 | 19 | def P[T] = new P[T] 20 | val p1: P[Unit] = P[Unit] ~ P[Unit] 21 | val p2: P[Int] = P[Unit] ~ P[Int] 22 | val p3: P[Int] = P[Int] ~ P[Unit] 23 | val p4: P[(Char, Int)] = P[Char] ~ P[Int] 24 | val p5: P[(Char, Int, String)] = P[Char] ~ P[Int] ~ P[String] 25 | val p6: P[(Char, Int, String)] = P[Char] ~ P[Int] ~ P[Unit] ~ P[String] ~ P[Unit] 26 | val p7: P[(Char, Int, String, (Int, Int))] = P[Char] ~ P[Int] ~ P[Unit] ~ P[String] ~ P[Unit] ~ P[(Int, Int)] 27 | class R[T] 28 | val p8: P[Unit] = P[Unit].rep 29 | val p9: P[Seq[Int]] = P[Int].rep 30 | val p10: P[Unit] = P[Unit].? 31 | val p11: P[Option[Int]] = P[Int].? 32 | } -------------------------------------------------------------------------------- /fastparse/test/src/fastparse/UtilTests.scala: -------------------------------------------------------------------------------- 1 | package fastparse 2 | import utest._ 3 | object UtilTests extends TestSuite { 4 | val tests = Tests{ 5 | test("hello1"){ 6 | val shortTxt = 7 | """' 8 | |""".stripMargin 9 | 10 | val lineStarts = fastparse.internal.Util.lineNumberLookup(shortTxt) 11 | lineStarts.toList ==> List(0, 2) 12 | val input = IndexedParserInput(shortTxt) 13 | 14 | val pretties = for(i <- 0 to shortTxt.length) yield input.prettyIndex(i) 15 | val expected = Vector( 16 | "1:1", "1:2", 17 | "2:1" 18 | ) 19 | assert(pretties == expected) 20 | } 21 | test("hello2"){ 22 | val txt = 23 | """a 24 | |bc 25 | |def 26 | | 27 | |ghij 28 | |lmnop""".stripMargin 29 | 30 | val lineStarts = fastparse.internal.Util.lineNumberLookup(txt) 31 | lineStarts.toList ==> List(0, 2, 5, 9, 10, 15) 32 | val input = IndexedParserInput(txt) 33 | 34 | val pretties = for(i <- 0 to txt.length) yield input.prettyIndex(i) 35 | val expected = Vector( 36 | "1:1", "1:2", 37 | "2:1", "2:2", "2:3", 38 | "3:1", "3:2", "3:3", "3:4", 39 | "4:1", 40 | "5:1", "5:2", "5:3", "5:4", "5:5", 41 | "6:1", "6:2", "6:3", "6:4", "6:5", "6:6" 42 | ) 43 | assert(pretties == expected) 44 | } 45 | 46 | test("unix"){ 47 | val txt = Array( 48 | "def myScalaVersion = \"2.13.2\"\n", 49 | "\n", 50 | "//hello\n", 51 | "println(doesntExis})" 52 | ).mkString 53 | 54 | val lineStarts = fastparse.internal.Util.lineNumberLookup(txt).toList 55 | 56 | assert(lineStarts == List(0, 30, 31, 39)) 57 | } 58 | 59 | test("carriageReturnOnly") { 60 | val txt = Array( 61 | "def myScalaVersion = \"2.13.2\"\r", 62 | "\r", 63 | "//hello\r", 64 | "println(doesntExis})" 65 | ).mkString 66 | 67 | val lineStarts = fastparse.internal.Util.lineNumberLookup(txt).toList 68 | 69 | assert(lineStarts == List(0, 30, 31, 39)) 70 | } 71 | 72 | test("windows"){ 73 | val txt = Array( 74 | "def myScalaVersion = \"2.13.2\"\r\n", 75 | "\r\n", 76 | "//hello\r\n", 77 | "println(doesntExis})" 78 | ).mkString 79 | 80 | val lineStarts = fastparse.internal.Util.lineNumberLookup(txt).toList 81 | 82 | assert(lineStarts == List(0, 31, 33, 42)) 83 | } 84 | test("reverseWindows"){ 85 | val txt = Array( 86 | "def myScalaVersion = \"2.13.2\"\n\r", 87 | "\n\r", 88 | "//hello\n\r", 89 | "println(doesntExis})" 90 | ).mkString 91 | 92 | val lineStarts = fastparse.internal.Util.lineNumberLookup(txt).toList 93 | 94 | assert(lineStarts == List(0, 31, 33, 42)) 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /fastparse/test/src/fastparse/WhitespaceMathTests.scala: -------------------------------------------------------------------------------- 1 | package test.fastparse 2 | 3 | import fastparse._ 4 | import utest._ 5 | 6 | /** 7 | * Same as MathTests, but demonstrating the use of whitespace 8 | */ 9 | object WhitespaceMathTests extends TestSuite{ 10 | import SingleLineWhitespace._ 11 | def eval(tree: (Int, Seq[(String, Int)])): Int = { 12 | val (base, ops) = tree 13 | ops.foldLeft(base){ case (left, (op, right)) => op match{ 14 | case "+" => left + right case "-" => left - right 15 | case "*" => left * right case "/" => left / right 16 | }} 17 | } 18 | def number[$: P]: P[Int] = P( CharIn("0-9").rep(1).!.map(_.toInt) ) 19 | def parens[$: P]: P[Int] = P( "(" ~/ addSub ~ ")" ) 20 | def factor[$: P]: P[Int] = P( number | parens ) 21 | 22 | def divMul[$: P]: P[Int] = P( factor ~ (CharIn("*/").! ~/ factor).rep ).map(eval) 23 | def addSub[$: P]: P[Int] = P( divMul ~ (CharIn("+\\-").! ~/ divMul).rep ).map(eval) 24 | def expr[$: P]: P[Int] = P( " ".rep ~ addSub ~ " ".rep ~ End ) 25 | 26 | val tests = Tests { 27 | test("pass"){ 28 | def check(str: String, num: Int) = { 29 | val Parsed.Success(value, _) = parse(str, expr(_)) 30 | assert(value == num) 31 | } 32 | 33 | test - check("1+1", 2) 34 | test - check("1+ 1* 2", 3) 35 | test - check("(1+ 1 * 2)+( 3*4*5)", 63) 36 | test - check("15/3", 5) 37 | test - check("63 /3", 21) 38 | test - check("(1+ 1*2)+(3 *4*5)/20", 6) 39 | test - check("((1+ 1*2)+(3*4*5))/3", 21) 40 | } 41 | 42 | test("fail"){ 43 | def check(input: String, expectedTrace: String) = { 44 | val failure = parse(input, expr(_)).asInstanceOf[Parsed.Failure] 45 | val actualTrace = failure.trace().longAggregateMsg 46 | assert(expectedTrace.trim == actualTrace.trim) 47 | } 48 | test - check( 49 | "( + )", 50 | """ Expected expr:1:1 / addSub:1:1 / divMul:1:1 / factor:1:1 / parens:1:1 / addSub:1:4 / divMul:1:4 / factor:1:4 / (number | parens):1:4, found "+ )" """ 51 | ) 52 | test - check( 53 | "1 + - ", 54 | """ Expected expr:1:1 / addSub:1:1 / divMul:1:7 / factor:1:7 / (number | parens):1:7, found "- " """ 55 | ) 56 | } 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /fastparse/test/src/fastparse/WhitespaceTests.scala: -------------------------------------------------------------------------------- 1 | package fastparse 2 | 3 | import utest._ 4 | 5 | /** 6 | * Same as MathTests, but demonstrating the use of whitespace 7 | */ 8 | object WhitespaceTests extends TestSuite{ 9 | val tests = Tests { 10 | def checkCommon(p0: Whitespace) = { 11 | val p = p0.apply(_) 12 | val Parsed.Success((), 0) = parse("", p) 13 | val Parsed.Success((), 0) = parse("/", p) 14 | val Parsed.Success((), 1) = parse(" /", p) 15 | val Parsed.Success((), 1) = parse(" / ", p) 16 | val Parsed.Success((), 2) = parse("//", p) 17 | val Parsed.Success((), 3) = parse(" //", p) 18 | val Parsed.Success((), 4) = parse(" // ", p) 19 | val Parsed.Success((), 13) = parse(" // / / // /*", p) 20 | val Parsed.Success((), 4) = parse("/**/", p) 21 | val Parsed.Success((), 5) = parse("/* */", p) 22 | val Parsed.Success((), 6) = parse("/****/", p) 23 | val Parsed.Success((), 9) = parse("/** * **/", p) 24 | val Parsed.Success((), 15) = parse("/** // * // **/", p) 25 | } 26 | test("scala"){ 27 | checkCommon(ScalaWhitespace.whitespace) 28 | // allow nested comments 29 | val Parsed.Failure(_, 11, _) = parse("/** /* /**/", ScalaWhitespace.whitespace.apply(_)) 30 | val Parsed.Success((), 8) = parse("/*/**/*/", ScalaWhitespace.whitespace.apply(_)) 31 | } 32 | test("java"){ 33 | checkCommon(JavaWhitespace.whitespace) 34 | // no nested comments 35 | val Parsed.Success((), 11) = parse("/** /* /**/", JavaWhitespace.whitespace.apply(_)) 36 | val Parsed.Success((), 6) = parse("/*/**/*/", JavaWhitespace.whitespace.apply(_)) 37 | } 38 | test("jsonnet"){ 39 | checkCommon(JsonnetWhitespace.whitespace) 40 | // no nested comments 41 | val Parsed.Success((), 11) = parse("/** /* /**/", JsonnetWhitespace.whitespace.apply(_)) 42 | val Parsed.Success((), 6) = parse("/*/**/*/", JsonnetWhitespace.whitespace.apply(_)) 43 | } 44 | } 45 | 46 | } -------------------------------------------------------------------------------- /perftests/bench1/src/perftests/CssParse.scala: -------------------------------------------------------------------------------- 1 | package perftests.string 2 | 3 | import perftests.Utils 4 | import utest._ 5 | import fastparse.all._ 6 | object CssParse extends TestSuite { 7 | val bootstrapStream = getClass.getResourceAsStream("/bootstrap.css") 8 | val bootstrapSource = scala.io.Source.fromInputStream(bootstrapStream).mkString 9 | def bootstrapIterator(size: Int) = bootstrapSource.grouped(size) 10 | val parser = cssparse.CssRulesParser.ruleList ~ End 11 | 12 | val tests = Tests { 13 | test("Bootstrap"){ 14 | Utils.benchmarkAll( 15 | "CssParse", 16 | parser, 17 | bootstrapSource, Some(bootstrapSource + "}"), 18 | bootstrapIterator 19 | ) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /perftests/bench1/src/perftests/Expr.scala: -------------------------------------------------------------------------------- 1 | package perftests 2 | 3 | sealed trait Offsetted{ 4 | def offset: Int 5 | } 6 | sealed trait Expr extends Offsetted 7 | object Expr{ 8 | case class Null(offset: Int) extends Expr 9 | case class True(offset: Int) extends Expr 10 | case class False(offset: Int) extends Expr 11 | case class Self(offset: Int) extends Expr 12 | case class Super(offset: Int) extends Expr 13 | case class $(offset: Int) extends Expr 14 | 15 | case class Str(offset: Int, value: String) extends Expr 16 | case class Num(offset: Int, value: Double) extends Expr 17 | case class Id(offset: Int, value: String) extends Expr 18 | case class Arr(offset: Int, value: Seq[Expr]) extends Expr 19 | case class Obj(offset: Int, value: ObjBody) extends Expr 20 | 21 | sealed trait FieldName 22 | 23 | object FieldName{ 24 | case class Fixed(value: String) extends FieldName 25 | case class Dyn(expr: Expr) extends FieldName 26 | } 27 | sealed trait Member 28 | 29 | object Member{ 30 | sealed trait Visibility 31 | object Visibility{ 32 | 33 | case object Normal extends Visibility 34 | case object Hidden extends Visibility 35 | case object Unhide extends Visibility 36 | } 37 | case class Field(offset: Int, 38 | fieldName: FieldName, 39 | plus: Boolean, 40 | args: Option[Params], 41 | sep: Visibility, 42 | rhs: Expr) extends Member 43 | case class BindStmt(value: Bind) extends Member 44 | case class AssertStmt(value: Expr, msg: Option[Expr]) extends Member 45 | } 46 | 47 | 48 | case class Parened(offset: Int, value: Expr) extends Expr 49 | case class Params(args: Seq[(String, Option[Expr])]) 50 | case class Args(args: Seq[(Option[String], Expr)]) 51 | 52 | case class UnaryOp(offset: Int, op: UnaryOp.Op, value: Expr) extends Expr 53 | object UnaryOp{ 54 | sealed trait Op 55 | case object `+` extends Op 56 | case object `-` extends Op 57 | case object `~` extends Op 58 | case object `!` extends Op 59 | } 60 | case class BinaryOp(offset: Int, lhs: Expr, op: BinaryOp.Op, rhs: Expr) extends Expr 61 | object BinaryOp{ 62 | sealed trait Op 63 | case object `*` extends Op 64 | case object `/` extends Op 65 | case object `%` extends Op 66 | case object `+` extends Op 67 | case object `-` extends Op 68 | case object `<<` extends Op 69 | case object `>>` extends Op 70 | case object `<` extends Op 71 | case object `>` extends Op 72 | case object `<=` extends Op 73 | case object `>=` extends Op 74 | case object `in` extends Op 75 | case object `==` extends Op 76 | case object `!=` extends Op 77 | case object `&` extends Op 78 | case object `^` extends Op 79 | case object `|` extends Op 80 | case object `&&` extends Op 81 | case object `||` extends Op 82 | } 83 | case class AssertExpr(offset: Int, asserted: Member.AssertStmt, returned: Expr) extends Expr 84 | case class LocalExpr(offset: Int, bindings: Seq[Bind], returned: Expr) extends Expr 85 | 86 | case class Bind(offset: Int, name: String, args: Option[Params], rhs: Expr) extends Offsetted 87 | case class Import(offset: Int, value: String) extends Expr 88 | case class ImportStr(offset: Int, value: String) extends Expr 89 | case class Error(offset: Int, value: Expr) extends Expr 90 | case class Apply(offset: Int, value: Expr, args: Args) extends Expr 91 | case class Select(offset: Int, value: Expr, name: String) extends Expr 92 | case class Lookup(offset: Int, value: Expr, index: Expr) extends Expr 93 | case class Slice(offset: Int, 94 | value: Expr, 95 | start: Option[Expr], 96 | end: Option[Expr], 97 | stride: Option[Expr]) extends Expr 98 | case class Function(offset: Int, params: Params, body: Expr) extends Expr 99 | case class IfElse(offset: Int, cond: Expr, then: Expr, `else`: Option[Expr]) extends Expr 100 | 101 | sealed trait CompSpec extends Expr 102 | case class IfSpec(offset: Int, cond: Expr) extends CompSpec 103 | case class ForSpec(offset: Int, name: String, cond: Expr) extends CompSpec 104 | 105 | case class Comp(offset: Int, value: Expr, first: ForSpec, rest: Seq[CompSpec]) extends Expr 106 | case class ObjExtend(offset: Int, base: Expr, ext: ObjBody) extends Expr 107 | 108 | sealed trait ObjBody 109 | object ObjBody{ 110 | case class MemberList(value: Seq[Member]) extends ObjBody 111 | case class ObjComp(preLocals: Seq[Member.BindStmt], 112 | key: Expr, 113 | value: Expr, 114 | postLocals: Seq[Member.BindStmt], 115 | first: ForSpec, 116 | rest: Seq[CompSpec]) extends ObjBody 117 | } 118 | 119 | } -------------------------------------------------------------------------------- /perftests/bench1/src/perftests/JsonParse.scala: -------------------------------------------------------------------------------- 1 | package perftests 2 | import fastparse.all._ 3 | import utest._ 4 | object JsonParse extends TestSuite { 5 | object Js { 6 | sealed trait Val extends Any { 7 | def value: Any 8 | def apply(i: Int): Val = this.asInstanceOf[Arr].value(i) 9 | def apply(s: java.lang.String): Val = 10 | this.asInstanceOf[Obj].value.find(_._1 == s).get._2 11 | } 12 | case class Str(value: java.lang.String) extends AnyVal with Val 13 | case class Obj(value: (java.lang.String, Val)*) extends AnyVal with Val 14 | case class Arr(value: Val*) extends AnyVal with Val 15 | case class Num(value: Double) extends AnyVal with Val 16 | case object False extends Val{ 17 | def value = false 18 | } 19 | case object True extends Val{ 20 | def value = true 21 | } 22 | case object Null extends Val{ 23 | def value = null 24 | } 25 | } 26 | case class NamedFunction[T, V](f: T => V, name: String) extends (T => V){ 27 | def apply(t: T) = f(t) 28 | override def toString() = name 29 | 30 | } 31 | // Here is the parser 32 | val StringChars = NamedFunction(!"\"\\".contains(_: Char), "StringChars") 33 | 34 | val space = P( CharsWhileIn(" \r\n").? ) 35 | val digits = P( CharsWhileIn("0123456789")) 36 | val exponent = P( CharIn("eE") ~ CharIn("+-").? ~ digits ) 37 | val fractional = P( "." ~ digits ) 38 | val integral = P( "0" | CharIn('1' to '9') ~ digits.? ) 39 | 40 | val number = P( CharIn("+-").? ~ integral ~ fractional.? ~ exponent.? ).!.map( 41 | x => Js.Num(x.toDouble) 42 | ) 43 | 44 | val `null` = P( "null" ).map(_ => Js.Null) 45 | val `false` = P( "false" ).map(_ => Js.False) 46 | val `true` = P( "true" ).map(_ => Js.True) 47 | 48 | val hexDigit = P( CharIn('0'to'9', 'a'to'f', 'A'to'F') ) 49 | val unicodeEscape = P( "u" ~ hexDigit ~ hexDigit ~ hexDigit ~ hexDigit ) 50 | val escape = P( "\\" ~ (CharIn("\"/\\bfnrt") | unicodeEscape) ) 51 | 52 | val strChars = P( CharsWhile(StringChars) ) 53 | val string = 54 | P( space ~ "\"" ~/ (strChars | escape).rep.! ~ "\"").map(Js.Str) 55 | 56 | val array = 57 | P( "[" ~/ jsonExpr.rep(sep=",".~/) ~ space ~ "]").map(Js.Arr(_:_*)) 58 | 59 | val pair = P( string.map(_.value) ~/ ":" ~/ jsonExpr ) 60 | 61 | val obj = 62 | P( "{" ~/ pair.rep(sep=",".~/) ~ space ~ "}").map(Js.Obj(_:_*)) 63 | 64 | val jsonExpr: P[Js.Val] = P( 65 | space ~ (obj | array | string | `true` | `false` | `null` | number) ~ space 66 | ) 67 | val crossValidationStream = getClass.getResourceAsStream("/fastparse/test.json") 68 | val crossValidationSource = scala.io.Source.fromInputStream(crossValidationStream).mkString 69 | def crossValidationIterator(size: Int) = crossValidationSource.grouped(size) 70 | 71 | val tests = Tests { 72 | test("CrossValidation"){ 73 | Utils.benchmarkAll( 74 | "JsonParse", 75 | jsonExpr ~ End, 76 | crossValidationSource, Some(crossValidationSource + "]"), 77 | crossValidationIterator 78 | ) 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /perftests/bench1/src/perftests/JsonnetParse.scala: -------------------------------------------------------------------------------- 1 | package perftests 2 | 3 | import fastparse.all._ 4 | import utest._ 5 | object JsonnetParse extends TestSuite { 6 | val names = Seq( 7 | "consul-alerts.jsonnet", 8 | "consul-dashboards.jsonnet", 9 | "grafana.jsonnet", 10 | "ksonnet-util.jsonnet", 11 | "oauth2-proxy.jsonnet", 12 | "prometheus.jsonnet", 13 | "prometheus-alertmanager.jsonnet", 14 | "prometheus-config.jsonnet", 15 | "prometheus-config-kops.jsonnet", 16 | "prometheus-grafana.jsonnet", 17 | "prometheus-kube-state-metrics.jsonnet", 18 | "prometheus-nginx.jsonnet", 19 | "prometheus-node-exporter.jsonnet" 20 | ) 21 | val crossValidationSource = names.map(n => 22 | scala.io.Source.fromInputStream(getClass.getResourceAsStream("/" + n)).mkString 23 | ).mkString("[\n", ",\n", "]") 24 | def crossValidationIterator(size: Int) = crossValidationSource.grouped(size) 25 | val parser = new FastParseParser() 26 | 27 | val tests = Tests { 28 | test("CrossValidation"){ 29 | Utils.benchmarkAll( 30 | "JsonnetParse", 31 | parser.document, 32 | crossValidationSource, Some("[ " + crossValidationSource), 33 | crossValidationIterator 34 | ) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /perftests/bench1/src/perftests/PythonParse.scala: -------------------------------------------------------------------------------- 1 | package perftests.string 2 | 3 | import perftests.Utils 4 | import utest._ 5 | import fastparse.all._ 6 | object PythonParse extends TestSuite { 7 | val crossValidationStream = getClass.getResourceAsStream("/cross_validation.py") 8 | val crossValidationSource = scala.io.Source.fromInputStream(crossValidationStream).mkString 9 | def crossValidationIterator(size: Int) = crossValidationSource.grouped(size) 10 | val parser = pythonparse.Statements.file_input ~ End 11 | 12 | val tests = Tests { 13 | test("CrossValidation"){ 14 | Utils.benchmarkAll( 15 | "PythonParse", 16 | parser, 17 | crossValidationSource, Some(crossValidationSource + " def"), 18 | crossValidationIterator 19 | ) 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /perftests/bench1/src/perftests/ScalaParse.scala: -------------------------------------------------------------------------------- 1 | package perftests.string 2 | 3 | import perftests.Utils 4 | import utest._ 5 | 6 | import scala.tools.nsc.{Global, Settings} 7 | import scalaparse.{Scala} 8 | import fastparse.all._ 9 | object ScalaParse extends TestSuite{ 10 | val genJsCodeStream = getClass.getResourceAsStream("/GenJSCode.scala") 11 | val genJsCodeSource = scala.io.Source.fromInputStream(genJsCodeStream).mkString 12 | def genJsCodeIterator(size: Int) = genJsCodeSource.grouped(size) 13 | 14 | val tests = Tests { 15 | test("GenJSCode"){ 16 | // var current = Thread.currentThread().getContextClassLoader 17 | // val files = collection.mutable.Buffer.empty[java.io.File] 18 | // files.appendAll( 19 | // System.getProperty("sun.boot.class.path") 20 | // .split(":") 21 | // .map(new java.io.File(_)) 22 | // ) 23 | // while (current != null) { 24 | // current match { 25 | // case t: java.net.URLClassLoader => 26 | // files.appendAll(t.getURLs.map(u => new java.io.File(u.toURI))) 27 | // case _ => 28 | // } 29 | // current = current.getParent 30 | // } 31 | // 32 | // val settings = new Settings() 33 | // settings.usejavacp.value = true 34 | // settings.embeddedDefaults[ScalacParser.type] 35 | // settings.classpath.append(files.mkString(":")) 36 | // val global = new Global(settings) 37 | // val run = new global.Run() 38 | // 39 | // println("Optimizing Parser") 40 | 41 | val parser = Scala.CompilationUnit 42 | println("Loaded " + genJsCodeSource.length + " bytes of input. Parsing...") 43 | 44 | Utils.benchmarkAll("ScalaParse", 45 | parser, 46 | genJsCodeSource, Some(genJsCodeSource + "*/"), 47 | genJsCodeIterator) 48 | } 49 | } 50 | } 51 | 52 | -------------------------------------------------------------------------------- /perftests/bench1/src/perftests/Utils.scala: -------------------------------------------------------------------------------- 1 | package perftests 2 | 3 | import fastparse.utils.IteratorParserInput 4 | import fastparse.core.{Parsed, Parser} 5 | import fastparse.utils.{IteratorParserInput, ReprOps} 6 | 7 | import scala.collection.mutable 8 | import scala.reflect.ClassTag 9 | 10 | object Utils { 11 | 12 | def time(f: () => Any, maxTime: Int = 10000): Int = { 13 | val start = System.currentTimeMillis() 14 | var count = 0 15 | while(System.currentTimeMillis() - start < maxTime){ 16 | f() 17 | count += 1 18 | } 19 | count 20 | } 21 | 22 | def benchmark(name: String, fs: Seq[() => Any], iterations: Int = 5, maxTime: Int = 10000): Seq[Seq[Int]] = { 23 | println(name) 24 | println(s"Max time - $maxTime ms. Iterations - $iterations.") 25 | (1 to iterations).map(i => { 26 | println(s"Iteration $i") 27 | fs.zipWithIndex.map(fi => { 28 | print(s"Benchmark ${fi._2}.") 29 | val res = time(fi._1, maxTime) 30 | println(s" Result: $res") 31 | res 32 | }) 33 | }) 34 | } 35 | 36 | def benchmarkIteratorBufferSizes[Elem, Repr](parser: Parser[_, Elem, Repr], 37 | sizes: Seq[Int], 38 | iteratorFactory: Int => Iterator[Repr]) 39 | (implicit repr: ReprOps[Elem, Repr], 40 | ct: ClassTag[Elem]): Unit = { 41 | 42 | class LoggedMaxBufferLengthParserInput(data: Iterator[Repr]) 43 | extends IteratorParserInput[Elem, Repr](data) { 44 | 45 | var maxInnerLength = 0 46 | 47 | override def dropBuffer(index: Int): Unit = { 48 | maxInnerLength = math.max(maxInnerLength, this.innerLength) 49 | super.dropBuffer(index) 50 | } 51 | } 52 | 53 | class LoggedDistributionBufferLengthParserInput(data: Iterator[Repr]) 54 | extends IteratorParserInput[Elem, Repr](data) { 55 | 56 | val drops = mutable.Map.empty[Int, Int].withDefaultValue(0) 57 | 58 | override def dropBuffer(index: Int): Unit = { 59 | drops(this.innerLength) = drops(this.innerLength) + 1 60 | super.dropBuffer(index) 61 | } 62 | } 63 | 64 | 65 | sizes.foreach(s => { 66 | println("Parsing for batch size " + s) 67 | val input = new LoggedMaxBufferLengthParserInput(iteratorFactory(s)) 68 | parser.parseInput(input) 69 | println(s"Batch size: $s. Max buffer size: ${input.maxInnerLength}.") 70 | }) 71 | 72 | val input = new LoggedDistributionBufferLengthParserInput(iteratorFactory(1)) 73 | parser.parseInput(input) 74 | println("Distibutions of buffer size:") 75 | 76 | val chunkSize = (input.drops.size - 11) / 10 77 | val lengths = input.drops.toSeq.sorted 78 | 79 | println(lengths.take(11).map(v => s"${v._1}: ${v._2}").mkString("\n")) 80 | if(lengths.length > 11) { 81 | println(lengths.drop(11).grouped(chunkSize).toList 82 | .map(chunk => (chunk.map(_._1).min, chunk.map(_._1).max, chunk.map(_._2).sum)) 83 | .map(v => s"${v._1}-${v._2}: ${v._3}").mkString("\n")) 84 | } 85 | } 86 | 87 | def benchmarkAll[Elem, Repr](name: String, 88 | parser: Parser[_, Elem, Repr], 89 | data: Repr, dataFailOpt: Option[Repr], 90 | iteratorFactory: Int => Iterator[Repr]) 91 | (implicit repr: ReprOps[Elem, Repr], 92 | ct: ClassTag[Elem]): Unit = { 93 | 94 | val results = Utils.benchmark(s"$name Benchmark", 95 | Seq( 96 | Some(() => parser.parse(data).get), 97 | dataFailOpt.map(dataFail => 98 | () => parser.parse(dataFail).asInstanceOf[Parsed.Failure[Elem, Repr]].extra.traced 99 | ) 100 | ).flatten 101 | ) 102 | println(results.map(_.mkString(" ")).mkString("\n")) 103 | 104 | // val sizes = Seq(1, 64, 4096) 105 | // Utils.benchmarkIteratorBufferSizes(parser, sizes, iteratorFactory) 106 | // 107 | // val iteratorResults = Utils.benchmark(s"$name Iterator Benchmark", 108 | // sizes.map(s => () => parser.parse(iteratorFactory(s)).asInstanceOf[Parsed.Success[_, _, _]]) 109 | // ) 110 | // 111 | // println(iteratorResults.map(_.mkString(" ")).mkString("\n")) 112 | } 113 | } 114 | 115 | 116 | -------------------------------------------------------------------------------- /perftests/bench2/src/perftests/CssParse.scala: -------------------------------------------------------------------------------- 1 | package perftests.string 2 | import fastparse._, NoWhitespace._ 3 | import perftests.Utils 4 | import utest._ 5 | object CssParse extends TestSuite { 6 | val bootstrapStream = getClass.getResourceAsStream("/bootstrap.css") 7 | val bootstrapSource = scala.io.Source.fromInputStream(bootstrapStream).mkString 8 | def bootstrapIterator(size: Int) = bootstrapSource.grouped(size) 9 | 10 | def parser[$: P] = cssparse.CssRulesParser.ruleList ~ End 11 | val tests = Tests { 12 | test("Bootstrap"){ 13 | Utils.benchmarkAll( 14 | "CssParse", 15 | parser(_), 16 | bootstrapSource, Some(bootstrapSource + "}"), 17 | bootstrapIterator 18 | ) 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /perftests/bench2/src/perftests/Expr.scala: -------------------------------------------------------------------------------- 1 | package perftests 2 | 3 | sealed trait Offsetted{ 4 | def offset: Int 5 | } 6 | sealed trait Expr extends Offsetted 7 | object Expr{ 8 | case class Null(offset: Int) extends Expr 9 | case class True(offset: Int) extends Expr 10 | case class False(offset: Int) extends Expr 11 | case class Self(offset: Int) extends Expr 12 | case class Super(offset: Int) extends Expr 13 | case class $(offset: Int) extends Expr 14 | 15 | case class Str(offset: Int, value: String) extends Expr 16 | case class Num(offset: Int, value: Double) extends Expr 17 | case class Id(offset: Int, value: String) extends Expr 18 | case class Arr(offset: Int, value: Seq[Expr]) extends Expr 19 | case class Obj(offset: Int, value: ObjBody) extends Expr 20 | 21 | sealed trait FieldName 22 | 23 | object FieldName{ 24 | case class Fixed(value: String) extends FieldName 25 | case class Dyn(expr: Expr) extends FieldName 26 | } 27 | sealed trait Member 28 | 29 | object Member{ 30 | sealed trait Visibility 31 | object Visibility{ 32 | 33 | case object Normal extends Visibility 34 | case object Hidden extends Visibility 35 | case object Unhide extends Visibility 36 | } 37 | case class Field(offset: Int, 38 | fieldName: FieldName, 39 | plus: Boolean, 40 | args: Option[Params], 41 | sep: Visibility, 42 | rhs: Expr) extends Member 43 | case class BindStmt(value: Bind) extends Member 44 | case class AssertStmt(value: Expr, msg: Option[Expr]) extends Member 45 | } 46 | 47 | 48 | case class Parened(offset: Int, value: Expr) extends Expr 49 | case class Params(args: Seq[(String, Option[Expr])]) 50 | case class Args(args: Seq[(Option[String], Expr)]) 51 | 52 | case class UnaryOp(offset: Int, op: UnaryOp.Op, value: Expr) extends Expr 53 | object UnaryOp{ 54 | sealed trait Op 55 | case object `+` extends Op 56 | case object `-` extends Op 57 | case object `~` extends Op 58 | case object `!` extends Op 59 | } 60 | case class BinaryOp(offset: Int, lhs: Expr, op: BinaryOp.Op, rhs: Expr) extends Expr 61 | object BinaryOp{ 62 | sealed trait Op 63 | case object `*` extends Op 64 | case object `/` extends Op 65 | case object `%` extends Op 66 | case object `+` extends Op 67 | case object `-` extends Op 68 | case object `<<` extends Op 69 | case object `>>` extends Op 70 | case object `<` extends Op 71 | case object `>` extends Op 72 | case object `<=` extends Op 73 | case object `>=` extends Op 74 | case object `in` extends Op 75 | case object `==` extends Op 76 | case object `!=` extends Op 77 | case object `&` extends Op 78 | case object `^` extends Op 79 | case object `|` extends Op 80 | case object `&&` extends Op 81 | case object `||` extends Op 82 | } 83 | case class AssertExpr(offset: Int, asserted: Member.AssertStmt, returned: Expr) extends Expr 84 | case class LocalExpr(offset: Int, bindings: Seq[Bind], returned: Expr) extends Expr 85 | 86 | case class Bind(offset: Int, name: String, args: Option[Params], rhs: Expr) extends Offsetted 87 | case class Import(offset: Int, value: String) extends Expr 88 | case class ImportStr(offset: Int, value: String) extends Expr 89 | case class Error(offset: Int, value: Expr) extends Expr 90 | case class Apply(offset: Int, value: Expr, args: Args) extends Expr 91 | case class Select(offset: Int, value: Expr, name: String) extends Expr 92 | case class Lookup(offset: Int, value: Expr, index: Expr) extends Expr 93 | case class Slice(offset: Int, 94 | value: Expr, 95 | start: Option[Expr], 96 | end: Option[Expr], 97 | stride: Option[Expr]) extends Expr 98 | case class Function(offset: Int, params: Params, body: Expr) extends Expr 99 | case class IfElse(offset: Int, cond: Expr, `then`: Expr, `else`: Option[Expr]) extends Expr 100 | 101 | sealed trait CompSpec extends Expr 102 | case class IfSpec(offset: Int, cond: Expr) extends CompSpec 103 | case class ForSpec(offset: Int, name: String, cond: Expr) extends CompSpec 104 | 105 | case class Comp(offset: Int, value: Expr, first: ForSpec, rest: Seq[CompSpec]) extends Expr 106 | case class ObjExtend(offset: Int, base: Expr, ext: ObjBody) extends Expr 107 | 108 | sealed trait ObjBody 109 | object ObjBody{ 110 | case class MemberList(value: Seq[Member]) extends ObjBody 111 | case class ObjComp(preLocals: Seq[Member.BindStmt], 112 | key: Expr, 113 | value: Expr, 114 | postLocals: Seq[Member.BindStmt], 115 | first: ForSpec, 116 | rest: Seq[CompSpec]) extends ObjBody 117 | } 118 | 119 | } -------------------------------------------------------------------------------- /perftests/bench2/src/perftests/JsonParse.scala: -------------------------------------------------------------------------------- 1 | package perftests 2 | import utest._ 3 | object JsonParse extends TestSuite { 4 | val crossValidationStream = getClass.getResourceAsStream("/fastparse/test.json") 5 | val crossValidationSource = scala.io.Source.fromInputStream(crossValidationStream).mkString 6 | def crossValidationIterator(size: Int) = crossValidationSource.grouped(size) 7 | import fastparse._, NoWhitespace._ 8 | def jsonDoc[$: P] = P( _root_.test.fastparse.Json.jsonExpr ~ End ) 9 | val tests = Tests { 10 | test("CrossValidation"){ 11 | Utils.benchmarkAll( 12 | "JsonParse", 13 | jsonDoc(_), 14 | crossValidationSource, Some(crossValidationSource + "]"), 15 | crossValidationIterator 16 | ) 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /perftests/bench2/src/perftests/JsonnetParse.scala: -------------------------------------------------------------------------------- 1 | package perftests 2 | import utest._ 3 | object JsonnetParse extends TestSuite { 4 | val names = Seq( 5 | "consul-alerts.jsonnet", 6 | "consul-dashboards.jsonnet", 7 | "grafana.jsonnet", 8 | "ksonnet-util.jsonnet", 9 | "oauth2-proxy.jsonnet", 10 | "prometheus.jsonnet", 11 | "prometheus-alertmanager.jsonnet", 12 | "prometheus-config.jsonnet", 13 | "prometheus-config-kops.jsonnet", 14 | "prometheus-grafana.jsonnet", 15 | "prometheus-kube-state-metrics.jsonnet", 16 | "prometheus-nginx.jsonnet", 17 | "prometheus-node-exporter.jsonnet" 18 | ) 19 | val crossValidationSource = names.map(n => 20 | scala.io.Source.fromInputStream(getClass.getResourceAsStream("/" + n)).mkString 21 | ).mkString("[\n", ",\n", "]") 22 | def crossValidationIterator(size: Int) = crossValidationSource.grouped(size) 23 | 24 | 25 | val tests = Tests { 26 | test("CrossValidation"){ 27 | Utils.benchmarkAll( 28 | "JsonnetParse", 29 | fastparseParser.document(_), 30 | crossValidationSource, Some("[ " + crossValidationSource), 31 | crossValidationIterator 32 | ) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /perftests/bench2/src/perftests/PythonParse.scala: -------------------------------------------------------------------------------- 1 | package perftests.string 2 | 3 | import perftests.Utils 4 | import utest._ 5 | import fastparse._, NoWhitespace._ 6 | object PythonParse extends TestSuite { 7 | val crossValidationStream = getClass.getResourceAsStream("/cross_validation.py") 8 | val crossValidationSource = scala.io.Source.fromInputStream(crossValidationStream).mkString 9 | def crossValidationIterator(size: Int) = crossValidationSource.grouped(size) 10 | def parser[$: P] = pythonparse.Statements.file_input ~ End 11 | val tests = Tests { 12 | test("CrossValidation"){ 13 | Utils.benchmarkAll( 14 | "PythonParse", 15 | parser(_), 16 | crossValidationSource, Some(crossValidationSource + " def"), 17 | crossValidationIterator 18 | ) 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /perftests/bench2/src/perftests/ScalaParse.scala: -------------------------------------------------------------------------------- 1 | package perftests.string 2 | 3 | import perftests.Utils 4 | import utest._ 5 | 6 | import scalaparse.{Scala} 7 | object ScalaParse extends TestSuite{ 8 | val genJsCodeStream = getClass.getResourceAsStream("/GenJSCode.scala") 9 | val genJsCodeSource = scala.io.Source.fromInputStream(genJsCodeStream).mkString 10 | def genJsCodeIterator(size: Int) = genJsCodeSource.grouped(size) 11 | 12 | val tests = Tests { 13 | test("GenJSCode"){ 14 | // var current = Thread.currentThread().getContextClassLoader 15 | // val files = collection.mutable.Buffer.empty[java.io.File] 16 | // files.appendAll( 17 | // System.getProperty("sun.boot.class.path") 18 | // .split(":") 19 | // .map(new java.io.File(_)) 20 | // ) 21 | // while (current != null) { 22 | // current match { 23 | // case t: java.net.URLClassLoader => 24 | // files.appendAll(t.getURLs.map(u => new java.io.File(u.toURI))) 25 | // case _ => 26 | // } 27 | // current = current.getParent 28 | // } 29 | // 30 | // val settings = new Settings() 31 | // settings.usejavacp.value = true 32 | // settings.embeddedDefaults[ScalacParser.type] 33 | // settings.classpath.append(files.mkString(":")) 34 | // val global = new Global(settings) 35 | // val run = new global.Run() 36 | // 37 | // println("Loaded " + genJsCodeSource.length + " bytes of input. Parsing...") 38 | 39 | Utils.benchmarkAll("ScalaParse", 40 | Scala.CompilationUnit(_), 41 | genJsCodeSource, Some(genJsCodeSource + "*/"), 42 | genJsCodeIterator) 43 | } 44 | } 45 | } 46 | 47 | -------------------------------------------------------------------------------- /perftests/bench2/src/perftests/Utils.scala: -------------------------------------------------------------------------------- 1 | package perftests 2 | 3 | import fastparse._ 4 | 5 | import scala.collection.mutable 6 | import scala.reflect.ClassTag 7 | 8 | object Utils { 9 | 10 | def time(f: () => Any, maxTime: Int = 10000): Int = { 11 | val start = System.currentTimeMillis() 12 | var count = 0 13 | while(System.currentTimeMillis() - start < maxTime){ 14 | f() 15 | count += 1 16 | } 17 | count 18 | } 19 | 20 | def benchmark(name: String, fs: Seq[() => Any], iterations: Int = 5, maxTime: Int = 10000): Seq[Seq[Int]] = { 21 | println(name) 22 | println(s"Max time - $maxTime ms. Iterations - $iterations.") 23 | (1 to iterations).map(i => { 24 | println(s"Iteration $i") 25 | fs.zipWithIndex.map(fi => { 26 | print(s"Benchmark ${fi._2}.") 27 | val res = time(fi._1, maxTime) 28 | println(s" Result: $res") 29 | res 30 | }) 31 | }) 32 | } 33 | 34 | def benchmarkIteratorBufferSizes(parser: P[_] => P[Any], 35 | sizes: Seq[Int], 36 | iteratorFactory: Int => Iterator[String]): Unit = { 37 | 38 | class LoggedMaxBufferLengthParserInput(data: Iterator[String]) 39 | extends IteratorParserInput(data) { 40 | 41 | var maxInnerLength = 0 42 | 43 | override def dropBuffer(index: Int): Unit = { 44 | maxInnerLength = math.max(maxInnerLength, this.innerLength) 45 | super.dropBuffer(index) 46 | } 47 | } 48 | 49 | class LoggedDistributionBufferLengthParserInput(data: Iterator[String]) 50 | extends IteratorParserInput(data) { 51 | 52 | val drops = mutable.Map.empty[Int, Int].withDefaultValue(0) 53 | 54 | override def dropBuffer(index: Int): Unit = { 55 | drops(this.innerLength) = drops(this.innerLength) + 1 56 | super.dropBuffer(index) 57 | } 58 | } 59 | 60 | 61 | sizes.foreach(s => { 62 | println("Parsing for batch size " + s) 63 | val input = new LoggedMaxBufferLengthParserInput(iteratorFactory(s)) 64 | parse(input, parser) 65 | println(s"Batch size: $s. Max buffer size: ${input.maxInnerLength}.") 66 | }) 67 | 68 | val input = new LoggedDistributionBufferLengthParserInput(iteratorFactory(1)) 69 | parse(input, parser) 70 | println("Distibutions of buffer size:") 71 | 72 | val chunkSize = (input.drops.size - 11) / 10 73 | val lengths = input.drops.toSeq.sorted 74 | 75 | println(lengths.take(11).map(v => s"${v._1}: ${v._2}").mkString("\n")) 76 | if(lengths.length > 11) { 77 | println(lengths.drop(11).grouped(chunkSize).toList 78 | .map(chunk => (chunk.map(_._1).min, chunk.map(_._1).max, chunk.map(_._2).sum)) 79 | .map(v => s"${v._1}-${v._2}: ${v._3}").mkString("\n")) 80 | } 81 | } 82 | 83 | def benchmarkAll(name: String, 84 | parser: P[_] => P[Any], 85 | data: ParserInputSource, dataFailOpt: Option[String], 86 | iteratorFactory: Int => Iterator[String]): Unit = { 87 | 88 | val results = Utils.benchmark(s"$name Benchmark", 89 | Seq( 90 | Some(() => parse(data, parser).get), 91 | dataFailOpt.map(dataFail => 92 | () => parse(dataFail, parser).asInstanceOf[Parsed.Failure].extra.traced 93 | ) 94 | ).flatten 95 | ) 96 | 97 | println(results.map(_.mkString(" ")).mkString("\n")) 98 | 99 | // val sizes = Seq(1, 64, 4096) 100 | // Utils.benchmarkIteratorBufferSizes(parser, sizes, iteratorFactory) 101 | // 102 | // val iteratorResults = Utils.benchmark(s"$name Iterator Benchmark", 103 | // sizes.map(s => () => parse(iteratorFactory(s), parser(_)).asInstanceOf[Parsed.Success[_]]) 104 | // ) 105 | // 106 | // println(iteratorResults.map(_.mkString(" ")).mkString("\n")) 107 | } 108 | } 109 | 110 | 111 | -------------------------------------------------------------------------------- /perftests/compare/src/compare/JsonBench.scala: -------------------------------------------------------------------------------- 1 | package compare 2 | import utest._ 3 | 4 | object JsonBench extends TestSuite{ 5 | def bench(f: => Unit) = { 6 | var start = System.currentTimeMillis() 7 | while(System.currentTimeMillis() - start < 10000) { 8 | f 9 | } 10 | start = System.currentTimeMillis() 11 | var count = 0 12 | while(System.currentTimeMillis() - start < 10000) { 13 | f 14 | count += 1 15 | } 16 | count 17 | } 18 | val txt = scala.io.Source 19 | .fromInputStream(getClass.getResourceAsStream("/fastparse/test.json")) 20 | .mkString 21 | 22 | val tests = Tests{ 23 | test("fastparse") - bench{ 24 | fastparse.parse(txt, _root_.test.fastparse.Json.jsonExpr(_)) 25 | } 26 | test("circe") - bench{ 27 | io.circe.parser.parse(txt).asInstanceOf[Right[_, _]] 28 | } 29 | test("argonaut") - bench{ 30 | argonaut.Parse.parse(txt).asInstanceOf[Right[_, _]] 31 | } 32 | test("ujson") - bench{ 33 | ujson.read(txt) 34 | } 35 | test("json4s") - bench{ 36 | org.json4s.native.JsonMethods.parse(txt) 37 | } 38 | test("play") - bench{ 39 | play.api.libs.json.Json.parse(txt) 40 | } 41 | test("combinators") - bench{ 42 | scala.util.parsing.json.JSON.parseRaw(txt).get 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /perftests/compare/src/compare/PythonBench.scala: -------------------------------------------------------------------------------- 1 | package compare 2 | 3 | import org.python.core.{CompileMode, CompilerFlags} 4 | import utest._ 5 | 6 | object PythonBench extends TestSuite{ 7 | def bench(f: => Unit) = { 8 | var start = System.currentTimeMillis() 9 | while(System.currentTimeMillis() - start < 10000) { 10 | f 11 | } 12 | start = System.currentTimeMillis() 13 | var count = 0 14 | while(System.currentTimeMillis() - start < 10000) { 15 | f 16 | count += 1 17 | } 18 | count 19 | } 20 | val txt = scala.io.Source 21 | .fromFile("perftests/resources/cross_validation.py") 22 | .mkString 23 | 24 | val tests = Tests{ 25 | test("fastparse") - bench{ 26 | fastparse.parse(txt, pythonparse.Statements.file_input(_)) 27 | } 28 | test("jython") - bench{ 29 | org.python.core.ParserFacade.parse(txt, CompileMode.exec, "", CompilerFlags.getCompilerFlags) 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /perftests/compare/src/compare/ScalaBench.scala: -------------------------------------------------------------------------------- 1 | package compare 2 | 3 | import scalaparse.ScalacParser 4 | import utest._ 5 | 6 | import scala.tools.nsc.{Global, Settings} 7 | 8 | object ScalaBench extends TestSuite{ 9 | def bench(f: => Unit) = { 10 | var start = System.currentTimeMillis() 11 | while(System.currentTimeMillis() - start < 10000) { 12 | f 13 | } 14 | start = System.currentTimeMillis() 15 | var count = 0 16 | while(System.currentTimeMillis() - start < 10000) { 17 | f 18 | count += 1 19 | } 20 | count 21 | } 22 | val txt = scala.io.Source 23 | .fromFile("perftests/resources/GenJSCode.scala") 24 | .mkString 25 | 26 | val tests = Tests{ 27 | test("fastparse") - bench{ 28 | 29 | fastparse.parse(txt, scalaparse.Scala.CompilationUnit(_)) 30 | } 31 | test("scalac") - bench{ 32 | ScalacParser.checkParseFails(txt) 33 | } 34 | 35 | } 36 | } -------------------------------------------------------------------------------- /perftests/resources/consul-alerts.jsonnet: -------------------------------------------------------------------------------- 1 | panel_settings { valueMaps: 1} 2 | 3 | -------------------------------------------------------------------------------- /perftests/resources/consul-dashboards.jsonnet: -------------------------------------------------------------------------------- 1 | 2 | { 3 | dashboards+:: { 4 | 'consul.json': 5 | g.dashboard('Consul') 6 | .addTemplate('job', 'consul_up', 'job') 7 | .addMultiTemplate('instance', 'consul_up{job="$job"}', 'instance') 8 | .addRow( 9 | g.row('Up') 10 | .addPanel( 11 | g.panel('$instance') + 12 | g.statPanel('consul_up{job="$job",instance=~"$instance"}', 'none') + 13 | panel_settings { 14 | valueMaps: [ 15 | { value: '0', op: '=', text: 'DOWN' }, 16 | { value: '1', op: '=', text: 'UP' }, 17 | ], 18 | colors: ['#d44a3a', 'rgba(237, 129, 40, 0.89)', '#299c46'], 19 | } 20 | ) + 21 | row_settings 22 | ) 23 | .addRow( 24 | g.row('Leader') 25 | .addPanel( 26 | g.panel('$instance') + 27 | g.statPanel(||| 28 | (rate(consul_raft_leader_lastcontact_count{job="$job",instance=~"$instance"}[1m]) > bool 0) 29 | or 30 | (consul_up{job="$job",instance=~"$instance"} == bool 0) 31 | |||, 'none') + 32 | panel_settings { 33 | valueMaps: [ 34 | { value: '0', op: '=', text: 'FOLLOWER' }, 35 | { value: '1', op: '=', text: 'LEADER' }, 36 | ], 37 | colors: ['rgba(237, 129, 40, 0.89)', 'rgba(237, 129, 40, 0.89)', '#299c46'], 38 | } 39 | ) + 40 | row_settings 41 | ) 42 | .addRow( 43 | g.row('Has Leader') 44 | .addPanel( 45 | g.panel('$instance') + 46 | g.statPanel('consul_raft_leader{job="$job",instance=~"$instance"}', 'none') + 47 | panel_settings { 48 | valueMaps: [ 49 | { value: '0', op: '=', text: 'NO LEADER' }, 50 | { value: '1', op: '=', text: 'HAS LEADER' }, 51 | ], 52 | colors: ['#d44a3a', 'rgba(237, 129, 40, 0.89)', '#299c46'], 53 | } 54 | ) + 55 | row_settings 56 | ) 57 | .addRow( 58 | g.row('# Peers') 59 | .addPanel( 60 | g.panel('$instance') + 61 | g.statPanel('consul_raft_peers{job="$job",instance=~"$instance"}', 'none') + 62 | panel_settings { 63 | thresholds: '1,2', 64 | colors: ['#d44a3a', 'rgba(237, 129, 40, 0.89)', '#299c46'], 65 | } 66 | ) + 67 | row_settings 68 | ) 69 | .addRow( 70 | g.row('Vault Server') 71 | .addPanel( 72 | g.panel('QPS') + 73 | g.queryPanel('sum(rate(consul_http_request_count{job="$job"}[1m])) by (instance)', '{{instance}}') + 74 | g.stack 75 | ) 76 | .addPanel( 77 | g.panel('Latency') + 78 | g.queryPanel('max(consul_http_request{job="$job", quantile="0.99"})', '99th Percentile') + 79 | g.queryPanel('max(consul_http_request{job="$job", quantile="0.5"})', '50th Percentile') + 80 | g.queryPanel('sum(rate(consul_http_request{job="$job"}[5m])) / sum(rate(consul_http_request{job="$job"}[5m]))', 'Average') + 81 | { yaxes: g.yaxes('ms') } 82 | ) 83 | ), 84 | }, 85 | } -------------------------------------------------------------------------------- /perftests/resources/oauth2-proxy.jsonnet: -------------------------------------------------------------------------------- 1 | 2 | local k = import 'ksonnet-util/kausal.libsonnet'; 3 | 4 | k { 5 | _config+:: { 6 | oauth_cookie_secret: error 'Must define a cookie secret', 7 | oauth_client_id: error 'Must define a client id', 8 | oauth_client_secret: error 'Must define a client secret', 9 | oauth_redirect_url: error 'Must define a redirect url', 10 | oauth_upstream: error 'Must define an upstream', 11 | oauth_email_domain: '*', 12 | oauth_pass_basic_auth: 'false', 13 | }, 14 | 15 | _images+:: { 16 | oauth2_proxy: 'a5huynh/oauth2_proxy:latest', 17 | }, 18 | 19 | local secret = $.core.v1.secret, 20 | 21 | oauth2_proxy_secret: 22 | secret.new('oauth2-proxy', { 23 | OAUTH2_PROXY_COOKIE_SECRET: std.base64($._config.oauth_cookie_secret), 24 | OAUTH2_PROXY_CLIENT_SECRET: std.base64($._config.oauth_client_secret), 25 | OAUTH2_PROXY_CLIENT_ID: std.base64($._config.oauth_client_id), 26 | }), 27 | 28 | local container = $.core.v1.container, 29 | local envFrom = container.envFromType, 30 | 31 | oauth2_proxy_container:: 32 | container.new('oauth2-proxy', $._images.oauth2_proxy) + 33 | container.withPorts($.core.v1.containerPort.new('http', 80)) + 34 | container.withArgs([ 35 | '-http-address=0.0.0.0:80', 36 | '-redirect-url=%s' % $._config.oauth_redirect_url, 37 | '-upstream=%s' % $._config.oauth_upstream, 38 | '-email-domain=%s' % $._config.oauth_email_domain, 39 | '-pass-basic-auth=%s' % $._config.oauth_pass_basic_auth, 40 | ]) + 41 | container.withEnvFrom( 42 | envFrom.new() + 43 | envFrom.mixin.secretRef.withName('oauth2-proxy'), 44 | ), 45 | 46 | local deployment = $.apps.v1beta1.deployment, 47 | 48 | oauth2_proxy_deployment: 49 | deployment.new('oauth2-proxy', 1, [$.oauth2_proxy_container]), 50 | 51 | oauth2_proxy_service: 52 | $.util.serviceFor($.oauth2_proxy_deployment), 53 | } -------------------------------------------------------------------------------- /perftests/resources/prometheus-alertmanager.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | alertmanager_config:: { 3 | templates: ['/etc/alertmanager/*.tmpl'], 4 | route: { 5 | group_by: ['alertname'], 6 | receiver: 'slack', 7 | }, 8 | 9 | receivers: [{ 10 | name: 'slack', 11 | slack_configs: [{ 12 | api_url: $._config.slack_url, 13 | channel: $._config.slack_channel, 14 | }], 15 | }], 16 | }, 17 | 18 | local configMap = $.core.v1.configMap, 19 | 20 | alertmanager_config_map: 21 | configMap.new('alertmanager-config') + 22 | configMap.withData({ 23 | 'alertmanager.yml': $.util.manifestYaml($.alertmanager_config), 24 | }), 25 | 26 | local container = $.core.v1.container, 27 | 28 | alertmanager_container:: 29 | container.new('alertmanager', $._images.alertmanager) + 30 | container.withPorts($.core.v1.containerPort.new('http-metrics', $._config.alertmanager_port)) + 31 | container.withArgs([ 32 | '--log.level=info', 33 | '--config.file=/etc/alertmanager/config/alertmanager.yml', 34 | '--web.listen-address=:%s' % $._config.alertmanager_port, 35 | '--web.external-url=%s%s' % [$._config.alertmanager_external_hostname, $._config.alertmanager_path], 36 | ]) + 37 | container.mixin.resources.withRequests({ 38 | cpu: '10m', 39 | memory: '40Mi', 40 | }), 41 | 42 | alertmanager_watch_container:: 43 | container.new('watch', $._images.watch) + 44 | container.withArgs([ 45 | '-v', 46 | '-t', 47 | '-p=/etc/alertmanager/config', 48 | 'curl', 49 | '-X', 50 | 'POST', 51 | '--fail', 52 | '-o', 53 | '-', 54 | '-sS', 55 | 'http://localhost:%s%s-/reload' % [$._config.alertmanager_port, $._config.alertmanager_path], 56 | ]) + 57 | container.mixin.resources.withRequests({ 58 | cpu: '10m', 59 | memory: '20Mi', 60 | }), 61 | 62 | local deployment = $.apps.v1beta1.deployment, 63 | 64 | alertmanager_deployment: 65 | deployment.new('alertmanager', 1, [ 66 | $.alertmanager_container, 67 | $.alertmanager_watch_container, 68 | ]) + 69 | deployment.mixin.spec.template.metadata.withAnnotations({ 'prometheus.io.path': '%smetrics' % $._config.alertmanager_path }) + 70 | $.util.configVolumeMount('alertmanager-config', '/etc/alertmanager/config'), 71 | 72 | alertmanager_service: 73 | $.util.serviceFor($.alertmanager_deployment), 74 | } -------------------------------------------------------------------------------- /perftests/resources/prometheus-config-kops.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | local k8s_pod_scrape(name, port) = { 3 | job_name: name, 4 | kubernetes_sd_configs: [{ 5 | role: 'pod', 6 | }], 7 | 8 | tls_config: { 9 | ca_file: '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', 10 | insecure_skip_verify: $._config.insecureSkipVerify, 11 | }, 12 | bearer_token_file: '/var/run/secrets/kubernetes.io/serviceaccount/token', 13 | 14 | relabel_configs: [ 15 | // Only keeps jobs matching / 16 | { 17 | source_labels: ['__meta_kubernetes_namespace', '__meta_kubernetes_pod_label_k8s_app'], 18 | separator: '/', 19 | action: 'keep', 20 | regex: name, 21 | }, 22 | 23 | // Rename instances to be the pod name 24 | { 25 | source_labels: ['__meta_kubernetes_pod_name'], 26 | action: 'replace', 27 | target_label: 'instance', 28 | }, 29 | 30 | // Override the port. 31 | { 32 | source_labels: ['__address__'], 33 | action: 'replace', 34 | target_label: '__address__', 35 | regex: '(.+?)(\\:\\d+)?', 36 | replacement: '$1:%s' % port, 37 | }, 38 | 39 | // But also include the namespace as a separate label, for routing alerts 40 | { 41 | source_labels: ['__meta_kubernetes_namespace'], 42 | action: 'replace', 43 | target_label: 'namespace', 44 | }, 45 | ], 46 | }, 47 | 48 | prometheus_config+:: { 49 | scrape_configs+: [ 50 | k8s_pod_scrape('kube-system/kube-apiserver', 443) { 51 | scheme: 'https', 52 | }, 53 | 54 | k8s_pod_scrape('kube-system/kube-scheduler', 10251), 55 | k8s_pod_scrape('kube-system/kube-controller-manager', 10252), 56 | 57 | // kops doesn't configure kube-proxy to listen on non-localhost, 58 | // can't scrape. 59 | // k8s_pod_scrape("kube-system/kube-proxy", 10249), 60 | 61 | // kops firewalls etcd on masters off from nodes, so we 62 | // can't scrape it with Prometheus. 63 | // k8s_pod_scrape("kube-system/etcd-server", 4001), 64 | // k8s_pod_scrape("kube-system/etcd-server-events", 4002), 65 | ], 66 | }, 67 | } -------------------------------------------------------------------------------- /perftests/resources/prometheus-grafana.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | grafana_config:: { 3 | sections: { 4 | 'auth.anonymous': { 5 | enabled: true, 6 | org_role: 'Admin', 7 | }, 8 | server: { 9 | http_port: 80, 10 | root_url: $._config.grafana_root_url, 11 | }, 12 | analytics: { 13 | reporting_enabled: false, 14 | }, 15 | users: { 16 | default_theme: 'light', 17 | }, 18 | }, 19 | }, 20 | 21 | local configMap = $.core.v1.configMap, 22 | 23 | grafana_config_map: 24 | configMap.new('grafana-config') + 25 | configMap.withData({ 'grafana.ini': std.manifestIni($.grafana_config) }), 26 | 27 | // Extension point for you to add your own dashboards. 28 | dashboards+:: {}, 29 | grafana_dashboards+:: {}, 30 | grafanaDashboards+:: $.dashboards + $.grafana_dashboards, 31 | 32 | dashboards_config_map: 33 | configMap.new('dashboards') + 34 | configMap.withDataMixin({ 35 | [name]: std.toString($.grafanaDashboards[name]) 36 | for name in std.objectFields($.grafanaDashboards) 37 | }), 38 | 39 | grafana_dashboard_provisioning_config_map: 40 | configMap.new('grafana-dashboard-provisioning') + 41 | configMap.withData({ 42 | 'dashboards.yml': $.util.manifestYaml({ 43 | apiVersion: 1, 44 | providers: [{ 45 | name: 'dashboards', 46 | orgId: 1, 47 | folder: '', 48 | type: 'file', 49 | disableDeletion: true, 50 | editable: false, 51 | options: { 52 | path: '/grafana/dashboards', 53 | }, 54 | }], 55 | }), 56 | }), 57 | 58 | grafana_datasource_config_map: 59 | configMap.new('grafana-datasources') + 60 | $.grafana_add_datasource('prometheus', 61 | 'http://prometheus.%(namespace)s.svc.%(cluster_dns_suffix)s%(prometheus_web_route_prefix)s' % $._config, 62 | default=true), 63 | 64 | grafana_add_datasource(name, url, default=false):: 65 | configMap.withDataMixin({ 66 | ['%s.yml' % name]: $.util.manifestYaml({ 67 | apiVersion: 1, 68 | datasources: [{ 69 | name: name, 70 | type: 'prometheus', 71 | access: 'proxy', 72 | url: url, 73 | isDefault: default, 74 | version: 1, 75 | editable: false, 76 | }], 77 | }), 78 | }), 79 | 80 | grafana_add_datasource_with_basicauth(name, url, username, password, default=false):: 81 | configMap.withDataMixin({ 82 | ['%s.yml' % name]: $.util.manifestYaml({ 83 | apiVersion: 1, 84 | datasources: [{ 85 | name: name, 86 | type: 'prometheus', 87 | access: 'proxy', 88 | url: url, 89 | isDefault: default, 90 | version: 1, 91 | editable: false, 92 | basicAuth: true, 93 | basicAuthUser: username, 94 | basicAuthPassword: password, 95 | }], 96 | }), 97 | }), 98 | 99 | local container = $.core.v1.container, 100 | 101 | grafana_container:: 102 | container.new('grafana', $._images.grafana) + 103 | container.withPorts($.core.v1.containerPort.new('grafana', 80)) + 104 | container.withCommand([ 105 | '/usr/share/grafana/bin/grafana-server', 106 | '--homepath=/usr/share/grafana', 107 | '--config=/etc/grafana-config/grafana.ini', 108 | ]) + 109 | $.util.resourcesRequests('10m', '40Mi'), 110 | 111 | local deployment = $.apps.v1beta1.deployment, 112 | 113 | grafana_deployment: 114 | deployment.new('grafana', 1, [$.grafana_container]) + 115 | deployment.mixin.spec.template.spec.securityContext.withRunAsUser(0) + 116 | $.util.configVolumeMount('grafana-config', '/etc/grafana-config') + 117 | $.util.configVolumeMount('grafana-dashboard-provisioning', '%(grafana_provisioning_dir)s/dashboards' % $._config) + 118 | $.util.configVolumeMount('grafana-datasources', '%(grafana_provisioning_dir)s/datasources' % $._config) + 119 | $.util.configVolumeMount('dashboards', '/grafana/dashboards'), 120 | 121 | grafana_service: 122 | $.util.serviceFor($.grafana_deployment), 123 | } -------------------------------------------------------------------------------- /perftests/resources/prometheus-kube-state-metrics.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | local policyRule = $.rbac.v1beta1.policyRule, 3 | 4 | kube_state_metrics_rbac: 5 | $.util.rbac('kube-state-metrics', [ 6 | policyRule.new() + 7 | policyRule.withApiGroups(['']) + 8 | policyRule.withResources([ 9 | 'configmaps', 10 | 'secrets', 11 | 'nodes', 12 | 'pods', 13 | 'services', 14 | 'resourcequotas', 15 | 'replicationcontrollers', 16 | 'limitranges', 17 | 'persistentvolumeclaims', 18 | 'persistentvolumes', 19 | 'namespaces', 20 | 'endpoints', 21 | ]) + 22 | policyRule.withVerbs(['list', 'watch']), 23 | 24 | policyRule.new() + 25 | policyRule.withApiGroups(['extensions']) + 26 | policyRule.withResources([ 27 | 'daemonsets', 28 | 'deployments', 29 | 'replicasets', 30 | ]) + 31 | policyRule.withVerbs(['list', 'watch']), 32 | 33 | policyRule.new() + 34 | policyRule.withApiGroups(['apps']) + 35 | policyRule.withResources([ 36 | 'statefulsets', 37 | ]) + 38 | policyRule.withVerbs(['list', 'watch']), 39 | 40 | policyRule.new() + 41 | policyRule.withApiGroups(['batch']) + 42 | policyRule.withResources([ 43 | 'cronjobs', 44 | 'jobs', 45 | ]) + 46 | policyRule.withVerbs(['list', 'watch']), 47 | 48 | policyRule.new() + 49 | policyRule.withApiGroups(['autoscaling']) + 50 | policyRule.withResources([ 51 | 'horizontalpodautoscalers', 52 | ]) + 53 | policyRule.withVerbs(['list', 'watch']), 54 | ]), 55 | 56 | local container = $.core.v1.container, 57 | local containerPort = $.core.v1.containerPort, 58 | 59 | kube_state_metrics_container:: 60 | container.new('kube-state-metrics', $._images.kubeStateMetrics) + 61 | container.withArgs([ 62 | '--port=80', 63 | '--telemetry-host=0.0.0.0', 64 | '--telemetry-port=81', 65 | ]) + 66 | container.withPorts([ 67 | containerPort.new('http-metrics', 80), 68 | containerPort.new('self-metrics', 81), 69 | ]) + 70 | $.util.resourcesRequests('25m', '20Mi') + 71 | $.util.resourcesLimits('100m', '60Mi'), 72 | 73 | local deployment = $.apps.v1beta1.deployment, 74 | 75 | kube_state_metrics_deployment: 76 | deployment.new('kube-state-metrics', 1, [ 77 | $.kube_state_metrics_container, 78 | ]) + 79 | // Stop the default pod discovery scraping this pod - we use a special 80 | // scrape config to preserve namespace etc labels. 81 | deployment.mixin.spec.template.metadata.withAnnotationsMixin({ 'prometheus.io.scrape': 'false' }) + 82 | if $._config.enable_rbac 83 | then deployment.mixin.spec.template.spec.withServiceAccount('kube-state-metrics') 84 | else {}, 85 | 86 | kube_state_metrics_service: 87 | $.util.serviceFor($.kube_state_metrics_deployment), 88 | } -------------------------------------------------------------------------------- /perftests/resources/prometheus-nginx.jsonnet: -------------------------------------------------------------------------------- 1 | 2 | { 3 | local configMap = $.core.v1.configMap, 4 | 5 | nginx_config_map: 6 | local vars = { 7 | location_stanzas: [ 8 | ||| 9 | location ~ ^/%(path)s(/?)(.*)$ { 10 | proxy_pass %(url)s$2$is_args$args; 11 | } 12 | ||| % service 13 | for service in $._config.admin_services 14 | ], 15 | locations: std.join('\n', self.location_stanzas), 16 | link_stanzas: [ 17 | ||| 18 |
  • %(title)s
    • 19 | ||| % service 20 | for service in $._config.admin_services 21 | ], 22 | links: std.join('\n', self.link_stanzas), 23 | }; 24 | 25 | configMap.new('nginx-config') + 26 | configMap.withData({ 27 | 'nginx.conf': ||| 28 | worker_processes 5; ## Default: 1 29 | error_log /dev/stderr; 30 | pid /tmp/nginx.pid; 31 | worker_rlimit_nofile 8192; 32 | 33 | events { 34 | worker_connections 4096; ## Default: 1024 35 | } 36 | 37 | http { 38 | default_type application/octet-stream; 39 | log_format main '$remote_addr - $remote_user [$time_local] $status ' 40 | '"$request" $body_bytes_sent "$http_referer" ' 41 | '"$http_user_agent" "$http_x_forwarded_for"'; 42 | access_log /dev/stderr main; 43 | sendfile on; 44 | tcp_nopush on; 45 | resolver kube-dns.kube-system.svc.%(cluster_dns_suffix)s; 46 | server { 47 | listen 80; 48 | %(locations)s 49 | location ~ /(index.html)? { 50 | root /etc/nginx; 51 | } 52 | } 53 | } 54 | ||| % ($._config + vars), 55 | 'index.html': ||| 56 | 57 | Admin 58 | 59 |

    Admin

    60 |
      61 | %(links)s 62 |
    63 | 64 | 65 | ||| % vars, 66 | }), 67 | 68 | local container = $.core.v1.container, 69 | 70 | nginx_container:: 71 | container.new('nginx', $._images.nginx) + 72 | container.withPorts($.core.v1.containerPort.new('http', 80)) + 73 | $.util.resourcesRequests('50m', '100Mi'), 74 | 75 | local deployment = $.apps.v1beta1.deployment, 76 | 77 | nginx_deployment: 78 | deployment.new('nginx', 1, [$.nginx_container]) + 79 | $.util.configVolumeMount('nginx-config', '/etc/nginx'), 80 | 81 | nginx_service: 82 | $.util.serviceFor($.nginx_deployment), 83 | 84 | local oauth2_proxy = import 'oauth2_proxy/oauth2-proxy.libsonnet', 85 | 86 | oauth2_proxy: 87 | if ! $._config.oauth_enabled 88 | then {} 89 | else oauth2_proxy { 90 | _config+:: $._config { 91 | oauth_upstream: 'http://nginx.%(namespace)s.svc.%(cluster_dns_suffix)s/' % $._config, 92 | }, 93 | }, 94 | } -------------------------------------------------------------------------------- /perftests/resources/prometheus-node-exporter.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | local container = $.core.v1.container, 3 | 4 | node_exporter_container:: 5 | container.new('node-exporter', $._images.nodeExporter) + 6 | container.withPorts($.core.v1.containerPort.new('http-metrics', 9100)) + 7 | container.withArgs([ 8 | '--path.procfs=/host/proc', 9 | '--path.sysfs=/host/sys', 10 | '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($|/)', 11 | ]) + 12 | container.mixin.securityContext.withPrivileged(true) + 13 | container.mixin.securityContext.withRunAsUser(0) + 14 | $.util.resourcesRequests('10m', '20Mi') + 15 | $.util.resourcesLimits('50m', '40Mi'), 16 | 17 | local daemonSet = $.extensions.v1beta1.daemonSet, 18 | 19 | node_exporter_daemonset: 20 | daemonSet.new('node-exporter', [$.node_exporter_container]) + 21 | daemonSet.mixin.spec.template.spec.withHostPid(true) + 22 | daemonSet.mixin.spec.template.spec.withHostNetwork(true) + 23 | daemonSet.mixin.spec.template.metadata.withAnnotationsMixin({ 'prometheus.io.scrape': 'false' }) + 24 | $.util.hostVolumeMount('proc', '/proc', '/host/proc') + 25 | $.util.hostVolumeMount('sys', '/sys', '/host/sys') + 26 | (if $._config.node_exporter_mount_root 27 | then $.util.hostVolumeMount('root', '/', '/rootfs') 28 | else {}), 29 | } -------------------------------------------------------------------------------- /perftests/resources/prometheus.jsonnet: -------------------------------------------------------------------------------- 1 | { 2 | local policyRule = $.rbac.v1beta1.policyRule, 3 | 4 | prometheus_rbac: 5 | $.util.rbac('prometheus', [ 6 | policyRule.new() + 7 | policyRule.withApiGroups(['']) + 8 | policyRule.withResources(['nodes', 'nodes/proxy', 'services', 'endpoints', 'pods']) + 9 | policyRule.withVerbs(['get', 'list', 'watch']), 10 | 11 | policyRule.new() + 12 | policyRule.withNonResourceUrls('/metrics') + 13 | policyRule.withVerbs(['get']), 14 | ]), 15 | 16 | local container = $.core.v1.container, 17 | 18 | prometheus_container:: 19 | container.new('prometheus', $._images.prometheus) + 20 | container.withPorts($.core.v1.containerPort.new('http-metrics', 80)) + 21 | container.withArgs([ 22 | '--config.file=/etc/prometheus/prometheus.yml', 23 | '--web.listen-address=:%s' % $._config.prometheus_port, 24 | '--web.external-url=%s%s' % [$._config.prometheus_external_hostname, $._config.prometheus_path], 25 | '--web.enable-lifecycle', 26 | '--web.route-prefix=%s' % $._config.prometheus_web_route_prefix, 27 | ]) + 28 | $.util.resourcesRequests('250m', '1536Mi') + 29 | $.util.resourcesLimits('500m', '2Gi'), 30 | 31 | prometheus_watch_container:: 32 | container.new('watch', $._images.watch) + 33 | container.withArgs([ 34 | '-v', 35 | '-t', 36 | '-p=/etc/prometheus', 37 | 'curl', 38 | '-X', 39 | 'POST', 40 | '--fail', 41 | '-o', 42 | '-', 43 | '-sS', 44 | 'http://localhost:%s%s-/reload' % [$._config.prometheus_port, $._config.prometheus_web_route_prefix], 45 | ]), 46 | 47 | local deployment = $.apps.v1beta1.deployment, 48 | 49 | prometheus_deployment: 50 | if $._config.stateful 51 | then {} 52 | else ( 53 | deployment.new('prometheus', 1, [ 54 | $.prometheus_container, 55 | $.prometheus_watch_container, 56 | ]) + 57 | $.util.configVolumeMount('prometheus-config', '/etc/prometheus') + 58 | deployment.mixin.spec.template.metadata.withAnnotations({ 'prometheus.io.path': '%smetrics' % $._config.prometheus_web_route_prefix }) + 59 | deployment.mixin.spec.template.spec.securityContext.withRunAsUser(0) + 60 | if $._config.enable_rbac 61 | then deployment.mixin.spec.template.spec.withServiceAccount('prometheus') 62 | else {} 63 | ), 64 | 65 | local pvc = $.core.v1.persistentVolumeClaim, 66 | 67 | prometheus_pvc:: 68 | if ! $._config.stateful 69 | then {} 70 | else ( 71 | pvc.new() + 72 | pvc.mixin.metadata.withName('prometheus-data') + 73 | pvc.mixin.spec.withAccessModes('ReadWriteOnce') + 74 | pvc.mixin.spec.resources.withRequests({ storage: '300Gi' }) 75 | ), 76 | 77 | local statefulset = $.apps.v1beta1.statefulSet, 78 | local volumeMount = $.core.v1.volumeMount, 79 | 80 | prometheus_statefulset: 81 | if ! $._config.stateful 82 | then {} 83 | else ( 84 | statefulset.new('prometheus', 1, [ 85 | $.prometheus_container.withVolumeMountsMixin( 86 | volumeMount.new('prometheus-data', '/prometheus') 87 | ), 88 | $.prometheus_watch_container, 89 | ], $.prometheus_pvc) + 90 | $.util.configVolumeMount('prometheus-config', '/etc/prometheus') + 91 | statefulset.mixin.spec.withServiceName('prometheus') + 92 | statefulset.mixin.spec.template.metadata.withAnnotations({ 'prometheus.io.path': '%smetrics' % $._config.prometheus_web_route_prefix }) + 93 | statefulset.mixin.spec.template.spec.securityContext.withRunAsUser(0) + 94 | if $._config.enable_rbac 95 | then statefulset.mixin.spec.template.spec.withServiceAccount('prometheus') 96 | else {} 97 | ), 98 | 99 | prometheus_service: 100 | $.util.serviceFor( 101 | if $._config.stateful 102 | then $.prometheus_statefulset 103 | else $.prometheus_deployment 104 | ), 105 | } -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.9.7 2 | -------------------------------------------------------------------------------- /project/build.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.lihaoyi" % "scalatex-sbt-plugin" % "0.3.11") 2 | -------------------------------------------------------------------------------- /pythonparse/src/pythonparse/Lexical.scala: -------------------------------------------------------------------------------- 1 | package pythonparse 2 | import fastparse.NoWhitespace._ 3 | /** 4 | * Python's lexical grammar; how basic tokens get parsed. This stuff is 5 | * sensitive to whitespace, which can only appear where it's explicitly 6 | * stated to be part of the grammar. 7 | * 8 | * Manually transcribed from https://docs.python.org/2/reference/lexical_analysis.html 9 | */ 10 | object Lexical { 11 | import fastparse._ 12 | 13 | def kw[$: P](s: String) = s ~ !(letter | digit | "_") 14 | def comment[$: P] = P( "#" ~ CharsWhile(_ != '\n', 0) ) 15 | def wscomment[$: P] = P( (CharsWhileIn(" \n") | Lexical.comment | "\\\n").rep ) 16 | def nonewlinewscomment[$: P] = P( (CharsWhileIn(" ") | Lexical.comment | "\\\n").rep ) 17 | 18 | def identifier[$: P]: P[Ast.identifier] = 19 | P( (letter|"_") ~ (letter | digit | "_").rep ).!.filter(!keywordList.contains(_)).map(Ast.identifier.apply) 20 | def letter[$: P] = P( lowercase | uppercase ) 21 | def lowercase[$: P] = P( CharIn("a-z") ) 22 | def uppercase[$: P] = P( CharIn("A-Z") ) 23 | def digit[$: P] = P( CharIn("0-9") ) 24 | 25 | val keywordList = Set( 26 | "and", "del", "from", "not", "while", 27 | "as", "elif", "global", "or", "with", 28 | "assert", "else", "if", "pass", "yield", 29 | "break", "except", "import", "print", 30 | "class", "exec", "in", "raise", 31 | "continue", "finally", "is", "return", 32 | "def", "for", "lambda", "try" 33 | ) 34 | 35 | def stringliteral[$: P]: P[String] = P( stringprefix.? ~ (longstring | shortstring) ) 36 | def stringprefix[$: P]: P[Unit] = P( 37 | "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR" | "b" | "B" | "br" | "Br" | "bR" | "BR" 38 | ) 39 | def shortstring[$: P]: P[String] = P( shortstring0("'") | shortstring0("\"") ) 40 | def shortstring0[$: P](delimiter: String) = P( delimiter ~ shortstringitem(delimiter).rep.! ~ delimiter) 41 | def shortstringitem[$: P](quote: String): P[Unit] = P( shortstringchar(quote) | escapeseq ) 42 | def shortstringchar[$: P](quote: String): P[Unit] = P( CharsWhile(!s"\\\n${quote(0)}".contains(_)) ) 43 | 44 | def longstring[$: P]: P[String] = P( longstring0("'''") | longstring0("\"\"\"") ) 45 | def longstring0[$: P](delimiter: String) = P( delimiter ~ longstringitem(delimiter).rep.! ~ delimiter) 46 | def longstringitem[$: P](quote: String): P[Unit] = P( longstringchar(quote) | escapeseq | !quote ~ quote.take(1) ) 47 | def longstringchar[$: P](quote: String): P[Unit] = P( CharsWhile(!s"\\${quote(0)}".contains(_)) ) 48 | 49 | def escapeseq[$: P]: P[Unit] = P( "\\" ~ AnyChar ) 50 | 51 | def negatable[T, $: P](p: => P[T])(implicit ev: Numeric[T]) = (("+" | "-").?.! ~ p).map { 52 | case ("-", i) => ev.negate(i) 53 | case (_, i) => i 54 | } 55 | 56 | def longinteger[$: P]: P[BigInt] = P( integer ~ ("l" | "L") ) 57 | def integer[$: P]: P[BigInt] = negatable[BigInt, Any](P( octinteger | hexinteger | bininteger | decimalinteger)) 58 | def decimalinteger[$: P]: P[BigInt] = P( nonzerodigit ~ digit.rep | "0" ).!.map(scala.BigInt(_)) 59 | def octinteger[$: P]: P[BigInt] = P( "0" ~ ("o" | "O") ~ octdigit.rep(1).! | "0" ~ octdigit.rep(1).! ).map(scala.BigInt(_, 8)) 60 | def hexinteger[$: P]: P[BigInt] = P( "0" ~ ("x" | "X") ~ hexdigit.rep(1).! ).map(scala.BigInt(_, 16)) 61 | def bininteger[$: P]: P[BigInt] = P( "0" ~ ("b" | "B") ~ bindigit.rep(1).! ).map(scala.BigInt(_, 2)) 62 | def nonzerodigit[$: P]: P[Unit] = P( CharIn("1-9") ) 63 | def octdigit[$: P]: P[Unit] = P( CharIn("0-7") ) 64 | def bindigit[$: P]: P[Unit] = P( "0" | "1" ) 65 | def hexdigit[$: P]: P[Unit] = P( digit | CharIn("a-f", "A-F") ) 66 | 67 | 68 | def floatnumber[$: P]: P[BigDecimal] = negatable[BigDecimal, Any](P( pointfloat | exponentfloat )) 69 | def pointfloat[$: P]: P[BigDecimal] = P( intpart.? ~ fraction | intpart ~ "." ).!.map(BigDecimal(_)) 70 | def exponentfloat[$: P]: P[BigDecimal] = P( (intpart | pointfloat) ~ exponent ).!.map(BigDecimal(_)) 71 | def intpart[$: P]: P[BigDecimal] = P( digit.rep(1) ).!.map(BigDecimal(_)) 72 | def fraction[$: P]: P[Unit] = P( "." ~ digit.rep(1) ) 73 | def exponent[$: P]: P[Unit] = P( ("e" | "E") ~ ("+" | "-").? ~ digit.rep(1) ) 74 | 75 | 76 | def imagnumber[$: P] = P( (floatnumber | intpart) ~ ("j" | "J") ) 77 | } 78 | -------------------------------------------------------------------------------- /pythonparse/test/resources/bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/com-lihaoyi/fastparse/ff1dfacb0db6c64ae0d5ba39e16c9f66c3d1c763/pythonparse/test/resources/bench.py -------------------------------------------------------------------------------- /pythonparse/test/resources/parse.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import ast 3 | ast.parse(open(sys.argv[1]).read()) -------------------------------------------------------------------------------- /pythonparse/test/src-jvm/pythonparse/ProjectTests.scala: -------------------------------------------------------------------------------- 1 | package pythonparse 2 | import concurrent.ExecutionContext.Implicits.global 3 | import scala.concurrent.duration.Duration 4 | import scala.concurrent.{Await, Future} 5 | import java.nio.file._ 6 | import utest._ 7 | import fastparse._ 8 | /** 9 | * Load external Python code and force feed it through the parser 10 | * to find out where it blows up 11 | */ 12 | object ProjectTests extends TestSuite{ 13 | 14 | def check(commitHash: String, 15 | ignored: Seq[String] = Nil) 16 | (implicit testPath: utest.framework.TestPath) = { 17 | val repo = "https://github.com/" + testPath.value.last 18 | val name = repo.split("/").last 19 | val path = Paths.get("out/repos/" + name) 20 | if (!Files.exists(path)) { 21 | println("Cloning " + path) 22 | 23 | new java.lang.ProcessBuilder() 24 | .command("git", "clone", repo, path.toString) 25 | .directory(new java.io.File(".")) 26 | .start() 27 | .waitFor() 28 | 29 | new java.lang.ProcessBuilder() 30 | .command("git", "checkout", commitHash) 31 | .directory(path.toFile) 32 | .start() 33 | .waitFor() 34 | 35 | } 36 | def listFiles(s: java.io.File): Iterator[String] = { 37 | val (dirs, files) = Option(s.listFiles()).toIterator 38 | .flatMap(_.toIterator) 39 | .partition(_.isDirectory) 40 | 41 | files.map(_.getPath) ++ dirs.flatMap(listFiles) 42 | } 43 | 44 | val pythonFiles: Seq[String] = listFiles(new java.io.File(path.toString)) 45 | .filter(path => path.toString.endsWith(".py") && !ignored.exists(path.endsWith)) 46 | .map(_.toString) 47 | .toSeq 48 | 49 | val grouped = Await.result(Future.sequence(pythonFiles.map { p => 50 | Future { 51 | print("-") 52 | import sys.process._ 53 | (Seq("python", "pythonparse/test/resources/parse.py", p).!, p) 54 | } 55 | }), Duration.Inf).groupBy(_._1).mapValues(_.map(_._2)) 56 | val selfParsed = grouped(0) groupBy { x => 57 | print(".") 58 | parse(new String(Files.readAllBytes(Paths.get(x))), pythonparse.Statements.file_input(_)).getClass 59 | } 60 | 61 | selfParsed.get(classOf[Parsed.Failure]) match{ 62 | case None => (grouped.mapValues(_.length), selfParsed.mapValues(_.length)) 63 | case Some(xs) => throw new Exception(xs.mkString("\n")) 64 | } 65 | 66 | } 67 | val tests = Tests { 68 | // Disabled since Github Actions dropped support for Python 2.7 69 | 70 | // "dropbox/changes" - check("37e23c3141b75e4785cf398d015e3dbca41bdd56") 71 | // "django/django" - check( 72 | // "399a8db33b14a1f707912ac48a185fb0a1204913", 73 | // ignored = Seq("tests/i18n/test_compilation.py") 74 | // ) 75 | // "mitsuhiko/flask" - check("9291ead32e2fc8b13cef825186c968944e9ff344") 76 | // "zulip/zulip" - check("b5c107ed27b337ed833ebe9c754889bf078d743e") 77 | // "ansible/ansible"- check("02cd88169764232fd63c776456178fe61d3a214a") 78 | // "kennethreitz/requests" - check("9713289e741960249c94fcb1686746f80e2f20b5") 79 | 80 | // test("test"){ 81 | // val txt = new String(Files.readAllBytes(Paths.get("out/repos/ansible/lib/ansible/modules/cloud/cloudstack/cs_instance.py"))) 82 | // parse(txt).read(pythonparse.Statements.file_input(_)) 83 | // } 84 | // test("bench"){ 85 | // val path = "pythonparse/jvm/src/test/resources/pythonparse/bench.py" 86 | // val data = Files.readAllBytes(Paths.get(path)) 87 | // val code = new String(data) 88 | // import NoWhitespace._ 89 | // parse(code).read(implicit ctx => pythonparse.Statements.file_input ~ End).get 90 | // } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /pythonparse/test/src/pythonparse/RegressionTests.scala: -------------------------------------------------------------------------------- 1 | package pythonparse 2 | import utest._ 3 | 4 | object RegressionTests extends TestSuite{ 5 | import Ast.expr._ 6 | import Ast.stmt._ 7 | import Ast.expr_context._ 8 | import Ast.operator._ 9 | import Ast.unaryop._ 10 | import Ast._ 11 | implicit def strName(s: String): Name = Name(identifier(s), Load) 12 | implicit def strIdent(s: String): identifier = identifier(s) 13 | val tests = Tests { 14 | test("multiple_comments") - TestUtils.check( 15 | Statements.file_input(_), 16 | Seq(Ast.stmt.Pass), 17 | """# a 18 | |# b 19 | |pass""".stripMargin 20 | ) 21 | 22 | test("multiple_newlines") - TestUtils.check( 23 | Statements.file_input(_), 24 | Seq(Expr("a"), Expr("b")), 25 | """a 26 | | 27 | |b""".stripMargin 28 | ) 29 | 30 | test("multi_line_function") - TestUtils.check( 31 | Statements.file_input(_), 32 | Seq(FunctionDef("session_config", arguments(Nil, None, None, Nil), Seq(Expr("a"), Expr("b")), Nil)), 33 | """def session_config(): 34 | | a 35 | | 36 | | b""".stripMargin 37 | ) 38 | 39 | test("backslash_breaks") - TestUtils.check( 40 | Statements.file_input(_), 41 | Seq(Expr(Attribute("a", "b", Load))), 42 | """a\ 43 | |.b 44 | |""".stripMargin 45 | ) 46 | test("multiline_string") - TestUtils.check( 47 | Statements.file_input(_), 48 | Seq(Expr(Str("\n"))), 49 | """''' 50 | |''' 51 | |""".stripMargin 52 | ) 53 | 54 | test("try_finally_no_except") - TestUtils.check( 55 | Statements.file_input(_), 56 | Seq(TryFinally(Seq(Expr("a")), Seq(Expr("b")))), 57 | """try: 58 | | a 59 | |finally: 60 | | b 61 | | 62 | |""".stripMargin 63 | ) 64 | test("indented_try_except_with_space") - TestUtils.check( 65 | Statements.file_input(_), 66 | Seq(FunctionDef("f", arguments(Nil, None, None, Nil), Seq( 67 | TryExcept( 68 | Seq(Pass), 69 | Seq(excepthandler.ExceptHandler(Some("s"), None, Seq(Pass))), 70 | Nil 71 | ) 72 | ), Nil)), 73 | """def f(): 74 | | try: 75 | | pass 76 | | 77 | | except s: 78 | | pass 79 | | 80 | |""".stripMargin 81 | ) 82 | test("indented_block_with_spaces_and_offset_comments") - TestUtils.check( 83 | Statements.file_input(_), 84 | Seq(FunctionDef( 85 | "post", 86 | arguments(Seq(Name("self", Param)), None, None, Nil), 87 | Seq(If(Num(1), Seq(Expr("a")), Nil)), 88 | Nil 89 | )), 90 | """def post(self): 91 | |#LOL 92 | | 93 | | if 1: 94 | | a 95 | |""".stripMargin 96 | ) 97 | test("indented_block_with_spaces_and_offset_comments") - TestUtils.check( 98 | Statements.file_input(_), 99 | Seq(While( 100 | "a", 101 | Seq( 102 | TryExcept( 103 | Seq(Expr("a")), 104 | Seq( 105 | excepthandler.ExceptHandler(None, None, Seq(Return(Some("a")))), 106 | excepthandler.ExceptHandler(None, None, Seq(Expr("a"))) 107 | ), 108 | Nil 109 | ) 110 | ), 111 | Nil 112 | )), 113 | """while a: 114 | | try: a 115 | | except: return a 116 | | except: a 117 | |""".stripMargin 118 | ) 119 | 120 | test("weird_comments") - TestUtils.check( 121 | Statements.file_input(_), 122 | Seq(While( 123 | Num(1), 124 | Seq(Expr("a")), 125 | Nil 126 | )), 127 | """while 1: 128 | | # 129 | | a 130 | |""".stripMargin 131 | ) 132 | test("ident_looking_string") - TestUtils.check( 133 | Statements.file_input(_), 134 | Seq(If( 135 | Call("match", Seq(Str("^[a-zA-Z0-9]")), Nil, None, None), 136 | Seq(Expr("a")), 137 | Nil 138 | )), 139 | """ 140 | |if match(r'^[a-zA-Z0-9]'): 141 | | a 142 | | 143 | |""".stripMargin 144 | ) 145 | test("same_line_comment") - TestUtils.check( 146 | Statements.file_input(_), 147 | Seq(If( 148 | "b", 149 | Seq(If( 150 | "c", 151 | Seq(Pass), 152 | Nil 153 | )), 154 | Nil 155 | )), 156 | """if b: # 157 | | if c: 158 | | pass 159 | |""".stripMargin 160 | ) 161 | test("chained_elifs") - TestUtils.check( 162 | Statements.file_input(_), 163 | Seq(While(Num(1), 164 | Seq( 165 | If("a", 166 | Seq(Expr("a")), 167 | Seq( 168 | If("b", 169 | Seq(Expr("b")), 170 | Seq(If("c", 171 | Seq(Expr("c")), 172 | Nil 173 | ))) 174 | )) 175 | ), 176 | Nil 177 | )), 178 | """while 1: 179 | | if a: 180 | | a 181 | | elif b: 182 | | b 183 | | elif c: 184 | | c 185 | |""".stripMargin 186 | ) 187 | test("bitand") - TestUtils.check( 188 | Statements.file_input(_), 189 | Seq(Expr(BinOp("a", BitAnd, "a"))), 190 | """a & a 191 | |""".stripMargin 192 | ) 193 | test("octal") - TestUtils.check( 194 | Statements.file_input(_), 195 | Seq(Expr(Num(0))), 196 | """0x0 197 | |""".stripMargin 198 | ) 199 | test("comment_after_decorator") - TestUtils.check( 200 | Statements.file_input(_), 201 | Seq(ClassDef("GenericForeignKeyTests", Nil, Seq(Pass), Seq("override_settings"))), 202 | """@override_settings # ForeignKey(unique=True) 203 | |class GenericForeignKeyTests: 204 | | pass 205 | |""".stripMargin 206 | ) 207 | test("while_block") - TestUtils.check( 208 | Statements.file_input(_), 209 | Seq(While(Num(1), Seq(Expr(Num(2)), Expr(Num(3))), Nil)), 210 | """while 1: 211 | | 212 | | 2 213 | | 3 214 | |""".stripMargin 215 | ) 216 | test("while_in_if") - TestUtils.check( 217 | Statements.file_input(_), 218 | Seq(If(Num(1), Seq(While(Num(0), Seq(Pass), Nil)), Seq(Pass))), 219 | """if 1: 220 | | while 0: 221 | | pass 222 | |else: 223 | | pass 224 | |""".stripMargin 225 | ) 226 | 227 | test("tab_indent") - TestUtils.check( 228 | Statements.file_input(_), 229 | Seq(While(Num(1), Seq(Pass), Nil)), 230 | "while 1:\n\tpass" 231 | ) 232 | 233 | test("negative_integer") - TestUtils.check( 234 | Statements.file_input(_), 235 | Seq(Expr(Num(-1))), 236 | "-1" 237 | ) 238 | test("unary_subtraction") - TestUtils.check( 239 | Statements.file_input(_), 240 | Seq(Expr(UnaryOp(USub, Name(identifier("foo"), Load)))), 241 | "-foo" 242 | ) 243 | } 244 | } 245 | 246 | -------------------------------------------------------------------------------- /pythonparse/test/src/pythonparse/TestUtils.scala: -------------------------------------------------------------------------------- 1 | package pythonparse 2 | 3 | import utest._ 4 | 5 | /** 6 | * Created by haoyi on 10/8/15. 7 | */ 8 | object TestUtils { 9 | import fastparse._ 10 | def check[T](rule: P[_] => P[T], expected: T, s: String) = { 11 | import fastparse.NoWhitespace._ 12 | def parseIt[$: P] = rule(P.current) ~ End 13 | val parsed = parse(s, parseIt(_)) 14 | val stringResult = parsed match { 15 | case f: Parsed.Failure => throw new Exception(f.trace().longTerminalsMsg) 16 | case s: Parsed.Success[T] => 17 | val result = s.value 18 | assert(result == expected) 19 | result 20 | } 21 | 22 | // for(chunkSize <- Seq(1, 4, 16, 64, 256, 1024)){ 23 | // val parsed = (rule ~ End).parse(s.grouped(chunkSize)) 24 | // val stringResult = parsed match { 25 | // case f: Result.Failure => throw new Exception(f.extra.traced.trace) 26 | // case s: Result.Success[T] => 27 | // val result = s.value 28 | // assert(result == expected) 29 | // } 30 | // } 31 | 32 | stringResult 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /readme/ApiHighlights.scalatex: -------------------------------------------------------------------------------- 1 | @import Main._ 2 | @val tests = wd/'fastparse/'test/'src/'fastparse 3 | @val main = wd/'fastparse/'src/'fastparse 4 | 5 | @sect{API Highlights} 6 | @sect{Parsing Results} 7 | @p 8 | The result of a parser comes in two flavors of @hl.scala{Parsed}; 9 | the first is a success (@hl.scala{Parsed.Success}) and the second 10 | is a failure (@hl.scala{Parsed.Failure}). @hl.scala{Parsed.Success} 11 | provides the parsed value - the value you are probably most 12 | interested in - and the index in the input string till where the 13 | parse was performed. @hl.scala{Parsed.Failure} allows you to 14 | retrieve the last parser that failed and the index where it failed. 15 | Additionally, failure provides an @hl.scala{Parsed.Failure.extra} 16 | field that provides precise details about the failure, in particular, 17 | and most importantly a complete stack trace of the involved parsers, 18 | which is accessible via @hl.scala{extra.traced}. 19 | 20 | @hl.ref(tests/"ExampleTests.scala", start = "sealed trait AndOr", end = "}") 21 | 22 | @p 23 | It is also possible to pattern match over @hl.scala{Parsed}, however, you may experience spurious warnings related to @a("SI-4440", href:="https://issues.scala-lang.org/browse/SI-4440"). 24 | In order to prevent these warnings @hl.scala{import fastparse.core.Result} in versions 0.2.x and @hl.scala{import fastparse.core.Parsed} in higher versions than 0.2.x. 25 | 26 | @p 27 | An overview of @hl.scala{Parsed}: 28 | 29 | @hl.ref(main/"Parsed.scala") 30 | 31 | @p 32 | Note how @hl.scala{Failure} only contains the parser which failed 33 | and a single index where the parse failed. Further debugging 34 | information is available via the @hl.scala{Failure.Extra} class. 35 | Especially the @hl.scala{TracedFailure} that is lazily-computed via 36 | @hl.scala{Extra.traced}, provides valuable information: It performs 37 | a whole new parse on the input data with additional instrumentation, 38 | and provides additional insight into why the parse failed. 39 | 40 | @p 41 | Computing the @hl.scala{Extra.traced} data is not done by default 42 | for performance reasons: the additional run takes about 3x longer 43 | than the initial run due to the instrumentation, for a total of 4x 44 | slowdown. If you want the information for debugging, though, it 45 | will be there. 46 | -------------------------------------------------------------------------------- /readme/Comparisons.scalatex: -------------------------------------------------------------------------------- 1 | @import Main._ 2 | @sect{Comparisons} 3 | @p 4 | FastParse differs from all other parser-combinator libraries in the 5 | Scala universe, in quite substantial ways: 6 | 7 | @ul 8 | @li 9 | Compared to @a("Parboiled2", href:="https://github.com/sirthias/parboiled2"), 10 | FastParse has much better error messages (both compile and run-time), 11 | has much simpler and more predictable behavior. Parboiled2 suffers from many 12 | @a("usability problems and bugs", href:="https://groups.google.com/forum/#!msg/scala-internals/4N-uK5YOtKI/9vAdsH1VhqAJ") 13 | that make it excruciating difficult to use, and also does not 14 | support higher-order rules. FastParse suffers from none of these 15 | problems. 16 | @li 17 | @a("Parboiled1", href:="https://github.com/sirthias/parboiled/wiki") 18 | is a Java library, and does not/cannot work on Scala.js 19 | @li 20 | @a("scala-parser-combinators", href:="https://github.com/scala/scala-parser-combinators") 21 | is similar, but poorly executed. It is ~2-300x slower than FastParse, 22 | has an awkward inheritance-based API, and is full of bugs despite 23 | being half a decade old. FastParse is faster, has self-contained 24 | pure-functional parsers, and fixes bugs e.g. by having the 25 | @hl.scala{.log} operator actually work. 26 | -------------------------------------------------------------------------------- /readme/ErrorReportingInternals.scalatex: -------------------------------------------------------------------------------- 1 | @import Main._ 2 | @val tests = wd/'fastparse/'test/'src/'fastparse 3 | @sect{Error Reporting Internals} 4 | @p 5 | This section goes into detail of how the FastParse error reporting 6 | algorithm works. In general, it should "just work" when you call 7 | @code{.longMsg}, @code{.longAggregateMsg}, or @code{.longTerminalsMsg}. 8 | Nevertheless, it is both complicated as well important enough that it 9 | is worth documenting in detail 10 | 11 | @p 12 | The two levels of error reporting that are most interesting are 13 | @code{.longAggregateMsg} and @code{.longTerminalsMsg}. Consider a failed 14 | parse of an example simplified arithmetic parser: 15 | 16 | @hl.ref(tests/"ExampleTests.scala", Seq("\"errorHandlingExplanation\"", "")) 17 | 18 | @p 19 | This fails on the @code{?} being invalid syntax. The following error reporting 20 | levels will treat this as follows: 21 | @ul 22 | @li 23 | @code{terminalMsgs} lists all the lowest-level terminal parsers which are 24 | tried at the given @code{traceIndex}, i.e. the character class @code{[0-9]} and the 25 | token @hl.scala{"("}. This is useful to answer the question "what token/char can I 26 | put at the error position to make my parse continue". The implementation 27 | of @code{terminalMsgs} is straightforward: we simply call 28 | @code{reportTerminalMsg} in every terminal parser, which collects all the 29 | messages in a big list and returns it. 30 | @li 31 | @code{aggregateMsgs} lists all high-level parsers which are tried at the given 32 | @code{traceIndex}, i.e. the named parsers @code{num} and @code{plus}. This is useful to 33 | answer the question "What construct was the parser trying to do when it 34 | failed" 35 | @p 36 | The implementation of @code{aggregateMsgs} is more interesting, since we need 37 | to define what "high level" parsers means, which is non-obvious. 38 | 39 | @sect{Definition of aggregateMsgs} 40 | @p 41 | Fastparse uses the following definition for @code{aggregateMsgs}: 42 | @ul 43 | @li 44 | @code{aggregateMsgs} should contain the parsers highest in the call stack, 45 | whose failure isn't immediately fatal to the parse (due to them being in 46 | @code{|}, @code{.rep}, @code{?}, or other "backtrackable" operators, but 47 | not past a @code{cut}) 48 | @p 49 | This is a useful definition because we already have the @code{failureStack} 50 | containing all (named) parsers whose failure *is* immediately fatal to the 51 | parse, both those at @code{traceIndex} and those earlier in the input. Thus 52 | there is no need to duplicate showing any of them in the @code{aggregateMsgs}, 53 | and we can instead go "one level deeper" to find the highest-level parsers 54 | within the deepest parser of the @code{failureStack} and show those instead. 55 | Thus, in the combined @code{longAggregateMsg}, the failure stack shows us 56 | exactly which parsers failing directly contributed to the failure at 57 | @code{traceIndex}, while the longAggregateMsg tells us what are the 58 | highest-level parsers FastParse was trying to parse at @code{traceIndex} before 59 | it finally failed. 60 | @sect{Implementation of aggregateMsgs} 61 | @p 62 | To collect the @code{aggregateMsgs}, We use the following algorithm: 63 | @ul 64 | @li 65 | When a parse which started at the given @code{traceIndex} fails without a cut: 66 | Over-write @code{aggregateMsgs} with it's @code{shortMsg} 67 | 68 | @li 69 | Otherwise: 70 | 71 | @ul 72 | @li 73 | If we are a terminal parser, we set our @code{aggregateMsgs} to Nil 74 | @li 75 | If we are a compound parser, we simply sum up the @code{aggregateMsgs} 76 | of all our constituent parts 77 | @p 78 | As mentioned earlier, the point of this is to provide the highest-level parsers which 79 | failed at the @code{traceIndex}, but are not already part of the @code{failureStack}. 80 | non-highest-level parsers do successfully write their message to 81 | @code{aggregateMsgs}, but they are subsequently over-written by the higher 82 | level parsers, until it reaches the point where @code{cut == true}, indicating 83 | that any further higher-level parsers will be in @code{failureStack} and using 84 | their message to stomp over the existing parse-failure-messages in 85 | @code{aggregateMsgs} would be wasteful. 86 | @sect{Edge Cases} 87 | @p 88 | These is an edge case where there is no given failure that occurs exactly at 89 | @code{traceIndex} e.g. 90 | @ul 91 | @li 92 | Parsing @hl.scala{"ax"} with @hl.scala{P( ("a" ~ "b") ~ "c" | "a" ~/ "d" )} 93 | @li 94 | The final failure @code{index} and thus @code{traceIndex} is at offset 1 95 | @li 96 | We would like to receive the aggregation @hl.scala{("b" | "d")} 97 | @li 98 | But @hl.scala{("a" ~ "b")} passes from offsets 0-2, @hl.scala{"c"} fails 99 | 100 | 101 | @p 102 | In such a case, we truncate the @code{shortMsg} at 103 | @code{traceIndex} to only include the portion we're interested in (which directly 104 | follows the failure). This then gets aggregated nicely to form the error 105 | message from-point-of-failure. 106 | @p 107 | A follow-on edge case is parsing @hl.scala{"ax"} with 108 | @hl.scala 109 | val inner = P( "a" ~ "b" ) 110 | P( inner ~ "c" | "a" ~/ "d" ) 111 | @ul 112 | @li 113 | Here, we find that the @code{inner} parser starts before the @code{traceIndex} and 114 | fails at @code{traceIndex}, 115 | @li 116 | But we want our aggregation to continue being @hl.scala{("b" | "d")}, rather than 117 | @hl.scala{(inner | "d")}. 118 | 119 | Thus, for opaque compound parsers like @code{inner} which do not expose their 120 | internals, we use @code{forceAggregate} to force it to expose it's internals 121 | when it's range covers the @code{traceIndex} but it isn't an exact match 122 | -------------------------------------------------------------------------------- /readme/FastParseInternals.scalatex: -------------------------------------------------------------------------------- 1 | @import Main._ 2 | @sect{Internals} 3 | @p 4 | FastParse is implemented as a set of methods that perform a 5 | recursive-descent parse on the given input, with all book-keeping 6 | information maintained in the @code{fastparse.ParsingRun[T]} objects 7 | (abbreviated @code{fastparse.P[T]}). @code{ParsingRun}s are mutable, 8 | heavily stateful objects that keep track of everything related to the 9 | parse: the current index, whether backtracking is allowed, any value we 10 | may want to return, etc.. By defining your parsers as: 11 | 12 | @hl.scala 13 | def myParser[_: P]: P[T] = P( ... ) 14 | 15 | @p 16 | We ensure that a @code{ParsingRun} object is required to start the 17 | parse, the same instance is made available implicitly to any other 18 | parsers @code{myParser} may call and finally returned with the relevant 19 | value of type @code{T} stored in it. 20 | 21 | @sect{Inlining} 22 | 23 | @p 24 | FastParse heavily relies on inlining to achieve performance; many 25 | of the FastParse operators such as @code{a ~ b}, @code{!a}, 26 | @code{a.rep} etc. are implemented using macros which inline their 27 | implementation code at the call-site. 28 | @p 29 | This inlining should be mostly transparent to you, as the operators 30 | would parse the same input and return the same values as if they 31 | were not inlined. The only thing you may notice is these operators 32 | do not appear in stack traces or profiles, as their code is inlined 33 | as part of the enclosing method (e.g. @hl.scala{def myParser}). 34 | 35 | @sect{Opacity} 36 | @p 37 | Fastparse parsers are opaque: as plain methods 38 | @hl.scala{def myParser[_: P]: P[T]}, the only thing you can do is 39 | invoke them. You cannot programmatically inspect the body of a 40 | parser, see what other parsers it depends on, or perform any 41 | transformations on it. This in turn allows us to perform additional 42 | optimizations to improve performance of the parsing run. 43 | 44 | @sect{Synchronous} 45 | @p 46 | As plain methods, Fastparse parsers are synchrous: calling a parser 47 | method does not return until the parse is complete. Even if 48 | parsing streaming input, FastParse will block 49 | on the input iterator until it either provides the text to parse 50 | or is exhausted. 51 | 52 | @sect{Stack-Limited} 53 | @p 54 | FastParse's parsing methods use the normal JVM method-call-stack 55 | to perform their recursive descent. This means that excessively 56 | deep or nested parses can cause a stack-overflow, which can be 57 | mitigated by the normal JVM flags to increase the stack size in 58 | memory. -------------------------------------------------------------------------------- /readme/GettingStarted.scalatex: -------------------------------------------------------------------------------- 1 | @import Main._ 2 | 3 | @val tests = wd/'fastparse/'test/'src/'fastparse 4 | @val main = wd/'fastparse/'src/'fastparse 5 | 6 | @hl.ref(tests/"MathTests.scala", "import fastparse._", "val tests") 7 | @hl.ref(tests/"MathTests.scala", Seq("\"pass\"", "")) 8 | @div(id := "splashdiv") 9 | @script(raw("""DemoMain.math(document.getElementById("splashdiv"))""")) 10 | @p 11 | FastParse is a Scala library for parsing strings and bytes into structured 12 | data. This lets you easily write a parser for any arbitrary textual data 13 | formats (e.g. program source code, JSON, ...) and have the parsers run at 14 | an acceptable speed, with great error debuggability and error reporting. 15 | Features include: 16 | 17 | @ul 18 | @li 19 | @sect.ref("Performance", "Comparable in speed to a hand-written parser"), 20 | 200x faster than @a("scala-parser-combinators", 21 | href:="https://github.com/scala/scala-parser-combinators") 22 | @li 23 | 1/10th the size of a hand-written parser 24 | @li 25 | Automatic, excellent @sect.ref("Debugging Parsers", "error-reporting and diagnostics"). 26 | @li 27 | @sect.ref{Streaming Parsing} of data-sets to avoid pre-loading 28 | everything into memory 29 | @li 30 | Compatible with both Scala-JVM and Scala.js 31 | 32 | @p 33 | This documentation serves as a thorough reference on how to use this library. 34 | For a more hands-on introduction, take a look at the following blog post: 35 | 36 | @ul 37 | @li 38 | @lnk("Easy Parsing with Parser Combinators", "http://www.lihaoyi.com/post/EasyParsingwithParserCombinators.html") 39 | 40 | @p 41 | FastParse is a project by @lnk("Li Haoyi", "http://www.lihaoyi.com/"). If 42 | you use FastParse and enjoyed it, please chip in to support our development 43 | at @lnk("https://www.patreon.com/lihaoyi", "https://www.patreon.com/lihaoyi"). 44 | 45 | @p 46 | The following sections will introduce you to FastParse and how to use it. 47 | 48 | @sect{Getting Started} 49 | @p 50 | To begin using FastParse, add the following to your build config 51 | 52 | @hl.scala 53 | "com.lihaoyi" %% "fastparse" % "3.1.1" // SBT 54 | ivy"com.lihaoyi::fastparse:3.1.1" // Mill 55 | 56 | @p 57 | To use with Scala.js, you'll need 58 | @hl.scala 59 | "com.lihaoyi" %%% "fastparse" % "3.1.1" // SBT 60 | ivy"com.lihaoyi::fastparse::3.1.1" // Mill 61 | 62 | -------------------------------------------------------------------------------- /readme/Performance.scalatex: -------------------------------------------------------------------------------- 1 | @import Main._ 2 | @sect{Performance} 3 | @p 4 | FastParse will never be able to compete with hand-written recursive 5 | descent parsers for speed, but for most use cases it is plenty fast 6 | enough. Here's a comparison of FastParse with alternatives, using 7 | Parboiled2's JSON parsing benchmark, which parses a ~21,500 line JSON file: 8 | 9 | @table(width := "100%", cls := "pure-table") 10 | @thead 11 | @th{Benchmark}@th{Score} 12 | @tbody 13 | @tr 14 | @td{fastparse}@td{159.5} 15 | @tr 16 | @td{circe}@td{332.4} 17 | @tr 18 | @td{argonaut}@td{149.1} 19 | @tr 20 | @td{uJson}@td{266.6} 21 | @tr 22 | @td{json4s}@td{100.9} 23 | @tr 24 | @td{play-json}@td{226.6} 25 | @tr 26 | @td{scala-parser-combinators}@td{0.9} 27 | 28 | 29 | @p 30 | These numbers are the number of iterations/second of parsing a sample 31 | @code{test.json} file, averaged over 200 runs. As you can see, the 32 | FastParse based parser comes within a factor of 4 of the fastest hand 33 | written parser (Jackson), is just as fast as the Parboiled2 based 34 | parser (slightly faster/slower depending if full tracing is enabled), 35 | and is almost 100x faster than the scala-parser-combinators library. 36 | 37 | @p 38 | In exchange for the perf hit compared to hand-rolled solutions, you get 39 | the @sect.ref("Json", "short, super-simple parser definition"), and 40 | excellent error free error reporting. While for super-high-performance 41 | use cases you may still want a hand-rolled parser, for many ad-hoc 42 | situations a FastParse parser would do just fine. 43 | 44 | @p 45 | A similar speed ratio can be seen in parsing a 46 | @a("sample Scala file", href:="https://github.com/scala-js/scala-js/blob/master/compiler/src/main/scala/org/scalajs/core/compiler/GenJSCode.scala") 47 | comparing FastParse andScalac's inbuilt hand-written Scala-language parser: 48 | 49 | @table(width := "100%", cls := "pure-table") 50 | @thead 51 | @th{Benchmark}@th{Score} 52 | @tbody 53 | @tr 54 | @td{fastparse}@td{203} 55 | @tr 56 | @td{scalac}@td{754} 57 | 58 | @p 59 | Or comparing Fastparse's Python parser with the Jython Python parser: 60 | 61 | @table(width := "100%", cls := "pure-table") 62 | @thead 63 | @th{Benchmark}@th{Score} 64 | @tbody 65 | @tr 66 | @td{fastparse}@td{406} 67 | @tr 68 | @td{jython}@td{472} 69 | @p 70 | In all these cases, you can see that the iterations-per-second 71 | performance of Fastparse parsers is comparable to various production 72 | quality parsers. While the Fastparse parser may be a few times slower, 73 | it is nonetheless competitive, and at the same time is usually less 74 | than 1/10th as much code to write, understand and debug. 75 | 76 | @sect{Improving Performance} 77 | 78 | @p 79 | There are many ways to improve performance of your FastParse parsers. 80 | If you study the example parsers included in the repo, those already 81 | have many of these techniques applied, and if you follow the same style 82 | you'll probably do ok. Nevertheless, here are some concrete tips: 83 | 84 | @ul 85 | @li 86 | @b{Understand your Parser's behavior}: using @sect.ref{Log}, or 87 | by @code{Instrumenting Parsers}. Often poor performance is due 88 | to parsers doing the wrong thing: attempting more alternatives 89 | than they need to, or backtracking and repeating the same 90 | parse many times. Understanding the flow of how your parser works 91 | is the first step in identifying these issues and fixing them 92 | 93 | @li 94 | @b{Avoid Backtracking}: FastParse parsers have unlimited backtracking, 95 | which is convenient for getting something working initially, but 96 | inconvenient when you want things to be fast. If you have a parser 97 | with lots of backtracking, see if you can factor out parts of it 98 | so they only get parsed once, e.g. turning @code{a ~ b | a ~ c} into 99 | @code{a ~ (b | c)} 100 | 101 | @li 102 | @b{Use @sect.ref{Cuts}}: although you can remove backtracking manually, 103 | it is easy to make a mistake and miss some of it, or for backtracking 104 | to creep back in as you make further changes to your parser. Cuts 105 | prevent that, ensuring that your parser never backtracks past certain 106 | points no matter what. 107 | 108 | @li 109 | @b{Use @sect.ref{Utilities}}: things like @sect.ref{CharPred}, 110 | @sect.ref{CharIn}, @sect.ref{CharsWhile}, @sect.ref{StringIn}, 111 | @sect.ref{CharsWhileIn} are orders of magnitude faster than 112 | implementing their behavior yourself with @code{|} and @code{.rep}. 113 | Use them where-ever possible 114 | 115 | @sect{Profiling} 116 | @p 117 | Since FastParse Parsers are just methods, you can use standard 118 | profiling techniques to show where in the parser time is being 119 | spent. For example, here is the @lnk("JProfiler", "https://www.ej-technologies.com/products/jprofiler/overview.html") 120 | profile of the @sect.ref{ScalaParse} Scala syntax parser: 121 | 122 | @img(width := "100%", src := "JProfiler.png") 123 | 124 | @p 125 | Using standard tools, you can easily dig into what parts of your 126 | parser are slow and why -------------------------------------------------------------------------------- /readme/Readme.scalatex: -------------------------------------------------------------------------------- 1 | @import Main._ 2 | 3 | @script 4 | (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ 5 | (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), 6 | m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) 7 | })(window,document,'script','https://www.google-analytics.com/analytics.js','ga'); 8 | 9 | ga('create', 'UA-27464920-6', 'auto'); 10 | ga('send', 'pageview'); 11 | 12 | 13 | @a( 14 | href:="https://github.com/lihaoyi/fastparse", 15 | position.absolute, 16 | top:=0,right:=0,border:=0, 17 | img( 18 | src:="https://camo.githubusercontent.com/a6677b08c955af8400f44c6298f40e7d19cc5b2d/68747470733a2f2f73332e616d617a6f6e6177732e636f6d2f6769746875622f726962626f6e732f666f726b6d655f72696768745f677261795f3664366436642e706e67", 19 | alt:="Fork me on GitHub" 20 | ) 21 | ) 22 | 23 | @sect("FastParse 3.1.1", "Fast to write, Fast running Parsers in Scala") 24 | @GettingStarted() 25 | 26 | @WritingParsers() 27 | 28 | @StreamingParsing() 29 | 30 | @ExampleParsers() 31 | 32 | @ApiHighlights() 33 | 34 | @Performance() 35 | 36 | @DebuggingParsers() 37 | 38 | @Comparisons() 39 | 40 | @FastParseInternals() 41 | 42 | @ErrorReportingInternals() 43 | 44 | @Changelog() 45 | 46 | -------------------------------------------------------------------------------- /readme/resources/JProfiler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/com-lihaoyi/fastparse/ff1dfacb0db6c64ae0d5ba39e16c9f66c3d1c763/readme/resources/JProfiler.png -------------------------------------------------------------------------------- /readme/resources/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/com-lihaoyi/fastparse/ff1dfacb0db6c64ae0d5ba39e16c9f66c3d1c763/readme/resources/favicon.png -------------------------------------------------------------------------------- /scalaparse/src/scalaparse/Core.scala: -------------------------------------------------------------------------------- 1 | package scalaparse 2 | 3 | import syntax.{Basic, Key} 4 | 5 | import scala.language.implicitConversions 6 | import syntax.Identifiers 7 | import fastparse._ 8 | import fastparse._, ScalaWhitespace._ 9 | 10 | import scala.annotation.{switch, tailrec} 11 | trait Core extends syntax.Literals{ 12 | 13 | 14 | implicit class TrailingCommaOps[+T](p0: => P[T]) { 15 | def repTC[R](min: Int = 0, max: Int = Int.MaxValue, exactly: Int = -1) 16 | (implicit ev: fastparse.Implicits.Repeater[T, R], 17 | ctx: P[_]): P[R] = 18 | p0.rep(min = min, sep = ",", max = max, exactly = exactly) ~ TrailingComma 19 | } 20 | // Aliases for common things. These things are used in almost every parser 21 | // in the file, so it makes sense to keep them short. 22 | 23 | import Key._ 24 | // Keywords that match themselves and nothing else 25 | def `=>`[$: P] = (O("=>") | O("⇒")).opaque("\"=>\"") 26 | def `<-`[$: P] = O("<-") | O("←").opaque("\"<-\"") 27 | def `:`[$: P] = O(":") 28 | def `=`[$: P] = O("=") 29 | def `@`[$: P] = O("@") 30 | //def `_`[$: P] = W("_") 31 | def Underscore[$: P] = W("_") 32 | def `this`[$: P] = W("this") 33 | def `type`[$: P] = W("type") 34 | def `val`[$: P] = W("val") 35 | def `var`[$: P] = W("var") 36 | def `def`[$: P] = W("def") 37 | def `with`[$: P] = W("with") 38 | def `package`[$: P] = W("package") 39 | def `object`[$: P] = W("object") 40 | def `class`[$: P] = W("class") 41 | def `case`[$: P] = W("case") 42 | def `trait`[$: P] = W("trait") 43 | def `extends`[$: P] = W("extends") 44 | def `implicit`[$: P] = W("implicit") 45 | def `try`[$: P] = W("try") 46 | def `new`[$: P] = W("new") 47 | def `macro`[$: P] = W("macro") 48 | def `import`[$: P] = W("import") 49 | def `else`[$: P] = W("else") 50 | def `super`[$: P] = W("super") 51 | def `catch`[$: P] = W("catch") 52 | def `finally`[$: P] = W("finally") 53 | def `do`[$: P] = W("do") 54 | def `yield`[$: P] = W("yield") 55 | def `while`[$: P] = W("while") 56 | def `<%`[$: P] = O("<%") 57 | def `override`[$: P] = W("override") 58 | def `#`[$: P] = O("#") 59 | def `forSome`[$: P] = W("forSome") 60 | def `for`[$: P] = W("for") 61 | def `abstract`[$: P] = W("abstract") 62 | def `throw`[$: P] = W("throw") 63 | def `return`[$: P] = W("return") 64 | def `lazy`[$: P] = W("lazy") 65 | def `if`[$: P] = W("if") 66 | def `match`[$: P] = W("match") 67 | def `>:`[$: P] = O(">:") 68 | def `<:`[$: P] = O("<:") 69 | def `final`[$: P] = W("final") 70 | def `sealed`[$: P] = W("sealed") 71 | def `private`[$: P] = W("private") 72 | def `protected`[$: P] = W("protected") 73 | 74 | 75 | // kinda-sorta keywords that are common patterns even if not 76 | // really-truly keywords 77 | def `*`[$: P] = O("*") 78 | // def `_*`[$: P] = P( `_` ~ `*` ) 79 | def `Underscore*`[$: P] = P( Underscore ~ `*` ) 80 | def `}`[$: P] = P( Semis.? ~ "}" ) 81 | def `{`[$: P] = P( "{" ~ Semis.? ) 82 | /** 83 | * helper printing function 84 | */ 85 | 86 | def Id[$: P] = P( WL ~ Identifiers.Id ) 87 | def VarId[$: P] = P( WL ~ Identifiers.VarId ) 88 | def BacktickId[$: P] = P( WL ~ Identifiers.BacktickId ) 89 | def ExprLiteral[$: P] = P( WL ~ Literals.Expr.Literal ) 90 | def PatLiteral[$: P] = P( WL ~ Literals.Pat.Literal ) 91 | 92 | def QualId[$: P] = P( WL ~ Id.rep(1, sep = ".") ) 93 | def Ids[$: P] = P( Id.rep(1, sep = ",") ) 94 | 95 | /** 96 | * Sketchy way to whitelist a few suffixes that come after a . select; 97 | * apart from these and IDs, everything else is illegal 98 | */ 99 | def PostDotCheck[$: P]: P[Unit] = P( WL ~ !(`super` | `this` | "{" | Underscore | `type`) ) 100 | def ClassQualifier[$: P] = P( "[" ~ Id ~ "]" ) 101 | def ThisSuper[$: P] = P( `this` | `super` ~ ClassQualifier.? ) 102 | def ThisPath[$: P]: P[Unit] = P( ThisSuper ~ ("." ~ PostDotCheck ~/ Id).rep ) 103 | def IdPath[$: P]: P[Unit] = P( Id ~ ("." ~ PostDotCheck ~/ (`this` | Id)).rep ~ ("." ~ ThisPath).? ) 104 | def StableId[$: P]: P[Unit] = P( ThisPath | IdPath ) 105 | } 106 | -------------------------------------------------------------------------------- /scalaparse/src/scalaparse/Exprs.scala: -------------------------------------------------------------------------------- 1 | package scalaparse 2 | 3 | import fastparse._, ScalaWhitespace._ 4 | trait Exprs extends Core with Types with Xml{ 5 | def AnonTmpl[$: P]: P[Unit] 6 | def BlockDef[$: P]: P[Unit] 7 | 8 | def Import[$: P]: P[Unit] = { 9 | def Selector: P[Unit] = P( (Id | Underscore) ~ (`=>` ~/ (Id | Underscore)).? ) 10 | def Selectors: P[Unit] = P( "{" ~/ Selector.repTC() ~ "}" ) 11 | def ImportExpr: P[Unit] = P( StableId ~ ("." ~/ (Underscore | Selectors)).? ) 12 | P( `import` ~/ ImportExpr.rep(1, sep = ","./) ) 13 | } 14 | 15 | // Depending on where an expression is located, subtle behavior around 16 | // semicolon inference and arrow-type-ascriptions like i: a => b 17 | // varies. 18 | 19 | // Expressions used as statements, directly within a {block} 20 | object StatCtx extends WsCtx(semiInference=true, arrowTypeAscriptions=false) 21 | // Expressions nested within other expressions 22 | object ExprCtx extends WsCtx(semiInference=false, arrowTypeAscriptions=true) 23 | // Expressions directly within a `val x = ...` or `def x = ...` 24 | object FreeCtx extends WsCtx(semiInference=true, arrowTypeAscriptions=true) 25 | 26 | def TypeExpr[$: P] = ExprCtx.Expr 27 | 28 | class WsCtx(semiInference: Boolean, arrowTypeAscriptions: Boolean){ 29 | 30 | def OneSemiMax[$: P] = if (semiInference) OneNLMax else Pass 31 | def NoSemis[$: P] = if (semiInference) NotNewline else Pass 32 | 33 | 34 | def Enumerators[$: P] = { 35 | def Generator = P( `<-` ~/ Expr ~ Guard.? ) 36 | def Assign = P( `=` ~/ Expr ) 37 | // CuttingSemis is a bit weird, and unlike other places within this parser 38 | // which just use `Semis`. This is because unlike other semicolons, semis 39 | // within a generator cannot be trailing: any set of semis *must* be followed 40 | // by another generator, assignment or guard! Thus we need to make sure we 41 | def CuttingSemis = P( WL ~~ ";"./.? ~~ WL ) 42 | def GenAssign = P( `val`.? ~ TypeOrBindPattern ~/ (Generator | Assign) ) 43 | def Enumerator = P( CuttingSemis ~~ (GenAssign | Guard) | Guard ) 44 | P( TypeOrBindPattern ~ Generator ~~ Enumerator.repX ) 45 | } 46 | 47 | def Expr[$: P]: P[Unit] = { 48 | def If = { 49 | def Else = P( Semis.? ~ `else` ~/ Expr ) 50 | P( `if` ~/ "(" ~ ExprCtx.Expr ~ ")" ~ Expr ~ Else.? ) 51 | } 52 | def While = P( `while` ~/ "(" ~ ExprCtx.Expr ~ ")" ~ Expr ) 53 | def Try = { 54 | def Catch = P( `catch` ~/ Expr ) 55 | def Finally = P( `finally` ~/ Expr ) 56 | P( `try` ~/ Expr ~ Catch.? ~ Finally.? ) 57 | } 58 | def DoWhile = P( `do` ~/ Expr ~ Semis.? ~ `while` ~ "(" ~ ExprCtx.Expr ~ ")" ) 59 | 60 | def For = { 61 | def Body= P( "(" ~/ ExprCtx.Enumerators ~ ")" | "{" ~/ StatCtx.Enumerators ~ "}" ) 62 | P( `for` ~/ Body ~ `yield`.? ~ Expr ) 63 | } 64 | def Throw = P( `throw` ~/ Expr ) 65 | def Return = P( `return` ~/ Expr.? ) 66 | def LambdaRhs = if (semiInference) P( BlockChunk ) else P( Expr ) 67 | 68 | 69 | def ImplicitLambda = P( `implicit` ~ (Id | Underscore) ~ (`:` ~ InfixType).? ~ `=>` ~ LambdaRhs.? ) 70 | def ParenedLambda = P( Parened ~~ (WL ~ `=>` ~ LambdaRhs.? | ExprSuffix ~~ PostfixSuffix ~ SuperPostfixSuffix) ) 71 | def PostfixLambda = P( PostfixExpr ~ (`=>` ~ LambdaRhs.? | SuperPostfixSuffix).? ) 72 | def SmallerExprOrLambda = P( ParenedLambda | PostfixLambda ) 73 | P( 74 | If | While | Try | DoWhile | For | Throw | Return | 75 | ImplicitLambda | SmallerExprOrLambda 76 | ) 77 | } 78 | 79 | def SuperPostfixSuffix[$: P] = P( (`=` ~/ Expr).? ~ MatchAscriptionSuffix.? ) 80 | def AscriptionType[$: P] = if (arrowTypeAscriptions) P( Type ) else P( InfixType ) 81 | def Ascription[$: P] = P( `:` ~/ (`Underscore*` | AscriptionType | Annot.rep(1)) ) 82 | def MatchAscriptionSuffix[$: P] = P(`match` ~/ "{" ~ CaseClauses | Ascription) 83 | def ExprPrefix[$: P] = P( WL ~ CharIn("\\-+!~") ~~ !syntax.Basic.OpChar ~ WS) 84 | def ExprSuffix[$: P] = P( (WL ~ "." ~/ Id | WL ~ TypeArgs | NoSemis ~ ArgList).repX ~~ (NoSemis ~ Underscore).? ) 85 | def PrefixExpr[$: P] = P( ExprPrefix.? ~ SimpleExpr ) 86 | 87 | // Intermediate `WL` needs to always be non-cutting, because you need to 88 | // backtrack out of `InfixSuffix` into `PostFixSuffix` if it doesn't work out 89 | def InfixSuffix[$: P] = P( NoSemis ~~ WL ~~ Id ~ TypeArgs.? ~~ OneSemiMax ~ PrefixExpr ~~ ExprSuffix) 90 | def PostFix[$: P] = P( NoSemis ~~ WL ~~ Id ~ Newline.? ) 91 | 92 | def PostfixSuffix[$: P] = P( InfixSuffix.repX ~~ PostFix.?) 93 | 94 | def PostfixExpr[$: P]: P[Unit] = P( PrefixExpr ~~ ExprSuffix ~~ PostfixSuffix ) 95 | 96 | def Parened[$: P] = P ( "(" ~/ TypeExpr.repTC() ~ ")" ) 97 | def SimpleExpr[$: P]: P[Unit] = { 98 | def New = P( `new` ~/ AnonTmpl ) 99 | 100 | P( XmlExpr | New | BlockExpr | ExprLiteral | StableId | Underscore | Parened ) 101 | } 102 | def Guard[$: P] : P[Unit] = P( `if` ~/ PostfixExpr ) 103 | } 104 | 105 | def SimplePattern[$: P]: P[Unit] = { 106 | def TupleEx = P( "(" ~/ Pattern.repTC() ~ ")" ) 107 | def Extractor = P( StableId ~ TypeArgs.? ~ TupleEx.? ) 108 | def Thingy = P( Underscore ~ (`:` ~/ TypePat).? ~ !("*" ~~ !syntax.Basic.OpChar) ) 109 | P( XmlPattern | Thingy | PatLiteral | TupleEx | Extractor | VarId ) 110 | } 111 | 112 | def BlockExpr[$: P]: P[Unit] = P( "{" ~/ (CaseClauses | Block ~ "}") ) 113 | 114 | def BlockLambdaHead[$: P]: P[Unit] = P( "(" ~ BlockLambdaHead ~ ")" | `this` | Id | Underscore ) 115 | 116 | def BlockLambda[$: P] = P( BlockLambdaHead ~ (`=>` | `:` ~ InfixType ~ `=>`.?) ) 117 | 118 | def BlockChunk[$: P] = { 119 | def Prelude = P( Annot.rep ~ LocalMod.rep ) 120 | def BlockStat = P( Import | Prelude ~ BlockDef | StatCtx.Expr ) 121 | P( BlockLambda.rep ~ BlockStat.rep(sep = Semis) ) 122 | } 123 | 124 | def BaseBlock[$: P](end: => P[Unit])(implicit name: sourcecode.Name): P[Unit] = { 125 | def BlockEnd = P( Semis.? ~ &(end) ) 126 | def Body = P( BlockChunk.repX(sep = Semis) ) 127 | P( Semis.? ~ BlockLambda.? ~ Body ~/ BlockEnd ) 128 | } 129 | 130 | def Block[$: P] = BaseBlock("}") 131 | def CaseBlock[$: P] = BaseBlock("}" | `case`) 132 | 133 | def Patterns[$: P]: P[Unit] = P( Pattern.rep(1, sep = ","./) ) 134 | def Pattern[$: P]: P[Unit] = P( (WL ~ TypeOrBindPattern).rep(1, sep = "|"./) ) 135 | def TypePattern[$: P] = P( (Underscore | BacktickId | VarId) ~ `:` ~ TypePat ) 136 | def TypeOrBindPattern[$: P]: P[Unit] = P( TypePattern | BindPattern ) 137 | def BindPattern[$: P]: P[Unit] = { 138 | def InfixPattern = P( SimplePattern ~ (Id ~/ SimplePattern).rep | `Underscore*` ) 139 | def Binding = P( (Id | Underscore) ~ `@` ) 140 | P( Binding ~ InfixPattern | InfixPattern | VarId ) 141 | } 142 | 143 | def TypePat[$: P] = P( CompoundType ) 144 | def ParenArgList[$: P] = P( "(" ~/ (Exprs ~ (`:` ~/ `Underscore*`).?).? ~ TrailingComma ~ ")" ) 145 | def ArgList[$: P]: P[Unit] = P( ParenArgList | OneNLMax ~ BlockExpr ) 146 | 147 | def CaseClauses[$: P]: P[Unit] = { 148 | // Need to lookahead for `class` and `object` because 149 | // the block { case object X } is not a case clause! 150 | def CaseClause: P[Unit] = P( `case` ~ !(`class` | `object`) ~/ Pattern ~ ExprCtx.Guard.? ~ `=>` ~ CaseBlock ) 151 | P( CaseClause.rep(1) ~ "}" ) 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /scalaparse/src/scalaparse/Scala.scala: -------------------------------------------------------------------------------- 1 | package scalaparse 2 | 3 | import language.implicitConversions 4 | import syntax._ 5 | import fastparse._ 6 | import fastparse._, ScalaWhitespace._ 7 | /** 8 | * Parser for Scala syntax. 9 | */ 10 | object Scala extends Core with Types with Exprs with Xml{ 11 | 12 | def TmplBody[$: P]: P[Unit] = { 13 | def Prelude = P( (Annot ~ OneNLMax).rep ~ Mod./.rep ) 14 | def TmplStat = P( Import | Prelude ~ BlockDef | StatCtx.Expr ) 15 | 16 | P( "{" ~/ BlockLambda.? ~ Semis.? ~ TmplStat.repX(sep = NoCut(Semis)) ~ Semis.? ~ `}` ) 17 | } 18 | 19 | def ValVarDef[$: P] = P( BindPattern.rep(1, ","./) ~ (`:` ~/ Type).? ~ (`=` ~/ FreeCtx.Expr).? ) 20 | 21 | def FunDef[$: P] = { 22 | def Body = P( WL ~ `=` ~/ `macro`.? ~ StatCtx.Expr | OneNLMax ~ "{" ~ Block ~ "}" ) 23 | P( FunSig ~ (`:` ~/ Type).? ~~ Body.? ) 24 | } 25 | 26 | def BlockDef[$: P]: P[Unit] = P( Dcl | TraitDef | ClsDef | ObjDef ) 27 | 28 | def ClsDef[$: P] = { 29 | def ClsAnnot = P( `@` ~ SimpleType ~ ArgList.? ) 30 | def Prelude = P( NotNewline ~ ( ClsAnnot.rep(1) ~ AccessMod.? | AccessMod) ) 31 | def ClsArgMod = P( Mod.rep ~ (`val` | `var`) ) 32 | def ClsArg = P( Annot.rep ~ ClsArgMod.? ~ Id ~ `:` ~ Type ~ (`=` ~ ExprCtx.Expr).? ) 33 | 34 | def ClsArgs = P( OneNLMax ~ "(" ~/ `implicit`.? ~ ClsArg.repTC() ~ ")" ) 35 | P( `case`.? ~ `class` ~/ Id ~ TypeArgList.? ~~ Prelude.? ~~ ClsArgs.repX ~ DefTmpl.? ) 36 | } 37 | 38 | def Constrs[$: P] = P( (WL ~ Constr).rep(1, `with`./) ) 39 | def EarlyDefTmpl[$: P] = P( TmplBody ~ (`with` ~/ Constr).rep ~ TmplBody.? ) 40 | def NamedTmpl[$: P] = P( Constrs ~ TmplBody.? ) 41 | 42 | def DefTmpl[$: P] = P( (`extends` | `<:`) ~ AnonTmpl | TmplBody ) 43 | def AnonTmpl[$: P] = P( EarlyDefTmpl | NamedTmpl | TmplBody ) 44 | 45 | def TraitDef[$: P] = P( `trait` ~/ Id ~ TypeArgList.? ~ DefTmpl.? ) 46 | 47 | def ObjDef[$: P]: P[Unit] = P( `case`.? ~ `object` ~/ Id ~ DefTmpl.? ) 48 | 49 | def Constr[$: P] = P( AnnotType ~~ (NotNewline ~ ParenArgList ).repX ) 50 | 51 | def PkgObj[$: P] = P( ObjDef ) 52 | def PkgBlock[$: P] = P( QualId ~/ `{` ~ TopStatSeq.? ~ `}` ) 53 | def Pkg[$: P] = P( `package` ~/ (PkgBlock | PkgObj) ) 54 | def TopStatSeq[$: P]: P[Unit] = { 55 | def Tmpl = P( (Annot ~~ OneNLMax).rep ~ Mod.rep ~ (TraitDef | ClsDef | ObjDef) ) 56 | def TopStat = P( Pkg | Import | Tmpl ) 57 | P( TopStat.repX(1, Semis) ) 58 | } 59 | def TopPkgSeq[$: P] = P( ((`package` ~ QualId) ~~ !(WS ~ "{")).repX(1, Semis) ) 60 | def CompilationUnit[$: P]: P[Unit] = { 61 | def Body = P( TopPkgSeq ~~ (Semis ~ TopStatSeq).? | TopStatSeq ) 62 | P( Semis.? ~ Body.? ~~ Semis.? ~ WL0 ~ End ) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /scalaparse/src/scalaparse/Types.scala: -------------------------------------------------------------------------------- 1 | package scalaparse 2 | 3 | import fastparse._ 4 | import fastparse._, ScalaWhitespace._ 5 | trait Types extends Core{ 6 | def TypeExpr[$: P]: P[Unit] 7 | def ValVarDef[$: P]: P[Unit] 8 | def FunDef[$: P]: P[Unit] 9 | 10 | def LocalMod[$: P]: P[Unit] = P( `abstract` | `final` | `sealed` | `implicit` | `lazy` ) 11 | def AccessMod[$: P]: P[Unit] = { 12 | def AccessQualifier = P( "[" ~/ (`this` | Id) ~ "]" ) 13 | P( (`private` | `protected`) ~ AccessQualifier.? ) 14 | } 15 | def Dcl[$: P]: P[Unit] = { 16 | P( (`val` | `var`) ~/ ValVarDef | `def` ~/ FunDef | `type` ~/ TypeDef ) 17 | } 18 | 19 | def Mod[$: P]: P[Unit] = P( LocalMod | AccessMod | `override` ) 20 | 21 | def ExistentialClause[$: P] = P( `forSome` ~/ `{` ~ Dcl.repX(1, Semis) ~ `}` ) 22 | def PostfixType[$: P] = P( InfixType ~ (`=>` ~/ Type | ExistentialClause).? ) 23 | def Type[$: P]: P[Unit] = P( `=>`.? ~~ PostfixType ~ TypeBounds ~ `*`.? ) 24 | 25 | 26 | // Can't cut after `*` because we may need to backtrack and settle for 27 | // the `*`-postfix rather than an infix type 28 | def InfixType[$: P] = P( CompoundType ~~ (NotNewline ~ (`*` | Id./) ~~ OneNLMax ~ CompoundType).repX ) 29 | 30 | def CompoundType[$: P] = { 31 | def Refinement = P( OneNLMax ~ `{` ~/ Dcl.repX(sep=Semis) ~ `}` ) 32 | def NamedType = P( (Pass ~ AnnotType).rep(1, `with`./) ) 33 | P( NamedType ~~ Refinement.? | Refinement ) 34 | } 35 | def NLAnnot[$: P] = P( NotNewline ~ Annot ) 36 | def AnnotType[$: P] = P(SimpleType ~~ NLAnnot.repX ) 37 | 38 | def TypeId[$: P] = P( StableId ) 39 | def SimpleType[$: P]: P[Unit] = { 40 | // Can't `cut` after the opening paren, because we might be trying to parse `()` 41 | // or `() => T`! only cut after parsing one type 42 | def TupleType = P( "(" ~/ Type.repTC() ~ ")" ) 43 | def BasicType = P( TupleType | Literals.NoInterp.Literal | TypeId ~ ("." ~ `type`).? | Underscore ) 44 | P( BasicType ~ (TypeArgs | `#` ~/ Id).rep ) 45 | } 46 | 47 | def TypeArgs[$: P] = P( "[" ~/ Type.repTC() ~ "]" ) 48 | 49 | 50 | def FunSig[$: P]: P[Unit] = { 51 | def FunArg = P( Annot.rep ~ Id ~ (`:` ~/ Type).? ~ (`=` ~/ TypeExpr).? ) 52 | def Args = P( FunArg.repTC(1) ) 53 | def FunArgs = P( OneNLMax ~ "(" ~/ `implicit`.? ~ Args.? ~ ")" ) 54 | def FunTypeArgs = P( "[" ~/ (Annot.rep ~ TypeArg).repTC(1) ~ "]" ) 55 | P( (Id | `this`) ~ FunTypeArgs.? ~~ FunArgs.rep ) 56 | } 57 | 58 | def TypeBounds[$: P]: P[Unit] = P( (`>:` ~/ Type).? ~ (`<:` ~/ Type).? ) 59 | def TypeArg[$: P]: P[Unit] = { 60 | def CtxBounds = P((`<%` ~/ Type).rep ~ (`:` ~/ Type).rep) 61 | P((Id | Underscore) ~ TypeArgList.? ~ TypeBounds ~ CtxBounds) 62 | } 63 | 64 | def Annot[$: P]: P[Unit] = P( `@` ~/ SimpleType ~ ("(" ~/ (Exprs ~ (`:` ~/ `Underscore*`).?).? ~ TrailingComma ~ ")").rep ) 65 | 66 | def TypeArgList[$: P]: P[Unit] = { 67 | def Variant: P[Unit] = P( Annot.rep ~ CharIn("+\\-").? ~ TypeArg ) 68 | P( "[" ~/ Variant.repTC(1) ~ "]" ) 69 | } 70 | def Exprs[$: P]: P[Unit] = P( TypeExpr.rep(1, ",") ) 71 | def TypeDef[$: P]: P[Unit] = P( Id ~ TypeArgList.? ~ (`=` ~/ Type | TypeBounds) ) 72 | } 73 | -------------------------------------------------------------------------------- /scalaparse/src/scalaparse/Xml.scala: -------------------------------------------------------------------------------- 1 | package scalaparse 2 | 3 | import syntax.Basic 4 | import fastparse._ 5 | import fastparse._ 6 | 7 | import scala.language.implicitConversions 8 | import NoWhitespace._ 9 | 10 | trait Xml { 11 | def WL[$: P]: P0 12 | def WS[$: P]: P0 13 | def Block[$: P]: P0 14 | def Patterns[$: P]: P[Unit] 15 | def XmlExpr[$: P] = P( WL ~ Xml.XmlContent.rep(min = 1, sep = WL.?) ) 16 | def XmlPattern[$: P] = P( WL ~ Xml.ElemPattern ) 17 | 18 | private[this] object Xml { 19 | def Element[$: P] = P( TagHeader ~/ ("/>" | ">" ~/ Content ~/ ETag ) ) 20 | def TagHeader[$: P] = P( "<" ~ Name ~/ (WL ~ Attribute).rep ~ WL.? ) 21 | def ETag[$: P] = P( "" ) 22 | 23 | def Attribute[$: P] = P( Name ~ Eq ~/ AttValue ) 24 | def Eq[$: P] = P( WL.? ~ "=" ~ WL.? ) 25 | def AttValue[$: P] = P( 26 | "\"" ~/ (CharQ | Reference).rep ~ "\"" | 27 | "'" ~/ (CharA | Reference).rep ~ "'" | 28 | ScalaExpr 29 | ) 30 | 31 | def Content[$: P] = P( (CharData | Reference | ScalaExpr | XmlContent).rep ) 32 | def XmlContent[$: P]: P[Unit] = P( Unparsed | CDSect | PI | Comment | Element ) 33 | 34 | def ScalaExpr[$: P] = P( "{" ~ WS ~ Block ~ WL ~ "}" ) 35 | 36 | def Unparsed[$: P] = P( UnpStart ~/ UnpData ~ UnpEnd ) 37 | def UnpStart[$: P] = P( "" ) 38 | def UnpEnd[$: P] = P( "" ) 39 | def UnpData[$: P] = P( (!UnpEnd ~ AnyChar).rep ) 40 | 41 | def CDSect[$: P] = P( CDStart ~/ CData ~ CDEnd ) 42 | def CDStart[$: P] = P( "" ~ Char).rep ) 44 | def CDEnd[$: P] = P( "]]>" ) 45 | 46 | def Comment[$: P] = P( "" ) 47 | def ComText[$: P] = P( (!"--" ~ Char).rep ~ ("-" ~ &("--")).? ) 48 | 49 | def PI[$: P] = P( "" ) 50 | def PITarget[$: P] = P( !(("X" | "x") ~ ("M" | "m") ~ ("L" | "l")) ~ Name ) 51 | def PIProcText[$: P] = P( WL ~ (!"?>" ~ Char).rep ) 52 | 53 | def Reference[$: P] = P( EntityRef | CharRef ) 54 | def EntityRef[$: P] = P( "&" ~ Name ~/ ";" ) 55 | def CharRef[$: P] = P( "&#" ~ Num ~/ ";" | "&#x" ~ HexNum ~/ ";" ) 56 | def Num[$: P] = P( CharIn("0-9").rep ) 57 | def HexNum[$: P] = P( CharIn("0-9a-fA-F").rep ) 58 | 59 | def CharData[$: P] = P( (!"{" ~ Char1 | "{{").rep(1) ) 60 | 61 | def Char[$: P] = P( AnyChar ) 62 | def Char1[$: P] = P( !("<" | "&") ~ Char ) 63 | def CharQ[$: P] = P( !"\"" ~ Char1 ) 64 | def CharA[$: P] = P( !"'" ~ Char1 ) 65 | 66 | def Name[$: P] = P( NameStart ~ NameChar.rep ) 67 | def NameStart[$: P] = P( CharPred(isNameStart) ).opaque("NameStart") 68 | def NameChar[$: P] = P( CharPred(isNameChar) ).opaque("NameChar") 69 | 70 | def ElemPattern[$: P]: P[Unit] = P( TagPHeader ~/ ("/>" | ">" ~/ ContentP ~/ ETag ) ) 71 | def TagPHeader[$: P] = P( "<" ~ Name ~ WL.? ) 72 | 73 | def ContentP[$: P]: P[Unit] = P( ( CharDataP | ScalaPatterns | ElemPattern ).rep ) 74 | def ScalaPatterns[$: P] = P( "{" ~ Patterns ~ WL ~ "}" ) 75 | def CharDataP[$: P] = P( "&" ~ CharData.? | CharData ) // matches weirdness of scalac parser on xml reference. 76 | 77 | //================================================================================ 78 | // From `scala.xml.parsing.TokenTests` 79 | //================================================================================ 80 | 81 | /** 82 | * {{{ 83 | * NameChar ::= Letter | Digit | '.' | '-' | '_' | ':' 84 | * | CombiningChar | Extender 85 | * }}} 86 | * See [4] and Appendix B of XML 1.0 specification. 87 | */ 88 | def isNameChar(ch: Char) = { 89 | import java.lang.Character._ 90 | // The constants represent groups Mc, Me, Mn, Lm, and Nd. 91 | 92 | isNameStart(ch) || (getType(ch).toByte match { 93 | case COMBINING_SPACING_MARK | 94 | ENCLOSING_MARK | NON_SPACING_MARK | 95 | MODIFIER_LETTER | DECIMAL_DIGIT_NUMBER => true 96 | case _ => ".-:" contains ch 97 | }) 98 | } 99 | 100 | /** 101 | * {{{ 102 | * NameStart ::= ( Letter | '_' ) 103 | * }}} 104 | * where Letter means in one of the Unicode general 105 | * categories `{ Ll, Lu, Lo, Lt, Nl }`. 106 | * 107 | * We do not allow a name to start with `:`. 108 | * See [3] and Appendix B of XML 1.0 specification 109 | */ 110 | def isNameStart(ch: Char) = { 111 | import java.lang.Character._ 112 | 113 | getType(ch).toByte match { 114 | case LOWERCASE_LETTER | 115 | UPPERCASE_LETTER | OTHER_LETTER | 116 | TITLECASE_LETTER | LETTER_NUMBER => true 117 | case _ => ch == '_' 118 | } 119 | } 120 | } 121 | } -------------------------------------------------------------------------------- /scalaparse/src/scalaparse/syntax/Basic.scala: -------------------------------------------------------------------------------- 1 | package scalaparse.syntax 2 | 3 | 4 | import fastparse._ 5 | import fastparse._ 6 | import fastparse.NoWhitespace._ 7 | import CharPredicates._ 8 | import scalaparse.syntax.Identifiers.NamedFunction 9 | object Basic { 10 | 11 | def UnicodeEscape[$: P] = P( "u" ~ HexDigit ~ HexDigit ~ HexDigit ~ HexDigit ) 12 | 13 | //Numbers and digits 14 | def Digit[$: P] = P( CharIn("0-9") ) 15 | 16 | def HexDigit[$: P] = P( CharIn("0-9a-fA-F") ) 17 | def HexNum[$: P] = P( "0x" ~ CharsWhileIn("0-9a-fA-F") ) 18 | def DecNum[$: P] = P( CharsWhileIn("0-9") ) 19 | def Exp[$: P] = P( CharIn("Ee") ~ CharIn("+\\-").? ~ DecNum ) 20 | def FloatType[$: P] = P( CharIn("fFdD") ) 21 | 22 | def WSChars[$: P] = P( NoTrace(CharsWhileIn("\u0020\u0009")) ) 23 | def Newline[$: P] = P( NoTrace(StringIn("\r\n", "\n")) ) 24 | def Semi[$: P] = P( ";" | Newline.rep(1) ) 25 | def OpChar[$: P] = P ( CharPred(isOpChar) ) 26 | 27 | val isOpChar = NamedFunction{ 28 | case '!' | '#' | '%' | '&' | '*' | '+' | '-' | '/' | 29 | ':' | '<' | '=' | '>' | '?' | '@' | '\\' | '^' | '|' | '~' => true 30 | case c => isOtherSymbol(c) || isMathSymbol(c) 31 | } 32 | 33 | val LetterDigitDollarUnderscore = NamedFunction( 34 | c => isLetter(c) | isDigit(c) | c == '$' | c == '_' 35 | ) 36 | val LowerChar = NamedFunction( 37 | c => isLower(c) || c == '$' | c == '_' 38 | ) 39 | val UpperChar = NamedFunction(isUpper) 40 | 41 | def Lower[$: P] = P( CharPred(LowerChar) ) 42 | def Upper[$: P] = P( CharPred(UpperChar) ) 43 | } 44 | /** 45 | * Most keywords don't just require the correct characters to match, 46 | * they have to ensure that subsequent characters *don't* match in 47 | * order for it to be a keyword. This enforces that rule for key-words 48 | * (W) and key-operators (O) which have different non-match criteria. 49 | */ 50 | object Key { 51 | def W[$: P](s: String) = P( s ~ !CharPred(Basic.LetterDigitDollarUnderscore) )(s"`$s`", implicitly) 52 | // If the operator is followed by a comment, stop early so we can parse the comment 53 | def O[$: P](s: String) = P( s ~ (!Basic.OpChar | &(NoTrace(StringIn("/*", "//")))) )(s"`$s`", implicitly) 54 | } 55 | -------------------------------------------------------------------------------- /scalaparse/src/scalaparse/syntax/Identifiers.scala: -------------------------------------------------------------------------------- 1 | package scalaparse.syntax 2 | import fastparse._ 3 | import fastparse._ 4 | import NoWhitespace._ 5 | import Basic._ 6 | import CharPredicates._ 7 | object Identifiers{ 8 | 9 | case class NamedFunction(f: Char => Boolean) 10 | (implicit name: sourcecode.Name) extends (Char => Boolean){ 11 | def apply(t: Char) = f(t) 12 | override def toString() = name.value 13 | 14 | } 15 | val OpCharNotSlash = NamedFunction(x => isOpChar(x) && x != '/') 16 | val NotBackTick = NamedFunction(_ != '`') 17 | 18 | def Operator[$: P] = P( 19 | !SymbolicKeywords ~ (!StringIn("/*", "//") ~ (CharsWhile(OpCharNotSlash) | "/")).rep(1) 20 | ).opaque("operator") 21 | 22 | def VarId[$: P] = P( VarId0(true) ).opaque("var-id") 23 | 24 | def VarId0[$: P](dollar: Boolean) = P( !Keywords ~ Lower ~ IdRest(dollar) ) 25 | 26 | def UppercaseId[$: P](dollar: Boolean) = P( !Keywords ~ Upper ~ IdRest(dollar) ) 27 | def PlainId[$: P] = P( UppercaseId(true) | VarId | Operator ~ (!OpChar | &(StringIn("/*", "//"))) ) 28 | .opaque("plain-id") 29 | 30 | def PlainIdNoDollar[$: P] = P( UppercaseId(false) | VarId0(false) | Operator ).opaque("plain-id") 31 | 32 | def BacktickId[$: P] = P( "`" ~ CharsWhile(NotBackTick) ~ "`" ) 33 | def Id[$: P]: P[Unit] = P( BacktickId | PlainId ).opaque("id") 34 | 35 | def IdRest[$: P](allowDollar: Boolean) = { 36 | 37 | val IdCharacter = 38 | if(allowDollar) NamedFunction(c => c == '$' || isLetter(c) || isDigit(c)) 39 | else NamedFunction(c => isLetter(c) || isDigit(c)) 40 | 41 | def IdUnderscoreChunk = P( CharsWhileIn("_", 0) ~ CharsWhile(IdCharacter) ) 42 | P( IdUnderscoreChunk.rep ~ (CharsWhileIn("_") ~ CharsWhile(isOpChar, 0)).? ) 43 | } 44 | 45 | def AlphabetKeywords[$: P] = P { 46 | StringIn("abstract", "case", "catch", "class", "def", "do", "else", 47 | "extends", "false", "finally", "final", "finally", "forSome", 48 | "for", "if", "implicit", "import", "lazy", "match", "new", 49 | "null", "object", "override", "package", "private", "protected", 50 | "return", "sealed", "super", "this", "throw", "trait", "try", 51 | "true", "type", "val", "var", "while", "with", "yield", "_", "macro") ~ 52 | !CharPred(Basic.LetterDigitDollarUnderscore) 53 | }.opaque("AlphabetKeywords") 54 | 55 | def SymbolicKeywords[$: P] = P{ 56 | StringIn(":", ";", "=>", "=", "<-", "<:", "<%", ">:", "#", "@", "\u21d2", "\u2190") ~ !OpChar 57 | }.opaque("SymbolicKeywords") 58 | 59 | def Keywords[$: P] = P( AlphabetKeywords | SymbolicKeywords ) 60 | } 61 | -------------------------------------------------------------------------------- /scalaparse/src/scalaparse/syntax/Literals.scala: -------------------------------------------------------------------------------- 1 | package scalaparse.syntax 2 | 3 | import fastparse._ 4 | 5 | import NoWhitespace._ 6 | import Identifiers._ 7 | 8 | trait Literals { l => 9 | def Block[$: P]: P[Unit] 10 | 11 | /** 12 | * Parses all whitespace, excluding newlines. This is only 13 | * really useful in e.g. {} blocks, where we want to avoid 14 | * capturing newlines so semicolon-inference would work 15 | */ 16 | def WS[$: P]: P[Unit] = P( NoTrace((Basic.WSChars | Literals.Comment).rep) ) 17 | 18 | /** 19 | * Parses whitespace, including newlines. 20 | * This is the default for most things 21 | */ 22 | def WL0[$: P]: P[Unit] = P( ScalaWhitespace.whitespace(P.current) ) 23 | def WL[$: P]: P[Unit] = P( NoCut(WL0) ) 24 | 25 | def Semis[$: P]: P[Unit] = P( NoTrace(NoCut(WS) ~ Basic.Semi.rep(1, NoCut(WS)) ~ NoCut(WS)) ) 26 | def Newline[$: P]: P[Unit] = P( WL ~ Basic.Newline ) 27 | 28 | def NotNewline[$: P]: P[Unit] = P( &( WS ~ !Basic.Newline ) ) 29 | def OneNLMax[$: P]: P[Unit] = { 30 | def ConsumeComments = P( (Basic.WSChars.? ~ NoTrace(Literals.Comment) ~ Basic.WSChars.? ~ Basic.Newline).rep ) 31 | P( NoCut(NoTrace(WS ~ Basic.Newline.? ~ ConsumeComments ~ NotNewline) )) 32 | } 33 | 34 | def TrailingComma[$: P]: P[Unit] = P( ("," ~ WS ~ Basic.Newline).? ) 35 | def Pattern[$: P]: P[Unit] 36 | 37 | object Literals{ 38 | import Basic._ 39 | def Float[$: P] = { 40 | def LeadingDotFloat = P( "." ~ DecNum ~ Exp.? ~ FloatType.? ) 41 | def FloatSuffix = P( LeadingDotFloat | Exp ~ FloatType.? | Exp.? ~ FloatType ) 42 | P( LeadingDotFloat | DecNum ~ FloatSuffix ) 43 | } 44 | 45 | def Int[$: P] = P( (HexNum | DecNum) ~ ("L" | "l").? ) 46 | 47 | def Bool[$: P] = P( Key.W("true") | Key.W("false") ) 48 | 49 | // Comments cannot have cuts in them, because they appear before every 50 | // terminal node. That means that a comment before any terminal will 51 | // prevent any backtracking from working, which is not what we want! 52 | def CommentChunk[$: P] = P( CharsWhile(c => c != '/' && c != '*') | MultilineComment | !"*/" ~ AnyChar ) 53 | def MultilineComment[$: P]: P[Unit] = P( "/*" ~/ CommentChunk.rep ~ "*/" ) 54 | def SameLineCharChunks[$: P] = P( CharsWhile(c => c != '\n' && c != '\r') | !Basic.Newline ~ AnyChar ) 55 | def LineComment[$: P] = P( "//" ~ SameLineCharChunks.rep ~ &(Basic.Newline | End) ) 56 | def Comment[$: P]: P[Unit] = P( MultilineComment | LineComment ) 57 | 58 | def Null[$: P] = Key.W("null") 59 | 60 | def OctalEscape[$: P] = P( Digit ~ Digit.? ~ Digit.? ) 61 | def Escape[$: P] = P( "\\" ~/ (CharIn("""btnfr'\\"]""") | OctalEscape | UnicodeEscape ) ) 62 | 63 | // Note that symbols can take on the same values as keywords! 64 | def Symbol[$: P] = P( Identifiers.PlainId | Identifiers.Keywords ) 65 | 66 | def Char[$: P] = { 67 | // scalac 2.10 crashes if PrintableChar below is substituted by its body 68 | def PrintableChar = CharPred(CharPredicates.isPrintableChar) 69 | 70 | P( (Escape | PrintableChar) ~ "'" ) 71 | } 72 | 73 | class InterpCtx(interp: Option[() => P[Unit]]) { 74 | def Literal[$: P] = P( ("-".? ~ (Float | Int)) | Bool | String | "'" ~/ (Char | Symbol) | Null ) 75 | def Interp[$: P] = interp match{ 76 | case None => P ( Fail ) 77 | case Some(p) => P( "$" ~ Identifiers.PlainIdNoDollar | ("${" ~ p() ~ WL ~ "}") | "$$" ) 78 | } 79 | 80 | def TQ[$: P] = P( "\"\"\"" ) 81 | /** 82 | * Helper to quickly gobble up large chunks of un-interesting 83 | * characters. We break out conservatively, even if we don't know 84 | * it's a "real" escape sequence: worst come to worst it turns out 85 | * to be a dud and we go back into a CharsChunk next rep 86 | */ 87 | def StringChars[$: P] = P( CharsWhile(c => c != '\n' && c != '"' && c != '\\' && c != '$') ) 88 | def NonTripleQuoteChar[$: P] = P( "\"" ~ "\"".? ~ !"\"" | CharIn("\\\\$\n") ) 89 | def TripleChars[$: P] = P( (StringChars | Interp | NonTripleQuoteChar).rep ) 90 | def TripleTail[$: P] = P( TQ ~ "\"".rep ) 91 | def SingleChars[$: P](allowSlash: Boolean) = { 92 | def LiteralSlash = P( if(allowSlash) "\\" else Fail ) 93 | def NonStringEnd = P( !CharIn("\n\"") ~ AnyChar ) 94 | P( (StringChars | Interp | LiteralSlash | Escape | NonStringEnd ).rep ) 95 | } 96 | def String[$: P] = { 97 | P { 98 | Id.filter(_ => interp.isDefined) ~ ( 99 | TQ ~/ TripleChars ~ TripleTail | 100 | "\"" ~/ SingleChars(true) ~ "\"" 101 | ) | 102 | TQ ~/ NoInterp.TripleChars ~ TripleTail | 103 | "\"" ~/ NoInterp.SingleChars(false) ~ "\"" 104 | } 105 | } 106 | 107 | } 108 | def NoInterp[$: P] = new InterpCtx(None) 109 | def Pat[$: P] = new InterpCtx(Some(() => l.Pattern)) 110 | def Expr[$: P] = new InterpCtx(Some(() => Block)) 111 | 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /scalaparse/test/src-2-jvm/scalaparse/TestMain.scala: -------------------------------------------------------------------------------- 1 | package scalaparse 2 | import fastparse._ 3 | 4 | /** 5 | * Created by lihaoyi on 13/5/17. 6 | */ 7 | object TestMain { 8 | def main(args: Array[String]): Unit = { 9 | val input = """ 10 | | object X{ 11 | | { 12 | | val x = 1 13 | | ; 14 | | """.stripMargin 15 | val res = parse(input, scalaparse.Scala.CompilationUnit(_)) 16 | // val fail = res.asInstanceOf[Result.Failure] 17 | println(res) 18 | // for(frame <- fail.extra.traced.stack){ 19 | // println(Result.Failure.formatParser(frame.parser, fail.extra.input, frame.index)) 20 | // } 21 | // println(fail.msg) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /scalaparse/test/src-2.12-jvm/scalaparse/ScalacParser.scala: -------------------------------------------------------------------------------- 1 | package scalaparse 2 | 3 | import scala.tools.nsc.{Global, Settings} 4 | 5 | object ScalacParser{ 6 | var current = Thread.currentThread().getContextClassLoader 7 | val files = collection.mutable.Buffer.empty[java.io.File] 8 | files.appendAll( 9 | System.getProperty("sun.boot.class.path") 10 | .split(":") 11 | .map(new java.io.File(_)) 12 | ) 13 | while(current != null){ 14 | current match{ 15 | case t: java.net.URLClassLoader => 16 | files.appendAll(t.getURLs.map(u => new java.io.File(u.toURI))) 17 | case _ => 18 | } 19 | current = current.getParent 20 | } 21 | 22 | val settings = new Settings() 23 | settings.usejavacp.value = true 24 | settings.embeddedDefaults[ScalacParser.type] 25 | settings.classpath.append(files.mkString(":")) 26 | 27 | val global = new Global(settings) 28 | 29 | def checkParseFails(input: String) = this.synchronized{ 30 | new global.Run() 31 | var fail = false 32 | import global.syntaxAnalyzer.Offset 33 | val cu = new global.CompilationUnit(global.newSourceFile(input)) 34 | val parser = new global.syntaxAnalyzer.UnitParser(cu, Nil){ 35 | override def newScanner() = new global.syntaxAnalyzer.UnitScanner(cu, Nil){ 36 | override def error(off: Offset, msg: String) = { 37 | fail = true 38 | } 39 | override def syntaxError(off: Offset, msg: String) = { 40 | fail = true 41 | } 42 | override def incompleteInputError(off: Offset, msg: String) = { 43 | fail = true 44 | } 45 | } 46 | override def incompleteInputError(msg: String) = { 47 | fail = true 48 | } 49 | override def syntaxError(offset: Offset, msg: String) = { 50 | fail = true 51 | } 52 | } 53 | parser.parse() 54 | fail 55 | } 56 | } -------------------------------------------------------------------------------- /scalaparse/test/src-2.13-jvm/scalaparse/ScalacParser.scala: -------------------------------------------------------------------------------- 1 | package scalaparse 2 | 3 | import scala.reflect.internal.util.CodeAction 4 | import scala.tools.nsc.{Global, Settings} 5 | 6 | object ScalacParser{ 7 | var current = Thread.currentThread().getContextClassLoader 8 | val files = collection.mutable.Buffer.empty[java.io.File] 9 | files.appendAll( 10 | System.getProperty("sun.boot.class.path") 11 | .split(":") 12 | .map(new java.io.File(_)) 13 | ) 14 | while(current != null){ 15 | current match{ 16 | case t: java.net.URLClassLoader => 17 | files.appendAll(t.getURLs.map(u => new java.io.File(u.toURI))) 18 | case _ => 19 | } 20 | current = current.getParent 21 | } 22 | 23 | val settings = new Settings() 24 | settings.usejavacp.value = true 25 | settings.embeddedDefaults[ScalacParser.type] 26 | settings.classpath.append(files.mkString(":")) 27 | 28 | val global = new Global(settings) 29 | 30 | def checkParseFails(input: String) = this.synchronized{ 31 | new global.Run() 32 | var fail = false 33 | import global.syntaxAnalyzer.Offset 34 | val cu = new global.CompilationUnit(global.newSourceFile(input)) 35 | val parser = new global.syntaxAnalyzer.UnitParser(cu, Nil){ 36 | override def newScanner() = new global.syntaxAnalyzer.UnitScanner(cu, Nil){ 37 | override def error(off: Offset, msg: String) = { 38 | fail = true 39 | } 40 | override def syntaxError(off: Offset, msg: String) = { 41 | fail = true 42 | } 43 | override def incompleteInputError(off: Offset, msg: String) = { 44 | fail = true 45 | } 46 | } 47 | 48 | override def incompleteInputError(msg: String, actions: List[CodeAction]): Unit = { 49 | fail = true 50 | super.incompleteInputError(msg, actions) 51 | } 52 | 53 | override def syntaxError(offset: global.syntaxAnalyzer.Offset, msg: String, actions: List[CodeAction]): Unit = { 54 | fail = true 55 | super.syntaxError(offset, msg, actions) 56 | } 57 | } 58 | parser.parse() 59 | fail 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /scalaparse/test/src/scalaparse/TestUtil.scala: -------------------------------------------------------------------------------- 1 | package scalaparse 2 | 3 | import utest.assert 4 | import fastparse._ 5 | 6 | 7 | 8 | /** 9 | * Created by haoyi on 5/3/15. 10 | */ 11 | object TestUtil { 12 | def checkNeg[T](input: String, 13 | terminals: String, 14 | aggregate: String, 15 | found: String) 16 | (implicit line: sourcecode.Line) = { 17 | // println("Checking Neg...\n" ) 18 | // println(input) 19 | parse(input, Scala.CompilationUnit(_)) match{ 20 | case f: Parsed.Failure => 21 | 22 | println("=" * 100) 23 | val trace = f.extra.trace(true) 24 | val index = f.index 25 | val parsedTerminals = trace.terminalAggregateString 26 | val parsedAggregate = trace.groupAggregateString 27 | val parsedFound = input.slice(f.index, f.index + 10) 28 | val stack = trace.longAggregateMsg 29 | 30 | assert( 31 | { implicitly(input) 32 | implicitly(stack) 33 | implicitly(index) 34 | implicitly(parsedFound) 35 | (aggregate == null || aggregate.trim == parsedAggregate.trim) && 36 | (terminals == null || terminals.trim == parsedTerminals.trim) && 37 | parsedFound.startsWith(found) 38 | } 39 | ) 40 | 41 | line.value 42 | case _: Parsed.Success[_] => 43 | assert({implicitly(input); false}) 44 | line.value 45 | } 46 | // for(chunkSize <- Seq(/*1, 4, 16, 64, 256, 1024*/)){ 47 | // val res = parse(input.grouped(chunkSize), Scala.CompilationUnit(_)) 48 | // res match{ 49 | // case f: Parsed.Failure => 50 | // 51 | //// val parsedExpected = f.lastParser.toString 52 | // val parsedFound = input.slice(f.index, f.index + 10) 53 | // // Note, here we check `expected.contains` rather than `expected ==`! 54 | // // This is because when parsing an `Iterator`, the `.extra.traced` that 55 | // // we normally use to get the stack trace doesn't work, so instead we 56 | // // do an approximate check to make sure the parser is somewhere in the 57 | // // expected output. OTOH, the `parsedFound` check can still be the same 58 | // // since that just depends on the `index` 59 | // assert( 60 | // { implicitly(input) 61 | // /*expected.trim.contains(parsedExpected.trim) && */parsedFound.startsWith(found) 62 | // } 63 | // ) 64 | // case s: Parsed.Success[_] => assert{implicitly(input); false} 65 | // } 66 | // } 67 | } 68 | 69 | def check[T](input: String, tag: String = "", skipIterator: Boolean = false) = { 70 | println("Checking...\n" ) 71 | // println(input) 72 | check0(input, input.length, tag) 73 | if (!skipIterator) { 74 | for(chunkSize <- Seq(1, 5, 18, 67, 260, 1029)) { 75 | println(chunkSize) 76 | check0(input.grouped(chunkSize), input.length, tag) 77 | } 78 | } 79 | 80 | if (!skipIterator) { 81 | for(chunkSize <- Seq(1, 3, 14, 61, 252, 1019)) { 82 | check0(ParserInputSource.FromReadable(input, chunkSize), input.length, tag) 83 | } 84 | } 85 | } 86 | def check0(input: ParserInputSource, inputLength: Int, tag: String) = { 87 | val res = parse(input, Scala.CompilationUnit(_)) 88 | res match{ 89 | case f: Parsed.Failure => 90 | // println(f.formatExpectedAsString) 91 | // println(f.formatTraces) 92 | println("TRACING") 93 | println(f) 94 | throw new Exception(tag + "\n" + input + "\n" + f.trace().msg) 95 | case s: Parsed.Success[_] => 96 | // println(parsed) 97 | assert(s.index == inputLength) 98 | } 99 | } 100 | } 101 | 102 | --------------------------------------------------------------------------------