├── .gitignore ├── project ├── build.properties ├── plugins.sbt ├── Build.scala └── PublishSettings.scala ├── LICENSE.txt ├── .travis.yml ├── Acceleration Parser Combinators with Macros - Uppsala 2014.pdf ├── FastParsers └── src │ ├── main │ └── scala │ │ └── fastparsers │ │ ├── parsers │ │ ├── Parser.scala │ │ ├── FlatMapParsers.scala │ │ ├── TokenParsers.scala │ │ ├── RepParsers.scala │ │ ├── FlatMapImpl.scala │ │ ├── BaseParsers.scala │ │ ├── TokenParsersImpl.scala │ │ └── ParserImplBase.scala │ │ ├── framework │ │ ├── saveAST.scala │ │ ├── parseresult │ │ │ ├── Failure.scala │ │ │ ├── Success.scala │ │ │ └── ParseResult.scala │ │ ├── ruleprocessing │ │ │ ├── ReduceRules.scala │ │ │ ├── MapRules.scala │ │ │ ├── RulesProcessing.scala │ │ │ ├── RuleCombiner.scala │ │ │ ├── RulesInliner.scala │ │ │ └── ParseRules.scala │ │ ├── implementations │ │ │ ├── FinalFastParserImpl.scala │ │ │ ├── FastParsers.scala │ │ │ ├── FastParsersCharArray.scala │ │ │ ├── FastParsersCharArrayNoInline.scala │ │ │ ├── FastParsersCharArrayDefaultErrors.scala │ │ │ ├── FastParsersCharArrayIgnoreResults.scala │ │ │ ├── FastArrayParsers.scala │ │ │ └── BaseImpl.scala │ │ └── getAST.scala │ │ ├── error │ │ ├── IgnoreParseError.scala │ │ ├── DefaultParseError.scala │ │ └── ParseError.scala │ │ ├── input │ │ ├── ArrayInput.scala │ │ ├── StringInput.scala │ │ ├── CharArrayInput.scala │ │ ├── StringLikeInput.scala │ │ ├── ParseInput.scala │ │ ├── ArrayLikeInput.scala │ │ └── InputWindow.scala │ │ └── tools │ │ ├── DefaultValue.scala │ │ ├── ToPosition.scala │ │ └── TreeTools.scala │ └── test │ ├── resources │ ├── tweet10ho │ ├── tweet11ho │ ├── tweet12ho │ ├── tweet13ho │ ├── tweet14ho │ ├── tweet16ho │ ├── tweet17ho │ ├── tweet19ho │ ├── tweet21ho │ ├── tweet28ho │ ├── tweet29ho │ ├── tweet32ho │ ├── tweet33ho │ ├── tweet54ho │ ├── tweet57ho │ ├── tweet5ho │ ├── tweet62ho │ ├── tweet7ho │ ├── tweet81ho │ ├── tweet8ho │ ├── tweet10 │ ├── tweet11 │ ├── tweet12 │ ├── tweet13 │ ├── tweet14 │ ├── tweet16 │ ├── tweet17 │ ├── tweet19 │ ├── tweet21 │ ├── tweet28 │ ├── tweet29 │ ├── tweet32 │ ├── tweet33 │ ├── tweet5 │ ├── tweet54 │ ├── tweet57 │ ├── tweet62 │ ├── tweet7 │ ├── tweet8 │ ├── tweet81 │ ├── tweet9ho │ ├── tweet100ho │ ├── tweet101ho │ ├── tweet15ho │ ├── tweet18ho │ ├── tweet20ho │ ├── tweet22ho │ ├── tweet23ho │ ├── tweet24ho │ ├── tweet25ho │ ├── tweet26ho │ ├── tweet27ho │ ├── tweet30ho │ ├── tweet31ho │ ├── tweet34ho │ ├── tweet35ho │ ├── tweet36ho │ ├── tweet37ho │ ├── tweet38ho │ ├── tweet39ho │ ├── tweet3ho │ ├── tweet40ho │ ├── tweet41ho │ ├── tweet42ho │ ├── tweet43ho │ ├── tweet44ho │ ├── tweet45ho │ ├── tweet46ho │ ├── tweet47ho │ ├── tweet48ho │ ├── tweet49ho │ ├── tweet4ho │ ├── tweet50ho │ ├── tweet51ho │ ├── tweet52ho │ ├── tweet53ho │ ├── tweet55ho │ ├── tweet56ho │ ├── tweet58ho │ ├── tweet59ho │ ├── tweet60ho │ ├── tweet61ho │ ├── tweet63ho │ ├── tweet64ho │ ├── tweet65ho │ ├── tweet66ho │ ├── tweet67ho │ ├── tweet68ho │ ├── tweet69ho │ ├── tweet6ho │ ├── tweet70ho │ ├── tweet71ho │ ├── tweet72ho │ ├── tweet73ho │ ├── tweet74ho │ ├── tweet75ho │ ├── tweet76ho │ ├── tweet77ho │ ├── tweet78ho │ ├── tweet79ho │ ├── tweet80ho │ ├── tweet82ho │ ├── tweet83ho │ ├── tweet84ho │ ├── tweet85ho │ ├── tweet86ho │ ├── tweet87ho │ ├── tweet88ho │ ├── tweet89ho │ ├── tweet90ho │ ├── tweet91ho │ ├── tweet92ho │ ├── tweet93ho │ ├── tweet94ho │ ├── tweet95ho │ ├── tweet96ho │ ├── tweet97ho │ ├── tweet98ho │ ├── tweet99ho │ ├── tweet2ho │ └── tweet9 │ └── scala │ ├── HttpParserSpecs.scala │ ├── JsonParserSpecs.scala │ ├── FastCharSequence.scala │ ├── ParsersMixinBenchmark.scala │ ├── CSVParsers.scala │ ├── HttpParserBenchmark.scala │ ├── CSVBoolHandWritten.scala │ ├── TestsHelper.scala │ ├── lms │ └── DataStructures.scala │ ├── JsonParserBenchmark.scala │ ├── JsonParsers.scala │ └── CSVParserBenchmark.scala ├── Examples └── src │ └── main │ └── scala │ ├── Calculator.scala │ ├── Test.scala │ ├── CSVBoolHandWritten.scala │ ├── DataStructures.scala │ └── Calculator2.scala └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .idea 3 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.5 2 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/begeric/FastParsers/HEAD/LICENSE.txt -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | libraryDependencies += "org.scala-lang" % "scala-compiler" % scalaVersion.value -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | sbt_args: -sbt-version 0.13.5 3 | scala: 4 | - "2.11.2" 5 | jdk: 6 | - oraclejdk8 7 | -------------------------------------------------------------------------------- /Acceleration Parser Combinators with Macros - Uppsala 2014.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/begeric/FastParsers/HEAD/Acceleration Parser Combinators with Macros - Uppsala 2014.pdf -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/parsers/Parser.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.parsers 2 | 3 | /** 4 | * Façade of a Parser. It only appears in the tree given to the macro in order to be expanded 5 | */ 6 | trait Parser[+T] 7 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/saveAST.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework 2 | 3 | import scala.annotation.StaticAnnotation 4 | 5 | /** 6 | * Created by Eric on 22.04.14. 7 | */ 8 | class saveAST(code: Any) extends StaticAnnotation 9 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/error/IgnoreParseError.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.error 2 | 3 | /** 4 | * Created by Eric on 23.04.14. 5 | */ 6 | trait IgnoreParseError extends DefaultParseError{ 7 | import c.universe._ 8 | override def pushError(error: String, inputPos: c.Tree): c.Tree = q"()" 9 | } 10 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet10ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:12:45 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830036538078387; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:12:45 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet11ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:13:15 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830039562280838; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:13:15 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet12ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:13:45 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830042585994215; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:13:45 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet13ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:14:16 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830045690262308; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:14:16 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet14ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:14:47 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830048713771254; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:14:47 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet16ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:15:47 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830054772130157; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:15:47 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet17ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:16:17 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830057797199665; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:16:17 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet19ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:17:18 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830063852908156; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:17:18 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet21ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:18:19 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830069908595001; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:18:19 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet28ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:21:51 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830091152263382; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:21:51 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet29ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:22:21 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830094176787564; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:22:21 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet32ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:23:52 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830103269185469; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:23:52 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet33ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:24:22 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830106295961953; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:24:22 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet54ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:35:00 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830170094703955; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:35:00 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet57ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:36:31 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830179192496004; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:36:31 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet5ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:10:13 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830021393800171; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:10:13 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet62ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:39:03 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830194372704851; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:39:03 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet7ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:11:14 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830027455579259; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:11:14 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet81ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:48:40 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830252056603337; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:48:40 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet8ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:11:44 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830030482071063; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:11:44 UTC 7 | 8 | 9 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/parseresult/Failure.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.parseresult 2 | 3 | /** 4 | * Extractor for ParseResult in case of Failure 5 | */ 6 | object Failure { 7 | def unapply[T,U](p: ParseResult[T,U]) = 8 | if (!p.success) Some(p.error) 9 | else None 10 | } 11 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/parseresult/Success.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.parseresult 2 | 3 | /** 4 | * Extractor for ParseResult in case of success 5 | */ 6 | object Success { 7 | def unapply[T,U](p: ParseResult[T,U]) = 8 | if (p.success) Some(p.result) 9 | else None 10 | } 11 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/ruleprocessing/ReduceRules.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.ruleprocessing 2 | 3 | import scala.collection.mutable.HashMap 4 | 5 | /** 6 | * Combine rules 7 | */ 8 | trait ReduceRules extends RulesProcessing { 9 | def combine(rules: HashMap[String, RuleInfo]): c.Tree 10 | } 11 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/input/ArrayInput.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.input 2 | 3 | /** 4 | * Created by Eric on 22.04.14. 5 | * Input to deal with arrays 6 | */ 7 | trait ArrayInput extends ArrayLikeInput { 8 | 9 | import c.universe._ 10 | def inputType = c.typecheck(tq"Array[$inputElemType]",c.TYPEmode).tpe 11 | 12 | } 13 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/ruleprocessing/MapRules.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.ruleprocessing 2 | 3 | import scala.collection.mutable.HashMap 4 | 5 | /** 6 | * Transform rules 7 | */ 8 | trait MapRules extends RulesProcessing { 9 | def process(rules: HashMap[String, RuleInfo]): HashMap[String, RuleInfo] = rules 10 | } 11 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet10: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:12:45 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830036538078387; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:12:45 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet11: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:13:15 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830039562280838; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:13:15 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet12: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:13:45 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830042585994215; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:13:45 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet13: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:14:16 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830045690262308; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:14:16 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet14: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:14:47 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830048713771254; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:14:47 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet16: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:15:47 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830054772130157; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:15:47 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet17: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:16:17 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830057797199665; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:16:17 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet19: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:17:18 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830063852908156; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:17:18 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet21: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:18:19 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830069908595001; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:18:19 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet28: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:21:51 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830091152263382; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:21:51 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet29: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:22:21 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830094176787564; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:22:21 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet32: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:23:52 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830103269185469; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:23:52 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet33: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:24:22 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830106295961953; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:24:22 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet5: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:10:13 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830021393800171; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:10:13 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet54: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:35:00 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830170094703955; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:35:00 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet57: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:36:31 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830179192496004; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:36:31 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet62: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:39:03 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830194372704851; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:39:03 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet7: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:11:14 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830027455579259; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:11:14 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet8: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:11:44 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830030482071063; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:11:44 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet81: -------------------------------------------------------------------------------- 1 | HTTP/1.1 401 Unauthorized 2 | content-length: 63 3 | content-type: application/json; charset=utf-8 4 | date: Wed, 04 Sep 2013 13:48:40 UTC 5 | server: tfe 6 | set-cookie: guest_id=v1%3A137830252056603337; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:48:40 UTC 7 | 8 | {"errors":[{"message":"Could not authenticate you","code":32}]} 9 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/implementations/FinalFastParserImpl.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.implementations 2 | 3 | /** 4 | * Created by Eric on 22.04.14. 5 | * Trait used to recognise other parsers. 6 | * val parser = FastParser {...} 7 | * The type of parser will be AnyRef extends FinalFastParserImpl {..} 8 | */ 9 | trait FinalFastParserImpl 10 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/input/StringInput.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.input 2 | 3 | /** 4 | * Created by Eric on 22.04.14. 5 | */ 6 | trait StringInput extends StringLikeInput { 7 | import c.universe._ 8 | 9 | def inputType = typeOf[String] 10 | 11 | override def inputWindowType: c.Type = typeOf[fastparsers.input.InputWindow.StringStruct] 12 | 13 | override def getInputWindow(start: c.Tree, end: c.Tree): c.Tree = q"new fastparsers.input.InputWindow.StringStruct($inputValue, $start, $end)" 14 | } 15 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/input/CharArrayInput.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.input 2 | 3 | /** 4 | * Created by Eric on 22.04.14. 5 | */ 6 | trait CharArrayInput extends StringLikeInput { 7 | import c.universe._ 8 | 9 | def inputType = typeOf[Array[Char]] 10 | override def inputWindowType: c.Type = typeOf[fastparsers.input.InputWindow.CharArrayStruct] 11 | 12 | override def getInputWindow(start: c.Tree, end: c.Tree): c.Tree = q"new fastparsers.input.InputWindow.CharArrayStruct($inputValue, $start, $end)" 13 | } 14 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/ruleprocessing/RulesProcessing.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.ruleprocessing 2 | 3 | import scala.reflect.macros.whitebox.Context 4 | 5 | /** 6 | * Created by Eric on 22.04.14. 7 | */ 8 | trait RulesProcessing { 9 | val c: Context 10 | type RuleType = c.Type 11 | type RuleCode = c.Tree 12 | type ParamInfo = (c.TermName, c.Type) 13 | 14 | case class RuleInfo(typ: RuleType,code: RuleCode, params: List[c.Tree], typeParams: List[c.universe.TypeDef], oldCode: c.Tree) 15 | } 16 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/parseresult/ParseResult.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.parseresult 2 | 3 | /** 4 | * 5 | * @param success If the result is a success 6 | * @param error The error message if any (only relevant if success == false) 7 | * @param result The result generated by the parser 8 | * @param inputPos The position in the fastparsers.input at which the parser has finished to read 9 | * @tparam T Type of the result 10 | * @tparam U Type of the error 11 | */ 12 | case class ParseResult[+T,U](success: Boolean, error: U, result: T, inputPos: Int) 13 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/parsers/FlatMapParsers.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.parsers 2 | 3 | import scala.annotation.compileTimeOnly 4 | 5 | /** 6 | * Created by Eric on 22.04.14. 7 | * Interface for flatMap combinator 8 | */ 9 | trait FlatMapParsers { 10 | 11 | implicit class flatmapparsers[T](p: Parser[T]) { 12 | @compileTimeOnly("can’t be used outside FastParser") 13 | def flatMap[U](f: T => Parser[U]): Parser[U] = ??? 14 | 15 | @compileTimeOnly("can’t be used outside FastParser") 16 | def >>[U](f: T => Parser[U]): Parser[U] = ??? 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/getAST.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework 2 | 3 | import scala.language.experimental.macros 4 | import scala.reflect.macros.whitebox.Context 5 | 6 | /** 7 | * Created by Eric on 22.04.14. 8 | */ 9 | object getAST { 10 | def get(code: Any):Any = macro getImpl 11 | 12 | def getImpl(c: Context)(code: c.Tree): c.Tree = { 13 | import c.universe._ 14 | 15 | val tmp = q"$code" 16 | val elem: c.Type = c.typecheck(tq"Char",c.TYPEmode).tpe 17 | val input: c.Type = typeOf[Array[Char]] 18 | c.abort(c.enclosingPosition,show(code)) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/scala/HttpParserSpecs.scala: -------------------------------------------------------------------------------- 1 | import org.scalatest.FunSuite 2 | 3 | import HttpParsers._ 4 | import scala.util.parsing.combinator._ 5 | import TestsHelper._ 6 | /** 7 | * Created by Eric on 05.04.14. 8 | */ 9 | class HttpParserSpecs extends FunSuite { 10 | 11 | (1 to 101).foreach{ i => 12 | test("httpparser " + "FastParsers/src/test/resources/tweet" + i){ 13 | compareImplementations("FastParsers/src/test/resources/tweet" + i, 14 | HTTPImpl1.httpparser.respAndMessage, 15 | HTTP, 16 | x => HTTP.parse(HTTP.respAndMessage, x)) 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/error/DefaultParseError.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.error 2 | 3 | /** 4 | * Created by Eric on 23.04.14. 5 | */ 6 | /** 7 | * An error manager that replicate the one used in Scala Parser combinators, so not really helpful. 8 | */ 9 | trait DefaultParseError extends ParseError{ 10 | import c.universe._ 11 | 12 | def errorType = tq"String" 13 | 14 | override def initError(cont: c.Tree): c.Tree = { 15 | q""" 16 | var error = " " 17 | $cont 18 | """ 19 | } 20 | 21 | def pushError(error: String, inputPos: c.Tree): c.Tree = 22 | q"""error = $error + " at " + $inputPos """ 23 | } 24 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/input/StringLikeInput.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.input 2 | 3 | /** 4 | * ArrayInput which work on Strings 5 | */ 6 | trait StringLikeInput extends ArrayLikeInput { 7 | 8 | import c.universe._ 9 | 10 | private val inputpositioned = TermName(c.freshName("inputpositioned")) 11 | 12 | def inputElemType = typeOf[Char] 13 | 14 | override def initInput(startpos: c.Tree, then: c.Tree) = 15 | super.initInput(startpos, 16 | q""" 17 | val $inputpositioned = new fastparsers.tools.ToPosition.IndexedCharSeqToPosition($inputValue) 18 | $then 19 | """ 20 | ) 21 | 22 | override def getPositionned(offset: c.Tree): c.Tree = q"$inputpositioned.get($offset)" 23 | } 24 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet9ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 5232 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:12:15 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:12:15 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830033507910847; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:12:15 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 174 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 9999f617cf8c0608 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet100ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 90767 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:58:18 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:58:18 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830309828896653; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:58:18 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 170 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: 49a51850d5528f6e 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet101ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 25227 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:58:48 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:58:48 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830312867459113; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:58:48 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 169 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: a71b616baa690e38 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet15ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 32239 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:15:17 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:15:17 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830051740804171; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:15:17 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 173 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 5859a6a377e55370 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet18ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 14892 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:16:48 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:16:48 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830060824634168; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:16:48 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 172 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: b8704b5c6c2f6845 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet20ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 52489 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:17:48 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:17:48 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830066877566667; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:17:48 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 171 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: f6a46168963c2c73 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet22ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 112064 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:18:49 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:18:49 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830072932875057; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:18:49 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 170 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: e60c3fa202e841c5 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet23ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 169167 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:19:19 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:19:19 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830075969863734; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:19:19 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 169 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 196edcbda76dbdab 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet24ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 77465 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:19:50 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:19:50 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830079009001043; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:19:50 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 168 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 249b86d29a5071a5 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet25ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 127230 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:20:20 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:20:20 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830082041804398; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:20:20 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 167 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: e0a22c67a3854fa3 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet26ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 54928 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:20:50 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:20:50 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830085081763226; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:20:50 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 166 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 455891bcfaa91df2 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet27ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 183857 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:21:21 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:21:21 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830088117182319; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:21:21 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 165 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 89906ac77192b540 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet30ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 76241 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:22:52 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:22:52 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830097203802569; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:22:52 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 164 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 36ce6cfe3a531467 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet31ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 53813 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:23:22 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:23:22 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830100235016916; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:23:22 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 179 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 02f18f15529212ce 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet34ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 117834 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:24:53 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:24:53 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830109321233677; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:24:53 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 178 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 9a1d794a7a16d294 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet35ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 23213 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:25:23 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:25:23 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830112357202089; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:25:23 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 177 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 9a8f2958a348acb3 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet36ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 68877 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:25:53 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:25:53 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830115385811008; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:25:53 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 176 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: bac18c8b423c241f 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet37ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 87529 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:26:24 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:26:24 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830118419657735; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:26:24 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 175 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: f5502d44b7769092 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet38ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 141677 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:26:54 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:26:54 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830121465923297; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:26:54 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 174 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 11389a50e8d9d24b 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet39ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 105634 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:27:25 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:27:25 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830124506567614; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:27:25 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 173 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 797de2e78a71ac22 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet3ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 59411 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:09:13 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:09:13 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830015319162047; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:09:13 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 177 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 4e036df3b5ee1ec1 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet40ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 174502 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:27:55 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:27:55 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830127544491725; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:27:55 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 172 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: cd1549f31ecbc365 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet41ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 146258 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:28:26 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:28:25 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830130586660757; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:28:26 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 171 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: faa09c4c69892bd8 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet42ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 73438 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:28:56 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:28:56 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830133627185762; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:28:56 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 170 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 0076df46c9f67ebb 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet43ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 104669 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:29:26 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:29:26 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830136659144014; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:29:26 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 169 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: ae0d95b39edee2e5 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet44ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 58570 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:29:57 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:29:56 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830139694203328; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:29:57 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 168 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 5cc552ec51414933 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet45ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 131907 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:30:27 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:30:27 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830142731040965; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:30:27 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 167 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: a2b7f157a3f9a39a 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet46ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 120526 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:30:57 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:30:57 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830145768625779; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:30:57 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 166 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 52c76ec9b96797cf 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet47ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 155959 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:31:28 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:31:28 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830148815639185; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:31:28 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 165 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 181efa6067038221 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet48ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 162559 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:31:58 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:31:58 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830151850135459; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:31:58 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 164 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: c6c24bdfd120a4ca 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet49ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 179274 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:32:29 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:32:28 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830154893599003; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:32:29 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 163 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: d31cf1554de12af0 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet4ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 179222 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:09:43 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:09:43 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830018351272411; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:09:43 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 176 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 792486f4201a23f7 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet50ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 116380 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:32:59 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:32:59 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830157941758107; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:32:59 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 162 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 5d562b82f8217cef 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet51ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 110679 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:33:30 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:33:29 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830160988359275; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:33:30 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 161 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 24924802ae9f6177 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet52ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 44039 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:34:00 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:34:00 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830164028593503; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:34:00 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 160 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 2aecf50ad90fa463 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet53ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 94878 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:34:30 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:34:30 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830167059145699; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:34:30 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 159 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: ebca673ec0689900 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet55ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 127847 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:35:31 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:35:31 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830173122336125; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:35:31 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 158 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 40090e72bb3a7da3 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet56ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 142331 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:36:01 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:36:01 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830176154870249; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:36:01 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 157 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 2d5c0eea44e2bacd 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet58ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 118836 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:37:02 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:37:02 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830182217428973; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:37:02 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 156 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: 7b939141c02fd8e3 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet59ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 166855 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:37:32 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:37:32 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830185252964433; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:37:32 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 155 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: ed92de8a38f175bb 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet60ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 154373 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:38:03 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:38:02 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830188295436857; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:38:03 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 154 17 | x-rate-limit-reset: 1378301902 18 | x-transaction: b6b6440e7503d926 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet61ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 173334 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:38:33 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:38:33 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830191332364676; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:38:33 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 179 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 2ab2cd9161782434 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet63ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 179685 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:39:34 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:39:33 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830197397685685; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:39:34 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 178 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 38d619b9e1a37889 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet64ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 108832 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:40:04 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:40:04 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830200436685353; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:40:04 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 177 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 67157fc28cfd7d9d 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet65ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 174791 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:40:34 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:40:34 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830203470335411; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:40:34 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 176 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 9d7778bac97928ef 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet66ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 54171 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:41:05 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:41:05 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830206515845047; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:41:05 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 175 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 14bcae1972c939c6 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet67ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 72533 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:41:35 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:41:35 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830209546242552; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:41:35 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 174 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 30472ed5bc1652a0 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet68ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 91705 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:42:05 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:42:05 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830212580527809; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:42:05 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 173 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 3b0b270ade4782ad 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet69ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 130852 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:42:36 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:42:36 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830215617515166; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:42:36 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 172 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: ec361671a1545ec8 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet6ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 158607 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:10:44 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:10:44 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830024420373609; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:10:44 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 175 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: b81a4ecfe9aef557 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet70ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 30149 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:43:06 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:43:06 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830218656530534; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:43:06 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 171 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: d45d6efea49f56a6 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet71ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 14651 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:43:36 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:43:36 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830221688083421; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:43:36 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 170 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 44a0f6da1c0bf6e9 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet72ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 45439 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:44:07 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:44:07 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830224716648147; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:44:07 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 169 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 00fdbae146e76b3e 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet73ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 91697 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:44:37 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:44:37 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830227748084148; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:44:37 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 168 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: a181644487e3d156 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet74ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 65529 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:45:07 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:45:07 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830230784555034; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:45:07 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 167 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: e67b8cd0d04d03b5 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet75ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 201927 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:45:38 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:45:38 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830233820239879; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:45:38 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 166 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: f4ba2e554660b55c 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet76ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 108793 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:46:08 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:46:08 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830236870137002; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:46:08 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 165 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: c4c96281afdf933f 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet77ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 94366 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:46:39 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:46:39 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830239909143627; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:46:39 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 164 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: b15693a19c6ad82a 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet78ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 72566 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:47:09 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:47:09 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830242947556902; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:47:09 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 163 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 410a740ac3c4e270 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet79ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 115505 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:47:39 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:47:39 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830245980113098; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:47:39 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 162 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: f633b0d38c764b1b 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet80ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 109325 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:48:10 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:48:10 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830249018408821; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:48:10 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 161 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 251a8d32e8c4f430 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet82ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 103650 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:49:11 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:49:10 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830255082588322; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:49:11 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 160 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 458495b851a0165a 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet83ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 135015 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:49:41 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:49:41 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830258132842208; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:49:41 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 159 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: b76c5ac500855105 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet84ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 87446 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:50:11 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:50:11 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830261175042965; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:50:11 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 158 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: e79aa39e2525e9c4 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet85ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 120175 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:50:42 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:50:42 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830264212642420; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:50:42 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 157 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 1d8d76320176905d 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet86ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 84591 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:51:12 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:51:12 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830267252688553; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:51:12 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 156 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: bd7135eac548faf1 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet87ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 151773 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:51:43 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:51:42 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830270290018635; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:51:43 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 155 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 0aa63d271a4e0b18 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet88ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 59777 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:52:13 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:52:13 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830273328354258; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:52:13 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 154 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 15009bedc49a48b8 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet89ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 45053 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:52:43 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:52:43 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830276362377697; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:52:43 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 153 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: ce53e833717f5677 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet90ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 158133 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:53:14 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:53:13 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830279395210875; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:53:14 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 152 17 | x-rate-limit-reset: 1378302813 18 | x-transaction: 9b472cc86c216694 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet91ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 47558 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:53:44 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:53:44 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830282434840447; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:53:44 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 179 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: 97462a6fd61ea0ff 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet92ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 110014 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:54:14 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:54:14 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830285468066987; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:54:14 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 178 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: a3d7ca742422a744 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet93ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 47250 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:54:45 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:54:45 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830288505605420; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:54:45 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 177 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: 44fbe4ab855be86f 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet94ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 137404 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:55:15 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:55:15 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830291541609147; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:55:15 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 176 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: f6e82f175f45c2c0 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet95ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 53267 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:55:46 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:55:46 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830294603611988; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:55:46 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 175 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: 5695532217f0c6a4 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet96ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 134692 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:56:16 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:56:16 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830297639090139; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:56:16 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 174 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: cffe95048435c50e 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet97ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 154283 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:56:47 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:56:47 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830300707669129; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:56:47 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 173 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: 7343c0c58b763626 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet98ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 111456 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:57:17 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:57:17 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830303752826604; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:57:17 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 172 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: 4c69878967c4d4bf 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet99ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 122281 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:57:48 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:57:47 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830306789095512; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:57:48 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 171 17 | x-rate-limit-reset: 1378303724 18 | x-transaction: eafc1e5872d4fda8 19 | 20 | 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet2ho: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 153483 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:08:42 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:08:42 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830012271879294; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:08:42 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 178 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 49bb634da730cfef 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /FastParsers/src/test/scala/JsonParserSpecs.scala: -------------------------------------------------------------------------------- 1 | import JsonParsers._ 2 | import TestsHelper._ 3 | import org.scalatest.FunSuite 4 | 5 | /** 6 | * Created by Eric on 05.04.14. 7 | */ 8 | class JsonParserSpecs extends FunSuite { 9 | (1 to 20).foreach{ i => 10 | test("jsonparser " + "FastParsers/src/test/resources/json" + i){ 11 | compareImplementations("FastParsers/src/test/resources/json" + i, 12 | JSonImpl1.jsonparser.value, 13 | JSON, 14 | x => JSON.parse(JSON.value, x)) 15 | } 16 | } 17 | test("jsonparser " + "FastParsers/src/test/resources/json.big" + 1){ 18 | compareImplementations("FastParsers/src/test/resources/json.big" + 1, 19 | JSonImpl1.jsonparser.value, 20 | JSON, 21 | x => JSON.parse(JSON.value, x)) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/error/ParseError.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.error 2 | 3 | import scala.reflect.macros.whitebox.Context 4 | 5 | /** 6 | * Created by Eric on 23.04.14. 7 | */ 8 | /** 9 | * Trait used to deal with error in Parsers implementation 10 | */ 11 | trait ParseError { 12 | val c: Context 13 | import c.universe._ 14 | 15 | def errorType: c.Tree 16 | 17 | val success = TermName(c.freshName("success")) 18 | 19 | /** 20 | * Initialize objects needed to deal with error and then run the code that follow 21 | * @param cont The code that follow 22 | */ 23 | def initError(cont: c.Tree): c.Tree = cont 24 | 25 | /** 26 | * @param error The error message 27 | * @param inputPos The position in the input at which the error occurs 28 | * @return The code that deals with the error 29 | */ 30 | def pushError(error: String, inputPos: c.Tree): c.Tree 31 | } 32 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/parsers/TokenParsers.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.parsers 2 | 3 | import fastparsers.input.InputWindow 4 | import InputWindow.InputWindow 5 | import scala.annotation.compileTimeOnly 6 | 7 | /** 8 | * Created by Eric on 22.04.14. 9 | * Parsers which parse tokens 10 | * StringRepr can be, for example, a String or an Array[Char] 11 | */ 12 | trait TokenParsers[StringRepr] { 13 | 14 | @compileTimeOnly("can’t be used outside FastParser") 15 | implicit def lit(str: StringRepr): Parser[StringRepr] = ??? 16 | 17 | @compileTimeOnly("can’t be used outside FastParser") 18 | def ident: Parser[InputWindow[StringRepr]] = ??? 19 | 20 | @compileTimeOnly("can’t be used outside FastParser") 21 | def number: Parser[InputWindow[StringRepr]] = ??? 22 | 23 | @compileTimeOnly("can’t be used outside FastParser") 24 | def decimalNumber: Parser[InputWindow[StringRepr]] = ??? 25 | 26 | @compileTimeOnly("can’t be used outside FastParser") 27 | def stringLit: Parser[InputWindow[StringRepr]] = ??? 28 | 29 | @compileTimeOnly("can’t be used outside FastParser") 30 | def whitespaces: Parser[InputWindow[StringRepr]] = ??? 31 | 32 | } 33 | -------------------------------------------------------------------------------- /FastParsers/src/test/scala/FastCharSequence.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Found on the net 3 | */ 4 | import java.lang.CharSequence 5 | 6 | class FastCharSequence(chars: Array[Char], val startBounds: Int, val endBounds: Int) extends CharSequence { 7 | def this(chars: Array[Char]) = this(chars, 0, chars.length) 8 | def this(input: String) = this(input.toCharArray) 9 | 10 | def length(): Int = endBounds - startBounds 11 | 12 | def charAt(index: Int): Char = { 13 | if (index < length) { 14 | chars(index + startBounds) 15 | } else { 16 | throw new IndexOutOfBoundsException(s"$boundsInfo index: $index") 17 | } 18 | } 19 | 20 | def subSequence(start: Int, end: Int): CharSequence = { 21 | if (start >= 0 && start <= length && end >= 0 && end <= length) { 22 | new FastCharSequence(chars, startBounds + start, startBounds + end) 23 | } else { 24 | throw new IndexOutOfBoundsException(s"$boundsInfo start: $start, end $end") 25 | } 26 | } 27 | 28 | override def toString(): String = new String(chars, startBounds, length) 29 | 30 | private def boundsInfo = s"current: (startBounds: $startBounds, endBounds: $endBounds, length: $length, chars length: ${chars.length})" 31 | } -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/input/ParseInput.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.input 2 | 3 | import scala.reflect.macros.whitebox.Context 4 | 5 | /** 6 | * General interface to work on an fastparsers.input in the macro world. 7 | */ 8 | trait ParseInput { 9 | val c: Context 10 | 11 | import c.universe._ 12 | 13 | def inputType: c.Type 14 | def inputElemType: c.Type 15 | def inputWindowType: c.Type = c.typecheck(tq"fastparsers.input.InputWindow.InputWindow[$inputElemType]",c.TYPEmode).tpe 16 | 17 | val inputValue = TermName(c.freshName("input")) 18 | 19 | def initInput(startpos: c.Tree, then: c.Tree): c.Tree 20 | 21 | def currentInput: c.Tree 22 | 23 | def advance: c.Tree 24 | 25 | def setpos(pos: c.Tree): c.Tree 26 | 27 | def mark(code: c.Tree => c.Tree): c.Tree 28 | 29 | def isEOI: c.Tree 30 | 31 | def isNEOI: c.Tree 32 | 33 | def pos: c.Tree 34 | 35 | def slice(begin: c.Tree, end: c.Tree): c.Tree 36 | 37 | def inputsize: c.Tree 38 | 39 | def getPositionned(offset: c.Tree): c.Tree = q"scala.util.parsing.input.NoPosition" 40 | 41 | def getInputWindow(start: c.Tree, end: c.Tree): c.Tree = q"new fastparsers.input.InputWindow.InputWindow[$inputElemType]($inputValue, $start, $end)" 42 | } 43 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/implementations/FastParsers.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.implementations 2 | 3 | import scala.language.experimental.macros 4 | import fastparsers.parsers._ 5 | import scala.reflect.macros.whitebox.Context 6 | import fastparsers.framework.ruleprocessing.{RuleCombiner, ParseRules, RulesInliner, RulesTransformer} 7 | import fastparsers.input.StringInput 8 | import fastparsers.error.IgnoreParseError 9 | 10 | /** 11 | * Example of a parser working on string. 12 | * Interface for FastParsersImpl 13 | */ 14 | object FastParsers extends BaseParsers[Char, String] with RepParsers with TokenParsers[String] with FlatMapParsers { 15 | def FastParser(rules: => Unit): FinalFastParserImpl = macro FastParsersImpl.FastParser 16 | } 17 | 18 | /** 19 | * Here is where the FastParsers implementation is composed to make an actual useful FastParsers 20 | */ 21 | 22 | class FastParsersImpl(val c: Context) extends BaseImpl 23 | with RulesTransformer with RulesInliner 24 | with ParseRules with BaseParsersImpl with RepParsersImpl 25 | with TokenParsersImpl with FlatMapImpl with RuleCombiner 26 | with StringInput with IgnoreParseError with DontIgnoreResults{ 27 | 28 | override def FastParser(rules: c.Tree) = super.FastParser(rules) //why ?? 29 | } 30 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/implementations/FastParsersCharArray.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.implementations 2 | 3 | import scala.language.experimental.macros 4 | import fastparsers.parsers._ 5 | import scala.reflect.macros.whitebox.Context 6 | import fastparsers.framework.ruleprocessing.{RuleCombiner, ParseRules, RulesInliner, RulesTransformer} 7 | import fastparsers.input.CharArrayInput 8 | import fastparsers.error.IgnoreParseError 9 | 10 | /** 11 | * Interface for CharArrayImpl 12 | */ 13 | object FastParsersCharArray extends BaseParsers[Char, Array[Char]] with RepParsers with TokenParsers[Array[Char]] with FlatMapParsers { 14 | def FastParsersCharArray(rules: => Unit): FinalFastParserImpl = macro CharArrayImpl.FastParser 15 | } 16 | 17 | /** 18 | * Implementation of Parsers that deal with CharArray (different from ArrayParser[Char] in that it can deal with 19 | * TokenParsers 20 | */ 21 | class CharArrayImpl(val c: Context) extends BaseImpl with RulesTransformer with RulesInliner 22 | with ParseRules with BaseParsersImpl with RepParsersImpl with FlatMapImpl with RuleCombiner 23 | with TokenParsersImpl with CharArrayInput 24 | with IgnoreParseError with IgnoreResults { 25 | override def FastParser(rules: c.Tree) = super.FastParser(rules) //why ?? 26 | } 27 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/input/ArrayLikeInput.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.input 2 | 3 | /** 4 | * Input on a array-like. 5 | * 6 | * Elem, Input, inputType and inputElemType must be set 7 | */ 8 | trait ArrayLikeInput extends ParseInput { 9 | 10 | import c.universe._ 11 | 12 | private val inputpos = TermName(c.freshName("inputpos")) 13 | private val inputlength = TermName(c.freshName("inputsize")) 14 | 15 | 16 | def initInput(startpos: c.Tree, then: c.Tree) = 17 | q""" 18 | var $inputpos = $startpos 19 | val $inputlength = $inputValue.size 20 | $then 21 | """ 22 | 23 | def currentInput = q"$inputValue($inputpos)" 24 | 25 | def advance = q"$inputpos = $inputpos + 1" 26 | 27 | def setpos(pos: c.Tree): c.Tree = q"$inputpos = $pos" 28 | 29 | def mark(code: c.Tree => c.Tree) = { 30 | val input_tmp = TermName(c.freshName) 31 | q""" 32 | val $input_tmp = $inputpos 33 | ${code(q"$inputpos = $input_tmp")} 34 | """ 35 | } 36 | 37 | def isEOI = q"$inputpos >= $inputlength" 38 | 39 | def isNEOI = q"$inputpos < $inputlength" 40 | 41 | def pos = q"$inputpos" 42 | 43 | def inputsize = q"$inputlength" 44 | 45 | def slice(begin: c.Tree, end: c.Tree) = { 46 | q"$inputValue.slice($begin,$end)" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /Examples/src/main/scala/Calculator.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by Eric on 07.04.14. 3 | */ 4 | 5 | import fastparsers.framework.implementations.FastParsers 6 | import fastparsers.framework.parseresult._ 7 | import fastparsers.parsers.Parser 8 | import scala.language.reflectiveCalls 9 | import scala.language.implicitConversions 10 | 11 | object Calculator { 12 | 13 | def main(args: Array[String]) { 14 | import FastParsers._ 15 | 16 | def exec(x:(Int,Option[(Function2[Int,Int,Int],Int)])):Int = x._2 match { 17 | case None => x._1 18 | case Some((op,y)) => op(x._1,y) 19 | } 20 | 21 | val code = FastParser{ 22 | def op1 = lit("+") ^^^ ((x:Int,y:Int) => x + y) | 23 | lit("-") ^^^ ((x:Int,y:Int) => x - y) 24 | 25 | def op2 = lit("*") ^^^ ((x:Int,y:Int) => x * y) | 26 | lit("/") ^^^ ((x:Int,y:Int) => x / y) 27 | 28 | def factor: Parser[Int] = number ^^(_.toString.toInt) | lit("(") ~> expr <~ ")" 29 | def term: Parser[Int] = factor ~ opt(op2 ~ term) ^^ exec 30 | def expr: Parser[Int] = term ~ opt(op1 ~ expr) ^^ exec 31 | } 32 | 33 | code.expr("21+(5+1 ) * 2 +2 *8") match { 34 | case Success(x) => println(x) 35 | case Failure(error) => println("failure : " + error) 36 | } 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/tools/DefaultValue.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.tools 2 | 3 | /** 4 | * Created by Eric on 10.04.14. 5 | 6 | see 7 | http://stackoverflow.com/questions/5260298/how-can-i-obtain-the-default-value-for-a-type-in-scala 8 | http://missingfaktor.blogspot.ch/2011/08/emulating-cs-default-keyword-in-scala.html 9 | */ 10 | class Default[+A](val default: A) 11 | 12 | trait LowerPriorityImplicits { 13 | // Stop AnyRefs from clashing with AnyVals 14 | implicit def defaultNull[A <: AnyRef]: Default[A] = new Default[A](null.asInstanceOf[A]) 15 | } 16 | 17 | object Default extends LowerPriorityImplicits { 18 | implicit object DefaultDouble extends Default[Double](0.0) 19 | implicit object DefaultFloat extends Default[Float](0.0F) 20 | implicit object DefaultInt extends Default[Int](0) 21 | implicit object DefaultLong extends Default[Long](0L) 22 | implicit object DefaultShort extends Default[Short](0) 23 | implicit object DefaultByte extends Default[Byte](0) 24 | implicit object DefaultChar extends Default[Char]('\u0000') 25 | implicit object DefaultBoolean extends Default[Boolean](false) 26 | implicit object DefaultUnit extends Default[Unit](()) 27 | implicit object DefaultAny extends Default[Any](null) 28 | 29 | def value[A](implicit value: Default[A]): A = value.default 30 | } -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/implementations/FastParsersCharArrayNoInline.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.implementations 2 | 3 | import scala.language.experimental.macros 4 | import fastparsers.parsers._ 5 | import scala.reflect.macros.whitebox.Context 6 | import fastparsers.framework.ruleprocessing.{RuleCombiner, ParseRules, RulesInliner, RulesTransformer} 7 | import fastparsers.input.CharArrayInput 8 | import fastparsers.error.IgnoreParseError 9 | 10 | /** 11 | * Interface for CharArrayImpl 12 | */ 13 | object FastParsersCharArrayNoInline extends BaseParsers[Char, Array[Char]] with RepParsers with TokenParsers[Array[Char]] with FlatMapParsers { 14 | def FastParsersCharArray(rules: => Unit): FinalFastParserImpl = macro CharArrayNoInlineImpl.FastParser 15 | } 16 | 17 | /** 18 | * Implementation of Parsers that deal with CharArray (different from ArrayParser[Char] in that it can deal with 19 | * TokenParsers 20 | */ 21 | class CharArrayNoInlineImpl(val c: Context) extends BaseImpl with RulesTransformer 22 | with ParseRules with BaseParsersImpl with RepParsersImpl with FlatMapImpl with RuleCombiner 23 | with TokenParsersImpl with CharArrayInput 24 | with IgnoreParseError with DontIgnoreResults { 25 | override def FastParser(rules: c.Tree) = super.FastParser(rules) //why ?? 26 | } 27 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/implementations/FastParsersCharArrayDefaultErrors.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.implementations 2 | 3 | import scala.language.experimental.macros 4 | import fastparsers.parsers._ 5 | import scala.reflect.macros.whitebox.Context 6 | import fastparsers.framework.ruleprocessing.{RuleCombiner, ParseRules, RulesInliner, RulesTransformer} 7 | import fastparsers.input.CharArrayInput 8 | import fastparsers.error.DefaultParseError 9 | 10 | /** 11 | * Interface for CharArrayImpl 12 | */ 13 | object FastParsersCharArrayDefaultErrors extends BaseParsers[Char, Array[Char]] with RepParsers with TokenParsers[Array[Char]] with FlatMapParsers { 14 | def FastParsersCharArray(rules: => Unit): FinalFastParserImpl = macro CharArrayDefaultErrorsImpl.FastParser 15 | } 16 | 17 | /** 18 | * Implementation of Parsers that deal with CharArray (different from ArrayParser[Char] in that it can deal with 19 | * TokenParsers 20 | */ 21 | class CharArrayDefaultErrorsImpl(val c: Context) extends BaseImpl with RulesTransformer 22 | with ParseRules with BaseParsersImpl with RepParsersImpl with FlatMapImpl with RuleCombiner 23 | with TokenParsersImpl with CharArrayInput 24 | with DefaultParseError with DontIgnoreResults { 25 | override def FastParser(rules: c.Tree) = super.FastParser(rules) //why ?? 26 | } 27 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/implementations/FastParsersCharArrayIgnoreResults.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.implementations 2 | 3 | import scala.language.experimental.macros 4 | import fastparsers.parsers._ 5 | import scala.reflect.macros.whitebox.Context 6 | import fastparsers.framework.ruleprocessing.{RuleCombiner, ParseRules, RulesInliner, RulesTransformer} 7 | import fastparsers.input.CharArrayInput 8 | import fastparsers.error.DefaultParseError 9 | 10 | /** 11 | * Interface for CharArrayImpl 12 | */ 13 | object FastParsersCharArrayIgnoreResults extends BaseParsers[Char, Array[Char]] with RepParsers with TokenParsers[Array[Char]] with FlatMapParsers { 14 | def FastParsersCharArray(rules: => Unit): FinalFastParserImpl = macro CharArrayIgnoreResultsImpl.FastParser 15 | } 16 | 17 | /** 18 | * Implementation of Parsers that deal with CharArray (different from ArrayParser[Char] in that it can deal with 19 | * TokenParsers 20 | */ 21 | class CharArrayIgnoreResultsImpl(val c: Context) extends BaseImpl with RulesTransformer 22 | with ParseRules with BaseParsersImpl with RepParsersImpl with FlatMapImpl with RuleCombiner 23 | with TokenParsersImpl with CharArrayInput 24 | with DefaultParseError with IgnoreResults { 25 | override def FastParser(rules: c.Tree) = super.FastParser(rules) //why ?? 26 | } 27 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/implementations/FastArrayParsers.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.implementations 2 | 3 | import scala.language.experimental.macros 4 | import fastparsers.parsers._ 5 | import scala.reflect.macros.whitebox.Context 6 | import fastparsers.framework.ruleprocessing.{RuleCombiner, ParseRules, RulesInliner, RulesTransformer} 7 | import fastparsers.input.ArrayInput 8 | import fastparsers.error.DefaultParseError 9 | 10 | /** 11 | * Created by Eric on 22.04.14. 12 | * Interface for ArrayParserImpl 13 | */ 14 | class FastArrayParsers[T] extends BaseParsers[T, Array[T]] with RepParsers with FlatMapParsers { 15 | def apply(rules: => Unit): FinalFastParserImpl = macro ArrayParserImpl.ArrayParserImpl[T] 16 | } 17 | 18 | /** 19 | * An implementation of Parser combinator that deal with Array[T] inputs 20 | */ 21 | object ArrayParserImpl { 22 | def ArrayParserImpl[T: context.WeakTypeTag](context: Context)(rules: context.Tree): context.Tree = { 23 | new BaseImpl with RulesTransformer with RulesInliner 24 | with ParseRules with BaseParsersImpl with RepParsersImpl 25 | with FlatMapImpl with RuleCombiner 26 | with ArrayInput with DefaultParseError with IgnoreResults { 27 | 28 | val c: context.type = context 29 | 30 | import c.universe._ 31 | 32 | def inputElemType = c.typecheck(tq"${implicitly[c.WeakTypeTag[T]]}",c.TYPEmode).tpe 33 | 34 | }.FastParser(rules) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Examples/src/main/scala/Test.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Created with IntelliJ IDEA. 3 | * User: Eric 4 | * Date: 12.02.14 5 | * Time: 15:57 6 | * To change this template use File | Settings | File Templates. 7 | */ 8 | 9 | //because warnings 10 | 11 | import fastparsers.framework.getAST 12 | import fastparsers.framework.implementations.{FastParsers, FastArrayParsers} 13 | import fastparsers.framework.parseresult._ 14 | import fastparsers.input.InputWindow 15 | import fastparsers.parsers.Parser 16 | import scala.collection.mutable.HashMap 17 | import scala.language.reflectiveCalls 18 | import scala.language.implicitConversions 19 | import scala.reflect.ClassTag 20 | 21 | 22 | object Test { 23 | 24 | def main(args: Array[String]) { 25 | 26 | import FastParsers._ 27 | val parser = FastParser { 28 | /*def rule = 'a' ~ 'c' 29 | def rule2(p: Parser[List[Char]]) = 'a' ~ p 30 | def rule3(y: Int) = rule2(repN('b', y)) 31 | 32 | def rule1(p: Parser[List[Char]], y: Int): Parser[Any] = 'a' ~ p ~ rule2(y) 33 | def rule2(x: Int): Parser[Any] = rule1(repN('c', x), x + 1) | 'b'*/ 34 | def parens[T](p: Parser[T]) = 'x' ~ p ~ 'x' 35 | def test = parens('a' ~ 'y') 36 | 37 | /*def parens(p: Parser[(Char, Char)]) = 'x' ~ p ~ 'x' 38 | def test = parens('a' ~ 'y') //TODO correct this bug*/ 39 | def test2 = 'x' ~ ('a' ~ 'y') ~ 'x' //~ ('a' ~ 'b') ~ 'x' 40 | 41 | } 42 | 43 | parser.test("xayx cacccb") match { 44 | case Success(x) => 45 | println(x) 46 | case Failure(msg) => println("failure : " + msg) 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/ruleprocessing/RuleCombiner.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.ruleprocessing 2 | 3 | import scala.reflect.macros.whitebox.Context 4 | import scala.collection.mutable.HashMap 5 | 6 | /** 7 | * Create the final parser object 8 | */ 9 | trait RuleCombiner extends ReduceRules { 10 | val c: Context 11 | 12 | import c.universe._ 13 | 14 | 15 | def combine(rules: HashMap[String, RuleInfo]) = { 16 | val anon = TypeName(c.freshName) 17 | val dmmy = TermName(c.freshName) //no joke : see http://stackoverflow.com/questions/14370842/getting-a-structural-type-with-an-anonymous-classs-methods-from-a-macro 18 | 19 | val methods = rules.values.map(_.code) 20 | 21 | val methodsEmpty = rules.keySet.map{ k => 22 | val rule = rules(k) 23 | val ruleName = TermName(k) 24 | rule.params match { 25 | case Nil => q"""@scala.annotation.compileTimeOnly("can't be used outside of FastParser") def $ruleName[..${rule.typeParams}]: fastparsers.parsers.Parser[${rule.typ}] = ???""" 26 | case params => q"""@scala.annotation.compileTimeOnly("can't be used outside of FastParser") def $ruleName[..${rule.typeParams}](..${rule.params}): fastparsers.parsers.Parser[${rule.typ}] = ???""" 27 | } 28 | } 29 | 30 | // 31 | 32 | val tree = q""" 33 | class $anon extends fastparsers.framework.implementations.FinalFastParserImpl { 34 | import scala.collection.mutable.ListBuffer 35 | import scala.reflect.runtime.universe._ 36 | ..$methodsEmpty 37 | ..$methods 38 | } 39 | val $dmmy = 0 40 | new $anon 41 | """ 42 | tree 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /project/Build.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | 4 | object FastParsersBuild extends Build { 5 | import PublishSettings._ 6 | 7 | def commonSettings = Seq( 8 | version := "0.1-SNAPSHOT", 9 | scalaVersion := "2.11.5" , 10 | //scalacOptions := Seq("-optimize"), 11 | libraryDependencies ++= Seq( 12 | "org.scala-lang" % "scala-compiler" % scalaVersion.value % "provided", 13 | "org.scala-lang" % "scala-reflect" % scalaVersion.value 14 | ) 15 | ) ++ publishSettings 16 | 17 | 18 | lazy val Examples = Project( 19 | id = "Examples", 20 | base = file("Examples"), 21 | dependencies = Seq(FastParsers), 22 | settings = commonSettings ++ Seq ( 23 | // include the macro classes and resources in the main jar 24 | mappings in (Compile, packageBin) ++= mappings.in(FastParsers, Compile, packageBin).value, 25 | // include the macro sources in the main source jar 26 | mappings in (Compile, packageSrc) ++= mappings.in(FastParsers, Compile, packageSrc).value 27 | ) 28 | ) 29 | 30 | lazy val FastParsers = Project( 31 | id = "FastParsers", 32 | base = file("FastParsers"), 33 | settings = commonSettings ++ publishableSettings ++ Seq ( 34 | resolvers ++= Seq( 35 | "Sonatype Releases" at "http://oss.sonatype.org/content/repositories/releases" 36 | ), 37 | //addSbtPlugin("org.scoverage" %% "sbt-scoverage" % "0.98.0"), 38 | 39 | libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.1" % "test", 40 | 41 | libraryDependencies += "org.scalacheck" %% "scalacheck" % "1.11.5" % "test", 42 | 43 | libraryDependencies += "com.storm-enroute" % "scalameter_2.11" % "0.6" % "test", 44 | 45 | testFrameworks += new TestFramework("org.scalameter.ScalaMeterFramework"), 46 | 47 | logBuffered := false 48 | ) 49 | ) 50 | } -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/parsers/RepParsers.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.parsers 2 | 3 | import scala.annotation.compileTimeOnly 4 | 5 | /** 6 | * Created by Eric on 22.04.14. 7 | * Interface for parsers representing repetition 8 | */ 9 | trait RepParsers { 10 | 11 | @compileTimeOnly("can’t be used outside FastParser") 12 | def rep[T](p: Parser[T], min: Int = 0, max: Int = -1): Parser[List[T]] = ??? 13 | 14 | @compileTimeOnly("can’t be used outside FastParser") 15 | def rep1[T](p: Parser[T]): Parser[List[T]] = ??? 16 | 17 | @compileTimeOnly("can’t be used outside FastParser") 18 | def repN[T](p: Parser[T], rep: Int): Parser[List[T]] = ??? 19 | 20 | @compileTimeOnly("can’t be used outside FastParser") 21 | def opt[T](p: Parser[T]): Parser[Option[T]] = ??? 22 | 23 | @compileTimeOnly("can’t be used outside FastParser") 24 | def repsep[T, U](p: Parser[T], sep: Parser[U]): Parser[List[T]] = ??? 25 | 26 | @compileTimeOnly("can’t be used outside FastParser") 27 | def repsep1[T, U](p: Parser[T], sep: Parser[U]): Parser[List[T]] = ??? 28 | 29 | 30 | @compileTimeOnly("can’t be used outside FastParser") 31 | def until[T,U](p: Parser[T], sep: Parser[U]): Parser[List[T]] = ??? 32 | 33 | implicit class repParser[T](p: Parser[T]) { 34 | @compileTimeOnly("can’t be used outside FastParser") 35 | def foldLeft[U](init: U, f: (U, T) => U): Parser[U] = ??? 36 | 37 | @compileTimeOnly("can’t be used outside FastParser") 38 | def foldRight[U, X >: T](init: U, f: (T, U) => U): Parser[U] = ??? 39 | 40 | @compileTimeOnly("can’t be used outside FastParser") 41 | def reduceLeft[U >: T](f: (U, T) => U): Parser[U] = ??? 42 | 43 | @compileTimeOnly("can’t be used outside FastParser") 44 | def reduceRight[U >: T](f: (T, U) => U): Parser[U] = ??? 45 | 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | FastParsers [![Build Status](https://travis-ci.org/begeric/FastParsers.svg)](https://travis-ci.org/begeric/FastParsers) 2 | =========== 3 | 4 | FastParsers is a Scala parser library which uses macros to transform easy-to-write parser combinators into efficient recursive-descent backtracking parsers. 5 | The generated parsers are about 20x faster than [Scala's parser combinator library](https://github.com/scala/scala-parser-combinators) even though its interface stay about the same. 6 | 7 | ### Example 8 | Here is an example of a basic JSON parser 9 | ```scala 10 | val jsonParser = FastParser { 11 | def value: Parser[Any] = obj | arr | stringLit | 12 | decimalNumber | "null" | "true" | "false" 13 | def obj: Parser[Any] = lit("{") ~> repsep (member, ",") <~ "}" 14 | def arr: Parser[Any] = lit("[") ~> repsep (value , ",") <~ "]" 15 | def member: Parser[Any] = stringLit ~ (lit(":") ~> value) 16 | } 17 | ``` 18 | This generate a *Parser Object* which allow you to call any of the transformed rules. 19 | ```scala 20 | val cnt = "{\"firstName \": \"John\" , \"age\": 25}" 21 | jsonParser.value (cnt) match { 22 | case Success ( result ) => 23 | println ("success : " + result ) 24 | case Failure ( error ) => 25 | println (" failure : " + error ) 26 | } 27 | ``` 28 | ### How to use 29 | A snapshot is available on _Sonatype_. To use it with SBT, add the following lines in your build: 30 | ```scala 31 | libraryDependencies += "com.github.begeric" % "fastparsers_2.11" % "0.1-SNAPSHOT" 32 | resolvers += Resolver.sonatypeRepo("snapshots") 33 | ``` 34 | 35 | ### More information 36 | For a more comprehensive presentation please take a look at the [presentation slides](https://github.com/begeric/FastParsers/blob/experiment/Acceleration%20Parser%20Combinators%20with%20Macros%20-%20Uppsala%202014.pdf). 37 | 38 | To have a better understanding of the internals please read the [paper describing the implementation](http://infoscience.epfl.ch/record/200905?ln=en). 39 | -------------------------------------------------------------------------------- /FastParsers/src/test/scala/ParsersMixinBenchmark.scala: -------------------------------------------------------------------------------- 1 | 2 | import JsonParsers._ 3 | import org.scalameter.api._ 4 | import org.scalameter.PerformanceTest 5 | import org.scalameter.reporting.LoggingReporter 6 | 7 | /** 8 | * Created by Eric on 12.05.14. 9 | */ 10 | object ParsersMixinBenchmark extends PerformanceTest { 11 | 12 | lazy val executor = LocalExecutor( 13 | new Executor.Warmer.Default, 14 | Aggregator.min, 15 | new Measurer.Default) 16 | lazy val reporter = new LoggingReporter 17 | lazy val persistor = Persistor.None 18 | 19 | val range = Gen.enumeration("size")(10) 20 | 21 | 22 | val bigFileName = "FastParsers/src/test/resources/json.big1" 23 | val bigFile = scala.io.Source.fromFile(bigFileName).getLines mkString "\n" 24 | val bigFileArray = bigFile.toCharArray 25 | 26 | /*performance of "JsonParser:@FastParsers" in { 27 | measure method "value" in { 28 | using(range) in { j => 29 | for (i <- 1 to j) 30 | JSonImpl1.jsonparser.value(bigFile) 31 | } 32 | } 33 | } 34 | 35 | performance of "JsonParser:@FastParsersCharArray" in { 36 | measure method "value" in { 37 | using(range) in { j => 38 | for (i <- 1 to j) 39 | JSonImpl2.jsonparser.value(bigFileArray) 40 | } 41 | } 42 | } 43 | 44 | performance of "JsonParser:@FastParsersCharArrayNoInline" in { 45 | measure method "value" in { 46 | using(range) in { j => 47 | for (i <- 1 to j) 48 | JSonImpl3.jsonparser.value(bigFileArray) 49 | } 50 | } 51 | } 52 | 53 | performance of "JsonParser:@FastParsersCharArrayDefaultErrors" in { 54 | measure method "value" in { 55 | using(range) in { j => 56 | for (i <- 1 to j) 57 | JSonImpl4.jsonparser.value(bigFileArray) 58 | } 59 | } 60 | } 61 | 62 | performance of "JsonParser:@FastParsersCharArrayIgnoreResults" in { 63 | measure method "value" in { 64 | using(range) in { j => 65 | for (i <- 1 to j) 66 | JSonImpl5.jsonparser.value(bigFileArray) 67 | } 68 | } 69 | }*/ 70 | 71 | } 72 | -------------------------------------------------------------------------------- /FastParsers/src/test/scala/CSVParsers.scala: -------------------------------------------------------------------------------- 1 | import fastparsers.input.InputWindow._ 2 | import scala.util.parsing.combinator._ 3 | import scala.util.parsing.input._ 4 | import scala.annotation.compileTimeOnly 5 | 6 | /** 7 | * Created by Eric on 05.04.14. 8 | */ 9 | object CSVParsers { 10 | 11 | sealed abstract class JSValue 12 | case class JSArray(arr: List[JSValue]) extends JSValue 13 | case class JSDouble(d: Double) extends JSValue 14 | case class JSDouble2(d: InputWindow[Array[Char]]) extends JSValue 15 | case class JSString(d: InputWindow[Array[Char]]) extends JSValue 16 | case class JSString2(d: String) extends JSValue 17 | case class JSBool(b: Boolean) extends JSValue 18 | object JTrue extends JSValue 19 | object JFalse extends JSValue 20 | 21 | 22 | val trueValue = "true".toCharArray 23 | val falseValue = "false".toCharArray 24 | val comma = ",".toCharArray 25 | val close = "]".toCharArray 26 | 27 | /*val x = new { 28 | @compileTimeOnly("dsad") 29 | def y = 2 30 | }*/ 31 | 32 | 33 | object CSVImpl1 { 34 | import fastparsers.framework.implementations.FastParsersCharArray._ 35 | import fastparsers.parsers.Parser 36 | val cvsParser = FastParsersCharArray { 37 | def cvs(p: Parser[JSValue]) = '[' ~> repsep(p, comma) <~ close ^^ JSArray 38 | def doubles = cvs(decimalNumber ^^ (x => JSDouble(x.toString.toDouble))) 39 | def bools = cvs((lit(trueValue) ~> success(JTrue)) | (lit(falseValue) ~> success(JFalse))) 40 | //def strings = cvs(stringLit ^^ (x => JSString2(x.toString))) 41 | def strings = cvs(stringLit ^^ JSString) 42 | } 43 | } 44 | 45 | object CSVImpl2 { 46 | import fastparsers.framework.implementations.FastParsersCharArray._ 47 | val cvsParserOpt = (FastParsersCharArray { 48 | def primBools = ('t' ~ 'r' ~ 'u' ~ 'e' ~> success(JTrue)) | ('f' ~ 'a' ~ 'l' ~'s' ~ 'e' ~> success(JFalse)) 49 | def bools = '[' ~> repsep(primBools, ',') <~ close// ^^ (x => JSArray(x)) 50 | }) 51 | } 52 | 53 | object CSV extends JavaTokenParsers { 54 | def cvs(p: Parser[JSValue]) = "[" ~> repsep(p, ",") <~ "]" ^^ (x => JSArray(x)) 55 | def doubles = cvs(floatingPointNumber ^^ (y => JSDouble(y.toDouble))) 56 | def bools = cvs(("true" ~> success(JTrue)) | ("false" ~> success(JFalse))) 57 | def strings = cvs(stringLiteral ^^ (y => JSString2(y))) 58 | } 59 | 60 | } -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/input/InputWindow.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.input 2 | 3 | import scala.language.implicitConversions 4 | 5 | object InputWindow { 6 | 7 | /** 8 | * Class used to represent a result by taking a reference to the original input and the position of its 9 | * subsequence in it 10 | */ 11 | class InputWindow[Input](val in: Input,val start: Int,val end: Int){ 12 | override def equals(x: Any) = x match { 13 | case s: InputWindow[Input] => s.in == in && s.start == start && s.end == end 14 | case _ => super.equals(x) 15 | } 16 | } 17 | 18 | class StringStruct(in: String,start: Int, end: Int) extends InputWindow[String](in, start, end){ 19 | 20 | private lazy val realValue = in.slice(start, end) 21 | 22 | def size = end - start 23 | def apply(n: Int) = in.charAt(n - start) 24 | 25 | override def toString = realValue 26 | override def equals(x: Any) = x match { 27 | case s: String => realValue == s 28 | case _ => super.equals(x) 29 | } 30 | } 31 | 32 | class CharArrayStruct(in: Array[Char],start: Int, end: Int) extends InputWindow[Array[Char]](in, start, end){ 33 | 34 | lazy val size = end - start 35 | 36 | private lazy val realValue = in.slice(start, end) 37 | private lazy val realString = { 38 | new String(in, start, end - start) 39 | /*val lb = new StringBuilder(size) 40 | var i = 0 41 | while (i < size){ 42 | lb.append(in(i + start)) 43 | i += 1 44 | } 45 | lb.result*/ 46 | } 47 | 48 | def apply(n: Int) = in(n + start) 49 | 50 | override def toString = realString 51 | 52 | override def equals(x: Any) = x match { 53 | case s: Array[Char] if s.length == end - start => 54 | var i = 0 55 | var cont = true 56 | val sLength = end - start 57 | while (cont && i < sLength){ 58 | cont = s(i) == in(start + i) 59 | i += 1 60 | } 61 | cont 62 | case s: String if s.length == end - start => 63 | var i = 0 64 | var cont = true 65 | val sLength = end - start 66 | while (cont && i < sLength){ 67 | cont = s(i) == in(start + i) 68 | i += 1 69 | } 70 | cont 71 | case _ => super.equals(x) 72 | } 73 | } 74 | 75 | //WHY DOESNT IT WORK ????? 76 | implicit def CharArrayStruct2String(in: CharArrayStruct) = in.toString 77 | implicit def StringStruct2String(in: StringStruct) = in.toString 78 | } -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/ruleprocessing/RulesInliner.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.ruleprocessing 2 | 3 | import fastparsers.tools.TreeTools 4 | import fastparsers.input.ParseInput 5 | import scala.collection.mutable.HashMap 6 | 7 | /** 8 | * Created by Eric on 23.04.14. 9 | * Inline rule calls. 10 | * 11 | * def rule1 = a ~ rule2 12 | * def rule2 = b ~ c 13 | * 14 | * becomes 15 | * 16 | * def rule1 = a ~ b ~ c 17 | * def rule2 = b ~ c 18 | * 19 | * If it cannot be inlined (recursive rules) then the rule will be simply called 20 | */ 21 | trait RulesInliner extends RulesTransformer { 22 | self: TreeTools with ParseInput => 23 | 24 | import c.universe._ 25 | 26 | override def transformRuleCalls(tree: c.Tree, 27 | enclosingRule: RuleInfo, 28 | rulesMap: HashMap[String, RuleInfo], 29 | expandedRules: HashMap[String, RuleInfo], 30 | rulesPath: List[String]): c.Tree = { 31 | 32 | def inlineExpand(ruleName: TermName, typeArgs: List[c.Type], args: List[c.Tree]): Option[c.Tree] = 33 | if (!rulesPath.contains(ruleName.toString)) { 34 | getValidRuleInfo(ruleName,rulesMap, typeArgs, args).collect[c.Tree] { 35 | case RuleInfo(typ, code, params, typeParams,_) => 36 | val substituted = subsituteParams(params.map(_.symbol), args, code) 37 | val newRulesPath = ruleName.toString :: rulesPath 38 | val transformedCode = transformRuleCalls(substituted, enclosingRule, rulesMap, expandedRules, newRulesPath) 39 | val substitutedType = typ.substituteTypes(typeParams.map(_.symbol), typeArgs) 40 | q"compound[${substitutedType}](${transformedCode})" 41 | } 42 | } 43 | else None 44 | 45 | def callParent = super.transformRuleCalls(tree, enclosingRule, rulesMap, expandedRules, rulesPath) 46 | 47 | tree match { 48 | //TODO correct bug with inlining when rule like that parens('a' ~ 'b') 49 | /*case q"${ruleName: TermName}[..$t](..$args)" => 50 | inlineExpand(ruleName, t.map(_.tpe), args) getOrElse callParent*/ 51 | case q"${ruleName: TermName}(..$args)" => 52 | inlineExpand(ruleName, Nil, args) getOrElse callParent 53 | case q"${ruleName: TermName}[..$t]" => 54 | inlineExpand(ruleName, t.map(_.tpe), Nil) getOrElse callParent 55 | case q"${ruleName: TermName}" => 56 | inlineExpand(ruleName, Nil, Nil) getOrElse callParent 57 | case _ => callParent 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/tools/ToPosition.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.tools 2 | 3 | /** 4 | * Created by Eric on 10.04.14. 5 | */ 6 | 7 | import java.util 8 | import scala.collection.mutable.ArrayBuffer 9 | import scala.util.parsing.input._ 10 | 11 | /** 12 | * Create a position object for an input 13 | */ 14 | object ToPosition { 15 | 16 | trait ToPosition[T] { 17 | def get(offset: Int): Position 18 | } 19 | 20 | type IndexedCharSeq[T] = { 21 | def apply(n: Int):Char 22 | def size: Int 23 | def slice(start: Int, end: Int): T 24 | } 25 | 26 | /** 27 | * TODOD change... 28 | */ 29 | 30 | implicit class StringProxy(s: String) { 31 | def apply(n: Int):Char = s.charAt(n) 32 | def size: Int = s.length 33 | def slice(start: Int, end: Int): String = s.substring(start,end) 34 | } 35 | 36 | implicit class CharArrayProxy(s: Array[Char]) { 37 | def apply(n: Int):Char = s(n) 38 | def size: Int = s.length 39 | def slice(start: Int, end: Int): Array[Char] = util.Arrays.copyOfRange(s, start, start + end); 40 | } 41 | 42 | /* 43 | Almost copy pasted from scala.util.parsing.fastparsers.input.OffsetPosition @ https://github.com/scala/scala/blob/v2.10.2/src/library/scala/util/parsing/fastparsers.input/OffsetPosition.scala 44 | */ 45 | class IndexedCharSeqToPosition[T](input: IndexedCharSeq[T]) extends ToPosition[IndexedCharSeq[T]] { 46 | 47 | val index = new ArrayBuffer[Int]() 48 | index += 0 49 | 50 | def computeIndexTill(pos: Int) { 51 | 52 | def compute(from: Int, to: Int) = { 53 | for (i <- from until to) 54 | if (input(i) == '\n') 55 | index += (i + 1) 56 | 57 | if (pos >= input.size) 58 | index += input.size 59 | } 60 | 61 | if (pos > index.last && index.last < input.size) 62 | compute(index.last + 1,Math.min(pos,input.size)) 63 | } 64 | 65 | def lineFromOffset(pos: Int) = { 66 | computeIndexTill(pos) 67 | var lo = 0 68 | var hi = index.length - 1 69 | while (lo + 1 < hi) { 70 | val mid = (hi + lo) / 2 71 | if (pos < index(mid)) hi = mid 72 | else lo = mid 73 | } 74 | lo + 1 75 | } 76 | 77 | 78 | def get(offset: Int) = { 79 | val l = lineFromOffset(offset) 80 | new Position { 81 | override protected def lineContents = input.slice(index(line - 1), index(line)).toString //TODO change that 82 | override def line = l 83 | override def column = offset - index(lineFromOffset(l) - 1) + 1 84 | } 85 | } 86 | } 87 | 88 | } -------------------------------------------------------------------------------- /FastParsers/src/test/scala/HttpParserBenchmark.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by Eric on 05.04.14. 3 | */ 4 | import org.scalameter.api._ 5 | import HttpParsers._ 6 | import scala.collection.mutable.ListBuffer 7 | import lms._ 8 | 9 | object HttpParserBenchmark extends PerformanceTest { 10 | 11 | /* configuration */ 12 | 13 | lazy val executor = LocalExecutor( 14 | new Executor.Warmer.Default, 15 | Aggregator.min, 16 | new Measurer.Default) 17 | lazy val reporter = new LoggingReporter 18 | lazy val persistor = Persistor.None 19 | 20 | /* inputs */ 21 | 22 | val range = Gen.enumeration("size")(10) 23 | 24 | val files = (1 to 100).foldLeft(new ListBuffer[(Array[Char], String, FastCharSequence)]){ (acc,i) => 25 | val fileName = "FastParsers/src/test/resources/tweet" + i + "ho" 26 | val data = scala.io.Source.fromFile(fileName).getLines mkString "\n" 27 | acc += ((data.toCharArray,data, new FastCharSequence(data.toCharArray))) 28 | acc 29 | }.toList 30 | 31 | //println(files.mkString.size) 32 | 33 | /* tests */ 34 | /*performance of "HttpParser response and message" in { 35 | measure method "FastParsers" in { 36 | using(range) in { j => 37 | for (i <- 1 to j; m <- files) 38 | httpparser.respAndMessage(m._1) 39 | } 40 | } 41 | 42 | measure method "Combinators" in { 43 | using(range) in { j => 44 | for (i <- 1 to j; m <- files) 45 | HTTP.parse(HTTP.respAndMessage, m._2) 46 | } 47 | } 48 | 49 | }*/ 50 | 51 | /*performance of "HttpParser response only" in { 52 | measure method "FastParsers" in { 53 | using(range) in { j => 54 | for (i <- 1 to j; m <- files) 55 | HTTPImpl3.httpparser.response(m._1) 56 | } 57 | } 58 | 59 | measure method "Combinators" in { 60 | using(range) in { j => 61 | for (i <- 1 to j; m <- files) 62 | HTTP.parse(HTTP.response, m._3) 63 | } 64 | } 65 | 66 | measure method "LMS" in { 67 | using(range) in { j => 68 | for (i <- 1 to j; m <- files) 69 | HttpResponseParser.apply(m._1) 70 | } 71 | } 72 | 73 | 74 | }*/ 75 | 76 | /*performance of "HttpParser@LMS" in { 77 | measure method "response" in { 78 | using(range) in { j => 79 | for (i <- 1 to j; m <- files) 80 | HttpResponseParser.apply(m._1) 81 | } 82 | } 83 | measure method "response" in { 84 | using(range) in { j => 85 | for (i <- 1 to j; m <- files) 86 | httpparser.response(m._2) 87 | } 88 | } 89 | }*/ 90 | 91 | 92 | } -------------------------------------------------------------------------------- /Examples/src/main/scala/CSVBoolHandWritten.scala: -------------------------------------------------------------------------------- 1 | object CSVBoolHandWritten { 2 | 3 | def apply(input: Array[Char]): List[Boolean] = { 4 | var pos = 0 5 | val inputsize = input.length 6 | val values = new scala.collection.mutable.ListBuffer[Boolean]() 7 | var result1 = false 8 | var result2 = false 9 | var result3 = false 10 | 11 | var success = false 12 | if (pos < inputsize && input(pos) == '['){ 13 | pos += 1; 14 | success = true 15 | while (success) { 16 | val tmp_pos = pos 17 | if (input(pos) == 't'){ 18 | pos += 1 19 | success = true 20 | } 21 | else 22 | success = false 23 | if (success) { 24 | if (input(pos) == 'r'){ 25 | pos += 1 26 | success = true 27 | } 28 | else 29 | success = false 30 | 31 | if (success) { 32 | if (input(pos) == 'u'){ 33 | pos += 1 34 | success = true 35 | } 36 | else 37 | success = false 38 | 39 | if (success) { 40 | if (input(pos) == 'e'){ 41 | pos += 1 42 | success = true 43 | } 44 | else 45 | success = false 46 | if (success) 47 | result1 = true 48 | } 49 | } 50 | } 51 | if (!success) { 52 | pos = tmp_pos 53 | if (input(pos) == 'f'){ 54 | pos += 1 55 | success = true 56 | } 57 | else 58 | success = false 59 | if (success) { 60 | if (input(pos) == 'a'){ 61 | pos += 1 62 | success = true 63 | } 64 | else 65 | success = false 66 | 67 | if (success) { 68 | if (input(pos) == 'l'){ 69 | pos += 1 70 | success = true 71 | } 72 | else 73 | success = false 74 | 75 | if (success) { 76 | if (input(pos) == 's'){ 77 | pos += 1 78 | success = true 79 | } 80 | else 81 | success = false 82 | if (success) { 83 | if (input(pos) == 'e'){ 84 | pos += 1 85 | success = true 86 | } 87 | else 88 | success = false 89 | if (success) 90 | result2 = false 91 | } 92 | } 93 | } 94 | } 95 | result3 = result2 96 | } 97 | else 98 | result3 = result1 99 | if (success){ 100 | values += (result3) 101 | if (input(pos) == ',') 102 | pos += 1 103 | else 104 | success = false 105 | } 106 | } 107 | if (pos < inputsize && input(pos) == ']') 108 | values.toList 109 | else 110 | Nil 111 | } 112 | else 113 | Nil 114 | } 115 | } -------------------------------------------------------------------------------- /FastParsers/src/test/scala/CSVBoolHandWritten.scala: -------------------------------------------------------------------------------- 1 | object CSVBoolHandWritten { 2 | 3 | def apply(input: Array[Char]): List[Boolean] = { 4 | var pos = 0 5 | val inputsize = input.length 6 | val values = new scala.collection.mutable.ListBuffer[Boolean]() 7 | var result1 = false 8 | var result2 = false 9 | var result3 = false 10 | 11 | var success = false 12 | if (pos < inputsize && input(pos) == '['){ 13 | pos += 1; 14 | success = true 15 | while (success) { 16 | val tmp_pos = pos 17 | if (input(pos) == 't'){ 18 | pos += 1 19 | success = true 20 | } 21 | else 22 | success = false 23 | if (success) { 24 | if (input(pos) == 'r'){ 25 | pos += 1 26 | success = true 27 | } 28 | else 29 | success = false 30 | 31 | if (success) { 32 | if (input(pos) == 'u'){ 33 | pos += 1 34 | success = true 35 | } 36 | else 37 | success = false 38 | 39 | if (success) { 40 | if (input(pos) == 'e'){ 41 | pos += 1 42 | success = true 43 | } 44 | else 45 | success = false 46 | if (success) 47 | result1 = true 48 | } 49 | } 50 | } 51 | if (!success) { 52 | pos = tmp_pos 53 | if (input(pos) == 'f'){ 54 | pos += 1 55 | success = true 56 | } 57 | else 58 | success = false 59 | if (success) { 60 | if (input(pos) == 'a'){ 61 | pos += 1 62 | success = true 63 | } 64 | else 65 | success = false 66 | 67 | if (success) { 68 | if (input(pos) == 'l'){ 69 | pos += 1 70 | success = true 71 | } 72 | else 73 | success = false 74 | 75 | if (success) { 76 | if (input(pos) == 's'){ 77 | pos += 1 78 | success = true 79 | } 80 | else 81 | success = false 82 | if (success) { 83 | if (input(pos) == 'e'){ 84 | pos += 1 85 | success = true 86 | } 87 | else 88 | success = false 89 | if (success) 90 | result2 = false 91 | } 92 | } 93 | } 94 | } 95 | result3 = result2 96 | } 97 | else 98 | result3 = result1 99 | if (success){ 100 | values += (result3) 101 | if (input(pos) == ',') 102 | pos += 1 103 | else 104 | success = false 105 | } 106 | } 107 | if (pos < inputsize && input(pos) == ']') 108 | values.toList 109 | else 110 | Nil 111 | } 112 | else 113 | Nil 114 | } 115 | } -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/implementations/BaseImpl.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.implementations 2 | 3 | import scala.language.experimental.macros 4 | import fastparsers.tools.TreeTools 5 | import fastparsers.input.ParseInput 6 | import scala.reflect.macros.whitebox.Context 7 | import scala.collection.mutable.HashMap 8 | import fastparsers.framework.ruleprocessing.{RuleCombiner, MapRules} 9 | import fastparsers.parsers.Parser 10 | 11 | /** 12 | * General trait which create the basic needs of a FastParsers implementation. 13 | * 14 | * All it does is create the map of rulenames with their code (whitout modification). 15 | * It must be composed with some Rule transformer which will expand the rules wich will 16 | * be combined to form the final object by the fastparsers.framework.ruleprocessing.RuleCombiner. 17 | * It must also be composed with a fastparsers.input.ParseInput to allow access on the fastparsers.input. 18 | */ 19 | trait BaseImpl extends TreeTools { 20 | self: MapRules with RuleCombiner with ParseInput => 21 | val c: Context 22 | 23 | import c.universe._ 24 | 25 | def FastParser(rules: c.Tree): c.Tree = { 26 | val map = getBasicStructure(rules) 27 | val transformedMap = process(map) 28 | combine(transformedMap) 29 | } 30 | 31 | /** 32 | * Expand each rule in a imperative style without considering other rules (i.e def rule2 = rule1 is not expanded to the code of rule1) 33 | * @return An HashMap containing (rulename, corresponging code) 34 | */ 35 | private def getBasicStructure(rules: c.Tree) = { 36 | 37 | def getReturnType(ruleCode: c.Tree): Type = c.typecheck(ruleCode).tpe match { 38 | //case TypeRef(_, y, List(z)) if y.typeSignature =:= typeOf[Parser[_]] => z 39 | case TypeRef(_, y, List(z)) if y.fullName == "fastparsers.parsers.Parser" => z //q"Any".tpe//q"var x:${d.tpe}" //check it is a code 40 | case v => c.abort(c.enclosingPosition, "incorrect parser type " + show(v)) 41 | } 42 | 43 | 44 | val rulesMap = new HashMap[String, RuleInfo]() 45 | c.typecheck(rules) match { 46 | case q"{..$body}" => 47 | body.foreach { 48 | case q"def $name[..$t](..$params): $d = $b" => 49 | rulesMap += name.toString -> RuleInfo(getReturnType(d), b, params,t, b)///not supported yet 50 | case q"def $name(..$params): $d = $b" => 51 | rulesMap += name.toString -> RuleInfo(getReturnType(d), b, params, Nil, b) 52 | case q"def $name: $d = $b" => 53 | rulesMap += name.toString -> RuleInfo(getReturnType(d), b, Nil, Nil, b) 54 | case q"()" => 55 | case x => c.abort(c.enclosingPosition, "body must only contain rule definition with the following form : def ruleName = body : " + x) 56 | } 57 | case _ => 58 | c.abort(c.enclosingPosition, "ill-formed body, cannot be empty") //TODO can be empty ? 59 | } 60 | rulesMap 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Examples/src/main/scala/DataStructures.scala: -------------------------------------------------------------------------------- 1 | 2 | case class Anon6507737(kind: Int, data: Any) 3 | 4 | case class Tuple2StringAnon6507737(_1: java.lang.String, _2: Anon6507737) 5 | 6 | case class OptionChar(value: Char, defined: Boolean) 7 | 8 | 9 | case class ParseResultAnon6507737(res: Anon6507737, empty: Boolean, next: Int) 10 | 11 | case class Tuple2Anon6507737ListAnon6507737(_1: Anon6507737, _2: scala.collection.immutable.List[Anon6507737]) 12 | 13 | case class Tuple2OptionCharString(_1: OptionChar, _2: java.lang.String) 14 | 15 | case class ParseResultListAnon6507737(res: scala.collection.immutable.List[Anon6507737], empty: Boolean, next: Int) 16 | 17 | case class Tuple2OptionCharInt(_1: OptionChar, _2: Int) 18 | 19 | case class Tuple2IntInt(_1: Int, _2: Int) 20 | 21 | case class Anon1680061013(input: Array[Char], start: Int, length: Int) 22 | 23 | case class StringReader(input: Array[Char], offset: Int) 24 | 25 | case class OptionAnon1323431030(value: Anon1323431030, defined: Boolean) 26 | 27 | case class Tuple2IntAnon1680061013(_1: Int, _2: Anon1680061013) 28 | 29 | case class Tuple2Anon1680061013Anon1680061013(_1: Anon1680061013, _2: Anon1680061013) 30 | 31 | case class OptionAnon6507737(value: Anon6507737, defined: Boolean) 32 | 33 | case class ParseResultAnon6507737B(res: Anon6507737, empty: Boolean, next: StringReader) 34 | 35 | case class ParseResultCharB(res: Char, empty: Boolean, next: StringReader) 36 | 37 | case class ParseResultListAnon6507737B(res: scala.collection.immutable.List[Anon6507737], empty: Boolean, next: StringReader) 38 | 39 | case class Tuple2StringString(_1: java.lang.String, _2: java.lang.String) 40 | 41 | case class Tuple2CharString(_1: Char, _2: java.lang.String) 42 | 43 | case class Anon1323431030(status: Int, contentLength: Int, connection: java.lang.String, chunked: Boolean, upgrade: Boolean) 44 | 45 | case class Tuple2Anon1323431030String(_1: Anon1323431030, _2: java.lang.String) 46 | 47 | case class ParseResultInt(res: Int, empty: Boolean, next: Int) 48 | 49 | case class ParseResultTuple2Anon1323431030String(res: Tuple2Anon1323431030String, empty: Boolean, next: Int) 50 | 51 | case class ParseResultAnon1323431030(res: Anon1323431030, empty: Boolean, next: Int) 52 | 53 | case class Tuple2IntAnon1323431030(_1: Int, _2: Anon1323431030) 54 | 55 | case class ParseResultString(res: java.lang.String, empty: Boolean, next: Int) 56 | 57 | case class ParseResultTuple2StringString(res: Tuple2StringString, empty: Boolean, next: Int) 58 | 59 | case class ParseResultChar(res: Char, empty: Boolean, next: Int) 60 | 61 | case class ParseResultListDouble(res: scala.collection.immutable.List[Double], empty: Boolean, next: Int) 62 | 63 | case class Tuple2DoubleListDouble(_1: Double, _2: scala.collection.immutable.List[Double]) 64 | 65 | case class ParseResultTuple2DoubleListDouble(res: Tuple2DoubleListDouble, empty: Boolean, next: Int) 66 | 67 | case class ParseResultTuple2Anon6507737ListAnon6507737(res: Tuple2Anon6507737ListAnon6507737, empty: Boolean, next: Int) -------------------------------------------------------------------------------- /Examples/src/main/scala/Calculator2.scala: -------------------------------------------------------------------------------- 1 | import fastparsers.framework.implementations.FastParsers 2 | import fastparsers.framework.parseresult._ 3 | import fastparsers.parsers.Parser 4 | import scala.collection.immutable.HashMap 5 | 6 | /** 7 | * Created by Eric on 07.04.14. 8 | */ 9 | object Calculator2 { 10 | 11 | def main(args: Array[String]) { 12 | import FastParsers._ 13 | 14 | var values = new HashMap[String,Int]() 15 | 16 | var numRes = 0 17 | def getNextVar = {numRes += 1;"res" + numRes} 18 | 19 | def getVar(v:String): Int = values.get(v) match { 20 | case None => 0 21 | case Some(x) => x 22 | } 23 | 24 | def setVar(name:String,value:Int) = { 25 | values += name -> value 26 | (name,value) 27 | } 28 | 29 | def setRes(value:Int) = setVar(getNextVar,value) 30 | 31 | def exec(x:(Int,Option[(Char,Int)])) = x match { 32 | case (y,None) => y 33 | case (y1,Some((op,y2))) => op match { 34 | case '+' => y1 + y2 35 | case '-' => y1 - y2 36 | case '*' => y1 * y2 37 | case '/' => y1 / y2 38 | } 39 | } 40 | 41 | def func(name:String,args:List[Int]):Int = (name,args) match { 42 | case ("abs",x::Nil) => Math.abs(x) 43 | case ("min",x::y::Nil) => Math.min(x,y) 44 | case ("max",x::y::Nil) => Math.max(x,y) 45 | case ("rnd",Nil) => (Math.random() * 100).toInt 46 | case ("rnd",x::y::Nil) => (Math.random() * (y - x)).toInt + x 47 | case ("pow",x::y::Nil) => Math.pow(x,y).toInt 48 | case ("sqrt",x::Nil) => Math.sqrt(x).toInt 49 | case _ => 50 | println("incorrect function " + name + " with args " + args.mkString) 51 | 0 52 | } 53 | 54 | def alpha(x:Char) = x >= 'a' && x <= 'z' || x >= 'A' && x <= 'Z' 55 | def alphanum(x:Char) = alpha(x) || x >= '0' && x <= '9' 56 | 57 | val parser = FastParser{ 58 | def op1 = lit("+") ^^^ '+' | lit("-") ^^^ '-' 59 | def op2 = lit("*") ^^^ '*' | lit("/") ^^^ '/' 60 | def funccall = ident ~ (lit("(") ~> repsep(expr,",") <~ ")") ^^ (x => func(x._1.toString,x._2)) 61 | def factor:Parser[Int] = number ^^(_.toString.toInt) | lit("(") ~> expr <~ ")" | funccall | (ident ^^ (x => getVar(x.toString))) 62 | def term:Parser[Int] = factor ~ opt(op2 ~ term) ^^ exec 63 | def expr:Parser[Int] = term ~ opt(op1 ~ expr) ^^ exec 64 | def assign = ident ~ (lit("=") ~ whitespaces ~> expr) 65 | def start = phrase(assign ^^ (x => setVar(x._1.toString,x._2)) | expr ^^ setRes) //the order is important !! (because of opt(op1 ~ expr)) 66 | } 67 | 68 | println("Please enter an expression (or enter an empty line to quit)") 69 | 70 | var cont = true 71 | while (cont) { 72 | scala.io.StdIn.readLine() match { 73 | case "" => cont = false 74 | case line => 75 | parser.start(line) match { 76 | case Success((res,value)) => 77 | println(res + " = " + value) 78 | case Failure(msg) => println("failure: " + msg) 79 | } 80 | } 81 | } 82 | } 83 | } -------------------------------------------------------------------------------- /FastParsers/src/test/scala/TestsHelper.scala: -------------------------------------------------------------------------------- 1 | import fastparsers.framework.implementations.FastParsers 2 | import fastparsers.framework.parseresult.{ParseResult, Success, Failure} 3 | import fastparsers.framework.parseresult.Success 4 | import org.scalatest.{FunSuite, FunSpec} 5 | import scala.util.parsing.combinator.Parsers 6 | 7 | /** 8 | * Created by Eric on 15.03.14. 9 | */ 10 | object TestsHelper extends FunSuite { 11 | import org.scalatest._ 12 | import FastParsers._ 13 | 14 | 15 | implicit def strToInput(s:String) = new Input(s) 16 | 17 | class Input(in:String){ 18 | def gives(result:Any) = InputAndResult(in,result) 19 | } 20 | 21 | case class InputAndResult(in:String, res:Any) 22 | 23 | def shouldSucced[T](rule:(String,Int) => ParseResult[Any,_])(tests:InputAndResult*) = { 24 | tests.foreach{ 25 | x => rule(x.in,0) match { 26 | case Success(result) => assert(result == x.res, "on " + x.in + " expected " + x.res + " got " + result) 27 | case Failure(msg) => fail("Didn't succeed : " + msg + " on \"" + x.in+"\"") 28 | } 29 | } 30 | } 31 | 32 | def shouldFail(rule:(String,Int) => ParseResult[Any,_])(tests:String*) = { 33 | tests.foreach{ 34 | str => rule(str,0) match { 35 | case Success(result) => fail("Wasn't supposed to succeed, result : " + result) 36 | case _ => 37 | } 38 | } 39 | } 40 | 41 | def compare(s1:String, s2:String) = { 42 | s1.zip(s2).zipWithIndex.foreach{case ((a1,a2),i) => 43 | if (a1 != a2) 44 | fail("error at " + i + " : " + s1.substring(i,i + 10) + " != " + s2.substring(i,i + 10)) 45 | } 46 | } 47 | 48 | def repeat[T](x:T,n:Int):List[T] = n match { 49 | case a if a <= 0 => Nil 50 | case _ => x::repeat(x,n - 1) 51 | } 52 | 53 | 54 | 55 | def compareImplementations(fileName: String, fast:(String,Int) => Any,combHelper:Parsers, comb:CharSequence => Any) { 56 | val file = scala.io.Source.fromFile(fileName).getLines mkString "\n" 57 | val fileArray = file.toCharArray 58 | val charSeq = new FastCharSequence(fileArray) 59 | 60 | def getFastParserResult = fast(file,0) match { 61 | case Success(result) => result 62 | case Failure(msg) => fail("error : " + msg) 63 | } 64 | 65 | def getCombinatorResult = comb(charSeq) match { 66 | case combHelper.Success(result,_) => result 67 | case combHelper.Failure(msg,_) => fail("error : " + msg) 68 | } 69 | 70 | val now = System.nanoTime 71 | val res1 = getFastParserResult 72 | val micros = (System.nanoTime - now) /1e6 73 | val now2 = System.nanoTime 74 | val res2 = getCombinatorResult 75 | val micros2 = (System.nanoTime - now2) /1e6 76 | //assert(res1 == res2) 77 | //println(res1 + " : " + res2) 78 | //assert(micros < micros2) 79 | //println(fileName.split('\\').last + " : FastParserResult @ " + micros + " : " + "CombinatorResult @ " + micros2) 80 | println(fileName.split('/').last + " (" + (file.length/1024) + "kb)" + " FastParsers is " + ((micros2 / micros)*100).toInt/100.0 + " times faster than Combinator") 81 | } 82 | 83 | } -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/framework/ruleprocessing/ParseRules.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.framework.ruleprocessing 2 | 3 | import fastparsers.error._ 4 | import fastparsers.input.ParseInput 5 | import fastparsers.parsers.{Parser, ParserImplBase} 6 | import fastparsers.tools.TreeTools 7 | import scala.collection.mutable.{ListBuffer, HashMap} 8 | 9 | /** 10 | * Create the "final" code for each rule 11 | */ 12 | trait ParseRules extends MapRules { 13 | self: ParseInput with ParserImplBase with ParseError with TreeTools => 14 | 15 | import c.universe._ 16 | import c.universe.internal._ 17 | //import c.internal.decorators._ 18 | 19 | override def process(rules: HashMap[String, RuleInfo]) = { 20 | val rulesMap = super.process(rules) 21 | 22 | val map = new HashMap[String, RuleInfo]() 23 | for (k <- rulesMap.keys) { 24 | val rule = rulesMap(k) 25 | map += ((k, rule.copy(code = createRuleDef(k, rule)))) 26 | } 27 | map 28 | } 29 | 30 | def convertParsersParams(params: List[c.Tree]) = params.map{ 31 | case valdef @ ValDef(m,n,t,v) => getInnerTypeOf[Parser[_]](t.tpe) match { 32 | case Some(List(innerType)) => 33 | ValDef(m,n,tq"($inputType, Int) => fastparsers.framework.parseresult.ParseResult[$innerType, $errorType]",v) 34 | case None => valdef 35 | } 36 | } 37 | 38 | def removeCompileTimeAnnotation(tree: c.Tree): c.Tree = new Transformer { 39 | override def transform(tree: c.Tree): c.Tree = tree match { 40 | case orig @ Select(qual, name) if orig.symbol.annotations.exists(_.tree.tpe =:= typeOf[scala.annotation.compileTimeOnly]) => 41 | val newStuff = setType(setSymbol(Select(qual, name),NoSymbol), orig.tpe) 42 | super.transform(newStuff) 43 | case _ => super.transform(tree) 44 | } 45 | }.transform(c.typecheck(callToString(tree))) 46 | 47 | private def createRuleDef(name: String, rule: RuleInfo): c.Tree = { 48 | val ruleName = TermName(name) 49 | val startPosition = TermName(c.freshName) 50 | val rs = new ResultsStruct(new ListBuffer[Result]()) 51 | val ruleCode = expand(rule.code, rs) 52 | val initResults = rs.results.map(x => q"var ${x._1}:${x._2} = ${zeroValue(x._2)}") 53 | val tupledResults = rs.combine 54 | 55 | 56 | val result = q"""fastparsers.framework.parseresult.ParseResult($success,error,if ($success) $tupledResults else ${zeroValue(tq"${rule.typ}")},$pos)""" 57 | 58 | val wrapCode = 59 | q""" 60 | var $success = false 61 | ..$initResults 62 | $ruleCode 63 | $result 64 | """ 65 | 66 | val code = initError(initInput(q"$startPosition", wrapCode)) 67 | 68 | val rewriteParams = convertParsersParams(rule.params) 69 | 70 | val replacedTree = removeCompileTimeAnnotation(rule.code)// @saveAST(${replacedTree}) 71 | 72 | val allParams = q"$inputValue: $inputType" :: (rewriteParams :+ q"val $startPosition: Int = 0") 73 | 74 | val rulecode = q"""def $ruleName[..${rule.typeParams}](..$allParams): 75 | fastparsers.framework.parseresult.ParseResult[${rule.typ}, $errorType] @fastparsers.framework.saveAST(${replacedTree}) = 76 | ${c.untypecheck(code)}""" 77 | rulecode 78 | } 79 | 80 | } -------------------------------------------------------------------------------- /project/PublishSettings.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | 4 | object PublishSettings { 5 | def publishSettings = Seq( 6 | organization := "com.github.begeric", 7 | publishMavenStyle := true, 8 | publishOnlyWhenOnMaster := publishOnlyWhenOnMasterImpl.value, 9 | publishTo <<= version { v: String => 10 | val nexus = "https://oss.sonatype.org/" 11 | if (v.trim.endsWith("SNAPSHOT")) 12 | Some("snapshots" at nexus + "content/repositories/snapshots") 13 | else 14 | Some("releases" at nexus + "service/local/staging/deploy/maven2") 15 | }, 16 | pomIncludeRepository := { x => false }, 17 | publishArtifact in Compile := false, 18 | publishArtifact in Test := false, 19 | pomExtra := ( 20 | https://github.com/begeric/FastParsers 21 | 2014 22 | 23 | 24 | MIT 25 | https://github.com/begeric/FastParsers/blob/experiment/LICENSE.txt 26 | repo 27 | 28 | 29 | 30 | git:github.com/begeric/FastParsers.git 31 | scm:git:git://github.com/begueric/FastParsers.git 32 | 33 | 34 | GitHub 35 | https://github.com/begueric/FastParsers/issues 36 | 37 | ), 38 | publishArtifact in (Compile, packageDoc) := false 39 | ) 40 | 41 | lazy val publishOnlyWhenOnMaster = taskKey[Unit]("publish task for Travis (don't publish when building pull requests, only publish when the build is triggered by merge into master)") 42 | def publishOnlyWhenOnMasterImpl = Def.taskDyn { 43 | import scala.util.Try 44 | val travis = Try(sys.env("TRAVIS")).getOrElse("false") == "true" 45 | val pr = Try(sys.env("TRAVIS_PULL_REQUEST")).getOrElse("false") != "false" 46 | val branch = Try(sys.env("TRAVIS_BRANCH")).getOrElse("??") 47 | val snapshot = version.value.trim.endsWith("SNAPSHOT") 48 | (travis, pr, branch, snapshot) match { 49 | case (true, false, "master", true) => publish 50 | case _ => Def.task () 51 | } 52 | } 53 | 54 | lazy val publishableSettings = Seq( 55 | publishArtifact in Compile := true, 56 | publishArtifact in Test := false, 57 | credentials ++= { 58 | val mavenSettingsFile = System.getenv("MAVEN_SETTINGS_FILE") 59 | if (mavenSettingsFile != null) { 60 | println("Loading Sonatype credentials from " + mavenSettingsFile) 61 | try { 62 | import scala.xml._ 63 | val settings = XML.loadFile(mavenSettingsFile) 64 | def readServerConfig(key: String) = (settings \\ "settings" \\ "servers" \\ "server" \\ key).head.text 65 | Some(Credentials( 66 | "Sonatype Nexus Repository Manager", 67 | "oss.sonatype.org", 68 | readServerConfig("username"), 69 | readServerConfig("password") 70 | )) 71 | } catch { 72 | case ex: Exception => 73 | println("Failed to load Maven settings from " + mavenSettingsFile + ": " + ex) 74 | None 75 | } 76 | } else { 77 | println("Do not load Credentials") 78 | None 79 | } 80 | }.toList 81 | ) 82 | } -------------------------------------------------------------------------------- /FastParsers/src/test/scala/lms/DataStructures.scala: -------------------------------------------------------------------------------- 1 | package lms 2 | 3 | case class Anon6507737(kind: Int, data: Any) 4 | 5 | case class Tuple2StringAnon6507737(_1: java.lang.String, _2: Anon6507737) 6 | 7 | case class OptionChar(value: Char, defined: Boolean) 8 | 9 | 10 | case class ParseResultAnon6507737(res: Anon6507737, empty: Boolean, next: Int) 11 | 12 | case class Tuple2Anon6507737ListAnon6507737(_1: Anon6507737, _2: scala.collection.immutable.List[Anon6507737]) 13 | 14 | case class Tuple2OptionCharString(_1: OptionChar, _2: java.lang.String) 15 | 16 | case class ParseResultListAnon6507737(res: scala.collection.immutable.List[Anon6507737], empty: Boolean, next: Int) 17 | 18 | case class Tuple2OptionCharInt(_1: OptionChar, _2: Int) 19 | 20 | case class Tuple2IntInt(_1: Int, _2: Int) 21 | 22 | case class Anon1680061013(input: Array[Char], start: Int, length: Int) 23 | 24 | case class StringReader(input: Array[Char], offset: Int) 25 | 26 | case class OptionAnon1323431030(value: Anon1323431030, defined: Boolean) 27 | 28 | case class Tuple2IntAnon1680061013(_1: Int, _2: Anon1680061013) 29 | 30 | case class Tuple2Anon1680061013Anon1680061013(_1: Anon1680061013, _2: Anon1680061013) 31 | 32 | case class OptionAnon6507737(value: Anon6507737, defined: Boolean) 33 | 34 | case class ParseResultAnon6507737B(res: Anon6507737, empty: Boolean, next: StringReader) 35 | 36 | case class ParseResultCharB(res: Char, empty: Boolean, next: StringReader) 37 | 38 | case class ParseResultListAnon6507737B(res: scala.collection.immutable.List[Anon6507737], empty: Boolean, next: StringReader) 39 | 40 | case class Tuple2StringString(_1: java.lang.String, _2: java.lang.String) 41 | 42 | case class Tuple2CharString(_1: Char, _2: java.lang.String) 43 | 44 | case class Anon1323431030(status: Int, contentLength: Int, connection: java.lang.String, chunked: Boolean, upgrade: Boolean) 45 | 46 | case class Tuple2Anon1323431030String(_1: Anon1323431030, _2: java.lang.String) 47 | 48 | case class ParseResultInt(res: Int, empty: Boolean, next: Int) 49 | 50 | case class ParseResultTuple2Anon1323431030String(res: Tuple2Anon1323431030String, empty: Boolean, next: Int) 51 | 52 | case class ParseResultAnon1323431030(res: Anon1323431030, empty: Boolean, next: Int) 53 | 54 | case class Tuple2IntAnon1323431030(_1: Int, _2: Anon1323431030) 55 | 56 | case class ParseResultString(res: java.lang.String, empty: Boolean, next: Int) 57 | 58 | case class ParseResultTuple2StringString(res: Tuple2StringString, empty: Boolean, next: Int) 59 | 60 | case class ParseResultChar(res: Char, empty: Boolean, next: Int) 61 | 62 | case class ParseResultListDouble(res: scala.collection.immutable.List[Double], empty: Boolean, next: Int) 63 | 64 | case class Tuple2DoubleListDouble(_1: Double, _2: scala.collection.immutable.List[Double]) 65 | 66 | case class ParseResultTuple2DoubleListDouble(res: Tuple2DoubleListDouble, empty: Boolean, next: Int) 67 | 68 | case class ParseResultTuple2Anon6507737ListAnon6507737(res: Tuple2Anon6507737ListAnon6507737, empty: Boolean, next: Int) 69 | 70 | case class Tuple2Anon1680061013String(_1: Anon1680061013, _2: java.lang.String) 71 | 72 | case class Tuple2OptionCharAnon1680061013(_1: OptionChar, _2: Anon1680061013) 73 | 74 | case class Tuple2Anon1680061013Anon6507737(_1: Anon1680061013, _2: Anon6507737) 75 | 76 | case class ParseResultListAnon1680061013(res: scala.collection.immutable.List[Anon1680061013], empty: Boolean, next: Int) 77 | 78 | case class ParseResultTuple2Anon1680061013ListAnon1680061013(res: Tuple2Anon1680061013ListAnon1680061013, empty: Boolean, next: Int) 79 | 80 | case class Tuple2Anon1680061013ListAnon1680061013(_1: Anon1680061013, _2: scala.collection.immutable.List[Anon1680061013]) 81 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/tools/TreeTools.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.tools 2 | 3 | import scala.reflect.macros.whitebox.Context 4 | import fastparsers.parsers.Parser 5 | 6 | /** 7 | * Created by Eric on 21.04.14. 8 | * Diverse tool to deal with ASTs 9 | */ 10 | trait TreeTools { 11 | val c: Context 12 | import c.universe._ 13 | 14 | /** 15 | * Substitute a symbol by a value in a tree 16 | * @param symbol 17 | * @param value 18 | * @param in 19 | * @return 20 | */ 21 | def substituteSymbol(symbol: Symbol, value: c.Tree => c.Tree, in: c.Tree) = new Transformer { 22 | override def transform(tree: c.Tree): c.Tree = 23 | if (tree.symbol == symbol) value(tree) 24 | else super.transform(tree) 25 | }.transform(in) 26 | 27 | /** 28 | * Substitute a TermName by a value in a tree 29 | * @param name 30 | * @param value 31 | * @param in 32 | * @return 33 | */ 34 | def substituteTermName(name: TermName, value: c.Tree => c.Tree, in: c.Tree) = new Transformer { 35 | override def transform(tree: c.Tree): c.Tree = tree match { 36 | case q"${treeName: TermName}" if treeName == name => value(tree) 37 | case q"$m val ${treeName: TermName}:$t = $v" if treeName == name => tree 38 | case _ => super.transform(tree) 39 | } 40 | }.transform(in) 41 | 42 | def substituteParamCall(name: TermName, value: c.Tree => c.Tree, in: c.Tree) = new Transformer { 43 | override def transform(tree: c.Tree): c.Tree = tree match { 44 | case q"$_.callParam[$t](${param: String})" if param == name.toString => value(tree) 45 | case q"${treeName: TermName}" if treeName == name => value(tree) 46 | case q"$m val ${treeName: TermName}:$t = $v" if treeName == name => tree 47 | case _ => super.transform(tree) 48 | } 49 | }.transform(in) 50 | 51 | def subsituteParams(params: List[Symbol], args: List[c.Tree], in: c.Tree) = { 52 | assert(params.size == args.size) 53 | params.zip(args).foldLeft(in){(acc,c) => substituteSymbol(c._1, _ => c._2,acc)} 54 | } 55 | 56 | def subsituteParams2(params: List[TermName], args: List[c.Tree], in: c.Tree) = { 57 | assert(params.size == args.size) 58 | params.zip(args).foldLeft(in){(acc,c) => substituteParamCall(c._1, _ => c._2,acc)} 59 | } 60 | 61 | 62 | def callToString(tree: c.Tree): c.Tree = new Transformer { 63 | override def transform(tree: c.Tree): c.Tree = tree match { 64 | case q"call[$t](${name: TermName},..$args)" => 65 | q"call[$t](${name.toString},..$args)" 66 | case _ => super.transform(tree) 67 | } 68 | }.transform(tree) 69 | 70 | def callToTermName(tree: c.Tree): c.Tree = new Transformer { 71 | override def transform(tree: c.Tree): c.Tree = tree match { 72 | case q"$x.call[$t](${name: String},..$args)" => q"$x.call[$t](${TermName(name)},..$args)" 73 | case _ => super.transform(tree) 74 | } 75 | }.transform(tree) 76 | 77 | 78 | /** 79 | * Get the template types of another Type. 80 | * getInnerTypeOf[Parser[_]](..Parser[String]..) get you Some(List(String)) 81 | * @param typ The type in which we want to recuperate the generic part 82 | * @tparam T 83 | * @return 84 | */ 85 | def getInnerTypeOf[T : TypeTag](typ: c.Type): Option[List[c.Type]] = typ match { 86 | case TypeRef(_, _, Nil) => None 87 | case TypeRef(_, _, z) if typ <:< typeOf[T] => Some(z) 88 | case _ => None 89 | } 90 | 91 | /** 92 | * 93 | * @param params 94 | * @return 95 | */ 96 | def getParserParams(params: List[c.Tree]) = 97 | params.map{case ValDef(_,name,tpt,_) => (name,tpt.tpe)} 98 | .filter(_._2 <:< typeOf[Parser[_]]) 99 | .map(x => (x._1,getInnerTypeOf[Parser[_]](x._2).get.head)) //TODO what do you mean its super ugly ? 100 | } 101 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/parsers/FlatMapImpl.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.parsers 2 | 3 | import scala.collection.mutable.HashMap 4 | import fastparsers.framework._ 5 | import fastparsers.input._ 6 | import fastparsers.tools._ 7 | import fastparsers.framework.ruleprocessing.RulesTransformer 8 | import fastparsers.error.ParseError 9 | 10 | /** 11 | * Created by Eric on 22.04.14. 12 | * Implements FlatMap for fastparsers in the macro world. 13 | * A preprocessing phase is needed to modify the code inside the lambda/partial function 14 | */ 15 | trait FlatMapImpl extends RulesTransformer with ParserImplBase { 16 | self: ParseInput with ParseError with TreeTools => 17 | 18 | import c.universe._ 19 | 20 | 21 | override def transformRuleCalls(tree: c.Tree, 22 | enclosingRule: RuleInfo, 23 | rulesMap: HashMap[String, RuleInfo], 24 | expandedRules: HashMap[String, RuleInfo], 25 | rulesPath: List[String]): c.Tree = { 26 | 27 | def expandCallRuleFlatMap(tree: c.Tree): c.Tree = { 28 | def expandBody(body: c.Tree) = body match { 29 | case q"{..$body;$parser}" => q"{..$body;${transformRuleCalls(parser, enclosingRule, rulesMap, expandedRules, rulesPath)}}" 30 | case _ => c.abort(c.enclosingPosition, "ill-formed body") 31 | } 32 | tree match { 33 | case q"(..$params => $x match {case ..$cases})" => 34 | val trcases = cases.map { 35 | case cq"$pat => $body" => cq"$pat => ${expandBody(body)}" 36 | } 37 | q"(..$params => $x match {case ..$trcases})" 38 | case q"(..$params => $body)" => q"(..$params => ${expandBody(body)})" 39 | case _ => c.abort(c.enclosingPosition, show(tree)) 40 | } 41 | } 42 | 43 | tree match { 44 | case q"$a flatMap[$d]($f)" => q"${transformRuleCalls(a, enclosingRule, rulesMap, expandedRules, rulesPath)} flatMap[$d](${expandCallRuleFlatMap(f)})" 45 | case q"$a >>[$d]($f)" => q"${transformRuleCalls(a, enclosingRule, rulesMap, expandedRules, rulesPath)} >>[$d](${expandCallRuleFlatMap(f)})" 46 | case _ => super.transformRuleCalls(tree, enclosingRule, rulesMap, expandedRules, rulesPath) 47 | } 48 | } 49 | 50 | 51 | 52 | override def expand(tree: c.Tree, rs: ResultsStruct) = tree match { 53 | case q"$_.flatmapparsers[$d]($a)" => expand(a, rs) 54 | case q"$a flatMap[$d]($f)" => parseFlatMap(a, f, d, rs) 55 | case q"$a >>[$d]($f)" => parseFlatMap(a, f, d, rs) 56 | case _ => super.expand(tree, rs) 57 | } 58 | 59 | override def prettyPrint(tree: c.Tree) = tree match { 60 | case q"$_.flatmapparsers[$d]($a)" => prettyPrint(a) 61 | case q"$a flatMap[$d]($f)" => prettyPrint(a) + ".flatMap(" + prettyPrint(f) + ")" 62 | case q"$a >>[$d]($f)" => prettyPrint(a) + ".>> (" + prettyPrint(f) + ")" 63 | case _ => super.prettyPrint(tree) 64 | } 65 | 66 | private def parseFlatMap(a: c.Tree, f: c.Tree, typ: c.Tree, rs: ResultsStruct) = f match { 67 | case q"(..$params => $body)" => 68 | var results_tmp = rs.temporary 69 | val result = rs.newVar(typ) 70 | val fm = TermName(c.freshName) 71 | c.untypecheck(mark { 72 | rollback => 73 | q""" 74 | ${expand(a, results_tmp)} 75 | if ($success) { 76 | val $fm = (..$params => ${expandFunction(body, result, rs)}) 77 | $fm.apply(${results_tmp.combine}) 78 | if (!$success) 79 | $rollback 80 | } 81 | else { 82 | $rollback 83 | } 84 | """ 85 | }) 86 | 87 | case _ => c.abort(c.enclosingPosition, "invalid function in rhs of flatMap") 88 | } 89 | 90 | private def expandFunction(func: c.Tree, result: TermName, rs: ResultsStruct) = { 91 | def expandBody(body: List[c.Tree], parser: c.Tree)(wrap: c.Tree => c.Tree) = { 92 | var results_tmp = rs.temporary 93 | val tree = wrap( 94 | q"""{ 95 | ..$body 96 | ${expand(parser, results_tmp)} 97 | $result = ${results_tmp.combine} 98 | } """) 99 | tree 100 | } 101 | 102 | func match { 103 | case q"$x match {case ..$cases}" => 104 | val trcases = cases.map { 105 | case cq"$pat => {..$body;$parser}" => expandBody(body, parser) { 106 | code => cq"$pat => $code" 107 | } 108 | } 109 | q"$x match {case ..$trcases}" 110 | case q"{..$body;$parser}" => expandBody(body, parser) { 111 | x => x 112 | } 113 | case _ => expand(func, rs) 114 | } 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/parsers/BaseParsers.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.parsers 2 | 3 | import fastparsers.input.InputWindow 4 | import InputWindow.InputWindow 5 | import scala.annotation.compileTimeOnly 6 | import scala.util.parsing.input.Positional 7 | 8 | /** 9 | * Interface for Base parsers 10 | */ 11 | 12 | trait BaseParsers[Elem, Input] { 13 | 14 | 15 | /* 16 | Used to deconstruct tuples, i.e write case a~b => instead of case (a,b) => 17 | */ 18 | object ~ { 19 | def unapply[T, U](x: Tuple2[T, U]): Option[Tuple2[T, U]] = Some((x._1, x._2)) 20 | } 21 | 22 | abstract class ElemOrRange 23 | 24 | @compileTimeOnly("can’t be used outside FastParser") 25 | implicit def toElemOrRange(elem: Elem): ElemOrRange = ??? 26 | 27 | @compileTimeOnly("can’t be used outside FastParser") 28 | implicit def toElemOrRange(elem: (Elem,Elem)): ElemOrRange = ??? 29 | 30 | 31 | 32 | 33 | @compileTimeOnly("toElem can’t be used outside FastParser") 34 | implicit def toElem(elem: Elem): Parser[Elem] = ??? 35 | 36 | @compileTimeOnly("toElem can’t be used outside FastParser") 37 | implicit def toElem(elem: (Elem, Elem)): Parser[Elem] = ??? 38 | 39 | @compileTimeOnly("range can’t be used outside FastParser") 40 | def range(a: Elem, b: Elem): Parser[Elem] = ??? 41 | 42 | @compileTimeOnly("accept can’t be used outside FastParser") 43 | def accept(p1: ElemOrRange, p2: ElemOrRange*):Parser[Elem] = ??? 44 | 45 | 46 | @compileTimeOnly("not can’t be used outside FastParser") 47 | def not(p1: ElemOrRange, p2: ElemOrRange*): Parser[Elem] = ??? 48 | 49 | @compileTimeOnly("acceptIf can’t be used outside FastParser") 50 | def acceptIf(f: Elem => Boolean): Parser[Elem] = ??? 51 | 52 | @compileTimeOnly("wildcard can’t be used outside FastParser") 53 | def wildcard: Parser[Elem] = ??? 54 | 55 | @compileTimeOnly("takeWhile can’t be used outside FastParser") 56 | def takeWhile(f: Elem => Boolean): Parser[Input] = ??? 57 | 58 | @compileTimeOnly("takeWhile can’t be used outside FastParser") 59 | def takeWhile2(f: Elem => Boolean): Parser[InputWindow[Input]] = ??? 60 | 61 | @compileTimeOnly("take can’t be used outside FastParser") 62 | def take(n: Int): Parser[Input] = ??? 63 | 64 | @compileTimeOnly("raw can’t be used outside FastParser") 65 | def raw[T](p:Parser[T]):Parser[InputWindow[Input]] = ??? 66 | 67 | @compileTimeOnly("guard can’t be used outside FastParser") 68 | def guard[T](p: Parser[T]): Parser[T] = ??? 69 | 70 | @compileTimeOnly("phrase can’t be used outside FastParser") 71 | def phrase[T](p: Parser[T]): Parser[T] = ??? 72 | 73 | @compileTimeOnly("failure can’t be used outside FastParser") 74 | def failure(msg: String): Parser[Any] = ??? 75 | 76 | @compileTimeOnly("success can’t be used outside FastParser") 77 | def success[T](v: T): Parser[T] = ??? 78 | 79 | 80 | @compileTimeOnly("position can’t be used outside FastParser") 81 | def position: Parser[Int] = ??? 82 | 83 | @compileTimeOnly("positioned can’t be used outside FastParser") 84 | def positioned[T <: Positional](p: Parser[T]): Parser[T] = ??? 85 | 86 | @compileTimeOnly("positioned can’t be used outside FastParser") 87 | def call[T](p: Any,params: Any*) : Parser[T] = ??? 88 | @compileTimeOnly("positioned can’t be used outside FastParser") 89 | def callParam[T](p: String) : Parser[T] = ??? 90 | def compound[T](p:Parser[T]): Parser[T] = ??? 91 | def foreignCall[T](p: Any, ruleName: Any, params: Any*) = ??? 92 | 93 | 94 | @compileTimeOnly("positioned can’t be used outside FastParser") 95 | def paramRule[T](p: Any, params: Any*): Parser[T] = ??? 96 | 97 | trait BaseParser[T] { 98 | @compileTimeOnly("~ can’t be used outside FastParser") 99 | def ~[U](p2: Parser[U]): Parser[(T, U)] = ??? 100 | 101 | @compileTimeOnly("~> can’t be used outside FastParser") 102 | def ~>[U](p2: Parser[U]): Parser[U] = ??? 103 | 104 | @compileTimeOnly("<~ can’t be used outside FastParser") 105 | def <~[U](p2: Parser[U]): Parser[T] = ??? 106 | 107 | @compileTimeOnly("|| can’t be used outside FastParser") 108 | def ||[U >: T](p2: Parser[U]): Parser[U] = ??? 109 | 110 | @compileTimeOnly("| can’t be used outside FastParser") 111 | def |[U >: T](p2: Parser[U]): Parser[U] = ??? 112 | 113 | @compileTimeOnly("^^ can’t be used outside FastParser") 114 | def ^^[U](f: T => U): Parser[U] = ??? 115 | 116 | @compileTimeOnly("map can’t be used outside FastParser") 117 | def map[U](f: T => U): Parser[U] = ??? 118 | 119 | @compileTimeOnly("^^^ can’t be used outside FastParser") 120 | def ^^^[U](f: U): Parser[U] = ??? 121 | 122 | @compileTimeOnly("filter can’t be used outside FastParser") 123 | def filter[U >: T](f: T => Boolean): Parser[T] = ??? 124 | 125 | @compileTimeOnly("withFailureMessage can’t be used outside FastParser") 126 | def withFailureMessage(msg: String): Parser[T] = ??? 127 | } 128 | 129 | 130 | implicit class elemParser(p1: Elem) extends BaseParser[Elem] 131 | implicit class baseParsers[T](p1: Parser[T]) extends BaseParser[T] 132 | 133 | } 134 | -------------------------------------------------------------------------------- /FastParsers/src/test/scala/JsonParserBenchmark.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by Eric on 05.04.14. 3 | */ 4 | 5 | import fastparsers.input.InputWindow 6 | import org.scalameter.api._ 7 | import JsonParsers._ 8 | import scala.collection.mutable.ListBuffer 9 | 10 | import lms._ 11 | import InputWindow._ 12 | 13 | object JsonParserBenchmark extends PerformanceTest { 14 | 15 | lazy val executor = LocalExecutor( 16 | new Executor.Warmer.Default, 17 | Aggregator.min, 18 | new Measurer.Default) 19 | lazy val reporter = new LoggingReporter 20 | lazy val persistor = Persistor.None 21 | 22 | val range = Gen.enumeration("size")(10) 23 | 24 | val files = (1 to 4).foldLeft(new ListBuffer[Array[Char]]){ (acc,i) => 25 | val filename = "FastParsers/src/test/resources/json" + i 26 | val data = scala.io.Source.fromFile(filename).getLines mkString "\n" 27 | acc.append(data.toCharArray) 28 | acc 29 | }.toList 30 | 31 | val bigFileName = "FastParsers/src/test/resources/" + "json.big1" 32 | val bigFile = scala.io.Source.fromFile(bigFileName).getLines mkString "\n" 33 | val bigFileArray = bigFile.toCharArray 34 | val bigFileSeq = new FastCharSequence(bigFileArray) 35 | 36 | val vbigFileName = "FastParsers/src/test/resources/" + "json.vbig" 37 | val vbigFile = scala.io.Source.fromFile(vbigFileName).getLines mkString "\n" 38 | val vbigFileArray = vbigFile.toCharArray 39 | val vbigFileSeq = new FastCharSequence(vbigFileArray) 40 | 41 | 42 | /*performance of "JsonParser on small inputs" in { 43 | measure method "FastParsers" in { 44 | using(range) in { j => 45 | for (i <- 1 to j; m <- files) 46 | JSonImplBoxed.jsonparser.value(m) 47 | } 48 | } 49 | measure method "LMS (gen2)" in { 50 | using(range) in { j => 51 | for (i <- 1 to j; m <- files) 52 | LMSJsonParserGen2.apply(m) 53 | } 54 | } 55 | 56 | /*measure method "Combinators" in { 57 | using(range) in { j => 58 | for (i <- 1 to j; m <- files) 59 | JSON.parse(JSON.value,new FastCharSequence(m)) 60 | } 61 | }*/ 62 | }*/ 63 | 64 | 65 | /* performance of "JsonParser on a big input" in { 66 | measure method "FastParsers" in { 67 | using(range) in { j => 68 | for (i <- 1 to j) 69 | JSonImplBoxed.jsonparser.value(bigFileArray) 70 | } 71 | } 72 | 73 | measure method "LMS (gen2)" in { 74 | using(range) in { j => 75 | for (i <- 1 to j) 76 | LMSJsonParserGen2.apply(bigFileArray) 77 | } 78 | } 79 | 80 | /* measure method "Combinators" in { 81 | using(range) in { j => 82 | for (i <- 1 to j) 83 | JSON.parse(JSON.value,bigFileSeq) 84 | } 85 | }*/ 86 | 87 | }*/ 88 | 89 | 90 | /*performance of "JsonParser on a very big input" in { 91 | measure method "FastParsers" in { 92 | using(range) in { j => 93 | for (i <- 1 to j) 94 | JSonImpl2.jsonparser.value(vbigFileArray) 95 | } 96 | } 97 | 98 | measure method "Combinators" in { 99 | using(range) in { j => 100 | for (i <- 1 to j) 101 | JSON.parse(JSON.value,vbigFileSeq) 102 | } 103 | } 104 | 105 | measure method "LMS (gen2)" in { 106 | using(range) in { j => 107 | for (i <- 1 to j) 108 | LMSJsonParserGen2.apply(vbigFileArray) 109 | } 110 | } 111 | }*/ 112 | 113 | 114 | performance of "Different JSonParser implementations" in { 115 | /*measure method "FastParsers" in { 116 | using(range) in { j => 117 | for (i <- 1 to j) 118 | JSonImpl2.jsonparser.value(bigFileArray) 119 | } 120 | }*/ 121 | 122 | /*measure method "FastParsers Boxed" in { 123 | using(range) in { j => 124 | for (i <- 1 to j) 125 | JSonImplBoxed.jsonparser.value(vbigFileArray) 126 | } 127 | } 128 | 129 | measure method "FastParsers no inline" in { 130 | using(range) in { j => 131 | for (i <- 1 to j) 132 | JSonImpl3.jsonparser.value(vbigFileArray) 133 | } 134 | } 135 | measure method "FastParsers no inline with errors reporting" in { 136 | using(range) in { j => 137 | for (i <- 1 to j) 138 | JSonImpl4.jsonparser.value(vbigFileArray) 139 | } 140 | } 141 | 142 | measure method "FastParsers no inline with errors reporting and ignore results" in { 143 | using(range) in { j => 144 | for (i <- 1 to j) 145 | JSonImpl5.jsonparser.value(vbigFileArray) 146 | } 147 | }*/ 148 | 149 | /*measure method "FastParsers InputWindow to String" in { 150 | using(range) in { j => 151 | for (i <- 1 to j) 152 | JSonImpl6.jsonparser.value(vbigFileArray) 153 | } 154 | }*/ 155 | 156 | /*measure method "FastParsers on string input" in { 157 | using(range) in { j => 158 | for (i <- 1 to j) 159 | JSonImpl1.jsonparser.value(bigFile) 160 | } 161 | }*/ 162 | 163 | /*measure method "FastParsers on bigFileSeq input" in { 164 | using(range) in { j => 165 | //for (i <- 1 to j) 166 | JSON.parse(JSON.value,bigFileSeq) 167 | } 168 | } 169 | 170 | measure method "FastParsers on bigFile input" in { 171 | using(range) in { j => 172 | //for (i <- 1 to j) 173 | JSON.parse(JSON.value,bigFile) 174 | } 175 | }*/ 176 | 177 | } 178 | 179 | } 180 | -------------------------------------------------------------------------------- /FastParsers/src/test/resources/tweet9: -------------------------------------------------------------------------------- 1 | HTTP/1.1 200 OK 2 | cache-control: no-cache, no-store, must-revalidate, pre-check=0, post-check=0 3 | content-length: 5232 4 | content-type: application/json;charset=utf-8 5 | date: Wed, 04 Sep 2013 13:12:15 GMT 6 | expires: Tue, 31 Mar 1981 05:00:00 GMT 7 | last-modified: Wed, 04 Sep 2013 13:12:15 GMT 8 | pragma: no-cache 9 | server: tfe 10 | set-cookie: lang=en 11 | set-cookie: guest_id=v1%3A137830033507910847; Domain=.twitter.com; Path=/; Expires=Fri, 04-Sep-2015 13:12:15 UTC 12 | status: 200 OK 13 | x-access-level: read 14 | x-frame-options: SAMEORIGIN 15 | x-rate-limit-limit: 180 16 | x-rate-limit-remaining: 174 17 | x-rate-limit-reset: 1378300992 18 | x-transaction: 9999f617cf8c0608 19 | 20 | {"statuses":[{"metadata":{"result_type":"recent","iso_language_code":"en"},"created_at":"Wed Sep 04 11:18:18 +0000 2013","id":375216244321837057,"id_str":"375216244321837057","text":"@twitterapi Is there any API to delete a tweet?","source":"web","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":6253282,"in_reply_to_user_id_str":"6253282","in_reply_to_screen_name":"twitterapi","user":{"id":415415472,"id_str":"415415472","name":"\u0bae\u0bca\u0b95\u0bcd\u0b95\u0bb0\u0bbe\u0b9a\u0bc1","screen_name":"mokrasu","location":"","description":"\u0b8e\u0ba9\u0bcd\u0ba9\u0bc1\u0b9f\u0bc8\u0baf \u0b8e\u0bb2\u0bcd\u0bb2\u0bbe \u0b9f\u0bbf\u0bb5\u0bbf\u0b9f\u0bcd\u0b9f\u0bbf\u0ba9\u0bcd \u0baa\u0bbf\u0ba9\u0bcd\u0ba9\u0bbe\u0bb2\u0bcd \u0b87\u0ba4\u0bc8 \u0b9a\u0bc7\u0bb0\u0bcd\u0ba4\u0bcd\u0ba4\u0bc1 \u0baa\u0b9f\u0bbf\u0b95\u0bcd\u0b95\u0bb5\u0bc1\u0bae\u0bcd \u201d\u0b87\u0ba4\u0bc1 \u0b95\u0bb1\u0bcd\u0baa\u0ba9\u0bc8\u0baf\u0bc7, \u0baf\u0bbe\u0bb0\u0bc8\u0baf\u0bc1\u0bae\u0bcd \u0baa\u0bc1\u0ba3\u0bcd\u0baa\u0b9f\u0bc1\u0ba4\u0bcd\u0ba4\u0bc1\u0bae\u0bcd \u0ba8\u0bcb\u0b95\u0bcd\u0b95\u0bae\u0bb2\u0bcd\u0bb2\u201d","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":2162,"friends_count":247,"listed_count":15,"created_at":"Fri Nov 18 09:23:41 +0000 2011","favourites_count":462,"utc_offset":19800,"time_zone":"Chennai","geo_enabled":false,"verified":false,"statuses_count":12843,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http:\/\/a0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_image_url_https":"https:\/\/si0.twimg.com\/images\/themes\/theme1\/bg.png","profile_background_tile":false,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/1660223875\/Mokkarasu_normal.JPG","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/1660223875\/Mokkarasu_normal.JPG","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"twitterapi","name":"Twitter API","id":6253282,"id_str":"6253282","indices":[0,11]}]},"favorited":false,"retweeted":false,"lang":"en"},{"metadata":{"result_type":"recent","iso_language_code":"en"},"created_at":"Wed Sep 04 09:29:08 +0000 2013","id":375188770368126976,"id_str":"375188770368126976","text":"@twitterapi how come the ME button is no longer there??","source":"web","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":6253282,"in_reply_to_user_id_str":"6253282","in_reply_to_screen_name":"twitterapi","user":{"id":1648593661,"id_str":"1648593661","name":"COYRedsss","screen_name":"1865clintonarms","location":"East Midlands","description":"Football Mad..Get the odd bit of info.\r\n#NFFC","url":"http:\/\/t.co\/buFX7hyvop","entities":{"url":{"urls":[{"url":"http:\/\/t.co\/buFX7hyvop","expanded_url":"http:\/\/giffgaff.com\/orders\/affiliate\/123inger","display_url":"giffgaff.com\/orders\/affilia\u2026","indices":[0,22]}]},"description":{"urls":[]}},"protected":false,"followers_count":552,"friends_count":1194,"listed_count":3,"created_at":"Mon Aug 05 20:04:13 +0000 2013","favourites_count":9,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":812,"lang":"en","contributors_enabled":false,"is_translator":false,"profile_background_color":"131516","profile_background_image_url":"http:\/\/a0.twimg.com\/profile_background_images\/378800000061436349\/905a526f66d25ea819207c60bb3f1da3.jpeg","profile_background_image_url_https":"https:\/\/si0.twimg.com\/profile_background_images\/378800000061436349\/905a526f66d25ea819207c60bb3f1da3.jpeg","profile_background_tile":true,"profile_image_url":"http:\/\/a0.twimg.com\/profile_images\/378800000408977568\/3fcf97759fbe7485bf73a3947a7306fa_normal.jpeg","profile_image_url_https":"https:\/\/si0.twimg.com\/profile_images\/378800000408977568\/3fcf97759fbe7485bf73a3947a7306fa_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/1648593661\/1377698413","profile_link_color":"009999","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"EFEFEF","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"twitterapi","name":"Twitter API","id":6253282,"id_str":"6253282","indices":[0,11]}]},"favorited":false,"retweeted":false,"lang":"en"}],"search_metadata":{"completed_in":0.048,"max_id":0,"max_id_str":"0","query":"to%3Atwitterapi","count":3,"since_id":0,"since_id_str":"0"}} 21 | -------------------------------------------------------------------------------- /FastParsers/src/test/scala/JsonParsers.scala: -------------------------------------------------------------------------------- 1 | 2 | import fastparsers.input.InputWindow 3 | import fastparsers.parsers.Parser 4 | import scala.util.parsing.combinator._ 5 | import scala.util.parsing.input._ 6 | 7 | /** 8 | * Created by Eric on 05.04.14. 9 | */ 10 | object JsonParsers { 11 | 12 | object JSonImpl1 { 13 | import fastparsers.framework.implementations.FastParsers._ 14 | val jsonparser = FastParser{ 15 | def value:Parser[Any] = whitespaces ~> (obj | arr | stringLit | decimalNumber | "null" | "true" | "false") 16 | def obj:Parser[Any] = '{' ~> repsep(member,",") <~ "}" 17 | def arr:Parser[Any] = '[' ~> repsep(value,",") <~ "]" 18 | def member:Parser[Any] = stringLit ~> ":" ~> value 19 | } 20 | } 21 | 22 | val nullValue = "null".toCharArray 23 | val trueValue = "true".toCharArray 24 | val falseValue = "false".toCharArray 25 | val closeBracket = "}".toCharArray 26 | val closeSBracket = "]".toCharArray 27 | val comma = ",".toCharArray 28 | val points = ":".toCharArray 29 | 30 | object JSonImpl2 { 31 | import fastparsers.framework.implementations.FastParsersCharArray._ 32 | val jsonparser = FastParsersCharArray{ 33 | def value:Parser[Any] = whitespaces ~> (obj | arr | stringLit | decimalNumber | nullValue | trueValue | falseValue) 34 | def obj:Parser[Any] = '{' ~> repsep(member,comma) <~ closeBracket 35 | def arr:Parser[Any] = '[' ~> repsep(value,comma) <~ closeSBracket 36 | def member:Parser[Any] = stringLit ~ (lit(points) ~> value) 37 | } 38 | } 39 | 40 | 41 | object JSonImpl3 { 42 | import fastparsers.framework.implementations.FastParsersCharArrayNoInline._ 43 | val jsonparser = FastParsersCharArray{ 44 | def value:Parser[Any] = whitespaces ~> (obj | arr | stringLit | decimalNumber | nullValue | trueValue | falseValue) 45 | def obj:Parser[Any] = '{' ~> repsep(member,comma) <~ closeBracket 46 | def arr:Parser[Any] = '[' ~> repsep(value,comma) <~ closeSBracket 47 | def member:Parser[Any] = stringLit ~ (lit(points) ~> value) 48 | } 49 | } 50 | 51 | object JSonImpl4 { 52 | import fastparsers.framework.implementations.FastParsersCharArrayDefaultErrors._ 53 | val jsonparser = FastParsersCharArray{ 54 | def value:Parser[Any] = whitespaces ~> (obj | arr | stringLit | decimalNumber | nullValue | trueValue | falseValue) 55 | def obj:Parser[Any] = '{' ~> repsep(member,comma) <~ closeBracket 56 | def arr:Parser[Any] = '[' ~> repsep(value,comma) <~ closeSBracket 57 | def member:Parser[Any] = stringLit ~ (lit(points) ~> value) 58 | } 59 | } 60 | 61 | object JSonImpl5 { 62 | import fastparsers.framework.implementations.FastParsersCharArrayIgnoreResults._ 63 | val jsonparser = FastParsersCharArray{ 64 | def value:Parser[Any] = whitespaces ~> (obj | arr | stringLit | decimalNumber | nullValue | trueValue | falseValue) 65 | def obj:Parser[Any] = '{' ~> repsep(member,comma) <~ closeBracket 66 | def arr:Parser[Any] = '[' ~> repsep(value,comma) <~ closeSBracket 67 | def member:Parser[Any] = stringLit ~ (lit(points) ~> value) 68 | } 69 | } 70 | 71 | object JSonImpl6 { 72 | import fastparsers.framework.implementations.FastParsersCharArray._ 73 | import fastparsers.input.InputWindow.InputWindow 74 | val jsonparser = FastParsersCharArray{ 75 | def value:Parser[Any] = whitespaces ~> (obj | arr | stringLit ^^ (_.toString) | decimalNumber | nullValue | trueValue | falseValue) 76 | def obj:Parser[Any] = '{' ~> repsep(member,comma) <~ closeBracket 77 | def arr:Parser[Any] = '[' ~> repsep(value,comma) <~ closeSBracket 78 | def member:Parser[Any] = stringLit ~ (lit(points) ~> value) ^^ {case (a, b) => (a.toString, b)} 79 | } 80 | } 81 | 82 | object JSonImplBoxed { 83 | import fastparsers.framework.implementations.FastParsersCharArray._ 84 | //GROS HACK 85 | import fastparsers.input.InputWindow.InputWindow 86 | 87 | sealed abstract class JSValue 88 | case class JSObject(map: List[(InputWindow[Array[Char]], JSValue)]) extends JSValue 89 | case class JSArray(arr: List[JSValue]) extends JSValue 90 | case class JSDouble(d: InputWindow[Array[Char]]) extends JSValue 91 | case class JSDouble2(d: Double) extends JSValue 92 | case class JSString(s: InputWindow[Array[Char]]) extends JSValue 93 | case class JSBool(b: Boolean) extends JSValue 94 | case object JSNull extends JSValue 95 | 96 | val nullValue = "null".toCharArray 97 | val trueValue = "true".toCharArray 98 | val falseValue = "false".toCharArray 99 | val closeBracket = "}".toCharArray 100 | val closeSBracket = "]".toCharArray 101 | val comma = ",".toCharArray 102 | val points = ":".toCharArray 103 | 104 | val jsonparser = FastParsersCharArray { 105 | def value:Parser[JSValue] = whitespaces ~> 106 | ( 107 | obj | 108 | arr | 109 | stringLit ^^ {x => JSString(x)} | 110 | decimalNumber ^^ {x => JSDouble2(x.toString.toDouble)} | 111 | //decimalNumber ^^ {x => JSDouble(x)} | 112 | lit(nullValue) ^^^ JSNull | 113 | lit(trueValue) ^^^ JSBool(true) | 114 | lit(falseValue) ^^^ JSBool(false) 115 | ) 116 | def obj:Parser[JSValue] = ('{' ~> repsep(member,comma) <~ closeBracket) ^^ {x => JSObject(x)} 117 | def arr:Parser[JSValue] = ('[' ~> repsep(value,comma) <~ closeSBracket) ^^ {x => JSArray(x)} 118 | def member:Parser[(InputWindow[Array[Char]], JSValue)] = stringLit ~ (lit(points) ~> value) 119 | } 120 | 121 | } 122 | 123 | object JSON extends JavaTokenParsers { 124 | def value: Parser[Any] = obj | arr | stringLiteral | 125 | floatingPointNumber | 126 | "null" | "true" | "false" 127 | def obj: Parser[Any] = "{" ~> repsep(member, ",") <~ "}" 128 | def arr: Parser[Any] = "[" ~> repsep(value, ",") <~ "]" 129 | def member: Parser[Any] = stringLiteral ~> ":" ~> value 130 | } 131 | 132 | //if needed 133 | def JsonEqual(a:Any,b:Any):Boolean = (a,b) match { 134 | case (x::xs,y::ys) => JsonEqual(x,y) && JsonEqual(xs,ys) 135 | case (Tuple2(x1,y1),JSON.~(x2,y2)) => JsonEqual(x1,x2) && JsonEqual(y1,y2) 136 | case (JSON.~(x1,y1),(x2,y2)) => JsonEqual(x1,x2) && JsonEqual(y1,y2) 137 | case _ => a == b 138 | 139 | } 140 | 141 | } 142 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/parsers/TokenParsersImpl.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.parsers 2 | 3 | import fastparsers.input.StringLikeInput 4 | import fastparsers.error.ParseError 5 | 6 | /** 7 | * Created by Eric on 22.04.14. 8 | * Implementation of TokenParsers 9 | */ 10 | trait TokenParsersImpl extends ParserImplBase { self: StringLikeInput with ParseError => 11 | 12 | import c.universe._ 13 | 14 | override def expand(tree: c.Tree, rs: ResultsStruct) = tree match { 15 | case q"$_.lit($str)" => parseLit(str, rs) 16 | case q"$_.ident" => parseIdentifier(rs) 17 | case q"$_.stringLit" => parseStringLit(rs) 18 | case q"$_.number" => parseNumber(rs) 19 | case q"$_.decimalNumber" => parseDecimalNumber(rs) 20 | case q"$_.whitespaces" => parseWhiteSpaces(rs) 21 | case _ => super.expand(tree, rs) 22 | } 23 | 24 | override def prettyPrint(tree: c.Tree) = tree match { 25 | case q"$_.lit($str)" => "lit(" + show(str) + ")" 26 | case q"$_.ident" => "ident" 27 | case q"$_.stringLit" => "stringLit" 28 | case q"$_.number" => "number" 29 | case q"$_.decimalNumber" => "decimalNumber" 30 | case q"$_.whitespaces" => "whitespaces" 31 | case _ => super.prettyPrint(tree) 32 | } 33 | 34 | private def skipWhiteSpace = { 35 | q""" 36 | while($isNEOI && ($currentInput == ' ' || $currentInput == '\t' || $currentInput == '\n' || $currentInput == '\r')) 37 | $advance 38 | """ 39 | } 40 | 41 | private def parseLit(str: c.Tree, rs: ResultsStruct) = { 42 | val tmpstr = TermName(c.freshName) 43 | val litsize = TermName(c.freshName) 44 | val i = TermName(c.freshName) 45 | //error = "`" + $str + "' expected but " + (if ($isEOI) "EOF" else $currentInput) + " found at " + $pos 46 | mark { rollback => 47 | q""" 48 | var $i = 0 49 | val $litsize = $str.length 50 | $skipWhiteSpace 51 | while ($isNEOI && $i < $litsize && $currentInput == $str.charAt($i)){ 52 | $i = $i + 1 53 | $advance 54 | } 55 | if ($i == $litsize){ 56 | $success = true 57 | ${rs.assignNew(str, inputType)} 58 | } 59 | else { 60 | $success = false 61 | ${pushError("`" + show(str) + "' expected but ... found", pos)} 62 | $rollback 63 | } 64 | """ 65 | } 66 | } 67 | 68 | private def parseIdentifier(rs: ResultsStruct) = { 69 | val beginpos = TermName(c.freshName) 70 | mark { rollback => 71 | q""" 72 | $skipWhiteSpace 73 | val $beginpos = $pos 74 | if ($isNEOI && Character.isJavaIdentifierStart($currentInput)){ 75 | $advance 76 | while ($isNEOI && Character.isJavaIdentifierPart($currentInput)) { 77 | $advance 78 | } 79 | ${rs.assignNew(getInputWindow(q"$beginpos", q"$pos"), inputWindowType)} 80 | $success = true 81 | } 82 | else { 83 | $rollback 84 | $success = false 85 | } 86 | """ 87 | } 88 | } 89 | 90 | private def parseStringLit(rs: ResultsStruct) = { 91 | val beginpos = TermName(c.freshName) 92 | mark { rollback => 93 | q""" 94 | $skipWhiteSpace 95 | val $beginpos = $pos 96 | if ($isNEOI && $currentInput == '\"'){ 97 | $advance 98 | while ($isNEOI && $currentInput != '\"'){ 99 | if ($currentInput == '\\'){ 100 | $advance 101 | } 102 | $advance 103 | } 104 | 105 | if ($isNEOI) { 106 | $success = true 107 | $advance 108 | ${rs.assignNew(getInputWindow(q"$beginpos", q"$pos"), inputWindowType)} 109 | } 110 | else { 111 | $success = false 112 | ${pushError("expected '\"' got EOF", pos)} 113 | $rollback 114 | } 115 | } 116 | else { 117 | $success = false 118 | ${pushError("expected '\"' got EOF", pos)} 119 | $rollback 120 | } 121 | """ 122 | } 123 | } 124 | 125 | 126 | private def parseNumber(rs: ResultsStruct) = { 127 | val beginpos = TermName(c.freshName) 128 | mark {rollback => 129 | q""" 130 | $skipWhiteSpace 131 | val $beginpos = $pos 132 | if ($isNEOI && $currentInput == '-'){ 133 | $advance 134 | } 135 | if ($isNEOI && $currentInput >= '0' && $currentInput <= '9') { 136 | $advance 137 | while ($isNEOI && $currentInput >= '0' && $currentInput <= '9'){ 138 | $advance 139 | } 140 | $success = true 141 | ${rs.assignNew(getInputWindow(q"$beginpos", q"$pos"), inputWindowType)} 142 | } 143 | else { 144 | $success = false 145 | ${pushError("expected '\"' got EOF", pos)} 146 | $rollback 147 | } 148 | """ 149 | } 150 | } 151 | 152 | private def parseDecimalNumber(rs: ResultsStruct) = { 153 | val isNeg = TermName(c.freshName) 154 | val beginPos = TermName(c.freshName) 155 | val result = rs.newVar(tq"$inputWindowType") 156 | mark { rollback => 157 | q""" 158 | $skipWhiteSpace 159 | var $isNeg = false 160 | val $beginPos = $pos 161 | $success = false 162 | if ($isNEOI && $currentInput == '-'){ 163 | $advance 164 | } 165 | if ($isNEOI && $currentInput >= '0' && $currentInput <= '9') { 166 | $advance 167 | while ($isNEOI && $currentInput >= '0' && $currentInput <= '9') 168 | $advance 169 | if ($isNEOI && $currentInput == '.') { 170 | $advance 171 | while ($isNEOI && $currentInput >= '0' && $currentInput <= '9') 172 | $advance 173 | } 174 | $success = true 175 | ${rs.assignTo(result, getInputWindow(q"$beginPos", q"$pos"))} 176 | } 177 | else if ($isNEOI && $currentInput == '.') { 178 | $advance 179 | if ($isNEOI && $currentInput >= '0' && $currentInput <= '9') { 180 | $advance 181 | while ($isNEOI && $currentInput >= '0' && $currentInput <= '9') 182 | $advance 183 | $success = true 184 | ${rs.assignTo(result, getInputWindow(q"$beginPos", q"$pos"))} 185 | } 186 | } 187 | 188 | """ 189 | } 190 | } 191 | 192 | private def parseWhiteSpaces(rs: ResultsStruct) = { 193 | val beginpos = TermName(c.freshName) 194 | val result = TermName(c.freshName) 195 | q""" 196 | val $beginpos = $pos 197 | $skipWhiteSpace 198 | ${rs.assignNew(getInputWindow(q"$beginpos", q"$pos"), inputWindowType)} 199 | $success = true 200 | """ 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /FastParsers/src/test/scala/CSVParserBenchmark.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Created by Eric on 05.04.14. 3 | */ 4 | 5 | import fastparsers.input.InputWindow 6 | import org.scalameter.api._ 7 | import CSVParsers._ 8 | import JsonParsers._ 9 | import scala.collection.mutable.ListBuffer 10 | 11 | import lms._ 12 | import InputWindow._ 13 | 14 | /*object CSVParserRun { 15 | def main(args: Array[String]) { 16 | println("cho tai!!") 17 | 18 | val data = "[1.223, -123.243, 212143223.211]".toCharArray 19 | 20 | //println(cvsParser.doubles(data)) 21 | //println(LMSCSVDoubleParserGen2.apply(data)) 22 | //println(LMSCSVDoubleParserGen3.apply(data)) 23 | 24 | val strlits = "[\"higher\", \"natty\", \"jah jah\"]".toCharArray 25 | println(cvsParser.strings(strlits)) 26 | println(LMSCSVStringLitParseGen.apply(strlits)) 27 | println(StringLitParseHandWritten.parseCSVStringLits(strlits, 0)) 28 | 29 | } 30 | }*/ 31 | 32 | object CSVParserBenchmark extends PerformanceTest { 33 | 34 | lazy val executor = LocalExecutor( 35 | new Executor.Warmer.Default, 36 | Aggregator.min, 37 | new Measurer.Default) 38 | lazy val reporter = new LoggingReporter 39 | lazy val persistor = Persistor.None 40 | 41 | val range = Gen.enumeration("size")(10) 42 | 43 | benchStringLits 44 | 45 | def benchDoubles = { 46 | val bigDoubleFileName = "FastParsers/src/test/resources/" + "csvDoubles.txt" 47 | val bigDoubleFile = scala.io.Source.fromFile(bigDoubleFileName).getLines mkString "\n" 48 | val bigDoubleFileArray = bigDoubleFile.toCharArray 49 | val bigDoubleFileSeq = new FastCharSequence(bigDoubleFileArray) 50 | 51 | performance of "CSV Double Parser" in { 52 | /*measure method "FastParsers" in { 53 | using(range) in { j => 54 | for (i <- 1 to j) 55 | CSVImpl1.cvsParser.doubles(bigDoubleFileArray) 56 | } 57 | }*/ 58 | 59 | /*measure method "JSON FastParsers" in { 60 | using(range) in { j => 61 | for (i <- 1 to j) 62 | JSonImpl2.jsonparser.value(bigDoubleFileArray) 63 | //cvsParser.doubles(bigDoubleFileArray) 64 | } 65 | }*/ 66 | 67 | /*measure method "LMS" in { 68 | using(range) in { j => 69 | //for (i <- 1 to j) 70 | LMSCSVDoubleParserGen.apply(bigDoubleFileArray) 71 | } 72 | } 73 | 74 | measure method "LMS2" in { 75 | using(range) in { j => 76 | //for (i <- 1 to j) 77 | LMSCSVDoubleParserGen2.apply(bigDoubleFileArray) 78 | } 79 | }*/ 80 | 81 | /* measure method "LMS3" in { 82 | using(range) in { j => 83 | for (i <- 1 to j) 84 | LMSCSVDoubleParserGen3.apply(bigDoubleFileArray) 85 | } 86 | }*/ 87 | 88 | //too slow 89 | /*measure method "Combinators" in { 90 | using(range) in { j => 91 | for (i <- 1 to j) 92 | CSV.parse(CSV.doubles, bigDoubleFileSeq) 93 | } 94 | }*/ 95 | } 96 | } 97 | 98 | def benchBools = { 99 | val bigBoolFileName = "FastParsers/src/test/resources/" + "csvBooleans.txt" 100 | val bigBoolFile = scala.io.Source.fromFile(bigBoolFileName).getLines mkString "\n" 101 | val bigBoolFileArray = bigBoolFile.toCharArray 102 | val bigBoolFileSeq = new FastCharSequence(bigBoolFileArray) 103 | 104 | 105 | performance of "CSVBooleanParser:Boolean" in { 106 | /*measure method "FastParsers" in { 107 | using(range) in { j => 108 | for (i <- 1 to j) 109 | CSVImpl1.cvsParser.bools(bigBoolFileArray) 110 | } 111 | } 112 | 113 | measure method "LMS" in { 114 | using(range) in { j => 115 | for (i <- 1 to j) 116 | LMSCSVBooleanParseGen.apply(bigBoolFileArray) 117 | } 118 | }*/ 119 | 120 | /*measure method "Handwritten" in { 121 | using(range) in { j => 122 | for (i <- 1 to j) 123 | CSVBoolHandWritten.apply(bigBoolFileArray) 124 | } 125 | }*/ 126 | 127 | /*measure method "Combinators" in { 128 | using(range) in { j => 129 | for (i <- 1 to j) 130 | CSV.parse(CSV.bools, bigBoolFileSeq) 131 | } 132 | }*/ 133 | } 134 | } 135 | 136 | def benchStringLits = { 137 | val bigStringLitFileName = "FastParsers/src/test/resources/" + "csvStringLits.txt" 138 | val bigStringLitFile = scala.io.Source.fromFile(bigStringLitFileName).getLines mkString "\n" 139 | val bigStringLitFileArray = bigStringLitFile.toCharArray 140 | val bigStringLitFileSeq = new FastCharSequence(bigStringLitFileArray) 141 | 142 | performance of "CSVStringLitParser:StringLit" in { 143 | 144 | //separating stringLit parsing in a different function 145 | /*measure method "Handwritten" in { 146 | using(range) in { j => 147 | for (i <- 1 to j) 148 | StringLitParseHandWritten.parseCSVStringLits(bigStringLitFileArray, 0) 149 | } 150 | } 151 | 152 | //inlining the parsing of string literal 153 | // attention: inlining of functions isNEOI, currentInput, advance makes no difference 154 | measure method "Handwritten2" in { 155 | using(range) in { j => 156 | for (i <- 1 to j) 157 | StringLitParseHandWritten.parseCSVStringLits2(bigStringLitFileArray, 0) 158 | } 159 | } 160 | 161 | //using CharArrayStructs: benchmarks now get interesting 162 | measure method "Handwritten3" in { 163 | using(range) in { j => 164 | for (i <- 1 to j) 165 | StringLitParseHandWritten.parseCSVStringLits3(bigStringLitFileArray, 0) 166 | } 167 | } 168 | 169 | //an extra boxing layer around the result of a parse 170 | measure method "Handwritten4" in { 171 | using(range) in { j => 172 | for (i <- 1 to j) 173 | StringLitParseHandWritten.parseCSVStringLits4(bigStringLitFileArray, 0) 174 | } 175 | } 176 | 177 | //code gen a la LMS for rep1 and stringlit, with extra layer of boxing for results 178 | // a | b = lift(lift(a) or b) 179 | // repsep(p,q) = p ~ rep(q ~ p) | success(Nil) 180 | // stringlit = '\\ ~ anything | not('\"') 181 | measure method "Handwritten4" in { 182 | using(range) in { j => 183 | for (i <- 1 to j) 184 | StringLitParseHandWritten.parseCSVStringLits4(bigStringLitFileArray, 0) 185 | } 186 | } 187 | 188 | measure method "Handwritten5" in { 189 | using(range) in { j => 190 | for (i <- 1 to j) 191 | StringLitParseHandWritten.parseCSVStringLits4(bigStringLitFileArray, 0) 192 | } 193 | }*/ 194 | 195 | /*measure method "FastParsers" in { 196 | using(range) in { j => 197 | for (i <- 1 to j) 198 | CSVImpl1.cvsParser.strings(bigStringLitFileArray) 199 | } 200 | } 201 | 202 | measure method "LMS" in { 203 | using(range) in { j => 204 | for (i <- 1 to j) 205 | LMSCSVStringLitParseGen.apply(bigStringLitFileArray) 206 | } 207 | } 208 | 209 | 210 | measure method "Combinators" in { 211 | using(range) in { j => 212 | for (i <- 1 to j) 213 | CSV.parse(CSV.strings, bigStringLitFileSeq) 214 | } 215 | }*/ 216 | } 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /FastParsers/src/main/scala/fastparsers/parsers/ParserImplBase.scala: -------------------------------------------------------------------------------- 1 | package fastparsers.parsers 2 | 3 | import fastparsers.error._ 4 | import fastparsers.input.ParseInput 5 | import scala.reflect.macros.whitebox.Context 6 | import scala.collection.mutable.ListBuffer 7 | 8 | 9 | /** 10 | * Provide the interface and the basics method needed to implement the transformation on parsers 11 | */ 12 | trait ParserImplBase { self: ParseInput with ParseError => 13 | val c: Context 14 | 15 | import c.universe._ 16 | 17 | /** 18 | * Represent a result variable 19 | * _1 : Variable name 20 | * _2 : Variable type 21 | * _3 : wheter or not it is temporary, which mean that it won't be used anymore to construct another result 22 | */ 23 | type Result = (TermName, c.Tree, Boolean) 24 | 25 | /** 26 | * Structure to deal with the results of a rule expansion 27 | * Put inside all variables which will need to be initialized at the begining of the rule expansion 28 | */ 29 | class ResultsStruct(var results: ListBuffer[Result]) { 30 | def this() = this(new ListBuffer[Result]()) 31 | 32 | @deprecated("will be removed") 33 | protected def setNoUse = results = results.map(x => (x._1, x._2, false)) 34 | protected def append(t: TermName, typ: c.Tree): Unit = append((t, typ, true)) 35 | @deprecated("will be removed") 36 | protected def append(rs: ResultsStruct): Unit = rs.results.foreach(append(_)) 37 | 38 | //TODO modifier !! 39 | def append(r: Result): Unit = results.append(r) 40 | 41 | 42 | def newVar(typ: c.Tree): TermName = { 43 | val t = c.freshName 44 | append(t, typ) 45 | t 46 | } 47 | def assignTo(result: TermName, value: c.Tree): c.Tree = { 48 | assert(results.exists(_._1 == result)) 49 | q"$result = $value" 50 | } 51 | 52 | /** 53 | * Automatically create a variable, assign it with code and register it 54 | * @todo change typ: c.Tree to typ: c.Type 55 | * @param code The code to assign to the newly created variable 56 | * @param typ The typ of the variable 57 | * @return The code representing the assignation 58 | */ 59 | def assignNew(code: c.Tree, typ: c.Type): c.Tree = assignNew(code, tq"$typ") 60 | 61 | def assignNew(code: c.Tree, typ: c.Tree): c.Tree = { 62 | val result = TermName(c.freshName) 63 | append(result,typ) 64 | q"$result = $code" 65 | } 66 | 67 | def temporary = new TemporaryResults(this) 68 | 69 | /** 70 | * Combine a list of results into either a tuple of result or into the same result 71 | */ 72 | def combine: c.Tree = { 73 | val usedResults = results.toList.filter(_._3) 74 | if (usedResults.size > 1) { 75 | //q"(..${usedResults.map(x => q"${x._1}")})" 76 | val first = usedResults(0) 77 | val sec = usedResults(1) 78 | val rest = usedResults.drop(2) 79 | rest.foldLeft(q"(${first._1},${sec._1})")((acc, e) => q"($acc,${e._1})") 80 | } 81 | else if (usedResults.size == 1) 82 | q"${usedResults(0)._1}" 83 | else 84 | q"Nil" 85 | } 86 | 87 | def combineType: c.Tree = { 88 | val usedResults = results.toList.filter(_._3) 89 | if (usedResults.size > 1) { 90 | val first = usedResults(0) 91 | val sec = usedResults(1) 92 | val rest = usedResults.drop(2) 93 | rest.foldLeft(tq"(${first._2},${sec._2})")((acc, e) => tq"($acc,${e._2})") 94 | } 95 | else if (usedResults.size == 1) 96 | tq"${usedResults(0)._2}" 97 | else 98 | tq"Unit" 99 | } 100 | } 101 | 102 | 103 | /** 104 | * Class which automatically append its result to its parents as already used results 105 | * @todo clarify 106 | * @param dependence 107 | */ 108 | class TemporaryResults(dependence: ResultsStruct) extends ResultsStruct { 109 | override def append(r: Result) = { 110 | super.append(r) 111 | dependence.append((r._1,r._2,false)) 112 | } 113 | } 114 | 115 | trait IgnoreResult { self : ResultsStruct => 116 | override def assignNew(code: c.Tree, typ: c.Tree) = q"()" 117 | override def combine = q"()"//c.abort(c.enclosingPosition, "cannot combine results while ignoring them") 118 | override def assignTo(result: TermName, value: c.Tree): c.Tree = q"()" 119 | override def temporary = new TemporaryResults(this) with IgnoreResult 120 | } 121 | 122 | 123 | /** 124 | * Get the "zero" value of a certain type 125 | * @todo not make it work on strings 126 | * @param typ 127 | * @return 128 | */ 129 | 130 | def zeroValue(typ: c.Tree) = { 131 | def fromString(str: String) = str match { 132 | case "Char" => q"' '" 133 | case "Int" => q"0" 134 | case "Long" => q"0L" 135 | case "Float" => q"0" 136 | case "Boolean" => q"false" 137 | case "Byte" => q"0" 138 | case "Double" => q"0.0D" 139 | case "String" => q"""""""" 140 | case "Unit" => q"()" 141 | case x if x.startsWith("List") => q"Nil" 142 | case _ => q"null" 143 | } 144 | c.typecheck(typ, c.TYPEmode).tpe match { 145 | case Ident(TypeName(name)) => fromString(name) 146 | case t if t =:= typeOf[Char] => q"' '" 147 | case t if t =:= typeOf[Int] => q"0" 148 | case t if t =:= typeOf[Long] => q"0L" 149 | case t if t =:= typeOf[Float] => q"0" 150 | case t if t =:= typeOf[Double] => q"0.0D" 151 | case t if t =:= typeOf[Boolean] => q"false" 152 | case t if t =:= typeOf[Unit] => q"()" 153 | case t if t =:= typeOf[String] => q"""""""" 154 | case t if t =:= typeOf[List[_]] => q"Nil" 155 | case t if t <:< typeOf[AnyRef] => q"null" 156 | case t if t =:= typeOf[Any] => q"null" 157 | case t => q"fastparsers.tools.Default.value[$t]" //NEED to find a better way !!!! 158 | /*case x @ TypeRef(_, y, _) => 159 | /*y match { 160 | case y : TypeName => c.echo(c.enclosingPosition, "stuff") 161 | case _ => c.echo(c.enclosingPosition, "nostuff") 162 | }*/ 163 | c.echo(c.enclosingPosition, showRaw(y.tpe)) 164 | q"fastparsers.tools.Default.value[$x]"*/ 165 | //case x => fromString(x.toString) 166 | } 167 | } 168 | 169 | /* def zeroValue(typ: c.Tree) = 170 | if (typ != null) typ.tpe match { 171 | case t if t =:= typeOf[Char] => q"' '" 172 | case t if t =:= typeOf[Int] => q"0" 173 | case t if t =:= typeOf[Float] => q"0" 174 | case t if t =:= typeOf[Double] => q"0.0D" 175 | case t if t =:= typeOf[String] => q"""""""" 176 | case t if t =:= typeOf[List[_]] => q"Nil" 177 | case _ => q"null" 178 | } 179 | else q"null" */ 180 | 181 | /** 182 | * Expand a combinator into an imperative code 183 | * @param tree the combinator to expand 184 | * @param r ResultsStruct used to register the results 185 | * @return The expanded code 186 | */ 187 | def expand(tree: c.Tree, r: ResultsStruct): c.Tree = c.abort(c.enclosingPosition, "Not implemented combinator " + show(tree)) 188 | 189 | /** 190 | * Prints a combinator in a readable way 191 | * @param tree Code containing the combinator to print 192 | */ 193 | def prettyPrint(tree: c.Tree): String = "?" //TODO change ? 194 | } 195 | 196 | trait IgnoreResultsPolicy extends ParserImplBase { self: ParseInput with ParseError => 197 | def ignoreResult(rs: ResultsStruct): ResultsStruct 198 | } 199 | 200 | trait DontIgnoreResults extends IgnoreResultsPolicy { self: ParseInput with ParseError => 201 | override def ignoreResult(rs: ResultsStruct): ResultsStruct = rs.temporary 202 | } 203 | 204 | trait IgnoreResults extends IgnoreResultsPolicy { self: ParseInput with ParseError => 205 | override def ignoreResult(rs: ResultsStruct): ResultsStruct = new ResultsStruct with IgnoreResult 206 | } 207 | 208 | --------------------------------------------------------------------------------