├── .gitignore ├── .scalafmt.conf ├── BUILD.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── build.sbt ├── circe ├── build.sbt └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ ├── io.idml.IdmlJson │ │ │ └── io.idml.functions.FunctionResolver │ └── scala │ │ └── io │ │ └── idml │ │ └── circe │ │ ├── CirceFunctions.scala │ │ ├── IdmlCirce.scala │ │ └── instances.scala │ └── test │ └── scala │ └── io │ └── idml │ └── circe │ ├── CirceFunctionsSpec.scala │ └── IdmlCirceSpec.scala ├── core ├── build.sbt └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── idml │ │ │ ├── AutoIdmlBuilder.java │ │ │ ├── Idml.java │ │ │ ├── IdmlBuilder.java │ │ │ └── StaticIdmlBuilder.java │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── io.idml.functions.FunctionResolver │ └── scala │ │ └── io │ │ └── idml │ │ ├── FunctionResolverService.scala │ │ ├── IdmlChain.scala │ │ ├── IdmlContext.scala │ │ ├── IdmlException.scala │ │ ├── IdmlJson.scala │ │ ├── IdmlListener.scala │ │ ├── IdmlListenerBase.scala │ │ ├── IdmlListenerWithState.scala │ │ ├── IdmlMapping.scala │ │ ├── IdmlParser.scala │ │ ├── PluginFunctionResolverService.scala │ │ ├── StaticFunctionResolverService.scala │ │ ├── UnmappedFieldsFinder.scala │ │ ├── ast │ │ ├── Assignment.scala │ │ ├── AstGenerator.scala │ │ ├── Coalesce.scala │ │ ├── ExecNav.scala │ │ ├── Filter.scala │ │ ├── IdmlFunction.scala │ │ ├── Match.scala │ │ ├── Node.scala │ │ ├── Pipeline.scala │ │ ├── Reassignment.scala │ │ ├── Variable.scala │ │ └── Wildcard.scala │ │ └── functions │ │ ├── AppendFunction.scala │ │ ├── ApplyArrayFunction.scala │ │ ├── ApplyFunction.scala │ │ ├── ArrayFunction.scala │ │ ├── AverageFunction.scala │ │ ├── BlacklistFunction.scala │ │ ├── BuiltinFunctionResolver.scala │ │ ├── ConcatFunction.scala │ │ ├── ExtractFunction.scala │ │ ├── FilterFunction.scala │ │ ├── FunctionResolver.scala │ │ ├── GetSizeFunction.scala │ │ ├── GroupByFunction.scala │ │ ├── GroupsByFunction.scala │ │ ├── IdmlFunction0.scala │ │ ├── IdmlFunction1.scala │ │ ├── IdmlFunction2.scala │ │ ├── IdmlFunction3.scala │ │ ├── IdmlFunctionN.scala │ │ ├── IdmlValueFunction.scala │ │ ├── IdmlValueFunctionResolver.scala │ │ ├── IdmlValueNaryFunctionResolver.scala │ │ ├── LanguageNameFunctions.scala │ │ ├── MapFunction.scala │ │ ├── PrependFunction.scala │ │ ├── SetSizeFunction.scala │ │ ├── SortFunction.scala │ │ ├── UniqueFunction.scala │ │ └── json │ │ ├── JsonFunctions.scala │ │ ├── ObjectModuleJson.scala │ │ ├── RandomModuleJson.scala │ │ └── UUIDModuleJson.scala │ └── test │ ├── resources │ └── mock_resource.txt │ └── scala │ └── io │ └── idml │ ├── AstString.scala │ ├── FunctionResolverServiceTest.scala │ ├── IdmlChainItTest.scala │ ├── IdmlChainTest.scala │ ├── IdmlListenerWithStateTest.scala │ ├── IdmlMappingSpec.scala │ ├── IdmlParserTest.scala │ ├── IdmlTest.scala │ ├── UnmappedFieldsFinderItTest.scala │ ├── UnmappedFieldsFinderTest.scala │ └── json │ └── JsonFunctionSuite.scala ├── datanodes ├── build.sbt ├── readme.md └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── idml │ │ │ └── utils │ │ │ └── IdmlUUID.java │ └── scala │ │ └── io │ │ └── idml │ │ ├── FieldTypeCounter.scala │ │ ├── IdmlArray.scala │ │ ├── IdmlBool.scala │ │ ├── IdmlDouble.scala │ │ ├── IdmlInt.scala │ │ ├── IdmlNothing.scala │ │ ├── IdmlNull.scala │ │ ├── IdmlObject.scala │ │ ├── IdmlString.scala │ │ ├── IdmlValue.scala │ │ ├── IdmlValueVisitor.scala │ │ ├── JIdmlValue.java │ │ └── datanodes │ │ ├── CompositeValue.scala │ │ ├── IArray.scala │ │ ├── IBool.scala │ │ ├── IDate.scala │ │ ├── IDateFormats.scala │ │ ├── IDomNode.scala │ │ ├── IDouble.scala │ │ ├── IEmail.scala │ │ ├── IInt.scala │ │ ├── IObject.scala │ │ ├── IString.scala │ │ ├── IUrl.scala │ │ ├── SgmlNode.scala │ │ ├── modules │ │ ├── ArrayModule.scala │ │ ├── DateModule.scala │ │ ├── DomModule.scala │ │ ├── EmailModule.scala │ │ ├── JavaApiModule.scala │ │ ├── MathsModule.scala │ │ ├── NavigationModule.scala │ │ ├── ObjectModule.scala │ │ ├── RegexModule.scala │ │ ├── SchemaModule.scala │ │ ├── StringModule.scala │ │ └── UrlModule.scala │ │ └── regex │ │ ├── PJavaRegex.scala │ │ ├── PRe2Regex.scala │ │ └── PRegexLike.scala │ └── test │ └── scala │ └── io │ └── idml │ └── datanodes │ ├── IArrayTest.scala │ ├── IDateTest.scala │ ├── IDoubleTest.scala │ ├── IEmailTest.scala │ ├── IIntTest.scala │ ├── IObjectTest.scala │ ├── IStringTest.scala │ ├── IdmlNullTest.scala │ ├── IdmlValueTest.scala │ └── PBoolTest.scala ├── docker.sh ├── docs ├── build.sbt ├── generate-index.sh ├── src │ └── main │ │ ├── resources │ │ └── microsite │ │ │ ├── README.md │ │ │ ├── css │ │ │ ├── override.css │ │ │ └── toc.css │ │ │ ├── data │ │ │ └── menu.yml │ │ │ ├── img │ │ │ ├── .directory │ │ │ ├── jumbotron_pattern.png │ │ │ ├── jumbotron_pattern2x.png │ │ │ ├── logo.svg │ │ │ ├── navbar_brand.png │ │ │ ├── navbar_brand2x.png │ │ │ ├── sidebar_brand.png │ │ │ └── sidebar_brand2x.png │ │ │ ├── js │ │ │ ├── idml-hilight.js │ │ │ └── toc.js │ │ │ └── layouts │ │ │ ├── README.md │ │ │ └── docsplus.html │ │ └── tut │ │ ├── basic-usage.md │ │ ├── dependencies.md │ │ ├── diagrams │ │ ├── chain.png │ │ ├── chain.puml │ │ ├── merge.png │ │ └── merge.puml │ │ ├── features │ │ ├── core-language.md │ │ ├── expressions.md │ │ └── predicates.md │ │ ├── functions │ │ ├── advanced.md │ │ ├── array.md │ │ ├── date.md │ │ ├── email.md │ │ ├── generateindex.py │ │ ├── index.md │ │ ├── maths.md │ │ ├── navigation.md │ │ ├── object.md │ │ ├── random.md │ │ ├── regex.md │ │ ├── schema.md │ │ ├── string.md │ │ ├── url.md │ │ └── uuid.md │ │ ├── getting-started.md │ │ ├── index.md │ │ ├── modules │ │ ├── geo.md │ │ ├── hashing.md │ │ └── jsoup.md │ │ ├── philosophy.md │ │ └── user-guide │ │ └── tour.md └── touchups.sh ├── geo ├── build.sbt └── src │ ├── main │ ├── resources │ │ ├── META-INF │ │ │ └── services │ │ │ │ └── io.idml.functions.FunctionResolver │ │ └── io │ │ │ └── idml │ │ │ └── geo │ │ │ ├── Countries.json │ │ │ └── Regions.json │ └── scala │ │ └── io │ │ └── idml │ │ └── geo │ │ ├── Geo.scala │ │ ├── Geo2Function.scala │ │ ├── GeoFunction.scala │ │ ├── GeoFunctionResolver.scala │ │ ├── IsoCountryFunction.scala │ │ ├── IsoRegionFunction.scala │ │ └── TimezoneFunction.scala │ └── test │ ├── resources │ └── io.idml.geo │ │ ├── CountrySuite.json │ │ ├── Geo2Suite.json │ │ ├── GeoSuite.json │ │ └── RegionSuite.json │ └── scala │ └── io │ └── idml │ └── geo │ ├── Admin1Spec.scala │ ├── Dev2680Test.scala │ ├── GeoFunctionsTest.scala │ └── TimezoneSpec.scala ├── hashing ├── build.sbt └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── io.idml.functions.FunctionResolver │ └── scala │ │ └── io │ │ └── idml │ │ └── hashing │ │ └── HashingFunctionResolver.scala │ └── test │ └── scala │ └── io │ └── idml │ └── hashing │ └── HashingSpec.scala ├── idmld ├── build.sbt └── src │ └── main │ └── scala │ └── io │ └── idml │ └── server │ ├── Server.scala │ └── WebsocketServer.scala ├── idmldoc-plugin ├── build.sbt └── src │ └── main │ └── scala │ └── io │ └── idml │ └── doc │ └── IdmlDocPlugin.scala ├── idmldoc ├── build.sbt └── src │ ├── main │ └── scala │ │ └── io │ │ └── idml │ │ └── doc │ │ ├── Main.scala │ │ ├── Markdown.scala │ │ └── Runners.scala │ └── test │ └── scala │ └── io │ └── idml │ └── idmldoc │ └── IdmlDocSpec.scala ├── idmltest-plugin ├── build.sbt └── src │ └── main │ └── scala │ └── io │ └── idml │ └── test │ └── IdmlTestPlugin.scala ├── idmltest ├── build.sbt └── src │ ├── main │ └── scala │ │ └── io │ │ └── idml │ │ └── test │ │ ├── CirceEitherEncoders.scala │ │ ├── DeterministicTime.scala │ │ ├── Main.scala │ │ ├── Runner.scala │ │ ├── RunnerUtils.scala │ │ ├── Test.scala │ │ ├── TestState.scala │ │ └── diffable │ │ ├── DiffableParser.scala │ │ ├── DiffablePrinter.scala │ │ └── TestDiff.scala │ └── test │ ├── resources │ └── tests │ │ ├── bad-multitest.json │ │ ├── basic-failed.json │ │ ├── basic-invalid-ref.json │ │ ├── basic-multitest.json │ │ ├── basic-pipeline.json │ │ ├── basic-ref-nofile.json │ │ ├── basic-ref.json │ │ ├── basic.json │ │ ├── create-me.json │ │ ├── idml │ │ ├── a.idml │ │ └── b.idml │ │ ├── inject-now.json │ │ ├── null-behaviour.json │ │ ├── outputs │ │ └── output.json │ │ ├── pipeline-ref.json │ │ ├── test.json │ │ └── two-tests.json │ └── scala │ └── io │ └── idml │ └── test │ ├── DeterministicTimeSpec.scala │ ├── DiffableFormatsSpec.scala │ ├── MainSpec.scala │ ├── RunnerSpec.scala │ └── TestsSpec.scala ├── idmltutor ├── build.sbt └── src │ └── main │ └── scala │ └── io │ └── idml │ └── tutor │ ├── Chapter1.scala │ ├── Colours.scala │ ├── JLine.scala │ ├── Main.scala │ ├── TutorialAlg.scala │ └── Utils.scala ├── jackson ├── build.sbt └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ ├── io.idml.IdmlJson │ │ │ └── io.idml.functions.FunctionResolver │ └── scala │ │ └── io │ │ └── idml │ │ └── jackson │ │ ├── DefaultIdmlJackson.scala │ │ ├── IdmlJackson.scala │ │ ├── JacksonFunctions.scala │ │ ├── JsonAstGenerator.scala │ │ ├── difftool │ │ ├── Diff.scala │ │ ├── DiffJacksonModule.scala │ │ └── DiffSerializer.scala │ │ └── serder │ │ ├── IdmlJacksonModule.scala │ │ ├── PValueDeserializer.scala │ │ └── PValueSerializer.scala │ └── test │ └── scala │ └── io │ └── idml │ └── jackson │ ├── ArrayParsingSpec.scala │ ├── FieldTypeCounterTest.scala │ ├── IDoubleParsingSpec.scala │ ├── IdmlJsonTest.scala │ ├── IntParsingSpec.scala │ ├── JacksonFunctionsSpec.scala │ ├── JsonAstGeneratorTest.scala │ ├── NullParsingSpec.scala │ ├── ObjectParsingSpec.scala │ ├── StringParsingSpec.scala │ ├── UUIDTest.scala │ └── difftool │ └── DiffTest.scala ├── jsoup ├── build.sbt └── src │ ├── main │ ├── resources │ │ └── META-INF │ │ │ └── services │ │ │ └── io.idml.functions.FunctionResolver │ └── scala │ │ └── io │ │ └── idml │ │ └── jsoup │ │ ├── IdmlJsoup.scala │ │ ├── JsoupConverter.scala │ │ ├── JsoupFunctionResolver.scala │ │ ├── ParseHtmlFunction.scala │ │ ├── ParseXmlFunction.scala │ │ └── StripTagsFunction.scala │ └── test │ ├── resources │ └── tests │ │ └── StripTagsSuite.json │ └── scala │ └── io │ └── idml │ └── jsoup │ ├── JsoupBench.scala │ ├── JsoupElementTest.scala │ └── JsoupItTest.scala ├── lang ├── build.sbt └── src │ ├── main │ ├── antlr4 │ │ ├── Mapping.g4 │ │ └── MappingTest.g4 │ └── scala │ │ └── io │ │ └── idml │ │ └── lang │ │ └── ThrowConsoleErrorListener.scala │ └── test │ ├── resources │ ├── atom.ini │ ├── literals.ini │ ├── regression │ │ ├── case-sensitive-ops.ini │ │ ├── coalesce-dynamic-paths.ini │ │ ├── fn-after-relative-path-filter.ini │ │ └── strings-with-quotes.ini │ └── support_ticket.ini │ └── scala │ └── io │ └── idml │ └── lang │ ├── MappingTestBase.scala │ └── ParseMapFileResourcesTest.scala ├── project ├── aether-deploy.sbt ├── assembly.sbt ├── build.properties ├── buildinfo.sbt ├── docsplugins.sbt ├── license.sbt ├── native-packager.sbt ├── proguard.sbt ├── sbt-antlr4.sbt ├── scalafmt.sbt └── sonatype.sbt ├── readme.md ├── repl ├── build.sbt └── src │ └── main │ └── scala │ └── io │ └── idmlrepl │ ├── JLineImpl.scala │ ├── Lexer.scala │ ├── Main.scala │ └── Repl.scala ├── test ├── build.sbt └── src │ └── test │ ├── resources │ ├── io.idml.ast │ │ ├── ArithmeticSuite.json │ │ ├── ArraysSuite.json │ │ ├── ArraysSuiteNestedArrays.ini │ │ ├── ArraysSuiteNestedObjects.ini │ │ ├── AssignmentsSuite.json │ │ ├── BackticksSuite.json │ │ ├── BooleanSuite.json │ │ ├── CoalesceSuite.json │ │ ├── FiltersSuite.json │ │ ├── IfSuite.json │ │ ├── IndexSuite.json │ │ ├── LiteralSuite.json │ │ ├── MatchSuite.json │ │ ├── ReassignmentSuite.json │ │ ├── SliceSuite.json │ │ ├── TemporaryVariableSuite.json │ │ ├── VariablesSuite.json │ │ └── WildcardsSuite.json │ ├── io.idml.functions │ │ ├── AppendSuite.json │ │ ├── ApplyArraySuite.json │ │ ├── ApplySuite.json │ │ ├── ArraySuite.json │ │ ├── BlacklistSuite.json │ │ ├── ConcatSuite.json │ │ ├── DataScienceArraysSuite.json │ │ ├── DataScienceNumbersSuite.json │ │ ├── DateSuite.json │ │ ├── EmailSuite.json │ │ ├── EmailSuiteMagicMethods.ini │ │ ├── EnumerateSuite.json │ │ ├── FilterSuite.json │ │ ├── FormatSuite.json │ │ ├── GetSizeFunctionSuite.json │ │ ├── GetSuite.json │ │ ├── GroupBySuite.json │ │ ├── GroupsBySuite.json │ │ ├── IndexOfSuite.json │ │ ├── IndexOfWithGet.ini │ │ ├── IsMatchSuite.json │ │ ├── LanguageNameSuite.json │ │ ├── LowercaseSuite.json │ │ ├── MapSuite.json │ │ ├── MatchSuite.json │ │ ├── MathsSuite.json │ │ ├── PrependSuite.json │ │ ├── RegexSuite.json │ │ ├── ReplaceSuite.json │ │ ├── SetFunctionsSuite.json │ │ ├── SetSizeFunctionSuite.json │ │ ├── SortSuite.json │ │ ├── SplitSuite.json │ │ ├── StripSuite.json │ │ ├── UniqueSuite.json │ │ ├── UppercaseSuite.json │ │ ├── UrlEncodeSuite.json │ │ ├── UrlSuite.json │ │ ├── UrlSuiteMagicMethods.ini │ │ ├── ZipSuite.ini │ │ └── ZipSuite.json │ └── mock_tests │ │ ├── load_me.ini │ │ ├── suite1_many.json │ │ ├── suite2_many.json │ │ ├── suite_chain_pending.json │ │ ├── suite_mapping_pending.json │ │ ├── suite_missing_object.json │ │ ├── suite_missing_tests.json │ │ ├── suite_shared_chain.json │ │ ├── suite_shared_mapping.json │ │ └── suite_single.json │ └── scala │ └── io │ └── idml │ ├── Base64Prop.scala │ ├── IdmlCirceProp.scala │ ├── IdmlScalaMeterBase.scala │ ├── IdmlScalaTestBase.scala │ ├── IdmlTestHarness.scala │ ├── IdmlTestHarnessTest.scala │ ├── UrlEncodeProp.scala │ ├── ast │ ├── AssignmentTest.scala │ └── AstItTest.scala │ └── functions │ ├── ApplyArrayFunctionTest.scala │ ├── ApplyFunctionTest.scala │ ├── ArrayFunctionTest.scala │ ├── AverageFunctionTest.scala │ ├── BuiltinFunctionResolverTest.scala │ ├── ExtractFunctionTest.scala │ ├── FunctionsItTest.scala │ ├── IdmlValueFunctionResolverTest.scala │ ├── IdmlValueFunctionTest.scala │ └── IdmlValueNAryFunctionResolverTest.scala ├── tool ├── build.sbt └── src │ ├── graal │ └── resource-config.json │ └── main │ ├── resources │ └── logback.xml │ └── scala │ └── io │ └── idml │ └── tool │ ├── DeclineHelpers.scala │ ├── IOCommandApp.scala │ ├── IdmlTool.scala │ ├── IdmlToolConfig.scala │ └── IdmlTools.scala └── utils ├── build.sbt └── src ├── main └── scala │ └── io │ └── idml │ └── utils │ ├── AutoComplete.scala │ ├── DocumentClassifier.scala │ ├── DocumentValidator.scala │ ├── FunctionReport.scala │ ├── PValueComparison.scala │ ├── Tracer.scala │ ├── configuration │ └── Pipeline.scala │ ├── folders │ └── Folders.scala │ ├── validators │ ├── MappingValidator.scala │ └── SchemaValidator.scala │ └── visitor │ ├── ExecNodeVisitor.scala │ └── VisitationStyle.scala └── test └── scala └── io └── idml └── utils ├── AutoCompleteSpec.scala ├── ConfigurationDslSpec.scala ├── DocumentClassifierTest.scala ├── DocumentValidatorTest.scala ├── PValueComparisonTest.scala └── TracerSpec.scala /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .idea 3 | dependency-reduced-pom.xml 4 | *.iml 5 | target/ 6 | lang/gen/ 7 | scalastyle-output.xml 8 | idml.jar 9 | .bloop 10 | .metals 11 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | runner.dialect = scala213 2 | version = 3.4.3 3 | align.preset = most 4 | maxColumn = 100 5 | rewrite.rules = [SortImports] 6 | danglingParentheses.preset = false 7 | -------------------------------------------------------------------------------- /BUILD.md: -------------------------------------------------------------------------------- 1 | Building 2 | ======== 3 | 4 | This project uses sbt to define it's structure and build processes. 5 | 6 | You can build all of the modules with `sbt package` 7 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | Contributions are always welcome and appreciated 5 | 6 | 1. Fork on GitHub 7 | 2. Create a feature branch (we use Gitflow for branching) 8 | 3. Commit your changes with tests 9 | 4. Run `sbt test` and ensure that the tests pass 10 | 5. Send a pull request against the relevant branch 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | from hseeberger/scala-sbt:8u212_1.2.8_2.13.0 as sbt 2 | copy . /build/ 3 | workdir /build/ 4 | run sbt "project tool" "docker:stage" 5 | workdir /build/tool/target/docker/stage 6 | 7 | FROM amazoncorretto:17-alpine3.18-full 8 | RUN apk update 9 | RUN apk add bash 10 | WORKDIR /opt/docker 11 | COPY --from=sbt --chown=daemon:daemon /build/tool/target/docker/stage/opt /opt 12 | EXPOSE 8081 13 | USER daemon 14 | ENTRYPOINT ["/opt/docker/bin/idml-tool"] 15 | CMD [] 16 | 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2013-2018 MediaSift Ltd 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /circe/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-circe" 2 | 3 | libraryDependencies ++= List( 4 | "io.circe" %% "circe-core" % "0.14.1", 5 | "io.circe" %% "circe-parser" % "0.14.1", 6 | "io.circe" %% "circe-testing" % "0.14.1" % "test", 7 | "org.scalatest" %% "scalatest" % "3.2.8" % "test" 8 | ) 9 | -------------------------------------------------------------------------------- /circe/src/main/resources/META-INF/services/io.idml.IdmlJson: -------------------------------------------------------------------------------- 1 | io.idml.circe.IdmlCirce -------------------------------------------------------------------------------- /circe/src/main/resources/META-INF/services/io.idml.functions.FunctionResolver: -------------------------------------------------------------------------------- 1 | io.idml.circe.CirceFunctions 2 | -------------------------------------------------------------------------------- /circe/src/main/scala/io/idml/circe/CirceFunctions.scala: -------------------------------------------------------------------------------- 1 | package io.idml.circe 2 | 3 | import io.idml.functions.json.JsonFunctions 4 | 5 | class CirceFunctions extends JsonFunctions(IdmlCirce) 6 | -------------------------------------------------------------------------------- /circe/src/main/scala/io/idml/circe/IdmlCirce.scala: -------------------------------------------------------------------------------- 1 | package io.idml.circe 2 | 3 | import io.circe.Json.Folder 4 | import io.circe._ 5 | import io.circe.syntax._ 6 | import io.idml.datanodes._ 7 | 8 | import scala.collection.mutable 9 | import scala.util.Try 10 | import cats._, cats.implicits._ 11 | 12 | import io.idml._ 13 | 14 | class IdmlCirce extends IdmlJson { 15 | import io.idml.circe.instances._ 16 | 17 | def read(in: String): Either[Throwable, IdmlValue] = 18 | io.circe.parser.decode[IdmlValue](in).leftMap[Throwable](e => new IdmlJsonReadingException(e)) 19 | 20 | /** Take a json string and transform it into a DataNode hierarchy */ 21 | override def parse(in: String): IdmlValue = read(in).toTry.get 22 | 23 | /** Take a json string and transform it into a DataNode hierarchy, if it's an object */ 24 | override def parseObject(in: String): IdmlObject = 25 | read(in) 26 | .flatMap { 27 | case o: IdmlObject => o.asRight 28 | case _ => (new IdmlJsonObjectException).asLeft 29 | } 30 | .toTry 31 | .get 32 | 33 | /** Render a DataNode hierarchy as compacted json */ 34 | override def compact(d: IdmlValue): String = d.asJson.noSpaces 35 | 36 | /** Render a DataNode hierarchy as pretty-printed json */ 37 | override def pretty(d: IdmlValue): String = d.asJson.spaces2 38 | } 39 | 40 | object IdmlCirce extends IdmlCirce 41 | -------------------------------------------------------------------------------- /circe/src/test/scala/io/idml/circe/CirceFunctionsSpec.scala: -------------------------------------------------------------------------------- 1 | package io.idml.circe 2 | 3 | import io.idml.json.JsonFunctionSuite 4 | 5 | class CirceFunctionsSpec extends JsonFunctionSuite("CirceFunctions", new CirceFunctions) 6 | -------------------------------------------------------------------------------- /circe/src/test/scala/io/idml/circe/IdmlCirceSpec.scala: -------------------------------------------------------------------------------- 1 | package io.idml.circe 2 | 3 | import io.idml.datanodes._ 4 | import org.scalatest.matchers.must.Matchers 5 | import org.scalatest.wordspec.AnyWordSpec 6 | import io.circe.syntax._ 7 | import io.idml.circe.instances._ 8 | 9 | class IdmlCirceSpec extends AnyWordSpec with Matchers { 10 | 11 | "IdmlCirce" should { 12 | "work" in { 13 | IdmlCirce.parse("""{"a":[1,2,3,"hello"]}""") must equal( 14 | IObject( 15 | "a" -> IArray( 16 | IInt(1), 17 | IInt(2), 18 | IInt(3), 19 | IString("hello") 20 | ) 21 | ) 22 | ) 23 | } 24 | "preserve key ordering" in { 25 | IObject( 26 | "a" -> IInt(1), 27 | "c" -> IInt(3), 28 | "b" -> IInt(2), 29 | "Z" -> IInt(0) 30 | ).asJson.noSpaces must equal( 31 | """{"Z":0,"a":1,"b":2,"c":3}""" 32 | ) 33 | } 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /core/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-core" 2 | 3 | libraryDependencies ++= Seq( 4 | "org.slf4j" % "slf4j-api" % "1.7.26", 5 | "org.tpolecat" %% "atto-core" % "0.9.4", 6 | "org.scalatest" %% "scalatest" % "3.2.8" % Test, 7 | "org.mockito" % "mockito-all" % "1.9.5" % Test, 8 | "org.scalatestplus" %% "mockito-3-4" % "3.2.8.0" % Test 9 | ) 10 | -------------------------------------------------------------------------------- /core/src/main/java/io/idml/AutoIdmlBuilder.java: -------------------------------------------------------------------------------- 1 | package io.idml; 2 | 3 | import java.util.Optional; 4 | 5 | public class AutoIdmlBuilder extends IdmlBuilder { 6 | public AutoIdmlBuilder() { 7 | super(Optional.of(new FunctionResolverService())); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /core/src/main/java/io/idml/StaticIdmlBuilder.java: -------------------------------------------------------------------------------- 1 | package io.idml; 2 | 3 | import io.idml.functions.FunctionResolver; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.Optional; 8 | 9 | public class StaticIdmlBuilder extends IdmlBuilder { 10 | List resolvers = new ArrayList(); 11 | 12 | public StaticIdmlBuilder() { 13 | super(Optional.empty()); 14 | List resolvers = new ArrayList<>(); 15 | this.functionResolver = Optional.of(new StaticFunctionResolverService(resolvers)); 16 | this.resolvers = resolvers; 17 | } 18 | 19 | public StaticIdmlBuilder(IdmlJson json) { 20 | super(Optional.empty()); 21 | resolvers.addAll(StaticFunctionResolverService.defaults(json)); 22 | this.functionResolver = Optional.of(new StaticFunctionResolverService(resolvers)); 23 | } 24 | 25 | public StaticIdmlBuilder withResolver(FunctionResolver resolver) { 26 | resolvers.add(resolver); 27 | this.functionResolver = Optional.of(new StaticFunctionResolverService(resolvers)); 28 | return this; 29 | } 30 | 31 | public StaticIdmlBuilder withResolverPrepend(FunctionResolver resolver) { 32 | resolvers.add(0, resolver); 33 | this.functionResolver = Optional.of(new StaticFunctionResolverService(resolvers)); 34 | return this; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/resources/META-INF/services/io.idml.functions.FunctionResolver: -------------------------------------------------------------------------------- 1 | io.idml.functions.BuiltinFunctionResolver 2 | io.idml.functions.IdmlValueFunctionResolver 3 | io.idml.functions.IdmlValueNaryFunctionResolver 4 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/IdmlChain.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import scala.collection.JavaConverters._ 4 | 5 | class IdmlChain(val transforms: Mapping*) extends Mapping { 6 | require(transforms != Nil, "Expected at least one transform") 7 | 8 | /** The first transform in the chain. This is the only one that will access input data */ 9 | protected val head = transforms.head 10 | 11 | /** The remaining transforms in the chain */ 12 | protected val tail = transforms.tail 13 | 14 | override def run(ctx: IdmlContext): IdmlContext = { 15 | ctx.enterChain() 16 | head.run(ctx) 17 | 18 | ctx.input = ctx.output 19 | ctx.scope = ctx.output 20 | ctx.cursor = ctx.output 21 | tail.foreach(_.run(ctx)) 22 | 23 | ctx.exitChain() 24 | 25 | ctx 26 | } 27 | 28 | } 29 | 30 | object IdmlChain { 31 | def of(transforms: java.util.List[Mapping]): IdmlChain = 32 | new IdmlChain(transforms.asScala.toSeq: _*) 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/IdmlException.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | /** Base class for an exception */ 4 | // scalastyle:off null 5 | abstract class IdmlException(msg: String = null, ex: Exception = null) 6 | extends RuntimeException(msg, ex) 7 | // scalastyle:on null 8 | 9 | /** The exception that is thrown when we couldn't resolve a function with this name and number of 10 | * parameters 11 | */ 12 | class UnknownFunctionException(msg: String) extends IdmlException(msg) 13 | 14 | /** The exception that is thrown when we asked to apply() a block that doesn't exist */ 15 | class UnknownBlockException(msg: String) extends IdmlException(msg) 16 | 17 | /** Thrown when there's no way to load functions.. this is probably a misconfiguration */ 18 | class NoFunctionResolversLoadedException(msg: String) extends IdmlException(msg) 19 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/IdmlListener.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.ast.{Assignment, Field, IdmlFunction, Maths, Pipeline} 4 | 5 | /** Hook into interpreter events */ 6 | abstract class IdmlListener { 7 | def enterAssignment(ctx: IdmlContext, assignment: Assignment): Unit 8 | 9 | def exitAssignment(ctx: IdmlContext, assignment: Assignment): Unit 10 | 11 | def enterChain(ctx: IdmlContext): Unit 12 | 13 | def exitChain(ctx: IdmlContext): Unit 14 | 15 | def enterPath(context: IdmlContext, path: Field): Unit 16 | 17 | def exitPath(context: IdmlContext, path: Field): Unit 18 | 19 | def enterPipl(context: IdmlContext, pipl: Pipeline): Unit 20 | 21 | def exitPipl(context: IdmlContext, pipl: Pipeline): Unit 22 | 23 | def enterFunc(ctx: IdmlContext, func: IdmlFunction): Unit 24 | 25 | def exitFunc(ctx: IdmlContext, func: IdmlFunction): Unit 26 | 27 | def enterMaths(context: IdmlContext, maths: Maths): Unit 28 | 29 | def exitMaths(context: IdmlContext, maths: Maths): Unit 30 | 31 | } 32 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/IdmlListenerBase.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.ast.{Assignment, Field, IdmlFunction, Maths, Pipeline} 4 | 5 | /** The implements all the IdmlListener methods with stubs. Extend at will 6 | */ 7 | class IdmlListenerBase extends IdmlListener { 8 | override def exitAssignment(ctx: IdmlContext, assignment: Assignment): Unit = () 9 | 10 | override def enterAssignment(ctx: IdmlContext, assignment: Assignment): Unit = () 11 | 12 | override def enterChain(ctx: IdmlContext): Unit = () 13 | 14 | override def exitChain(ctx: IdmlContext): Unit = () 15 | 16 | override def enterPath(context: IdmlContext, path: Field): Unit = () 17 | 18 | override def exitPath(context: IdmlContext, path: Field): Unit = () 19 | 20 | override def enterPipl(context: IdmlContext, pipl: Pipeline): Unit = () 21 | 22 | override def exitPipl(context: IdmlContext, pipl: Pipeline): Unit = () 23 | 24 | override def enterMaths(context: IdmlContext, maths: Maths): Unit = () 25 | 26 | override def exitMaths(context: IdmlContext, maths: Maths): Unit = () 27 | 28 | override def enterFunc(ctx: IdmlContext, func: IdmlFunction): Unit = () 29 | 30 | override def exitFunc(ctx: IdmlContext, func: IdmlFunction): Unit = () 31 | } 32 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/IdmlListenerWithState.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | /** Helper that adds state */ 4 | abstract class IdmlListenerWithState[A] extends IdmlListenerBase { 5 | 6 | /** Create a new starting state object */ 7 | protected def defaultState(ctx: IdmlContext): A 8 | 9 | /** Return the listener state */ 10 | def state(ctx: IdmlContext): A = { 11 | ctx.state.getOrElseUpdate(this.getClass, defaultState(ctx)).asInstanceOf[A] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/IdmlMapping.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.datanodes.IObject 4 | import io.idml.ast.Document 5 | import scala.collection.JavaConverters._ 6 | 7 | abstract class Mapping { 8 | def run(ctx: IdmlContext): IdmlContext 9 | def run(input: IdmlValue): IdmlObject = run(new IdmlContext(input, IObject())).output 10 | def run(input: IdmlValue, output: IdmlObject): IdmlObject = run( 11 | new IdmlContext(input, output)).output 12 | } 13 | 14 | object Mapping { 15 | import cats._, cats.data._, cats.implicits._ 16 | 17 | def fromMultipleMappings(ms: List[Mapping]): Mapping = 18 | fromMultipleMappings(ms.asJava) 19 | 20 | def fromMultipleMappings(ms: java.util.List[Mapping]): Mapping = 21 | (ctx: IdmlContext) => { 22 | val result = ms.asScala.toList.map { m => 23 | ctx.output = IObject() 24 | m.run(ctx) 25 | ctx.output.asInstanceOf[IObject] 26 | } 27 | ctx.output = 28 | NonEmptyList.fromList(result).map(_.reduceLeft(_ deepMerge _)).getOrElse(IObject()) 29 | ctx 30 | } 31 | } 32 | 33 | class IdmlMapping(val nodes: Document) extends Mapping { 34 | override def run(ctx: IdmlContext): IdmlContext = { 35 | ctx.doc = nodes 36 | nodes.invoke(ctx) 37 | ctx 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/PluginFunctionResolverService.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import java.net.{URL, URLClassLoader} 4 | import java.util.ServiceLoader 5 | 6 | import io.idml.functions.FunctionResolver 7 | 8 | /** A factory for functions that utilizes the Java ServiceLoader pattern, with a custom classloader 9 | */ 10 | class PluginFunctionResolverService(urls: Array[URL]) extends FunctionResolverService { 11 | 12 | // make a classloader with the URLs we got given 13 | protected val classLoader = new URLClassLoader(urls) 14 | 15 | /** Function resolver */ 16 | override protected val loader = ServiceLoader.load(classOf[FunctionResolver], classLoader) 17 | 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/ast/Coalesce.scala: -------------------------------------------------------------------------------- 1 | package io.idml.ast 2 | 3 | import io.idml.{EmptyCoalesce, IdmlContext, IdmlNothing} 4 | 5 | /** Try to execute a series of pipls and pick the first with a value */ 6 | case class Coalesce(exps: List[Pipeline]) extends Expression { 7 | def invoke(ctx: IdmlContext) { 8 | val tmp = ctx.scope 9 | ctx.scope = ctx.cursor 10 | ctx.cursor = exps.view 11 | .map(_.eval(ctx)) 12 | .find(!_.isInstanceOf[IdmlNothing]) 13 | .getOrElse(EmptyCoalesce) 14 | ctx.scope = tmp 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/ast/ExecNav.scala: -------------------------------------------------------------------------------- 1 | package io.idml.ast 2 | 3 | import io.idml.datanodes.IObject 4 | import io.idml.{IdmlContext, IdmlObject} 5 | 6 | /** Nav expressions set the cursor to a starting value */ 7 | trait ExecNav extends Expression 8 | 9 | /** The base of the expression is a literal value */ 10 | case class ExecNavLiteral(literal: Literal) extends ExecNav { 11 | def invoke(ctx: IdmlContext) { 12 | literal.invoke(ctx) 13 | } 14 | } 15 | 16 | /** The base of the expression is a variable */ 17 | case object ExecNavVariable extends ExecNav { 18 | def invoke(ctx: IdmlContext) { 19 | ctx.cursor = ctx.output 20 | } 21 | } 22 | 23 | /** The base of the expression is a relative path */ 24 | case object ExecNavRelative extends ExecNav { 25 | def invoke(ctx: IdmlContext) { 26 | ctx.cursor = ctx.scope 27 | } 28 | } 29 | 30 | /** The base of the expression is an absolute path */ 31 | case object ExecNavAbsolute extends ExecNav { 32 | def invoke(ctx: IdmlContext) { 33 | ctx.cursor = ctx.input 34 | } 35 | } 36 | 37 | case object ExecNavTemp extends ExecNav { 38 | def invoke(ctx: IdmlContext): Unit = { 39 | ctx.cursor = ctx.state.getOrElseUpdate(Variable.stateKey, IObject()).asInstanceOf[IdmlObject] 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/ast/Match.scala: -------------------------------------------------------------------------------- 1 | package io.idml.ast 2 | 3 | import io.idml.{EmptyCoalesce, IdmlContext, IdmlNothing, IdmlValue} 4 | 5 | case class Match(input: Pipeline, cases: List[Case]) extends Expression { 6 | override def invoke(ctx: IdmlContext): Unit = { 7 | val tmp = ctx.scope 8 | ctx.scope = input.eval(ctx) 9 | ctx.cursor = cases.view 10 | .find(_.matches(ctx)) 11 | .map(_.eval(ctx)) 12 | .getOrElse(EmptyCoalesce) 13 | ctx.scope = tmp 14 | } 15 | } 16 | 17 | case class Case(value: Predicate, result: Pipeline) extends Expression { 18 | def matches(ctx: IdmlContext): Boolean = value.predicate(ctx, ctx.cursor) 19 | 20 | override def invoke(ctx: IdmlContext): Unit = { 21 | if (matches(ctx)) { 22 | ctx.cursor = result.eval(ctx) 23 | } else { 24 | ctx.cursor = EmptyCoalesce 25 | } 26 | } 27 | } 28 | 29 | case class If(pred: Predicate, `then`: Pipeline, `else`: Option[Pipeline]) extends Expression { 30 | override def invoke(ctx: IdmlContext): Unit = { 31 | ctx.cursor = (pred.predicate(ctx, ctx.cursor), `else`) match { 32 | case (true, _) => `then`.eval(ctx) 33 | case (false, Some(other)) => other.eval(ctx) 34 | case _ => EmptyCoalesce 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/ast/Wildcard.scala: -------------------------------------------------------------------------------- 1 | package io.idml.ast 2 | 3 | import io.idml.datanodes.{IArray, IObject} 4 | import io.idml.{IdmlArray, IdmlContext, IdmlNothing, IdmlObject, IdmlValue, NoFields} 5 | 6 | import scala.collection.mutable 7 | 8 | /** Perform wildcard operations like a.*.b */ 9 | case class Wildcard(tail: Pipeline) extends Expression { 10 | 11 | def invokeForObject(ctx: IdmlContext, obj: IdmlObject) { 12 | val res = mutable.Map[String, IdmlValue]() 13 | obj.fields foreach { case (key, value) => 14 | ctx.cursor = value 15 | tail.eval(ctx) match { 16 | case n: IdmlNothing => () 17 | case n: Any => res(key) = n 18 | } 19 | } 20 | ctx.cursor = new IObject(res) 21 | } 22 | 23 | def invokeForArray(ctx: IdmlContext, arr: IdmlArray) { 24 | val res = mutable.Buffer[IdmlValue]() 25 | arr.items foreach { value => 26 | ctx.cursor = value 27 | tail.eval(ctx) match { 28 | case n: IdmlNothing => () 29 | case n: Any => res.append(n) 30 | } 31 | } 32 | ctx.cursor = new IArray(res) 33 | } 34 | 35 | def invoke(ctx: IdmlContext) { 36 | ctx.cursor match { 37 | case obj: IdmlObject => invokeForObject(ctx, obj) 38 | case arr: IdmlArray => invokeForArray(ctx, arr) 39 | case _ => ctx.cursor = NoFields 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/AppendFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IArray 4 | import io.idml.{IdmlArray, IdmlNothing, IdmlValue, InvalidCaller} 5 | import io.idml.ast.Pipeline 6 | 7 | /** Append an argument to the calling array if it evaluated successfully */ 8 | case class AppendFunction(arg: Pipeline) extends IdmlFunction1 { 9 | override def name: String = "append" 10 | 11 | override def apply(cursor: IdmlValue, input: IdmlValue): IdmlValue = { 12 | cursor match { 13 | case nothing: IdmlNothing => 14 | nothing 15 | case arr: IdmlArray => 16 | input match { 17 | case nothing: IdmlNothing => 18 | // Do nothing to the cursor 19 | arr 20 | case _ => 21 | arr.deepCopy 22 | // Create a new array with the original values and a new one on the end 23 | IArray(arr.items.map(_.deepCopy) :+ input) 24 | } 25 | case _ => 26 | InvalidCaller 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/ApplyArrayFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.ast.IdmlFunction 4 | import io.idml.{IdmlArray, IdmlContext, NoIndex} 5 | 6 | import scala.collection.immutable 7 | 8 | /** Invoke a named mapping block within the document */ 9 | case class ApplyArrayFunction(n: String) extends IdmlFunction { 10 | def name: String = "applyArray" 11 | def args: immutable.Nil.type = Nil 12 | 13 | /** Execute the underlying block, if it exists */ 14 | override def invoke(ctx: IdmlContext) { 15 | 16 | // Preserve the existing scope and output object 17 | val oldScope = ctx.scope 18 | val oldOutput = ctx.output 19 | 20 | val block = findBlock(ctx, n) 21 | 22 | ctx.enterFunc(this) 23 | 24 | ctx.cursor match { 25 | case array: IdmlArray => applyBlock(ctx, block, array) 26 | case _ => ctx.cursor = NoIndex 27 | } 28 | 29 | ctx.scope = oldScope 30 | ctx.output = oldOutput 31 | 32 | // FIXME: Note ambiguity of this event. Right now it's after the scope has been restored but could otherwise be before 33 | ctx.exitFunc(this) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/ApplyFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.ast.IdmlFunction 4 | import io.idml.{IdmlArray, IdmlContext, IdmlNothing} 5 | 6 | import scala.collection.immutable 7 | 8 | /** Invoke a named mapping block within the document */ 9 | case class ApplyFunction(n: String) extends IdmlFunction { 10 | def name: String = "apply" 11 | def args: immutable.Nil.type = Nil 12 | 13 | /** Execute the underlying block, if it exists */ 14 | override def invoke(ctx: IdmlContext) { 15 | // Preserve the existing scope and output object 16 | val oldScope = ctx.scope 17 | val oldOutput = ctx.output 18 | 19 | val block = findBlock(ctx, n) 20 | 21 | ctx.enterFunc(this) 22 | 23 | ctx.cursor match { 24 | case _: IdmlNothing => () 25 | case array: IdmlArray => applyBlockToArray(ctx, block, array) 26 | case _ => applyBlock(ctx, block, ctx.cursor) 27 | } 28 | 29 | ctx.scope = oldScope 30 | ctx.output = oldOutput 31 | 32 | ctx.exitFunc(this) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/ArrayFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IArray 4 | import io.idml.ast.{IdmlFunction, Node} 5 | import io.idml.{IdmlArray, IdmlContext, IdmlValue, InvalidCaller} 6 | 7 | import scala.collection.{immutable, mutable} 8 | 9 | /** Applies an expression to a series of nodes */ 10 | case class ArrayFunction(expr: Node) extends IdmlFunction { 11 | 12 | def args: immutable.Nil.type = Nil 13 | 14 | def name: String = "array" 15 | 16 | /** Applies the expression to each item in the cursor */ 17 | override def invoke(ctx: IdmlContext): Unit = { 18 | // Preserve context 19 | val oldScope = ctx.scope 20 | val oldOutput = ctx.output 21 | 22 | // Iterate items in the array 23 | ctx.cursor match { 24 | case array: IdmlArray => 25 | val results: mutable.Buffer[IdmlValue] = array.items.map { item => 26 | ctx.scope = item 27 | ctx.cursor = item 28 | expr.invoke(ctx) 29 | ctx.cursor 30 | } 31 | ctx.cursor = IArray(results) 32 | case _ => 33 | ctx.cursor = InvalidCaller 34 | } 35 | 36 | ctx.scope = oldScope 37 | ctx.output = oldOutput 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/AverageFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IDouble 4 | import io.idml.{IdmlArray, IdmlNothing, IdmlValue, InvalidCaller} 5 | 6 | /** Calculate an average value */ 7 | case object AverageFunction extends IdmlFunction0 { 8 | 9 | def name: String = "average" 10 | 11 | protected def apply(cursor: IdmlValue): IdmlValue = { 12 | cursor match { 13 | case nothing: IdmlNothing => 14 | nothing 15 | case array: IdmlArray if array.items.size > 0 => 16 | // FIXME: turn Idml into a numeric and replace this with sum() 17 | val sum = array.items.reduce((l, r) => l.+(r)) 18 | sum / IDouble(array.items.size) 19 | case other: Any => InvalidCaller 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/BlacklistFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.ast.{IdmlFunction, Pipeline} 4 | import io.idml.{IdmlContext, IdmlNothing, IdmlObject, InvalidParameters} 5 | 6 | case class BlacklistFunction(args: List[Pipeline]) extends IdmlFunction { 7 | 8 | /** Strip outs blacklisted fields */ 9 | override def invoke(ctx: IdmlContext): Unit = { 10 | ctx.cursor = ctx.cursor match { 11 | case nothing: IdmlNothing => 12 | nothing 13 | case obj: IdmlObject => 14 | val keys = args.flatMap(_.eval(ctx).toStringOption) 15 | if (keys.size != args.size) { 16 | InvalidParameters 17 | } else { 18 | val blacklisted = obj.deepCopy.asInstanceOf[IdmlObject] 19 | keys.foreach(blacklisted.fields.remove) 20 | blacklisted 21 | } 22 | } 23 | } 24 | 25 | override def name: String = "blacklist" 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/ConcatFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.{IArray, IDouble, IString} 4 | import io.idml.ast.{IdmlFunction, Pipeline} 5 | import io.idml._ 6 | 7 | import scala.collection.immutable 8 | 9 | case class ConcatFunction(sep: String) extends IdmlFunction { 10 | 11 | def name: String = "concat" 12 | 13 | def args: immutable.Nil.type = Nil 14 | 15 | override def invoke(ctx: IdmlContext): Unit = { 16 | ctx.cursor = ctx.cursor match { 17 | case IArray(items) => 18 | items 19 | .foldLeft(Option.empty[String]) { 20 | case (None, i: IdmlString) => Some(i.value) 21 | case (Some(a), i: IdmlString) => Some(a + sep + i.value) 22 | case (None, n: IdmlNothing) => None 23 | case (a @ Some(_), _) => a 24 | case (None, v: IdmlValue) => v.toStringOption 25 | } 26 | .map(IString.apply) 27 | .getOrElse(MissingField) 28 | case _ => InvalidCaller 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/ExtractFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IArray 4 | import io.idml.{IdmlArray, IdmlContext, IdmlNothing, IdmlValue, InvalidCaller, NoFields} 5 | import io.idml.ast.{IdmlFunction, Node} 6 | 7 | import scala.collection.{immutable, mutable} 8 | 9 | case class ExtractFunction(expr: Node) extends IdmlFunction { 10 | def args: immutable.Nil.type = Nil 11 | 12 | def name: String = "extract" 13 | 14 | protected def extractOpt(ctx: IdmlContext, item: IdmlValue): Option[IdmlValue] = { 15 | ctx.scope = item 16 | ctx.cursor = item 17 | expr.invoke(ctx) 18 | if (ctx.cursor.isInstanceOf[IdmlNothing]) { 19 | None 20 | } else { 21 | Some(ctx.cursor) 22 | } 23 | } 24 | 25 | /** Applies the expression to each item in the cursor */ 26 | override def invoke(ctx: IdmlContext): Unit = { 27 | // Preserve context 28 | val oldScope = ctx.scope 29 | val oldOutput = ctx.output 30 | 31 | // Iterate items in the array 32 | ctx.cursor match { 33 | case nothing: IdmlNothing => 34 | nothing 35 | case array: IdmlArray => 36 | val results: mutable.Buffer[IdmlValue] = 37 | array.items.flatMap(extractOpt(ctx, _)) 38 | if (results.nonEmpty) { 39 | ctx.cursor = IArray(results) 40 | } else { 41 | ctx.cursor = NoFields 42 | } 43 | case _ => 44 | ctx.cursor = InvalidCaller 45 | } 46 | 47 | ctx.scope = oldScope 48 | ctx.output = oldOutput 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/FilterFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IArray 4 | import io.idml._ 5 | import io.idml.ast._ 6 | 7 | import scala.collection.{immutable, mutable} 8 | 9 | case class FilterFunction(expr: Node) extends IdmlFunction { 10 | def args: immutable.Nil.type = Nil 11 | 12 | def name: String = "filter" 13 | 14 | protected def filterOpt(ctx: IdmlContext, item: IdmlValue): Option[IdmlValue] = { 15 | ctx.scope = item 16 | ctx.cursor = item 17 | expr.invoke(ctx) 18 | if (ctx.cursor.isInstanceOf[IdmlNothing] || ctx.cursor == Filtered || ctx.cursor.toBoolOption 19 | .contains(false)) { 20 | None 21 | } else { 22 | Some(ctx.cursor) 23 | } 24 | } 25 | 26 | override def invoke(ctx: IdmlContext): Unit = { 27 | // Preserve context 28 | val oldScope = ctx.scope 29 | val oldOutput = ctx.output 30 | 31 | // Iterate items in the array 32 | ctx.cursor match { 33 | case nothing: IdmlNothing => 34 | nothing 35 | case array: IdmlArray => 36 | val results: mutable.Buffer[IdmlValue] = 37 | array.items.flatMap(x => filterOpt(ctx, x)) 38 | if (results.nonEmpty) { 39 | ctx.cursor = IArray(results) 40 | } else { 41 | ctx.cursor = NoFields 42 | } 43 | case v: IdmlValue => 44 | ctx.cursor = filterOpt(ctx, v).getOrElse(Filtered) 45 | case _ => 46 | ctx.cursor = InvalidCaller 47 | } 48 | 49 | ctx.scope = oldScope 50 | ctx.output = oldOutput 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/FunctionResolver.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.ast.{Argument, IdmlFunction, IdmlFunctionMetadata, Pipeline} 4 | 5 | /** A class that may be able to resolve a function */ 6 | abstract class FunctionResolver { 7 | def providedFunctions(): List[IdmlFunctionMetadata] 8 | def resolve(name: String, args: List[Argument]): Option[IdmlFunction] 9 | } 10 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/GetSizeFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IInt 4 | import io.idml.{IdmlArray, IdmlContext, IdmlNothing, IdmlString, InvalidCaller} 5 | import io.idml.ast.IdmlFunction 6 | 7 | import scala.collection.immutable 8 | 9 | /** Get the size of something */ 10 | object GetSizeFunction extends IdmlFunction { 11 | override def name: String = "size" 12 | override def args: immutable.Nil.type = Nil 13 | 14 | override def invoke(ctx: IdmlContext): Unit = { 15 | ctx.cursor = ctx.cursor match { 16 | case nothing: IdmlNothing => 17 | nothing 18 | case array: IdmlArray => 19 | IInt(array.items.size) 20 | case string: IdmlString => 21 | IInt(string.value.length) 22 | case _ => 23 | InvalidCaller 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/IdmlFunction0.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.{IdmlContext, IdmlValue} 4 | import io.idml.ast.IdmlFunction 5 | 6 | /** Base implementation of a function with no parameters 7 | */ 8 | abstract class IdmlFunction0 extends IdmlFunction { 9 | 10 | /** The implementation of a variable-length function 11 | * 12 | * @param cursor 13 | * The call site 14 | * @return 15 | * The function return value 16 | */ 17 | protected def apply(cursor: IdmlValue): IdmlValue 18 | 19 | /** Invocation logic for handling variable-length functions 20 | * 21 | * @param ctx 22 | * The execution context 23 | */ 24 | override def invoke(ctx: IdmlContext): Unit = { 25 | ctx.enterFunc(this) 26 | ctx.cursor = apply(ctx.cursor) 27 | ctx.exitFunc(this) 28 | } 29 | 30 | val args = Nil 31 | } 32 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/IdmlFunction1.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.{IdmlContext, IdmlValue} 4 | import io.idml.ast.{IdmlFunction, Pipeline} 5 | 6 | /** Base implementation of a function with 3 parameters 7 | */ 8 | abstract class IdmlFunction1 extends IdmlFunction { 9 | 10 | /** The ast node for the parameter 11 | */ 12 | val arg: Pipeline 13 | 14 | /** The implementation of a variable-length function 15 | * 16 | * @param cursor 17 | * The call site 18 | * @param val1 19 | * The fully-evaluated first parameter 20 | * @return 21 | * The function return value 22 | */ 23 | protected def apply(cursor: IdmlValue, val1: IdmlValue): IdmlValue 24 | 25 | /** Invocation logic for handling variable-length functions 26 | * 27 | * @param ctx 28 | * The execution context 29 | */ 30 | override def invoke(ctx: IdmlContext): Unit = { 31 | ctx.enterFunc(this) 32 | val val1 = arg.eval(ctx) 33 | ctx.cursor = apply(ctx.cursor, val1) 34 | ctx.exitFunc(this) 35 | } 36 | 37 | val args = List(arg) 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/IdmlFunction2.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.{IdmlContext, IdmlValue} 4 | import io.idml.ast.{IdmlFunction, Pipeline} 5 | 6 | /** Base implementation of a function with 3 parameters 7 | */ 8 | abstract class IdmlFunction2 extends IdmlFunction { 9 | 10 | /** The ast node for the first parameters 11 | */ 12 | val arg1: Pipeline 13 | 14 | /** The ast node for the second parameters 15 | */ 16 | val arg2: Pipeline 17 | 18 | val args = List(arg1, arg2) 19 | 20 | /** The implementation of a variable-length function 21 | * 22 | * @param cursor 23 | * The call site 24 | * @param val1 25 | * The fully-evaluated first parameter 26 | * @param val2 27 | * The fully-evaluated second parameter 28 | * @return 29 | * The function return value 30 | */ 31 | protected def apply(cursor: IdmlValue, val1: IdmlValue, val2: IdmlValue): IdmlValue 32 | 33 | /** Invocation logic for handling variable-length functions 34 | * 35 | * @param ctx 36 | * The execution context 37 | */ 38 | override def invoke(ctx: IdmlContext): Unit = { 39 | ctx.enterFunc(this) 40 | val val1 = arg1.eval(ctx) 41 | val val2 = arg2.eval(ctx) 42 | ctx.cursor = apply(ctx.cursor, val1, val2) 43 | ctx.exitFunc(this) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/IdmlFunction3.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.{IdmlContext, IdmlValue} 4 | import io.idml.ast.{IdmlFunction, Pipeline} 5 | 6 | /** Base implementation of a function with 3 parameters 7 | */ 8 | abstract class IdmlFunction3 extends IdmlFunction { 9 | 10 | /** The ast node for the first parameters 11 | */ 12 | val arg1: Pipeline 13 | 14 | /** The ast node for the second parameters 15 | */ 16 | val arg2: Pipeline 17 | 18 | /** The ast node for the third parameters 19 | */ 20 | val arg3: Pipeline 21 | 22 | val args = List(arg1, arg2, arg3) 23 | 24 | /** The implementation of a variable-length function 25 | * 26 | * @param cursor 27 | * The call site 28 | * @param val1 29 | * The fully-evaluated first parameter 30 | * @param val2 31 | * The fully-evaluated second parameter 32 | * @param val3 33 | * The fully-evaluated parameter parameter 34 | * @return 35 | * The function return value 36 | */ 37 | protected def apply( 38 | cursor: IdmlValue, 39 | val1: IdmlValue, 40 | val2: IdmlValue, 41 | val3: IdmlValue): IdmlValue 42 | 43 | /** Invocation logic for handling variable-length functions 44 | * 45 | * @param ctx 46 | * The execution context 47 | */ 48 | override def invoke(ctx: IdmlContext): Unit = { 49 | ctx.enterFunc(this) 50 | val val1 = arg1.eval(ctx) 51 | val val2 = arg2.eval(ctx) 52 | val val3 = arg3.eval(ctx) 53 | ctx.cursor = apply(ctx.cursor, val1, val2, val3) 54 | ctx.exitFunc(this) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/IdmlFunctionN.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.{IdmlContext, IdmlValue} 4 | import io.idml.ast.{IdmlFunction, Pipeline} 5 | 6 | /** Base implementation of a function with a variable parameter list 7 | */ 8 | abstract class IdmlFunctionN extends IdmlFunction { 9 | 10 | /** The ast nodes for the function arguments 11 | */ 12 | val args: List[Pipeline] 13 | 14 | /** The implementation of a variable-length function 15 | * 16 | * @param cursor 17 | * The call site 18 | * @param args 19 | * The fully-evaluated arguments 20 | * @return 21 | * The function return value 22 | */ 23 | protected def apply(cursor: IdmlValue, args: Seq[IdmlValue]): IdmlValue 24 | 25 | /** Invocation logic for handling variable-length functions 26 | * 27 | * @param ctx 28 | * The execution context 29 | */ 30 | override def invoke(ctx: IdmlContext): Unit = { 31 | ctx.enterFunc(this) 32 | val results = args.map(_.eval(ctx)) 33 | ctx.cursor = apply(ctx.cursor, results) 34 | ctx.exitFunc(this) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/IdmlValueFunctionResolver.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import java.lang.reflect.Modifier 4 | 5 | import io.idml.IdmlValue 6 | import io.idml.ast.{Argument, IdmlFunctionMetadata, Pipeline} 7 | 8 | import scala.collection.JavaConverters._ 9 | 10 | /** Companion object for IdmlValueFunction */ 11 | class IdmlValueFunctionResolver extends FunctionResolver { 12 | 13 | /** An extractor that acts as a piece of syntactic sugar for evaluating IdmlValue functions */ 14 | def resolve(name: String, args: List[Argument]): Option[IdmlValueFunction] = 15 | args match { 16 | case as if as.forall(_.isInstanceOf[Pipeline]) => 17 | try { 18 | val classes = as.map(x => classOf[IdmlValue]) 19 | val method = classOf[IdmlValue].getMethod(name, classes: _*) 20 | try Some(IdmlValueFunction(method, as.asInstanceOf[List[Pipeline]])) 21 | catch { 22 | case _: IllegalArgumentException => None 23 | } 24 | } catch { 25 | case _: NoSuchMethodException => None 26 | } 27 | case _ => None 28 | } 29 | 30 | lazy val functions = classOf[IdmlValue] 31 | .getMethods() 32 | .filter(_.getReturnType == classOf[IdmlValue]) 33 | .map { m => 34 | IdmlFunctionMetadata(m.getName, m.getParameters.map(_.getName -> "").toList, "") 35 | } 36 | .toSet 37 | .toList 38 | override def providedFunctions(): List[IdmlFunctionMetadata] = functions 39 | } 40 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/IdmlValueNaryFunctionResolver.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.ast.{Argument, IdmlFunctionMetadata, Pipeline} 4 | import io.idml.IdmlValue 5 | 6 | /** Alternative matcher to create IdmlValueFunction objects for functions which take N arguments */ 7 | class IdmlValueNaryFunctionResolver extends FunctionResolver { 8 | 9 | /** An extractor that acts as a piece of syntactic sugar for evaluating IdmlValue functions */ 10 | def resolve(name: String, args: List[Argument]): Option[IdmlValueFunction] = 11 | args match { 12 | case as if as.forall(_.isInstanceOf[Pipeline]) => 13 | try { 14 | val method = 15 | classOf[IdmlValue].getMethod(name, classOf[Seq[IdmlValue]]) 16 | try Some(IdmlValueFunction(method, as.asInstanceOf[List[Pipeline]], isNAry = true)) 17 | catch { 18 | case _: IllegalArgumentException => None 19 | } 20 | } catch { 21 | case _: NoSuchMethodException => None 22 | } 23 | case _ => None 24 | 25 | } 26 | override def providedFunctions(): List[IdmlFunctionMetadata] = List.empty // todo 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/MapFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IArray 4 | import io.idml.ast.{IdmlFunction, Node} 5 | import io.idml._ 6 | 7 | import scala.collection.{immutable, mutable} 8 | 9 | /* 10 | * This is functionally the same as ExtractFunction, but is kept as a separate type for usability reasons 11 | */ 12 | case class MapFunction(expr: Node) extends IdmlFunction { 13 | def args: immutable.Nil.type = Nil 14 | 15 | def name: String = "map" 16 | 17 | protected def extractOpt(ctx: IdmlContext, item: IdmlValue): Option[IdmlValue] = { 18 | ctx.scope = item 19 | ctx.cursor = item 20 | expr.invoke(ctx) 21 | if (ctx.cursor.isInstanceOf[IdmlNothing]) { 22 | None 23 | } else { 24 | Some(ctx.cursor) 25 | } 26 | } 27 | 28 | /** Applies the expression to each item in the cursor */ 29 | override def invoke(ctx: IdmlContext): Unit = { 30 | // Preserve context 31 | val oldScope = ctx.scope 32 | val oldOutput = ctx.output 33 | 34 | // Iterate items in the array 35 | ctx.cursor match { 36 | case nothing: IdmlNothing => 37 | nothing 38 | case array: IdmlArray => 39 | val results: mutable.Buffer[IdmlValue] = 40 | array.items.flatMap(extractOpt(ctx, _)) 41 | if (results.nonEmpty) { 42 | ctx.cursor = IArray(results) 43 | } else { 44 | ctx.cursor = NoFields 45 | } 46 | case _ => 47 | ctx.cursor = InvalidCaller 48 | } 49 | 50 | ctx.scope = oldScope 51 | ctx.output = oldOutput 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/PrependFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IArray 4 | import io.idml.{IdmlArray, IdmlNothing, IdmlValue, InvalidCaller} 5 | import io.idml.ast.Pipeline 6 | 7 | /** Prepend an argument to the calling array if it evaluated successfully */ 8 | case class PrependFunction(arg: Pipeline) extends IdmlFunction1 { 9 | override def name: String = "prepend" 10 | 11 | protected def apply(cursor: IdmlValue, val1: IdmlValue): IdmlValue = { 12 | cursor match { 13 | case nothing: IdmlNothing => 14 | nothing 15 | case arr: IdmlArray => 16 | val1 match { 17 | case nothing: IdmlNothing => 18 | // Do nothing to the cursor 19 | arr 20 | case prependee: Any => 21 | arr.deepCopy 22 | // Create a new array with the original values and a new one on the end 23 | IArray(prependee +: arr.items.map(_.deepCopy)) 24 | } 25 | case other: Any => 26 | InvalidCaller 27 | } 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/SetSizeFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IString 4 | import io.idml.{IdmlArray, IdmlContext, IdmlNothing, IdmlString, InvalidCaller, InvalidParameters} 5 | import io.idml.ast.{IdmlFunction, Pipeline} 6 | 7 | import scala.collection.immutable 8 | 9 | /** Set the size of something */ 10 | case class SetSizeFunction(arg: Pipeline) extends IdmlFunction { 11 | override def name: String = "size" 12 | 13 | override def args: immutable.Nil.type = Nil 14 | 15 | override def invoke(ctx: IdmlContext): Unit = { 16 | arg.eval(ctx).toIntOption.filter(_ >= 0) match { 17 | case Some(maxSize) => 18 | ctx.cursor = ctx.cursor match { 19 | case nothing: IdmlNothing => 20 | nothing 21 | case array: IdmlArray => 22 | array.slice(None, Some(maxSize)) 23 | case string: IdmlString if string.value == "" || string.value.length <= maxSize => 24 | string 25 | case string: IdmlString => 26 | IString(string.value.substring(0, maxSize)) 27 | case other: Any => 28 | InvalidCaller 29 | } 30 | case None => 31 | ctx.cursor = InvalidParameters 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/SortFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IArray 4 | import io.idml.ast.{IdmlFunction, Node} 5 | import io.idml._ 6 | 7 | import scala.collection.{immutable, mutable} 8 | 9 | case class SortFunction(expr: Node) extends IdmlFunction { 10 | def args: immutable.Nil.type = Nil 11 | 12 | def name: String = "sort" 13 | 14 | protected def extractOpt(ctx: IdmlContext, item: IdmlValue): Option[IdmlValue] = { 15 | ctx.scope = item 16 | ctx.cursor = item 17 | expr.invoke(ctx) 18 | if (ctx.cursor.isInstanceOf[IdmlNothing]) { 19 | None 20 | } else { 21 | Some(ctx.cursor) 22 | } 23 | } 24 | 25 | override def invoke(ctx: IdmlContext): Unit = { 26 | // Preserve context 27 | val oldScope = ctx.scope 28 | val oldOutput = ctx.output 29 | 30 | // Iterate items in the array 31 | ctx.cursor match { 32 | case nothing: IdmlNothing => 33 | nothing 34 | case array: IdmlArray => 35 | val results: mutable.Buffer[IdmlValue] = 36 | array.items.flatMap(x => extractOpt(ctx, x).map(x -> _)).sortBy(_._2).map(_._1) 37 | if (results.nonEmpty) { 38 | ctx.cursor = IArray(results) 39 | } else { 40 | ctx.cursor = NoFields 41 | } 42 | case _ => 43 | ctx.cursor = InvalidCaller 44 | } 45 | 46 | ctx.scope = oldScope 47 | ctx.output = oldOutput 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/json/ObjectModuleJson.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions.json 2 | 3 | import io.idml._ 4 | import io.idml.ast.Pipeline 5 | import io.idml.functions.IdmlFunction0 6 | 7 | class ObjectModuleJson(json: IdmlJson) { 8 | def serialize(input: IdmlValue): IdmlValue = 9 | input match { 10 | case o: IdmlObject => IdmlValue(json.compact(o)) 11 | case _ => InvalidCaller 12 | } 13 | 14 | val serializeFunction: IdmlFunction0 = new IdmlFunction0 { 15 | override protected def apply(cursor: IdmlValue): IdmlValue = serialize(cursor) 16 | override def name: String = "serialize" 17 | } 18 | 19 | def parseJson(input: IdmlValue): IdmlValue = 20 | input match { 21 | case s: IdmlString => json.parseEither(s.value).getOrElse(InvalidCaller) 22 | case _ => InvalidCaller 23 | } 24 | 25 | val parseJsonFunction: IdmlFunction0 = new IdmlFunction0 { 26 | override protected def apply(cursor: IdmlValue): IdmlValue = 27 | parseJson(cursor) 28 | override def name: String = "parseJson" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/src/main/scala/io/idml/functions/json/UUIDModuleJson.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions.json 2 | 3 | import java.nio.charset.StandardCharsets.UTF_8 4 | import java.util.UUID 5 | 6 | import io.idml.datanodes.IString 7 | import io.idml.functions.IdmlFunction0 8 | import io.idml.utils.IdmlUUID 9 | import io.idml.{CastUnsupported, IdmlJson, IdmlObject, IdmlString, IdmlValue} 10 | 11 | class UUIDModuleJson(json: IdmlJson) extends ObjectModuleJson(json) { 12 | 13 | val uuid3Function: IdmlFunction0 = new IdmlFunction0 { 14 | override protected def apply(cursor: IdmlValue): IdmlValue = 15 | cursor match { 16 | case o: IdmlObject => 17 | IString(UUID.nameUUIDFromBytes(serialize(o).toStringOption.get.getBytes(UTF_8)).toString) 18 | case n: IdmlString => IString(UUID.nameUUIDFromBytes(n.value.getBytes(UTF_8)).toString) 19 | case _ => CastUnsupported 20 | } 21 | override def name: String = "uuid3" 22 | } 23 | 24 | val uuid5Function: IdmlFunction0 = new IdmlFunction0 { 25 | override protected def apply(cursor: IdmlValue): IdmlValue = 26 | cursor match { 27 | case o: IdmlObject => 28 | IString( 29 | IdmlUUID.nameUUIDFromBytes5(serialize(o).toStringOption.get.getBytes(UTF_8)).toString) 30 | case n: IdmlString => IString(IdmlUUID.nameUUIDFromBytes5(n.value.getBytes(UTF_8)).toString) 31 | } 32 | override def name: String = "uuid5" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/test/resources/mock_resource.txt: -------------------------------------------------------------------------------- 1 | This only exists so ResourceResolver can be integration tested -------------------------------------------------------------------------------- /core/src/test/scala/io/idml/IdmlChainItTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.datanodes.{IObject, ITrue} 4 | import org.scalatest.funsuite.AnyFunSuite 5 | 6 | class IdmlChainItTest extends AnyFunSuite { 7 | 8 | test("Test mapping chain order works properly") { 9 | val idml = Idml.autoBuilder().build() 10 | val chain = idml.chain( 11 | idml.compile("x = a"), 12 | idml.compile("y = x \n z = a") 13 | ) 14 | val output = chain.run(IObject("a" -> ITrue)) 15 | 16 | assert(output.get("x") === ITrue) 17 | assert(output.get("z") === MissingField) 18 | assert(output.get("y") === ITrue) 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /core/src/test/scala/io/idml/IdmlListenerWithStateTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import java.util.concurrent.atomic.AtomicInteger 4 | import org.scalatest.funsuite.AnyFunSuite 5 | import org.scalatestplus.mockito.MockitoSugar 6 | 7 | class IdmlListenerWithStateTest extends AnyFunSuite with MockitoSugar { 8 | 9 | /** A test implementation. Typically this would be a more complex object than an integer */ 10 | class TestImpl extends IdmlListenerWithState[AtomicInteger] { 11 | override protected def defaultState(ctx: IdmlContext) = 12 | new AtomicInteger(1) 13 | } 14 | 15 | test("The defaultState function provides a starting value") { 16 | val ctx = new IdmlContext() 17 | val listener = new TestImpl() 18 | assert(listener.state(ctx).get() === 1) 19 | } 20 | 21 | test("The state can be returned and updated") { 22 | val ctx = new IdmlContext() 23 | val listener = new TestImpl() 24 | listener.state(ctx).set(2) 25 | assert(listener.state(ctx).get() === 2) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /core/src/test/scala/io/idml/IdmlParserTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.lang.DocumentParseException 4 | import io.idml.ast._ 5 | import org.mockito.Answers 6 | import org.scalatest.funsuite.AnyFunSuite 7 | import org.scalatestplus.mockito.MockitoSugar 8 | 9 | class IdmlParserTest extends AnyFunSuite with MockitoSugar { 10 | 11 | test("Parses text") { 12 | new IdmlParser().parse(null, "a = b").nodes == Document( 13 | Map( 14 | "main" -> Block( 15 | "main", 16 | List(Assignment(List("a"), Pipeline(List(ExecNavRelative, Field("b"))))))) 17 | ) 18 | } 19 | 20 | test("Throws parse error when input is invalid") { 21 | intercept[DocumentParseException](new IdmlParser().parse(null, ":-D")) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /core/src/test/scala/io/idml/IdmlTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import org.scalatest.funsuite.AnyFunSuite 4 | import org.mockito.Mockito._ 5 | import org.scalatestplus.mockito.MockitoSugar 6 | 7 | import scala.collection.JavaConverters._ 8 | 9 | class IdmlTest extends AnyFunSuite with MockitoSugar { 10 | 11 | test("Passes strings to parser") { 12 | val parser = mock[IdmlParser] 13 | 14 | val funcs = mock[FunctionResolverService] 15 | val ptolemy = new Idml( 16 | parser, 17 | funcs, 18 | List.empty[IdmlListener].asJava 19 | ) 20 | 21 | ptolemy.compile("abc") 22 | 23 | verify(parser).parse(funcs, "abc") 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /datanodes/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-datanodes" 2 | 3 | libraryDependencies ++= Seq( 4 | "joda-time" % "joda-time" % "2.9.9", // FIXME: Create date and time plugin 5 | "org.joda" % "joda-convert" % "1.7", // FIXME: Create date and time plugin 6 | "javax.mail" % "mail" % "1.5.0-b01", // FIXME: Create email plugin 7 | "com.google.guava" % "guava" % "27.0-jre", 8 | "org.typelevel" %% "spire" % "0.17.0", 9 | "org.typelevel" %% "cats-core" % "2.6.0", 10 | "com.google.re2j" % "re2j" % "1.2", 11 | "org.scalatest" %% "scalatest" % "3.2.8" % Test 12 | ) 13 | -------------------------------------------------------------------------------- /datanodes/src/main/java/io/idml/utils/IdmlUUID.java: -------------------------------------------------------------------------------- 1 | package io.idml.utils; 2 | 3 | import java.security.MessageDigest; 4 | import java.security.NoSuchAlgorithmException; 5 | import java.util.UUID; 6 | 7 | /** 8 | * This class exists because all the fun parts of java.util.UUID are private and I can't copy paste the logic into scala 9 | * reliably. 10 | * 11 | * This code is from OpenJDK and this file, and this file only, is licensed under the GPL with linking exception 12 | */ 13 | public class IdmlUUID { 14 | public static UUID nameUUIDFromBytes5(byte[] name) { 15 | MessageDigest md; 16 | try { 17 | md = MessageDigest.getInstance("SHA-1"); 18 | } catch (NoSuchAlgorithmException nsae) { 19 | throw new InternalError("SHA-1 not supported", nsae); 20 | } 21 | byte[] md5Bytes = md.digest(name); 22 | md5Bytes[6] &= 0x0f; /* clear version */ 23 | md5Bytes[6] |= 0x50; /* set to version 5 */ 24 | md5Bytes[8] &= 0x3f; /* clear variant */ 25 | md5Bytes[8] |= 0x80; /* set to IETF variant */ 26 | return create(md5Bytes); 27 | } 28 | 29 | public static UUID create(byte[] data) { 30 | long msb = 0; 31 | long lsb = 0; 32 | assert data.length == 16 : "data must be 16 bytes in length"; 33 | for (int i = 0; i < 8; i++) 34 | msb = (msb << 8) | (data[i] & 0xff); 35 | for (int i = 8; i < 16; i++) 36 | lsb = (lsb << 8) | (data[i] & 0xff); 37 | return new UUID(msb, lsb); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/FieldTypeCounter.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | /** Counts fields of particular types */ 4 | case class FieldTypeCounter( 5 | var nothing: Int = 0, 6 | var strings: Int = 0, 7 | var doubles: Int = 0, 8 | var ints: Int = 0, 9 | var bools: Int = 0, 10 | var objects: Int = 0, 11 | var arrays: Int = 0, 12 | var nulls: Int = 0 13 | ) extends IdmlValueVisitor { 14 | 15 | override def visitNothing(path: Seq[String], n: IdmlNothing): Unit = { 16 | nothing += 1 17 | super.visitNothing(path, n) 18 | } 19 | 20 | override def visitString(path: Seq[String], s: IdmlString): Unit = { 21 | strings += 1 22 | super.visitString(path, s) 23 | } 24 | 25 | override def visitDouble(path: Seq[String], d: IdmlDouble): Unit = { 26 | doubles += 1 27 | super.visitDouble(path, d) 28 | } 29 | 30 | override def visitInt(path: Seq[String], i: IdmlInt): Unit = { 31 | ints += 1 32 | super.visitInt(path, i) 33 | } 34 | 35 | override def visitBool(path: Seq[String], b: IdmlBool): Unit = { 36 | bools += 1 37 | super.visitBool(path, b) 38 | } 39 | 40 | override def visitNull(path: Seq[String]): Unit = { 41 | nulls += 1 42 | super.visitNull(path) 43 | } 44 | 45 | override def visitObject(path: Seq[String], obj: IdmlObject) { 46 | objects += 1 47 | super.visitObject(path, obj) 48 | } 49 | 50 | override def visitArray(path: Seq[String], array: IdmlArray) { 51 | arrays += 1 52 | super.visitArray(path, array) 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/IdmlBool.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | /** The IdmlValue that contains boolean values */ 4 | trait IdmlBool extends IdmlValue { 5 | 6 | def formatValue: Boolean = value 7 | 8 | override def bool(): IdmlBool = this 9 | 10 | /** The boolean value of this IdmlValue */ 11 | def value: Boolean 12 | 13 | override def equals(o: Any): Boolean = 14 | o match { 15 | case n: IdmlBool => n.value == value 16 | case _ => false 17 | } 18 | 19 | override def toBoolOption: Some[Boolean] = Some(value) 20 | 21 | override def hashCode(): Int = value.hashCode() 22 | 23 | override def toStringOption: Option[String] = if (value) Some("true") else Some("false") 24 | 25 | } 26 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/IdmlDouble.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.datanodes.{IBool, IFalse, ITrue} 4 | 5 | /** The IdmlValue that contains floating point numbers */ 6 | trait IdmlDouble extends IdmlValue { 7 | 8 | def formatValue: Double = value 9 | 10 | /** The floating point number for this IdmlValue */ 11 | def value: Double 12 | 13 | override def equals(o: Any): Boolean = 14 | o match { 15 | case n: IdmlDouble => n.value == value 16 | case n: IdmlInt => n.value == value 17 | case _ => false 18 | } 19 | override def hashCode(): Int = value.hashCode() 20 | 21 | override def bool(): IBool = if (value == 0) IFalse else ITrue 22 | } 23 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/IdmlInt.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.datanodes.{IBool, IFalse, IString, ITrue} 4 | 5 | /** The IdmlValue for containing natural numbers */ 6 | trait IdmlInt extends IdmlValue { 7 | 8 | @deprecated(message = "Use toStringOption and related functions instead", since = "1.3.0") 9 | def formatValue: Long = value 10 | 11 | /** The natural number for this IdmlValue */ 12 | def value: Long 13 | 14 | override def equals(o: Any): Boolean = 15 | o match { 16 | case n: IdmlDouble => n.value == value 17 | case n: IdmlInt => n.value == value 18 | case _ => false 19 | } 20 | 21 | override def string(): IString = IString(value.toString) 22 | 23 | override def bool(): IBool = if (value == 0) IFalse else ITrue 24 | 25 | override def hashCode(): Int = value.hashCode() 26 | 27 | override def toStringOption: Some[String] = Some(value.toString) 28 | } 29 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/IdmlNull.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | /** The null value */ 4 | case object IdmlNull extends IdmlValue 5 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/IdmlObject.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.datanodes.IBool 4 | 5 | import scala.collection.mutable 6 | 7 | /** The IdmlValue that represents objects */ 8 | abstract class IdmlObject extends IdmlValue { 9 | 10 | /** The underlying field container for this object */ 11 | def fields: mutable.Map[String, IdmlValue] 12 | 13 | override def equals(o: Any): Boolean = 14 | o match { 15 | case n: IdmlObject => n.fields == fields 16 | case _ => false 17 | } 18 | 19 | override def hashCode(): Int = fields.hashCode() 20 | 21 | /** Iterate over values without keys */ 22 | override def iterator: Iterator[IdmlValue] = fields.values.iterator 23 | 24 | /** Get fields if present */ 25 | override def get(name: String): IdmlValue = { 26 | fields.getOrElse(name, MissingField) 27 | } 28 | 29 | /** Remove fields if able */ 30 | override def remove(name: String) { 31 | fields.remove(name) 32 | } 33 | 34 | /** True if we have no fields */ 35 | override def isEmpty: IBool = IBool(fields.isEmpty) 36 | 37 | /** No-op as we are already an object */ 38 | // scalastyle:off method.name 39 | override def `object`(): IdmlObject = this 40 | // scalastyle:on method.name 41 | 42 | override def toStringOption: Option[String] = 43 | Some( 44 | "{" + fields.toList 45 | .sortBy(_._1) 46 | .flatMap { case (k, v) => v.toStringOption.map(k -> _) } 47 | .map { case (k, v) => s""""$k":$v""" } 48 | .mkString(",") + "}" 49 | ) 50 | 51 | } 52 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/JIdmlValue.java: -------------------------------------------------------------------------------- 1 | package io.idml; 2 | 3 | import io.idml.datanodes.*; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * The java-friendly API for constructing IdmlValues 9 | * 10 | * You'll find `of` methods which allow you to construct the AST types from java 11 | * 12 | * The corresponding `asX` methods are on IdmlValue itself, and return Optionals 13 | */ 14 | public class JIdmlValue { 15 | public static IdmlValue of(int i) { 16 | return new IInt(i); 17 | } 18 | public static IdmlValue of(double d) { 19 | return new IDouble(d); 20 | } 21 | public static IdmlValue of(long l) { 22 | return new IInt(l); 23 | } 24 | public static IdmlValue of(boolean b) { 25 | return new IBool(b); 26 | } 27 | public static IdmlValue of(IdmlValue... v) { 28 | return IArray.of(v); 29 | } 30 | public static IdmlValue of(Map kv) { 31 | return IObject.of(kv); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/CompositeValue.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.IdmlValue 4 | 5 | /** Indicates we should not track sub-paths of this value */ 6 | trait CompositeValue extends IdmlValue 7 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/IArray.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.{IdmlArray, IdmlValue} 4 | 5 | import scala.collection.mutable 6 | 7 | object IArray { 8 | val empty = IArray() 9 | 10 | /** Construct an array from zero or more items */ 11 | def apply(items: IdmlValue*): IArray = { 12 | IArray(mutable.Buffer(items: _*)) 13 | } 14 | 15 | /** Extractor for PArray */ 16 | def unapply(value: IdmlValue): Option[Seq[IdmlValue]] = 17 | value match { 18 | case arr: IArray => Some(arr.items.toSeq) 19 | case _ => None 20 | } 21 | 22 | def of(arr: Array[IdmlValue]): IArray = { 23 | IArray(arr.toBuffer) 24 | } 25 | } 26 | 27 | /** The standard implementation of an array. Encapsulates a mutable array buffer */ 28 | case class IArray(var items: mutable.Buffer[IdmlValue]) extends IdmlArray { 29 | 30 | /** Clone this structure by making a deep copy of every element */ 31 | override def deepCopy: IArray = new IArray(items.map(_.deepCopy)) 32 | } 33 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/IBool.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.IdmlBool 4 | 5 | /** The default IdmlValue implementation of a boolean */ 6 | case class IBool(value: Boolean) extends IdmlBool { 7 | 8 | // scalastyle:off method.name 9 | def ||(r: IBool): Boolean = value || r.value 10 | // scalastyle:on method.name 11 | 12 | } 13 | 14 | /** The PBool for true */ 15 | object ITrue extends IBool(true) 16 | 17 | /** The PBool for false */ 18 | object IFalse extends IBool(false) 19 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/IDate.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.datanodes.modules.DateModule 4 | import io.idml.IdmlString 5 | import org.joda.time.DateTime 6 | import org.joda.time.format.DateTimeFormatter 7 | 8 | /** */ 9 | case class IDate(dateVal: DateTime, format: DateTimeFormatter = DateModule.DefaultDateFormat) 10 | extends IdmlString 11 | with CompositeValue { 12 | def value: String = dateVal.toString(format) 13 | override def int(): IInt = IInt(dateVal.getMillis()) 14 | } 15 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/IDouble.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.{IdmlDouble, IdmlValue} 4 | 5 | /** The default IdmlValue implementation of a floating point number */ 6 | case class IDouble(value: Double) extends IdmlDouble { 7 | 8 | /** Transform this value into a floating point number */ 9 | override def float(): IDouble = this 10 | 11 | /** Transform this value into a natural number */ 12 | override def int(): IdmlValue = IdmlValue(value.toLong) 13 | 14 | override def toDoubleOption = Some(value) 15 | } 16 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/IInt.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.{IdmlInt, IdmlValue} 4 | 5 | /** The default IdmlValue implementation of an integer */ 6 | case class IInt(value: Long) extends IdmlInt { 7 | 8 | /** Transform this value into a natural number */ 9 | override def int(): IInt = this 10 | 11 | /** Transform this value into a floating point */ 12 | override def float(): IdmlValue = IdmlValue(value.toDouble) 13 | 14 | // FIXME: PInt is currently implemented as PLong! 15 | override def toIntOption: Some[Int] = Some(value.toInt) 16 | 17 | override def toLongOption: Some[Long] = Some(value) 18 | 19 | override def toDoubleOption = Some(value.toDouble) 20 | } 21 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/IString.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.IdmlString 4 | 5 | /** The standard implementation of a string */ 6 | case class IString(value: String) extends IdmlString 7 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/IUrl.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import java.net.URL 4 | 5 | import io.idml.{IdmlString, IdmlValue, MissingField} 6 | 7 | /** Represents a valid URL */ 8 | class IUrl(url: URL) extends IdmlString with CompositeValue { 9 | 10 | /** The URL represented as a string */ 11 | val value: String = url.toString 12 | 13 | /** Override get(..) to provide custom field accessors */ 14 | override def get(name: String): IdmlValue = 15 | name match { 16 | case "host" => IdmlValue(url.getHost) 17 | case "protocol" => IdmlValue(url.getProtocol) 18 | case "query" => IdmlValue(url.getQuery) 19 | case "path" => IdmlValue(url.getPath) 20 | case _ => MissingField 21 | } 22 | 23 | override def toString: String = value 24 | } 25 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/SgmlNode.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.{IdmlArray, IdmlObject, IdmlValue} 4 | 5 | import scala.collection.mutable 6 | 7 | case class SgmlNode( 8 | name: String, 9 | items: mutable.Buffer[IdmlValue], 10 | attrs: Map[String, String], 11 | body: String) 12 | extends IdmlObject 13 | with IdmlArray 14 | with CompositeValue { 15 | lazy val contents: mutable.Map[String, IdmlValue] = mutable.Map(attrs.toList.map { case (k, v) => 16 | k -> IdmlValue(v) 17 | }: _*) 18 | 19 | def fields: mutable.Map[String, IdmlValue] = contents 20 | } 21 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/modules/DomModule.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes.modules 2 | 3 | import io.idml.{CastUnsupported, IdmlValue, MissingField} 4 | import io.idml.datanodes.{IDomElement, IDomNode, IString} 5 | 6 | trait DomModule { 7 | this: IdmlValue => 8 | 9 | def tagName: IdmlValue = 10 | this match { 11 | case e: IDomElement => Option(e.name).map(IString).getOrElse(MissingField) 12 | case _ => CastUnsupported 13 | } 14 | 15 | // This is overridden in the DOM element type 16 | def attributes(): IdmlValue = MissingField 17 | 18 | def text: IdmlValue = 19 | this match { 20 | case e: IDomNode => 21 | e.getText 22 | case _ => CastUnsupported 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/modules/EmailModule.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes.modules 2 | 3 | import javax.mail.internet.InternetAddress 4 | 5 | import io.idml.datanodes.{IArray, IEmail} 6 | import io.idml.{CastFailed, CastUnsupported, IdmlArray, IdmlNothing, IdmlString, IdmlValue} 7 | 8 | import scala.util.Try 9 | 10 | /** Adds email manipulation behaviour to data nodes */ 11 | trait EmailModule { 12 | this: IdmlValue => 13 | 14 | /** Construct a new Email by parsing a string */ 15 | def email(): IdmlValue = 16 | this match { 17 | case _: IEmail | _: IdmlNothing => this 18 | case n: IdmlString => 19 | Try(new InternetAddress(n.value, true)) 20 | .map(new IEmail(_)) 21 | .getOrElse(CastFailed) 22 | case a: IdmlArray => new IArray(a.items.map { _.email() }) 23 | case _ => CastUnsupported 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/modules/JavaApiModule.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes.modules 2 | 3 | import java.util.Optional 4 | import java.lang._ 5 | 6 | import io.idml.IdmlValue 7 | import io.idml.datanodes.{IArray, IBool, IDouble, IInt, IObject, IString} 8 | 9 | import scala.collection.JavaConverters._ 10 | 11 | trait JavaApiModule { 12 | this: IdmlValue => 13 | 14 | def asBoolean(): Optional[Boolean] = 15 | this match { 16 | case b: IBool => Optional.of(b.value.asInstanceOf[java.lang.Boolean]) 17 | case _ => Optional.empty[java.lang.Boolean] 18 | } 19 | 20 | def asString(): Optional[java.lang.String] = 21 | this match { 22 | case s: IString => Optional.of(s.value) 23 | case _ => Optional.empty[String] 24 | } 25 | 26 | def asLong(): Optional[java.lang.Long] = 27 | this match { 28 | case l: IInt => Optional.of(l.value) 29 | case _ => Optional.empty[Long] 30 | } 31 | 32 | def asDouble(): Optional[java.lang.Double] = 33 | this match { 34 | case d: IDouble => Optional.of(d.value) 35 | case _ => Optional.empty[java.lang.Double] 36 | } 37 | 38 | def asObject(): Optional[java.util.Map[String, IdmlValue]] = 39 | this match { 40 | case o: IObject => Optional.of(o.fields.asJava) 41 | case _ => Optional.empty[java.util.Map[String, IdmlValue]] 42 | } 43 | 44 | def asList(): Optional[java.util.List[IdmlValue]] = 45 | this match { 46 | case a: IArray => Optional.of(a.items.asJava) 47 | case _ => Optional.empty[java.util.List[IdmlValue]] 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/modules/ObjectModule.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes.modules 2 | 3 | import io.idml.datanodes.{IArray, IString} 4 | import io.idml._ 5 | 6 | import scala.util.Try 7 | 8 | /** Adds object-like behaviour */ 9 | trait ObjectModule { 10 | this: IdmlValue => 11 | 12 | /** Remove a field by name */ 13 | def remove(path: String) {} 14 | 15 | def values(): IdmlValue = 16 | this match { 17 | case o: IdmlObject => IArray(o.fields.values.toBuffer) 18 | case _ => InvalidCaller 19 | } 20 | 21 | def keys(): IdmlValue = 22 | this match { 23 | case o: IdmlObject => IArray(o.fields.keys.map(IString.apply).toBuffer[IdmlValue]) 24 | case _ => InvalidCaller 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/regex/PJavaRegex.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes.regex 2 | 3 | import java.util.regex.Pattern 4 | 5 | import scala.collection.mutable.ListBuffer 6 | 7 | /** Implementation of PRegexLike for the standard Java regular expressions 8 | * @param regex 9 | */ 10 | class PJavaRegex(regex: String) extends PRegexLike(regex) { 11 | val pattern = Pattern.compile(regex) 12 | 13 | override def matches(target: String): List[List[String]] = { 14 | val r = pattern.matcher(target) 15 | val builder = ListBuffer.empty[List[String]] 16 | while (r.find()) { 17 | val result = (1 to r.groupCount()).toList.map { i => 18 | r.group(i) 19 | } 20 | builder.append(result) 21 | } 22 | builder.toList 23 | } 24 | // scalastyle:off method.name 25 | override def `match`(target: String): List[String] = { 26 | val r = pattern.matcher(target) 27 | if (r.matches()) { 28 | val results = 1 to r.groupCount() map { i => 29 | r.group(i) 30 | } 31 | results.toList 32 | } else { 33 | List[String]() 34 | } 35 | } 36 | // scalastyle:on method.name 37 | 38 | override def replace(target: String, replacement: String): String = { 39 | target.replaceAll(pattern.pattern(), replacement) 40 | } 41 | 42 | override def isMatch(target: String): Boolean = { 43 | pattern.matcher(target).matches() 44 | } 45 | 46 | override def split(target: String): List[String] = { 47 | pattern.split(target).toList 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/regex/PRe2Regex.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes.regex 2 | import com.google.re2j.Pattern 3 | 4 | import scala.collection.JavaConverters._ 5 | import scala.collection.mutable.ListBuffer 6 | 7 | class PRe2Regex(regex: String) extends PRegexLike(regex) { 8 | private val pattern = Pattern.compile(regex) 9 | 10 | override def matches(target: String): List[List[String]] = { 11 | val r = pattern.matcher(target) 12 | val builder = ListBuffer.empty[List[String]] 13 | while (r.find()) { 14 | val result = (1 to r.groupCount()).toList.map { i => 15 | r.group(i) 16 | } 17 | builder.append(result) 18 | } 19 | builder.toList 20 | } 21 | override def `match`(target: String): List[String] = { 22 | val r = pattern.matcher(target) 23 | if (r.matches()) { 24 | val results = 1 to r.groupCount() map { i => 25 | r.group(i) 26 | } 27 | results.toList 28 | } else { 29 | List[String]() 30 | } 31 | } 32 | override def split(target: String): List[String] = pattern.split(target).toList 33 | override def isMatch(target: String): Boolean = pattern.matches(target) 34 | override def replace(target: String, replacement: String): String = 35 | pattern.matcher(target).replaceAll(replacement) 36 | } 37 | -------------------------------------------------------------------------------- /datanodes/src/main/scala/io/idml/datanodes/regex/PRegexLike.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes.regex 2 | 3 | abstract case class PRegexLike(regex: String) { 4 | // scalastyle:off method.name 5 | def `match`(target: String): List[String] 6 | // scalastyle:on method.name 7 | def split(target: String): List[String] 8 | def isMatch(target: String): Boolean 9 | def replace(target: String, replacement: String): String 10 | def matches(target: String): List[List[String]] 11 | } 12 | 13 | object PRegexFactory { 14 | var regexType: Option[String] = None 15 | 16 | def getRegex(i: String): PRegexLike = { 17 | regexType match { 18 | case Some("java") => 19 | new PRe2Regex(i) // can re-add java later 20 | case Some("re2") => 21 | new PRe2Regex(i) 22 | case None => 23 | new PRe2Regex(i) 24 | case _ => 25 | new PRe2Regex(i) 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /datanodes/src/test/scala/io/idml/datanodes/IDoubleTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import org.scalatest.funsuite.AnyFunSuite 4 | import org.scalatest.matchers.must.Matchers 5 | 6 | /** Test the behaviour of the PDouble class */ 7 | class IDoubleTest extends AnyFunSuite with Matchers { 8 | // Equality 9 | test("float min == min")(new IDouble(Float.MinValue) must equal(new IDouble(Float.MinValue))) 10 | test("float max == max")(new IDouble(Float.MaxValue) must equal(new IDouble(Float.MaxValue))) 11 | test("float max != min")(new IDouble(Float.MaxValue) must not equal new IDouble(Float.MinValue)) 12 | test("double min == min")(new IDouble(Double.MinValue) must equal(new IDouble(Double.MinValue))) 13 | test("double max == max")(new IDouble(Double.MaxValue) must equal(new IDouble(Double.MaxValue))) 14 | test("double max != min")( 15 | new IDouble(Double.MaxValue) must not equal new IDouble(Double.MinValue)) 16 | 17 | test("float = long")(new IDouble(1000f) must equal(new IDouble(1000L))) 18 | test("double = long")(new IDouble(1000.0) must equal(new IDouble(1000L))) 19 | 20 | // Comparison with other numerical types 21 | test("PDouble(int) == PInt(int)")(new IDouble(1000) must equal(new IInt(1000))) 22 | test("PDouble(long) == PInt(long)")(new IDouble(1000L) must equal(new IInt(1000L))) 23 | test("PDouble(float) == PInt(int)")(new IDouble(1000.0) must equal(new IInt(1000))) 24 | test("PDouble(double) == PInt(int)")(new IDouble(1000f) must equal(new IInt(1000))) 25 | 26 | } 27 | -------------------------------------------------------------------------------- /datanodes/src/test/scala/io/idml/datanodes/IdmlNullTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.IdmlNull 4 | import org.scalatest.funsuite.AnyFunSuite 5 | import org.scalatest.matchers.must.Matchers 6 | 7 | /** Test the functionality of the IdmlNull object */ 8 | class IdmlNullTest extends AnyFunSuite with Matchers { 9 | 10 | // Equality 11 | test("null == null")(IdmlNull must equal(IdmlNull)) 12 | } 13 | -------------------------------------------------------------------------------- /datanodes/src/test/scala/io/idml/datanodes/IdmlValueTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import io.idml.{IdmlNull, IdmlValue} 4 | import org.scalatest.funsuite.AnyFunSuite 5 | import org.scalatest.matchers.must.Matchers 6 | 7 | /** Test the behaviour of the IdmlValue class */ 8 | class IdmlValueTest extends AnyFunSuite with Matchers { 9 | 10 | // Companion object 11 | test("companion - null")(IdmlValue(null) must equal(IdmlNull)) 12 | test("companion - string")(IdmlValue("abc") must equal(IdmlValue("abc"))) 13 | test("companion - true")(IdmlValue(v = true) must equal(ITrue)) 14 | test("companion - false")(IdmlValue(v = false) must equal(IFalse)) 15 | test("companion - long")(IdmlValue(v = Long.MaxValue) must equal(new IInt(Long.MaxValue))) 16 | test("companion - int")(IdmlValue(v = Int.MaxValue) must equal(new IInt(Int.MaxValue))) 17 | test("companion - float")(IdmlValue(v = Float.MaxValue) must equal(new IDouble(Float.MaxValue))) 18 | test("companion - double")( 19 | IdmlValue(v = Double.MaxValue - 1000) must equal(new IDouble(Double.MaxValue - 1000))) 20 | 21 | } 22 | -------------------------------------------------------------------------------- /datanodes/src/test/scala/io/idml/datanodes/PBoolTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.datanodes 2 | 3 | import org.scalatest.funsuite.AnyFunSuite 4 | import org.scalatest.matchers.must.Matchers 5 | 6 | /** Test the behaviour of the PBool class */ 7 | class PBoolTest extends AnyFunSuite with Matchers { 8 | // Equality 9 | test("true == true")(ITrue must equal(ITrue)) 10 | test("false == false")(IFalse must equal(IFalse)) 11 | test("true != false")(IFalse must not equal ITrue) 12 | test("false != true")(IFalse must not equal ITrue) 13 | 14 | // bool 15 | test("true.bool() == true")(ITrue.bool() must equal(ITrue)) 16 | test("false.bool() == false")(IFalse.bool() must equal(IFalse)) 17 | } 18 | -------------------------------------------------------------------------------- /docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "Trying to publish docker image" 3 | set -e 4 | sbt "project tool" "docker:publish" 5 | -------------------------------------------------------------------------------- /docs/generate-index.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo "Generating index" 3 | cd docs/src/main/tut/functions 4 | ./generateindex.py > index.md 5 | cd ../../../../../ 6 | -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/README.md: -------------------------------------------------------------------------------- 1 | The docsplus layout is from cats-effect, and is used unmodified from the original under the Apache license. 2 | -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/css/override.css: -------------------------------------------------------------------------------- 1 | #site-header .navbar-wrapper nav ul li { 2 | margin-right: 50px !important; 3 | } 4 | -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/css/toc.css: -------------------------------------------------------------------------------- 1 | 2 | #toc.styling { 3 | border-top: 1px solid #eee; 4 | border-bottom: 1px solid #eee; 5 | padding: 0; 6 | padding-top: 10px; 7 | padding-bottom: 10px; 8 | margin-bottom: 20px; 9 | } 10 | 11 | #toc h2 { 12 | margin: 0; 13 | margin-top: 12.5px; 14 | margin-bottom: 12.5px; 15 | color: #888; 16 | font-size: 24px; 17 | } 18 | 19 | #toc ul { 20 | margin: 0; 21 | margin-right: 10px; 22 | list-style-position: inside; 23 | padding: 0; 24 | } 25 | 26 | #toc ul li { 27 | margin-left: 10px; 28 | } 29 | 30 | #toc > ul > li { 31 | margin-left: 0; 32 | } 33 | -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/.directory: -------------------------------------------------------------------------------- 1 | [Dolphin] 2 | PreviewsShown=true 3 | Timestamp=2018,5,18,11,9,25 4 | Version=3 5 | -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/jumbotron_pattern.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDML/idml/d1732f24a42593b2f0ec27a7293d5faae5a7c2dd/docs/src/main/resources/microsite/img/jumbotron_pattern.png -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/jumbotron_pattern2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDML/idml/d1732f24a42593b2f0ec27a7293d5faae5a7c2dd/docs/src/main/resources/microsite/img/jumbotron_pattern2x.png -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/navbar_brand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDML/idml/d1732f24a42593b2f0ec27a7293d5faae5a7c2dd/docs/src/main/resources/microsite/img/navbar_brand.png -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/navbar_brand2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDML/idml/d1732f24a42593b2f0ec27a7293d5faae5a7c2dd/docs/src/main/resources/microsite/img/navbar_brand2x.png -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/sidebar_brand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDML/idml/d1732f24a42593b2f0ec27a7293d5faae5a7c2dd/docs/src/main/resources/microsite/img/sidebar_brand.png -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/img/sidebar_brand2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDML/idml/d1732f24a42593b2f0ec27a7293d5faae5a7c2dd/docs/src/main/resources/microsite/img/sidebar_brand2x.png -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/js/idml-hilight.js: -------------------------------------------------------------------------------- 1 | console.log("registering IDML highlighter"); 2 | 3 | hljs.registerLanguage("idml", function(h) { 4 | return { 5 | cI: false, 6 | k: 'let if then else match not substr this root and or contains exists in cs true false', 7 | c: [ 8 | { 9 | cN: 'comment', 10 | b: /#/, e: /\n/ 11 | }, 12 | { 13 | cN: 'string', 14 | b: /"/, e: /"/, 15 | }, 16 | { 17 | cN: 'number', 18 | b: hljs.CNR 19 | }, 20 | { 21 | cN: 'variable', 22 | v: [ 23 | {b: /\$[a-zA-Z]+/}, 24 | {b: /\@[a-zA-Z]+/} 25 | ] 26 | }, 27 | { 28 | cN: 'title', 29 | b: /\[[a-zA-Z]+\]/ 30 | }, 31 | 32 | ], 33 | }; 34 | }); 35 | hljs.configure({languages:['scala','java','bash', 'idml']}); 36 | hljs.initHighlighting.called = false; 37 | hljs.initHighlighting(); 38 | -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/layouts/README.md: -------------------------------------------------------------------------------- 1 | The docsplus layout is from cats-effect, and is used unmodified from the original under the Apache license. 2 | -------------------------------------------------------------------------------- /docs/src/main/resources/microsite/layouts/docsplus.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: docs 3 | --- 4 | 5 |

{{ page.title }}

6 | 7 | 8 | 9 | {{ content }} 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /docs/src/main/tut/diagrams/chain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDML/idml/d1732f24a42593b2f0ec27a7293d5faae5a7c2dd/docs/src/main/tut/diagrams/chain.png -------------------------------------------------------------------------------- /docs/src/main/tut/diagrams/chain.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | input -> output: map1 3 | output -> output: map2 4 | output -> output: map3 5 | output -> output: map4 6 | output -> output: map5 7 | @enduml -------------------------------------------------------------------------------- /docs/src/main/tut/diagrams/merge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDML/idml/d1732f24a42593b2f0ec27a7293d5faae5a7c2dd/docs/src/main/tut/diagrams/merge.png -------------------------------------------------------------------------------- /docs/src/main/tut/diagrams/merge.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | input -> output: map1 3 | input -> output: map2 4 | input -> output: map3 5 | input -> output: map4 6 | input -> output: map5 7 | @enduml -------------------------------------------------------------------------------- /docs/src/main/tut/functions/advanced.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: docsplus 3 | title: Advanced 4 | section: language 5 | --- 6 | 7 | Advanced functions which resolve differently to normal functions. 8 | 9 | ## apply and applyArray 10 | 11 | Invokes a [Block](features/core-language.html#blocks) on an item or array of items. 12 | 13 | ## array and extract 14 | 15 | These both do what you'd call `flatMap` in functional languages, they're used to transform items in an array and will drop items which resolve to Nothing. 16 | 17 | ``` 18 | {"input": [1, 2, 3, 4]} 19 | 20 | output = input.extract(this + 1) 21 | 22 | {"output" : [ 2, 3, 4, 5 ]} 23 | ``` 24 | 25 | ## blacklist 26 | 27 | Removes a blacklisted field from an object. 28 | 29 | ``` 30 | {"input": {"a": 1, "b": 2, "c": 3}} 31 | 32 | output = input.blacklist("b") 33 | 34 | {"output" : {"a" : 1, "c" : 3 }} 35 | ``` 36 | 37 | This would be the same as 38 | 39 | ``` 40 | output = input 41 | output.b = deleted() 42 | ``` 43 | 44 | ## average 45 | 46 | Averages an array of ints and doubles, always returns a double. 47 | 48 | ``` 49 | {"input": [1,2,3,4,5,6,7,8,9]} 50 | 51 | output = input.average() 52 | 53 | {"output" : 5.0} 54 | ``` 55 | 56 | ## append 57 | 58 | Appends an item to a list. 59 | 60 | ## prepend 61 | 62 | Prepends an item to a list. 63 | 64 | ## size 65 | 66 | Gets the size of an array or string when called with no arguments, trims it down to a size when called with an integer. 67 | 68 | ## concat 69 | 70 | Joins two strings together, mostly replaced by `+`. 71 | 72 | -------------------------------------------------------------------------------- /docs/src/main/tut/functions/generateindex.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from __future__ import print_function 3 | import os 4 | from glob import glob 5 | 6 | 7 | def getsections(): 8 | for filename in (filename for filename in glob("*.md") if filename != "index.md"): 9 | with open(filename, 'r') as f: 10 | yield (True, filename, "") 11 | for section in filter(lambda x:x.startswith("## "), f.readlines()): 12 | yield (False, filename, section.replace("## ", "").strip()) 13 | 14 | def generateindex(sections): 15 | yield "---" 16 | yield "layout: docsplus" 17 | yield "title: Functions" 18 | yield "section: language" 19 | yield "---" 20 | for (toggle, filename, section) in sections: 21 | if toggle: 22 | yield "## {}".format(filename.replace(".md", "")) 23 | else: 24 | yield "[{}]({})".format(section, "/functions/{}#{}".format(filename.replace(".md", ".html"), section.lower().replace(" ", "-").replace(",", ""))) 25 | 26 | if __name__=="__main__": 27 | print('\n\n'.join(list(generateindex(getsections())))) 28 | -------------------------------------------------------------------------------- /docs/src/main/tut/functions/object.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: docsplus 3 | title: Object 4 | section: language 5 | --- 6 | 7 | Object functions are used to manipulate and work with objects. 8 | 9 | ## serialize 10 | 11 | This function is called on an Object and returns a String containing a JSON serialized copy of the object 12 | 13 | ```idml:input 14 | {} 15 | ``` 16 | 17 | ```idml:code:inline 18 | a.b = 1 19 | a.c = 2 20 | 21 | serialized = @a.serialize() 22 | ``` 23 | 24 | ## keys 25 | 26 | This function is called on an object and returns an array of its keys. 27 | 28 | ## values 29 | 30 | This function is called on an object and returns an array of its values. 31 | -------------------------------------------------------------------------------- /docs/src/main/tut/functions/random.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: docsplus 3 | title: Random 4 | section: language 5 | --- 6 | 7 | The Random function is used to generate random numbers. 8 | 9 | 10 | ## random 11 | 12 | The random function *takes a seed* and a range within which to generate the random number, remember to use a seed which makes sense from your input object [so that replayed data gets the same random number](../user-guide/philosophy.html#determinism). 13 | 14 | The seed may be: 15 | * a string, this is murmurhashed and fed in 16 | * an integer, this is used to seed directly 17 | * an object, this is serialized and murmurhashed 18 | 19 | ### generating a random number on the whole input 20 | ``` 21 | sample = root.random() # generates a random 64-bit integer seeded by the entire input object 22 | sample1 = root.random(0, 100) # generates a random integer between 0 and 100, seeded by the entire input 23 | sample2 = root.random(0.0, 100.0) # generates a random float between 0.0 and 100.0, seeded by the entire input 24 | ``` 25 | 26 | ### seeding with a couple fields 27 | ``` 28 | sample = (url + title).random(0, 100) # seed with two strings and range between 0 and 100 29 | ``` 30 | -------------------------------------------------------------------------------- /docs/src/main/tut/functions/schema.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IDML/idml/d1732f24a42593b2f0ec27a7293d5faae5a7c2dd/docs/src/main/tut/functions/schema.md -------------------------------------------------------------------------------- /docs/src/main/tut/functions/uuid.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: docsplus 3 | title: UUID 4 | section: language 5 | --- 6 | 7 | The uuid functions allow the generation of UUIDs. 8 | 9 | Design Note: Type 1, 2 and 4 are not included because they are not deterministic, [see Philosophy](/philosophy.html#determinism). 10 | 11 | ## uuid3 12 | 13 | Generates version 3 UUIDs with MD5. 14 | 15 | This function takes a string or object, and when called with no arguments will run against the current root object. 16 | 17 | ``` 18 | {"input": "hello world"} 19 | 20 | idml> output = input.uuid3() 21 | 22 | {"output": "5eb63bbb-e01e-3ed0-93cb-22bb8f5acdc3"} 23 | ``` 24 | 25 | ### hash the entire input object 26 | 27 | ``` 28 | {} 29 | 30 | hash1 = root.uuid3() 31 | hash2 = uuid3() 32 | # as long as you're not in a block these are equivalent 33 | 34 | { 35 | "hash2" : "99914b93-2bd3-3a50-b983-c5e7c90ae93b", 36 | "hash1" : "99914b93-2bd3-3a50-b983-c5e7c90ae93b" 37 | } 38 | ``` 39 | 40 | ## uuid5 41 | 42 | Generates version 5 UUIDs with SHA-1. 43 | 44 | This function takes a string or object, and when called with no arguments will run against the current root object. 45 | 46 | ``` 47 | {"input": "hello world"} 48 | 49 | idml> output = input.uuid5() 50 | 51 | {"output": "2aae6c35-c94f-5fb4-95db-e95f408b9ce9"} 52 | ``` 53 | 54 | ### hash the entire input object 55 | 56 | ``` 57 | {} 58 | 59 | hash1 = root.uuid5() 60 | hash2 = uuid5() 61 | # as long as you're not in a block these are equivalent 62 | 63 | { 64 | "hash2" : "bf21a9e8-fbc5-5384-afb0-5b4fa0859e09", 65 | "hash1" : "bf21a9e8-fbc5-5384-afb0-5b4fa0859e09" 66 | } 67 | ``` 68 | 69 | 70 | -------------------------------------------------------------------------------- /docs/src/main/tut/index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: home 3 | technologies: 4 | - first: ["JVM", "IDML is a JVM-first data transform language"] 5 | - second: ["Pure", "Transformations are always pure, and will return the same output every time"] 6 | - third: ["Safe", "All actions can fail safely to ensure as much of your data was mapped as possible"] 7 | --- 8 | 9 | # Overview 10 | 11 | IDML is a data preparation language designed to process unstructured data at high volumes. 12 | 13 | 14 | 15 | ## Benefits 16 | 17 | * __Accessible__ - Designed for Analysts, Product Managers, Tech Writers, Support, Sales Engineers and Software Engineers alike 18 | * __Concise__ - Free yourself from error-prone boilerplate code like null pointer checks and parse exception handlers 19 | * __Extensible__ - Out of the box it has support for things like regular expressions, email and geolocation but it's easy to add new modules 20 | * __High-performance__ - Built for firehoses 21 | * __Java-based__ - Integrates with the emerging big data stack, including Hadoop MapReduce and Kafka 22 | -------------------------------------------------------------------------------- /docs/src/main/tut/modules/hashing.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: docsplus 3 | title: Hashing 4 | section: language 5 | --- 6 | 7 | The hashing module includes common hashing algorithms which may be called on strings 8 | 9 | This module is optional and must be included in the list of FunctionResolvers passed to the Ptolemy engine constructor. 10 | 11 | 12 | ### Included hashing algorithms 13 | 14 | | Algorithm | Function Call | Output | 15 | |---|---|---| 16 | | [xxHash32](https://cyan4973.github.io/xxHash/) | `input.xxHash32()` | 32-bit hex | 17 | | [xxHash64](https://cyan4973.github.io/xxHash/) | `input.xxHash64()` | 64-bit hex | 18 | | [cityHash](https://github.com/google/cityhash) | `input.cityHash()` | 64-bit hex | 19 | | [murmurHash3](https://github.com/aappleby/smhasher) | `input.murmurHash3()` | 64-bit hex | 20 | | [sha1](https://en.wikipedia.org/wiki/SHA-1) | `input.sha1()` | 20-bit hex | 21 | | [sha256](https://en.wikipedia.org/wiki/SHA-2) | `input.sha256()` | 256-bit hex | 22 | | [sha512](https://en.wikipedia.org/wiki/SHA-2) | `input.sha512()` | 512-bit hex | 23 | | [md5](https://en.wikipedia.org/wiki/MD5) | `input.md5()` | 128-bit hex | 24 | | [crc32](https://en.wikipedia.org/wiki/Cyclic_redundancy_check) | `input.crc32()` | 32-bit hex | 25 | 26 | Note: you can use the functions available on strings in the `Maths` set of functions to parse these into integers if you really need to, they are [parseHex](/functions/maths.html#parsehex) and [parseHexUnsigned](/functions/maths.html#parsehexunsigned), you probably want the latter. 27 | -------------------------------------------------------------------------------- /docs/src/main/tut/modules/jsoup.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: docsplus 3 | title: jsoup (HTML and XML) 4 | section: language 5 | --- 6 | 7 | This module is optional and must be included in the list of FunctionResolvers passed to the Ptolemy engine constructor. 8 | 9 | It includes the `PtolemyJsoup` class which can be used to parse XML or HTML into input which can be read by the interpreter. 10 | 11 | ## PtolemyJsoup 12 | 13 | When coding against this, you can use `PtolemyJsoup.parseXml` and `PtolemyJsoup.parseHtml` to load data. 14 | 15 | ## stripTags 16 | 17 | This module also includes the `stripTags()` function, which can be called on a string to strip all XML and HTML tags out of it. 18 | 19 | ``` 20 | "hello world, this has some tags in".stripTags() # "hello world, this has some tags in" 21 | ``` 22 | 23 | -------------------------------------------------------------------------------- /docs/src/main/tut/philosophy.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: docs 3 | title: Philosophy 4 | section: language 5 | --- 6 | 7 | # Philosophy 8 | 9 | ## Purity 10 | 11 | Running a program has no side-effects 12 | 13 | ## Determinism 14 | 15 | Running a program on the same input will always produce the same output 16 | 17 | ## Reliability 18 | 19 | No runtime errors, once parsed a program will always run 20 | 21 | ## Flow 22 | 23 | Code should be written left to right 24 | -------------------------------------------------------------------------------- /docs/touchups.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo "Doing touchups" 3 | sed -i 's:
    • :
    • :g' docs/target/**/*.html 4 | 5 | sed -i 's|

    View on GitHub

    |

    ReleasesLanguage DocsAPI Docs

    |g' docs/target/**/*.html 6 | -------------------------------------------------------------------------------- /geo/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-geo" 2 | 3 | libraryDependencies ++= Seq( 4 | // "org.xerial" % "sqlite-jdbc" % "3.21.0.1", 5 | // "org.tpolecat" %% "doobie-core" % "0.6.0-M2", 6 | // "org.tpolecat" %% "doobie-hikari" % "0.6.0-M2", 7 | "net.iakovlev" % "timeshape" % "2018d.6", 8 | "org.scalatest" %% "scalatest" % "3.2.8" % Test, 9 | "com.storm-enroute" %% "scalameter" % "0.19" % Test 10 | ) 11 | -------------------------------------------------------------------------------- /geo/src/main/resources/META-INF/services/io.idml.functions.FunctionResolver: -------------------------------------------------------------------------------- 1 | io.idml.geo.DefaultGeoFunctionResolver 2 | -------------------------------------------------------------------------------- /geo/src/main/scala/io/idml/geo/Geo.scala: -------------------------------------------------------------------------------- 1 | package io.idml.geo 2 | 3 | import io.idml.datanodes.CompositeValue 4 | import io.idml.{CastFailed, IdmlDouble, IdmlObject, IdmlValue} 5 | 6 | import scala.collection.mutable 7 | 8 | /** A geo-tag object */ 9 | case class Geo(lat: Double, long: Double) extends IdmlObject with CompositeValue { 10 | 11 | /** The fields */ 12 | val fields = mutable.Map( 13 | "latitude" -> IdmlValue(lat), 14 | "longitude" -> IdmlValue(long) 15 | ) 16 | } 17 | 18 | /** Helpers for geo-tagging */ 19 | object Geo { 20 | 21 | /** Transform arbitrary nodes into geo-tags */ 22 | def apply(lat: IdmlValue, long: IdmlValue): IdmlValue = 23 | (lat, long) match { 24 | case (lat: IdmlDouble, long: IdmlDouble) => 25 | Geo(lat.value, long.value) 26 | case _ => CastFailed 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /geo/src/main/scala/io/idml/geo/Geo2Function.scala: -------------------------------------------------------------------------------- 1 | package io.idml.geo 2 | 3 | import io.idml.IdmlValue 4 | import io.idml.ast.Pipeline 5 | import io.idml.functions.IdmlFunction2 6 | 7 | /** 2-argument constructor for geolocation 8 | */ 9 | case class Geo2Function(arg1: Pipeline, arg2: Pipeline) extends IdmlFunction2 { 10 | override protected def apply(cursor: IdmlValue, lat: IdmlValue, long: IdmlValue): IdmlValue = { 11 | Geo(lat.float(), long.float()) 12 | } 13 | 14 | override def name: String = "geo" 15 | } 16 | -------------------------------------------------------------------------------- /geo/src/main/scala/io/idml/geo/GeoFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.geo 2 | 3 | import io.idml.{CastFailed, CastUnsupported, IdmlArray, IdmlObject, IdmlValue} 4 | import io.idml.functions.IdmlFunction0 5 | 6 | /** Constructor for geolocation objects 7 | */ 8 | case object GeoFunction extends IdmlFunction0 { 9 | override protected def apply(cursor: IdmlValue): IdmlValue = { 10 | cursor match { 11 | case obj: IdmlObject => 12 | Geo(obj.get("latitude").float(), obj.get("longitude").float()) 13 | case arr: IdmlArray if arr.size > 1 => 14 | val lat = arr.items.head.float() 15 | val long = arr.items(1).float() 16 | if (lat.isNothing || lat.isNothing) { 17 | CastFailed 18 | } else { 19 | Geo(lat, long) 20 | } 21 | case arr: IdmlArray => CastFailed 22 | case _ => CastUnsupported 23 | } 24 | } 25 | 26 | override def name: String = "geo" 27 | } 28 | -------------------------------------------------------------------------------- /geo/src/main/scala/io/idml/geo/IsoCountryFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.geo 2 | 3 | import java.nio.charset.Charset 4 | 5 | import io.idml.{IdmlJson, IdmlValue} 6 | import io.idml.ast.Pipeline 7 | import io.idml.functions.IdmlFunction1 8 | import com.google.common.io.Resources 9 | 10 | /** Turns iso countries into country names */ 11 | class IsoCountryFunction(countries: => IdmlValue, val arg: Pipeline) extends IdmlFunction1 { 12 | 13 | override protected def apply(cursor: IdmlValue, country: IdmlValue): IdmlValue = { 14 | countries.get(country) 15 | } 16 | 17 | override def name: String = "country" 18 | } 19 | -------------------------------------------------------------------------------- /geo/src/main/scala/io/idml/geo/IsoRegionFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.geo 2 | 3 | import java.nio.charset.Charset 4 | 5 | import io.idml.ast.{IdmlFunction, Pipeline} 6 | import io.idml.functions.IdmlFunction2 7 | import io.idml.{IdmlJson, IdmlValue} 8 | import com.google.common.io.Resources 9 | 10 | /** Transform ISO3166 country and region codes into region name */ 11 | class IsoRegionFunction(regions: => IdmlValue, val arg1: Pipeline, val arg2: Pipeline) 12 | extends IdmlFunction2 { 13 | override protected def apply( 14 | cursor: IdmlValue, 15 | country: IdmlValue, 16 | region: IdmlValue): IdmlValue = { 17 | regions.get(country).get(region) 18 | } 19 | override def name: String = "region" 20 | } 21 | -------------------------------------------------------------------------------- /geo/src/main/scala/io/idml/geo/TimezoneFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.geo 2 | 3 | import cats.implicits._ 4 | import io.idml._ 5 | import io.idml.ast.Pipeline 6 | import io.idml.datanodes.{IInt, IObject, IString} 7 | import io.idml.functions.IdmlFunction0 8 | import net.iakovlev.timeshape.TimeZoneEngine 9 | 10 | object TimezoneFunction { 11 | lazy val engine: TimeZoneEngine = TimeZoneEngine.initialize() 12 | 13 | def query(g: Geo): Option[String] = { 14 | engine.query(g.lat, g.long).map[Option[String]](r => Some(r.toString)).orElse(None) 15 | } 16 | 17 | case object TimezoneFunction extends IdmlFunction0 { 18 | override protected def apply(cursor: IdmlValue): IdmlValue = { 19 | cursor match { 20 | case g: Geo => query(g).map(IString).getOrElse(MissingField) 21 | case _ => InvalidParameters 22 | } 23 | } 24 | 25 | override def name: String = "timezone" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /geo/src/test/resources/io.idml.geo/CountrySuite.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapping": "output = country(country)", 3 | 4 | "tests": [ 5 | { 6 | "name": "find a country name", 7 | "input": {"country": "DE"}, 8 | "output": {"output": "Germany"} 9 | }, 10 | { 11 | "name": "unknown country name", 12 | "input": {"country": "XX"}, 13 | "output": {} 14 | }, 15 | { 16 | "name": "invalid input type", 17 | "input": {"country": true}, 18 | "output": {} 19 | }, 20 | { 21 | "name": "missing country name", 22 | "input": {}, 23 | "output": {} 24 | } 25 | ] 26 | } -------------------------------------------------------------------------------- /geo/src/test/resources/io.idml.geo/Geo2Suite.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapping": "output = geo(input.a, input.b)", 3 | 4 | "tests": [ 5 | { 6 | "name": "accepts two floats", 7 | "input": {"input": {"a": 4.5, "b": 12.0}}, 8 | "output": {"output": {"latitude": 4.5, "longitude": 12.0}} 9 | }, 10 | { 11 | "name": "accepts two floats - strings", 12 | "input": {"input": {"a": "4.5", "b": "12"}}, 13 | "output": {"output": {"latitude": 4.5, "longitude": 12.0}} 14 | }, 15 | { 16 | "name": "accepts Reading", 17 | "input": {"input": {"a": 51.4542, "b": 0.9731}}, 18 | "output": {"output": {"latitude": 51.4542, "longitude": 0.9731}} 19 | }, 20 | { 21 | "name": "rejects invalid coordinates", 22 | "input": {"input": {"a": "dog", "b": "cat"}}, 23 | "output": {} 24 | }, 25 | { 26 | "name": "rejects partially missing", 27 | "input": {"input": {"a": 51.4542}}, 28 | "output": {} 29 | }, 30 | { 31 | "name": "rejects missing", 32 | "input": {}, 33 | "output": {} 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /geo/src/test/resources/io.idml.geo/GeoSuite.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapping": "output = input.geo()", 3 | 4 | "tests": [ 5 | { 6 | "name": "arrays", 7 | "input": {"input": [7.5, 1.0]}, 8 | "output": {"output": {"latitude": 7.5, "longitude": 1.0}} 9 | }, 10 | { 11 | "name": "objects", 12 | "input": {"input": {"longitude": 7.5, "latitude": 2.0}}, 13 | "output": {"output": {"longitude": 7.5, "latitude": 2.0}} 14 | }, 15 | { 16 | "name": "arrays - ints and strings", 17 | "input": {"input": ["7.5", 1]}, 18 | "output": {"output": {"latitude": 7.5, "longitude": 1.0}} 19 | }, 20 | 21 | { 22 | "name": "objects - ints and strings", 23 | "input": {"input": {"longitude": "7.5", "latitude": 2}}, 24 | "output": {"output": {"longitude": 7.5, "latitude": 2.0}} 25 | } 26 | ] 27 | } -------------------------------------------------------------------------------- /geo/src/test/scala/io/idml/geo/Admin1Spec.scala: -------------------------------------------------------------------------------- 1 | package io.idml.geo 2 | 3 | class Admin1Spec {} 4 | -------------------------------------------------------------------------------- /geo/src/test/scala/io/idml/geo/GeoFunctionsTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.geo 2 | 3 | import io.idml.IdmlScalaTestBase 4 | 5 | class GeoFunctionsTest extends IdmlScalaTestBase("io.idml.geo") 6 | -------------------------------------------------------------------------------- /geo/src/test/scala/io/idml/geo/TimezoneSpec.scala: -------------------------------------------------------------------------------- 1 | package io.idml.geo 2 | import io.idml.datanodes.IString 3 | import org.scalatest.wordspec.AnyWordSpec 4 | import org.scalatest.matchers.must 5 | 6 | class TimezoneSpec extends AnyWordSpec with must.Matchers { 7 | 8 | "the timezone function" should { 9 | "know the timezone of Reading" in { 10 | TimezoneFunction.query(Geo(51.459, -0.9722)) must equal(Some("Europe/London")) 11 | } 12 | "know the timezone of Christchurch, NZ" in { 13 | TimezoneFunction.query(Geo(-43.53, 172.62)) must equal(Some("Pacific/Auckland")) 14 | } 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /hashing/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-hashing" 2 | 3 | libraryDependencies ++= Seq( 4 | "net.openhft" % "zero-allocation-hashing" % "0.8", 5 | "org.lz4" % "lz4-java" % "1.8.0", 6 | "org.scalatest" %% "scalatest" % "3.2.8" % Test 7 | ) 8 | -------------------------------------------------------------------------------- /hashing/src/main/resources/META-INF/services/io.idml.functions.FunctionResolver: -------------------------------------------------------------------------------- 1 | io.idml.hashing.HashingFunctionResolver 2 | -------------------------------------------------------------------------------- /idmld/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idmld" 2 | 3 | lazy val http4sVersion = "0.21.22" 4 | lazy val circeVersion = "0.13.0" 5 | 6 | libraryDependencies ++= Seq( 7 | "org.http4s" %% "http4s-core" % http4sVersion, 8 | "org.http4s" %% "http4s-dsl" % http4sVersion, 9 | "org.http4s" %% "http4s-circe" % http4sVersion, 10 | "org.http4s" %% "http4s-blaze-server" % http4sVersion, 11 | "io.circe" %% "circe-generic" % circeVersion, 12 | "io.circe" %% "circe-parser" % circeVersion, 13 | "ch.qos.logback" % "logback-classic" % "1.0.1" 14 | ) 15 | -------------------------------------------------------------------------------- /idmld/src/main/scala/io/idml/server/Server.scala: -------------------------------------------------------------------------------- 1 | package io.idml.server 2 | 3 | import cats.effect._ 4 | import io.idml.FunctionResolverService 5 | import org.http4s.server.blaze.BlazeBuilder 6 | 7 | import scala.concurrent.ExecutionContext.Implicits.global 8 | 9 | object Server extends IOApp { 10 | override def run(args: List[String]): IO[ExitCode] = 11 | BlazeBuilder[IO] 12 | .mountService(WebsocketServer.service(new FunctionResolverService), "/") 13 | .bindHttp(8081, "localhost") 14 | .serve 15 | .compile 16 | .lastOrError 17 | } 18 | -------------------------------------------------------------------------------- /idmldoc-plugin/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idmldoc-plugin" 2 | 3 | sbtPlugin := true 4 | 5 | addSbtPlugin("com.47deg" % "sbt-microsites" % "1.2.0" % Provided) 6 | -------------------------------------------------------------------------------- /idmldoc/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idmldoc" 2 | 3 | libraryDependencies ++= Seq( 4 | "org.tpolecat" %% "atto-core" % "0.9.3", 5 | "com.lihaoyi" %% "fastparse" % "2.3.2", 6 | "co.fs2" %% "fs2-io" % "2.5.9", 7 | "org.typelevel" %% "cats-effect" % "2.5.0", 8 | "com.monovore" %% "decline" % "1.4.0" 9 | ) 10 | 11 | libraryDependencies ++= Seq( 12 | "org.scalatest" %% "scalatest" % "3.2.8" % Test 13 | ) 14 | 15 | mainClass in Compile := Some("io.idml.doc.Main") 16 | -------------------------------------------------------------------------------- /idmldoc/src/main/scala/io/idml/doc/Markdown.scala: -------------------------------------------------------------------------------- 1 | package io.idml.doc 2 | 3 | import cats._ 4 | import cats.implicits._ 5 | import fastparse._, NoWhitespace._ 6 | import fastparse.{parse => fparse} 7 | 8 | object Markdown { 9 | trait Node 10 | case class Text(s: String) extends Node 11 | case class Code(label: String, content: String) extends Node 12 | 13 | def backticks[_: P] = P("```") 14 | def codeBlock[_: P]: P[Code] = 15 | (backticks ~ (CharsWhile(_ != '\n').!).? ~ (!backticks ~ AnyChar).rep(1).! ~ backticks).map { 16 | case (l, c) => Code(l.getOrElse(""), c) 17 | } 18 | def text[_: P]: P[Text] = (!backticks ~ AnyChar).rep(1).!.map(Text.apply) 19 | def node[_: P]: P[Node] = codeBlock | text 20 | def document[_: P]: P[List[Node]] = node.rep.map(_.toList) 21 | 22 | def parse(s: String) = fparse(s, document(_)) 23 | def render(ns: List[Node]): String = 24 | ns.map { 25 | case Text(t) => t 26 | case Code(label, content) => 27 | s"""```$label 28 | |${content.stripPrefix("\n").stripSuffix("\n")} 29 | |```""".stripMargin 30 | }.mkString 31 | } 32 | -------------------------------------------------------------------------------- /idmltest-plugin/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idmltest-plugin" 2 | 3 | sbtPlugin := true 4 | -------------------------------------------------------------------------------- /idmltest/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-test" 2 | 3 | libraryDependencies ++= Seq( 4 | "co.fs2" %% "fs2-io" % "2.5.5", 5 | "org.typelevel" %% "cats-effect" % "2.5.0", 6 | "com.monovore" %% "decline" % "1.4.0", 7 | "io.circe" %% "circe-generic" % "0.14.1", 8 | "io.circe" %% "circe-parser" % "0.14.1", 9 | "io.circe" %% "circe-literal" % "0.14.1" % Test, 10 | "io.circe" %% "circe-yaml" % "0.13.1", 11 | "com.lihaoyi" %% "fansi" % "0.2.13", 12 | "org.gnieh" %% "diffson-circe" % "4.1.1", 13 | "org.tpolecat" %% "atto-core" % "0.9.4", 14 | "com.googlecode.java-diff-utils" % "diffutils" % "1.2.1" 15 | ) 16 | 17 | libraryDependencies ++= Seq( 18 | "org.scalatest" %% "scalatest" % "3.2.8" % Test 19 | ) 20 | 21 | mainClass in Compile := Some("io.idml.test.Main") 22 | -------------------------------------------------------------------------------- /idmltest/src/main/scala/io/idml/test/CirceEitherEncoders.scala: -------------------------------------------------------------------------------- 1 | package io.idml.test 2 | import io.circe.{Decoder, Encoder, Json} 3 | import cats._, cats.implicits._ 4 | 5 | trait CirceEitherEncoders { 6 | implicit def goodEitherDecoder[A, B](implicit 7 | a: Decoder[A], 8 | b: Decoder[B]): Decoder[Either[A, B]] = { 9 | a.map(_.asLeft[B]) or b.map(_.asRight[A]) 10 | } 11 | 12 | implicit def goodEitherEncoder[A, B](implicit 13 | a: Encoder[A], 14 | b: Encoder[B]): Encoder[Either[A, B]] = 15 | (e: Either[A, B]) => e.bimap(a.apply, b.apply).merge 16 | } 17 | -------------------------------------------------------------------------------- /idmltest/src/main/scala/io/idml/test/DeterministicTime.scala: -------------------------------------------------------------------------------- 1 | package io.idml.test 2 | import io.idml.{IdmlContext, IdmlValue} 3 | import io.idml.ast.{Argument, IdmlFunction, IdmlFunctionMetadata, Pipeline} 4 | import io.idml.datanodes.IDate 5 | import io.idml.functions.{FunctionResolver, IdmlFunction0} 6 | import org.joda.time.{DateTime, DateTimeZone} 7 | 8 | class DeterministicTime(val time: Long = 0) extends FunctionResolver { 9 | override def providedFunctions(): List[IdmlFunctionMetadata] = 10 | List( 11 | IdmlFunctionMetadata("now", List.empty, "output the current time in a deterministic way") 12 | ) 13 | override def resolve(name: String, args: List[Argument]): Option[IdmlFunction] = 14 | (name, args) match { 15 | case ("now", Nil) => Some(DeterministicTime.now(time)) 16 | case ("microtime", Nil) => Some(DeterministicTime.microtime(time)) 17 | case _ => None 18 | } 19 | } 20 | 21 | object DeterministicTime { 22 | def now(time: Long) = 23 | new IdmlFunction0 { 24 | override protected def apply(cursor: IdmlValue): IdmlValue = IDate( 25 | new DateTime(time, DateTimeZone.UTC)) 26 | override def name: String = "now" 27 | } 28 | def microtime(time: Long) = 29 | new IdmlFunction0 { 30 | override protected def apply(cursor: IdmlValue): IdmlValue = IdmlValue(time * 1000) 31 | override def name: String = "microtime" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /idmltest/src/main/scala/io/idml/test/TestState.scala: -------------------------------------------------------------------------------- 1 | package io.idml.test 2 | import cats.effect.ExitCode 3 | import cats._, cats.implicits._ 4 | import cats.kernel.Order 5 | 6 | sealed trait TestState 7 | object TestState { 8 | def toExitCode(ts: TestState): ExitCode = 9 | ts match { 10 | case Error => ExitCode.Error 11 | case Failed => ExitCode.Error 12 | case Success => ExitCode.Success 13 | case Updated => ExitCode.Success 14 | } 15 | implicit val testStateMonoid: Monoid[TestState] = new Monoid[TestState] { 16 | override def empty: TestState = TestState.Success 17 | override def combine(x: TestState, y: TestState): TestState = 18 | Set(x, y) match { 19 | case tss if tss.contains(Error) => Error 20 | case tss if tss.contains(Failed) => Failed 21 | case tss if tss.contains(Updated) => Updated 22 | case tss if tss.contains(Success) => Success 23 | } 24 | } 25 | implicit val testStateOrdering: Ordering[TestState] = Order[Int] 26 | .contramap[TestState] { 27 | case Success => 0 28 | case Failed => 1 29 | case Error => 2 30 | case Updated => 3 31 | } 32 | .toOrdering 33 | case object Error extends TestState 34 | case object Failed extends TestState 35 | case object Success extends TestState 36 | case object Updated extends TestState 37 | def error: TestState = Error 38 | def failed: TestState = Failed 39 | def success: TestState = Success 40 | def updated: TestState = Updated 41 | } 42 | -------------------------------------------------------------------------------- /idmltest/src/main/scala/io/idml/test/diffable/DiffableParser.scala: -------------------------------------------------------------------------------- 1 | package io.idml.test.diffable 2 | import io.circe.Json 3 | 4 | import atto._, Atto._ 5 | import cats._, cats.implicits._ 6 | 7 | object DiffableParser { 8 | 9 | lazy val json: Parser[Json] = jstring | jnull | jnumber | jarray | jobject 10 | val jstring = stringLiteral.map(Json.fromString) 11 | val jnull = string("null").map(_ => Json.Null) 12 | val jnumber = bigDecimal.map(Json.fromBigDecimal) 13 | val jarray = for { 14 | _ <- char('[') <* skipWhitespace 15 | items <- (skipWhitespace *> json <* skipWhitespace <* opt(char(',')) <* skipWhitespace).many 16 | _ <- char(']') <* skipWhitespace 17 | } yield Json.arr(items: _*) 18 | 19 | val jobjectkv: Parser[(String, Json)] = for { 20 | key <- stringLiteral <* skipWhitespace 21 | _ <- char(':') <* skipWhitespace 22 | value <- json <* skipWhitespace 23 | _ <- opt(char(',') <* skipWhitespace) 24 | } yield key -> value 25 | 26 | val jobject = for { 27 | _ <- char('{') <* skipWhitespace 28 | kvs <- jobjectkv.many 29 | _ <- char('}') <* skipWhitespace 30 | } yield Json.obj(kvs: _*) 31 | 32 | def parse(s: String): ParseResult[Json] = json.parseOnly(s).done 33 | } 34 | -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/bad-multitest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bad multitest", 3 | "code": "r = a + b", 4 | "input": [{ 5 | "a": 2, 6 | "b": 1 7 | },{ 8 | "a": 2, 9 | "b": 2 10 | }], 11 | "output": [{ 12 | "r": 3 13 | }] 14 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/basic-failed.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "basic test", 3 | "code": "r = a + b", 4 | "input": { 5 | "a": 2, 6 | "b": 1 7 | }, 8 | "output": { 9 | "r": 4 10 | } 11 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/basic-invalid-ref.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "basic test with a ref", 3 | "code": "r = a + b", 4 | "input": { 5 | "a": 2, 6 | "b": 1 7 | }, 8 | "output": { 9 | "$ref": "outputs/non-existent.json" 10 | } 11 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/basic-multitest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "basic multitest", 3 | "code": "r = a + b", 4 | "input": [{ 5 | "a": 2, 6 | "b": 1 7 | },{ 8 | "a": 2, 9 | "b": 2 10 | }], 11 | "output": [{ 12 | "r": 3 13 | },{ 14 | "r": 4 15 | }] 16 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/basic-pipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "basic pipeline test", 3 | "code": { 4 | "pipeline": "a", 5 | "database": { 6 | "a": "r = a + b" 7 | } 8 | }, 9 | "input": { 10 | "a": 2, 11 | "b": 1 12 | }, 13 | "output": { 14 | "r": 3 15 | } 16 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/basic-ref-nofile.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "basic test with a nonexistent ref", 3 | "code": "r = a + b", 4 | "input": { 5 | "a": 2, 6 | "b": 1 7 | }, 8 | "output": { 9 | "$ref": "outputs/output-nonexistent.json" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/basic-ref.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "basic test with a ref", 3 | "code": "r = a + b", 4 | "input": { 5 | "a": 2, 6 | "b": 1 7 | }, 8 | "output": { 9 | "$ref": "outputs/output.json" 10 | } 11 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/basic.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "basic test", 3 | "code": { 4 | "pipeline": "a", 5 | "database": { 6 | "a": "r = a + b" 7 | } 8 | }, 9 | "input": { 10 | "a": 2, 11 | "b": 1 12 | }, 13 | "output": { 14 | "r": 3 15 | } 16 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/create-me.json: -------------------------------------------------------------------------------- 1 | { 2 | "r" : 4 3 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/idml/a.idml: -------------------------------------------------------------------------------- 1 | a = 1 2 | m = "a" 3 | -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/idml/b.idml: -------------------------------------------------------------------------------- 1 | a = a + 1 2 | m = "b" -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/inject-now.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "injected now test", 3 | "code" : "r = now()", 4 | "input" : { 5 | 6 | }, 7 | "output" : { 8 | "r" : "Mon, 01 Apr 2019 11:26:07 +0000" 9 | }, 10 | "time" : 1554117967 11 | } 12 | -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/null-behaviour.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "null behaviour", 3 | "code": "a = a\nb = b\nfoo.bar.baz.blargh = b", 4 | "input": { 5 | "b": null 6 | }, 7 | "output": { 8 | "foo": { 9 | "bar": { 10 | "baz": { 11 | } 12 | } 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/outputs/output.json: -------------------------------------------------------------------------------- 1 | { 2 | "r": 3 3 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/pipeline-ref.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pipeline reference test", 3 | "code": { 4 | "pipeline": "a|b", 5 | "database": { 6 | "a": { 7 | "$ref": "idml/a.idml" 8 | }, 9 | "b": { 10 | "$ref": "idml/b.idml" 11 | } 12 | } 13 | }, 14 | "input": { 15 | }, 16 | "output": { 17 | "a": 2, 18 | "m": "b" 19 | } 20 | } -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "example test", 3 | "code" : "r = a + b", 4 | "input" : { 5 | "a" : 2, 6 | "b" : 2 7 | }, 8 | "output" : { 9 | "$ref" : "create-me.json" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /idmltest/src/test/resources/tests/two-tests.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "basic test aaa", 4 | "code": "r = a + b", 5 | "input": { 6 | "a": 2, 7 | "b": 1 8 | }, 9 | "output": { 10 | "r": 3 11 | } 12 | }, 13 | { 14 | "name": "basic test bbb", 15 | "code": "r = a + b", 16 | "input": { 17 | "a": 2, 18 | "b": 1 19 | }, 20 | "output": { 21 | "r": 3 22 | } 23 | } 24 | ] -------------------------------------------------------------------------------- /idmltutor/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idmltutor" 2 | 3 | libraryDependencies ++= List( 4 | "org.jline" % "jline" % "3.13.3", 5 | "org.tpolecat" %% "atto-core" % "0.9.4", 6 | "com.lihaoyi" %% "fansi" % "0.2.13", 7 | "io.circe" %% "circe-literal" % "0.14.1" 8 | ) 9 | 10 | libraryDependencies ++= Seq( 11 | "org.mockito" % "mockito-all" % "1.9.5" % Test, 12 | "org.scalatest" %% "scalatest" % "3.2.8" % Test 13 | ) 14 | 15 | mainClass in Compile := Some("io.idml.tutor.Main") 16 | -------------------------------------------------------------------------------- /idmltutor/src/main/scala/io/idml/tutor/Colours.scala: -------------------------------------------------------------------------------- 1 | package io.idml.tutor 2 | 3 | import fansi.Color._ 4 | import fansi.Underlined 5 | 6 | object Colours { 7 | def cyan(s: String): String = Cyan(s).render 8 | def red(s: String): String = Red(s).render 9 | def green(s: String): String = Green(s).render 10 | def grey(s: String): String = DarkGray(s).render 11 | def yellow(s: String): String = Yellow(s).render 12 | def underlined(s: String): String = Underlined.On(s).render 13 | } 14 | -------------------------------------------------------------------------------- /idmltutor/src/main/scala/io/idml/tutor/Main.scala: -------------------------------------------------------------------------------- 1 | package io.idml.tutor 2 | 3 | import cats.effect.{ExitCode, IO, IOApp} 4 | import com.monovore.decline.{Command, Opts} 5 | import fansi.Color.{Cyan, Green, Red} 6 | 7 | object Main { 8 | import Colours._ 9 | 10 | val banner = 11 | """ 12 | " # ""# m m 13 | mmm mmm# mmmmm # mm#mm m m mm#mm mmm m mm 14 | # #" "# # # # # # # # # #" "# #" " 15 | # # # # # # # # # # # # # # 16 | mm#mm "#m## # # # "mm "mm "mm"# "mm "#m#" # 17 | """ 18 | 19 | def execute(): Command[IO[ExitCode]] = 20 | Command("tutor", "run the IDML tutor") { 21 | Opts.apply( 22 | for { 23 | jline <- JLine[IO]("idmltutor") 24 | _ <- jline.printAbove(banner) 25 | _ <- jline.printAbove(""" 26 | |Welcome to the IDML tutor, this is a utility for learning IDML 27 | | 28 | |Please select the option you'd like: 29 | | start - start at chapter 1 30 | | quit - quit 31 | | 32 | """.stripMargin) 33 | s <- jline.readLine(cyan("# ")) 34 | _ <- s match { 35 | case "start" => Chapter1(new TutorialAlg[IO](jline)) 36 | case _ => IO.unit 37 | } 38 | } yield ExitCode.Success 39 | ) 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /idmltutor/src/main/scala/io/idml/tutor/Utils.scala: -------------------------------------------------------------------------------- 1 | package io.idml.tutor 2 | 3 | import com.google.common.base.{CharMatcher, Strings} 4 | 5 | object Utils { 6 | 7 | def center(s: String, width: Int, realLength: Int) = { 8 | val padSize = width - realLength 9 | if (padSize <= 0) { 10 | s 11 | } else { 12 | Strings.padEnd( 13 | Strings.padStart(s, realLength + padSize / 2, ' '), 14 | width, 15 | ' ' 16 | ) 17 | } 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /jackson/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-jackson" 2 | 3 | libraryDependencies ++= List( 4 | "org.json4s" %% "json4s-jackson" % "4.0.3", 5 | "org.scalatest" %% "scalatest" % "3.2.8" % Test 6 | ) 7 | -------------------------------------------------------------------------------- /jackson/src/main/resources/META-INF/services/io.idml.IdmlJson: -------------------------------------------------------------------------------- 1 | io.idml.jackson.DefaultIdmlJackson -------------------------------------------------------------------------------- /jackson/src/main/resources/META-INF/services/io.idml.functions.FunctionResolver: -------------------------------------------------------------------------------- 1 | io.idml.jackson.JacksonFunctions -------------------------------------------------------------------------------- /jackson/src/main/scala/io/idml/jackson/DefaultIdmlJackson.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson 2 | 3 | class DefaultIdmlJackson extends IdmlJackson(IdmlJackson.newDefaultObjectMapper) 4 | -------------------------------------------------------------------------------- /jackson/src/main/scala/io/idml/jackson/JacksonFunctions.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson 2 | 3 | import io.idml.functions.json.JsonFunctions 4 | 5 | class JacksonFunctions extends JsonFunctions(IdmlJackson.default) 6 | -------------------------------------------------------------------------------- /jackson/src/main/scala/io/idml/jackson/difftool/DiffJacksonModule.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson.difftool 2 | 3 | import io.idml.IdmlValue 4 | import com.fasterxml.jackson.core.Version 5 | import com.fasterxml.jackson.databind.{BeanDescription, JavaType, Module, SerializationConfig} 6 | import com.fasterxml.jackson.databind.Module.SetupContext 7 | import com.fasterxml.jackson.databind.ser.Serializers 8 | 9 | class DiffJacksonModule extends Module { 10 | 11 | def getModuleName: String = "DiffModule" 12 | 13 | def version(): Version = Version.unknownVersion() 14 | 15 | def setupModule(ctxt: SetupContext) { 16 | ctxt.addSerializers(DiffJacksonSerializerResolver) 17 | } 18 | } 19 | 20 | /** An object that activates the de-serialization of PValues */ 21 | private object DiffJacksonSerializerResolver extends Serializers.Base { 22 | private val pvalue = classOf[IdmlValue] 23 | override def findSerializer( 24 | config: SerializationConfig, 25 | theType: JavaType, 26 | beanDesc: BeanDescription): DiffSerializer = { 27 | if (!pvalue.isAssignableFrom(theType.getRawClass)) { 28 | // scalastyle:off null 29 | null 30 | // scalastyle:on null 31 | } else { 32 | new DiffSerializer 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /jackson/src/main/scala/io/idml/jackson/difftool/DiffSerializer.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson.difftool 2 | 3 | import io.idml.{IdmlArray, IdmlNothing, IdmlValue} 4 | import com.fasterxml.jackson.core.JsonGenerator 5 | import com.fasterxml.jackson.databind.SerializerProvider 6 | import io.idml.jackson.serder.PValueSerializer 7 | 8 | class DiffSerializer extends PValueSerializer { 9 | override def serialize(value: IdmlValue, json: JsonGenerator, provider: SerializerProvider) { 10 | value match { 11 | case arr: IdmlArray if Diff.isDiff(arr) => 12 | val l = arr.get(1) 13 | val r = arr.get(2) 14 | if (!l.isInstanceOf[IdmlNothing]) { 15 | json.writeRaw("") 16 | serialize(l, json, provider) 17 | json.writeRaw("") 18 | } 19 | if (!r.isInstanceOf[IdmlNothing]) { 20 | json.writeRaw("") 21 | serialize(r, json, provider) 22 | json.writeRaw("") 23 | } 24 | case _ => 25 | super.serialize(value, json, provider) 26 | } 27 | } 28 | 29 | override def isEmpty(value: IdmlValue): Boolean = 30 | value.isInstanceOf[IdmlNothing] 31 | } 32 | -------------------------------------------------------------------------------- /jackson/src/main/scala/io/idml/jackson/serder/PValueSerializer.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson.serder 2 | 3 | import io.idml.{ 4 | IdmlArray, 5 | IdmlBool, 6 | IdmlDouble, 7 | IdmlInt, 8 | IdmlNothing, 9 | IdmlNull, 10 | IdmlObject, 11 | IdmlString, 12 | IdmlValue 13 | } 14 | import com.fasterxml.jackson.core.JsonGenerator 15 | import com.fasterxml.jackson.databind.{JsonSerializer, SerializerProvider} 16 | import io.idml.datanodes.IDomElement 17 | 18 | /** The Jackson serializer for PValues */ 19 | class PValueSerializer extends JsonSerializer[IdmlValue] { 20 | def serialize(value: IdmlValue, json: JsonGenerator, provider: SerializerProvider) { 21 | value match { 22 | case n: IdmlInt => json.writeNumber(n.value) 23 | case n: IdmlDouble => json.writeNumber(n.value) 24 | case n: IdmlString => json.writeString(n.value) 25 | case n: IdmlBool => json.writeBoolean(n.value) 26 | 27 | case n: IdmlObject => 28 | json.writeStartObject() 29 | n.fields.filterNot(_._2.isInstanceOf[IdmlNothing]).toList.sortBy(_._1).foreach { 30 | case (k, v) => 31 | json.writeObjectField(k, v) 32 | } 33 | json.writeEndObject() 34 | 35 | case n: IdmlArray => 36 | json.writeStartArray() 37 | n.items filterNot (_.isInstanceOf[IdmlNothing]) foreach json.writeObject 38 | json.writeEndArray() 39 | 40 | case _: IdmlNothing => () 41 | case IdmlNull => json.writeNull() 42 | } 43 | } 44 | 45 | override def isEmpty(value: IdmlValue): Boolean = 46 | value.isInstanceOf[IdmlNothing] 47 | } 48 | -------------------------------------------------------------------------------- /jackson/src/test/scala/io/idml/jackson/IDoubleParsingSpec.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson 2 | 3 | import io.idml.datanodes.IDouble 4 | import org.scalatest.funsuite.AnyFunSuite 5 | import org.scalatest.matchers.must.Matchers 6 | 7 | class IDoubleParsingSpec extends AnyFunSuite with Matchers { 8 | import IdmlJackson.default._ 9 | 10 | // Parsing 11 | test("parse(float min)")( 12 | pending 13 | ) // (DataNodes.parse(Float.MinValue.toString) must equal(new PDouble(Float.MinValue))) 14 | test("parse(float max)")( 15 | pending 16 | ) // (DataNodes.parse(Float.MaxValue.toString) must equal(new PDouble(Float.MaxValue))) 17 | test("parse(double min)")( 18 | parse(Double.MinValue.toString) must equal(new IDouble(Double.MinValue))) 19 | test("parse(double max)")( 20 | parse(Double.MaxValue.toString) must equal(new IDouble(Double.MaxValue))) 21 | 22 | // Generation 23 | test("generate(float min)")( 24 | pending 25 | ) // (Float.MinValue.toString must equal(compact(new PDouble(Float.MinValue)))) 26 | test("generate(float max)")( 27 | pending 28 | ) // (Float.MaxValue.toString must equal(compact(new PDouble(Float.MaxValue)))) 29 | test("generate(double min)")( 30 | Double.MinValue.toString must equal(compact(new IDouble(Double.MinValue)))) 31 | test("generate(double max)")( 32 | Double.MaxValue.toString must equal(compact(new IDouble(Double.MaxValue)))) 33 | } 34 | -------------------------------------------------------------------------------- /jackson/src/test/scala/io/idml/jackson/IdmlJsonTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson 2 | 3 | import java.nio.charset.Charset 4 | 5 | import io.idml.IdmlJson 6 | import org.scalatest.funsuite.AnyFunSuite 7 | 8 | class IdmlJsonTest extends AnyFunSuite { 9 | 10 | test("scala literal is equivalent to pile-of-poo byte array (sanity check)") { 11 | // The 'pile of poo', a 4-byte unicode character http://www.fileformat.info/info/unicode/char/1F4A9/index.htm 12 | val utf8PooBytes: Array[Byte] = 13 | Array('"'.toByte, 0xf0, 0x9f, 0x92, 0xa9, '"'.toByte).map(o => 14 | o.toByte.ensuring(o == 0 || _ != 0)) 15 | assert(utf8PooBytes === "\"\uD83D\uDCA9\"".getBytes(Charset.forName("UTF-8"))) 16 | } 17 | 18 | test("scala source pile of poo is serialized as a utf8 pile of poo") { 19 | assert( 20 | IdmlJackson.default 21 | .compact(IdmlJackson.default.parse("\"\uD83D\uDCA9\"")) === "\"\uD83D\uDCA9\"" 22 | ) 23 | } 24 | 25 | test("ascii escaped scala pile of poo is serialized as a utf8 pile of poo") { 26 | assert( 27 | IdmlJackson.default 28 | .compact(IdmlJackson.default.parse("\"\\uD83D\\uDCA9\"")) === "\"\uD83D\uDCA9\"" 29 | ) 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /jackson/src/test/scala/io/idml/jackson/IntParsingSpec.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson 2 | 3 | import io.idml.datanodes.IInt 4 | import org.scalatest.funsuite.AnyFunSuite 5 | import org.scalatest.matchers.must.Matchers 6 | 7 | class IntParsingSpec extends AnyFunSuite with Matchers { 8 | import IdmlJackson.default._ 9 | 10 | // Parsing 11 | test("parse(int min)")(parse(Int.MinValue.toString) === IInt(Int.MinValue)) 12 | test("parse(int max)")(parse(Int.MaxValue.toString) === IInt(Int.MaxValue)) 13 | test("parse(long min)")(parse(Long.MinValue.toString) === IInt(Long.MinValue)) 14 | test("parse(long max)")(parse(Long.MaxValue.toString) === IInt(Long.MaxValue)) 15 | 16 | // Generation 17 | test("generate(int min)")(Int.MinValue.toString === compact(IInt(Int.MinValue))) 18 | test("generate(int max)")(Int.MaxValue.toString === compact(IInt(Int.MaxValue))) 19 | test("generate(long min)")(Long.MinValue.toString === compact(IInt(Long.MinValue))) 20 | test("generate(long max)")(Long.MaxValue.toString === compact(IInt(Long.MaxValue))) 21 | 22 | } 23 | -------------------------------------------------------------------------------- /jackson/src/test/scala/io/idml/jackson/JacksonFunctionsSpec.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson 2 | 3 | import io.idml.json.JsonFunctionSuite 4 | 5 | class JacksonFunctionsSpec extends JsonFunctionSuite("JacksonFunctions", new JacksonFunctions()) 6 | -------------------------------------------------------------------------------- /jackson/src/test/scala/io/idml/jackson/NullParsingSpec.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson 2 | 3 | import io.idml.IdmlNull 4 | import org.scalatest.funsuite.AnyFunSuite 5 | import org.scalatest.matchers.must.Matchers 6 | 7 | class NullParsingSpec extends AnyFunSuite with Matchers { 8 | import IdmlJackson.default._ 9 | // Parsing 10 | test("parse(null)")(parse("null") must equal(IdmlNull)) 11 | 12 | // Generation 13 | test("generate(null)")("null" must equal(compact(IdmlNull))) 14 | 15 | } 16 | -------------------------------------------------------------------------------- /jackson/src/test/scala/io/idml/jackson/StringParsingSpec.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson 2 | 3 | import io.idml.IdmlJson 4 | import io.idml.datanodes.IString 5 | import org.scalatest.funsuite.AnyFunSuite 6 | import org.scalatest.matchers.must.Matchers 7 | 8 | class StringParsingSpec extends AnyFunSuite with Matchers { 9 | import IdmlJackson.default._ 10 | 11 | // Parsing 12 | test("parse string")(parse("\"a string\"") must equal(new IString("a string"))) 13 | 14 | // Generation 15 | test("generate string")("\"a string\"" must equal(compact(new IString("a string")))) 16 | } 17 | -------------------------------------------------------------------------------- /jackson/src/test/scala/io/idml/jackson/UUIDTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jackson 2 | 3 | import io.idml.{IdmlContext, IdmlValue} 4 | import io.idml.datanodes.{IObject, IString} 5 | import org.scalatest.funsuite.AnyFunSuite 6 | import org.scalatest.matchers.must.Matchers 7 | 8 | class UUIDTest extends AnyFunSuite with Matchers { 9 | val funcs = new JacksonFunctions 10 | import funcs.uuid._ 11 | 12 | def v3(pv: IdmlValue): IdmlValue = uuid3Function.eval(new IdmlContext(), pv) 13 | def v5(pv: IdmlValue): IdmlValue = uuid5Function.eval(new IdmlContext(), pv) 14 | 15 | /* 16 | I generated these using the python uuid3 and uuid5 methods to make sure they're the same 17 | */ 18 | test("uuid3")( 19 | v3(IString("helloworld")) must equal(IString("fc5e038d-38a5-3032-8854-41e7fe7010b0"))) 20 | test("uuid5")( 21 | v5(IString("helloworld")) must equal(IString("6adfb183-a4a2-594a-af92-dab5ade762a4"))) 22 | test("uuid3 on an object")( 23 | v3(IObject()) must equal(IString("99914b93-2bd3-3a50-b983-c5e7c90ae93b"))) 24 | test("uuid5 on an object")( 25 | v5(IObject()) must equal(IString("bf21a9e8-fbc5-5384-afb0-5b4fa0859e09"))) 26 | 27 | } 28 | -------------------------------------------------------------------------------- /jsoup/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-jsoup" 2 | 3 | libraryDependencies ++= Seq( 4 | "org.jsoup" % "jsoup" % "1.8.1", 5 | "org.scalatest" %% "scalatest" % "3.2.8" % Test, 6 | "com.storm-enroute" %% "scalameter" % "0.19" % Test 7 | ) 8 | -------------------------------------------------------------------------------- /jsoup/src/main/resources/META-INF/services/io.idml.functions.FunctionResolver: -------------------------------------------------------------------------------- 1 | io.idml.jsoup.JsoupFunctionResolver 2 | -------------------------------------------------------------------------------- /jsoup/src/main/scala/io/idml/jsoup/IdmlJsoup.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jsoup 2 | 3 | import io.idml.IdmlValue 4 | import org.jsoup.Jsoup 5 | import org.jsoup.parser.Parser 6 | 7 | object IdmlJsoup { 8 | 9 | /** Parse an xml document 10 | */ 11 | def parseXml(xml: String): IdmlValue = 12 | JsoupConverter(Jsoup.parse(xml, "", Parser.xmlParser())) 13 | 14 | /** Parse a html document 15 | */ 16 | def parseHtml(html: String): IdmlValue = 17 | JsoupConverter(Jsoup.parse(html, "", Parser.htmlParser())) 18 | 19 | } 20 | -------------------------------------------------------------------------------- /jsoup/src/main/scala/io/idml/jsoup/JsoupConverter.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jsoup 2 | 3 | import io.idml.datanodes.{IDomElement, IDomText} 4 | import io.idml.IdmlValue 5 | import org.jsoup.nodes.{Element, TextNode} 6 | 7 | import scala.collection.JavaConverters.iterableAsScalaIterableConverter 8 | import cats._, cats.data._, cats.implicits._ 9 | 10 | object JsoupConverter { 11 | def apply(el: Element): IdmlValue = go(el).value 12 | 13 | // Recursive implementation of the tree transform that uses Eval to be stack-safe 14 | private def go(el: Element): Eval[IDomElement] = { 15 | val name = el.tagName() 16 | val attrs = el.attributes().asList().asScala.map { a => a.getKey -> a.getValue }.toMap 17 | val children = el.childNodes().asScala.toList.traverse { 18 | case t: TextNode => Eval.now(Some(IDomText(t.getWholeText))) 19 | case e: Element => go(e).map(Some(_)) 20 | case _ => Eval.now(None) 21 | } 22 | children.map { c => 23 | IDomElement(name, attrs, c.flatten) 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /jsoup/src/main/scala/io/idml/jsoup/JsoupFunctionResolver.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jsoup 2 | 3 | import io.idml.ast.{Argument, IdmlFunction, IdmlFunctionMetadata} 4 | import io.idml.functions.FunctionResolver 5 | 6 | class JsoupFunctionResolver extends FunctionResolver { 7 | override def resolve(name: String, args: List[Argument]): Option[IdmlFunction] = { 8 | (name, args) match { 9 | case ("stripTags", Nil) => Some(StripTagsFunction) 10 | case ("parseXml", Nil) => Some(ParseXmlFunction) 11 | case ("parseHtml", Nil) => Some(ParseHtmlFunction) 12 | case _ => None 13 | } 14 | } 15 | override def providedFunctions(): List[IdmlFunctionMetadata] = 16 | List( 17 | IdmlFunctionMetadata("stripTags", List.empty, "remove XML tags from this string"), 18 | IdmlFunctionMetadata("parseXml", List.empty, "parse this string as XML"), 19 | IdmlFunctionMetadata("parseHtml", List.empty, "parse this string as HTML") 20 | ) 21 | } 22 | -------------------------------------------------------------------------------- /jsoup/src/main/scala/io/idml/jsoup/ParseHtmlFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jsoup 2 | 3 | import io.idml.ast.{IdmlFunction, Pipeline} 4 | import io.idml.datanodes.IString 5 | import io.idml.{IdmlContext, InvalidCaller} 6 | 7 | object ParseHtmlFunction extends IdmlFunction { 8 | override def name: String = "parseHtml" 9 | 10 | override def invoke(ctx: IdmlContext): Unit = { 11 | ctx.cursor = ctx.cursor match { 12 | case IString(str) => 13 | IdmlJsoup.parseHtml(str) 14 | case _ => 15 | InvalidCaller 16 | } 17 | } 18 | 19 | override def args: List[Pipeline] = Nil 20 | } 21 | -------------------------------------------------------------------------------- /jsoup/src/main/scala/io/idml/jsoup/ParseXmlFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jsoup 2 | 3 | import io.idml.ast.{IdmlFunction, Pipeline} 4 | import io.idml.datanodes.IString 5 | import io.idml.{IdmlContext, InvalidCaller} 6 | import org.jsoup.Jsoup 7 | import org.jsoup.nodes.Document 8 | import org.jsoup.parser.Parser 9 | import org.jsoup.safety.Whitelist 10 | 11 | object ParseXmlFunction extends IdmlFunction { 12 | override def name: String = "parseXml" 13 | 14 | override def invoke(ctx: IdmlContext): Unit = { 15 | ctx.cursor = ctx.cursor match { 16 | case IString(str) => 17 | IdmlJsoup.parseXml(str) 18 | case _ => 19 | InvalidCaller 20 | } 21 | } 22 | 23 | override def args: List[Pipeline] = Nil 24 | } 25 | -------------------------------------------------------------------------------- /jsoup/src/main/scala/io/idml/jsoup/StripTagsFunction.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jsoup 2 | 3 | import io.idml.datanodes.IString 4 | import io.idml.{IdmlContext, InvalidCaller} 5 | import io.idml.ast.{IdmlFunction, Pipeline} 6 | import org.jsoup.Jsoup 7 | import org.jsoup.nodes.Document 8 | import org.jsoup.parser.Parser 9 | import org.jsoup.safety.Whitelist 10 | 11 | object StripTagsFunction extends IdmlFunction { 12 | override def name: String = "stripTags" 13 | 14 | override def invoke(ctx: IdmlContext): Unit = { 15 | ctx.cursor = ctx.cursor match { 16 | case IString(str) => 17 | IString( 18 | Parser.unescapeEntities( 19 | Jsoup.clean( 20 | str, 21 | "", 22 | Whitelist.none(), 23 | new Document.OutputSettings().prettyPrint(false)), 24 | false 25 | ) 26 | ) 27 | case _ => 28 | InvalidCaller 29 | } 30 | } 31 | 32 | override def args: List[Pipeline] = Nil 33 | } 34 | -------------------------------------------------------------------------------- /jsoup/src/test/scala/io/idml/jsoup/JsoupBench.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jsoup 2 | 3 | import io.idml.IdmlScalaMeterBase 4 | 5 | object JsoupBench extends IdmlScalaMeterBase("tests") 6 | -------------------------------------------------------------------------------- /jsoup/src/test/scala/io/idml/jsoup/JsoupElementTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jsoup 2 | 3 | import io.idml.MissingField 4 | import io.idml.datanodes.{IDomElement, IDomText, IString} 5 | import org.jsoup.Jsoup 6 | import org.jsoup.nodes.Element 7 | import org.jsoup.parser.Parser 8 | import org.scalatest.funsuite.AnyFunSuite 9 | import org.scalatest.matchers.must.Matchers 10 | 11 | import scala.language.implicitConversions 12 | 13 | class JsoupElementTest extends AnyFunSuite with Matchers { 14 | 15 | test("get(missing_tag) returns empty element") { 16 | IdmlJsoup 17 | .parseXml("abcdef") 18 | .get("c") 19 | .asInstanceOf[IDomElement] 20 | .items 21 | .toList must equal(List.empty) 22 | } 23 | 24 | test("get(nested_tag)[0] returns nested tag") { 25 | IdmlJsoup.parseXml("abcdef").get("b").get(0) must equal( 26 | IDomElement("b", Map.empty, List(IDomText("def")))) 27 | } 28 | 29 | test("get(deeply_nested_tag) returns missing empty array") { 30 | IdmlJsoup 31 | .parseXml("abcdef") 32 | .get("b") 33 | .asInstanceOf[IDomElement] 34 | .items 35 | .toList must equal(List.empty) 36 | } 37 | 38 | test("pull some text out") { 39 | IdmlJsoup.parseXml("not mehelloworld").get("b").text must equal( 40 | IString("helloworld")) 41 | IdmlJsoup.parseXml("not mehelloworld").get("a").text must equal( 42 | IString("not me")) 43 | IdmlJsoup.parseXml("not mehelloworld").text must equal( 44 | IString("not mehelloworld")) 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /jsoup/src/test/scala/io/idml/jsoup/JsoupItTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.jsoup 2 | 3 | import io.idml.IdmlScalaTestBase 4 | 5 | /** Execute all jsoup integration tests */ 6 | class JsoupItTest extends IdmlScalaTestBase("tests") 7 | -------------------------------------------------------------------------------- /lang/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-lang" 2 | 3 | enablePlugins(Antlr4Plugin) 4 | 5 | antlr4GenListener in Antlr4 := true 6 | 7 | antlr4GenVisitor in Antlr4 := true 8 | 9 | antlr4Version in Antlr4 := "4.5" 10 | 11 | antlr4Dependency in Antlr4 := 12 | "org.antlr" % "antlr4" % "4.5" exclude ("org.antlr", "ST4") exclude ("org.antlr", "antlr-runtime") // BSD license 13 | 14 | libraryDependencies ++= Seq( 15 | "com.google.guava" % "guava" % "27.0-jre", // Apache License 2 16 | "org.scalatest" %% "scalatest" % "3.2.8" % Test // Apache License 2 17 | ) 18 | -------------------------------------------------------------------------------- /lang/src/main/antlr4/MappingTest.g4: -------------------------------------------------------------------------------- 1 | grammar MappingTest; 2 | 3 | document : (testOptions | testMappings | testCase)* text?; 4 | 5 | // Grammar rules for interpreter actions 6 | testOptions : text? Options; 7 | testMappings : text? Mappings; 8 | testCase : text? Input text? (Output | Exception | Metrics); 9 | 10 | // Grammar rules for documentation 11 | text : (header | other)+; 12 | header : Header; 13 | other : Any+; 14 | 15 | // Comments 16 | Comment1 : '//' ~( '\r' | '\n' )* -> skip; 17 | Comment2 : '/*' .*? '*/' -> skip; 18 | 19 | // Lexer symbols for interpreter actions 20 | Mappings : '+++' .*? '+++'; 21 | Input : '<<<' .*? '<<<'; 22 | Output : '>>>' .*? '>>>'; 23 | Exception : '!!!' .*? '!!!'; 24 | Metrics : '~~~' .*? '~~~'; 25 | Options : '@@@' .*? '@@@'; 26 | 27 | // Lexer symbols for documentation 28 | Header : '#' ~( '\r' | '\n' )*; 29 | Any : .+?; 30 | -------------------------------------------------------------------------------- /lang/src/main/scala/io/idml/lang/ThrowConsoleErrorListener.scala: -------------------------------------------------------------------------------- 1 | package io.idml.lang 2 | 3 | import org.antlr.v4.runtime.{ConsoleErrorListener, RecognitionException, Recognizer} 4 | 5 | /** Thrown when IDML failed to parse */ 6 | // scalastyle:off null 7 | class DocumentParseException(str: String, ex: Exception = null) extends Exception(str, ex) 8 | // scalastyle:on null 9 | 10 | /** Causes an antlr parser or lexer to throw an exception when an error is encountered */ 11 | class ThrowConsoleErrorListener extends ConsoleErrorListener { 12 | override def syntaxError( 13 | recognizer: Recognizer[_, _], 14 | offendingSymbol: AnyRef, 15 | line: Int, 16 | charPositionInLine: Int, 17 | msg: String, 18 | ex: RecognitionException 19 | ) { 20 | throw new DocumentParseException("Line " + line + ":" + charPositionInLine + " " + msg, ex) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /lang/src/test/resources/literals.ini: -------------------------------------------------------------------------------- 1 | strings.hello = "hello world" 2 | strings.empty = "" 3 | ints.zero = 0 4 | ints.one = 1 5 | ints.maxint = 2147483647 6 | ints.minint = -2147483648 7 | floats.pi = 3.14159265359 8 | floats.negativepi = -3.14159265359 9 | floats.e = 2.71828182845904523536028747135266249775724709369995 10 | floats.negativee = -2.71828182845904523536028747135266249775724709369995 11 | floats.zero = 0.0 12 | floats.negativeone = -1.0 13 | bools.t = true 14 | bools.f = false 15 | -------------------------------------------------------------------------------- /lang/src/test/resources/regression/case-sensitive-ops.ini: -------------------------------------------------------------------------------- 1 | a = [ a in "" ] a 2 | 3 | # operators are case sensitive. doesn't seem to be an obvious way of fixing this. 4 | # maybe https://gist.github.com/sharwell/9424666 ? 5 | #a = { a In "" } a -------------------------------------------------------------------------------- /lang/src/test/resources/regression/coalesce-dynamic-paths.ini: -------------------------------------------------------------------------------- 1 | # Try out a full range of weird looking paths 2 | a = (a.a | 3 | a.a.a [ a exists ] | 4 | a.fn().a | 5 | a.fn().a [ a exists ] | 6 | now() | 7 | this | 8 | this [ this exists ] 9 | ).a -------------------------------------------------------------------------------- /lang/src/test/resources/regression/fn-after-relative-path-filter.ini: -------------------------------------------------------------------------------- 1 | # can be applied either as a pre-condition to the apply or a post-condition to the dot 2 | a = this [ a exists ] .apply("a") 3 | a = [ a exists ] apply("a") -------------------------------------------------------------------------------- /lang/src/test/resources/regression/strings-with-quotes.ini: -------------------------------------------------------------------------------- 1 | string_with_quotes = "a\"b\"c" -------------------------------------------------------------------------------- /lang/src/test/scala/io/idml/lang/MappingTestBase.scala: -------------------------------------------------------------------------------- 1 | package io.idml.lang 2 | 3 | import java.nio.charset.Charset 4 | import com.google.common.io.Resources 5 | import org.antlr.v4.runtime._ 6 | import org.scalatest.funsuite.AnyFunSuite 7 | import org.scalatest.matchers.must.Matchers 8 | 9 | class MappingTestBase extends AnyFunSuite with Matchers { 10 | 11 | def test(filename: String) { 12 | super.test("Parsing " + filename) { 13 | val str = Resources.toString(Resources.getResource(filename), Charset.defaultCharset()) 14 | val input = new ANTLRInputStream(str) 15 | val lexer = new MappingLexer(input) 16 | val tokens = new CommonTokenStream(lexer) 17 | val parser = new MappingParser(tokens) 18 | 19 | lexer.removeErrorListeners() 20 | parser.removeErrorListeners() 21 | // parser.addErrorListener(new DiagnosticErrorListener) 22 | parser.addErrorListener(new ThrowConsoleErrorListener) 23 | 24 | parser.document() 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /lang/src/test/scala/io/idml/lang/ParseMapFileResourcesTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.lang 2 | 3 | /** Parse embedded resources */ 4 | class ParseMapFileResourcesTest extends MappingTestBase { 5 | test("regression/case-sensitive-ops.ini") 6 | test("regression/coalesce-dynamic-paths.ini") 7 | test("regression/fn-after-relative-path-filter.ini") 8 | test("regression/strings-with-quotes.ini") 9 | 10 | test("support_ticket.ini") 11 | test("atom.ini") 12 | test("literals.ini") 13 | } 14 | -------------------------------------------------------------------------------- /project/aether-deploy.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("no.arktekk.sbt" % "aether-deploy" % "0.21") 2 | addSbtPlugin("ch.epfl.scala" % "sbt-scalafix" % "0.9.18-1") 3 | -------------------------------------------------------------------------------- /project/assembly.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") 2 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.5.1 2 | -------------------------------------------------------------------------------- /project/buildinfo.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0") 2 | -------------------------------------------------------------------------------- /project/docsplugins.sbt: -------------------------------------------------------------------------------- 1 | //addSbtPlugin("io.idml" % "idmldoc-plugin" % "1.0.23") 2 | //addSbtPlugin("com.47deg" % "sbt-microsites" % "1.3.4") 3 | //libraryDependencies += "com.47deg" %% "org-policies-core" % "0.10.0" 4 | //addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.1") 5 | //libraryDependencies += "com.google.guava" % "guava" % "10.0.1" 6 | -------------------------------------------------------------------------------- /project/license.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbt" % "sbt-license-report" % "1.2.0") 2 | -------------------------------------------------------------------------------- /project/native-packager.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.6.1") 2 | addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") 3 | -------------------------------------------------------------------------------- /project/proguard.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.lightbend.sbt" % "sbt-proguard" % "0.4.0") 2 | -------------------------------------------------------------------------------- /project/sbt-antlr4.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.simplytyped" % "sbt-antlr4" % "0.8.3") 2 | -------------------------------------------------------------------------------- /project/scalafmt.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.2") 2 | -------------------------------------------------------------------------------- /project/sonatype.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.12") 2 | addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.1.2") 3 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Ingestion Data Mapping Language 2 | 3 | [![Build Status](https://cloud.drone.io/api/badges/IDML/idml/status.svg)](https://cloud.drone.io/IDML/idml) 4 | 5 | 6 | See http://idml.io/ 7 | 8 | ## History 9 | 10 | IDML has been developed and maintained by [DataSift](https://datasift.com) (now a [Meltwater](https://www.meltwater.com) company), where it's been used to deliver high throughput social media firehoses and to allow customers to onboard data easily. 11 | 12 | ## Releasing to Sonatype 13 | 14 | If you're a member of the `io.idml` organisation on sonatype you can perform a release with these commands: 15 | 16 | 1. add `default-key $KEYID` to `~/.gnupg/gpg.conf` to specify the key to release with 17 | 2. `++ publishSigned` 18 | 3. `sonatypePrepare` 19 | 4. `sonatypeBundleUpload` 20 | 5. `sonatypeRelease` 21 | 6. `project tool` then `assembly` to build the tool jar 22 | 7. get the tool jar from `./tool/target/scala-2.12/idml-tool-$VERSION-assembly.jar` 23 | 8. create a release on github with the format `$VERSION: Title of Release` 24 | 9. Upload the assembly jar to that release and write release notes 25 | 26 | 27 | ## Special Thanks 28 | 29 | This project has had many contributors before being open sourced, these include: 30 | 31 | * Andi Miller 32 | * Jon Davey 33 | * Stuart Dallas 34 | * Courtney Robinson 35 | * James Bloomer 36 | -------------------------------------------------------------------------------- /test/build.sbt: -------------------------------------------------------------------------------- 1 | name := "test" 2 | 3 | publishArtifact := false 4 | 5 | libraryDependencies ++= List( 6 | "org.scalatest" %% "scalatest" % "3.2.8" % Test, 7 | "org.mockito" % "mockito-all" % "1.9.0" % Test, 8 | "org.scalatestplus" %% "mockito-3-4" % "3.2.8.0" % Test, 9 | "com.storm-enroute" %% "scalameter" % "0.19" % Test, 10 | "io.circe" %% "circe-testing" % "0.13.0" % Test, 11 | "org.scalatest" %% "scalatest" % "3.2.8" % Test 12 | ) 13 | -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.ast/ArraysSuiteNestedArrays.ini: -------------------------------------------------------------------------------- 1 | [main] 2 | animals = animales.apply("row") 3 | 4 | [row] 5 | name = this[0] 6 | species = this[1] 7 | type = root.input_type -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.ast/ArraysSuiteNestedObjects.ini: -------------------------------------------------------------------------------- 1 | [main] 2 | animals = animales.apply("row") 3 | 4 | [row] 5 | name = nom.required() 6 | species = espece.required() 7 | type = root.input_type -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.ast/BackticksSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | { 4 | "name": "when labels contain characters that prevent them from being written as normal labels, text can be surrounded by backticks", 5 | "mapping": "x = `backticks: my incredible message!`.string()", 6 | "input": {"backticks: my incredible message!": "abc"}, 7 | "output": {"x": "abc"} 8 | }, 9 | 10 | { 11 | "name": "root label", 12 | "mapping": "`root` = `root`.string()", 13 | "input": { 14 | "root": "root_value" 15 | }, 16 | "output": { 17 | "root": "root_value" 18 | } 19 | }, 20 | 21 | 22 | { 23 | "name": "this label", 24 | "mapping": "`this` = `this`.string()", 25 | "input": { 26 | "this": "this_value" 27 | }, 28 | "output": { 29 | "this": "this_value" 30 | } 31 | } 32 | 33 | 34 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.ast/BooleanSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "true", 4 | "mapping": "t = true", 5 | "input": {}, 6 | "output": {"t": true} 7 | }, 8 | 9 | { 10 | "name": "false", 11 | "mapping": "f = false", 12 | "input": {}, 13 | "output": {"f": false} 14 | }, 15 | 16 | { 17 | "name": "filter with true", 18 | "mapping": "output = input [ root.condition == true ]", 19 | "input": {"input": "tokendata", "condition": true}, 20 | "output": {"output": "tokendata"} 21 | }, 22 | 23 | { 24 | "name": "filter with false", 25 | "mapping": "output = input [ root.condition == false ]", 26 | "input": {"input": "tokendata", "condition": false}, 27 | "output": {"output": "tokendata"} 28 | }, 29 | 30 | { 31 | "name": "bypassing booleans with backticks", 32 | "mapping": "output = `true`", 33 | "input": {"true": "hello"} , 34 | "output": {"output": "hello"} 35 | }, 36 | 37 | { 38 | "name": "bypassing booleans with backticks in absolute paths", 39 | "mapping": "output = root.`true`", 40 | "input": {"true": "hello"}, 41 | "output": {"output": "hello"} 42 | }, 43 | 44 | { 45 | "name": "name a variable true using backticks", 46 | "mapping": "`true` = thing", 47 | "input": {"thing": "hello world"}, 48 | "output": {"true": "hello world"} 49 | } 50 | 51 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.ast/IfSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "if expressions should assign", 4 | "mapping": "result = if 1 == 1 then true else false", 5 | "input": {}, 6 | "output": {"result": true} 7 | }, 8 | 9 | { 10 | "name": "if expressions should work on complex things", 11 | "mapping": "result = if a.thingy exists then 2 + 2 else 7 * 3", 12 | "input": {"a": {"thingy": "I exist"}}, 13 | "output": {"result": 4} 14 | }, 15 | 16 | { 17 | "name": "if expressions should work without an else", 18 | "mapping": "result = if 2>1 then \"yes\"", 19 | "input": {}, 20 | "output": {"result": "yes"} 21 | }, 22 | 23 | { 24 | "name": "if expressions should work without an else part 2", 25 | "mapping": "result = if 1>2 then \"yes\"", 26 | "input": {}, 27 | "output": {} 28 | }, 29 | { 30 | "name": "array 'contains' operator", 31 | "mapping": "output = if xs contains 2 then \"yes\" else \"no\"", 32 | "input": {"xs": [1,2,3]}, 33 | "output": {"output": "yes"} 34 | }, 35 | { 36 | "name": "array 'contains' operator 2", 37 | "mapping": "output = if xs contains 4 then \"yes\" else \"no\"", 38 | "input": {}, 39 | "output": {"output": "no"} 40 | }, 41 | { 42 | "name": "array 'in' operator", 43 | "mapping": "output = if 2 in xs then \"yes\" else \"no\"", 44 | "input": {"xs": [1,2,3]}, 45 | "output": {"output": "yes"} 46 | }, 47 | { 48 | "name": "array 'in' operator 2", 49 | "mapping": "output = if 4 in xs then \"yes\" else \"no\"", 50 | "input": {}, 51 | "output": {"output": "no"} 52 | } 53 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.ast/MatchSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "matching on a literal", 4 | "mapping": "output = match 2\n| this == 2 => \"two\"", 5 | "input": {}, 6 | "output": {"output": "two"} 7 | }, 8 | 9 | { 10 | "name": "matching on a literal and falling through", 11 | "mapping": "output = match 2\n| this == 1 => \"one\"", 12 | "input": {}, 13 | "output": {} 14 | }, 15 | 16 | { 17 | "name": "matching on input", 18 | "mapping": "output = match input\n| this == 1 => \"one\"\n| this == 2 => \"two\"", 19 | "input": {"input": 2}, 20 | "output": {"output": "two"} 21 | }, 22 | 23 | { 24 | "name": "matching on input with a complex predicate", 25 | "mapping": "output = match input\n| this < 10 and this > 5 => true\n", 26 | "input": {"input": 7}, 27 | "output": {"output": true} 28 | }, 29 | 30 | { 31 | "name": "matching on input with a complex predicate and failing", 32 | "mapping": "output = match input\n| this < 10 and this > 5 => true\n", 33 | "input": {"input": 42}, 34 | "output": {} 35 | }, 36 | 37 | { 38 | "name": "underscore should be a final case block", 39 | "mapping": "output = match input | this == true => 1 | _ => 2 | _ => 3", 40 | "input": {}, 41 | "output": {"output": 2} 42 | }, 43 | 44 | { 45 | "name": "falling through should not happen", 46 | "mapping": "output = match input | _ => a | _ => b | _ => 3", 47 | "input": {}, 48 | "output": {} 49 | } 50 | 51 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.ast/TemporaryVariableSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "the temporary variable operator should assign temporary variables", 4 | "mapping": "let foo = \"bar\"", 5 | "input": {}, 6 | "output": {} 7 | }, 8 | 9 | { 10 | "name": "temporary variables should be usable", 11 | "mapping": "let hello = \"hello\"\nlet world = \"world\"\nresult = $hello + \" \" + $world", 12 | "input": {}, 13 | "output": {"result": "hello world"} 14 | } 15 | 16 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.ast/VariablesSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "variable assignment - missing value", 4 | "mapping": "b = a \n c = @b", 5 | "input": {}, 6 | "output": {} 7 | }, 8 | 9 | { 10 | "name": "variable assignment", 11 | "mapping": "b = a \n c = @b", 12 | "input": {"a": 10}, 13 | "output": {"b": 10, "c": 10} 14 | } 15 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.ast/WildcardsSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "top-level wildcard", 4 | "mapping": "x = *", 5 | "input": {"b": 10, "c": 11}, 6 | "output": {"x": {"b": 10, "c": 11}} 7 | }, 8 | { 9 | "name": "depth-1 wildcard", 10 | "mapping": "x = a.*", 11 | "input": {"a": {"b": 10, "c": 11}}, 12 | "output": {"x": {"b": 10, "c": 11}} 13 | } 14 | 15 | /*, 16 | { 17 | "name": "expressions after wildcards - we'll find a.b.c", 18 | "mapping": "x = a.*.c", 19 | "input": {"a": {"b": {"c": 10}}}, 20 | "output": {"x": 10} 21 | }, 22 | { 23 | "name": "expressions after wildcards - don't find a.c", 24 | "mapping": "x = a.*.c", 25 | "input": {"a": {"b": {"c": 10}, "c": 11}}, 26 | "output": {"x": {"b": {"c": 10}}} 27 | }, 28 | { 29 | "name": "expressions after wildcards - don't find a.c.d", 30 | "mapping": "x = a.*.c", 31 | "input": {"a": {"b": {"c": 10}, "c": {"d": 11}}}, 32 | "output": {"x": 10} 33 | }, 34 | { 35 | "name": "expressions after wildcards - find a.b.c and a.c.c", 36 | "mapping": "x = a.*.c", 37 | "input": {"a": {"b": {"c": 10}}}, 38 | "output": {"x": {"b": 10}} 39 | }, 40 | { 41 | "name": "multiple wildcards", 42 | "mapping": "x = a.*.*", 43 | "input": {"a": {"b": {"c": 10}, "c": {"d": 11}, "e": 12}}, 44 | "output": {"x": {"b": {"c": 10}, "c": {"d": 11}}} 45 | }*/ 46 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/AppendSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "can be used in mappings", 4 | "mapping": "a = b.append(true)", 5 | "input": {}, 6 | "output": {} 7 | }, 8 | 9 | { 10 | "name": "does nothing if the caller is missing", 11 | "mapping": "a = b.append(true)", 12 | "input": {}, 13 | "output": {} 14 | }, 15 | 16 | { 17 | "name": "does nothing if the caller is not an array", 18 | "mapping": "a = b.append(true)", 19 | "input": {"b": true}, 20 | "output": {} 21 | }, 22 | 23 | { 24 | "name": "passes through the caller untouched if the parameter is missing", 25 | "mapping": "a = b.append(missing)", 26 | "input": {"b": [true]}, 27 | "output": {"a": [true]} 28 | }, 29 | 30 | { 31 | "name": "can add a value to the end of an array", 32 | "mapping": "a = b.append(false)", 33 | "input": {"b": [true]}, 34 | "output": {"a": [true, false]} 35 | }, 36 | 37 | { 38 | "name": "does not modify the original array", 39 | "mapping": "a = b.append(false)\n c = b", 40 | "input": {"b": [true]}, 41 | "output": {"a": [true, false], "c": [true]} 42 | } 43 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/ApplyArraySuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "this becomes the whole array rather than the individual item", 4 | "mapping": "[main] a = x.applyArray(\"my_block\") \n [my_block] b = this[0] + 1 \n c = this[1] + 2", 5 | "input": { 6 | "x": [ 7 | 10, 8 | 20 9 | ] 10 | }, 11 | "output": { 12 | "a": { 13 | "b": 11, 14 | "c": 22 15 | } 16 | } 17 | } 18 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/ArraySuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "apply a function to each item in an array", 4 | "mapping": "output = input.array(int())", 5 | "input": {"input": ["123", "456", "789"]} , 6 | "output": {"output": [123, 456, 789]} 7 | }, 8 | { 9 | "name": "combineAll should merge objects with the same keys", 10 | "mapping": "x = [{\"x\":{\"a\":1}}, {\"x\":{\"b\":2}},{\"x\":{\"a\":3}}].combineAll().x", 11 | "input": {}, 12 | "output": {"x": {"a": 3, "b": 2}} 13 | } 14 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/BlacklistSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "can blacklist one field", 4 | "mapping": "a = b.blacklist(\"p\")", 5 | "input": { 6 | "b": { 7 | "p": true, 8 | "q": true 9 | } 10 | }, 11 | "output": { 12 | "a": { 13 | "q": true 14 | } 15 | } 16 | }, 17 | 18 | { 19 | "name": "can blacklist two fields", 20 | "mapping": "a = b.blacklist(\"p\", \"q\")", 21 | "input": { 22 | "b": { 23 | "p": true, 24 | "q": true, 25 | "r": true 26 | } 27 | }, 28 | "output": { 29 | "a": { 30 | "r": true 31 | } 32 | } 33 | }, 34 | 35 | 36 | { 37 | "name": "object is unchanged if nothing is blacklisted", 38 | "mapping": "a = b.blacklist()", 39 | "input": { 40 | "b": { 41 | "p": true 42 | } 43 | }, 44 | "output": { 45 | "a": { 46 | "p": true 47 | } 48 | } 49 | }, 50 | 51 | { 52 | "name": "blacklist doesn't affect original object", 53 | "mapping": "a = b.blacklist(\"p\", \"q\") \n c = b", 54 | "input": { 55 | "b": { 56 | "p": true, 57 | "q": true, 58 | "r": true 59 | } 60 | }, 61 | "output": { 62 | "a": { 63 | "r": true 64 | }, 65 | "c": { 66 | "p": true, 67 | "q": true, 68 | "r": true 69 | } 70 | } 71 | } 72 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/ConcatSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "concat some strings", 4 | "mapping": "output = input.concat(\" \")", 5 | "input": {"input": ["hello", "world"]}, 6 | "output": {"output": "hello world" } 7 | }, 8 | { 9 | "name": "concat sparse arrays", 10 | "mapping": "output = input.a.wrapArray().append(input.b).append(input.c).concat(\" \")", 11 | "input": {"input": {"b": "hello"}}, 12 | "output": {"output": "hello"} 13 | }, 14 | { 15 | "name": "concat sparse arrays 2", 16 | "mapping": "output = input.a.wrapArray().append(input.b).append(input.c).concat(\" \")", 17 | "input": {"input": {"a": "hello"}}, 18 | "output": {"output": "hello"} 19 | }, 20 | { 21 | "name": "concat sparse arrays 3", 22 | "mapping": "output = input.a.wrapArray().append(input.b).append(input.c).concat(\" \")", 23 | "input": {"input": {"c": "hello"}}, 24 | "output": {"output": "hello"} 25 | }, 26 | { 27 | "name": "concat sparse arrays 4", 28 | "mapping": "output = input.a.wrapArray().append(input.b).append(input.c).concat(\" \")", 29 | "input": {"input": {}}, 30 | "output": {} 31 | } 32 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/DataScienceArraysSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "softmax should work the way it says it does on the wikipedia page", 4 | "mapping": "output = input.softmax().extract(this.round(3))", 5 | "input": {"input": [1,2,3,4,1,2,3]}, 6 | "output": {"output": [0.024, 0.064, 0.175, 0.475, 0.024, 0.064, 0.175]} 7 | }, 8 | { 9 | "name": "average should do a normal average", 10 | "mapping": "output = input.average()", 11 | "input": {"input": [1,2,3,4,5]}, 12 | "output": {"output": 3.0} 13 | }, 14 | { 15 | "name": "median should work on uneven length lists", 16 | "mapping": "output = input.median()", 17 | "input": {"input": [1,2,3,4,5]}, 18 | "output": {"output": 3} 19 | }, 20 | { 21 | "name": "median should work on even length lists", 22 | "mapping": "output = input.median()", 23 | "input": {"input": [1,2,3,4]}, 24 | "output": {"output": 2.5} 25 | }, 26 | { 27 | "name": "variance should calculate variance", 28 | "mapping": "output = input.variance()", 29 | "input": {"input": [1,2,3,4,5]}, 30 | "output": {"output": 2.0} 31 | }, 32 | { 33 | "name": "stdDev should calculate standard deviation", 34 | "mapping": "output = input.stdDev().round(3)", 35 | "input": {"input": [1,2,3,4,5]}, 36 | "output": {"output": 1.414} 37 | } 38 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/DataScienceNumbersSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "exp should raise an exponent", 4 | "mapping": "output = (2).exp().round(3)", 5 | "input": {}, 6 | "output": {"output": 7.389} 7 | }, 8 | { 9 | "name": "pow should let you apply powers", 10 | "mapping": "output = (2).pow(3)", 11 | "input": {}, 12 | "output": {"output": 8.0} 13 | }, 14 | { 15 | "name": "sqrt should do a square root", 16 | "mapping": "output = (9).sqrt()", 17 | "input": {}, 18 | "output": {"output": 3.0} 19 | } 20 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/EmailSuiteMagicMethods.ini: -------------------------------------------------------------------------------- 1 | name = input.email().name 2 | address = input.email().address 3 | username = input.email().username 4 | domain = input.email().domain 5 | -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/EnumerateSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "enumerate", 4 | "mapping": "output = input.enumerate()", 5 | "input": { 6 | "input": ["a", "b", "c", "d"] 7 | }, 8 | "output": { 9 | "output": [ 10 | [ 0, "a" ], [ 1, "b" ], [ 2, "c" ], [ 3, "d" ] 11 | ] 12 | } 13 | } 14 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/GetSizeFunctionSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "get the size of an empty array", 4 | "mapping": "a = b.size()", 5 | "input": {"b": []}, 6 | "output": {"a": 0} 7 | }, 8 | 9 | { 10 | "name": "get the size of an array with two items", 11 | "mapping": "a = b.size()", 12 | "input": {"b": [true, true]}, 13 | "output": {"a": 2} 14 | }, 15 | 16 | { 17 | "name": "get the size of an empty string", 18 | "mapping": "a = b.size()", 19 | "input": {"b": ""}, 20 | "output": {"a": 0} 21 | }, 22 | 23 | { 24 | "name": "get the size of a string of size four", 25 | "mapping": "a = b.size()", 26 | "input": {"b": "1234"}, 27 | "output": {"a": 4} 28 | }, 29 | 30 | { 31 | "name": "get the size of a missing field", 32 | "mapping": "a = b.size()", 33 | "input": {}, 34 | "output": {} 35 | }, 36 | 37 | { 38 | "name": "get the size of something else", 39 | "mapping": "a = b.size()", 40 | "input": {"b": true}, 41 | "output": {} 42 | } 43 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/GetSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "get with integer", 4 | "mapping": "x = a.get(1) \n y = a[1] ", 5 | "input": {"a": [123, 456]}, 6 | "output": {"x": 456, "y": 456} 7 | }, 8 | { 9 | "name": "get with string", 10 | "mapping": "x = get(\"b\") \n y = b", 11 | "input": {"a": 123, "b": 456}, 12 | "output": {"x": 456, "y": 456} 13 | }, 14 | { 15 | "name": "get with path - int", 16 | "mapping": "x = a.get(b)", 17 | "input": {"b": 1, "a": [123, 456]}, 18 | "output": {"x": 456} 19 | }, 20 | { 21 | "name": "get with path - string", 22 | "mapping": "x = a.get(b)", 23 | "input": {"b": "my_field", "a": {"other_field": 123, "my_field": 456}}, 24 | "output": {"x": 456} 25 | } 26 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/IndexOfSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "combining indexOf with get for late-bound field lookup", 4 | 5 | "mapping": "@/io.idml.functions/IndexOfWithGet.ini", 6 | 7 | "input": { 8 | "input_type": "vet data", 9 | "headers": ["name", "species"], 10 | "input": [ 11 | ["bob", "dog"], 12 | ["terry", "cat"], 13 | ["steve", "chicken"] 14 | ] 15 | }, 16 | 17 | "output": { 18 | "output": [ 19 | { 20 | "name" : "bob", 21 | "species" : "dog", 22 | "type" : "vet data" 23 | }, 24 | { 25 | "name" : "terry", 26 | "species" : "cat", 27 | "type" : "vet data" 28 | }, 29 | { 30 | "name" : "steve", 31 | "species" : "chicken", 32 | "type" : "vet data" 33 | } 34 | ] 35 | } 36 | } 37 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/IndexOfWithGet.ini: -------------------------------------------------------------------------------- 1 | # combining indexOf with get for late-bound field lookup 2 | [main] 3 | output = input.apply("row") 4 | 5 | [row] 6 | name = get(root.headers.indexOf("name")) 7 | species = get(root.headers.indexOf("species")) 8 | type = root.input_type -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/IsMatchSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "one character match", 4 | "mapping": "output = input.isMatch(\"a\")", 5 | "input": {"input": "a"}, 6 | "output": {"output": true} 7 | }, 8 | { 9 | "name": "multi character match", 10 | "mapping": "output = input.isMatch(\"foo\")", 11 | "input": {"input": "foo"}, 12 | "output": {"output": true} 13 | }, 14 | { 15 | "name": "wildcard match", 16 | "mapping": "output = input.isMatch(\".*coo.*\")", 17 | "input": {"input": "cats are cool"}, 18 | "output": {"output": true} 19 | } 20 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/LowercaseSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "lower case 1", 4 | "mapping": "output = input.lowercase()", 5 | "input": {"input": "abc"}, 6 | "output": {"output": "abc"} 7 | }, 8 | { 9 | "name": "lower case 2", 10 | "mapping": "output = input.lowercase()", 11 | "input": {"input": "ABC"}, 12 | "output": {"output": "abc"} 13 | }, 14 | { 15 | "name": "lower case 3", 16 | "mapping": "output = input.lowercase()", 17 | "input": {"input": "Abc"}, 18 | "output": {"output": "abc"} 19 | }, 20 | { 21 | "name": "lower case 4", 22 | "mapping": "output = input.lowercase()", 23 | "input": {"input": false}, 24 | "output": {"output": false} 25 | }, 26 | { 27 | "name": "lower case 5", 28 | "mapping": "output = input.lowercase()", 29 | "input": {"input": 1234}, 30 | "output": {"output": 1234} 31 | }, 32 | { 33 | "name": "lower case 6", 34 | "mapping": "output = input.lowercase()", 35 | "input": {"input": null}, 36 | "output": {"output": null} 37 | }, 38 | { 39 | "name": "lower case 7", 40 | "mapping": "output = input.lowercase()", 41 | "input": {"input": {"input": "Abc"}}, 42 | "output": {"output": {"input": "Abc"}} 43 | } 44 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/MapSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "map should map arrays", 4 | "mapping": "result = xs.map(this * 2)", 5 | "input": {"xs":[1,2,3]}, 6 | "output": {"result":[2,4,6]} 7 | } 8 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/MatchSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "match with no groups or results", 4 | "mapping": "output = input.match(\"a\")", 5 | "input": {"input": "a"}, 6 | "output": {"output": []} 7 | }, 8 | 9 | { 10 | "name": "match with no groups with regex style argument", 11 | "mapping": "output = input.match(\"a\")", 12 | "input": {"input": "a"}, 13 | "output": {"output": []} 14 | }, 15 | 16 | { 17 | "name": "match with no groups", 18 | "mapping": " output = input.match(\"(a+)\")", 19 | "input": {"input": "aaa"}, 20 | "output": {"output": ["aaa"]} 21 | }, 22 | 23 | { 24 | "name": "match a load of groups", 25 | "mapping": "output = input.match(\"(foo) (bar) (baz)\")", 26 | "input": {"input": "foo bar baz"}, 27 | "output": {"output": ["foo", "bar", "baz"]} 28 | } 29 | 30 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/PrependSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "can be used in mappings", 4 | "mapping": "a = b.prepend(true)", 5 | "input": {}, 6 | "output": {} 7 | }, 8 | 9 | { 10 | "name": "does nothing if the caller is missing", 11 | "mapping": "a = b.prepend(true)", 12 | "input": {}, 13 | "output": {} 14 | }, 15 | 16 | { 17 | "name": "does nothing if the caller is not an array", 18 | "mapping": "a = b.prepend(true)", 19 | "input": {"b": true}, 20 | "output": {} 21 | }, 22 | 23 | { 24 | "name": "passes through the caller untouched if the parameter is missing", 25 | "mapping": "a = b.prepend(missing)", 26 | "input": {"b": [true]}, 27 | "output": {"a": [true]} 28 | }, 29 | 30 | { 31 | "name": "adds a value to the start of an array", 32 | "mapping": "a = b.prepend(false)", 33 | "input": {"b": [true]}, 34 | "output": {"a": [false, true]} 35 | }, 36 | 37 | { 38 | "name": "does not modify the original array", 39 | "mapping": "a = b.prepend(false)\n c = b", 40 | "input": {"b": [true]}, 41 | "output": {"a": [false, true], "c": [true]} 42 | } 43 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/RegexSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "pull multiple matches out", 4 | "mapping": "result = input.matches(\"(a)\")", 5 | "input": {"input": "aaa"}, 6 | "output": {"result": [["a"], ["a"], ["a"]]} 7 | }, 8 | { 9 | "name": "do a full match and pull groups out", 10 | "mapping": "result = input.match(\"(a)\")", 11 | "input": {"input": "a"}, 12 | "output": {"result": ["a"]} 13 | } 14 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/ReplaceSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "replace a letter", 4 | "mapping": " output = input.replace(\"f\", \"z\")", 5 | "input": {"input": "foo"}, 6 | "output": {"output": "zoo"} 7 | }, 8 | { 9 | "name": "replace a word", 10 | "mapping": "output = input.replace(\"bar\", \"zoo\")", 11 | "input": {"input": "foo bar baz"}, 12 | "output": {"output": "foo zoo baz"} 13 | }, 14 | { 15 | "name": "replace a lot of letters", 16 | "mapping": "output = input.replace(\".\", \"dog\")", 17 | "input": {"input": "foo"}, 18 | "output": {"output": "dogdogdog"} 19 | } 20 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/SplitSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "split by a character", 4 | "mapping": "output = input.split(\"a\")", 5 | "input": {"input": "abababab"}, 6 | "output": {"output": ["", "b", "b", "b", "b"]} 7 | }, 8 | { 9 | "name": "split by a regex", 10 | "mapping": "output = input.split(\"(a|b)\")", 11 | "input": {"input": "acbcab"} , 12 | "output": {"output": ["", "c", "c"]} 13 | }, 14 | { 15 | "name": "split on whitespace", 16 | "mapping": "output = input.split(\"[ \\t\\n]+\")", 17 | "input": {"input": "foo bar baz \n tabbed"}, 18 | "output": {"output": ["foo", "bar", "baz", "tabbed"]} 19 | } 20 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/StripSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "strip should strip whitespace from strings", 4 | "mapping": "output = input.strip()", 5 | "input": {"input": " hello world \t\n"}, 6 | "output": {"output": "hello world"} 7 | }, 8 | { 9 | "name": "strip should strip weird whitespace", 10 | "mapping": "output = input.strip()", 11 | "input": {"input": " foo bar \u00a0\n\t"}, 12 | "output": {"output": "foo bar"} 13 | } 14 | ] 15 | -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/UniqueSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "unique should work the same as normal unique", 4 | "mapping": "output = input.unique(this)", 5 | "input": {"input": [1,1,2,2,3,3]}, 6 | "output": {"output": [1,2,3]} 7 | }, 8 | { 9 | "name": "unique should take a basic transform and return the first for each key", 10 | "mapping": "output = input.unique(this.lowercase())", 11 | "input": {"input": ["a", "A", "b", "B"]}, 12 | "output": {"output": ["a", "b"]} 13 | }, 14 | { 15 | "name": "unique should return the untransformed item", 16 | "mapping": "output = input.unique(this.uppercase())", 17 | "input": {"input": ["a", "A", "b", "B"]}, 18 | "output": {"output": ["a", "b"]} 19 | } 20 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/UppercaseSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "upper case 1", 4 | "mapping": "output = input.uppercase()", 5 | "input": {"input": "abc"}, 6 | "output": {"output": "ABC"} 7 | }, 8 | { 9 | "name": "upper case 2", 10 | "mapping": "output = input.uppercase()", 11 | "input": {"input": "ABC"}, 12 | "output": {"output": "ABC"} 13 | }, 14 | { 15 | "name": "upper case 3", 16 | "mapping": "output = input.uppercase()", 17 | "input": {"input": "Abc"}, 18 | "output": {"output": "ABC"} 19 | }, 20 | { 21 | "name": "upper case 4", 22 | "mapping": "output = input.uppercase()", 23 | "input": {"input": false}, 24 | "output": {"output": false} 25 | }, 26 | { 27 | "name": "upper case 5", 28 | "mapping": "output = input.uppercase()", 29 | "input": {"input": 1234}, 30 | "output": {"output": 1234} 31 | }, 32 | { 33 | "name": "upper case 6", 34 | "mapping": "output = input.uppercase()", 35 | "input": {"input": null}, 36 | "output": {"output": null} 37 | }, 38 | { 39 | "name": "upper case 7", 40 | "mapping": "output = input.uppercase()", 41 | "input": {"input": {"input": "Abc"}}, 42 | "output": {"output": {"input": "Abc"}} 43 | }, 44 | { 45 | "name": "DQ-904", 46 | "mapping": "web.content = email.number.uppercase()", 47 | "input": { 48 | "email": { 49 | "number": "aBc aca" 50 | } 51 | }, 52 | "output": { 53 | "web": { 54 | "content": "ABC ACA" 55 | } 56 | } 57 | } 58 | ] -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/UrlEncodeSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "urlEncode encodes strings", 4 | "mapping": "a = b.urlEncode()", 5 | "input": {"b": "hello world? 1 + 1 = 2"}, 6 | "output": {"a": "hello+world%3F+1+%2B+1+%3D+2" } 7 | }, 8 | 9 | { 10 | "name": "urlDecode decodes strings", 11 | "mapping": "a = b.urlDecode()", 12 | "input": {"b": "hello+world%3F+1+%2B+1+%3D+2"}, 13 | "output": {"a": "hello world? 1 + 1 = 2" } 14 | }, 15 | 16 | { 17 | "name": "urlEncode then urlDecode produces input unchanged", 18 | "mapping": "a = b.urlEncode().urlDecode()", 19 | "input": {"b": "hello world? 1 + 1 = 2"}, 20 | "output": {"a": "hello world? 1 + 1 = 2"} 21 | }, 22 | 23 | { 24 | "name": "urlEncode ignores regular characters", 25 | "mapping": "a = b.urlEncode()", 26 | "input": {"b": "foo"}, 27 | "output": {"a": "foo"} 28 | }, 29 | 30 | { 31 | "name": "urlDecode ignores regular characters", 32 | "mapping": "a = b.urlDecode()", 33 | "input": {"b": "foo"}, 34 | "output": {"a": "foo"} 35 | } 36 | ] 37 | -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/UrlSuiteMagicMethods.ini: -------------------------------------------------------------------------------- 1 | protocol = x.url().protocol 2 | host = x.url().host 3 | path = x.url().path 4 | query = x.url().query -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/ZipSuite.ini: -------------------------------------------------------------------------------- 1 | output = names.zip(species) -------------------------------------------------------------------------------- /test/src/test/resources/io.idml.functions/ZipSuite.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "zip", 4 | "mapping": "@/io.idml.functions/ZipSuite.ini", 5 | "input": { 6 | "input_type": "vet data", 7 | "names": ["bob", "terry", "steve"], 8 | "species": ["dog", "cat", "chicken"] 9 | }, 10 | "output": { 11 | "output": [ 12 | [ "bob", "dog" ], [ "terry", "cat" ], [ "steve", "chicken" ] 13 | ] 14 | } 15 | } 16 | ] -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/load_me.ini: -------------------------------------------------------------------------------- 1 | a = b -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/suite1_many.json: -------------------------------------------------------------------------------- 1 | { 2 | "tests": [ 3 | { 4 | "name": "This is a mock test", 5 | "input": {}, 6 | "mapping": "a = b" 7 | }, 8 | { 9 | "name": "This is another mock test", 10 | "input": {}, 11 | "mapping": "a = b" 12 | } 13 | ] 14 | } -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/suite2_many.json: -------------------------------------------------------------------------------- 1 | { 2 | "tests": [ 3 | { 4 | "name": "This is a mock test", 5 | "input": {}, 6 | "mapping": "a = b" 7 | }, 8 | { 9 | "name": "This is another mock test", 10 | "input": {}, 11 | "mapping": "a = b" 12 | } 13 | ] 14 | } -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/suite_chain_pending.json: -------------------------------------------------------------------------------- 1 | { 2 | "tests": [ 3 | { 4 | "name": "This is a mock test", 5 | "chain": ["a = b"], 6 | "input": {"b": true}, 7 | "output": {"a": true} 8 | }, 9 | { 10 | "name": "This is another mock test", 11 | "chain": ["a = b"], 12 | "input": {"b": false}, 13 | "pending": true, 14 | "output": {"a": "broken"} 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/suite_mapping_pending.json: -------------------------------------------------------------------------------- 1 | { 2 | "tests": [ 3 | { 4 | "name": "This is a mock test", 5 | "mapping": "a = b", 6 | "input": {"b": true}, 7 | "output": {"a": true} 8 | }, 9 | { 10 | "name": "This is another mock test", 11 | "input": {}, 12 | "pending": true, 13 | "mapping": "a = b", 14 | "ouput": {"a": "broken"} 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/suite_missing_object.json: -------------------------------------------------------------------------------- 1 | ["expects an object :-("] -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/suite_missing_tests.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | } -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/suite_shared_chain.json: -------------------------------------------------------------------------------- 1 | { 2 | "chain": [ 3 | "a = b", 4 | "c = a" 5 | ], 6 | "tests": [ 7 | { 8 | "name": "This is a mock test", 9 | "input": {"b": true}, 10 | "output": {"a": true, "c": true} 11 | }, 12 | { 13 | "name": "This is another mock test", 14 | "input": {"b": false}, 15 | "output": {"a": false, "c": false} 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/suite_shared_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "mapping": "a = b", 3 | "tests": [ 4 | { 5 | "name": "This is a mock test", 6 | "input": {"b": true}, 7 | "output": {"a": true} 8 | }, 9 | { 10 | "name": "This is another mock test", 11 | "input": {"b": false}, 12 | "output": {"a": false} 13 | } 14 | ] 15 | } -------------------------------------------------------------------------------- /test/src/test/resources/mock_tests/suite_single.json: -------------------------------------------------------------------------------- 1 | { 2 | "tests": [ 3 | { 4 | "name": "This is a mock test", 5 | "input": {}, 6 | "mapping": "a = b" 7 | }, 8 | { 9 | "name": "This is another mock test", 10 | "input": {}, 11 | "mapping": "a = b" 12 | } 13 | ] 14 | } -------------------------------------------------------------------------------- /test/src/test/scala/io/idml/Base64Prop.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.datanodes.IString 4 | import org.scalacheck._ 5 | import org.scalacheck.Prop.forAll 6 | import org.scalacheck.Properties 7 | 8 | class Base64Prop extends Properties("Base64") { 9 | 10 | property("base64 should be reversible") = forAll { s: String => 11 | IString(s).base64encode().base64decode() == IString(s) 12 | } 13 | property("base64mime should be reversible") = forAll { s: String => 14 | IString(s).base64mimeEncode().base64mimeDecode() == IString(s) 15 | } 16 | property("base64 should be reversible") = forAll { s: String => 17 | IString(s).base64urlsafeEncode().base64urlsafeDecode() == IString(s) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /test/src/test/scala/io/idml/IdmlCirceProp.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import com.fasterxml.jackson.core.JsonParseException 4 | import io.circe.Json 5 | import io.circe.testing.ArbitraryInstances 6 | import io.idml.circe.IdmlCirce 7 | import io.idml.jackson.IdmlJackson 8 | import org.scalacheck._ 9 | import org.scalacheck.Prop.forAll 10 | 11 | import scala.util.Try 12 | 13 | class IdmlJsonProperties extends Properties("IdmlJson") with ArbitraryInstances { 14 | 15 | property("IdmlCirce should be able to parse anything IdmlJackson can") = forAll { j: Json => 16 | val str = j.noSpaces 17 | val jacksonResult = IdmlJackson.default.parseEither(str) 18 | val circeResult = IdmlCirce.parseEither(str) 19 | (jacksonResult, circeResult) match { 20 | case (Right(jr), Right(cr)) => 21 | jr == cr 22 | case (Right(_), Left(_)) => 23 | false 24 | case (Left(_), _) => 25 | true // this JSON is a bit hard for jackson, so we'll give it some leeway 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /test/src/test/scala/io/idml/UrlEncodeProp.scala: -------------------------------------------------------------------------------- 1 | package io.idml 2 | 3 | import io.idml.datanodes.IString 4 | import org.scalacheck.Prop.forAll 5 | import org.scalacheck.Properties 6 | 7 | class UrlEncodeProp extends Properties("UrlEncode") { 8 | 9 | property("urlencode should be reversible") = forAll { s: String => 10 | IString(s).urlEncode().urlDecode() == IString(s) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /test/src/test/scala/io/idml/ast/AstItTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.ast 2 | 3 | import io.idml.IdmlScalaTestBase 4 | 5 | class AstItTest extends IdmlScalaTestBase("io.idml.ast", findUnmappedFields = true) 6 | -------------------------------------------------------------------------------- /test/src/test/scala/io/idml/functions/BuiltinFunctionResolverTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IString 4 | import io.idml.ast._ 5 | import org.scalatest.funsuite.AnyFunSuite 6 | 7 | class BuiltinFunctionResolverTest extends AnyFunSuite { 8 | 9 | val stringLiteral = ExecNavLiteral(Literal(IString("my_block"))) 10 | val stringPipl = Pipeline(List(stringLiteral)) 11 | val stringArgs = List(stringPipl) 12 | 13 | test("resolves apply(string)") { 14 | new BuiltinFunctionResolver().resolve("apply", stringArgs) 15 | } 16 | 17 | test("resolves applyArray(string)") { 18 | new BuiltinFunctionResolver().resolve("applyArray", stringArgs) 19 | } 20 | 21 | test("resolves array(string)") { 22 | new BuiltinFunctionResolver().resolve("array", stringArgs) 23 | } 24 | 25 | test("resolves extract(string)") { 26 | assert( 27 | new BuiltinFunctionResolver().resolve("extract", stringArgs) === Some( 28 | ExtractFunction(stringPipl))) 29 | } 30 | 31 | test("no matches") { 32 | assert(new BuiltinFunctionResolver().resolve("missing", Nil) === None) 33 | assert(new BuiltinFunctionResolver().resolve("missing", stringArgs) === None) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /test/src/test/scala/io/idml/functions/ExtractFunctionTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes._ 4 | import io.idml.ast.{ExecNavRelative, Field, Pipeline} 5 | import io.idml.{IdmlContext, InvalidCaller, NoFields} 6 | import org.scalatest.funsuite.AnyFunSuite 7 | 8 | class ExtractFunctionTest extends AnyFunSuite { 9 | 10 | def extract = ExtractFunction(Pipeline(List(ExecNavRelative, Field("a")))) 11 | 12 | test("extract returns the identical nothing when given nothing") { 13 | val ctx = new IdmlContext(NoFields) 14 | extract.invoke(ctx) 15 | assert(ctx.cursor === NoFields) 16 | } 17 | 18 | test("extract returns invalid caller if something that isn't an array is used") { 19 | val ctx = new IdmlContext(IString("abc")) 20 | extract.invoke(ctx) 21 | assert(ctx.cursor === InvalidCaller) 22 | } 23 | 24 | test("extract applies function to each element in an array") { 25 | val ctx = 26 | new IdmlContext(IArray(IObject("a" -> ITrue), IObject("a" -> IFalse))) 27 | extract.invoke(ctx) 28 | assert(ctx.cursor === IArray(ITrue, IFalse)) 29 | } 30 | 31 | test("extract returns nothing if no results are returned") { 32 | val ctx = 33 | new IdmlContext(IArray(IObject("b" -> ITrue), IObject("b" -> IFalse))) 34 | extract.invoke(ctx) 35 | assert(ctx.cursor === NoFields) 36 | } 37 | 38 | test("extract filters out missing fields") { 39 | val ctx = 40 | new IdmlContext(IArray(IObject("b" -> ITrue), IObject("a" -> IFalse))) 41 | extract.invoke(ctx) 42 | assert(ctx.cursor === IArray(IFalse)) 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /test/src/test/scala/io/idml/functions/FunctionsItTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.IdmlScalaTestBase 4 | 5 | class FunctionsItTest extends IdmlScalaTestBase("io.idml.functions") 6 | -------------------------------------------------------------------------------- /test/src/test/scala/io/idml/functions/IdmlValueFunctionResolverTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IString 4 | import io.idml.IdmlValue 5 | import io.idml.ast.{ExecNavLiteral, Literal, Pipeline} 6 | import org.scalatest.funsuite.AnyFunSuite 7 | 8 | class IdmlValueFunctionResolverTest extends AnyFunSuite { 9 | 10 | val pv = classOf[IdmlValue] 11 | 12 | val pipl = Pipeline(List(ExecNavLiteral(Literal(IString(""))))) 13 | 14 | test("return none if a method can't be found") { 15 | assert(new IdmlValueFunctionResolver().resolve("missing", Nil) === None) 16 | } 17 | 18 | test("return a function if a 0-arity method can be found") { 19 | assert( 20 | new IdmlValueFunctionResolver().resolve("int", Nil) === Some( 21 | IdmlValueFunction(pv.getMethod("int"), Nil))) 22 | } 23 | 24 | test("return a function if a 1-arity function can be found") { 25 | assert( 26 | new IdmlValueFunctionResolver() 27 | .resolve("default", List(pipl)) === Some( 28 | IdmlValueFunction(pv.getMethod("default", classOf[IdmlValue]), List(pipl), isNAry = false)) 29 | ) 30 | } 31 | 32 | test("return a function if a 2-arity function can be found") { 33 | assert( 34 | new IdmlValueFunctionResolver() 35 | .resolve("slice", List(pipl, pipl)) === Some( 36 | IdmlValueFunction( 37 | pv.getMethod("slice", classOf[IdmlValue], classOf[IdmlValue]), 38 | List(pipl, pipl), 39 | isNAry = false) 40 | ) 41 | ) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /test/src/test/scala/io/idml/functions/IdmlValueNAryFunctionResolverTest.scala: -------------------------------------------------------------------------------- 1 | package io.idml.functions 2 | 3 | import io.idml.datanodes.IString 4 | import io.idml.IdmlValue 5 | import io.idml.ast.{ExecNavLiteral, Literal, Pipeline} 6 | import org.scalatest.funsuite.AnyFunSuite 7 | 8 | class IdmlValueNAryFunctionResolverTest extends AnyFunSuite { 9 | 10 | val pv = classOf[IdmlValue] 11 | 12 | val pipl = Pipeline(List(ExecNavLiteral(Literal(IString(""))))) 13 | 14 | test("return none if a method can't be found") { 15 | assert(new IdmlValueNaryFunctionResolver().resolve("missing", Nil) === None) 16 | } 17 | 18 | test("return a function if a 1-arity function can be found") { 19 | assert( 20 | new IdmlValueNaryFunctionResolver() 21 | .resolve("format", List(pipl)) === Some( 22 | IdmlValueFunction( 23 | pv.getMethod("format", classOf[Seq[IdmlValue]]), 24 | List(pipl), 25 | isNAry = true) 26 | ) 27 | ) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tool/src/graal/resource-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "resources": [ 3 | {"pattern": "org/jline/utils/.*caps$"}, 4 | {"pattern": "org/jline/utils/capabilities\\.txt$"} 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /tool/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | System.err 4 | 5 | %date [%thread] - 5level %logger{36} - %msg%n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /tool/src/main/scala/io/idml/tool/DeclineHelpers.scala: -------------------------------------------------------------------------------- 1 | package io.idml.tool 2 | 3 | import java.io.File 4 | 5 | import cats._ 6 | import cats.effect._ 7 | import cats.implicits._ 8 | import cats.syntax._ 9 | import cats.data._ 10 | import com.monovore.decline._ 11 | 12 | object DeclineHelpers { 13 | 14 | implicit val readFile: Argument[File] = new Argument[File] { 15 | 16 | override def read(string: String): ValidatedNel[String, File] = 17 | try { 18 | Validated 19 | .valid(new File(string)) 20 | .ensure(NonEmptyList.of(s"Invalid File: $string does not exist."))(_.exists()) 21 | .ensure(NonEmptyList.of(s"Invalid File: $string cannot be read."))(_.canRead()) 22 | } catch { 23 | case npe: NullPointerException => 24 | Validated.invalidNel(s"Invalid File: $string (${npe.getMessage})") 25 | } 26 | 27 | override def defaultMetavar: String = "file" 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /tool/src/main/scala/io/idml/tool/IOCommandApp.scala: -------------------------------------------------------------------------------- 1 | package io.idml.tool 2 | 3 | import cats.effect.{ExitCode, IO, IOApp} 4 | import com.monovore.decline.{Command, Opts, PlatformApp, Visibility} 5 | 6 | trait IOCommandApp[T] extends IOApp { 7 | def name: String 8 | def header: String 9 | def commandLine: Opts[T] 10 | def main(t: T): IO[ExitCode] 11 | def helpFlag: Boolean = true 12 | def version: String = "" 13 | 14 | def command: Command[IO[ExitCode]] = { 15 | val showVersion = 16 | if (version.isEmpty) Opts.never 17 | else 18 | Opts 19 | .flag("version", "Print the version number and exit.", visibility = Visibility.Partial) 20 | .map { _ => 21 | IO { System.err.println(version); ExitCode.Success } 22 | } 23 | Command(name, header, helpFlag)(showVersion orElse commandLine.map(main)) 24 | } 25 | 26 | override def run(args: List[String]): IO[ExitCode] = 27 | command.parse(PlatformApp.ambientArgs getOrElse args, sys.env) match { 28 | case Left(help) => IO { System.err.println(help); ExitCode.Error } 29 | case Right(r) => r 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /tool/src/main/scala/io/idml/tool/IdmlTool.scala: -------------------------------------------------------------------------------- 1 | package io.idml.tool 2 | 3 | import java.io.File 4 | import java.net.URI 5 | 6 | import cats._ 7 | import cats.effect._ 8 | import cats.implicits._ 9 | import cats.syntax._ 10 | import cats.data._ 11 | import com.monovore.decline._ 12 | import DeclineHelpers._ 13 | import io.idml.BuildInfo 14 | import io.idml.tool.IOCommandApp 15 | 16 | object IdmlTool extends IOCommandApp[IO[ExitCode]] { 17 | override def name: String = "idml" 18 | override def header: String = "IDML command line tools" 19 | override def version: String = BuildInfo.version 20 | override def commandLine: Opts[IO[ExitCode]] = 21 | NonEmptyList 22 | .of( 23 | io.idmlrepl.Main.execute(), 24 | IdmlTools.apply, 25 | IdmlTools.server, 26 | io.idml.test.Main.execute(), 27 | io.idml.tutor.Main.execute() 28 | ) 29 | .map(c => Opts.subcommand(c)) 30 | .reduceK 31 | 32 | override def main(c: IO[ExitCode]): IO[ExitCode] = c 33 | } 34 | -------------------------------------------------------------------------------- /tool/src/main/scala/io/idml/tool/IdmlToolConfig.scala: -------------------------------------------------------------------------------- 1 | package io.idml.tool 2 | 3 | import java.io.File 4 | 5 | case class IdmlToolConfig( 6 | files: List[File] = List.empty, 7 | pretty: Boolean = false, 8 | unmapped: Boolean = false, 9 | strict: Boolean = false, 10 | traceFile: Option[File] = None 11 | ) 12 | -------------------------------------------------------------------------------- /utils/build.sbt: -------------------------------------------------------------------------------- 1 | name := "idml-utils" 2 | 3 | libraryDependencies ++= Seq( 4 | "io.higherkindness" %% "droste-core" % "0.8.0", 5 | "io.higherkindness" %% "droste-macros" % "0.8.0", 6 | "org.scalatest" %% "scalatest" % "3.2.8" % Test 7 | ) 8 | -------------------------------------------------------------------------------- /utils/src/main/scala/io/idml/utils/DocumentValidator.scala: -------------------------------------------------------------------------------- 1 | package io.idml.utils 2 | 3 | import io.idml.utils.validators.{MappingValidator, SchemaValidator} 4 | import io.idml.ast.Document 5 | import io.idml.{FunctionResolverService, Idml, IdmlParser} 6 | 7 | /** Validates IDML documents */ 8 | object DocumentValidator { 9 | 10 | /** Ensures an IDML document is valid. 11 | * 12 | * If it returns successfully, the document is valid, otherwise it will throw an exception 13 | */ 14 | def validate(str: String): Unit = { 15 | validate(new IdmlParser().parse(new FunctionResolverService, str).nodes) 16 | } 17 | 18 | /** Ensures that an IDML document is valid. 19 | * 20 | * If it returns successfully, the document is valid, otherwise it will throw an exception 21 | */ 22 | def validate(doc: Document): Unit = { 23 | DocumentClassifier.classify(doc) match { 24 | case MixedDocumentType => 25 | throw new ClassificationException("Document cannot be both schemas and mappings") 26 | case SchemaDocumentType => 27 | if (!SchemaValidator.validate(doc)) { 28 | throw new ClassificationException( 29 | "Document is a schema but contains inappropriate content for a schema") 30 | } 31 | case MappingDocumentType => 32 | if (!MappingValidator.validate(doc)) { 33 | throw new ClassificationException( 34 | "Document is a mapping but contains inappropriate content for a mapping") 35 | } 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /utils/src/main/scala/io/idml/utils/PValueComparison.scala: -------------------------------------------------------------------------------- 1 | package io.idml.utils 2 | 3 | import io.idml.{IdmlArray, IdmlObject, IdmlValue} 4 | 5 | /** A utility for doing a deep comparison of PValues and producing meaningful feedback should they 6 | * not be the same 7 | */ 8 | object PValueComparison { 9 | 10 | /** Perform a deep comparison of two PValues */ 11 | def assertEqual(left: IdmlValue, right: IdmlValue, path: List[String] = Nil): Unit = { 12 | (left, right) match { 13 | case (l: IdmlObject, r: IdmlObject) => 14 | l.fields.keys.foreach { k => 15 | assertEqual(l.get(k), r.get(k), k :: path) 16 | } 17 | require( 18 | l.fields.keys == r.fields.keys, 19 | s"${path.reverse.mkString(".")} actual: ${l.fields}, expected: ${r.fields}") 20 | case (l: IdmlArray, r: IdmlArray) => 21 | l.items.zip(r.items).zipWithIndex.foreach { case ((ll: IdmlValue, rr: IdmlValue), i: Int) => 22 | assertEqual(ll, rr, i.toString :: path) 23 | } 24 | require( 25 | l.items.length == r.items.length, 26 | s"${path.reverse.mkString(".")} actual: ${l.items}, expected: ${r.items}") 27 | case (l, r) => 28 | require(l == r, s"${path.reverse.mkString(".")} actual: $l, expected: $r") 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /utils/src/main/scala/io/idml/utils/folders/Folders.scala: -------------------------------------------------------------------------------- 1 | package io.idml.utils.folders 2 | 3 | object Folders {} 4 | -------------------------------------------------------------------------------- /utils/src/main/scala/io/idml/utils/validators/MappingValidator.scala: -------------------------------------------------------------------------------- 1 | package io.idml.utils.validators 2 | 3 | import io.idml.utils.visitor.{ExecNodeVisitor, StructureAgnosticVisitationStyle} 4 | import io.idml.ast.Document 5 | 6 | /** Object which provides a validate function, allowing you to ask if a mapping is valid 7 | */ 8 | object MappingValidator { 9 | 10 | /** Visitor which detects functions not allowed inside mappings 11 | */ 12 | class MappingValidatorVisitor extends ExecNodeVisitor with StructureAgnosticVisitationStyle { 13 | var valid = true 14 | 15 | override def visitFunc(ctx: ExecFuncContext): Unit = { 16 | if (ctx.expr.name == "array") { 17 | valid = false 18 | } 19 | } 20 | 21 | } 22 | 23 | /** Check whether a document is a valid Schema 24 | * @param doc 25 | * parsed Document tree 26 | * @return 27 | * boolean indicating if it's valid 28 | */ 29 | def validate(doc: Document): Boolean = { 30 | val visitor = new MappingValidatorVisitor() 31 | visitor.visit(doc) 32 | visitor.valid 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /utils/src/main/scala/io/idml/utils/validators/SchemaValidator.scala: -------------------------------------------------------------------------------- 1 | package io.idml.utils.validators 2 | 3 | import io.idml.utils.visitor.{ExecNodeVisitor, StructureAgnosticVisitationStyle} 4 | import io.idml.ast.Document 5 | 6 | /** Object which provides a validate function, allowing you to ask if a schema is valid 7 | */ 8 | object SchemaValidator { 9 | 10 | /** Visitor which detects functions not allowed inside schemas 11 | */ 12 | class SchemaValidatorVisitor extends ExecNodeVisitor with StructureAgnosticVisitationStyle { 13 | var valid = true 14 | 15 | override def visitFunc(ctx: ExecFuncContext): Unit = { 16 | if (ctx.expr.name == "extract") { 17 | valid = false 18 | } 19 | } 20 | 21 | } 22 | 23 | /** Check whether a document is a valid Schema 24 | * @param doc 25 | * parsed Document tree 26 | * @return 27 | * boolean indicating if it's valid 28 | */ 29 | def validate(doc: Document): Boolean = { 30 | val visitor = new SchemaValidatorVisitor() 31 | visitor.visit(doc) 32 | visitor.valid 33 | } 34 | } 35 | --------------------------------------------------------------------------------