├── .github └── workflows │ ├── snyk.yml │ ├── snyk_test.yml │ └── test_and_deploy.yml ├── .gitignore ├── .scalafmt.conf ├── .snyk ├── CHANGELOG ├── LICENSE-2.0.txt ├── README.md ├── benchmark ├── build.sbt └── src │ └── test │ └── scala │ └── com.snowplowanalytics.snowplow.analytics.scalasdk │ └── benchmark │ ├── OrderedBenchmark.scala │ └── ToTsvBenchmark.scala ├── build.sbt ├── project ├── BuildSettings.scala ├── Dependencies.scala ├── build.properties ├── plugins.sbt └── travis-deploy-key.enc └── src ├── main ├── scala-2 │ └── com.snowplowanalytics.snowplow.analytics.scalasdk │ │ └── decode │ │ ├── Parser.scala │ │ └── RowDecoderCompanion.scala ├── scala-3 │ └── com.snowplowanalytics.snowplow.analytics.scalasdk │ │ └── decode │ │ ├── Parser.scala │ │ └── RowDecoderCompanion.scala └── scala │ └── com.snowplowanalytics.snowplow.analytics.scalasdk │ ├── Common.scala │ ├── Data.scala │ ├── Event.scala │ ├── ParsingError.scala │ ├── SnowplowEvent.scala │ ├── decode │ ├── RowDecoder.scala │ ├── TSVParser.scala │ ├── ValueDecoder.scala │ └── package.scala │ ├── encode │ └── TsvEncoder.scala │ └── validate │ └── package.scala ├── site-preprocess └── index.html └── test └── scala └── com.snowplowanalytics.snowplow.analytics.scalasdk ├── EventGen.scala ├── EventSpec.scala ├── ParsingErrorSpec.scala ├── SnowplowEventSpec.scala └── decode └── ValueDecoderSpec.scala /.github/workflows/snyk.yml: -------------------------------------------------------------------------------- 1 | name: Snyk 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | 7 | jobs: 8 | check-vulnerabilities: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v2 13 | 14 | - name: Snyk monitor - Check for vulnerabilities 15 | uses: snyk/actions/scala@master 16 | with: 17 | command: monitor 18 | args: --project-name=analytics-sdk-scala 19 | env: 20 | SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/snyk_test.yml: -------------------------------------------------------------------------------- 1 | name: Snyk 2 | 3 | on: push 4 | 5 | 6 | jobs: 7 | check-vulnerabilities: 8 | runs-on: ubuntu-latest 9 | 10 | steps: 11 | - uses: actions/checkout@v2 12 | 13 | - name: Snyk monitor - Check for vulnerabilities 14 | uses: snyk/actions/scala@master 15 | with: 16 | command: test 17 | env: 18 | SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} -------------------------------------------------------------------------------- /.github/workflows/test_and_deploy.yml: -------------------------------------------------------------------------------- 1 | name: Test and deploy 2 | 3 | on: push 4 | 5 | jobs: 6 | test_and_deploy: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v2 11 | 12 | - name: Set up JDK 11 13 | uses: actions/setup-java@v1 14 | with: 15 | java-version: 11 16 | 17 | - name: Run tests 18 | run: sbt 'set coverageEnabled := true' +test 19 | 20 | - name: Aggregate coverage data 21 | run: sbt coverageAggregate 22 | 23 | - name: Check formatting 24 | run: sbt scalafmtCheck 25 | 26 | - name: Publish to Maven central 27 | if: startsWith(github.ref, 'refs/tags/') 28 | env: 29 | PGP_PASSPHRASE: ${{ secrets.SONA_PGP_PASSPHRASE }} 30 | PGP_SECRET: ${{ secrets.SONA_PGP_SECRET }} 31 | SONATYPE_USERNAME: ${{ secrets.SONA_USER }} 32 | SONATYPE_PASSWORD: ${{ secrets.SONA_PASS }} 33 | run: sbt ci-release 34 | 35 | - name: Generate API website 36 | if: startsWith(github.ref, 'refs/tags/') 37 | run: sbt makeSite 38 | 39 | - name: Publish website 40 | if: startsWith(github.ref, 'refs/tags/') 41 | run: | 42 | echo Publishing Scaladoc 43 | git fetch 44 | git checkout gh-pages 45 | cp -r target/site/* . 46 | git config user.name "GitHub Actions" 47 | git config user.email "<>" 48 | git add index.html $project_version 49 | git commit -m "Added Scaladoc for $project_version" 50 | git push origin gh-pages 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | # sbt specific 5 | dist/* 6 | target/ 7 | lib_managed/ 8 | src_managed/ 9 | project/boot/ 10 | project/plugins/project/ 11 | 12 | # Vagrant 13 | .vagrant 14 | VERSION 15 | .bloop 16 | .metals 17 | metals.sbt 18 | .vscode 19 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = "2.6.1" 2 | style = default 3 | align.preset = none 4 | align.openParenCallSite = true 5 | align.arrowEnumeratorGenerator = true 6 | maxColumn = 140 7 | docstrings = JavaDoc 8 | optIn.breakChainOnFirstMethodDot = true 9 | spaces.afterKeywordBeforeParen = true 10 | continuationIndent.callSite = 2 11 | continuationIndent.defnSite = 2 12 | verticalMultiline.atDefnSite = true 13 | verticalMultiline.arityThreshold = 3 14 | verticalMultiline.newlineAfterOpenParen = true 15 | verticalMultiline.newlineBeforeImplicitKW = true 16 | verticalMultiline.excludeDanglingParens = [] 17 | importSelectors = noBinPack 18 | rewrite.rules = [ 19 | AsciiSortImports, 20 | RedundantBraces, 21 | RedundantParens, 22 | PreferCurlyFors 23 | ] 24 | runner.dialect = scala212 25 | 26 | fileOverride { 27 | "glob:**/scala-3/**/*.scala" { 28 | runner.dialect = dotty 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.snyk: -------------------------------------------------------------------------------- 1 | ignore: 2 | 'SNYK-JAVA-ORGTYPELEVEL-2331743': 3 | - '*': 4 | reason: No fix available 5 | expires: 2022-10-01T17:33:45.004Z -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | Version 3.2.2 (2024-11-06) 2 | -------------------------- 3 | Fix for `_schema_version` overriding non-object entity data (#137) 4 | 5 | Version 3.2.1 (2024-01-10) 6 | -------------------------- 7 | Use only version for `_schema_version` context field (#134) 8 | 9 | Version 3.2.0 (2023-12-01) 10 | -------------------------- 11 | Parse Event from a ByteBuffer (#130) 12 | Add _schema_version field to each context during toShreddedJson transformation (#132) 13 | 14 | Version 3.1.0 (2023-03-17) 15 | -------------------------- 16 | Disable validation of field lengths when parsing event (#127) 17 | Build with Scala-3 (#124) 18 | 19 | Version 3.0.1 (2022-05-10) 20 | -------------------------- 21 | 3.0.0 was not published to maven (#121) 22 | 23 | Version 3.0.0 (2022-02-09) 24 | -------------------------- 25 | Event parser should fail on oversized fields (#115) 26 | Fix scoverage failures in github actions (#119) 27 | Update github workflows fix snyk vulnerability scanning (#120) 28 | 29 | Version 2.1.0 (2020-11-09) 30 | -------------------------- 31 | Update README to point to docs (#110) 32 | Add scalafmt (#112) 33 | Migrate from Travis to GH actions (#113) 34 | Integrate Snyk (#111) 35 | Add toTSV method (#97) 36 | Replace toJsonMap function with a lazy value (#107) 37 | Add benchmarking module (#108) 38 | 39 | Version 2.0.1 (2020-06-10) 40 | -------------------------- 41 | Fix Travis publish condition (#105) 42 | 43 | Version 2.0.0 (2020-06-09) 44 | -------------------------- 45 | Remove run manifest (#102) 46 | Add Scala 2.13 support (#101) 47 | Bump sbt-scoverage to 1.6.1 (#104) 48 | Bump Scala to 2.12.11 (#100) 49 | Bump sbt to 1.3.10 (#99) 50 | Bump iglu-core-circe to 1.0.0 (#98) 51 | 52 | Version 1.0.0 (2019-11-06) 53 | -------------------------- 54 | Make parsing errors type-safe (#75) 55 | Add function to create minimal event (#81) 56 | Deprecate run manifest (#86) 57 | Fix empty contexts and unstruct_event decoding bug (#92) 58 | Integrate MiMa (#87) 59 | Integrate scoverage (#90) 60 | Integrate sbt-gh-pages to create GH Pages from Scaladoc (#91) 61 | Remove Vagrant setup (#84) 62 | Add Travis CI secret key (#93) 63 | Add encryption label to .travis.yml (#94) 64 | Extend copyright notice to 2019 (#85) 65 | 66 | Version 0.4.2 (2019-08-06) 67 | -------------------------- 68 | Bump iglu-core to 0.5.1 (#73) 69 | Bump SBT to 1.2.8 (#74) 70 | Add decoder for Event (#78) 71 | Change Travis distribution to Trusty (#82) 72 | 73 | Version 0.4.1 (2018-03-05) 74 | -------------------------- 75 | Fix refr_dvce_tstamp field naming (#70) 76 | 77 | Version 0.4.0 (2018-02-13) 78 | -------------------------- 79 | Remove deprecated EventTransformer API (#68) 80 | Add type-safe Event API (#53) 81 | Bump Scala 2.11 to 2.11.12 in CI (#69) 82 | Bump Scala 2.12 to 2.12.8 (#66) 83 | Bump scalacheck to 1.14.0 (#65) 84 | Bump specs2 to 4.4.1 (#64) 85 | Bump aws-java-sdk to 1.11.490 (#63) 86 | Drop Scala 2.10 (#59) 87 | 88 | Version 0.3.2 (2018-08-24) 89 | -------------------------- 90 | Fix specification assuming unstruct_event field contains contexts envelope (#57) 91 | Fix non-flattening algorithm not returning inventory items (#58) 92 | 93 | Version 0.3.1 (2018-07-24) 94 | -------------------------- 95 | Add option to not flatten self-describing fields (#56) 96 | 97 | Version 0.3.0 (2018-03-06) 98 | -------------------------- 99 | Add Scala 2.12 support (#41) 100 | Bump json4s to 3.2.11 (#46) 101 | Bump aws-java-sdk to 1.11.289 (#48) 102 | Bump Scala 2.11 to 2.11.12 (#47) 103 | Bump SBT to 1.1.1 (#49) 104 | Extend copyright notice to 2018 (#51) 105 | Change se_value to Double (#52) 106 | 107 | Version 0.2.1 (2017-11-20) 108 | -------------------------- 109 | Fix non-merging matching contexts (#44) 110 | 111 | Version 0.2.0 (2017-05-24) 112 | -------------------------- 113 | Bump SBT to 0.13.15 (#32) 114 | Bump specs2 to 3.8.9 (#33) 115 | Add support for checking and setting a DynamoDB-backed run manifest (#31) 116 | Add transformWithInventory to the JSON EventTransformer (#34) 117 | JSON Event Transformer: don't add null unstruct_event and contexts fields to output (#11) 118 | Replace Scalaz Validation with Scala Either (#20) 119 | Use standard regular expression for schema URIs (#22) 120 | Allow empty custom contexts (#27) 121 | Add CI/CD to project (#18) 122 | Add Sonatype credentials to .travis.yml (#39) 123 | Add Bintray credentials to .travis.yml (#17) 124 | Update README markdown in according with CommonMark (#28) 125 | Migrate setup guide from README to dedicated snowplow/snowplow wiki page (#29) 126 | Migrate usage guide from README to dedicated snowplow/snowplow wiki page (#30) 127 | 128 | Version 0.1.1 (2016-07-27) 129 | -------------------------- 130 | Allow organisations in Iglu schema URIs to contain hyphens (#12) 131 | 132 | Version 0.1.0 (2016-03-22) 133 | -------------------------- 134 | Initial release 135 | -------------------------------------------------------------------------------- /LICENSE-2.0.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Snowplow Scala Analytics SDK 2 | 3 | [![Build Status][travis-image]][travis] 4 | [![Release][release-image]][releases] 5 | [![License][license-image]][license] 6 | 7 | ## Overview 8 | 9 | **[Snowplow][snowplow]** Analytics SDK for Scala lets you work with **[Snowplow enriched events][enriched-events]** in your Scala event processing and data modeling jobs. 10 | 11 | Use this SDK with **[Apache Spark][spark]**, **[AWS Lambda][lambda]**, **[Apache Flink][flink]**, **[Scalding][scalding]**, **[Apache Samza][samza]** and other Scala/JVM-compatible data processing frameworks. 12 | 13 | ## Documentation 14 | 15 | Setup guide and user guide can be found on our [documentation website](https://docs.snowplowanalytics.com/docs/modeling-your-data/analytics-sdk/analytics-sdk-scala/). 16 | 17 | Scaladoc for this project can be found as Github pages [here][scala-doc]. 18 | 19 | ## Benchmarking 20 | 21 | This project comes with [sbt-jmh](https://github.com/ktoso/sbt-jmh). 22 | 23 | Benchmarks need to be added [here](./benchmark/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/benchmark/). 24 | 25 | They can be run with `sbt "project benchmark" "+jmh:run -i 10 -wi 3 -f2 -t3"`. 26 | 27 | To get details about the parameters `jmh:run -h`. 28 | 29 | ## Copyright and license 30 | 31 | The Snowplow Scala Analytics SDK is copyright 2016-2019 Snowplow Analytics Ltd. 32 | 33 | Licensed under the **[Apache License, Version 2.0][license]** (the "License"); 34 | you may not use this software except in compliance with the License. 35 | 36 | Unless required by applicable law or agreed to in writing, software 37 | distributed under the License is distributed on an "AS IS" BASIS, 38 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 39 | See the License for the specific language governing permissions and 40 | limitations under the License. 41 | 42 | [travis-image]: https://travis-ci.org/snowplow/snowplow-scala-analytics-sdk.png?branch=master 43 | [travis]: http://travis-ci.org/snowplow/snowplow-scala-analytics-sdk 44 | 45 | [license-image]: http://img.shields.io/badge/license-Apache--2-blue.svg?style=flat 46 | [license]: http://www.apache.org/licenses/LICENSE-2.0 47 | 48 | [release-image]: http://img.shields.io/badge/release-3.2.2-blue.svg?style=flat 49 | [releases]: https://github.com/snowplow/snowplow-scala-analytics-sdk/releases 50 | 51 | [scala-doc]: http://snowplow.github.io/snowplow-scala-analytics-sdk/ 52 | 53 | [enriched-events]: https://docs.snowplowanalytics.com/docs/understanding-your-pipeline/canonical-event/ 54 | 55 | [spark]: http://spark.apache.org/ 56 | [lambda]: https://aws.amazon.com/lambda/ 57 | [flink]: https://flink.apache.org/ 58 | [scalding]: https://github.com/twitter/scalding 59 | [samza]: http://samza.apache.org/ 60 | -------------------------------------------------------------------------------- /benchmark/build.sbt: -------------------------------------------------------------------------------- 1 | sourceDirectory in Jmh := (sourceDirectory in Test).value 2 | classDirectory in Jmh := (classDirectory in Test).value 3 | dependencyClasspath in Jmh := (dependencyClasspath in Test).value 4 | // rewire tasks, so that 'jmh:run' automatically invokes 'jmh:compile' (otherwise a clean 'jmh:run' would fail) 5 | compile in Jmh := (compile in Jmh).dependsOn(compile in Test).value 6 | run in Jmh := (run in Jmh).dependsOn(Keys.compile in Jmh).evaluated -------------------------------------------------------------------------------- /benchmark/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/benchmark/OrderedBenchmark.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2020 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk.benchmark 14 | 15 | import org.openjdk.jmh.annotations._ 16 | 17 | import java.util.concurrent.TimeUnit 18 | import java.util.UUID 19 | import java.time.Instant 20 | 21 | import com.snowplowanalytics.snowplow.analytics.scalasdk.Event 22 | 23 | @State(Scope.Thread) 24 | @BenchmarkMode(Array(Mode.AverageTime, Mode.Throughput)) 25 | @OutputTimeUnit(TimeUnit.MICROSECONDS) 26 | class OrderedBenchmark { 27 | @Benchmark 28 | def ordered(state : States.AtomicEventState): Unit = { 29 | state.event.ordered 30 | } 31 | } 32 | 33 | object States { 34 | @State(Scope.Benchmark) 35 | class AtomicEventState { 36 | var event: Event = _ 37 | 38 | @Setup(Level.Trial) 39 | def init(): Unit = { 40 | val uuid = UUID.randomUUID() 41 | val timestamp = Instant.now() 42 | val vCollector = "2.0.0" 43 | val vTracker = "scala_0.7.0" 44 | event = Event.minimal(uuid, timestamp, vCollector, vTracker) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /benchmark/src/test/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/benchmark/ToTsvBenchmark.scala: -------------------------------------------------------------------------------- 1 | package com.snowplowanalytics.snowplow.analytics.scalasdk.benchmark 2 | 3 | import org.openjdk.jmh.annotations._ 4 | 5 | import java.util.concurrent.TimeUnit 6 | import java.util.UUID 7 | import java.time.Instant 8 | 9 | import com.snowplowanalytics.snowplow.analytics.scalasdk.Event 10 | 11 | @State(Scope.Thread) 12 | @BenchmarkMode(Array(Mode.AverageTime, Mode.Throughput)) 13 | @OutputTimeUnit(TimeUnit.MICROSECONDS) 14 | class ToTsvBenchmark { 15 | @Benchmark 16 | def toTsv(state : ToTsvBenchmark.AtomicEventState): Unit = { 17 | state.event.toTsv 18 | } 19 | } 20 | 21 | object ToTsvBenchmark { 22 | @State(Scope.Benchmark) 23 | class AtomicEventState { 24 | var event: Event = _ 25 | 26 | @Setup(Level.Trial) 27 | def init(): Unit = { 28 | val uuid = UUID.randomUUID() 29 | val timestamp = Instant.now() 30 | val vCollector = "2.0.0" 31 | val vTracker = "scala_0.7.0" 32 | event = Event.minimal(uuid, timestamp, vCollector, vTracker) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | 14 | lazy val root = project 15 | .in(file(".")) 16 | .settings( 17 | Seq[Setting[_]]( 18 | name := "snowplow-scala-analytics-sdk", 19 | organization := "com.snowplowanalytics", 20 | description := "Scala analytics SDK for Snowplow", 21 | scalaVersion := "2.13.10", 22 | crossScalaVersions := Seq("2.12.17", "2.13.10", "3.2.1") 23 | ) 24 | ) 25 | .enablePlugins(SiteScaladocPlugin) 26 | .enablePlugins(PreprocessPlugin) 27 | .settings(BuildSettings.dynVerSettings) 28 | .settings(BuildSettings.buildSettings) 29 | .settings(BuildSettings.publishSettings) 30 | .settings(BuildSettings.mimaSettings) 31 | .settings(BuildSettings.scoverageSettings) 32 | .settings(BuildSettings.sbtSiteSettings) 33 | .settings(BuildSettings.formattingSettings) 34 | .settings( 35 | Seq( 36 | shellPrompt := { _ => name.value + " > " } 37 | ) 38 | ) 39 | .settings( 40 | libraryDependencies ++= Seq( 41 | // Scala 42 | Dependencies.igluCore, 43 | Dependencies.cats, 44 | Dependencies.circeParser, 45 | Dependencies.circeGeneric, 46 | // Scala (test only) 47 | Dependencies.specs2, 48 | Dependencies.specs2Scalacheck, 49 | Dependencies.scalacheck, 50 | Dependencies.circeLiteral 51 | ) 52 | ) 53 | 54 | lazy val benchmark = project 55 | .in(file("benchmark")) 56 | .dependsOn(root % "test->test") 57 | .enablePlugins(JmhPlugin) 58 | -------------------------------------------------------------------------------- /project/BuildSettings.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | 14 | // SBT 15 | import sbt._ 16 | import Keys._ 17 | 18 | 19 | // Mima plugin 20 | import com.typesafe.tools.mima.plugin.MimaKeys._ 21 | import com.typesafe.tools.mima.core.{ProblemFilters, DirectMissingMethodProblem} 22 | 23 | // Scoverage plugin 24 | import scoverage.ScoverageKeys._ 25 | 26 | 27 | import sbtdynver.DynVerPlugin.autoImport._ 28 | 29 | import com.typesafe.sbt.site.SitePlugin.autoImport._ 30 | import com.typesafe.sbt.site.SiteScaladocPlugin.autoImport._ 31 | import com.typesafe.sbt.site.preprocess.PreprocessPlugin.autoImport._ 32 | 33 | import org.scalafmt.sbt.ScalafmtPlugin.autoImport._ 34 | 35 | object BuildSettings { 36 | 37 | // Basic settings for our app 38 | lazy val buildSettings = Seq( 39 | scalacOptions ++= Seq( 40 | "-deprecation", 41 | "-encoding", "UTF-8", 42 | "-feature", 43 | "-unchecked" 44 | ), 45 | scalacOptions ++= { 46 | if (scalaVersion.value.startsWith("3")) { 47 | Seq("-Xmax-inlines", "150") 48 | } else { 49 | Seq( 50 | "-Ywarn-dead-code", 51 | "-Ywarn-numeric-widen", 52 | "-Ywarn-value-discard" 53 | ) 54 | } 55 | } 56 | ) 57 | 58 | lazy val dynVerSettings = Seq( 59 | ThisBuild / dynverVTagPrefix := false, // Otherwise git tags required to have v-prefix 60 | ThisBuild / dynverSeparator := "-" // to be compatible with docker 61 | ) 62 | 63 | lazy val publishSettings = Seq[Setting[_]]( 64 | publishArtifact := true, 65 | Test / publishArtifact := false, 66 | licenses += ("Apache-2.0", url("http://www.apache.org/licenses/LICENSE-2.0.html")), 67 | pomIncludeRepository := { _ => false }, 68 | homepage := Some(url("http://snowplowanalytics.com")), 69 | developers := List( 70 | Developer( 71 | "Snowplow Analytics Ltd", 72 | "Snowplow Analytics Ltd", 73 | "support@snowplowanalytics.com", 74 | url("https://snowplowanalytics.com") 75 | ) 76 | ) 77 | ) 78 | 79 | // If new version introduces breaking changes, clear out the lists of previous version. 80 | // Otherwise, add previous version to set without removing other versions. 81 | val mimaPreviousVersionsScala2 = Set("3.0.1") 82 | val mimaPreviousVersionsScala3 = Set() 83 | lazy val mimaSettings = Seq( 84 | mimaPreviousArtifacts := { 85 | val versionsForBuild = 86 | CrossVersion.partialVersion(scalaVersion.value) match { 87 | case Some((3, _)) => 88 | mimaPreviousVersionsScala3 89 | case _ => 90 | mimaPreviousVersionsScala2 91 | } 92 | 93 | versionsForBuild.map { organization.value %% name.value % _ } 94 | }, 95 | ThisBuild / mimaFailOnNoPrevious := false, 96 | mimaBinaryIssueFilters ++= Seq( 97 | // DeriveParser should not have been public in previous versions 98 | ProblemFilters.exclude[DirectMissingMethodProblem]("com.snowplowanalytics.snowplow.analytics.scalasdk.decode.Parser#DeriveParser.get") 99 | ), 100 | Test / test := (Test / test).dependsOn(mimaReportBinaryIssues).value 101 | ) 102 | 103 | val scoverageSettings = Seq( 104 | coverageMinimumStmtTotal := 50, 105 | // Excluded because of shapeless, which would generate 1000x500KB statements driving coverage OOM 106 | coverageExcludedFiles := """.*\/Event.*;""", 107 | coverageFailOnMinimum := true, 108 | coverageHighlighting := false 109 | ) 110 | 111 | lazy val sbtSiteSettings = Seq( 112 | SiteScaladoc / siteSubdirName := s"${version.value}", 113 | Preprocess / preprocessVars := Map("VERSION" -> version.value) 114 | ) 115 | 116 | lazy val formattingSettings = Seq( 117 | scalafmtConfig := file(".scalafmt.conf"), 118 | scalafmtOnCompile := true 119 | ) 120 | 121 | } 122 | -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | import sbt._ 14 | 15 | object Dependencies { 16 | 17 | object V { 18 | val igluCore = "1.1.3" 19 | val cats = "2.9.0" 20 | val circe = "0.14.3" 21 | // Scala (test only) 22 | val specs2 = "4.19.0" 23 | val scalaCheck = "1.17.0" 24 | } 25 | 26 | val igluCore = "com.snowplowanalytics" %% "iglu-core-circe" % V.igluCore 27 | val cats = "org.typelevel" %% "cats-core" % V.cats 28 | val circeParser = "io.circe" %% "circe-parser" % V.circe 29 | val circeGeneric = "io.circe" %% "circe-generic" % V.circe 30 | // Scala (test only) 31 | val specs2 = "org.specs2" %% "specs2-core" % V.specs2 % Test 32 | val specs2Scalacheck = "org.specs2" %% "specs2-scalacheck" % V.specs2 % Test 33 | val scalacheck = "org.scalacheck" %% "scalacheck" % V.scalaCheck % Test 34 | val circeLiteral = "io.circe" %% "circe-literal" % V.circe % Test 35 | } 36 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.8.2 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe" % "sbt-mima-plugin" % "1.1.1") 2 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "2.0.7") 3 | addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.4.0") 4 | addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0") 5 | addSbtPlugin("pl.project13.scala" % "sbt-jmh" % "0.4.0") 6 | addSbtPlugin("com.geirsson" % "sbt-ci-release" % "1.5.7") 7 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.6") 8 | addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1") 9 | addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.10.0-RC1") 10 | 11 | libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always 12 | -------------------------------------------------------------------------------- /project/travis-deploy-key.enc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/snowplow/snowplow-scala-analytics-sdk/2ba728c6d1dfaca2242baabf515d73717adf5e99/project/travis-deploy-key.enc -------------------------------------------------------------------------------- /src/main/scala-2/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/Parser.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk.decode 14 | 15 | import cats.implicits._ 16 | import shapeless._ 17 | import shapeless.ops.record._ 18 | import shapeless.ops.hlist._ 19 | import cats.data.{NonEmptyList, Validated} 20 | import java.nio.ByteBuffer 21 | import scala.collection.mutable.ListBuffer 22 | import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.{FieldNumberMismatch, NotTSV, RowDecodingError} 23 | 24 | private[scalasdk] trait Parser[A] extends TSVParser[A] { 25 | 26 | /** Heterogeneous TSV values */ 27 | type HTSV <: HList 28 | 29 | def expectedNumFields: Int 30 | 31 | /** Evidence allowing to transform TSV line into `HList` */ 32 | protected def decoder: RowDecoder[HTSV] 33 | 34 | /** Evidence that `A` is isomorphic to `HTSV` */ 35 | protected def generic: Generic.Aux[A, HTSV] 36 | 37 | def parse(row: String): DecodeResult[A] = { 38 | val values = row.split("\t", -1) 39 | if (values.length == 1) 40 | Validated.Invalid(NotTSV) 41 | else if (values.length != expectedNumFields) 42 | Validated.Invalid(FieldNumberMismatch(values.length)) 43 | else { 44 | val decoded = decoder(values.toList).leftMap(e => RowDecodingError(e)) 45 | decoded.map(decodedValue => generic.from(decodedValue)) 46 | } 47 | } 48 | 49 | def parseBytes(row: ByteBuffer): DecodeResult[A] = { 50 | val values = Parser.splitBuffer(row) 51 | if (values.length == 1) 52 | Validated.Invalid(NotTSV) 53 | else if (values.length != expectedNumFields) 54 | Validated.Invalid(FieldNumberMismatch(values.length)) 55 | else { 56 | val decoded = decoder.decodeBytes(values.result()).leftMap(e => RowDecodingError(e)) 57 | decoded.map(decodedValue => generic.from(decodedValue)) 58 | } 59 | } 60 | } 61 | 62 | object Parser { 63 | 64 | private val tab: Byte = '\t'.toByte 65 | 66 | private def splitBuffer(row: ByteBuffer): ListBuffer[ByteBuffer] = { 67 | var current = row.duplicate 68 | val builder = ListBuffer(current) 69 | (row.position() until row.limit()).foreach { i => 70 | if (row.get(i) === tab) { 71 | current.limit(i) 72 | current = row.duplicate.position(i + 1) 73 | builder += current 74 | } 75 | } 76 | builder 77 | } 78 | 79 | private[scalasdk] sealed trait DeriveParser[A] { 80 | 81 | def knownKeys[R <: HList, K <: HList, L <: HList]( 82 | implicit lgen: LabelledGeneric.Aux[A, R], 83 | keys: Keys.Aux[R, K], 84 | gen: Generic.Aux[A, L], 85 | toTraversableAux: ToTraversable.Aux[K, List, Symbol] 86 | ): List[String] = 87 | keys().toList.map(_.name) 88 | 89 | /** 90 | * Get instance of parser after all evidences are given 91 | * @tparam R full class representation with field names and types 92 | * @tparam K evidence of field names 93 | * @tparam L evidence of field types 94 | */ 95 | def get[R <: HList, K <: HList, L <: HList]( 96 | maxLengths: Map[String, Int] 97 | )( 98 | implicit lgen: LabelledGeneric.Aux[A, R], 99 | keys: Keys.Aux[R, K], 100 | gen: Generic.Aux[A, L], 101 | toTraversableAux: ToTraversable.Aux[K, List, Symbol], 102 | deriveRowDecoder: RowDecoder.DeriveRowDecoder[L] 103 | ): TSVParser[A] = 104 | new Parser[A] { 105 | type HTSV = L 106 | val keyList = keys().toList 107 | val expectedNumFields: Int = keyList.length 108 | val decoder: RowDecoder[L] = deriveRowDecoder.get(keyList, maxLengths) 109 | val generic: Generic.Aux[A, L] = gen 110 | } 111 | } 112 | 113 | /** Derive a TSV parser for `A` */ 114 | private[scalasdk] def deriveFor[A]: DeriveParser[A] = 115 | new DeriveParser[A] {} 116 | } 117 | -------------------------------------------------------------------------------- /src/main/scala-2/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/RowDecoderCompanion.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk.decode 14 | 15 | import shapeless._ 16 | import cats.syntax.validated._ 17 | import cats.syntax.either._ 18 | import cats.syntax.apply._ 19 | import java.nio.ByteBuffer 20 | import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo.UnhandledRowDecodingError 21 | 22 | private[scalasdk] trait RowDecoderCompanion { 23 | import HList.ListCompat._ 24 | 25 | sealed trait DeriveRowDecoder[L] { 26 | def get(knownKeys: List[Key], maxLengths: Map[String, Int]): RowDecoder[L] 27 | } 28 | 29 | object DeriveRowDecoder { 30 | def apply[L](implicit fromRow: DeriveRowDecoder[L]): DeriveRowDecoder[L] = fromRow 31 | } 32 | 33 | /** Parse TSV row into HList */ 34 | private def parse[H: ValueDecoder, T <: HList]( 35 | key: Key, 36 | tailDecoder: RowDecoder[T], 37 | maxLength: Option[Int], 38 | row: List[String] 39 | ): RowDecodeResult[H :: T] = 40 | row match { 41 | case h :: t => 42 | val hv: RowDecodeResult[H] = ValueDecoder[H].parse(key, h, maxLength).toValidatedNel 43 | val tv: RowDecodeResult[T] = tailDecoder.apply(t) 44 | (hv, tv).mapN(_ :: _) 45 | case Nil => UnhandledRowDecodingError("Not enough values, format is invalid").invalidNel 46 | } 47 | 48 | /** Parse TSV row into HList */ 49 | private def parseBytes[H: ValueDecoder, T <: HList]( 50 | key: Key, 51 | tailDecoder: RowDecoder[T], 52 | maxLength: Option[Int], 53 | row: List[ByteBuffer] 54 | ): RowDecodeResult[H :: T] = 55 | row match { 56 | case h :: t => 57 | val hv: RowDecodeResult[H] = ValueDecoder[H].parseBytes(key, h, maxLength).toValidatedNel 58 | val tv: RowDecodeResult[T] = tailDecoder.decodeBytes(t) 59 | (hv, tv).mapN(_ :: _) 60 | case Nil => UnhandledRowDecodingError("Not enough values, format is invalid").invalidNel 61 | } 62 | 63 | implicit def hnilFromRow: DeriveRowDecoder[HNil] = 64 | new DeriveRowDecoder[HNil] { 65 | def get(knownKeys: List[Key], maxLengths: Map[String, Int]): RowDecoder[HNil] = 66 | new RowDecoder[HNil] { 67 | def apply(row: List[String]): RowDecodeResult[HNil] = 68 | row match { 69 | case Nil => 70 | HNil.validNel 71 | case _ => 72 | UnhandledRowDecodingError("Not enough values, format is invalid").invalidNel 73 | } 74 | 75 | def decodeBytes(row: List[ByteBuffer]): RowDecodeResult[HNil] = 76 | row match { 77 | case Nil => 78 | HNil.validNel 79 | case _ => 80 | UnhandledRowDecodingError("Not enough values, format is invalid").invalidNel 81 | } 82 | } 83 | } 84 | 85 | implicit def hconsFromRow[H: ValueDecoder, T <: HList: DeriveRowDecoder]: DeriveRowDecoder[H :: T] = 86 | new DeriveRowDecoder[H :: T] { 87 | def get(knownKeys: List[Key], maxLengths: Map[String, Int]): RowDecoder[H :: T] = 88 | knownKeys match { 89 | case key :: tailKeys => 90 | val tailDecoder = DeriveRowDecoder.apply[T].get(tailKeys, maxLengths) 91 | val maxLength = maxLengths.get(key.name) 92 | new RowDecoder[H :: T] { 93 | def apply(row: List[String]): RowDecodeResult[H :: T] = parse(key, tailDecoder, maxLength, row) 94 | def decodeBytes(row: List[ByteBuffer]): RowDecodeResult[H :: T] = parseBytes(key, tailDecoder, maxLength, row) 95 | } 96 | case Nil => 97 | // Shapeless type checking makes this impossible 98 | throw new IllegalStateException 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/scala-3/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/Parser.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk.decode 14 | 15 | import cats.implicits._ 16 | import cats.data.{NonEmptyList, Validated} 17 | import java.nio.ByteBuffer 18 | import scala.collection.mutable.ListBuffer 19 | import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.{FieldNumberMismatch, NotTSV, RowDecodingError} 20 | import scala.deriving._ 21 | import scala.compiletime._ 22 | 23 | private[scalasdk] trait Parser[A] extends TSVParser[A] { 24 | 25 | /** List of field names defined on `A` */ 26 | def expectedNumFields: Int 27 | 28 | protected def decoder: RowDecoder[A] 29 | 30 | def parse(row: String): DecodeResult[A] = { 31 | val values = row.split("\t", -1) 32 | if (values.length == 1) Validated.Invalid(NotTSV) 33 | else if (values.length != expectedNumFields) Validated.Invalid(FieldNumberMismatch(values.length)) 34 | else decoder(values.toList).leftMap(e => RowDecodingError(e)) 35 | } 36 | 37 | def parseBytes(row: ByteBuffer): DecodeResult[A] = { 38 | val values = Parser.splitBuffer(row) 39 | if (values.length == 1) Validated.Invalid(NotTSV) 40 | else if (values.length != expectedNumFields) Validated.Invalid(FieldNumberMismatch(values.length)) 41 | else decoder.decodeBytes(values.result()).leftMap(e => RowDecodingError(e)) 42 | } 43 | } 44 | 45 | object Parser { 46 | 47 | private val tab: Byte = '\t'.toByte 48 | 49 | private def splitBuffer(row: ByteBuffer): ListBuffer[ByteBuffer] = { 50 | var current = row.duplicate 51 | val builder = ListBuffer(current) 52 | (row.position() until row.limit()).foreach { i => 53 | if (row.get(i) === tab) { 54 | current.limit(i) 55 | current = row.duplicate.position(i + 1) 56 | builder += current 57 | } 58 | } 59 | builder 60 | } 61 | 62 | private[scalasdk] sealed trait DeriveParser[A] { 63 | inline def knownKeys(implicit mirror: Mirror.ProductOf[A]): List[String] = 64 | constValueTuple[mirror.MirroredElemLabels].toArray.map(_.toString).toList 65 | 66 | inline def get(maxLengths: Map[String, Int])(implicit mirror: Mirror.ProductOf[A]): TSVParser[A] = 67 | new Parser[A] { 68 | val knownKeys: List[Symbol] = constValueTuple[mirror.MirroredElemLabels].toArray.map(s => Symbol(s.toString)).toList 69 | val expectedNumFields = knownKeys.length 70 | val decoder: RowDecoder[A] = RowDecoder.DeriveRowDecoder.of[A].get(knownKeys, maxLengths) 71 | } 72 | } 73 | 74 | /** Derive a TSV parser for `A` */ 75 | private[scalasdk] def deriveFor[A]: DeriveParser[A] = 76 | new DeriveParser[A] {} 77 | } 78 | -------------------------------------------------------------------------------- /src/main/scala-3/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/RowDecoderCompanion.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk.decode 14 | 15 | import cats.syntax.validated._ 16 | import cats.syntax.either._ 17 | import cats.syntax.apply._ 18 | import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo.UnhandledRowDecodingError 19 | import java.nio.ByteBuffer 20 | import scala.deriving._ 21 | import scala.compiletime._ 22 | 23 | private[scalasdk] trait RowDecoderCompanion { 24 | 25 | sealed trait DeriveRowDecoder[L] { self => 26 | def get(knownKeys: List[Key], maxLengths: Map[String, Int]): RowDecoder[L] 27 | 28 | def map[B](f: L => B): DeriveRowDecoder[B] = 29 | new DeriveRowDecoder[B] { 30 | def get(knownKeys: List[Key], maxLengths: Map[String, Int]): RowDecoder[B] = 31 | self.get(knownKeys, maxLengths).map(f) 32 | } 33 | } 34 | 35 | object DeriveRowDecoder { 36 | inline def of[L](implicit m: Mirror.ProductOf[L]): DeriveRowDecoder[L] = { 37 | val instance = summonInline[DeriveRowDecoder[m.MirroredElemTypes]] 38 | instance.map(tuple => m.fromTuple(tuple)) 39 | } 40 | } 41 | 42 | private def parse[H: ValueDecoder, T <: Tuple]( 43 | key: Key, 44 | tailDecoder: RowDecoder[T], 45 | maxLength: Option[Int], 46 | row: List[String] 47 | ): RowDecodeResult[H *: T] = 48 | row match { 49 | case h :: t => 50 | val hv: RowDecodeResult[H] = ValueDecoder[H].parse(key, h, maxLength).toValidatedNel 51 | val tv: RowDecodeResult[T] = tailDecoder.apply(t) 52 | (hv, tv).mapN(_ *: _) 53 | case Nil => UnhandledRowDecodingError("Not enough values, format is invalid").invalidNel 54 | } 55 | 56 | private def parseBytes[H: ValueDecoder, T <: Tuple]( 57 | key: Key, 58 | tailDecoder: RowDecoder[T], 59 | maxLength: Option[Int], 60 | row: List[ByteBuffer] 61 | ): RowDecodeResult[H *: T] = 62 | row match { 63 | case h :: t => 64 | val hv: RowDecodeResult[H] = ValueDecoder[H].parseBytes(key, h, maxLength).toValidatedNel 65 | val tv: RowDecodeResult[T] = tailDecoder.decodeBytes(t) 66 | (hv, tv).mapN(_ *: _) 67 | case Nil => UnhandledRowDecodingError("Not enough values, format is invalid").invalidNel 68 | } 69 | 70 | implicit def hnilFromRow: DeriveRowDecoder[EmptyTuple] = 71 | new DeriveRowDecoder[EmptyTuple] { 72 | def get(knownKeys: List[Key], maxLengths: Map[String, Int]): RowDecoder[EmptyTuple] = 73 | new RowDecoder[EmptyTuple] { 74 | def apply(row: List[String]): RowDecodeResult[EmptyTuple] = 75 | row match { 76 | case Nil => 77 | EmptyTuple.validNel 78 | case _ => 79 | UnhandledRowDecodingError("Not enough values, format is invalid").invalidNel 80 | } 81 | 82 | def decodeBytes(row: List[ByteBuffer]): RowDecodeResult[EmptyTuple] = 83 | row match { 84 | case Nil => 85 | EmptyTuple.validNel 86 | case _ => 87 | UnhandledRowDecodingError("Not enough values, format is invalid").invalidNel 88 | } 89 | } 90 | } 91 | 92 | implicit def hconsFromRow[H: ValueDecoder, T <: Tuple: DeriveRowDecoder]: DeriveRowDecoder[H *: T] = 93 | new DeriveRowDecoder[H *: T] { 94 | def get(knownKeys: List[Key], maxLengths: Map[String, Int]): RowDecoder[H *: T] = 95 | knownKeys match { 96 | case key :: tailKeys => 97 | val tailDecoder = summon[DeriveRowDecoder[T]].get(tailKeys, maxLengths) 98 | val maxLength = maxLengths.get(key.name) 99 | new RowDecoder[H *: T] { 100 | def apply(row: List[String]): RowDecodeResult[H *: T] = parse(key, tailDecoder, maxLength, row) 101 | def decodeBytes(row: List[ByteBuffer]): RowDecodeResult[H *: T] = parseBytes(key, tailDecoder, maxLength, row) 102 | } 103 | case Nil => 104 | // Shapeless type checking makes this impossible 105 | throw new IllegalStateException 106 | } 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/Common.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk 14 | 15 | import com.snowplowanalytics.iglu.core.{SchemaCriterion, SchemaKey, SchemaVer} 16 | 17 | object Common { 18 | 19 | val UnstructEventCriterion = 20 | SchemaCriterion("com.snowplowanalytics.snowplow", "unstruct_event", "jsonschema", 1, 0) 21 | 22 | val ContextsCriterion = 23 | SchemaCriterion("com.snowplowanalytics.snowplow", "contexts", "jsonschema", 1, 0) 24 | 25 | val UnstructEventUri = 26 | SchemaKey("com.snowplowanalytics.snowplow", "unstruct_event", "jsonschema", SchemaVer.Full(1, 0, 0)) 27 | 28 | val ContextsUri = 29 | SchemaKey("com.snowplowanalytics.snowplow", "contexts", "jsonschema", SchemaVer.Full(1, 0, 0)) 30 | } 31 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/Data.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk 14 | 15 | import com.snowplowanalytics.iglu.core.SchemaKey 16 | 17 | /** 18 | * Common data types for enriched event 19 | */ 20 | object Data { 21 | 22 | /** 23 | * The type (contexts/derived_contexts/unstruct_event) and Iglu URI of a shredded type 24 | */ 25 | case class ShreddedType(shredProperty: ShredProperty, schemaKey: SchemaKey) 26 | 27 | /** 28 | * Known contexts types of enriched event 29 | */ 30 | sealed trait ContextsType { 31 | def field: String 32 | } 33 | 34 | case object DerivedContexts extends ContextsType { 35 | def field = "derived_contexts" 36 | } 37 | 38 | case object CustomContexts extends ContextsType { 39 | def field = "contexts" 40 | } 41 | 42 | /** 43 | * Field types of enriched event that can be shredded (self-describing JSONs) 44 | */ 45 | sealed trait ShredProperty { 46 | 47 | /** 48 | * Canonical field name 49 | */ 50 | def name: String 51 | 52 | /** 53 | * Result output prefix 54 | */ 55 | def prefix: String 56 | } 57 | 58 | case class Contexts(contextType: ContextsType) extends ShredProperty { 59 | def name = contextType.field 60 | def prefix = "contexts_" 61 | } 62 | 63 | case object UnstructEvent extends ShredProperty { 64 | def name = "unstruct_event" 65 | def prefix = name + "_" 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/Event.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2020 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk 14 | 15 | // java 16 | import java.time.Instant 17 | import java.util.UUID 18 | import java.time.format.DateTimeFormatter 19 | import java.nio.ByteBuffer 20 | 21 | // circe 22 | import io.circe.{Decoder, Encoder, Json, JsonObject} 23 | import io.circe.Json.JString 24 | import io.circe.generic.semiauto._ 25 | import io.circe.syntax._ 26 | 27 | // iglu 28 | import com.snowplowanalytics.iglu.core.SelfDescribingData 29 | import com.snowplowanalytics.iglu.core.circe.implicits._ 30 | 31 | // This library 32 | import com.snowplowanalytics.snowplow.analytics.scalasdk.decode.{DecodeResult, Key, Parser, TSVParser} 33 | import com.snowplowanalytics.snowplow.analytics.scalasdk.SnowplowEvent.{Contexts, UnstructEvent} 34 | import com.snowplowanalytics.snowplow.analytics.scalasdk.SnowplowEvent._ 35 | import com.snowplowanalytics.snowplow.analytics.scalasdk.validate.FIELD_SIZES 36 | import com.snowplowanalytics.snowplow.analytics.scalasdk.encode.TsvEncoder 37 | 38 | /** 39 | * Case class representing a canonical Snowplow event. 40 | * 41 | * @see https://docs.snowplowanalytics.com/docs/understanding-your-pipeline/canonical-event/ 42 | */ 43 | // format: off 44 | case class Event( 45 | app_id: Option[String], 46 | platform: Option[String], 47 | etl_tstamp: Option[Instant], 48 | collector_tstamp: Instant, 49 | dvce_created_tstamp: Option[Instant], 50 | event: Option[String], 51 | event_id: UUID, 52 | txn_id: Option[Int], 53 | name_tracker: Option[String], 54 | v_tracker: Option[String], 55 | v_collector: String, 56 | v_etl: String, 57 | user_id: Option[String], 58 | user_ipaddress: Option[String], 59 | user_fingerprint: Option[String], 60 | domain_userid: Option[String], 61 | domain_sessionidx: Option[Int], 62 | network_userid: Option[String], 63 | geo_country: Option[String], 64 | geo_region: Option[String], 65 | geo_city: Option[String], 66 | geo_zipcode: Option[String], 67 | geo_latitude: Option[Double], 68 | geo_longitude: Option[Double], 69 | geo_region_name: Option[String], 70 | ip_isp: Option[String], 71 | ip_organization: Option[String], 72 | ip_domain: Option[String], 73 | ip_netspeed: Option[String], 74 | page_url: Option[String], 75 | page_title: Option[String], 76 | page_referrer: Option[String], 77 | page_urlscheme: Option[String], 78 | page_urlhost: Option[String], 79 | page_urlport: Option[Int], 80 | page_urlpath: Option[String], 81 | page_urlquery: Option[String], 82 | page_urlfragment: Option[String], 83 | refr_urlscheme: Option[String], 84 | refr_urlhost: Option[String], 85 | refr_urlport: Option[Int], 86 | refr_urlpath: Option[String], 87 | refr_urlquery: Option[String], 88 | refr_urlfragment: Option[String], 89 | refr_medium: Option[String], 90 | refr_source: Option[String], 91 | refr_term: Option[String], 92 | mkt_medium: Option[String], 93 | mkt_source: Option[String], 94 | mkt_term: Option[String], 95 | mkt_content: Option[String], 96 | mkt_campaign: Option[String], 97 | contexts: Contexts, 98 | se_category: Option[String], 99 | se_action: Option[String], 100 | se_label: Option[String], 101 | se_property: Option[String], 102 | se_value: Option[Double], 103 | unstruct_event: UnstructEvent, 104 | tr_orderid: Option[String], 105 | tr_affiliation: Option[String], 106 | tr_total: Option[Double], 107 | tr_tax: Option[Double], 108 | tr_shipping: Option[Double], 109 | tr_city: Option[String], 110 | tr_state: Option[String], 111 | tr_country: Option[String], 112 | ti_orderid: Option[String], 113 | ti_sku: Option[String], 114 | ti_name: Option[String], 115 | ti_category: Option[String], 116 | ti_price: Option[Double], 117 | ti_quantity: Option[Int], 118 | pp_xoffset_min: Option[Int], 119 | pp_xoffset_max: Option[Int], 120 | pp_yoffset_min: Option[Int], 121 | pp_yoffset_max: Option[Int], 122 | useragent: Option[String], 123 | br_name: Option[String], 124 | br_family: Option[String], 125 | br_version: Option[String], 126 | br_type: Option[String], 127 | br_renderengine: Option[String], 128 | br_lang: Option[String], 129 | br_features_pdf: Option[Boolean], 130 | br_features_flash: Option[Boolean], 131 | br_features_java: Option[Boolean], 132 | br_features_director: Option[Boolean], 133 | br_features_quicktime: Option[Boolean], 134 | br_features_realplayer: Option[Boolean], 135 | br_features_windowsmedia: Option[Boolean], 136 | br_features_gears: Option[Boolean], 137 | br_features_silverlight: Option[Boolean], 138 | br_cookies: Option[Boolean], 139 | br_colordepth: Option[String], 140 | br_viewwidth: Option[Int], 141 | br_viewheight: Option[Int], 142 | os_name: Option[String], 143 | os_family: Option[String], 144 | os_manufacturer: Option[String], 145 | os_timezone: Option[String], 146 | dvce_type: Option[String], 147 | dvce_ismobile: Option[Boolean], 148 | dvce_screenwidth: Option[Int], 149 | dvce_screenheight: Option[Int], 150 | doc_charset: Option[String], 151 | doc_width: Option[Int], 152 | doc_height: Option[Int], 153 | tr_currency: Option[String], 154 | tr_total_base: Option[Double], 155 | tr_tax_base: Option[Double], 156 | tr_shipping_base: Option[Double], 157 | ti_currency: Option[String], 158 | ti_price_base: Option[Double], 159 | base_currency: Option[String], 160 | geo_timezone: Option[String], 161 | mkt_clickid: Option[String], 162 | mkt_network: Option[String], 163 | etl_tags: Option[String], 164 | dvce_sent_tstamp: Option[Instant], 165 | refr_domain_userid: Option[String], 166 | refr_dvce_tstamp: Option[Instant], 167 | derived_contexts: Contexts, 168 | domain_sessionid: Option[String], 169 | derived_tstamp: Option[Instant], 170 | event_vendor: Option[String], 171 | event_name: Option[String], 172 | event_format: Option[String], 173 | event_version: Option[String], 174 | event_fingerprint: Option[String], 175 | true_tstamp: Option[Instant] 176 | ) { 177 | // format: on 178 | 179 | /** 180 | * Extracts metadata from the event containing information about the types and Iglu URIs of its shred properties 181 | */ 182 | def inventory: Set[Data.ShreddedType] = { 183 | val unstructEvent = unstruct_event.data.toSet 184 | .map((ue: SelfDescribingData[Json]) => Data.ShreddedType(Data.UnstructEvent, ue.schema)) 185 | 186 | val derivedContexts = derived_contexts.data.toSet 187 | .map((ctx: SelfDescribingData[Json]) => Data.ShreddedType(Data.Contexts(Data.DerivedContexts), ctx.schema)) 188 | 189 | val customContexts = contexts.data.toSet 190 | .map((ctx: SelfDescribingData[Json]) => Data.ShreddedType(Data.Contexts(Data.CustomContexts), ctx.schema)) 191 | 192 | customContexts ++ derivedContexts ++ unstructEvent 193 | } 194 | 195 | /** 196 | * Returns the event as a map of keys to Circe JSON values, while dropping inventory fields 197 | */ 198 | def atomic: Map[String, Json] = jsonMap - "contexts" - "unstruct_event" - "derived_contexts" 199 | 200 | /** 201 | * Returns the event as a list of key/Circe JSON value pairs. 202 | * Unlike `jsonMap` and `atomic`, these keys use the ordering of the canonical event model 203 | */ 204 | def ordered: List[(String, Option[Json])] = 205 | Event.fieldNames.map(key => (key, jsonMap.get(key))) 206 | 207 | /** 208 | * Returns a compound JSON field containing information about an event's latitude and longitude, 209 | * or None if one of these fields doesn't exist 210 | */ 211 | def geoLocation: Option[(String, Json)] = 212 | for { 213 | lat <- geo_latitude 214 | lon <- geo_longitude 215 | } yield "geo_location" -> s"$lat,$lon".asJson 216 | 217 | /** 218 | * Transforms the event to a validated JSON whose keys are the field names corresponding to the 219 | * EnrichedEvent POJO of the Scala Common Enrich project. If the lossy argument is true, any 220 | * self-describing events in the fields (unstruct_event, contexts and derived_contexts) are returned 221 | * in a "shredded" format (e.g. "unstruct_event_com_acme_1_myField": "value"), otherwise a standard 222 | * self-describing format is used. 223 | * 224 | * @param lossy Whether unstruct_event, contexts and derived_contexts should be flattened 225 | */ 226 | def toJson(lossy: Boolean): Json = 227 | if (lossy) 228 | JsonObject 229 | .fromMap( 230 | atomic ++ contexts.toShreddedJson.toMap ++ derived_contexts.toShreddedJson.toMap ++ unstruct_event.toShreddedJson.toMap ++ geoLocation 231 | ) 232 | .asJson 233 | else 234 | this.asJson 235 | 236 | /** Create the TSV representation of this event. */ 237 | def toTsv: String = TsvEncoder.encode(this) 238 | 239 | /** 240 | * This event as a map of keys to Circe JSON values 241 | */ 242 | private lazy val jsonMap: Map[String, Json] = this.asJsonObject.toMap 243 | } 244 | 245 | object Event { 246 | 247 | @deprecated("Event.unsafe functionality is merged into Event.parse method", "3.1.0") 248 | object unsafe { 249 | implicit def unsafeEventDecoder: Decoder[Event] = Event.eventDecoder 250 | } 251 | 252 | /** 253 | * Automatically derived Circe encoder 254 | */ 255 | implicit val jsonEncoder: Encoder.AsObject[Event] = deriveEncoder[Event] 256 | 257 | implicit def eventDecoder: Decoder[Event] = deriveDecoder[Event] 258 | 259 | /** 260 | * Derive a TSV parser for the Event class 261 | * 262 | * @param truncateAtomicFields A map from field names, e.g. "app_id", to maximum string lengths. 263 | * If supplied, the event fields are truncated to not exceed these maximum values. 264 | * 265 | * @note Enrich already performs atomic field length validation since version 3.0.0. Only supply 266 | * a non-empty map if atomic field lengths are important to you, and either you are parsing 267 | * events generated by an older version of enrich, or if you run enrich with the 268 | * `featureFlags.acceptInvalid` config option on. 269 | */ 270 | def parser(truncateAtomicFields: Map[String, Int] = Map.empty): TSVParser[Event] = 271 | Parser.deriveFor[Event].get(truncateAtomicFields) 272 | 273 | private lazy val stdParser: TSVParser[Event] = parser() 274 | 275 | /** 276 | * Converts a string with an enriched event TSV to an Event instance, 277 | * or a ValidatedNel containing information about errors 278 | * 279 | * @param line Enriched event TSV line 280 | */ 281 | def parse(line: String): DecodeResult[Event] = 282 | stdParser.parse(line) 283 | 284 | def parseBytes(bytes: ByteBuffer): DecodeResult[Event] = 285 | stdParser.parseBytes(bytes) 286 | 287 | private lazy val fieldNames: List[String] = 288 | Parser.deriveFor[Event].knownKeys 289 | 290 | /** 291 | * Creates an event with only required fields. 292 | * All optional fields are set to [[None]]. 293 | */ 294 | def minimal( 295 | id: UUID, 296 | collectorTstamp: Instant, 297 | vCollector: String, 298 | vEtl: String 299 | ): Event = 300 | Event( 301 | None, 302 | None, 303 | None, 304 | collectorTstamp, 305 | None, 306 | None, 307 | id, 308 | None, 309 | None, 310 | None, 311 | vCollector, 312 | vEtl, 313 | None, 314 | None, 315 | None, 316 | None, 317 | None, 318 | None, 319 | None, 320 | None, 321 | None, 322 | None, 323 | None, 324 | None, 325 | None, 326 | None, 327 | None, 328 | None, 329 | None, 330 | None, 331 | None, 332 | None, 333 | None, 334 | None, 335 | None, 336 | None, 337 | None, 338 | None, 339 | None, 340 | None, 341 | None, 342 | None, 343 | None, 344 | None, 345 | None, 346 | None, 347 | None, 348 | None, 349 | None, 350 | None, 351 | None, 352 | None, 353 | Contexts(Nil), 354 | None, 355 | None, 356 | None, 357 | None, 358 | None, 359 | UnstructEvent(None), 360 | None, 361 | None, 362 | None, 363 | None, 364 | None, 365 | None, 366 | None, 367 | None, 368 | None, 369 | None, 370 | None, 371 | None, 372 | None, 373 | None, 374 | None, 375 | None, 376 | None, 377 | None, 378 | None, 379 | None, 380 | None, 381 | None, 382 | None, 383 | None, 384 | None, 385 | None, 386 | None, 387 | None, 388 | None, 389 | None, 390 | None, 391 | None, 392 | None, 393 | None, 394 | None, 395 | None, 396 | None, 397 | None, 398 | None, 399 | None, 400 | None, 401 | None, 402 | None, 403 | None, 404 | None, 405 | None, 406 | None, 407 | None, 408 | None, 409 | None, 410 | None, 411 | None, 412 | None, 413 | None, 414 | None, 415 | None, 416 | None, 417 | None, 418 | None, 419 | None, 420 | None, 421 | None, 422 | None, 423 | Contexts(Nil), 424 | None, 425 | None, 426 | None, 427 | None, 428 | None, 429 | None, 430 | None, 431 | None 432 | ) 433 | } 434 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/ParsingError.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk 14 | 15 | import cats.data.NonEmptyList 16 | import cats.syntax.either._ 17 | import io.circe._ 18 | import io.circe.syntax._ 19 | import com.snowplowanalytics.snowplow.analytics.scalasdk.decode.Key 20 | import com.snowplowanalytics.snowplow.analytics.scalasdk.decode.Key._ 21 | 22 | /** 23 | * Represents an error raised when parsing a TSV line. 24 | */ 25 | sealed trait ParsingError extends Product with Serializable 26 | 27 | object ParsingError { 28 | 29 | /** 30 | * Represents an error indicating a non-TSV line. 31 | */ 32 | case object NotTSV extends ParsingError 33 | 34 | /** 35 | * Represents an error indicating the number of actual fields is not equal 36 | * to the number of expected fields. 37 | * @param fieldCount The number of fields in the TSV line. 38 | */ 39 | final case class FieldNumberMismatch(fieldCount: Int) extends ParsingError 40 | 41 | /** 42 | * Represents an error raised when trying to decode the values in a line. 43 | * @param errors A non-empty list of errors encountered when trying to decode the values. 44 | */ 45 | final case class RowDecodingError(errors: NonEmptyList[RowDecodingErrorInfo]) extends ParsingError 46 | 47 | /** 48 | * Contains information about the reasons behind errors raised when trying to decode the values in a line. 49 | */ 50 | sealed trait RowDecodingErrorInfo extends Product with Serializable 51 | 52 | object RowDecodingErrorInfo { 53 | 54 | /** 55 | * Represents cases where tha value in a field is not valid, 56 | * e.g. an invalid timestamp, an invalid UUID, etc. 57 | * @param key The name of the field. 58 | * @param value The value of field. 59 | * @param message The error message. 60 | */ 61 | final case class InvalidValue( 62 | key: Key, 63 | value: String, 64 | message: String 65 | ) extends RowDecodingErrorInfo 66 | 67 | /** 68 | * Represents unhandled errors raised when trying to decode a line. 69 | * For example, while parsing a list of tuples to [[HList]] in 70 | * [[RowDecoder]], type checking should make it impossible to get more or less values 71 | * than expected. 72 | * @param message The error message. 73 | */ 74 | final case class UnhandledRowDecodingError(message: String) extends RowDecodingErrorInfo 75 | 76 | implicit val analyticsSdkRowDecodingErrorInfoCirceEncoder: Encoder[RowDecodingErrorInfo] = 77 | Encoder.instance { 78 | case InvalidValue(key, value, message) => 79 | Json.obj( 80 | "type" := "InvalidValue", 81 | "key" := key.name, 82 | "value" := value, 83 | "message" := message 84 | ) 85 | case UnhandledRowDecodingError(message: String) => 86 | Json.obj( 87 | "type" := "UnhandledRowDecodingError", 88 | "message" := message 89 | ) 90 | } 91 | 92 | implicit val analyticsSdkRowDecodingErrorInfoCirceDecoder: Decoder[RowDecodingErrorInfo] = 93 | Decoder.instance { cursor => 94 | for { 95 | errorType <- cursor.downField("type").as[String] 96 | result <- errorType match { 97 | case "InvalidValue" => 98 | for { 99 | key <- cursor.downField("key").as[Key] 100 | value <- cursor.downField("value").as[String] 101 | message <- cursor.downField("message").as[String] 102 | } yield InvalidValue(key, value, message) 103 | 104 | case "UnhandledRowDecodingError" => 105 | cursor 106 | .downField("message") 107 | .as[String] 108 | .map(UnhandledRowDecodingError(_)) 109 | } 110 | } yield result 111 | } 112 | } 113 | 114 | implicit val analyticsSdkParsingErrorCirceEncoder: Encoder[ParsingError] = 115 | Encoder.instance { 116 | case NotTSV => 117 | Json.obj("type" := "NotTSV") 118 | case FieldNumberMismatch(fieldCount) => 119 | Json.obj( 120 | "type" := "FieldNumberMismatch", 121 | "fieldCount" := fieldCount 122 | ) 123 | case RowDecodingError(errors) => 124 | Json.obj( 125 | "type" := "RowDecodingError", 126 | "errors" := errors.asJson 127 | ) 128 | } 129 | 130 | implicit val analyticsSdkParsingErrorCirceDecoder: Decoder[ParsingError] = 131 | Decoder.instance { cursor => 132 | for { 133 | error <- cursor.downField("type").as[String] 134 | result <- error match { 135 | case "NotTSV" => 136 | NotTSV.asRight 137 | case "FieldNumberMismatch" => 138 | cursor 139 | .downField("fieldCount") 140 | .as[Int] 141 | .map(FieldNumberMismatch(_)) 142 | case "RowDecodingError" => 143 | cursor 144 | .downField("errors") 145 | .as[NonEmptyList[RowDecodingErrorInfo]] 146 | .map(RowDecodingError(_)) 147 | case _ => 148 | DecodingFailure(s"Error type $error is not an Analytics SDK Parsing Error.", cursor.history).asLeft 149 | } 150 | } yield result 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/SnowplowEvent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2020 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk 14 | 15 | // circe 16 | import io.circe.syntax._ 17 | import io.circe.{Decoder, DecodingFailure, Encoder, Json, JsonObject} 18 | import io.circe.CursorOp.DownField 19 | 20 | //cats 21 | import cats.implicits._ 22 | 23 | // iglu 24 | import com.snowplowanalytics.iglu.core.circe.CirceIgluCodecs._ 25 | import com.snowplowanalytics.iglu.core.{SchemaKey, SelfDescribingData} 26 | 27 | object SnowplowEvent { 28 | 29 | /** 30 | * A JSON representation of an atomic event's unstruct_event field. 31 | * 32 | * @param data the unstruct event as self-describing JSON, or None if the field is missing 33 | */ 34 | case class UnstructEvent(data: Option[SelfDescribingData[Json]]) extends AnyVal { 35 | def toShreddedJson: Option[(String, Json)] = 36 | data.map { 37 | case SelfDescribingData(s, d) => 38 | (transformSchema(Data.UnstructEvent, s.vendor, s.name, s.version.model), d) 39 | } 40 | } 41 | 42 | implicit final val unstructCirceEncoder: Encoder[UnstructEvent] = 43 | Encoder.instance { unstructEvent => 44 | if (unstructEvent.data.isEmpty) Json.Null 45 | else 46 | JsonObject( 47 | ("schema", Common.UnstructEventUri.toSchemaUri.asJson), 48 | ("data", unstructEvent.data.asJson) 49 | ).asJson 50 | } 51 | 52 | implicit val unstructEventDecoder: Decoder[UnstructEvent] = Decoder.forProduct1("data")(UnstructEvent.apply).recover { 53 | case DecodingFailure(_, DownField("data") :: _) => UnstructEvent(None) 54 | } 55 | 56 | /** 57 | * A JSON representation of an atomic event's contexts or derived_contexts fields. 58 | * 59 | * @param data the context as self-describing JSON, or None if the field is missing 60 | */ 61 | case class Contexts(data: List[SelfDescribingData[Json]]) extends AnyVal { 62 | def toShreddedJson: Map[String, Json] = 63 | data.groupBy(x => (x.schema.vendor, x.schema.name, x.schema.format, x.schema.version.model)).map { 64 | case ((vendor, name, _, model), contextsSdd) => 65 | val transformedName = transformSchema(Data.Contexts(Data.CustomContexts), vendor, name, model) 66 | val transformedData = contextsSdd.map(addSchemaVersionToData).asJson 67 | (transformedName, transformedData) 68 | } 69 | } 70 | 71 | private def addSchemaVersionToData(contextSdd: SelfDescribingData[Json]): Json = 72 | if (contextSdd.data.isObject) { 73 | val version = Json.obj("_schema_version" -> contextSdd.schema.version.asString.asJson) 74 | contextSdd.data.deepMerge(version) 75 | } else contextSdd.data 76 | 77 | implicit final val contextsCirceEncoder: Encoder[Contexts] = 78 | Encoder.instance { contexts => 79 | if (contexts.data.isEmpty) JsonObject.empty.asJson 80 | else 81 | JsonObject( 82 | ("schema", Common.ContextsUri.toSchemaUri.asJson), 83 | ("data", contexts.data.asJson) 84 | ).asJson 85 | } 86 | 87 | implicit val contextsDecoder: Decoder[Contexts] = Decoder.forProduct1("data")(Contexts.apply).recover { 88 | case DecodingFailure(_, DownField("data") :: _) => Contexts(List()) 89 | } 90 | 91 | /** 92 | * @param shredProperty Type of self-describing entity 93 | * @param vendor Iglu schema vendor 94 | * @param name Iglu schema name 95 | * @param model Iglu schema model 96 | * @return the schema, transformed into an Elasticsearch-compatible column name 97 | */ 98 | def transformSchema( 99 | shredProperty: Data.ShredProperty, 100 | vendor: String, 101 | name: String, 102 | model: Int 103 | ): String = { 104 | // Convert dots & dashes in schema vendor to underscore 105 | val snakeCaseVendor = vendor.replaceAll("""[\.\-]""", "_").toLowerCase 106 | 107 | // Convert PascalCase in schema name to snake_case 108 | val snakeCaseName = name.replaceAll("""[\.\-]""", "_").replaceAll("([^A-Z_])([A-Z])", "$1_$2").toLowerCase 109 | 110 | s"${shredProperty.prefix}${snakeCaseVendor}_${snakeCaseName}_$model" 111 | } 112 | 113 | def transformSchema(shredProperty: Data.ShredProperty, schema: SchemaKey): String = 114 | transformSchema(shredProperty, schema.vendor, schema.name, schema.version.model) 115 | } 116 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/RowDecoder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk 14 | package decode 15 | 16 | import java.nio.ByteBuffer 17 | 18 | private[scalasdk] trait RowDecoder[L] extends Serializable { self => 19 | def apply(row: List[String]): RowDecodeResult[L] 20 | def decodeBytes(row: List[ByteBuffer]): RowDecodeResult[L] 21 | def map[B](f: L => B): RowDecoder[B] = 22 | new RowDecoder[B] { 23 | def apply(row: List[String]): RowDecodeResult[B] = self.apply(row).map(f) 24 | def decodeBytes(row: List[ByteBuffer]): RowDecodeResult[B] = self.decodeBytes(row).map(f) 25 | } 26 | } 27 | 28 | object RowDecoder extends RowDecoderCompanion 29 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/TSVParser.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2023 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk.decode 14 | 15 | import java.nio.ByteBuffer 16 | 17 | /** Parser for a TSV-encoded string */ 18 | trait TSVParser[A] extends Serializable { 19 | def parseBytes(bytes: ByteBuffer): DecodeResult[A] 20 | def parse(row: String): DecodeResult[A] 21 | } 22 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueDecoder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk 14 | package decode 15 | 16 | // java 17 | import com.snowplowanalytics.snowplow.analytics.scalasdk.validate.FIELD_SIZES 18 | 19 | import java.time.Instant 20 | import java.time.format.DateTimeParseException 21 | import java.util.UUID 22 | import java.nio.ByteBuffer 23 | import java.nio.charset.StandardCharsets 24 | 25 | // cats 26 | import cats.syntax.either._ 27 | import cats.syntax.option._ 28 | import cats.syntax.show._ 29 | 30 | // iglu 31 | import com.snowplowanalytics.iglu.core.SelfDescribingData 32 | import com.snowplowanalytics.iglu.core.circe.implicits._ 33 | 34 | // circe 35 | import io.circe.jawn.JawnParser 36 | import io.circe.{Error, Json, ParsingFailure} 37 | 38 | // This library 39 | import com.snowplowanalytics.snowplow.analytics.scalasdk.Common.{ContextsCriterion, UnstructEventCriterion} 40 | import com.snowplowanalytics.snowplow.analytics.scalasdk.SnowplowEvent.{Contexts, UnstructEvent} 41 | import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo 42 | import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo._ 43 | 44 | private[decode] trait ValueDecoder[A] { 45 | def parse( 46 | key: Key, 47 | value: String, 48 | maxLength: Option[Int] 49 | ): DecodedValue[A] 50 | 51 | def parseBytes( 52 | key: Key, 53 | value: ByteBuffer, 54 | maxLength: Option[Int] 55 | ): DecodedValue[A] = 56 | parse(key, StandardCharsets.UTF_8.decode(value).toString, maxLength) 57 | } 58 | 59 | private[decode] object ValueDecoder { 60 | 61 | private val parser: JawnParser = new JawnParser 62 | 63 | def apply[A](implicit readA: ValueDecoder[A]): ValueDecoder[A] = readA 64 | 65 | def fromFunc[A](f: ((Key, String, Option[Int])) => DecodedValue[A]): ValueDecoder[A] = 66 | new ValueDecoder[A] { 67 | def parse( 68 | key: Key, 69 | value: String, 70 | maxLength: Option[Int] 71 | ): DecodedValue[A] = f((key, value, maxLength)) 72 | } 73 | 74 | implicit final val stringColumnDecoder: ValueDecoder[String] = 75 | fromFunc[String] { 76 | case (key, value, Some(maxLength)) if value.length > maxLength => 77 | value.substring(0, maxLength).asRight 78 | case (key, "", _) => 79 | InvalidValue(key, "", s"Field ${key.name} cannot be empty").asLeft 80 | case (_, value, _) => 81 | value.asRight 82 | } 83 | 84 | implicit final val stringOptionColumnDecoder: ValueDecoder[Option[String]] = 85 | fromFunc[Option[String]] { 86 | case (key, value, Some(maxLength)) if value.length > maxLength => 87 | value.substring(0, maxLength).some.asRight 88 | case (_, "", _) => 89 | none[String].asRight 90 | case (_, value, _) => 91 | value.some.asRight 92 | 93 | } 94 | 95 | implicit final val intColumnDecoder: ValueDecoder[Option[Int]] = 96 | fromFunc[Option[Int]] { 97 | case (key, value, _) => 98 | if (value.isEmpty) none[Int].asRight 99 | else 100 | try value.toInt.some.asRight 101 | catch { 102 | case _: NumberFormatException => 103 | InvalidValue(key, value, s"Cannot parse key ${key.name} into integer").asLeft 104 | } 105 | } 106 | 107 | implicit final val uuidColumnDecoder: ValueDecoder[UUID] = 108 | fromFunc[UUID] { 109 | case (key, value, _) => 110 | if (value.isEmpty) 111 | InvalidValue(key, value, s"Field ${key.name} cannot be empty").asLeft 112 | else 113 | try UUID.fromString(value).asRight[RowDecodingErrorInfo] 114 | catch { 115 | case _: IllegalArgumentException => 116 | InvalidValue(key, value, s"Cannot parse key ${key.name} into UUID").asLeft 117 | } 118 | } 119 | 120 | implicit final val boolColumnDecoder: ValueDecoder[Option[Boolean]] = 121 | fromFunc[Option[Boolean]] { 122 | case (key, value, _) => 123 | value match { 124 | case "0" => false.some.asRight 125 | case "1" => true.some.asRight 126 | case "" => none[Boolean].asRight 127 | case _ => InvalidValue(key, value, s"Cannot parse key ${key.name} into boolean").asLeft 128 | } 129 | } 130 | 131 | implicit final val doubleColumnDecoder: ValueDecoder[Option[Double]] = 132 | fromFunc[Option[Double]] { 133 | case (key, value, _) => 134 | if (value.isEmpty) 135 | none[Double].asRight 136 | else 137 | try value.toDouble.some.asRight 138 | catch { 139 | case _: NumberFormatException => 140 | InvalidValue(key, value, s"Cannot parse key ${key.name} into double").asLeft 141 | } 142 | } 143 | 144 | implicit final val instantColumnDecoder: ValueDecoder[Instant] = 145 | fromFunc[Instant] { 146 | case (key, value, _) => 147 | if (value.isEmpty) 148 | InvalidValue(key, value, s"Field ${key.name} cannot be empty").asLeft 149 | else { 150 | val tstamp = reformatTstamp(value) 151 | try Instant.parse(tstamp).asRight 152 | catch { 153 | case _: DateTimeParseException => 154 | InvalidValue(key, value, s"Cannot parse key ${key.name} into datetime").asLeft 155 | } 156 | } 157 | } 158 | 159 | implicit final val instantOptionColumnDecoder: ValueDecoder[Option[Instant]] = 160 | fromFunc[Option[Instant]] { 161 | case (key, value, _) => 162 | if (value.isEmpty) 163 | none[Instant].asRight[RowDecodingErrorInfo] 164 | else { 165 | val tstamp = reformatTstamp(value) 166 | try Instant.parse(tstamp).some.asRight 167 | catch { 168 | case _: DateTimeParseException => 169 | InvalidValue(key, value, s"Cannot parse key ${key.name} into datetime").asLeft 170 | } 171 | } 172 | } 173 | 174 | implicit final val unstructuredJson: ValueDecoder[UnstructEvent] = { 175 | def fromJsonParseResult( 176 | result: Either[ParsingFailure, Json], 177 | key: Key, 178 | originalValue: => String 179 | ): DecodedValue[UnstructEvent] = { 180 | def asLeft(error: Error): RowDecodingErrorInfo = InvalidValue(key, originalValue, error.show) 181 | result 182 | .flatMap(_.as[SelfDescribingData[Json]]) 183 | .leftMap(asLeft) match { 184 | case Right(SelfDescribingData(schema, data)) if UnstructEventCriterion.matches(schema) => 185 | data.as[SelfDescribingData[Json]].leftMap(asLeft).map(_.some).map(UnstructEvent.apply) 186 | case Right(SelfDescribingData(schema, _)) => 187 | InvalidValue(key, originalValue, s"Unknown payload: ${schema.toSchemaUri}").asLeft[UnstructEvent] 188 | case Left(error) => error.asLeft[UnstructEvent] 189 | } 190 | } 191 | new ValueDecoder[UnstructEvent] { 192 | def parse( 193 | key: Key, 194 | value: String, 195 | maxLength: Option[Int] 196 | ): DecodedValue[UnstructEvent] = 197 | if (value.isEmpty) 198 | UnstructEvent(None).asRight[RowDecodingErrorInfo] 199 | else 200 | fromJsonParseResult(parser.parse(value), key, value) 201 | 202 | override def parseBytes( 203 | key: Key, 204 | value: ByteBuffer, 205 | maxLength: Option[Int] 206 | ): DecodedValue[UnstructEvent] = 207 | if (!value.hasRemaining()) 208 | UnstructEvent(None).asRight[RowDecodingErrorInfo] 209 | else 210 | fromJsonParseResult(parser.parseByteBuffer(value), key, StandardCharsets.UTF_8.decode(value).toString) 211 | } 212 | } 213 | 214 | implicit final val contexts: ValueDecoder[Contexts] = { 215 | def fromJsonParseResult( 216 | result: Either[ParsingFailure, Json], 217 | key: Key, 218 | originalValue: => String 219 | ): DecodedValue[Contexts] = { 220 | def asLeft(error: Error): RowDecodingErrorInfo = InvalidValue(key, originalValue, error.show) 221 | result 222 | .flatMap(_.as[SelfDescribingData[Json]]) 223 | .leftMap(asLeft) match { 224 | case Right(SelfDescribingData(schema, data)) if ContextsCriterion.matches(schema) => 225 | data.as[List[SelfDescribingData[Json]]].leftMap(asLeft).map(Contexts.apply) 226 | case Right(SelfDescribingData(schema, _)) => 227 | InvalidValue(key, originalValue, s"Unknown payload: ${schema.toSchemaUri}").asLeft[Contexts] 228 | case Left(error) => error.asLeft[Contexts] 229 | } 230 | } 231 | new ValueDecoder[Contexts] { 232 | def parse( 233 | key: Key, 234 | value: String, 235 | maxLength: Option[Int] 236 | ): DecodedValue[Contexts] = 237 | if (value.isEmpty) 238 | Contexts(List.empty).asRight[RowDecodingErrorInfo] 239 | else 240 | fromJsonParseResult(parser.parse(value), key, value) 241 | 242 | override def parseBytes( 243 | key: Key, 244 | value: ByteBuffer, 245 | maxLength: Option[Int] 246 | ): DecodedValue[Contexts] = 247 | if (!value.hasRemaining()) 248 | Contexts(List.empty).asRight[RowDecodingErrorInfo] 249 | else 250 | fromJsonParseResult(parser.parseByteBuffer(value), key, StandardCharsets.UTF_8.decode(value).toString) 251 | } 252 | } 253 | 254 | /** 255 | * Converts a timestamp to an ISO-8601 format usable by Instant.parse() 256 | * 257 | * @param tstamp Timestamp of the form YYYY-MM-DD hh:mm:ss 258 | * @return ISO-8601 timestamp 259 | */ 260 | private def reformatTstamp(tstamp: String): String = tstamp.replaceAll(" ", "T") + "Z" 261 | } 262 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016-2019 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk 14 | 15 | import cats.data.{Validated, ValidatedNel} 16 | import cats.syntax.either._ 17 | import com.snowplowanalytics.snowplow.analytics.scalasdk.ParsingError.RowDecodingErrorInfo 18 | import io.circe.{Decoder, Encoder} 19 | import io.circe.syntax._ 20 | 21 | package object decode { 22 | 23 | /** Expected name of the field */ 24 | type Key = Symbol 25 | 26 | object Key { 27 | implicit val analyticsSdkKeyCirceEncoder: Encoder[Key] = 28 | Encoder.instance(_.toString.stripPrefix("'").asJson) 29 | 30 | implicit val analyticsSdkKeyCirceDecoder: Decoder[Key] = 31 | Decoder.instance(_.as[String].map(Symbol(_))) 32 | } 33 | 34 | /** Result of single-value parsing */ 35 | type DecodedValue[A] = Either[RowDecodingErrorInfo, A] 36 | 37 | /** Result of row decode process */ 38 | type RowDecodeResult[A] = ValidatedNel[RowDecodingErrorInfo, A] 39 | 40 | /** Result of TSV line parsing, which is either an event or parse error */ 41 | type DecodeResult[A] = Validated[ParsingError, A] 42 | } 43 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/encode/TsvEncoder.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2020-2020 Snowplow Analytics Ltd. All rights reserved. 3 | * 4 | * This program is licensed to you under the Apache License Version 2.0, 5 | * and you may not use this file except in compliance with the Apache License Version 2.0. 6 | * You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0. 7 | * 8 | * Unless required by applicable law or agreed to in writing, 9 | * software distributed under the Apache License Version 2.0 is distributed on an 10 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the Apache License Version 2.0 for the specific language governing permissions and limitations there under. 12 | */ 13 | package com.snowplowanalytics.snowplow.analytics.scalasdk.encode 14 | 15 | import java.time.format.DateTimeFormatter 16 | import java.time.Instant 17 | import java.util.UUID 18 | 19 | import io.circe.syntax._ 20 | 21 | import com.snowplowanalytics.snowplow.analytics.scalasdk.SnowplowEvent._ 22 | import com.snowplowanalytics.snowplow.analytics.scalasdk.Event 23 | 24 | object TsvEncoder { 25 | sealed trait FieldEncoder[T] { 26 | def encodeField(t: T): String 27 | } 28 | 29 | implicit object StringEncoder extends FieldEncoder[String] { 30 | def encodeField(str: String) = str 31 | } 32 | 33 | implicit object InstantEncoder extends FieldEncoder[Instant] { 34 | def encodeField(inst: Instant): String = 35 | DateTimeFormatter.ISO_INSTANT 36 | .format(inst) 37 | .replace("T", " ") 38 | .dropRight(1) // remove trailing 'Z' 39 | } 40 | 41 | implicit object UuidEncoder extends FieldEncoder[UUID] { 42 | def encodeField(uuid: UUID): String = uuid.toString 43 | } 44 | 45 | implicit object IntEncoder extends FieldEncoder[Int] { 46 | def encodeField(int: Int): String = int.toString 47 | } 48 | 49 | implicit object DoubleEncoder extends FieldEncoder[Double] { 50 | def encodeField(doub: Double): String = doub.toString 51 | } 52 | 53 | implicit object BooleanEncoder extends FieldEncoder[Boolean] { 54 | def encodeField(bool: Boolean): String = if (bool) "1" else "0" 55 | } 56 | 57 | implicit object ContextsEncoder extends FieldEncoder[Contexts] { 58 | def encodeField(ctxts: Contexts): String = 59 | if (ctxts.data.isEmpty) 60 | "" 61 | else 62 | ctxts.asJson.noSpaces 63 | } 64 | 65 | implicit object UnstructEncoder extends FieldEncoder[UnstructEvent] { 66 | def encodeField(unstruct: UnstructEvent): String = 67 | if (unstruct.data.isDefined) 68 | unstruct.asJson.noSpaces 69 | else 70 | "" 71 | } 72 | 73 | def encode[A](a: A)(implicit ev: FieldEncoder[A]): String = 74 | ev.encodeField(a) 75 | 76 | def encode[A](optA: Option[A])(implicit ev: FieldEncoder[A]): String = 77 | optA.map(a => ev.encodeField(a)).getOrElse("") 78 | 79 | def encode(event: Event): String = 80 | encode(event.app_id) + "\t" + 81 | encode(event.platform) + "\t" + 82 | encode(event.etl_tstamp) + "\t" + 83 | encode(event.collector_tstamp) + "\t" + 84 | encode(event.dvce_created_tstamp) + "\t" + 85 | encode(event.event) + "\t" + 86 | encode(event.event_id) + "\t" + 87 | encode(event.txn_id) + "\t" + 88 | encode(event.name_tracker) + "\t" + 89 | encode(event.v_tracker) + "\t" + 90 | encode(event.v_collector) + "\t" + 91 | encode(event.v_etl) + "\t" + 92 | encode(event.user_id) + "\t" + 93 | encode(event.user_ipaddress) + "\t" + 94 | encode(event.user_fingerprint) + "\t" + 95 | encode(event.domain_userid) + "\t" + 96 | encode(event.domain_sessionidx) + "\t" + 97 | encode(event.network_userid) + "\t" + 98 | encode(event.geo_country) + "\t" + 99 | encode(event.geo_region) + "\t" + 100 | encode(event.geo_city) + "\t" + 101 | encode(event.geo_zipcode) + "\t" + 102 | encode(event.geo_latitude) + "\t" + 103 | encode(event.geo_longitude) + "\t" + 104 | encode(event.geo_region_name) + "\t" + 105 | encode(event.ip_isp) + "\t" + 106 | encode(event.ip_organization) + "\t" + 107 | encode(event.ip_domain) + "\t" + 108 | encode(event.ip_netspeed) + "\t" + 109 | encode(event.page_url) + "\t" + 110 | encode(event.page_title) + "\t" + 111 | encode(event.page_referrer) + "\t" + 112 | encode(event.page_urlscheme) + "\t" + 113 | encode(event.page_urlhost) + "\t" + 114 | encode(event.page_urlport) + "\t" + 115 | encode(event.page_urlpath) + "\t" + 116 | encode(event.page_urlquery) + "\t" + 117 | encode(event.page_urlfragment) + "\t" + 118 | encode(event.refr_urlscheme) + "\t" + 119 | encode(event.refr_urlhost) + "\t" + 120 | encode(event.refr_urlport) + "\t" + 121 | encode(event.refr_urlpath) + "\t" + 122 | encode(event.refr_urlquery) + "\t" + 123 | encode(event.refr_urlfragment) + "\t" + 124 | encode(event.refr_medium) + "\t" + 125 | encode(event.refr_source) + "\t" + 126 | encode(event.refr_term) + "\t" + 127 | encode(event.mkt_medium) + "\t" + 128 | encode(event.mkt_source) + "\t" + 129 | encode(event.mkt_term) + "\t" + 130 | encode(event.mkt_content) + "\t" + 131 | encode(event.mkt_campaign) + "\t" + 132 | encode(event.contexts) + "\t" + 133 | encode(event.se_category) + "\t" + 134 | encode(event.se_action) + "\t" + 135 | encode(event.se_label) + "\t" + 136 | encode(event.se_property) + "\t" + 137 | encode(event.se_value) + "\t" + 138 | encode(event.unstruct_event) + "\t" + 139 | encode(event.tr_orderid) + "\t" + 140 | encode(event.tr_affiliation) + "\t" + 141 | encode(event.tr_total) + "\t" + 142 | encode(event.tr_tax) + "\t" + 143 | encode(event.tr_shipping) + "\t" + 144 | encode(event.tr_city) + "\t" + 145 | encode(event.tr_state) + "\t" + 146 | encode(event.tr_country) + "\t" + 147 | encode(event.ti_orderid) + "\t" + 148 | encode(event.ti_sku) + "\t" + 149 | encode(event.ti_name) + "\t" + 150 | encode(event.ti_category) + "\t" + 151 | encode(event.ti_price) + "\t" + 152 | encode(event.ti_quantity) + "\t" + 153 | encode(event.pp_xoffset_min) + "\t" + 154 | encode(event.pp_xoffset_max) + "\t" + 155 | encode(event.pp_yoffset_min) + "\t" + 156 | encode(event.pp_yoffset_max) + "\t" + 157 | encode(event.useragent) + "\t" + 158 | encode(event.br_name) + "\t" + 159 | encode(event.br_family) + "\t" + 160 | encode(event.br_version) + "\t" + 161 | encode(event.br_type) + "\t" + 162 | encode(event.br_renderengine) + "\t" + 163 | encode(event.br_lang) + "\t" + 164 | encode(event.br_features_pdf) + "\t" + 165 | encode(event.br_features_flash) + "\t" + 166 | encode(event.br_features_java) + "\t" + 167 | encode(event.br_features_director) + "\t" + 168 | encode(event.br_features_quicktime) + "\t" + 169 | encode(event.br_features_realplayer) + "\t" + 170 | encode(event.br_features_windowsmedia) + "\t" + 171 | encode(event.br_features_gears) + "\t" + 172 | encode(event.br_features_silverlight) + "\t" + 173 | encode(event.br_cookies) + "\t" + 174 | encode(event.br_colordepth) + "\t" + 175 | encode(event.br_viewwidth) + "\t" + 176 | encode(event.br_viewheight) + "\t" + 177 | encode(event.os_name) + "\t" + 178 | encode(event.os_family) + "\t" + 179 | encode(event.os_manufacturer) + "\t" + 180 | encode(event.os_timezone) + "\t" + 181 | encode(event.dvce_type) + "\t" + 182 | encode(event.dvce_ismobile) + "\t" + 183 | encode(event.dvce_screenwidth) + "\t" + 184 | encode(event.dvce_screenheight) + "\t" + 185 | encode(event.doc_charset) + "\t" + 186 | encode(event.doc_width) + "\t" + 187 | encode(event.doc_height) + "\t" + 188 | encode(event.tr_currency) + "\t" + 189 | encode(event.tr_total_base) + "\t" + 190 | encode(event.tr_tax_base) + "\t" + 191 | encode(event.tr_shipping_base) + "\t" + 192 | encode(event.ti_currency) + "\t" + 193 | encode(event.ti_price_base) + "\t" + 194 | encode(event.base_currency) + "\t" + 195 | encode(event.geo_timezone) + "\t" + 196 | encode(event.mkt_clickid) + "\t" + 197 | encode(event.mkt_network) + "\t" + 198 | encode(event.etl_tags) + "\t" + 199 | encode(event.dvce_sent_tstamp) + "\t" + 200 | encode(event.refr_domain_userid) + "\t" + 201 | encode(event.refr_dvce_tstamp) + "\t" + 202 | encode(event.derived_contexts) + "\t" + 203 | encode(event.domain_sessionid) + "\t" + 204 | encode(event.derived_tstamp) + "\t" + 205 | encode(event.event_vendor) + "\t" + 206 | encode(event.event_name) + "\t" + 207 | encode(event.event_format) + "\t" + 208 | encode(event.event_version) + "\t" + 209 | encode(event.event_fingerprint) + "\t" + 210 | encode(event.true_tstamp) 211 | } 212 | -------------------------------------------------------------------------------- /src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/validate/package.scala: -------------------------------------------------------------------------------- 1 | package com.snowplowanalytics.snowplow.analytics.scalasdk 2 | 3 | package object validate { 4 | 5 | val FIELD_SIZES: Map[String, Int] = Map( 6 | "app_id" -> 255, 7 | "platform" -> 255, 8 | "event" -> 128, 9 | "event_id" -> 36, 10 | "name_tracker" -> 128, 11 | "v_tracker" -> 100, 12 | "v_collector" -> 100, 13 | "v_etl" -> 100, 14 | "user_id" -> 255, 15 | "user_ipaddress" -> 128, 16 | "user_fingerprint" -> 128, 17 | "domain_userid" -> 128, 18 | "network_userid" -> 128, 19 | "geo_country" -> 2, 20 | "geo_region" -> 3, 21 | "geo_city" -> 75, 22 | "geo_zipcode" -> 15, 23 | "geo_region_name" -> 100, 24 | "ip_isp" -> 100, 25 | "ip_organization" -> 128, 26 | "ip_domain" -> 128, 27 | "ip_netspeed" -> 100, 28 | "page_url" -> 4096, 29 | "page_title" -> 2000, 30 | "page_referrer" -> 4096, 31 | "page_urlscheme" -> 16, 32 | "page_urlhost" -> 255, 33 | "page_urlpath" -> 3000, 34 | "page_urlquery" -> 6000, 35 | "page_urlfragment" -> 3000, 36 | "refr_urlscheme" -> 16, 37 | "refr_urlhost" -> 255, 38 | "refr_urlpath" -> 6000, 39 | "refr_urlquery" -> 6000, 40 | "refr_urlfragment" -> 3000, 41 | "refr_medium" -> 25, 42 | "refr_source" -> 50, 43 | "refr_term" -> 255, 44 | "mkt_medium" -> 255, 45 | "mkt_source" -> 255, 46 | "mkt_term" -> 255, 47 | "mkt_content" -> 500, 48 | "mkt_campaign" -> 255, 49 | "se_category" -> 1000, 50 | "se_action" -> 1000, 51 | "se_label" -> 4096, 52 | "se_property" -> 1000, 53 | "tr_orderid" -> 255, 54 | "tr_affiliation" -> 255, 55 | "tr_city" -> 255, 56 | "tr_state" -> 255, 57 | "tr_country" -> 255, 58 | "ti_orderid" -> 255, 59 | "ti_sku" -> 255, 60 | "ti_name" -> 255, 61 | "ti_category" -> 255, 62 | "useragent" -> 1000, 63 | "br_name" -> 50, 64 | "br_family" -> 50, 65 | "br_version" -> 50, 66 | "br_type" -> 50, 67 | "br_renderengine" -> 50, 68 | "br_lang" -> 255, 69 | "br_colordepth" -> 12, 70 | "os_name" -> 50, 71 | "os_family" -> 50, 72 | "os_manufacturer" -> 50, 73 | "os_timezone" -> 255, 74 | "dvce_type" -> 50, 75 | "doc_charset" -> 128, 76 | "tr_currency" -> 3, 77 | "ti_currency" -> 3, 78 | "base_currency" -> 3, 79 | "geo_timezone" -> 64, 80 | "mkt_clickid" -> 128, 81 | "mkt_network" -> 64, 82 | "etl_tags" -> 500, 83 | "refr_domain_userid" -> 128, 84 | "domain_sessionid" -> 128, 85 | "event_vendor" -> 1000, 86 | "event_name" -> 1000, 87 | "event_format" -> 128, 88 | "event_version" -> 128, 89 | "event_fingerprint" -> 128 90 | ) 91 | 92 | private def validateStr( 93 | k: String, 94 | value: String 95 | ): List[String] = 96 | if (value.length > FIELD_SIZES.getOrElse(k, Int.MaxValue)) 97 | List(s"Field $k longer than maximum allowed size ${FIELD_SIZES.getOrElse(k, Int.MaxValue)}") 98 | else 99 | List.empty[String] 100 | 101 | private def validateStr( 102 | k: String, 103 | v: Option[String] 104 | ): List[String] = 105 | v match { 106 | case Some(value) => validateStr(k, value) 107 | case None => List.empty[String] 108 | } 109 | 110 | def validator(e: Event): List[String] = 111 | validateStr("app_id", e.app_id) ++ 112 | validateStr("platform", e.platform) ++ 113 | validateStr("event", e.event) ++ 114 | validateStr("name_tracker", e.name_tracker) ++ 115 | validateStr("v_tracker", e.v_tracker) ++ 116 | validateStr("v_collector", e.v_collector) ++ 117 | validateStr("v_etl", e.v_etl) ++ 118 | validateStr("user_id", e.user_id) ++ 119 | validateStr("user_ipaddress", e.user_ipaddress) ++ 120 | validateStr("user_fingerprint", e.user_fingerprint) ++ 121 | validateStr("domain_userid", e.domain_userid) ++ 122 | validateStr("network_userid", e.network_userid) ++ 123 | validateStr("geo_country", e.geo_country) ++ 124 | validateStr("geo_region", e.geo_region) ++ 125 | validateStr("geo_city", e.geo_city) ++ 126 | validateStr("geo_zipcode", e.geo_zipcode) ++ 127 | validateStr("geo_region_name", e.geo_region_name) ++ 128 | validateStr("ip_isp", e.ip_isp) ++ 129 | validateStr("ip_organization", e.ip_organization) ++ 130 | validateStr("ip_domain", e.ip_domain) ++ 131 | validateStr("ip_netspeed", e.ip_netspeed) ++ 132 | validateStr("page_url", e.page_url) ++ 133 | validateStr("page_title", e.page_title) ++ 134 | validateStr("page_referrer", e.page_referrer) ++ 135 | validateStr("page_urlscheme", e.page_urlscheme) ++ 136 | validateStr("page_urlhost", e.page_urlhost) ++ 137 | validateStr("page_urlpath", e.page_urlpath) ++ 138 | validateStr("page_urlquery", e.page_urlquery) ++ 139 | validateStr("page_urlfragment", e.page_urlfragment) ++ 140 | validateStr("refr_urlscheme", e.refr_urlscheme) ++ 141 | validateStr("refr_urlhost", e.refr_urlhost) ++ 142 | validateStr("refr_urlpath", e.refr_urlpath) ++ 143 | validateStr("refr_urlquery", e.refr_urlquery) ++ 144 | validateStr("refr_urlfragment", e.refr_urlfragment) ++ 145 | validateStr("refr_medium", e.refr_medium) ++ 146 | validateStr("refr_source", e.refr_source) ++ 147 | validateStr("refr_term", e.refr_term) ++ 148 | validateStr("mkt_medium", e.mkt_medium) ++ 149 | validateStr("mkt_source", e.mkt_source) ++ 150 | validateStr("mkt_term", e.mkt_term) ++ 151 | validateStr("mkt_content", e.mkt_content) ++ 152 | validateStr("mkt_campaign", e.mkt_campaign) ++ 153 | validateStr("se_category", e.se_category) ++ 154 | validateStr("se_action", e.se_action) ++ 155 | validateStr("se_label", e.se_label) ++ 156 | validateStr("se_property", e.se_property) ++ 157 | validateStr("tr_orderid", e.tr_orderid) ++ 158 | validateStr("tr_affiliation", e.tr_affiliation) ++ 159 | validateStr("tr_city", e.tr_city) ++ 160 | validateStr("tr_state", e.tr_state) ++ 161 | validateStr("tr_country", e.tr_country) ++ 162 | validateStr("ti_orderid", e.ti_orderid) ++ 163 | validateStr("ti_sku", e.ti_sku) ++ 164 | validateStr("ti_name", e.ti_name) ++ 165 | validateStr("ti_category", e.ti_category) ++ 166 | validateStr("useragent", e.useragent) ++ 167 | validateStr("br_name", e.br_name) ++ 168 | validateStr("br_family", e.br_family) ++ 169 | validateStr("br_version", e.br_version) ++ 170 | validateStr("br_type", e.br_type) ++ 171 | validateStr("br_renderengine", e.br_renderengine) ++ 172 | validateStr("br_lang", e.br_lang) ++ 173 | validateStr("br_colordepth", e.br_colordepth) ++ 174 | validateStr("os_name", e.os_name) ++ 175 | validateStr("os_family", e.os_family) ++ 176 | validateStr("os_manufacturer", e.os_manufacturer) ++ 177 | validateStr("os_timezone", e.os_timezone) ++ 178 | validateStr("dvce_type", e.dvce_type) ++ 179 | validateStr("doc_charset", e.doc_charset) ++ 180 | validateStr("tr_currency", e.tr_currency) ++ 181 | validateStr("ti_currency", e.ti_currency) ++ 182 | validateStr("base_currency", e.base_currency) ++ 183 | validateStr("geo_timezone", e.geo_timezone) ++ 184 | validateStr("mkt_clickid", e.mkt_clickid) ++ 185 | validateStr("mkt_network", e.mkt_network) ++ 186 | validateStr("etl_tags", e.etl_tags) ++ 187 | validateStr("refr_domain_userid", e.refr_domain_userid) ++ 188 | validateStr("domain_sessionid", e.domain_sessionid) ++ 189 | validateStr("event_vendor", e.event_vendor) ++ 190 | validateStr("event_name", e.event_name) ++ 191 | validateStr("event_format", e.event_format) ++ 192 | validateStr("event_version", e.event_version) ++ 193 | validateStr("event_fingerprint", e.event_fingerprint) 194 | 195 | } 196 | -------------------------------------------------------------------------------- /src/site-preprocess/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |
4 | 5 |