├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── build_and_test.yaml │ ├── publish_github_draft.yaml │ └── publish_maven.yaml ├── .gitignore ├── .idea ├── .gitignore └── .name ├── LICENSE ├── README.md ├── build.gradle ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle ├── src ├── main │ ├── java │ │ └── com │ │ │ └── vaticle │ │ │ └── typedb │ │ │ └── osi │ │ │ └── loader │ │ │ ├── cli │ │ │ ├── LoadOptions.java │ │ │ └── TypeDBLoaderCLI.java │ │ │ ├── config │ │ │ ├── Configuration.java │ │ │ ├── ConfigurationHandler.java │ │ │ └── ConfigurationValidation.java │ │ │ ├── generator │ │ │ ├── AppendAttributeGenerator.java │ │ │ ├── AppendAttributeOrInsertThingGenerator.java │ │ │ ├── AttributeGenerator.java │ │ │ ├── EntityGenerator.java │ │ │ ├── Generator.java │ │ │ └── RelationGenerator.java │ │ │ ├── io │ │ │ ├── FileLogger.java │ │ │ └── FileToInputStream.java │ │ │ ├── loader │ │ │ ├── AsyncLoaderWorker.java │ │ │ └── TypeDBLoader.java │ │ │ ├── preprocessor │ │ │ └── RegexPreprocessor.java │ │ │ ├── status │ │ │ └── MigrationStatus.java │ │ │ ├── type │ │ │ └── AttributeValueType.java │ │ │ └── util │ │ │ ├── GeneratorUtil.java │ │ │ ├── TypeDBUtil.java │ │ │ └── Util.java │ └── resources │ │ ├── log4j2.xml │ │ └── logback.xml └── test │ ├── java │ └── com │ │ └── vaticle │ │ └── typedb │ │ └── osi │ │ └── loader │ │ ├── cli │ │ └── TypeDBLoaderCLITest.java │ │ ├── config │ │ ├── ConfigurationTest.java │ │ └── ConfigurationValidationTest.java │ │ ├── generator │ │ ├── AppendAttributeGeneratorTest.java │ │ ├── AppendAttributeOrInsertThingGeneratorTest.java │ │ ├── AttributeGeneratorTest.java │ │ ├── EntityGeneratorTest.java │ │ └── RelationGeneratorTest.java │ │ ├── io │ │ └── ErrorLoggerTest.java │ │ ├── loader │ │ └── TypeDBLoaderTest.java │ │ ├── preprocessor │ │ └── RegexPreprocessorTest.java │ │ ├── status │ │ └── MigrationStatusTest.java │ │ └── util │ │ └── QueryUtilTest.java │ └── resources │ ├── bugfixing │ └── issue10 │ │ ├── config.json │ │ ├── label.csv │ │ ├── schema.gql │ │ ├── tag.csv │ │ └── text.csv │ ├── generic │ ├── config.json │ ├── configNoSchema.json │ ├── configSchemaNotFound.json │ ├── configValidationTest.json │ ├── empty.csv │ ├── entity1.tsv │ ├── entity2.tsv │ ├── entity3.tsv │ ├── rel1.tsv │ └── schema.gql │ └── phoneCalls │ ├── append-call-rating.csv │ ├── append-fb-preprocessed.csv │ ├── append-twitter-nickname.csv │ ├── call.csv │ ├── communication-channel-pm.csv │ ├── communication-channel.csv │ ├── company.csv │ ├── config.json │ ├── configValidationTest.json │ ├── contract.csv.gz │ ├── in-use.csv │ ├── is-in-use.csv │ ├── person-append-or-insert.csv │ ├── person.csv │ ├── person1.csv │ ├── person2.csv │ ├── person3.csv │ ├── person_nextfile.csv │ ├── schema-updated.gql │ └── schema.gql └── typedbloader.png /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | **For the title of this PR:** A succinct and user-oriented title. It should not contain any code and be use the present tense. 2 | 3 | ## What is the goal of this PR? 4 | 5 | { Describe the goal and new behaviour of this PR from the user's point of view, and reference related GitHub issues. } 6 | 7 | ## What are the changes implemented in this PR? 8 | 9 | { Explain what you implemented, why your changes are the best way to achieve the goal(s) above. 10 | 11 | This would allow the reviewer to understand your intentions in the code much better. Please reference the GitHub issues to be automatically closed, such like 'closes #number'. } 12 | -------------------------------------------------------------------------------- /.github/workflows/build_and_test.yaml: -------------------------------------------------------------------------------- 1 | name: Build and test 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | build: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Set up JDK 11 16 | uses: actions/setup-java@v1 17 | with: 18 | java-version: '11' 19 | - name: Build libs and distributions with Gradle 20 | run: ./gradlew build -x test 21 | - name: Test with Gradle 22 | run: ./gradlew test --tests com.vaticle.typedb.osi.loader.config.ConfigurationTest --tests com.vaticle.typedb.osi.loader.io.* --tests com.vaticle.typedb.osi.loader.preprocessor.* 23 | -------------------------------------------------------------------------------- /.github/workflows/publish_github_draft.yaml: -------------------------------------------------------------------------------- 1 | name: Github release draft 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | create-draft: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | - name: Set up JDK 11 14 | uses: actions/setup-java@v1 15 | with: 16 | java-version: '11' 17 | - name: Build libs and distributions with Gradle 18 | run: ./gradlew build -x test 19 | 20 | - name: Create release notes draft 21 | uses: actions/create-release@v1 22 | id: create_release 23 | with: 24 | release_name: TypeDB Loader ${{ github.ref_name }} 25 | draft: true 26 | prerelease: false 27 | tag_name: ${{ github.ref_name }} 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | 31 | - name: Upload binary zip 32 | uses: actions/upload-release-asset@v1 33 | with: 34 | upload_url: ${{ steps.create_release.outputs.upload_url }} 35 | asset_path: build/distributions/typedb-loader-${{ github.ref_name }}.zip 36 | asset_name: typedb-loader-${{ github.ref_name }}.zip 37 | asset_content_type: application/zip 38 | env: 39 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 40 | 41 | - name: Upload binary tar 42 | uses: actions/upload-release-asset@v1 43 | with: 44 | upload_url: ${{ steps.create_release.outputs.upload_url }} 45 | asset_path: build/distributions/typedb-loader-${{ github.ref_name }}.tar 46 | asset_name: typedb-loader-${{ github.ref_name }}.tar 47 | asset_content_type: application/gzip 48 | env: 49 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 50 | -------------------------------------------------------------------------------- /.github/workflows/publish_maven.yaml: -------------------------------------------------------------------------------- 1 | name: Maven release 2 | on: 3 | release: 4 | types: [published] 5 | jobs: 6 | publish-maven: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v2 10 | - uses: actions/setup-java@v1 11 | with: 12 | java-version: '11' 13 | - name: Publish package 14 | run: gradle publish 15 | env: 16 | REPO_TYPEDB_OSI_USERNAME: ${{ secrets.REPO_TYPEDB_OSI_USERNAME }} 17 | REPO_TYPEDB_OSI_PASSWORD: ${{ secrets.REPO_TYPEDB_OSI_PASSWORD }} 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.toptal.com/developers/gitignore/api/java,gradle,intellij 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=java,gradle,intellij 4 | 5 | ### Intellij ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider 7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 8 | 9 | # User-specific stuff 10 | .idea/**/workspace.xml 11 | .idea/**/tasks.xml 12 | .idea/**/usage.statistics.xml 13 | .idea/**/dictionaries 14 | .idea/**/shelf 15 | 16 | # Generated files 17 | .idea/**/contentModel.xml 18 | 19 | # Sensitive or high-churn files 20 | .idea/**/dataSources/ 21 | .idea/**/dataSources.ids 22 | .idea/**/dataSources.local.xml 23 | .idea/**/sqlDataSources.xml 24 | .idea/**/dynamic.xml 25 | .idea/**/uiDesigner.xml 26 | .idea/**/dbnavigator.xml 27 | 28 | # Gradle 29 | .idea/**/gradle.xml 30 | .idea/**/libraries 31 | 32 | # Gradle and Maven with auto-import 33 | # When using Gradle or Maven with auto-import, you should exclude module files, 34 | # since they will be recreated, and may cause churn. Uncomment if using 35 | # auto-import. 36 | .idea/artifacts 37 | .idea/compiler.xml 38 | .idea/jarRepositories.xml 39 | .idea/modules.xml 40 | .idea/*.iml 41 | .idea/modules 42 | *.iml 43 | *.ipr 44 | 45 | # CMake 46 | cmake-build-*/ 47 | 48 | # Mongo Explorer plugin 49 | .idea/**/mongoSettings.xml 50 | 51 | # File-based project format 52 | *.iws 53 | 54 | # IntelliJ 55 | out/ 56 | 57 | # mpeltonen/sbt-idea plugin 58 | .idea_modules/ 59 | 60 | # JIRA plugin 61 | atlassian-ide-plugin.xml 62 | 63 | # Cursive Clojure plugin 64 | .idea/replstate.xml 65 | 66 | # Crashlytics plugin (for Android Studio and IntelliJ) 67 | com_crashlytics_export_strings.xml 68 | crashlytics.properties 69 | crashlytics-build.properties 70 | fabric.properties 71 | 72 | # Editor-based Rest Client 73 | .idea/httpRequests 74 | 75 | # Android studio 3.1+ serialized cache file 76 | .idea/caches/build_file_checksums.ser 77 | 78 | ### Intellij Patch ### 79 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 80 | 81 | modules.xml 82 | .idea/misc.xml 83 | 84 | # Sonarlint plugin 85 | # https://plugins.jetbrains.com/plugin/7973-sonarlint 86 | .idea/**/sonarlint/ 87 | 88 | # SonarQube Plugin 89 | # https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin 90 | .idea/**/sonarIssues.xml 91 | 92 | # Markdown Navigator plugin 93 | # https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced 94 | .idea/**/markdown-navigator.xml 95 | .idea/**/markdown-navigator-enh.xml 96 | .idea/**/markdown-navigator/ 97 | 98 | # Cache file creation bug 99 | # See https://youtrack.jetbrains.com/issue/JBR-2257 100 | .idea/$CACHE_FILE$ 101 | 102 | # CodeStream plugin 103 | # https://plugins.jetbrains.com/plugin/12206-codestream 104 | .idea/codestream.xml 105 | 106 | ### Java ### 107 | # Compiled class file 108 | *.class 109 | 110 | # Log file 111 | *.log 112 | 113 | # BlueJ files 114 | *.ctxt 115 | 116 | # Mobile Tools for Java (J2ME) 117 | .mtj.tmp/ 118 | 119 | # Package Files # 120 | *.jar 121 | *.war 122 | *.nar 123 | *.ear 124 | *.zip 125 | *.tar.gz 126 | *.rar 127 | 128 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 129 | hs_err_pid* 130 | 131 | ### Gradle ### 132 | .gradle 133 | build/ 134 | 135 | # Ignore Gradle GUI config 136 | gradle-app.setting 137 | 138 | # Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) 139 | !gradle-wrapper.jar 140 | 141 | # Cache of project 142 | .gradletasknamecache 143 | 144 | # # Work around https://youtrack.jetbrains.com/issue/IDEA-116898 145 | # gradle/wrapper/gradle-wrapper.properties 146 | 147 | ### Gradle Patch ### 148 | **/build/ 149 | 150 | grami-data-warn.log 151 | grami-log.log 152 | 153 | *.log 154 | 155 | .idea/aws.xml 156 | .idea/deployment.xml 157 | .idea/libraries-with-intellij-classes.xml 158 | .idea/sshConfigs.xml 159 | .idea/vcs.xml 160 | .idea/webServers.xml 161 | 162 | src/test/resources/_* 163 | src/test/java/_* -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Datasource local storage ignored files 5 | /dataSources/ 6 | /dataSources.local.xml 7 | # Editor-based HTTP Client requests 8 | /httpRequests/ 9 | -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | typedb-loader -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![TypeDBLoader_icon](https://github.com/bayer-science-for-a-better-life/grami/blob/master/typedbloader.png?raw=true) 2 | --- 3 | --- 4 | 5 | ### 6 | 7 | [![TypeDB Loader Test](https://github.com/bayer-science-for-a-better-life/grami/actions/workflows/testandbuild.yaml/badge.svg)](https://github.com/bayer-science-for-a-better-life/grami/actions/workflows/testandbuild.yaml) 8 | [![TypeDB Loader Build](https://github.com/bayer-science-for-a-better-life/grami/actions/workflows/release.yaml/badge.svg)](https://github.com/bayer-science-for-a-better-life/grami/actions/workflows/release.yaml) 9 | 10 | ### 11 | 12 | --- 13 | 14 | If your [TypeDB](https://github.com/vaticle/typedb) project 15 | 16 | - has a lot of data 17 | - and you want/need to focus on schema design, inference, and querying 18 | 19 | Use TypeDB Loader to take care of your data migration for you. TypeDB Loader streams data from files and migrates them 20 | into TypeDB **at scale**! 21 | 22 | ## Features: 23 | 24 | - Data Input: 25 | - data is streamed to reduce memory requirements 26 | - supports any tabular data file with your separator of choice (i.e.: csv, tsv, whatever-sv...) 27 | - supports gzipped files 28 | - ignores unnecessary columns 29 | - [Attribute](https://github.com/typedb-osi/typedb-loader/wiki/02-Loading-Attributes), [Entity](https://github.com/typedb-osi/typedb-loader/wiki/03-Loading-Entities), [Relation](https://github.com/typedb-osi/typedb-loader/wiki/04-Loading-Relations) 30 | Loading: 31 | - load required/optional attributes of any TypeDB type (string, boolean, long, double, datetime) 32 | - load required/optional role players (attribute / entity / relation) 33 | - load list-like attribute columns as n attributes (recommended procedure until attribute lists are fully supported 34 | by TypeDB) 35 | - load list-like player columns as n players for a relation 36 | - load entity if not present - if present, either do not write or append attributes 37 | - [Appending Attributes](https://github.com/typedb-osi/typedb-loader/wiki/05-Appending-Attributes) to existing things 38 | - [Append-Attribute-Or-Insert-Entity](https://github.com/typedb-osi/typedb-loader/wiki/06-Append-Or-Insert) for entities 39 | - Data Validation: 40 | - validate input data rows and log issues for easy diagnosis input data-related issues (i.e. missing 41 | attributes/players, invalid characters...) 42 | - Configuration Validation: 43 | - write your configuration with confidence: warnings will display useful information for fine tuning, errors will 44 | let you know what you forgot. All BEFORE the database is touched. 45 | - Performance: 46 | - parallelized asynchronous writes to TypeDB to make the most of your hardware configuration, optimized with 47 | engineers @vaticle 48 | - Stop/Restart (in re-implementation, currently NOT available): 49 | - tracking of your migration status to stop/restart, or restart after failure 50 | 51 | - [Basic Column Preprocessing using RegEx's](https://github.com/typedb-osi/typedb-loader/wiki/08-Preprocessing) 52 | 53 | Create a Loading 54 | Configuration ([example](https://github.com/typedb-osi/typedb-loader/blob/master/src/test/resources/phoneCalls/config.json)) 55 | and use TypeDB Loader 56 | 57 | - as an [executable CLI](https://github.com/typedb-osi/typedb-loader/wiki/10-TypeDB-Loader-as-Executable-CLI) - no 58 | coding 59 | - in [your own Java project](https://github.com/typedb-osi/typedb-loader/wiki/09-TypeDB-Loader-as-Dependency) - easy API 60 | 61 | ## How it works: 62 | 63 | To illustrate how to use TypeDB Loader, we will use a slightly extended version of the "phone-calls" 64 | example [dataset](https://github.com/typedb-osi/typedb-loader/tree/master/src/test/resources/phoneCalls) 65 | and [schema](https://github.com/typedb-osi/typedb-loader/blob/master/src/test/resources/phoneCalls/schema.gql) from the 66 | TypeDB developer documentation: 67 | 68 | ### Configuration 69 | 70 | The configuration file tells TypeDB Loader what things you want to insert for each of your data files and how to do it. 71 | 72 | Here are some example: 73 | 74 | - [Attribute Examples](https://github.com/typedb-osi/typedb-loader/wiki/02-Loading-Attributes) 75 | - [Entity Examples](https://github.com/typedb-osi/typedb-loader/wiki/03-Loading-Entities) 76 | - [Relation Examples](https://github.com/typedb-osi/typedb-loader/wiki/04-Loading-Relations) 77 | - [Nested Relation - Match by Attribute(s) Example](https://github.com/typedb-osi/typedb-loader/wiki/04-Loading-Relations#loading-relations-with-entityrelation-players-matched-on-attribute-ownerships-incl-nested-relations) 78 | - [Nested Relation - Match by Player(s) Example](https://github.com/typedb-osi/typedb-loader/wiki/04-Loading-Relations#loading-relations-relation-players-matching-on-players-in-playing-relation-incl-nested-relations) 79 | - [Attribute-Player Relation Example](https://github.com/typedb-osi/typedb-loader/wiki/04-Loading-Relations#loading-relations-with-attribute-players) 80 | - [Custom Migration Order Example](https://github.com/typedb-osi/typedb-loader/wiki/07-Custom-Load-Order) 81 | 82 | For detailed documentation, please refer to the [WIKI](https://github.com/bayer-science-for-a-better-life/grami/wiki). 83 | 84 | The [config](https://github.com/typedb-osi/typedb-loader/tree/master/src/test/resources/phoneCalls/config.json) in the 85 | phone-calls test is a good starting example of a configuration. 86 | 87 | ### Migrate Data 88 | 89 | Once your configuration files are complete, you can use TypeDB Loader in one of two ways: 90 | 91 | 1. As an executable command line interface - no coding required: 92 | 93 | ```Shell 94 | ./bin/typedbloader load \ 95 | -tdb localhost:1729 \ 96 | -c /path/to/your/config.json \ 97 | -db databaseName \ 98 | -cm 99 | ``` 100 | 101 | [See details here](https://github.com/typedb-osi/typedb-loader/wiki/10-TypeDB-Loader-as-Executable-CLI) 102 | 103 | 2. As a dependency in your own Java code: 104 | 105 | ```Java 106 | 107 | import com.vaticle.typedb.osi.loader.cli.LoadOptions; 108 | import com.vaticle.typedb.osi.loader.loader.TypeDBLoader; 109 | 110 | public class LoadingData { 111 | 112 | public void loadData() { 113 | String uri = "localhost:1729"; 114 | String config = "path/to/your/config.json"; 115 | String database = "databaseName"; 116 | 117 | String[] args = { 118 | "load", 119 | "-tdb", uri, 120 | "-c", config, 121 | "-db", database, 122 | "-cm" 123 | }; 124 | 125 | LoadOptions options = LoadOptions.parse(args); 126 | TypeDBLoader loader = new TypeDBLoader(options); 127 | loader.load(); 128 | } 129 | } 130 | ``` 131 | 132 | [See details here](https://github.com/typedb-osi/typedb-loader/wiki/09-TypeDB-Loader-as-Dependency) 133 | 134 | ## Step-by-Step Tutorial 135 | 136 | A complete tutorial for TypeDB version >= 2.5.0 is in work and will be published. 137 | 138 | An example of configuration and usage of TypeDB Loader on real data can be 139 | found [in the TypeDB Examples](https://github.com/vaticle/typedb-examples/tree/master/biology/catalogue_of_life). 140 | 141 | A complete tutorial for TypeDB (Grakn) version < 2.0 can be 142 | found [on Medium](https://medium.com/@hkuich/introducing-grami-a-data-migration-tool-for-grakn-d4051582f867). 143 | 144 | There is an [example repository](https://github.com/bayer-science-for-a-better-life/grami-example) for your convenience. 145 | 146 | ## Connecting to TypeDB Cluster 147 | 148 | To connect to TypeDB Cluster, a set of options is provided: 149 | 150 | ``` 151 | --typedb-cluster= 152 | --username= 153 | --password // can be asked for interactively 154 | --tls-enabled 155 | --tls-root-ca= 156 | ``` 157 | 158 | ## Compatibility Table 159 | 160 | Ranges are [inclusive, inclusive]. 161 | 162 | | TypeDB Loader | TypeDB Driver (internal) | TypeDB | TypeDB Cloud | 163 | |:-------------:|:------------------------:|:---------------:|:---------------:| 164 | | 1.9.x | 2.26.6 | 2.25.x - | 2.25.x - | 165 | | 1.8.0 | 2.25.6 | 2.25.x - | 2.25.x - | 166 | | 1.7.0 | 2.18.1 | 2.18.x 2.23.x | 2.18.x 2.23.x | 167 | | 1.6.0 | 2.14.2 | 2.14.x - 2.17.x | 2.14.x - 2.16.x | 168 | | 1.2.0 - 1.5.x | 2.8.0 - 2.14.0 | 2.8.0 - 2.14.0 | N/A | 169 | | 1.1.0 - 1.1.x | 2.8.0 | 2.8.x | N/A | 170 | | 1.0.0 | 2.5.0 - 2.7.1 | 2.5.x - 2.7.x | N/A | 171 | | 0.1.1 | 2.0.0 - 2.4.x | 2.0.x - 2.4.x | N/A | 172 | | <0.1 | 1.8.0 | 1.8.x | N/A | 173 | 174 | * [Type DB](https://github.com/vaticle/typedb) 175 | 176 | Find the Readme for GraMi for grakn < 177 | 2.0 [here](https://github.com/bayer-science-for-a-better-life/grami/blob/b3d6d272c409d6c40254354027b49f90b255e1c3/README.md) 178 | 179 | ### Package hosting 180 | Package repository hosting is graciously provided by [Cloudsmith](https://cloudsmith.com). 181 | Cloudsmith is the only fully hosted, cloud-native, universal package management solution, that 182 | enables your organization to create, store and share packages in any format, to any place, with total 183 | confidence. 184 | 185 | ## Contributions 186 | 187 | TypeDB Loader was built @[Bayer AG](https://www.bayer.com/) in the Semantic and Knowledge Graph Technology Group with 188 | the support of the engineers @[Vaticle](https://github.com/vaticle). 189 | 190 | ## Licensing 191 | 192 | This repository includes software developed at [Bayer AG](https://www.bayer.com/). It is released under 193 | the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0). 194 | 195 | ## Credits 196 | 197 | Icon in banner by [Freepik](https://www.freepik.com") from [Flaticon](https://www.flaticon.com/) 198 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'application' 4 | id 'maven-publish' 5 | } 6 | 7 | group 'com.vaticle.typedb-osi' 8 | version '1.9.1' 9 | 10 | repositories { 11 | mavenCentral() 12 | maven { 13 | url "https://repo.typedb.com/public/public-release/maven/" 14 | } 15 | } 16 | 17 | dependencies { 18 | implementation("com.vaticle.typedb:typedb-driver:2.26.6") 19 | implementation("com.vaticle.typeql:typeql-lang:2.26.6") 20 | implementation("com.google.code.gson:gson:2.8.6") 21 | implementation("org.slf4j:slf4j-api:1.7.25") 22 | implementation("org.apache.logging.log4j:log4j-api:2.17.1") 23 | implementation("org.apache.logging.log4j:log4j-core:2.17.1") 24 | implementation("info.picocli:picocli:4.6.1") 25 | implementation("org.apache.commons:commons-csv:1.8") 26 | implementation("commons-io:commons-io:2.8.0") 27 | compileOnly("info.picocli:picocli-codegen:4.5.1") 28 | testImplementation("junit:junit:4.12") 29 | } 30 | 31 | mainClassName = 'com.vaticle.typedb.osi.loader.cli.TypeDBLoaderCLI' 32 | 33 | publishing { 34 | publications { 35 | maven(MavenPublication) { 36 | artifactId = 'typedb-loader' 37 | from components.java 38 | } 39 | } 40 | repositories { 41 | maven { 42 | name = "TypeDBOSI" 43 | url = "https://maven.cloudsmith.io/typedb/osi/" 44 | credentials { 45 | username = System.getenv("REPO_TYPEDB_OSI_USERNAME") 46 | password = System.getenv("REPO_TYPEDB_OSI_PASSWORD") 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typedb-osi/typedb-loader/9a5fc166cbe8c26b4f2934a008b19991fccb62a1/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | # 4 | # Copyright 2015 the original author or authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | ## 21 | ## Gradle start up script for UN*X 22 | ## 23 | ############################################################################## 24 | 25 | # Attempt to set APP_HOME 26 | # Resolve links: $0 may be a link 27 | PRG="$0" 28 | # Need this for relative symlinks. 29 | while [ -h "$PRG" ] ; do 30 | ls=`ls -ld "$PRG"` 31 | link=`expr "$ls" : '.*-> \(.*\)$'` 32 | if expr "$link" : '/.*' > /dev/null; then 33 | PRG="$link" 34 | else 35 | PRG=`dirname "$PRG"`"/$link" 36 | fi 37 | done 38 | SAVED="`pwd`" 39 | cd "`dirname \"$PRG\"`/" >/dev/null 40 | APP_HOME="`pwd -P`" 41 | cd "$SAVED" >/dev/null 42 | 43 | APP_NAME="Gradle" 44 | APP_BASE_NAME=`basename "$0"` 45 | 46 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 47 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 48 | 49 | # Use the maximum available, or set MAX_FD != -1 to use that value. 50 | MAX_FD="maximum" 51 | 52 | warn () { 53 | echo "$*" 54 | } 55 | 56 | die () { 57 | echo 58 | echo "$*" 59 | echo 60 | exit 1 61 | } 62 | 63 | # OS specific support (must be 'true' or 'false'). 64 | cygwin=false 65 | msys=false 66 | darwin=false 67 | nonstop=false 68 | case "`uname`" in 69 | CYGWIN* ) 70 | cygwin=true 71 | ;; 72 | Darwin* ) 73 | darwin=true 74 | ;; 75 | MINGW* ) 76 | msys=true 77 | ;; 78 | NONSTOP* ) 79 | nonstop=true 80 | ;; 81 | esac 82 | 83 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 84 | 85 | # Determine the Java command to use to start the JVM. 86 | if [ -n "$JAVA_HOME" ] ; then 87 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 88 | # IBM's JDK on AIX uses strange locations for the executables 89 | JAVACMD="$JAVA_HOME/jre/sh/java" 90 | else 91 | JAVACMD="$JAVA_HOME/bin/java" 92 | fi 93 | if [ ! -x "$JAVACMD" ] ; then 94 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 95 | 96 | Please set the JAVA_HOME variable in your environment to match the 97 | location of your Java installation." 98 | fi 99 | else 100 | JAVACMD="java" 101 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 102 | 103 | Please set the JAVA_HOME variable in your environment to match the 104 | location of your Java installation." 105 | fi 106 | 107 | # Increase the maximum file descriptors if we can. 108 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 109 | MAX_FD_LIMIT=`ulimit -H -n` 110 | if [ $? -eq 0 ] ; then 111 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 112 | MAX_FD="$MAX_FD_LIMIT" 113 | fi 114 | ulimit -n $MAX_FD 115 | if [ $? -ne 0 ] ; then 116 | warn "Could not set maximum file descriptor limit: $MAX_FD" 117 | fi 118 | else 119 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 120 | fi 121 | fi 122 | 123 | # For Darwin, add options to specify how the application appears in the dock 124 | if $darwin; then 125 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 126 | fi 127 | 128 | # For Cygwin or MSYS, switch paths to Windows format before running java 129 | if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then 130 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 131 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 132 | JAVACMD=`cygpath --unix "$JAVACMD"` 133 | 134 | # We build the pattern for arguments to be converted via cygpath 135 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 136 | SEP="" 137 | for dir in $ROOTDIRSRAW ; do 138 | ROOTDIRS="$ROOTDIRS$SEP$dir" 139 | SEP="|" 140 | done 141 | OURCYGPATTERN="(^($ROOTDIRS))" 142 | # Add a user-defined pattern to the cygpath arguments 143 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 144 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 145 | fi 146 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 147 | i=0 148 | for arg in "$@" ; do 149 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 150 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 151 | 152 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 153 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 154 | else 155 | eval `echo args$i`="\"$arg\"" 156 | fi 157 | i=`expr $i + 1` 158 | done 159 | case $i in 160 | 0) set -- ;; 161 | 1) set -- "$args0" ;; 162 | 2) set -- "$args0" "$args1" ;; 163 | 3) set -- "$args0" "$args1" "$args2" ;; 164 | 4) set -- "$args0" "$args1" "$args2" "$args3" ;; 165 | 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 166 | 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 167 | 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 168 | 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 169 | 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 170 | esac 171 | fi 172 | 173 | # Escape application args 174 | save () { 175 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 176 | echo " " 177 | } 178 | APP_ARGS=`save "$@"` 179 | 180 | # Collect all arguments for the java command, following the shell quoting and substitution rules 181 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 182 | 183 | exec "$JAVACMD" "$@" 184 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto init 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto init 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :init 68 | @rem Get command-line arguments, handling Windows variants 69 | 70 | if not "%OS%" == "Windows_NT" goto win9xME_args 71 | 72 | :win9xME_args 73 | @rem Slurp the command line arguments. 74 | set CMD_LINE_ARGS= 75 | set _SKIP=2 76 | 77 | :win9xME_args_slurp 78 | if "x%~1" == "x" goto execute 79 | 80 | set CMD_LINE_ARGS=%* 81 | 82 | :execute 83 | @rem Setup the command line 84 | 85 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 86 | 87 | @rem Execute Gradle 88 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 89 | 90 | :end 91 | @rem End local scope for the variables with windows NT shell 92 | if "%ERRORLEVEL%"=="0" goto mainEnd 93 | 94 | :fail 95 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 96 | rem the _cmd.exe /c_ return code! 97 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 98 | exit /b 1 99 | 100 | :mainEnd 101 | if "%OS%"=="Windows_NT" endlocal 102 | 103 | :omega 104 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'typedb-loader' 2 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/cli/LoadOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.cli; 18 | 19 | import picocli.CommandLine; 20 | 21 | import javax.annotation.Nullable; 22 | 23 | @CommandLine.Command(name = "load", description = "load data and/or schema", mixinStandardHelpOptions = true) 24 | public class LoadOptions { 25 | @CommandLine.Spec 26 | CommandLine.Model.CommandSpec spec; 27 | 28 | @CommandLine.Option(names = {"-c", "--config"}, description = "config file in JSON format", required = true) 29 | public String dataConfigFilePath; 30 | 31 | @CommandLine.Option(names = {"-db", "--database"}, description = "target database in your TypeDB instance", required = true) 32 | public String databaseName; 33 | 34 | @CommandLine.Option(names = {"-tdb", "--typedb"}, description = "Connect to TypeDB Core server in format: server:port (default: localhost:1729)", defaultValue = "localhost:1729") 35 | public String typedbURI; 36 | 37 | @CommandLine.Option(names = {"-tdbc", "--typedb-cluster"}, description = "Connect to TypeDB Cluster instead of TypeDB Core. Specify a cluster server with 'server:port'.") 38 | public String typedbClusterURI; 39 | 40 | @CommandLine.Option(names = {"--username"}, description = "Username") 41 | public @Nullable String username; 42 | 43 | @CommandLine.Option(names = {"--password"}, description = "Password", interactive = true, arity = "0..1") 44 | public @Nullable String password; 45 | 46 | @CommandLine.Option(names = {"--tls-enabled"}, description = "Connect to TypeDB Cluster with TLS encryption") 47 | public boolean tlsEnabled; 48 | 49 | @CommandLine.Option( names = {"--tls-root-ca"}, description = "Path to the TLS root CA file") 50 | public @Nullable String tlsRootCAPath; 51 | 52 | @CommandLine.Option(names = {"-cm", "--cleanMigration"}, description = "optional - delete old schema and data and restart migration from scratch - default: continue previous migration, if exists", defaultValue = "false") 53 | public boolean cleanMigration; 54 | 55 | @CommandLine.Option(names = {"-ls", "--loadSchema"}, description = "optional - reload schema when continuing a migration (ignored when clean migration)", defaultValue = "false") 56 | public boolean loadSchema; 57 | 58 | @CommandLine.Option(names = {"-mi", "--allowMultiInsert"}, description = "Allow match-inserts to match multiple answers and insert for each.", defaultValue = "false") 59 | public boolean multiInsert; 60 | 61 | public static LoadOptions parse(String[] args) { 62 | CommandLine commandLine = new CommandLine(new TypeDBLoaderCLI()) 63 | .addSubcommand("load", new LoadOptions()); 64 | if (args.length == 0) { 65 | commandLine.usage(commandLine.getOut()); 66 | System.exit(0); 67 | } 68 | 69 | CommandLine.ParseResult arguments = commandLine.parseArgs(args); 70 | if (arguments.isUsageHelpRequested()) { 71 | commandLine.usage(commandLine.getOut()); 72 | System.exit(0); 73 | } else if (arguments.isVersionHelpRequested()) { 74 | commandLine.printVersionHelp(commandLine.getOut()); 75 | System.exit(0); 76 | } else if (arguments.subcommand().isUsageHelpRequested()) { 77 | commandLine.getSubcommands().get("load").usage(commandLine.getOut()); 78 | System.exit(0); 79 | } else if (!arguments.hasSubcommand()) { 80 | commandLine.getErr().println("Missing subcommand"); 81 | commandLine.usage(commandLine.getOut()); 82 | System.exit(0); 83 | } else { 84 | return arguments.subcommand().asCommandLineList().get(0).getCommand(); 85 | } 86 | throw new RuntimeException("Illegal state"); 87 | } 88 | 89 | public void print() { 90 | spec.commandLine().getOut().println("############## TypeDB Loader ###############"); 91 | spec.commandLine().getOut().println("TypeDB Loader started with parameters:"); 92 | spec.commandLine().getOut().println("\tconfiguration: " + dataConfigFilePath); 93 | spec.commandLine().getOut().println("\tdatabase name: " + databaseName); 94 | if (typedbClusterURI != null) { 95 | spec.commandLine().getOut().println("\tTypeDB cluster URI: " + typedbClusterURI); 96 | spec.commandLine().getOut().println("\tTypeDB cluster username: " + username); 97 | spec.commandLine().getOut().println("\tTypeDB cluster TLS enabled: " + tlsEnabled); 98 | spec.commandLine().getOut().println("\tTypeDB cluster TLS path: " + (tlsRootCAPath == null ? "N/A" : tlsRootCAPath)); 99 | } else { 100 | spec.commandLine().getOut().println("\tTypeDB server URI: " + typedbURI); 101 | } 102 | spec.commandLine().getOut().println("\tdelete database and all data in it for a clean new migration?: " + cleanMigration); 103 | spec.commandLine().getOut().println("\treload schema (if not doing clean migration): " + loadSchema); 104 | } 105 | } -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/cli/TypeDBLoaderCLI.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.cli; 18 | 19 | import com.vaticle.typedb.osi.loader.loader.TypeDBLoader; 20 | import picocli.CommandLine; 21 | 22 | @CommandLine.Command(description = "Welcome to the CLI of TypeDB Loader - your TypeDB data loading tool", name = "typedb-loader", version = "1.0.0-alpha", mixinStandardHelpOptions = true) 23 | public class TypeDBLoaderCLI { 24 | 25 | public static void main(String[] args) { 26 | LoadOptions options = LoadOptions.parse(args); 27 | options.print(); 28 | TypeDBLoader loader = new TypeDBLoader(options); 29 | loader.load(); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/config/ConfigurationHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.config; 18 | 19 | public enum ConfigurationHandler { 20 | ATTRIBUTES, 21 | ENTITIES, 22 | RELATIONS, 23 | APPEND_ATTRIBUTE, 24 | APPEND_ATTRIBUTE_OR_INSERT_THING; 25 | 26 | @Override 27 | public String toString() { 28 | return super.toString().toLowerCase(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/generator/AppendAttributeGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.generator; 18 | 19 | import com.vaticle.typedb.driver.api.TypeDBTransaction; 20 | import com.vaticle.typedb.driver.api.answer.ConceptMap; 21 | import com.vaticle.typedb.driver.common.exception.TypeDBDriverException; 22 | import com.vaticle.typedb.osi.loader.config.Configuration; 23 | import com.vaticle.typedb.osi.loader.io.FileLogger; 24 | import com.vaticle.typedb.osi.loader.util.GeneratorUtil; 25 | import com.vaticle.typedb.osi.loader.util.TypeDBUtil; 26 | import com.vaticle.typedb.osi.loader.util.Util; 27 | import com.vaticle.typeql.lang.TypeQL; 28 | import com.vaticle.typeql.lang.builder.ConceptVariableBuilder; 29 | import com.vaticle.typeql.lang.pattern.constraint.ThingConstraint; 30 | import com.vaticle.typeql.lang.pattern.statement.ThingStatement; 31 | import com.vaticle.typeql.lang.query.TypeQLInsert; 32 | import org.apache.commons.io.FilenameUtils; 33 | import org.apache.logging.log4j.LogManager; 34 | import org.apache.logging.log4j.Logger; 35 | 36 | import java.io.IOException; 37 | import java.util.ArrayList; 38 | import java.util.Iterator; 39 | 40 | import static com.vaticle.typedb.osi.loader.util.TypeDBUtil.safeInsert; 41 | 42 | public class AppendAttributeGenerator implements Generator { 43 | private static final Logger dataLogger = LogManager.getLogger("com.vaticle.typedb.osi.loader.error"); 44 | private final String filePath; 45 | private final String[] header; 46 | private final Configuration.Generator.AppendAttribute appendConfiguration; 47 | private final char fileSeparator; 48 | 49 | public AppendAttributeGenerator(String filePath, Configuration.Generator.AppendAttribute appendConfiguration, char fileSeparator) throws IOException { 50 | this.filePath = filePath; 51 | this.header = Util.getFileHeader(filePath, fileSeparator); 52 | this.appendConfiguration = appendConfiguration; 53 | this.fileSeparator = fileSeparator; 54 | } 55 | 56 | @Override 57 | public void write(TypeDBTransaction tx, String[] row, boolean allowMultiInsert) { 58 | String fileName = FilenameUtils.getName(filePath); 59 | String fileNoExtension = FilenameUtils.removeExtension(fileName); 60 | String originalRow = String.join(Character.toString(fileSeparator), row); 61 | 62 | if (row.length > header.length) { 63 | FileLogger.getLogger().logMalformed(fileName, originalRow); 64 | dataLogger.error("Malformed Row detected in <" + filePath + "> - written to <" + fileNoExtension + "_malformed.log" + ">"); 65 | } 66 | 67 | TypeQLInsert query = generateMatchInsertStatement(row); 68 | 69 | if (appendAttributeInsertStatementValid(query)) { 70 | try { 71 | Iterator answers = TypeDBUtil.executeMatch(tx, query); 72 | if (!answers.hasNext()) { 73 | FileLogger.getLogger().logNoMatches(fileName, originalRow); 74 | dataLogger.error("Match-insert failed - File <" + filePath + "> row <" + originalRow + "> generates query <" + query + "> which matched no answers."); 75 | } else { 76 | safeInsert(tx, query, answers, allowMultiInsert, filePath, originalRow, dataLogger); 77 | } 78 | } catch (TypeDBDriverException typeDBDriverException) { 79 | FileLogger.getLogger().logUnavailable(fileName, originalRow); 80 | dataLogger.error("TypeDB Unavailable - Row in <" + filePath + "> not inserted - written to <" + fileNoExtension + "_unavailable.log" + ">"); 81 | } 82 | } else { 83 | FileLogger.getLogger().logInvalid(fileName, originalRow); 84 | dataLogger.error("Invalid Row detected in <" + filePath + "> - written to <" + fileNoExtension + "_invalid.log" + "> - invalid Statement: <" + query.toString().replace("\n", " ") + ">"); 85 | } 86 | } 87 | 88 | public TypeQLInsert generateMatchInsertStatement(String[] row) { 89 | if (row.length > 0) { 90 | ThingStatement.Thing entityMatchStatement = TypeQL.cVar("thing") 91 | .isa(appendConfiguration.getMatch().getType()); 92 | for (Configuration.Definition.Attribute consAtt : appendConfiguration.getMatch().getOwnerships()) { 93 | ArrayList constraintValues = GeneratorUtil.generateValueConstraintsConstrainingAttribute( 94 | row, header, filePath, fileSeparator, consAtt); 95 | for (ThingConstraint.Predicate constraintValue : constraintValues) { 96 | entityMatchStatement.constrain(GeneratorUtil.valueToHasConstraint(consAtt.getAttribute(), constraintValue)); 97 | } 98 | } 99 | 100 | ConceptVariableBuilder insertUnboundVar = TypeQL.cVar("thing"); 101 | ThingStatement.Thing insertStatement = null; 102 | for (Configuration.Definition.Attribute attributeToAppend : appendConfiguration.getInsert().getOwnerships()) { 103 | ArrayList constraintValues = GeneratorUtil.generateValueConstraintsConstrainingAttribute( 104 | row, header, filePath, fileSeparator, attributeToAppend); 105 | for (ThingConstraint.Predicate constraintValue : constraintValues) { 106 | if (insertStatement == null) { 107 | insertStatement = insertUnboundVar.constrain(GeneratorUtil.valueToHasConstraint(attributeToAppend.getAttribute(), constraintValue)); 108 | } else { 109 | insertStatement.constrain(GeneratorUtil.valueToHasConstraint(attributeToAppend.getAttribute(), constraintValue)); 110 | } 111 | } 112 | } 113 | 114 | if (insertStatement != null) { 115 | return TypeQL.match(entityMatchStatement).insert(insertStatement); 116 | } else { 117 | return TypeQL.insert(TypeQL.cVar("null").isa("null").has("null", "null")); 118 | } 119 | } else { 120 | return TypeQL.insert(TypeQL.cVar("null").isa("null").has("null", "null")); 121 | } 122 | } 123 | 124 | public boolean appendAttributeInsertStatementValid(TypeQLInsert insert) { 125 | if (insert == null) return false; 126 | if (!insert.toString().contains("isa " + appendConfiguration.getMatch().getType())) return false; 127 | for (Configuration.Definition.Attribute ownershipThingGetter : appendConfiguration.getMatch().getOwnerships()) { 128 | if (!insert.toString().contains("has " + ownershipThingGetter.getAttribute())) return false; 129 | } 130 | if (appendConfiguration.getInsert().getRequiredOwnerships() != null) { 131 | for (Configuration.Definition.Attribute attribute : appendConfiguration.getInsert().getRequiredOwnerships()) { 132 | if (!insert.toString().contains("has " + attribute.getAttribute())) return false; 133 | } 134 | } 135 | return true; 136 | } 137 | 138 | public char getFileSeparator() { 139 | return this.fileSeparator; 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/generator/AppendAttributeOrInsertThingGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.generator; 18 | 19 | import com.vaticle.typedb.driver.api.TypeDBTransaction; 20 | import com.vaticle.typedb.driver.api.answer.ConceptMap; 21 | import com.vaticle.typedb.driver.common.exception.TypeDBDriverException; 22 | import com.vaticle.typedb.osi.loader.config.Configuration; 23 | import com.vaticle.typedb.osi.loader.io.FileLogger; 24 | import com.vaticle.typedb.osi.loader.util.GeneratorUtil; 25 | import com.vaticle.typedb.osi.loader.util.TypeDBUtil; 26 | import com.vaticle.typedb.osi.loader.util.Util; 27 | import com.vaticle.typeql.lang.TypeQL; 28 | import com.vaticle.typeql.lang.builder.ConceptVariableBuilder; 29 | import com.vaticle.typeql.lang.pattern.constraint.ThingConstraint; 30 | import com.vaticle.typeql.lang.pattern.statement.ThingStatement; 31 | import com.vaticle.typeql.lang.query.TypeQLInsert; 32 | import org.apache.commons.io.FilenameUtils; 33 | import org.apache.logging.log4j.LogManager; 34 | import org.apache.logging.log4j.Logger; 35 | 36 | import java.io.IOException; 37 | import java.util.ArrayList; 38 | import java.util.Iterator; 39 | import java.util.stream.Stream; 40 | 41 | import static com.vaticle.typedb.osi.loader.util.TypeDBUtil.safeInsert; 42 | 43 | public class AppendAttributeOrInsertThingGenerator implements Generator { 44 | private static final Logger dataLogger = LogManager.getLogger("com.vaticle.typedb.osi.loader.error"); 45 | private final String filePath; 46 | private final String[] header; 47 | private final Configuration.Generator.AppendAttributeOrInsertThing appendOrInsertConfiguration; 48 | private final char fileSeparator; 49 | 50 | public AppendAttributeOrInsertThingGenerator(String filePath, Configuration.Generator.AppendAttributeOrInsertThing appendOrInsertConfiguration, char fileSeparator) throws IOException { 51 | this.filePath = filePath; 52 | this.header = Util.getFileHeader(filePath, fileSeparator); 53 | this.appendOrInsertConfiguration = appendOrInsertConfiguration; 54 | this.fileSeparator = fileSeparator; 55 | } 56 | 57 | @Override 58 | public void write(TypeDBTransaction tx, String[] row, boolean allowMultiInsert) { 59 | String fileName = FilenameUtils.getName(filePath); 60 | String fileNoExtension = FilenameUtils.removeExtension(fileName); 61 | String originalRow = String.join(Character.toString(fileSeparator), row); 62 | 63 | if (row.length > header.length) { 64 | FileLogger.getLogger().logMalformed(fileName, originalRow); 65 | dataLogger.error("Malformed Row detected in <" + filePath + "> - written to <" + fileNoExtension + "_malformed.log" + ">"); 66 | } 67 | 68 | TypeQLInsert appendQuery = generateMatchInsertStatement(row); 69 | TypeQLInsert insertQuery = generateThingInsertStatement(row); 70 | 71 | if (appendAttributeInsertStatementValid(appendQuery)) { 72 | try { 73 | Iterator answers = TypeDBUtil.executeMatch(tx, appendQuery); 74 | if (!answers.hasNext()) { 75 | if (thingInsertStatementValid(insertQuery)) { 76 | tx.query().insert(insertQuery); 77 | } else { 78 | FileLogger.getLogger().logInvalid(fileName, originalRow); 79 | dataLogger.error("Invalid Row detected in <" + filePath + "> - written to <" + fileNoExtension + "_invalid.log" + "> - invalid Statement: <" + insertQuery.toString().replace("\n", " ") + ">"); 80 | } 81 | } else { 82 | safeInsert(tx, appendQuery, answers, allowMultiInsert, filePath, originalRow, dataLogger); 83 | } 84 | } catch (TypeDBDriverException typeDBDriverException) { 85 | FileLogger.getLogger().logUnavailable(fileName, originalRow); 86 | dataLogger.error("TypeDB Unavailable - Row in <" + filePath + "> not inserted - written to <" + fileNoExtension + "_unavailable.log" + ">"); 87 | } 88 | } else { 89 | if (thingInsertStatementValid(insertQuery)) { 90 | tx.query().insert(insertQuery); 91 | } else { 92 | FileLogger.getLogger().logInvalid(fileName, originalRow); 93 | dataLogger.error("Invalid Row detected in <" + filePath + "> - written to <" + fileNoExtension + "_invalid.log" + "> - invalid Statements: <" + appendQuery.toString().replace("\n", " ") + "> and <" + insertQuery.toString().replace("\n", " ") + ">"); 94 | } 95 | } 96 | } 97 | 98 | public TypeQLInsert generateMatchInsertStatement(String[] row) { 99 | if (row.length > 0) { 100 | ThingStatement.Thing entityMatchStatement = TypeQL.cVar("thing") 101 | .isa(appendOrInsertConfiguration.getMatch().getType()); 102 | for (Configuration.Definition.Attribute ownershipThingGetter : appendOrInsertConfiguration.getMatch().getOwnerships()) { 103 | ArrayList constraintValues = GeneratorUtil.generateValueConstraintsConstrainingAttribute( 104 | row, header, filePath, fileSeparator, ownershipThingGetter); 105 | for (ThingConstraint.Predicate constraintValue : constraintValues) { 106 | entityMatchStatement.constrain(GeneratorUtil.valueToHasConstraint(ownershipThingGetter.getAttribute(), constraintValue)); 107 | } 108 | } 109 | 110 | ConceptVariableBuilder insertUnboundVar = TypeQL.cVar("thing"); 111 | ThingStatement.Thing insertStatement = null; 112 | for (Configuration.Definition.Attribute attributeToAppend : appendOrInsertConfiguration.getInsert().getOwnerships()) { 113 | ArrayList constraintValues = GeneratorUtil.generateValueConstraintsConstrainingAttribute( 114 | row, header, filePath, fileSeparator, attributeToAppend); 115 | for (ThingConstraint.Predicate constraintValue : constraintValues) { 116 | if (insertStatement == null) { 117 | insertStatement = insertUnboundVar.constrain(GeneratorUtil.valueToHasConstraint(attributeToAppend.getAttribute(), constraintValue)); 118 | } else { 119 | insertStatement.constrain(GeneratorUtil.valueToHasConstraint(attributeToAppend.getAttribute(), constraintValue)); 120 | } 121 | } 122 | } 123 | 124 | if (insertStatement != null) { 125 | return TypeQL.match(entityMatchStatement).insert(insertStatement); 126 | } else { 127 | return TypeQL.insert(TypeQL.cVar("null").isa("null").has("null", "null")); 128 | } 129 | } else { 130 | return TypeQL.insert(TypeQL.cVar("null").isa("null").has("null", "null")); 131 | } 132 | } 133 | 134 | public TypeQLInsert generateThingInsertStatement(String[] row) { 135 | if (row.length > 0) { 136 | ThingStatement.Thing insertStatement = GeneratorUtil.generateBoundThingVar(appendOrInsertConfiguration.getMatch().getType()); 137 | 138 | for (Configuration.Definition.Attribute attribute : appendOrInsertConfiguration.getMatch().getOwnerships()) { 139 | ArrayList constraintValues = GeneratorUtil.generateValueConstraintsConstrainingAttribute( 140 | row, header, filePath, fileSeparator, attribute); 141 | for (ThingConstraint.Predicate constraintValue : constraintValues) { 142 | insertStatement.constrain(GeneratorUtil.valueToHasConstraint(attribute.getAttribute(), constraintValue)); 143 | } 144 | } 145 | 146 | GeneratorUtil.constrainThingWithHasAttributes(row, header, filePath, fileSeparator, insertStatement, appendOrInsertConfiguration.getInsert().getOwnerships()); 147 | 148 | return TypeQL.insert(insertStatement); 149 | } else { 150 | return TypeQL.insert(TypeQL.cVar("null").isa("null").has("null", "null")); 151 | } 152 | } 153 | 154 | public boolean appendAttributeInsertStatementValid(TypeQLInsert insert) { 155 | if (insert == null) return false; 156 | if (!insert.toString().contains("isa " + appendOrInsertConfiguration.getMatch().getType())) return false; 157 | for (Configuration.Definition.Attribute ownershipThingGetter : appendOrInsertConfiguration.getMatch().getOwnerships()) { 158 | if (!insert.toString().contains("has " + ownershipThingGetter.getAttribute())) return false; 159 | } 160 | if (appendOrInsertConfiguration.getInsert().getRequiredOwnerships() != null) { 161 | for (Configuration.Definition.Attribute attribute : appendOrInsertConfiguration.getInsert().getRequiredOwnerships()) { 162 | if (!insert.toString().contains("has " + attribute.getAttribute())) return false; 163 | } 164 | } 165 | return true; 166 | } 167 | 168 | public boolean thingInsertStatementValid(TypeQLInsert insert) { 169 | if (insert == null) return false; 170 | if (!insert.toString().contains("isa " + appendOrInsertConfiguration.getMatch().getType())) return false; 171 | for (Configuration.Definition.Attribute ownershipThingGetter : appendOrInsertConfiguration.getMatch().getOwnerships()) { 172 | if (ownershipThingGetter.getRequired() != null && ownershipThingGetter.getRequired()) { 173 | if (!insert.toString().contains(", has " + ownershipThingGetter.getAttribute())) return false; 174 | } 175 | } 176 | if (appendOrInsertConfiguration.getInsert().getRequiredOwnerships() != null) { 177 | for (Configuration.Definition.Attribute attribute : appendOrInsertConfiguration.getInsert().getRequiredOwnerships()) { 178 | if (!insert.toString().contains("has " + attribute.getAttribute())) return false; 179 | } 180 | } 181 | return true; 182 | } 183 | 184 | public char getFileSeparator() { 185 | return this.fileSeparator; 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/generator/AttributeGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.generator; 18 | 19 | import com.vaticle.typedb.driver.api.TypeDBTransaction; 20 | import com.vaticle.typedb.driver.common.exception.TypeDBDriverException; 21 | import com.vaticle.typedb.osi.loader.config.Configuration; 22 | import com.vaticle.typedb.osi.loader.io.FileLogger; 23 | import com.vaticle.typedb.osi.loader.util.GeneratorUtil; 24 | import com.vaticle.typedb.osi.loader.util.Util; 25 | import com.vaticle.typeql.lang.TypeQL; 26 | import com.vaticle.typeql.lang.pattern.constraint.ThingConstraint; 27 | import com.vaticle.typeql.lang.query.TypeQLInsert; 28 | import org.apache.commons.io.FilenameUtils; 29 | import org.apache.logging.log4j.LogManager; 30 | import org.apache.logging.log4j.Logger; 31 | 32 | import java.io.IOException; 33 | import java.util.ArrayList; 34 | import java.util.List; 35 | 36 | public class AttributeGenerator implements Generator { 37 | private static final Logger dataLogger = LogManager.getLogger("com.vaticle.typedb.osi.loader.error"); 38 | private final String filePath; 39 | private final String[] header; 40 | private final Configuration.Generator.Attribute attributeConfiguration; 41 | private final char fileSeparator; 42 | 43 | public AttributeGenerator(String filePath, Configuration.Generator.Attribute attributeConfiguration, char fileSeparator) throws IOException { 44 | this.filePath = filePath; 45 | this.header = Util.getFileHeader(filePath, fileSeparator); 46 | this.attributeConfiguration = attributeConfiguration; 47 | this.fileSeparator = fileSeparator; 48 | } 49 | 50 | @Override 51 | public void write(TypeDBTransaction tx, String[] row, boolean allowMultiInsert) { 52 | 53 | String fileName = FilenameUtils.getName(filePath); 54 | String fileNoExtension = FilenameUtils.removeExtension(fileName); 55 | String originalRow = String.join(Character.toString(fileSeparator), row); 56 | 57 | if (row.length > header.length) { 58 | FileLogger.getLogger().logMalformed(fileName, originalRow); 59 | dataLogger.error("Malformed Row detected in <" + filePath + "> - written to <" + fileNoExtension + "_malformed.log" + ">"); 60 | } 61 | 62 | for (TypeQLInsert statement : generateInsertStatements(row)) { 63 | if (isValid(statement)) { 64 | try { 65 | tx.query().insert(statement); 66 | } catch (TypeDBDriverException driverException) { 67 | FileLogger.getLogger().logUnavailable(fileName, originalRow); 68 | dataLogger.error("TypeDB Unavailable - Row in <" + filePath + "> not inserted - written to <" + fileNoExtension + "_unavailable.log" + ">"); 69 | } 70 | } else { 71 | FileLogger.getLogger().logInvalid(fileName, originalRow); 72 | dataLogger.error("Invalid Row detected in <" + filePath + "> - written to <" + fileNoExtension + "_invalid.log" + "> - invalid Statement: <" + statement.toString().replace("\n", " ") + ">"); 73 | } 74 | } 75 | } 76 | 77 | public List generateInsertStatements(String[] row) { 78 | if (row.length > 0) { 79 | ArrayList constraints = GeneratorUtil.generateValueConstraintsConstrainingAttribute( 80 | row, header, filePath, fileSeparator, attributeConfiguration.getInsert()); 81 | 82 | List insertStatements = new ArrayList<>(); 83 | for (ThingConstraint.Predicate constraint : constraints) { 84 | insertStatements.add(TypeQL.insert( 85 | TypeQL.cVar("a") 86 | .constrain(constraint) 87 | .isa(attributeConfiguration.getInsert().getAttribute()) 88 | )); 89 | } 90 | return insertStatements; 91 | } else { 92 | return List.of(TypeQL.insert(TypeQL.cVar("null").isa("null").has("null", "null"))); 93 | } 94 | 95 | } 96 | 97 | private boolean isValid(TypeQLInsert insert) { 98 | return insert.toString().contains("isa " + attributeConfiguration.getInsert().getAttribute()); 99 | } 100 | 101 | public char getFileSeparator() { 102 | return this.fileSeparator; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/generator/EntityGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.generator; 18 | 19 | import com.vaticle.typedb.driver.api.TypeDBTransaction; 20 | import com.vaticle.typedb.driver.common.exception.TypeDBDriverException; 21 | import com.vaticle.typedb.osi.loader.config.Configuration; 22 | import com.vaticle.typedb.osi.loader.io.FileLogger; 23 | import com.vaticle.typedb.osi.loader.util.GeneratorUtil; 24 | import com.vaticle.typedb.osi.loader.util.TypeDBUtil; 25 | import com.vaticle.typedb.osi.loader.util.Util; 26 | import com.vaticle.typeql.lang.TypeQL; 27 | import com.vaticle.typeql.lang.pattern.statement.ThingStatement; 28 | import com.vaticle.typeql.lang.query.TypeQLInsert; 29 | import org.apache.commons.io.FilenameUtils; 30 | import org.apache.logging.log4j.LogManager; 31 | import org.apache.logging.log4j.Logger; 32 | 33 | import java.io.IOException; 34 | 35 | public class EntityGenerator implements Generator { 36 | private static final Logger dataLogger = LogManager.getLogger("com.vaticle.typedb.osi.loader.error"); 37 | private final String filePath; 38 | private final String[] header; 39 | private final Configuration.Generator.Entity entityConfiguration; 40 | private final char fileSeparator; 41 | 42 | public EntityGenerator(String filePath, Configuration.Generator.Entity entityConfiguration, char fileSeparator) throws IOException { 43 | this.filePath = filePath; 44 | this.header = Util.getFileHeader(filePath, fileSeparator); 45 | this.entityConfiguration = entityConfiguration; 46 | this.fileSeparator = fileSeparator; 47 | } 48 | 49 | @Override 50 | public void write(TypeDBTransaction tx, String[] row, boolean allowMultiInsert) { 51 | String fileName = FilenameUtils.getName(filePath); 52 | String fileNoExtension = FilenameUtils.removeExtension(fileName); 53 | String originalRow = String.join(Character.toString(fileSeparator), row); 54 | 55 | if (row.length > header.length) { 56 | FileLogger.getLogger().logMalformed(fileName, originalRow); 57 | dataLogger.error("Malformed Row detected in <" + filePath + "> - written to <" + fileNoExtension + "_malformed.log" + ">"); 58 | } 59 | 60 | TypeQLInsert query = generateThingInsertStatement(row); 61 | if (valid(query)) { 62 | try { 63 | tx.query().insert(query); 64 | } catch (TypeDBDriverException typeDBDriverException) { 65 | FileLogger.getLogger().logUnavailable(fileName, originalRow); 66 | dataLogger.error("TypeDB Unavailable - Row in <" + filePath + "> not inserted - written to <" + fileNoExtension + "_unavailable.log" + ">"); 67 | } 68 | } else { 69 | FileLogger.getLogger().logInvalid(fileName, originalRow); 70 | dataLogger.error("Invalid Row detected in <" + filePath + "> - written to <" + fileNoExtension + "_invalid.log" + "> - invalid Statement: <" + query.toString().replace("\n", " ") + ">"); 71 | } 72 | } 73 | 74 | public TypeQLInsert generateThingInsertStatement(String[] row) { 75 | if (row.length > 0) { 76 | ThingStatement insertStatement = GeneratorUtil.generateBoundThingVar(entityConfiguration.getInsert().getEntity()); 77 | 78 | GeneratorUtil.constrainThingWithHasAttributes(row, header, filePath, fileSeparator, insertStatement, entityConfiguration.getInsert().getOwnerships()); 79 | 80 | return TypeQL.insert(insertStatement); 81 | } else { 82 | return TypeQL.insert(TypeQL.cVar("null").isa("null").has("null", "null")); 83 | } 84 | } 85 | 86 | public boolean valid(TypeQLInsert insert) { 87 | if (insert == null) return false; 88 | if (!insert.toString().contains("isa " + entityConfiguration.getInsert().getEntity())) return false; 89 | for (Configuration.Definition.Attribute attribute : entityConfiguration.getInsert().getRequiredOwnerships()) { 90 | if (!insert.toString().contains("has " + attribute.getAttribute())) return false; 91 | } 92 | return true; 93 | } 94 | 95 | public char getFileSeparator() { 96 | return this.fileSeparator; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/generator/Generator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.generator; 18 | 19 | import com.vaticle.typedb.driver.api.TypeDBTransaction; 20 | 21 | public interface Generator { 22 | void write(TypeDBTransaction tx, String[] row, boolean allowMultiInsert); 23 | char getFileSeparator(); 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/io/FileLogger.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.io; 18 | 19 | import org.apache.commons.io.FilenameUtils; 20 | 21 | import java.io.File; 22 | import java.io.FileWriter; 23 | import java.io.IOException; 24 | import java.time.ZoneId; 25 | import java.time.format.DateTimeFormatter; 26 | import java.util.Date; 27 | 28 | public class FileLogger { 29 | 30 | private static FileLogger logger = null; 31 | private final String directoryString; 32 | 33 | private FileLogger() { 34 | DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd-HH_mm_ss").withZone(ZoneId.systemDefault()); 35 | File directory = new File(DATE_TIME_FORMATTER.format(new Date().toInstant())); 36 | this.directoryString = directory.toString(); 37 | if (! directory.exists()){ 38 | directory.mkdir(); 39 | } 40 | } 41 | 42 | public static synchronized FileLogger getLogger() { 43 | if (logger == null) { 44 | logger = new FileLogger(); 45 | } 46 | return logger; 47 | } 48 | 49 | public synchronized void logMalformed(String sourceFile, String errorString) { 50 | try { 51 | FileWriter fw = new FileWriter(directoryString + "/" + FilenameUtils.removeExtension(sourceFile) + "_malformed.log", true); 52 | fw.append(errorString.replace("null", "")); 53 | fw.append("\n"); 54 | fw.flush(); 55 | fw.close(); 56 | } catch (IOException ioException) { 57 | ioException.printStackTrace(); 58 | } 59 | } 60 | 61 | public synchronized void logInvalid(String sourceFile, String errorString) { 62 | try { 63 | FileWriter fw = new FileWriter(directoryString + "/" + FilenameUtils.removeExtension(sourceFile) + "_invalid.log", true); 64 | fw.append(errorString.replace("null", "")); 65 | fw.append("\n"); 66 | fw.flush(); 67 | fw.close(); 68 | } catch (IOException ioException) { 69 | ioException.printStackTrace(); 70 | } 71 | } 72 | 73 | public synchronized void logUnavailable(String sourceFile, String errorString) { 74 | try { 75 | FileWriter fw = new FileWriter(directoryString + "/" + FilenameUtils.removeExtension(sourceFile) + "_unavailable.log", true); 76 | fw.append(errorString.replace("null", "")); 77 | fw.append("\n"); 78 | fw.flush(); 79 | fw.close(); 80 | } catch (IOException ioException) { 81 | ioException.printStackTrace(); 82 | } 83 | } 84 | 85 | public void logNoMatches(String sourceFile, String row) { 86 | try { 87 | FileWriter fw = new FileWriter(directoryString + "/" + FilenameUtils.removeExtension(sourceFile) + "_no_matches.log", true); 88 | fw.append(row.replace("null", "")); 89 | fw.append("\n"); 90 | fw.flush(); 91 | fw.close(); 92 | } catch (IOException ioException) { 93 | ioException.printStackTrace(); 94 | } 95 | } 96 | 97 | public void logTooManyMatches(String sourceFile, String row) { 98 | try { 99 | FileWriter fw = new FileWriter(directoryString + "/" + FilenameUtils.removeExtension(sourceFile) + "_too_many_matches.log", true); 100 | fw.append(row.replace("null", "")); 101 | fw.append("\n"); 102 | fw.flush(); 103 | fw.close(); 104 | } catch (IOException ioException) { 105 | ioException.printStackTrace(); 106 | } 107 | } 108 | 109 | public synchronized void logColumnWarnings(String sourceFile, String errorString) { 110 | try { 111 | FileWriter fw = new FileWriter(directoryString + "/" + FilenameUtils.removeExtension(sourceFile) + "_column_type.log", true); 112 | fw.append(errorString.replace("null", "")); 113 | fw.append("\n"); 114 | fw.flush(); 115 | fw.close(); 116 | } catch (IOException ioException) { 117 | ioException.printStackTrace(); 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/io/FileToInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.io; 18 | 19 | import java.io.BufferedInputStream; 20 | import java.io.FileInputStream; 21 | import java.io.IOException; 22 | import java.io.InputStream; 23 | import java.util.zip.GZIPInputStream; 24 | 25 | public class FileToInputStream { 26 | 27 | public static InputStream getInputStream(String filepath) { 28 | try { 29 | if (filepath == null) { 30 | return null; 31 | } else if (filepath.endsWith(".gz")) { 32 | return new BufferedInputStream(new GZIPInputStream(new FileInputStream(filepath)), 128_000); 33 | } else { 34 | return new BufferedInputStream(new FileInputStream(filepath), 128_000); 35 | } 36 | } catch (IOException e) { 37 | e.printStackTrace(); 38 | return null; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/loader/TypeDBLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.loader; 18 | 19 | import com.vaticle.typedb.driver.TypeDB; 20 | import com.vaticle.typedb.driver.api.TypeDBDriver; 21 | import com.vaticle.typedb.driver.api.TypeDBSession; 22 | import com.vaticle.typedb.common.concurrent.NamedThreadFactory; 23 | import com.vaticle.typedb.osi.loader.cli.LoadOptions; 24 | import com.vaticle.typedb.osi.loader.config.Configuration; 25 | import com.vaticle.typedb.osi.loader.config.ConfigurationValidation; 26 | import com.vaticle.typedb.osi.loader.util.TypeDBUtil; 27 | import com.vaticle.typedb.osi.loader.util.Util; 28 | 29 | import java.time.Instant; 30 | import java.util.ArrayList; 31 | import java.util.HashMap; 32 | 33 | public class TypeDBLoader { 34 | 35 | private final LoadOptions options; 36 | private final Configuration dc; 37 | 38 | public TypeDBLoader(LoadOptions options) { 39 | this.options = options; 40 | this.dc = Util.initializeConfig(options.dataConfigFilePath); 41 | } 42 | 43 | public void load() { 44 | Util.info("validating your config..."); 45 | TypeDBDriver schemaDriver = TypeDBUtil.getDriver(options); 46 | ConfigurationValidation cv = new ConfigurationValidation(dc); 47 | HashMap> validationReport = new HashMap<>(); 48 | ArrayList errors = new ArrayList<>(); 49 | ArrayList warnings = new ArrayList<>(); 50 | validationReport.put("warnings", warnings); 51 | validationReport.put("errors", errors); 52 | cv.validateSchemaPresent(validationReport); 53 | 54 | if (validationReport.get("errors").size() == 0) { 55 | if (options.cleanMigration) { 56 | TypeDBUtil.cleanAndDefineSchemaToDatabase(schemaDriver, options.databaseName, dc.getGlobalConfig().getSchema()); 57 | Util.info("cleaned database and migrated schema..."); 58 | } else if (options.loadSchema) { 59 | TypeDBUtil.loadAndDefineSchema(schemaDriver, options.databaseName, dc.getGlobalConfig().getSchema()); 60 | Util.info("loaded schema..."); 61 | } 62 | } else { 63 | validationReport.get("errors").forEach(Util::error); 64 | System.exit(1); 65 | } 66 | 67 | TypeDBSession schemaSession = TypeDBUtil.getSchemaSession(schemaDriver, options.databaseName); 68 | cv.validateConfiguration(validationReport, schemaSession); 69 | 70 | if (validationReport.get("warnings").size() > 0) { 71 | validationReport.get("warnings").forEach(Util::warn); 72 | } 73 | if (validationReport.get("errors").size() > 0) { 74 | validationReport.get("errors").forEach(Util::error); 75 | schemaSession.close(); 76 | schemaDriver.close(); 77 | System.exit(1); 78 | } 79 | schemaSession.close(); 80 | schemaDriver.close(); 81 | Util.info("finished validating your config..."); 82 | 83 | Instant start = Instant.now(); 84 | try { 85 | AsyncLoaderWorker asyncLoaderWorker = null; 86 | try (TypeDBDriver driver = TypeDBUtil.getDriver(options)) { 87 | Runtime.getRuntime().addShutdownHook( 88 | NamedThreadFactory.create(AsyncLoaderWorker.class, "shutdown").newThread(driver::close) 89 | ); 90 | asyncLoaderWorker = new AsyncLoaderWorker(dc, options); 91 | asyncLoaderWorker.run(driver); 92 | } finally { 93 | if (asyncLoaderWorker != null) asyncLoaderWorker.close(); 94 | } 95 | } catch (Throwable e) { 96 | Util.error(e.getMessage(), e); 97 | Util.error("TERMINATED WITH ERROR"); 98 | } finally { 99 | Instant end = Instant.now(); 100 | Util.info("TypeDB Loader finished in: {}", Util.printDuration(start, end)); 101 | } 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/preprocessor/RegexPreprocessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.preprocessor; 18 | 19 | public class RegexPreprocessor { 20 | String match; 21 | String replace; 22 | 23 | public RegexPreprocessor(String match, String replace) { 24 | this.match = match; 25 | this.replace = replace; 26 | } 27 | 28 | public String applyProcessor(String value) { 29 | return value.replaceAll(match, replace); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/status/MigrationStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.status; 18 | 19 | 20 | public class MigrationStatus { 21 | private final String conceptName; 22 | private boolean isCompleted; 23 | private int migratedRows; 24 | 25 | public MigrationStatus(String conceptName, boolean isCompleted, int migratedRows) { 26 | this.conceptName = conceptName; 27 | this.isCompleted = isCompleted; 28 | this.migratedRows = migratedRows; 29 | } 30 | 31 | public int getMigratedRows() { 32 | return migratedRows; 33 | } 34 | 35 | public void setMigratedRows(int migratedRows) { 36 | this.migratedRows = migratedRows; 37 | } 38 | 39 | public boolean isCompleted() { 40 | return isCompleted; 41 | } 42 | 43 | public void setCompleted(boolean completed) { 44 | isCompleted = completed; 45 | } 46 | 47 | @Override 48 | public String toString() { 49 | return "MigrationStatus{" + 50 | "conceptName='" + conceptName + '\'' + 51 | ", isCompleted=" + isCompleted + 52 | ", migratedRows=" + migratedRows + 53 | '}'; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/type/AttributeValueType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.type; 18 | 19 | public enum AttributeValueType { 20 | STRING, 21 | LONG, 22 | DOUBLE, 23 | BOOLEAN, 24 | DATETIME, 25 | INVALID; 26 | 27 | // @Override 28 | // public String toString() { 29 | // return super.toString().toLowerCase(); 30 | // } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/util/GeneratorUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.util; 18 | 19 | import com.vaticle.typedb.osi.loader.config.Configuration; 20 | import com.vaticle.typedb.osi.loader.io.FileLogger; 21 | import com.vaticle.typedb.osi.loader.preprocessor.RegexPreprocessor; 22 | import com.vaticle.typedb.osi.loader.type.AttributeValueType; 23 | import com.vaticle.typeql.lang.TypeQL; 24 | import com.vaticle.typeql.lang.common.TypeQLToken; 25 | import com.vaticle.typeql.lang.pattern.constraint.Predicate; 26 | import com.vaticle.typeql.lang.pattern.constraint.ThingConstraint; 27 | import com.vaticle.typeql.lang.pattern.statement.ThingStatement; 28 | import org.apache.commons.io.FilenameUtils; 29 | import org.apache.logging.log4j.LogManager; 30 | import org.apache.logging.log4j.Logger; 31 | 32 | import java.time.DateTimeException; 33 | import java.time.LocalDate; 34 | import java.time.LocalDateTime; 35 | import java.time.LocalTime; 36 | import java.time.format.DateTimeFormatter; 37 | import java.util.ArrayList; 38 | import java.util.Arrays; 39 | 40 | public class GeneratorUtil { 41 | 42 | private static final Logger dataLogger = LogManager.getLogger("com.vaticle.typedb.osi.loader.error"); 43 | 44 | //TODO: remove this function and all the complications it provides... this would need to be either solved by a regex preprocessor, or is already solved by CSV.withIgnoreSurroundingSpaces() 45 | private static String cleanToken(String token) { 46 | String cleaned = token.replace("\"", ""); 47 | cleaned = cleaned.replace("\\", ""); 48 | cleaned = cleaned.trim(); 49 | return cleaned; 50 | } 51 | 52 | public static void constrainThingWithHasAttributes(String[] row, 53 | String[] header, 54 | String filePath, 55 | char fileSeparator, 56 | ThingStatement insertStatement, 57 | Configuration.Definition.Attribute[] attributes) { 58 | for (Configuration.Definition.Attribute attribute : attributes) { 59 | ArrayList constraintValues = GeneratorUtil.generateValueConstraintsConstrainingAttribute( 60 | row, header, filePath, fileSeparator, attribute); 61 | for (ThingConstraint.Predicate constraintValue : constraintValues) { 62 | insertStatement.constrain(GeneratorUtil.valueToHasConstraint(attribute.getAttribute(), constraintValue)); 63 | } 64 | } 65 | } 66 | 67 | public static int getColumnIndexByName(String[] header, String column) { 68 | return Arrays.asList(header).indexOf(column); 69 | } 70 | 71 | public static ThingStatement.Thing generateBoundThingVar(String schemaType) { 72 | return TypeQL.cVar("e").isa(schemaType); 73 | } 74 | 75 | public static ThingConstraint.Has valueToHasConstraint(String attributeSchemaType, 76 | ThingConstraint.Predicate valueConstraints) { 77 | if (valueConstraints != null) { 78 | return new ThingConstraint.Has(attributeSchemaType, valueConstraints); 79 | } 80 | return null; 81 | } 82 | 83 | public static ArrayList generateValueConstraintsConstrainingAttribute(String[] row, 84 | String[] header, 85 | String filepath, 86 | char fileSeparator, 87 | Configuration.Definition.Attribute attribute) { 88 | 89 | String attributeType = attribute.getAttribute(); 90 | AttributeValueType attributeValueType = attribute.getConceptValueType(); 91 | String listSeparator = attribute.getListSeparator(); 92 | Configuration.PreprocessorConfig preprocessor = attribute.getPreprocessorConfig(); 93 | String token; 94 | 95 | int colIdx = GeneratorUtil.getColumnIndexByName(header, attribute.getColumn()); 96 | if (colIdx < row.length) { 97 | token = row[colIdx]; 98 | } else { 99 | return new ArrayList<>(); 100 | } 101 | 102 | 103 | ArrayList valueConstraints = new ArrayList<>(); 104 | if (token != null && !token.isEmpty()) { 105 | String cleanedToken = cleanToken(token); 106 | if (listSeparator == null) { 107 | ThingConstraint.Predicate valueConstraint = generateValueConstraint(attributeType, attributeValueType, cleanedToken, preprocessor, row, filepath, fileSeparator); 108 | if (valueConstraint != null) { 109 | valueConstraints.add(valueConstraint); 110 | } 111 | } else { 112 | for (String exploded : cleanedToken.split(listSeparator)) { 113 | String cleanedExplodedToken = cleanToken(exploded); 114 | if (!cleanedExplodedToken.isEmpty()) { 115 | ThingConstraint.Predicate valueConstraint = generateValueConstraint(attributeType, attributeValueType, cleanedExplodedToken, preprocessor, row, filepath, fileSeparator); 116 | if (valueConstraint != null) { 117 | valueConstraints.add(valueConstraint); 118 | } 119 | } 120 | } 121 | } 122 | } 123 | return valueConstraints; 124 | } 125 | 126 | 127 | public static ThingConstraint.Predicate generateValueConstraint(String attributeSchemaType, 128 | AttributeValueType attributeValueType, 129 | String cleanedValue, 130 | Configuration.PreprocessorConfig preprocessorConfig, 131 | String[] row, 132 | String filepath, 133 | char fileSeparator) { 134 | String fileName = FilenameUtils.getName(filepath); 135 | String fileNoExtension = FilenameUtils.removeExtension(fileName); 136 | String originalRow = String.join(Character.toString(fileSeparator), row); 137 | 138 | if (preprocessorConfig != null) { 139 | cleanedValue = applyPreprocessor(cleanedValue, preprocessorConfig); 140 | } 141 | 142 | ThingConstraint.Predicate constraint = null; 143 | switch (attributeValueType) { 144 | case STRING: 145 | constraint = new ThingConstraint.Predicate(new Predicate.String(TypeQLToken.Predicate.Equality.EQ, cleanedValue)); 146 | break; 147 | case LONG: 148 | try { 149 | constraint = new ThingConstraint.Predicate(new Predicate.Long(TypeQLToken.Predicate.Equality.EQ, Long.parseLong(cleanedValue))); 150 | } catch (NumberFormatException numberFormatException) { 151 | FileLogger.getLogger().logColumnWarnings(fileName, originalRow); 152 | dataLogger.warn(String.format("column of type long for variable <%s> with non- value <%s> - skipping column - faulty row written to <%s_column_type.log>", attributeSchemaType, cleanedValue, fileNoExtension)); 153 | } 154 | break; 155 | case DOUBLE: 156 | try { 157 | constraint = new ThingConstraint.Predicate(new Predicate.Double(TypeQLToken.Predicate.Equality.EQ, Double.parseDouble(cleanedValue))); 158 | } catch (NumberFormatException numberFormatException) { 159 | FileLogger.getLogger().logColumnWarnings(fileName, originalRow); 160 | dataLogger.warn(String.format("column of type double for variable <%s> with non- value <%s> - skipping column - faulty row written to <%s_column_type.log>", attributeSchemaType, cleanedValue, fileNoExtension)); 161 | } 162 | break; 163 | case BOOLEAN: 164 | if (cleanedValue.equalsIgnoreCase("true")) { 165 | constraint = new ThingConstraint.Predicate(new Predicate.Boolean(TypeQLToken.Predicate.Equality.EQ, true)); 166 | } else if (cleanedValue.equalsIgnoreCase("false")) { 167 | constraint = new ThingConstraint.Predicate(new Predicate.Boolean(TypeQLToken.Predicate.Equality.EQ, false)); 168 | } else { 169 | FileLogger.getLogger().logColumnWarnings(fileName, originalRow); 170 | dataLogger.warn(String.format("column of type boolean for variable <%s> with non- value <%s> - skipping column - faulty row written to <%s_column_type.log>", attributeSchemaType, cleanedValue, fileNoExtension)); 171 | } 172 | break; 173 | case DATETIME: 174 | try { 175 | DateTimeFormatter isoDateFormatter = DateTimeFormatter.ISO_DATE; 176 | String[] dt = cleanedValue.split("T"); 177 | LocalDate date = LocalDate.parse(dt[0], isoDateFormatter); 178 | LocalDateTime dateTime; 179 | if (dt.length > 1) { 180 | LocalTime time = LocalTime.parse(dt[1], DateTimeFormatter.ISO_TIME); 181 | dateTime = date.atTime(time); 182 | } else { 183 | dateTime = date.atStartOfDay(); 184 | } 185 | constraint = new ThingConstraint.Predicate(new Predicate.DateTime(TypeQLToken.Predicate.Equality.EQ, dateTime)); 186 | } catch (DateTimeException dateTimeException) { 187 | FileLogger.getLogger().logColumnWarnings(fileName, originalRow); 188 | dataLogger.warn(String.format("column of type datetime for variable <%s> with non- datetime value <%s> - skipping column - faulty row written to <%s_column_type.log>", attributeSchemaType, cleanedValue, fileNoExtension)); 189 | } 190 | break; 191 | default: 192 | dataLogger.warn("column type not valid - must be either: string, long, double, boolean, or datetime"); 193 | } 194 | return constraint; 195 | } 196 | 197 | private static String applyPreprocessor(String cleanedValue, 198 | Configuration.PreprocessorConfig preprocessorConfig) { 199 | Configuration.PreprocessorConfig.PreprocessorParameters params = preprocessorConfig.getParameters(); 200 | String processorType = preprocessorConfig.getType(); 201 | switch (processorType) { 202 | case "regex": 203 | return applyRegexPreprocessor(cleanedValue, params.getRegexMatch(), params.getRegexReplace()); 204 | default: 205 | throw new IllegalArgumentException("Preprocessor of type: <" + processorType + "> as specified in data config does not exist"); 206 | } 207 | } 208 | 209 | private static String applyRegexPreprocessor(String stringToProcess, 210 | String matchString, 211 | String replaceString) { 212 | RegexPreprocessor rpp = new RegexPreprocessor(matchString, replaceString); 213 | return rpp.applyProcessor(stringToProcess); 214 | } 215 | 216 | } 217 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/util/TypeDBUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.util; 18 | 19 | import com.vaticle.typedb.driver.TypeDB; 20 | import com.vaticle.typedb.driver.api.TypeDBDriver; 21 | import com.vaticle.typedb.driver.api.TypeDBCredential; 22 | import com.vaticle.typedb.driver.api.TypeDBSession; 23 | import com.vaticle.typedb.driver.api.TypeDBTransaction; 24 | import com.vaticle.typedb.driver.api.answer.ConceptMap; 25 | import com.vaticle.typedb.driver.api.concept.Concept; 26 | import com.vaticle.typedb.osi.loader.cli.LoadOptions; 27 | import com.vaticle.typedb.osi.loader.io.FileLogger; 28 | import com.vaticle.typeql.lang.TypeQL; 29 | import com.vaticle.typeql.lang.pattern.statement.Statement; 30 | import com.vaticle.typeql.lang.pattern.statement.ThingStatement; 31 | import com.vaticle.typeql.lang.query.TypeQLDefine; 32 | import com.vaticle.typeql.lang.query.TypeQLInsert; 33 | import org.apache.commons.io.FilenameUtils; 34 | import org.apache.logging.log4j.Logger; 35 | 36 | import java.nio.file.Path; 37 | import java.util.ArrayList; 38 | import java.util.Iterator; 39 | import java.util.List; 40 | 41 | import static com.vaticle.typedb.osi.loader.util.Util.loadSchemaFromFile; 42 | 43 | public class TypeDBUtil { 44 | 45 | public static TypeDBDriver getDriver(LoadOptions options) { 46 | if (options.typedbClusterURI != null) { 47 | TypeDBCredential credential; 48 | if (options.tlsEnabled) { 49 | if (options.tlsRootCAPath == null) 50 | throw new RuntimeException("When TLS is enabled a Root CA path must be provided"); 51 | credential = new TypeDBCredential(options.username, options.password, Path.of(options.tlsRootCAPath)); 52 | } else { 53 | credential = new TypeDBCredential(options.username, options.password, false); 54 | } 55 | return TypeDB.cloudDriver(options.typedbClusterURI, credential); 56 | } else { 57 | return TypeDB.coreDriver(options.typedbURI); 58 | } 59 | } 60 | 61 | public static TypeDBDriver getCoreDriver(String typedbURI) { 62 | return TypeDB.coreDriver(typedbURI); 63 | } 64 | 65 | public static TypeDBSession getDataSession(TypeDBDriver driver, String databaseName) { 66 | return driver.session(databaseName, TypeDBSession.Type.DATA); 67 | } 68 | 69 | public static TypeDBSession getSchemaSession(TypeDBDriver driver, String databaseName) { 70 | return driver.session(databaseName, TypeDBSession.Type.SCHEMA); 71 | } 72 | 73 | private static void createDatabase(TypeDBDriver driver, String databaseName) { 74 | driver.databases().create(databaseName); 75 | } 76 | 77 | private static void deleteDatabaseIfExists(TypeDBDriver driver, String databaseName) { 78 | if (driver.databases().contains(databaseName)) { 79 | driver.databases().get(databaseName).delete(); 80 | } 81 | } 82 | 83 | public static void cleanAndDefineSchemaToDatabase(TypeDBDriver driver, String databaseName, String schemaPath) { 84 | deleteDatabaseIfExists(driver, databaseName); 85 | createDatabase(driver, databaseName); 86 | loadAndDefineSchema(driver, databaseName, schemaPath); 87 | } 88 | 89 | private static void defineToTypeDB(TypeDBDriver driver, String databaseName, String schemaAsString) { 90 | TypeDBSession schemaSession = getSchemaSession(driver, databaseName); 91 | TypeQLDefine q = TypeQL.parseQuery(schemaAsString); 92 | 93 | try (TypeDBTransaction writeTransaction = schemaSession.transaction(TypeDBTransaction.Type.WRITE)) { 94 | writeTransaction.query().define(q); 95 | writeTransaction.commit(); 96 | writeTransaction.close(); 97 | schemaSession.close(); 98 | } 99 | 100 | Util.info("Defined schema to database <" + databaseName + ">"); 101 | } 102 | 103 | public static void loadAndDefineSchema(TypeDBDriver driver, String databaseName, String schemaPath) { 104 | String schema = loadSchemaFromFile(schemaPath); 105 | defineToTypeDB(driver, databaseName, schema); 106 | } 107 | 108 | public static Iterator executeMatch(TypeDBTransaction tx, TypeQLInsert query) { 109 | if (!query.match().isPresent()) throw new RuntimeException("Expected TypeQL 'match' to be present"); 110 | return tx.query().get(query.match().get().get()).iterator(); 111 | } 112 | 113 | public static void safeInsert(TypeDBTransaction tx, TypeQLInsert query, Iterator matches, boolean allowMultiInsert, String filePath, String row, Logger dataLogger) { 114 | assert query.match().isPresent(); 115 | String fileName = FilenameUtils.getName(filePath); 116 | ConceptMap answer = matches.next(); 117 | if (!allowMultiInsert && matches.hasNext()) { 118 | FileLogger.getLogger().logTooManyMatches(fileName, row); 119 | dataLogger.error("Match-insert skipped - File <" + filePath + "> row <" + row + "> generates query <" + query + "> which matched more than 1 answer."); 120 | } else { 121 | tx.query().insert(TypeDBUtil.replaceMatchWithAnswer(query, answer)); 122 | while (matches.hasNext()) { 123 | answer = matches.next(); 124 | tx.query().insert(TypeDBUtil.replaceMatchWithAnswer(query, answer)); 125 | } 126 | } 127 | } 128 | 129 | public static TypeQLInsert replaceMatchWithAnswer(TypeQLInsert query, ConceptMap ans) { 130 | assert query.match().isPresent(); 131 | List> insertVars = query.asInsert().statements(); 132 | List matchVars = new ArrayList<>(); 133 | ans.variables().forEach((var) -> { 134 | Concept concept = ans.get(var); 135 | if (concept.isThing()) matchVars.add(TypeQL.cVar(var).iid(concept.asThing().getIID())); 136 | else if (concept.asType().getLabel().scope().isPresent()) { 137 | matchVars.add(TypeQL.cVar(var).type(concept.asType().getLabel().scope().get(), concept.asType().getLabel().name())); 138 | } else matchVars.add(TypeQL.cVar(var).type(concept.asType().getLabel().name())); 139 | }); 140 | return TypeQL.match(matchVars).insert(insertVars); 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/main/java/com/vaticle/typedb/osi/loader/util/Util.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.util; 18 | 19 | import com.google.gson.Gson; 20 | import com.google.gson.reflect.TypeToken; 21 | import com.vaticle.typedb.driver.api.TypeDBSession; 22 | import com.vaticle.typedb.osi.loader.config.Configuration; 23 | import org.apache.commons.csv.CSVFormat; 24 | import org.apache.commons.csv.CSVParser; 25 | import org.apache.commons.csv.CSVRecord; 26 | import org.apache.logging.log4j.LogManager; 27 | import org.apache.logging.log4j.Logger; 28 | 29 | import java.io.BufferedReader; 30 | import java.io.File; 31 | import java.io.FileInputStream; 32 | import java.io.FileNotFoundException; 33 | import java.io.IOException; 34 | import java.io.InputStream; 35 | import java.io.InputStreamReader; 36 | import java.lang.reflect.Type; 37 | import java.time.Duration; 38 | import java.time.Instant; 39 | import java.util.Objects; 40 | 41 | import static com.vaticle.typedb.osi.loader.io.FileToInputStream.getInputStream; 42 | import static java.nio.charset.StandardCharsets.UTF_8; 43 | 44 | public class Util { 45 | 46 | private static final Logger appLogger = LogManager.getLogger("com.vaticle.typedb.osi.loader.loader"); 47 | //TODO: disallow duplicate header names, and why does ignoreEmptyLines not work??? 48 | private static final CSVFormat CSV_FORMAT = CSVFormat.DEFAULT.withEscape('\\').withIgnoreSurroundingSpaces().withNullString(""); 49 | private static final CSVFormat TSV_FORMAT = CSVFormat.DEFAULT.withDelimiter('\t').withEscape('\\').withIgnoreSurroundingSpaces().withNullString(""); 50 | 51 | public static String[] getFileHeader(String filePath, char separator) throws IOException, IllegalArgumentException { 52 | BufferedReader br = newBufferedReader(filePath); 53 | if (separator == ',') { 54 | return parseCSV(br.readLine()); 55 | } else if (separator == '\t') { 56 | return parseTSV(br.readLine()); 57 | } else { 58 | throw new IllegalArgumentException("currently supported separators are: <,>, <\t>"); 59 | } 60 | } 61 | 62 | public static String getAbsPath(String p) { 63 | File file = new File(p); 64 | return file.getAbsolutePath(); 65 | } 66 | 67 | public static Configuration initializeConfig(String dcPath) { 68 | try { 69 | BufferedReader bufferedReader = newBufferedReader(dcPath); 70 | Type DataConfigType = new TypeToken() { 71 | }.getType(); 72 | return new Gson().fromJson(bufferedReader, DataConfigType); 73 | } catch (FileNotFoundException e) { 74 | appLogger.error("Data Configuration file not found - please check the path: " + dcPath); 75 | return null; 76 | } 77 | } 78 | 79 | public static String loadSchemaFromFile(String schemaPath) { 80 | String schema = ""; 81 | try { 82 | BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(schemaPath), UTF_8)); 83 | StringBuilder sb = new StringBuilder(); 84 | String line = br.readLine(); 85 | while (line != null) { 86 | sb.append(line).append("\n"); 87 | line = br.readLine(); 88 | } 89 | schema = sb.toString(); 90 | } catch (IOException e) { 91 | e.printStackTrace(); 92 | } 93 | return schema; 94 | } 95 | 96 | public static BufferedReader newBufferedReader(String filePath) throws FileNotFoundException { 97 | InputStream is = getInputStream(filePath); 98 | if (is != null) { 99 | return new BufferedReader(new InputStreamReader(is, UTF_8)); 100 | } else { 101 | throw new FileNotFoundException(); 102 | } 103 | } 104 | 105 | public static String[] parseBySeparator(String line, char separator) throws IllegalArgumentException { 106 | if (separator == ',') { 107 | return parseCSV(line); 108 | } else if (separator == '\t') { 109 | return parseTSV(line); 110 | } else { 111 | throw new IllegalArgumentException("currently supported separators are: <,>, <\t>"); 112 | } 113 | } 114 | 115 | public static String[] parseCSV(String line) { 116 | if (!line.isEmpty()) { 117 | try { 118 | CSVRecord csv = CSVParser.parse(line, CSV_FORMAT).getRecords().get(0); 119 | return parse(csv); 120 | } catch (IOException ioException) { 121 | line = line.replace("\"", ""); 122 | try { 123 | CSVRecord csv = CSVParser.parse(line, CSV_FORMAT).getRecords().get(0); 124 | return parse(csv); 125 | } catch (IOException ioException2) { 126 | //TODO LOG INTO ERRORS 127 | return new String[0]; 128 | } 129 | } 130 | } else { 131 | return new String[0]; 132 | } 133 | } 134 | 135 | public static String[] parseTSV(String line) { 136 | if (!line.isEmpty()) { 137 | try { 138 | CSVRecord tsv = CSVParser.parse(line, TSV_FORMAT).getRecords().get(0); 139 | return parse(tsv); 140 | } catch (IOException ioException) { 141 | line = line.replace("\"", ""); 142 | try { 143 | CSVRecord tsv = CSVParser.parse(line, TSV_FORMAT).getRecords().get(0); 144 | return parse(tsv); 145 | } catch (IOException ioException2) { 146 | //TODO LOG INTO ERRORS 147 | return new String[0]; 148 | } 149 | } 150 | } else { 151 | return new String[0]; 152 | } 153 | } 154 | 155 | private static String[] parse(CSVRecord record) { 156 | String[] arr = new String[record.size()]; 157 | for (int i = 0; i < record.size(); i++) { 158 | arr[i] = record.get(i); 159 | if (arr[i] != null) { 160 | if ((arr[i].equals("\\N") || arr[i].equalsIgnoreCase("null"))) arr[i] = null; 161 | else arr[i] = escapeDoubleQuotes(arr[i]); 162 | } 163 | } 164 | return arr; 165 | } 166 | 167 | public static String escapeDoubleQuotes(String toFormat) { 168 | return toFormat.replaceAll("\"", "\\\\\""); 169 | } 170 | 171 | public static void debug(String message, 172 | Object... objects) { 173 | if (appLogger.isDebugEnabled()) 174 | appLogger.debug(message, objects); 175 | } 176 | 177 | public static void error(String message, 178 | Object... objects) { 179 | appLogger.error(message, objects); 180 | } 181 | 182 | public static void warn(String message, 183 | Object... objects) { 184 | appLogger.warn(message, objects); 185 | 186 | } 187 | 188 | public static void info(String message, 189 | Object... objects) { 190 | appLogger.info(message, objects); 191 | } 192 | 193 | public static void trace(String message, 194 | Object... objects) { 195 | appLogger.trace(message, objects); 196 | } 197 | 198 | public static double calculateRate(double count, 199 | Instant start, 200 | Instant end) { 201 | return count * Duration.ofSeconds(1).toMillis() / Duration.between(start, end).toMillis(); 202 | } 203 | 204 | public static String printDuration(Instant start, 205 | Instant end) { 206 | return Duration.between(start, end).toString() 207 | .substring(2) 208 | .replaceAll("(\\d[HMS])(?!$)", "$1 ") 209 | .toLowerCase(); 210 | } 211 | 212 | public static void setConstrainingAttributeConceptType(Configuration.Definition.Attribute[] attributes, TypeDBSession session) { 213 | for (Configuration.Definition.Attribute attribute : attributes) { 214 | attribute.setConceptValueType(session); 215 | } 216 | } 217 | 218 | public static String playerType(Configuration.Definition.Player player) { 219 | if (player.getMatch().getOwnerships() != null) { 220 | return "byAttribute"; 221 | } else if (player.getMatch().getAttribute() != null) { 222 | return "attribute"; 223 | } else if (player.getMatch().getPlayers() != null) { 224 | return "byPlayer"; 225 | } else { 226 | return ""; 227 | } 228 | } 229 | 230 | public static Integer getRowsPerCommit(Configuration dc, Configuration.Generator.GeneratorConfig config) { 231 | if (config != null) { 232 | return Objects.requireNonNullElseGet(config.getRowsPerCommit(), () -> dc.getGlobalConfig().getRowsPerCommit()); 233 | } else { 234 | return dc.getGlobalConfig().getRowsPerCommit(); 235 | } 236 | } 237 | 238 | public static Character getSeparator(Configuration dc, Configuration.Generator.GeneratorConfig config) { 239 | if (config != null) { 240 | return Objects.requireNonNullElseGet(config.getSeparator(), () -> dc.getGlobalConfig().getSeparator()); 241 | } else { 242 | return dc.getGlobalConfig().getSeparator(); 243 | } 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/cli/TypeDBLoaderCLITest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.cli; 18 | 19 | import com.vaticle.typedb.driver.api.TypeDBDriver; 20 | import com.vaticle.typedb.driver.api.TypeDBSession; 21 | import com.vaticle.typedb.driver.api.TypeDBTransaction; 22 | import com.vaticle.typedb.osi.loader.loader.TypeDBLoader; 23 | import com.vaticle.typedb.osi.loader.util.TypeDBUtil; 24 | import com.vaticle.typeql.lang.TypeQL; 25 | import org.junit.Test; 26 | 27 | import static org.junit.Assert.assertEquals; 28 | import static org.junit.Assert.assertFalse; 29 | import static org.junit.Assert.assertTrue; 30 | 31 | public class TypeDBLoaderCLITest { 32 | 33 | @Test 34 | public void migrateTest() { 35 | String config = "src/test/resources/phoneCalls/config.json"; 36 | String database = "typedb_loader_cli_test"; 37 | String uri = "127.0.0.1:1729"; 38 | String[] args = { 39 | "load", 40 | "-c", config, 41 | "-db", database, 42 | "-tdb", uri, 43 | "-cm" 44 | }; 45 | 46 | LoadOptions options = LoadOptions.parse(args); 47 | TypeDBLoader loader = new TypeDBLoader(options); 48 | loader.load(); 49 | 50 | assertEquals(options.dataConfigFilePath, config); 51 | assertEquals(options.databaseName, database); 52 | assertEquals(options.typedbURI, uri); 53 | assertTrue(options.cleanMigration); 54 | assertFalse(options.loadSchema); 55 | } 56 | 57 | @Test 58 | public void migrateTestContinue() { 59 | String config = "src/test/resources/phoneCalls/config.json"; 60 | String db = "typedb_loader_cli_test"; 61 | String uri = "127.0.0.1:1729"; 62 | String[] cleanLoadArgs = { 63 | "load", 64 | "-c", config, 65 | "-db", db, 66 | "-tdb", uri, 67 | "-cm" 68 | }; 69 | 70 | LoadOptions cleanLoadOptions = LoadOptions.parse(cleanLoadArgs); 71 | assertEquals(cleanLoadOptions.dataConfigFilePath, config); 72 | assertEquals(cleanLoadOptions.databaseName, db); 73 | assertEquals(cleanLoadOptions.typedbURI, uri); 74 | assertTrue(cleanLoadOptions.cleanMigration); 75 | assertFalse(cleanLoadOptions.loadSchema); 76 | 77 | // run import once 78 | new TypeDBLoader(cleanLoadOptions).load(); 79 | // delete all the data 80 | clearData(uri, db); 81 | 82 | String[] continueArgs = { 83 | "load", 84 | "-c", config, 85 | "-db", db, 86 | "-tdb", uri 87 | }; 88 | LoadOptions continueLoadOptions = LoadOptions.parse(continueArgs); 89 | assertEquals(continueLoadOptions.dataConfigFilePath, config); 90 | assertEquals(continueLoadOptions.databaseName, db); 91 | assertEquals(continueLoadOptions.typedbURI, uri); 92 | assertFalse(continueLoadOptions.cleanMigration); 93 | assertFalse(continueLoadOptions.loadSchema); 94 | 95 | // load all data and schema again 96 | new TypeDBLoader(continueLoadOptions).load(); 97 | 98 | String[] continueLoadSchema = { 99 | "load", 100 | "-c", config, 101 | "-db", db, 102 | "-tdb", uri, 103 | "--loadSchema" 104 | }; 105 | LoadOptions continueLoadSchemaOptions = LoadOptions.parse(continueLoadSchema); 106 | assertEquals(continueLoadSchemaOptions.dataConfigFilePath, config); 107 | assertEquals(continueLoadSchemaOptions.databaseName, db); 108 | assertEquals(continueLoadSchemaOptions.typedbURI, uri); 109 | assertFalse(continueLoadSchemaOptions.cleanMigration); 110 | assertTrue(continueLoadSchemaOptions.loadSchema); 111 | 112 | // load all data and schema again 113 | new TypeDBLoader(continueLoadSchemaOptions).load(); 114 | } 115 | 116 | private void clearData(String uri, String db) { 117 | System.out.println("Cleaning all previous loaded data in: " + db); 118 | TypeDBDriver driver = TypeDBUtil.getCoreDriver(uri); 119 | try (TypeDBSession session = TypeDBUtil.getDataSession(driver, db)) { 120 | try (TypeDBTransaction txn = session.transaction(TypeDBTransaction.Type.WRITE)) { 121 | txn.query().delete(TypeQL.parseQuery("match $x isa thing; delete $x isa thing;").asDelete()); 122 | txn.commit(); 123 | } 124 | } 125 | driver.close(); 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/config/ConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.config; 18 | 19 | import com.vaticle.typedb.osi.loader.util.Util; 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | import java.io.File; 24 | 25 | public class ConfigurationTest { 26 | 27 | @Test 28 | public void dcPhoneCallsTest() { 29 | Configuration config = Util.initializeConfig(new File("src/test/resources/phoneCalls/config.json").getAbsolutePath()); 30 | //general 31 | Assert.assertNotNull(config); 32 | 33 | //default config 34 | Assert.assertEquals(',', config.getGlobalConfig().getSeparator().charValue()); 35 | 36 | //attributes 37 | String att = "is-in-use"; 38 | Assert.assertEquals("src/test/resources/phoneCalls/is-in-use.csv", config.getAttributes().get(att).getData()[0]); 39 | Assert.assertEquals("values", config.getAttributes().get(att).getInsert().getColumn()); 40 | Assert.assertEquals("is-in-use", config.getAttributes().get(att).getInsert().getAttribute()); 41 | Assert.assertEquals(',', config.getAttributes().get(att).getConfig().getSeparator().charValue()); 42 | Assert.assertNull(config.getAttributes().get(att).getInsert().getConceptValueType()); 43 | Assert.assertNull(config.getAttributes().get(att).getInsert().getListSeparator()); 44 | Assert.assertNull(config.getAttributes().get(att).getInsert().getPreprocessorConfig()); 45 | Assert.assertEquals(50, config.getAttributes().get(att).getConfig().getRowsPerCommit().intValue()); 46 | 47 | //entities 48 | String entity = "person"; 49 | Assert.assertEquals("src/test/resources/phoneCalls/person.csv", config.getEntities().get(entity).getData()[0]); 50 | Assert.assertEquals("src/test/resources/phoneCalls/person_nextfile.csv", config.getEntities().get(entity).getData()[1]); 51 | Assert.assertEquals(',', config.getEntities().get(entity).getConfig().getSeparator().charValue()); 52 | Assert.assertEquals("person", config.getEntities().get(entity).getInsert().getEntity()); 53 | Assert.assertEquals(50, config.getEntities().get(entity).getConfig().getRowsPerCommit().intValue()); 54 | Configuration.Definition.Attribute[] attributes = config.getEntities().get(entity).getInsert().getOwnerships(); 55 | Assert.assertEquals("first-name", attributes[0].getAttribute()); 56 | Assert.assertNull(attributes[0].getConceptValueType()); 57 | Assert.assertEquals("first_name", attributes[0].getColumn()); 58 | Assert.assertFalse(attributes[0].getRequired()); 59 | Assert.assertNull(attributes[0].getListSeparator()); 60 | Assert.assertNull(attributes[0].getPreprocessorConfig()); 61 | Assert.assertTrue(attributes[2].getRequired()); 62 | Assert.assertEquals(";", attributes[5].getListSeparator()); 63 | 64 | //entity-relations 65 | String relation = "contract"; 66 | Assert.assertEquals("src/test/resources/phoneCalls/contract.csv.gz", config.getRelations().get(relation).getData()[0]); 67 | Assert.assertEquals(',', config.getRelations().get(relation).getConfig().getSeparator().charValue()); 68 | Assert.assertEquals("contract", config.getRelations().get(relation).getInsert().getRelation()); 69 | Assert.assertEquals(50, config.getRelations().get(relation).getConfig().getRowsPerCommit().intValue()); 70 | Assert.assertNull(config.getRelations().get(relation).getInsert().getOwnerships()); 71 | 72 | Configuration.Definition.Player[] players = config.getRelations().get(relation).getInsert().getPlayers(); 73 | Configuration.Definition.Player player = players[0]; 74 | Assert.assertEquals("provider", player.getRole()); 75 | Assert.assertTrue(player.getRequired()); 76 | Configuration.Definition.Thing roleMatch = player.getMatch(); 77 | Assert.assertEquals("company", roleMatch.getType()); 78 | Configuration.Definition.Attribute consAtt = roleMatch.getOwnerships()[0]; 79 | Assert.assertEquals("name", consAtt.getAttribute()); 80 | Assert.assertEquals("company_name", consAtt.getColumn()); 81 | Assert.assertNull(consAtt.getListSeparator()); 82 | Assert.assertTrue(consAtt.getRequired()); 83 | 84 | player = players[1]; 85 | Assert.assertEquals("customer", player.getRole()); 86 | Assert.assertTrue(player.getRequired()); 87 | roleMatch = player.getMatch(); 88 | Assert.assertEquals("person", roleMatch.getType()); 89 | consAtt = player.getMatch().getOwnerships()[0]; 90 | Assert.assertEquals("phone-number", consAtt.getAttribute()); 91 | Assert.assertEquals("person_id", consAtt.getColumn()); 92 | Assert.assertEquals("###", consAtt.getListSeparator()); 93 | 94 | relation = "call"; 95 | Assert.assertEquals("src/test/resources/phoneCalls/call.csv", config.getRelations().get(relation).getData()[0]); 96 | Assert.assertEquals(',', config.getRelations().get(relation).getConfig().getSeparator().charValue()); 97 | Assert.assertEquals(50, config.getRelations().get(relation).getConfig().getRowsPerCommit().intValue()); 98 | Assert.assertEquals("call", config.getRelations().get(relation).getInsert().getRelation()); 99 | 100 | players = config.getRelations().get(relation).getInsert().getPlayers(); 101 | player = players[0]; 102 | Assert.assertEquals("caller", player.getRole()); 103 | Assert.assertTrue(player.getRequired()); 104 | roleMatch = player.getMatch(); 105 | Assert.assertEquals("person", roleMatch.getType()); 106 | consAtt = player.getMatch().getOwnerships()[0]; 107 | Assert.assertEquals("phone-number", consAtt.getAttribute()); 108 | Assert.assertEquals("caller_id", consAtt.getColumn()); 109 | Assert.assertNull(consAtt.getListSeparator()); 110 | 111 | player = players[1]; 112 | Assert.assertEquals("callee", player.getRole()); 113 | Assert.assertTrue(player.getRequired()); 114 | roleMatch = player.getMatch(); 115 | Assert.assertEquals("person", roleMatch.getType()); 116 | consAtt = player.getMatch().getOwnerships()[0]; 117 | Assert.assertEquals("phone-number", consAtt.getAttribute()); 118 | Assert.assertEquals("callee_id", consAtt.getColumn()); 119 | Assert.assertNull(consAtt.getListSeparator()); 120 | 121 | attributes = config.getRelations().get(relation).getInsert().getOwnerships(); 122 | Configuration.Definition.Attribute attribute = attributes[0]; 123 | Assert.assertEquals("started-at", attribute.getAttribute()); 124 | Assert.assertEquals("started_at", attribute.getColumn()); 125 | Assert.assertTrue(attribute.getRequired()); 126 | Assert.assertNull(attribute.getPreprocessorConfig()); 127 | Assert.assertNull(attribute.getListSeparator()); 128 | 129 | attribute = attributes[1]; 130 | Assert.assertEquals("duration", attribute.getAttribute()); 131 | Assert.assertEquals("duration", attribute.getColumn()); 132 | Assert.assertTrue(attribute.getRequired()); 133 | Assert.assertNull(attribute.getPreprocessorConfig()); 134 | Assert.assertNull(attribute.getListSeparator()); 135 | 136 | //Nested-Relation by Attribute: 137 | relation = "communication-channel"; 138 | Assert.assertEquals("src/test/resources/phoneCalls/communication-channel.csv", config.getRelations().get(relation).getData()[0]); 139 | Assert.assertEquals(',', config.getRelations().get(relation).getConfig().getSeparator().charValue()); 140 | Assert.assertEquals(50, config.getRelations().get(relation).getConfig().getRowsPerCommit().intValue()); 141 | Assert.assertEquals("communication-channel", config.getRelations().get(relation).getInsert().getRelation()); 142 | 143 | players = config.getRelations().get(relation).getInsert().getPlayers(); 144 | player = players[0]; 145 | Assert.assertEquals("peer", player.getRole()); 146 | Assert.assertTrue(player.getRequired()); 147 | roleMatch = player.getMatch(); 148 | Assert.assertEquals("person", roleMatch.getType()); 149 | consAtt = roleMatch.getOwnerships()[0]; 150 | Assert.assertEquals("phone-number", consAtt.getAttribute()); 151 | Assert.assertEquals("peer_1", consAtt.getColumn()); 152 | Assert.assertNull(consAtt.getListSeparator()); 153 | 154 | player = players[1]; 155 | Assert.assertEquals("peer", player.getRole()); 156 | Assert.assertTrue(player.getRequired()); 157 | roleMatch = player.getMatch(); 158 | Assert.assertEquals("person", roleMatch.getType()); 159 | consAtt = roleMatch.getOwnerships()[0]; 160 | Assert.assertEquals("phone-number", consAtt.getAttribute()); 161 | Assert.assertEquals("peer_2", consAtt.getColumn()); 162 | Assert.assertNull(consAtt.getListSeparator()); 163 | 164 | player = players[2]; 165 | Assert.assertEquals("past-call", player.getRole()); 166 | Assert.assertTrue(player.getRequired()); 167 | roleMatch = player.getMatch(); 168 | Assert.assertEquals("call", roleMatch.getType()); 169 | consAtt = roleMatch.getOwnerships()[0]; 170 | Assert.assertEquals("started-at", consAtt.getAttribute()); 171 | Assert.assertEquals("call_started_at", consAtt.getColumn()); 172 | Assert.assertEquals("###", consAtt.getListSeparator()); 173 | 174 | //Nested-Relation by Players: 175 | relation = "communication-channel-pm"; 176 | Assert.assertEquals("src/test/resources/phoneCalls/communication-channel-pm.csv", config.getRelations().get(relation).getData()[0]); 177 | Assert.assertEquals(',', config.getRelations().get(relation).getConfig().getSeparator().charValue()); 178 | Assert.assertEquals(50, config.getRelations().get(relation).getConfig().getRowsPerCommit().intValue()); 179 | Assert.assertEquals("communication-channel", config.getRelations().get(relation).getInsert().getRelation()); 180 | 181 | players = config.getRelations().get(relation).getInsert().getPlayers(); 182 | player = players[0]; 183 | Assert.assertEquals("peer", player.getRole()); 184 | Assert.assertTrue(player.getRequired()); 185 | roleMatch = player.getMatch(); 186 | Assert.assertEquals("person", roleMatch.getType()); 187 | consAtt = roleMatch.getOwnerships()[0]; 188 | Assert.assertEquals("phone-number", consAtt.getAttribute()); 189 | Assert.assertEquals("peer_1", consAtt.getColumn()); 190 | Assert.assertNull(consAtt.getListSeparator()); 191 | 192 | player = players[1]; 193 | Assert.assertEquals("peer", player.getRole()); 194 | Assert.assertTrue(player.getRequired()); 195 | roleMatch = player.getMatch(); 196 | Assert.assertEquals("person", roleMatch.getType()); 197 | consAtt = roleMatch.getOwnerships()[0]; 198 | Assert.assertEquals("phone-number", consAtt.getAttribute()); 199 | Assert.assertEquals("peer_2", consAtt.getColumn()); 200 | Assert.assertNull(consAtt.getListSeparator()); 201 | 202 | player = players[2]; 203 | Assert.assertEquals("past-call", player.getRole()); 204 | Assert.assertTrue(player.getRequired()); 205 | roleMatch = player.getMatch(); 206 | Assert.assertEquals("call", roleMatch.getType()); 207 | Configuration.Definition.Player[] playerPlayers = roleMatch.getPlayers(); 208 | Assert.assertEquals("caller", playerPlayers[0].getRole()); 209 | Assert.assertEquals("callee", playerPlayers[1].getRole()); 210 | Assert.assertEquals("person", playerPlayers[0].getMatch().getType()); 211 | Assert.assertEquals("person", playerPlayers[1].getMatch().getType()); 212 | Assert.assertEquals("peer_1", playerPlayers[0].getMatch().getOwnerships()[0].getColumn()); 213 | Assert.assertEquals("peer_2", playerPlayers[1].getMatch().getOwnerships()[0].getColumn()); 214 | 215 | //appendAttributes 216 | String appendAttribute = "append-twitter"; 217 | Assert.assertEquals("src/test/resources/phoneCalls/append-twitter-nickname.csv", config.getAppendAttribute().get(appendAttribute).getData()[0]); 218 | Assert.assertEquals(',', config.getAppendAttribute().get(appendAttribute).getConfig().getSeparator().charValue()); 219 | Assert.assertEquals(50, config.getAppendAttribute().get(appendAttribute).getConfig().getRowsPerCommit().intValue()); 220 | Assert.assertEquals(2, config.getAppendAttribute().get(appendAttribute).getInsert().getOwnerships().length); 221 | 222 | Assert.assertEquals("person", config.getAppendAttribute().get(appendAttribute).getMatch().getType()); 223 | Assert.assertEquals(1, config.getAppendAttribute().get(appendAttribute).getMatch().getOwnerships().length); 224 | 225 | Assert.assertEquals("phone-number", config.getAppendAttribute().get(appendAttribute).getMatch().getOwnerships()[0].getAttribute()); 226 | Assert.assertEquals("phone_number", config.getAppendAttribute().get(appendAttribute).getMatch().getOwnerships()[0].getColumn()); 227 | 228 | attributes = config.getAppendAttribute().get(appendAttribute).getInsert().getOwnerships(); 229 | Assert.assertEquals("twitter-username", attributes[0].getAttribute()); 230 | Assert.assertNull(attributes[0].getConceptValueType()); 231 | Assert.assertEquals("twitter", attributes[0].getColumn()); 232 | Assert.assertEquals("###", attributes[0].getListSeparator()); 233 | Assert.assertTrue(attributes[0].getRequired()); 234 | Assert.assertNull(attributes[0].getPreprocessorConfig()); 235 | 236 | } 237 | 238 | } 239 | -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/config/ConfigurationValidationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | package com.vaticle.typedb.osi.loader.config; 19 | 20 | import com.vaticle.typedb.driver.api.TypeDBDriver; 21 | import com.vaticle.typedb.driver.api.TypeDBSession; 22 | import com.vaticle.typedb.osi.loader.util.TypeDBUtil; 23 | import com.vaticle.typedb.osi.loader.util.Util; 24 | import org.apache.logging.log4j.LogManager; 25 | import org.apache.logging.log4j.Logger; 26 | import org.junit.Assert; 27 | import org.junit.Test; 28 | 29 | import java.io.File; 30 | import java.util.ArrayList; 31 | import java.util.HashMap; 32 | 33 | public class ConfigurationValidationTest { 34 | 35 | private static final Logger appLogger = LogManager.getLogger("com.bayer.dt.tdl.loader"); 36 | private static final String typedbURI = "localhost:1729"; 37 | private static final String databaseName = "config-validation-test"; 38 | 39 | @Test 40 | public void validateNoSchemaDataConfig() { 41 | Configuration dc = Util.initializeConfig(new File("src/test/resources/generic/configNoSchema.json").getAbsolutePath()); 42 | assert dc != null; 43 | 44 | ConfigurationValidation cv = new ConfigurationValidation(dc); 45 | 46 | HashMap> validationReport = new HashMap<>(); 47 | ArrayList errors = new ArrayList<>(); 48 | ArrayList warnings = new ArrayList<>(); 49 | validationReport.put("warnings", warnings); 50 | validationReport.put("errors", errors); 51 | cv.validateSchemaPresent(validationReport); 52 | 53 | validationReport.get("warnings").forEach(appLogger::warn); 54 | validationReport.get("errors").forEach(appLogger::error); 55 | 56 | Assert.assertTrue(validationReport.get("errors").stream().anyMatch(message -> message.equals( 57 | "defaultConfig.schema: missing required field" 58 | ))); 59 | } 60 | 61 | @Test 62 | public void validateSchemaFileNotFoundDataConfig() { 63 | Configuration dc = Util.initializeConfig(new File("src/test/resources/generic/configSchemaNotFound.json").getAbsolutePath()); 64 | assert dc != null; 65 | 66 | ConfigurationValidation cv = new ConfigurationValidation(dc); 67 | 68 | HashMap> validationReport = new HashMap<>(); 69 | ArrayList errors = new ArrayList<>(); 70 | ArrayList warnings = new ArrayList<>(); 71 | validationReport.put("warnings", warnings); 72 | validationReport.put("errors", errors); 73 | cv.validateSchemaPresent(validationReport); 74 | 75 | validationReport.get("warnings").forEach(appLogger::warn); 76 | validationReport.get("errors").forEach(appLogger::error); 77 | 78 | Assert.assertTrue(validationReport.get("errors").stream().anyMatch(message -> message.equals( 79 | "defaultConfig.schema - schema file not found under: " 80 | ))); 81 | } 82 | 83 | 84 | @Test 85 | public void validateErrorDataConfig() { 86 | Configuration dc = Util.initializeConfig(new File("src/test/resources/generic/configValidationTest.json").getAbsolutePath()); 87 | assert dc != null; 88 | 89 | TypeDBDriver schemaDriver = TypeDBUtil.getCoreDriver(typedbURI); 90 | 91 | ConfigurationValidation cv = new ConfigurationValidation(dc); 92 | 93 | HashMap> validationReport = new HashMap<>(); 94 | ArrayList errors = new ArrayList<>(); 95 | ArrayList warnings = new ArrayList<>(); 96 | validationReport.put("warnings", warnings); 97 | validationReport.put("errors", errors); 98 | 99 | cv.validateSchemaPresent(validationReport); 100 | if (validationReport.get("errors").size() == 0) { 101 | TypeDBUtil.cleanAndDefineSchemaToDatabase(schemaDriver, databaseName, dc.getGlobalConfig().getSchema()); 102 | appLogger.info("cleaned database and migrated schema..."); 103 | } 104 | TypeDBSession schemaSession = TypeDBUtil.getSchemaSession(schemaDriver, databaseName); 105 | cv.validateConfiguration(validationReport, schemaSession); 106 | schemaSession.close(); 107 | schemaDriver.close(); 108 | validationReport.get("warnings").forEach(appLogger::warn); 109 | validationReport.get("errors").forEach(appLogger::error); 110 | 111 | // overall 112 | Assert.assertEquals(2, validationReport.get("warnings").size()); 113 | Assert.assertEquals(10, validationReport.get("errors").size()); 114 | } 115 | 116 | @Test 117 | public void validateErrorDataConfigPhoneCalls() { 118 | Configuration dc = Util.initializeConfig(new File("src/test/resources/phoneCalls/configValidationTest.json").getAbsolutePath()); 119 | assert dc != null; 120 | 121 | TypeDBDriver schemaDriver = TypeDBUtil.getCoreDriver(typedbURI); 122 | 123 | ConfigurationValidation cv = new ConfigurationValidation(dc); 124 | 125 | HashMap> validationReport = new HashMap<>(); 126 | ArrayList errors = new ArrayList<>(); 127 | ArrayList warnings = new ArrayList<>(); 128 | validationReport.put("warnings", warnings); 129 | validationReport.put("errors", errors); 130 | 131 | cv.validateSchemaPresent(validationReport); 132 | if (validationReport.get("errors").size() == 0) { 133 | TypeDBUtil.cleanAndDefineSchemaToDatabase(schemaDriver, databaseName, dc.getGlobalConfig().getSchema()); 134 | appLogger.info("cleaned database and migrated schema..."); 135 | } 136 | TypeDBSession schemaSession = TypeDBUtil.getSchemaSession(schemaDriver, databaseName); 137 | cv.validateConfiguration(validationReport, schemaSession); 138 | schemaSession.close(); 139 | schemaDriver.close(); 140 | validationReport.get("warnings").forEach(appLogger::warn); 141 | validationReport.get("errors").forEach(appLogger::error); 142 | 143 | //TODO: cover all cases from synthetic, remove synthetic, and add all test cases for relations 144 | } 145 | 146 | @Test 147 | public void validateDataConfig() { 148 | Configuration dc = Util.initializeConfig(new File("src/test/resources/generic/config.json").getAbsolutePath()); 149 | assert dc != null; 150 | TypeDBDriver schemaDriver = TypeDBUtil.getCoreDriver(typedbURI); 151 | 152 | ConfigurationValidation cv = new ConfigurationValidation(dc); 153 | 154 | HashMap> validationReport = new HashMap<>(); 155 | ArrayList errors = new ArrayList<>(); 156 | ArrayList warnings = new ArrayList<>(); 157 | validationReport.put("warnings", warnings); 158 | validationReport.put("errors", errors); 159 | cv.validateSchemaPresent(validationReport); 160 | 161 | if (validationReport.get("errors").size() == 0) { 162 | TypeDBUtil.cleanAndDefineSchemaToDatabase(schemaDriver, databaseName, dc.getGlobalConfig().getSchema()); 163 | appLogger.info("cleaned database and migrated schema..."); 164 | } 165 | 166 | TypeDBSession schemaSession = TypeDBUtil.getSchemaSession(schemaDriver, databaseName); 167 | cv.validateConfiguration(validationReport, schemaSession); 168 | schemaSession.close(); 169 | schemaDriver.close(); 170 | 171 | validationReport.get("warnings").forEach(appLogger::warn); 172 | validationReport.get("errors").forEach(appLogger::error); 173 | Assert.assertEquals(1, validationReport.get("warnings").size()); 174 | Assert.assertEquals(0, validationReport.get("errors").size()); 175 | } 176 | 177 | @Test 178 | public void validatePhoneCallsDataConfig() { 179 | Configuration dc = Util.initializeConfig(new File("src/test/resources/phoneCalls/config.json").getAbsolutePath()); 180 | assert dc != null; 181 | 182 | TypeDBDriver schemaDriver = TypeDBUtil.getCoreDriver(typedbURI); 183 | ConfigurationValidation cv = new ConfigurationValidation(dc); 184 | 185 | HashMap> validationReport = new HashMap<>(); 186 | ArrayList errors = new ArrayList<>(); 187 | ArrayList warnings = new ArrayList<>(); 188 | validationReport.put("warnings", warnings); 189 | validationReport.put("errors", errors); 190 | cv.validateSchemaPresent(validationReport); 191 | if (validationReport.get("errors").size() == 0) { 192 | TypeDBUtil.cleanAndDefineSchemaToDatabase(schemaDriver, databaseName, dc.getGlobalConfig().getSchema()); 193 | appLogger.info("cleaned database and migrated schema..."); 194 | } 195 | 196 | TypeDBSession schemaSession = TypeDBUtil.getSchemaSession(schemaDriver, databaseName); 197 | cv.validateConfiguration(validationReport, schemaSession); 198 | schemaSession.close(); 199 | schemaDriver.close(); 200 | 201 | validationReport.get("warnings").forEach(appLogger::warn); 202 | validationReport.get("errors").forEach(appLogger::error); 203 | 204 | Assert.assertEquals(7, validationReport.get("warnings").size()); 205 | Assert.assertEquals(0, validationReport.get("errors").size()); 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/generator/AppendAttributeGeneratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.generator; 18 | 19 | import com.vaticle.typedb.driver.api.TypeDBDriver; 20 | import com.vaticle.typedb.driver.api.TypeDBSession; 21 | import com.vaticle.typedb.osi.loader.config.Configuration; 22 | import com.vaticle.typedb.osi.loader.util.TypeDBUtil; 23 | import com.vaticle.typedb.osi.loader.util.Util; 24 | import com.vaticle.typeql.lang.TypeQL; 25 | import com.vaticle.typeql.lang.query.TypeQLInsert; 26 | import org.junit.Assert; 27 | import org.junit.Test; 28 | 29 | import java.io.File; 30 | import java.io.IOException; 31 | import java.util.ArrayList; 32 | import java.util.Iterator; 33 | import java.util.List; 34 | import java.util.Objects; 35 | 36 | public class AppendAttributeGeneratorTest { 37 | 38 | @Test 39 | public void phoneCallsPersonTest() throws IOException { 40 | String dbName = "append-attribute-generator-test"; 41 | String sp = new File("src/test/resources/phoneCalls/schema.gql").getAbsolutePath(); 42 | TypeDBDriver driver = TypeDBUtil.getCoreDriver("localhost:1729"); 43 | TypeDBUtil.cleanAndDefineSchemaToDatabase(driver, dbName, sp); 44 | 45 | String dcp = new File("src/test/resources/phoneCalls/config.json").getAbsolutePath(); 46 | Configuration dc = Util.initializeConfig(dcp); 47 | assert dc != null; 48 | ArrayList appendKeys = new ArrayList<>(List.of("append-twitter", "append-fakebook", "append-call-rating")); 49 | TypeDBSession session = TypeDBUtil.getDataSession(driver, dbName); 50 | for (String appendkey : appendKeys) { 51 | Configuration.Definition.Attribute[] hasAttributes = dc.getAppendAttribute().get(appendkey).getInsert().getOwnerships(); 52 | if (hasAttributes != null) { 53 | Util.setConstrainingAttributeConceptType(hasAttributes, session); 54 | } 55 | if (dc.getAppendAttribute().get(appendkey).getMatch() != null && dc.getAppendAttribute().get(appendkey).getMatch().getOwnerships() != null) { 56 | Configuration.Definition.Attribute[] thingGetterAttributes = dc.getAppendAttribute().get(appendkey).getMatch().getOwnerships(); 57 | Util.setConstrainingAttributeConceptType(thingGetterAttributes, session); 58 | } 59 | } 60 | session.close(); 61 | driver.close(); 62 | 63 | testTwitter(dc, appendKeys); 64 | testFakebook(dc, appendKeys); 65 | testCallAppend(dc, appendKeys); 66 | } 67 | 68 | private void testTwitter(Configuration dc, ArrayList appendKeys) throws IOException { 69 | String dp = new File("src/test/resources/phoneCalls/append-twitter-nickname.csv").getAbsolutePath(); 70 | AppendAttributeGenerator gen = new AppendAttributeGenerator(dp, 71 | dc.getAppendAttribute().get(appendKeys.get(0)), 72 | Objects.requireNonNullElseGet(dc.getAppendAttribute().get(appendKeys.get(0)).getConfig().getSeparator(), () -> dc.getGlobalConfig().getSeparator())); 73 | Iterator iterator = Util.newBufferedReader(dp).lines().skip(1).iterator(); 74 | 75 | TypeQLInsert statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 76 | TypeQLInsert tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+7 171 898 0853\";\n" + 77 | "insert $thing has twitter-username \"@jojo\", has nick-name \"another\";").asInsert(); 78 | Assert.assertEquals(tmp, statement); 79 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 80 | 81 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 82 | tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+263 498 495 0617\";\n" + 83 | "insert $thing has twitter-username \"@hui\", has twitter-username \"@bui\", has nick-name \"yetanoter\";").asInsert(); 84 | Assert.assertEquals(tmp, statement); 85 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 86 | 87 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 88 | tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+370 351 224 5176\";\n" + 89 | "insert $thing has twitter-username \"@lalulix\", has nick-name \"one more\";").asInsert(); 90 | Assert.assertEquals(tmp, statement); 91 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 92 | 93 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 94 | tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+81 308 988 7153\";\n" + 95 | "insert $thing has twitter-username \"@go34\";").asInsert(); 96 | Assert.assertEquals(tmp, statement); 97 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 98 | 99 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 100 | tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+54 398 559 0423\";\n" + 101 | "insert $thing has twitter-username \"@hadaaa\";").asInsert(); 102 | Assert.assertEquals(tmp, statement); 103 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 104 | 105 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 106 | tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+7 690 597 4443\";\n" + 107 | "insert $thing has nick-name \"not inserted\";").asInsert(); 108 | Assert.assertEquals(tmp, statement); 109 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 110 | 111 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 112 | tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+63 815 962 6097\";\n" + 113 | "insert $thing has twitter-username \"@kuka\";").asInsert(); 114 | Assert.assertEquals(tmp, statement); 115 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 116 | 117 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 118 | tmp = TypeQL.parseQuery("insert $null isa null, has null \"null\";").asInsert(); 119 | Assert.assertEquals(tmp, statement); 120 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 121 | 122 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 123 | tmp = TypeQL.parseQuery("match $thing isa person;\n" + 124 | "insert $thing has twitter-username \"@notinserted\";").asInsert(); 125 | Assert.assertEquals(tmp, statement); 126 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 127 | 128 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 129 | tmp = TypeQL.parseQuery("insert $null isa null, has null \"null\";").asInsert(); 130 | Assert.assertEquals(tmp, statement); 131 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 132 | 133 | } 134 | 135 | private void testFakebook(Configuration dc, ArrayList appendKeys) throws IOException { 136 | String dp = new File("src/test/resources/phoneCalls/append-fb-preprocessed.csv").getAbsolutePath(); 137 | AppendAttributeGenerator gen = new AppendAttributeGenerator(dp, 138 | dc.getAppendAttribute().get(appendKeys.get(1)), 139 | Objects.requireNonNullElseGet(dc.getAppendAttribute().get(appendKeys.get(1)).getConfig().getSeparator(), () -> dc.getGlobalConfig().getSeparator())); 140 | Iterator iterator = Util.newBufferedReader(dp).lines().skip(1).iterator(); 141 | 142 | TypeQLInsert statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 143 | TypeQLInsert tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+36 318 105 5629\";\n" + 144 | "insert $thing has fakebook-link \"fakebook.com/personOne\";").asInsert(); 145 | Assert.assertEquals(tmp, statement); 146 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 147 | 148 | iterator.next(); 149 | iterator.next(); 150 | 151 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 152 | tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+62 533 266 3426\";\n" + 153 | "insert $thing has fakebook-link \"insertedWithoutAppliedRegex\";").asInsert(); 154 | Assert.assertEquals(tmp, statement); 155 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 156 | 157 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 158 | tmp = TypeQL.parseQuery("match $thing isa person;\n" + 159 | "insert $thing has fakebook-link \"@notinserted\";").asInsert(); 160 | Assert.assertEquals(tmp, statement); 161 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 162 | 163 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 164 | tmp = TypeQL.parseQuery("insert $null isa null, has null \"null\";").asInsert(); 165 | Assert.assertEquals(tmp, statement); 166 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 167 | } 168 | 169 | private void testCallAppend(Configuration dc, ArrayList appendKeys) throws IOException { 170 | String dp = new File("src/test/resources/phoneCalls/append-call-rating.csv").getAbsolutePath(); 171 | AppendAttributeGenerator gen = new AppendAttributeGenerator(dp, 172 | dc.getAppendAttribute().get(appendKeys.get(2)), 173 | Objects.requireNonNullElseGet(dc.getAppendAttribute().get(appendKeys.get(2)).getConfig().getSeparator(), () -> dc.getGlobalConfig().getSeparator())); 174 | Iterator iterator = Util.newBufferedReader(dp).lines().skip(1).iterator(); 175 | 176 | TypeQLInsert statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 177 | TypeQLInsert tmp = TypeQL.parseQuery("match $thing isa call, has started-at 2018-09-19T01:00:38;\n" + 178 | "insert $thing has call-rating 5;").asInsert(); 179 | Assert.assertEquals(tmp, statement); 180 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 181 | 182 | iterator.next(); 183 | iterator.next(); 184 | iterator.next(); 185 | iterator.next(); 186 | 187 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 188 | tmp = TypeQL.parseQuery("insert $null isa null, has null \"null\";").asInsert(); 189 | Assert.assertEquals(tmp, statement); 190 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 191 | 192 | statement = gen.generateMatchInsertStatement(Util.parseCSV(iterator.next())); 193 | tmp = TypeQL.parseQuery("match $thing isa call;\n" + 194 | "insert $thing has call-rating 4;").asInsert(); 195 | Assert.assertEquals(tmp, statement); 196 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/generator/AppendAttributeOrInsertThingGeneratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.generator; 18 | 19 | 20 | import com.vaticle.typedb.driver.api.TypeDBDriver; 21 | import com.vaticle.typedb.driver.api.TypeDBSession; 22 | import com.vaticle.typedb.osi.loader.config.Configuration; 23 | import com.vaticle.typedb.osi.loader.util.TypeDBUtil; 24 | import com.vaticle.typedb.osi.loader.util.Util; 25 | import com.vaticle.typeql.lang.TypeQL; 26 | import com.vaticle.typeql.lang.query.TypeQLInsert; 27 | import org.junit.Assert; 28 | import org.junit.Test; 29 | 30 | import java.io.File; 31 | import java.io.IOException; 32 | import java.util.ArrayList; 33 | import java.util.Iterator; 34 | import java.util.List; 35 | import java.util.Objects; 36 | 37 | public class AppendAttributeOrInsertThingGeneratorTest { 38 | 39 | @Test 40 | public void phoneCallsPersonTest() throws IOException { 41 | String dbName = "append-or-insert-generator-test"; 42 | String sp = new File("src/test/resources/phoneCalls/schema.gql").getAbsolutePath(); 43 | TypeDBDriver driver = TypeDBUtil.getCoreDriver("localhost:1729"); 44 | TypeDBUtil.cleanAndDefineSchemaToDatabase(driver, dbName, sp); 45 | 46 | String dcp = new File("src/test/resources/phoneCalls/config.json").getAbsolutePath(); 47 | Configuration dc = Util.initializeConfig(dcp); 48 | assert dc != null; 49 | ArrayList appendOrInsertKeys = new ArrayList<>(List.of("append-or-insert-person")); 50 | TypeDBSession session = TypeDBUtil.getDataSession(driver, dbName); 51 | for (String appendOrInsertkey : appendOrInsertKeys) { 52 | if (dc.getAppendAttributeOrInsertThing().get(appendOrInsertkey).getInsert().getOwnerships() != null) { 53 | Util.setConstrainingAttributeConceptType(dc.getAppendAttributeOrInsertThing().get(appendOrInsertkey).getInsert().getOwnerships(), session); 54 | } 55 | if (dc.getAppendAttributeOrInsertThing().get(appendOrInsertkey).getMatch() != null && dc.getAppendAttributeOrInsertThing().get(appendOrInsertkey).getMatch().getOwnerships() != null) { 56 | Util.setConstrainingAttributeConceptType(dc.getAppendAttributeOrInsertThing().get(appendOrInsertkey).getMatch().getOwnerships(), session); 57 | } 58 | } 59 | session.close(); 60 | driver.close(); 61 | 62 | testPerson(dc, appendOrInsertKeys); 63 | } 64 | 65 | private void testPerson(Configuration dc, ArrayList appendKeys) throws IOException { 66 | String dp = new File("src/test/resources/phoneCalls/person-append-or-insert.csv").getAbsolutePath(); 67 | AppendAttributeOrInsertThingGenerator gen = new AppendAttributeOrInsertThingGenerator(dp, 68 | dc.getAppendAttributeOrInsertThing().get(appendKeys.get(0)), 69 | Objects.requireNonNullElseGet(dc.getAppendAttributeOrInsertThing().get(appendKeys.get(0)).getConfig().getSeparator(), () -> dc.getGlobalConfig().getSeparator())); 70 | Iterator iterator = Util.newBufferedReader(dp).lines().skip(1).iterator(); 71 | 72 | String[] row = Util.parseCSV(iterator.next()); 73 | TypeQLInsert statement = gen.generateMatchInsertStatement(row); 74 | TypeQLInsert tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+7 171 898 0853\";\n" + 75 | "insert $thing has first-name \"Melli\", has last-name \"Winchcum\", has city \"London\", has age 55, has nick-name \"Mel\";").asInsert(); 76 | Assert.assertEquals(tmp, statement); 77 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 78 | statement = gen.generateThingInsertStatement(row); 79 | tmp = TypeQL.parseQuery("insert $e isa person, has phone-number \"+7 171 898 0853\", has first-name \"Melli\", has last-name \"Winchcum\", has city \"London\", has age 55, has nick-name \"Mel\";").asInsert(); 80 | Assert.assertEquals(tmp, statement); 81 | Assert.assertTrue(gen.thingInsertStatementValid(statement)); 82 | 83 | row = Util.parseCSV(iterator.next()); 84 | statement = gen.generateMatchInsertStatement(row); 85 | tmp = TypeQL.parseQuery("match $thing isa person;\n" + 86 | "insert $thing has first-name \"Sakura\", has city \"Fire Village\", has age 13;").asInsert(); 87 | Assert.assertEquals(tmp, statement); 88 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 89 | statement = gen.generateThingInsertStatement(row); 90 | tmp = TypeQL.parseQuery("insert $e isa person, has first-name \"Sakura\", has city \"Fire Village\", has age 13;").asInsert(); 91 | Assert.assertEquals(tmp, statement); 92 | Assert.assertTrue(gen.thingInsertStatementValid(statement)); 93 | 94 | iterator.next(); 95 | 96 | row = Util.parseCSV(iterator.next()); 97 | statement = gen.generateMatchInsertStatement(row); 98 | tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+62 107 666 3334\";\n" + 99 | "insert $thing has first-name \"Sasuke\", has city \"Fire Village\", has age 13;").asInsert(); 100 | Assert.assertEquals(tmp, statement); 101 | Assert.assertTrue(gen.appendAttributeInsertStatementValid(statement)); 102 | statement = gen.generateThingInsertStatement(row); 103 | tmp = TypeQL.parseQuery("insert $e isa person, has phone-number \"+62 107 666 3334\", has first-name \"Sasuke\", has city \"Fire Village\", has age 13;").asInsert(); 104 | Assert.assertEquals(tmp, statement); 105 | Assert.assertTrue(gen.thingInsertStatementValid(statement)); 106 | 107 | iterator.next(); 108 | iterator.next(); 109 | 110 | row = Util.parseCSV(iterator.next()); 111 | statement = gen.generateMatchInsertStatement(row); 112 | tmp = TypeQL.parseQuery("match $thing isa person, has phone-number \"+62 107 321 3333\";\n" + 113 | "insert $thing has first-name \"Missing\", has last-name \"Age\", has city \"notinsertcity\", has nick-name \"notinsertnickname\";").asInsert(); 114 | Assert.assertEquals(tmp, statement); 115 | Assert.assertFalse(gen.appendAttributeInsertStatementValid(statement)); 116 | statement = gen.generateThingInsertStatement(row); 117 | tmp = TypeQL.parseQuery("insert $e isa person, has phone-number \"+62 107 321 3333\", has first-name \"Missing\", has last-name \"Age\", has city \"notinsertcity\", has nick-name \"notinsertnickname\";").asInsert(); 118 | Assert.assertEquals(tmp, statement); 119 | Assert.assertFalse(gen.thingInsertStatementValid(statement)); 120 | 121 | 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/generator/AttributeGeneratorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.generator; 18 | 19 | import com.vaticle.typedb.driver.api.TypeDBDriver; 20 | import com.vaticle.typedb.driver.api.TypeDBSession; 21 | import com.vaticle.typedb.osi.loader.config.Configuration; 22 | import com.vaticle.typedb.osi.loader.util.TypeDBUtil; 23 | import com.vaticle.typedb.osi.loader.util.Util; 24 | import com.vaticle.typeql.lang.TypeQL; 25 | import com.vaticle.typeql.lang.query.TypeQLInsert; 26 | import org.junit.Assert; 27 | import org.junit.Test; 28 | 29 | import java.io.File; 30 | import java.io.IOException; 31 | import java.util.Iterator; 32 | import java.util.List; 33 | import java.util.Objects; 34 | 35 | public class AttributeGeneratorTest { 36 | 37 | @Test 38 | public void generateInsertStatementsTest() throws IOException { 39 | String dbName = "attribute-generator-test"; 40 | String sp = new File("src/test/resources/phoneCalls/schema.gql").getAbsolutePath(); 41 | TypeDBDriver driver = TypeDBUtil.getCoreDriver("localhost:1729"); 42 | TypeDBUtil.cleanAndDefineSchemaToDatabase(driver, dbName, sp); 43 | 44 | String dp = new File("src/test/resources/phoneCalls/is-in-use.csv").getAbsolutePath(); 45 | String dcp = new File("src/test/resources/phoneCalls/config.json").getAbsolutePath(); 46 | Configuration dc = Util.initializeConfig(dcp); 47 | assert dc != null; 48 | String attributeKey = "is-in-use"; 49 | AttributeGenerator gen = new AttributeGenerator(dp, 50 | dc.getAttributes().get(attributeKey), 51 | Objects.requireNonNullElseGet(dc.getAttributes().get(attributeKey).getConfig().getSeparator(), () -> dc.getGlobalConfig().getSeparator())); 52 | 53 | TypeDBSession session = TypeDBUtil.getDataSession(driver, dbName); 54 | dc.getAttributes().get(attributeKey).getInsert().setConceptValueType(session); 55 | session.close(); 56 | driver.close(); 57 | 58 | Iterator iterator = Util.newBufferedReader(dp).lines().skip(1).iterator(); 59 | 60 | String[] row = Util.parseCSV(iterator.next()); 61 | List insertStatements = gen.generateInsertStatements(row); 62 | TypeQLInsert tmp = TypeQL.parseQuery("insert $a \"yes\" isa is-in-use;").asInsert(); 63 | Assert.assertEquals(tmp, insertStatements.get(0)); 64 | 65 | row = Util.parseCSV(iterator.next()); 66 | insertStatements = gen.generateInsertStatements(row); 67 | tmp = TypeQL.parseQuery("insert $a \"no\" isa is-in-use;").asInsert(); 68 | Assert.assertEquals(tmp, insertStatements.get(0)); 69 | 70 | try { 71 | Util.parseCSV(iterator.next()); 72 | } catch (IndexOutOfBoundsException indexOutOfBoundsException) { 73 | Assert.assertEquals("Index 0 out of bounds for length 0", indexOutOfBoundsException.getMessage()); 74 | } 75 | 76 | row = Util.parseCSV(iterator.next()); 77 | insertStatements = gen.generateInsertStatements(row); 78 | tmp = TypeQL.parseQuery("insert $a \"5\" isa is-in-use;").asInsert(); 79 | Assert.assertEquals(tmp, insertStatements.get(0)); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/io/ErrorLoggerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.io; 18 | 19 | import org.junit.Test; 20 | 21 | public class ErrorLoggerTest { 22 | 23 | @Test 24 | public void errorFileLogger() { 25 | FileLogger logger = FileLogger.getLogger(); 26 | logger.logMalformed("entities.tsv", "shucks! There was a malformed row error"); 27 | logger.logUnavailable("entities.tsv", "shucks! There was a connection error"); 28 | logger.logInvalid("entities.tsv", "shucks! There was a invalid row error"); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/preprocessor/RegexPreprocessorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.preprocessor; 18 | 19 | import org.junit.Assert; 20 | import org.junit.Test; 21 | 22 | public class RegexPreprocessorTest { 23 | 24 | @Test 25 | public void regexPreprocessorTest() { 26 | RegexPreprocessor rpp = new RegexPreprocessor("^.*(fakebook\\.com.*)/$", "$1"); 27 | String test = "https://www.fakebook.com/personOne/"; 28 | String res = "fakebook.com/personOne"; 29 | Assert.assertEquals(res, rpp.applyProcessor(test)); 30 | 31 | test = "www.fakebook.com/personOne/"; 32 | res = "fakebook.com/personOne"; 33 | Assert.assertEquals(res, rpp.applyProcessor(test)); 34 | 35 | test = "fakebook.com/personOne/"; 36 | res = "fakebook.com/personOne"; 37 | Assert.assertEquals(res, rpp.applyProcessor(test)); 38 | 39 | test = "personOne/"; 40 | res = "personOne/"; 41 | Assert.assertEquals(res, rpp.applyProcessor(test)); 42 | 43 | test = "insertedWithoutAppliedRegex"; 44 | res = "insertedWithoutAppliedRegex"; 45 | Assert.assertEquals(res, rpp.applyProcessor(test)); 46 | 47 | test = ""; 48 | res = ""; 49 | Assert.assertEquals(res, rpp.applyProcessor(test)); 50 | } 51 | 52 | } -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/status/MigrationStatusTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.status; 18 | 19 | import org.junit.Test; 20 | 21 | import java.io.IOException; 22 | 23 | public class MigrationStatusTest { 24 | 25 | String typedbURI = "localhost:1729"; 26 | 27 | @Test 28 | public void migrationStatusTest() throws IOException { 29 | //TODO 30 | } 31 | } 32 | 33 | -------------------------------------------------------------------------------- /src/test/java/com/vaticle/typedb/osi/loader/util/QueryUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2021 Bayer AG 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.vaticle.typedb.osi.loader.util; 18 | 19 | import com.vaticle.typedb.driver.api.TypeDBSession; 20 | import com.vaticle.typedb.osi.loader.config.Configuration; 21 | 22 | import java.time.LocalDate; 23 | import java.time.LocalDateTime; 24 | import java.time.LocalTime; 25 | import java.time.format.DateTimeFormatter; 26 | 27 | public class QueryUtilTest { 28 | public static LocalDateTime getDT(String dtString) { 29 | DateTimeFormatter isoDateFormatter = DateTimeFormatter.ISO_DATE; 30 | String[] dt = dtString.split("T"); 31 | LocalDate date = LocalDate.parse(dt[0], isoDateFormatter); 32 | if (dt.length > 1) { 33 | LocalTime time = LocalTime.parse(dt[1], DateTimeFormatter.ISO_TIME); 34 | return date.atTime(time); 35 | } else { 36 | return date.atStartOfDay(); 37 | } 38 | } 39 | public static void setPlayerAttributeTypes(Configuration.Generator.Relation relation, int playerIndex, TypeDBSession session) { 40 | if (relation.getInsert().getPlayers()[playerIndex].getMatch() != null) { 41 | Configuration.Definition.Thing currentPlayerMatch = relation.getInsert().getPlayers()[playerIndex].getMatch(); 42 | if (currentPlayerMatch != null) { 43 | // if attribute player 44 | if (currentPlayerMatch.getAttribute() != null) { 45 | System.out.println("attribute player, index " + playerIndex + " " + relation.getInsert().getRelation()); 46 | currentPlayerMatch.getAttribute().setAttribute(currentPlayerMatch.getType()); 47 | currentPlayerMatch.getAttribute().setConceptValueType(session); 48 | } 49 | // if byAttribute player 50 | else if (currentPlayerMatch.getOwnerships() != null) { 51 | System.out.println("byAttribute player, index " + playerIndex + " " + relation.getInsert().getRelation()); 52 | for (Configuration.Definition.Attribute ownership : currentPlayerMatch.getOwnerships()) { 53 | ownership.setConceptValueType(session); 54 | } 55 | } 56 | } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/test/resources/bugfixing/issue10/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "globalConfig": { 3 | "separator": ',', 4 | "rowsPerCommit": 50, 5 | "parallelisation": 4, 6 | "schema": "src/test/resources/bugfixing.issue10/schema.gql" 7 | }, 8 | "entities": { 9 | "text": { 10 | "data": ["src/test/resources/bugfixing/issue10/text.csv"], 11 | "insert": { 12 | "entity": "text", 13 | "ownerships": [ 14 | { 15 | "attribute": "uid-name", 16 | "column": "uid" 17 | } 18 | ], 19 | } 20 | }, 21 | "label": { 22 | "data": ["src/test/resources/bugfixing/issue10/label.csv"], 23 | "insert": { 24 | "entity": "label", 25 | "ownerships": [ 26 | { 27 | "attribute": "name", 28 | "column": "name" 29 | } 30 | ], 31 | }, 32 | } 33 | }, 34 | "relations": { 35 | "tag": { 36 | "data": ["src/test/resources/bugfixing/issue10/tag.csv"], 37 | "insert": { 38 | "relation": "tag", 39 | "players": [ 40 | { 41 | "role": "tagger", 42 | "match": { 43 | "type": "label", 44 | "ownerships": [ 45 | { 46 | "attribute": "name", 47 | "column": "label_name" 48 | } 49 | ] 50 | } 51 | }, 52 | { 53 | "role": "tagged", 54 | "match": { 55 | "type": "text", 56 | "ownerships": [ 57 | { 58 | "attribute": "uid", 59 | "column": "text_id" 60 | } 61 | ] 62 | } 63 | } 64 | ] 65 | } 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /src/test/resources/bugfixing/issue10/label.csv: -------------------------------------------------------------------------------- 1 | name 2 | blue 3 | flower 4 | might -------------------------------------------------------------------------------- /src/test/resources/bugfixing/issue10/schema.gql: -------------------------------------------------------------------------------- 1 | define 2 | 3 | text sub entity, 4 | owns uid @key, 5 | plays tag:tagged; 6 | uid sub attribute, value long; 7 | 8 | label sub entity, 9 | owns name @key, 10 | plays tag:tagger; 11 | name sub attribute, value string; 12 | 13 | tag sub relation, 14 | relates tagger, 15 | relates tagged; -------------------------------------------------------------------------------- /src/test/resources/bugfixing/issue10/tag.csv: -------------------------------------------------------------------------------- 1 | label_name,text_id 2 | blue,28974 3 | flower,83682 4 | might,263684 -------------------------------------------------------------------------------- /src/test/resources/bugfixing/issue10/text.csv: -------------------------------------------------------------------------------- 1 | uid 2 | 28974 3 | 83682 4 | 263684 -------------------------------------------------------------------------------- /src/test/resources/generic/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "globalConfig": { 3 | "separator": ',', 4 | "rowsPerCommit": 100, 5 | "parallelisation": 4, 6 | "schema": "src/test/resources/generic/schema.gql" 7 | }, 8 | "entities": { 9 | "entity1": { 10 | "data": [ 11 | "src/test/resources/generic/entity1.tsv" 12 | ], 13 | "config": { 14 | "rowsPerCommit": 50, 15 | "separator": "\t" 16 | }, 17 | "insert": { 18 | "entity": "entity1", 19 | "ownerships": [ 20 | { 21 | "attribute": "entity1-id", 22 | "column": "entity1-id", 23 | "required": false 24 | }, 25 | { 26 | "attribute": "entity1-name", 27 | "column": "entity1-name", 28 | "required": false, 29 | "listSeparator": "###" 30 | }, 31 | { 32 | "attribute": "entity1-exp", 33 | "column": "entity1-exp", 34 | "required": true, 35 | "listSeparator": "###" 36 | } 37 | ] 38 | } 39 | }, 40 | "entity2": { 41 | "data": [ 42 | "src/test/resources/generic/entity2.tsv" 43 | ], 44 | "config": { 45 | "rowsPerCommit": 50, 46 | "separator": "\t" 47 | }, 48 | "insert": { 49 | "entity": "entity2", 50 | "ownerships": [ 51 | { 52 | "attribute": "entity2-id", 53 | "column": "entity2-id", 54 | "required": false 55 | }, 56 | { 57 | "attribute": "entity2-bool", 58 | "column": "entity2-bool", 59 | "required": false 60 | }, 61 | { 62 | "attribute": "entity2-double", 63 | "column": "entity2-double", 64 | "required": true, 65 | "listSeparator": "###" 66 | } 67 | ] 68 | } 69 | }, 70 | "entity3": { 71 | "data": [ 72 | "src/test/resources/generic/entity3.tsv" 73 | ], 74 | "config": { 75 | "rowsPerCommit": 50, 76 | "separator": "\t" 77 | }, 78 | "insert": { 79 | "entity": "entity3", 80 | "ownerships": [ 81 | { 82 | "attribute": "entity3-id", 83 | "column": "entity3-id", 84 | "required": false 85 | }, 86 | { 87 | "attribute": "entity3-int", 88 | "column": "entity3-int", 89 | "required": false, 90 | "listSeparator": "###" 91 | } 92 | ] 93 | } 94 | } 95 | }, 96 | "relations": { 97 | "rel1": { 98 | "data": [ 99 | "src/test/resources/generic/rel1.tsv" 100 | ], 101 | "config": { 102 | "rowsPerCommit": 50, 103 | "separator": "\t" 104 | }, 105 | "insert": { 106 | "relation": "rel1", 107 | "players": [ 108 | { 109 | "role": "player-one", 110 | "required": true, 111 | "match": { 112 | "type": "entity1", 113 | "ownerships": [ 114 | { 115 | "attribute": "entity1-id", 116 | "column": "entity1-id", 117 | "listSeparator": "###" 118 | } 119 | ] 120 | } 121 | }, 122 | { 123 | "role": "player-two", 124 | "required": true, 125 | "match": { 126 | "type": "entity2", 127 | "ownerships": [ 128 | { 129 | "attribute": "entity2-id", 130 | "column": "entity2-id" 131 | } 132 | ] 133 | } 134 | }, 135 | { 136 | "role": "player-optional", 137 | "required": false, 138 | "match": { 139 | "type": "entity3", 140 | "ownerships": [ 141 | { 142 | "attribute": "entity3-id", 143 | "column": "entity3-id" 144 | } 145 | ] 146 | } 147 | } 148 | ], 149 | "ownerships": [ 150 | { 151 | "attribute": "relAt-1", 152 | "column": "relAt-1", 153 | "required": true, 154 | "listSeparator": "###" 155 | }, 156 | { 157 | "attribute": "relAt-2", 158 | "column": "relAt-2", 159 | "required": false, 160 | "listSeparator": "###" 161 | } 162 | ] 163 | } 164 | } 165 | } 166 | } -------------------------------------------------------------------------------- /src/test/resources/generic/configNoSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "globalConfig": { 3 | "separator": ',', 4 | "rowsPerCommit": 200, 5 | "parallelisation": 64 6 | } 7 | } -------------------------------------------------------------------------------- /src/test/resources/generic/configSchemaNotFound.json: -------------------------------------------------------------------------------- 1 | { 2 | "globalConfig": { 3 | "separator": ',', 4 | "rowsPerCommit": 200, 5 | "parallelisation": 64, 6 | "schema": "src/test/resources/synthetic/schema-not-found.gql" 7 | } 8 | } -------------------------------------------------------------------------------- /src/test/resources/generic/configValidationTest.json: -------------------------------------------------------------------------------- 1 | { 2 | "globalConfig": { 3 | "separator": ',', 4 | "rowsPerCommit": 200, 5 | "parallelisation": 64, 6 | "schema": "src/test/resources/generic/schema.gql" 7 | }, 8 | "attributes": { 9 | "names": { 10 | "data": [ 11 | "src/test/resources/generic/names.csv" 12 | ], 13 | "insert": { 14 | "attribute": "name", 15 | "column": "doesnotexist" 16 | } 17 | }, 18 | "no-file": { 19 | "data": [ 20 | "src/test/resources/generic/nam.csv", 21 | "src/test/resources/generic/na.csv" 22 | ], 23 | "insert": { 24 | "column": "values", 25 | "attribute": "doesnotexist" 26 | } 27 | }, 28 | "empty-file": { 29 | "data": [ 30 | "src/test/resources/generic/empty.csv", 31 | "src/test/resources/generic/nam.csv" 32 | ], 33 | "insert": { 34 | "column": "values", 35 | "attribute": "doesnotexist" 36 | } 37 | } 38 | }, 39 | "entities": { 40 | "missing-attributes": { 41 | "data": [ 42 | "src/test/resources/generic/names.csv" 43 | ], 44 | "column": "name", 45 | "conceptType": "person" 46 | }, 47 | "person": { 48 | "data": [ 49 | "src/test/resources/generic/names.csv", 50 | "src/test/resources/generic/empty.csv", 51 | "src/test/resources/generic/notfound.csv", 52 | "src/test/resources/generic/notfound-other.csv" 53 | ], 54 | "config": { 55 | "separator": "," 56 | }, 57 | "insert": { 58 | "entity": "doesnotexist", 59 | "ownerships": [ 60 | { 61 | "attribute": "name", 62 | "column": "name", 63 | "required": false 64 | }, 65 | { 66 | "attribute": "doesnotexist", 67 | "column": "name", 68 | "required": false 69 | }, 70 | { 71 | "attribute": "name", 72 | "column": "doesnotexist", 73 | "required": false 74 | }, 75 | { 76 | "attribute": "name", 77 | "column": "name" 78 | }, 79 | { 80 | "column": "name", 81 | "required": false 82 | }, 83 | { 84 | "attribute": "name", 85 | "required": false 86 | } 87 | ] 88 | } 89 | } 90 | } 91 | } -------------------------------------------------------------------------------- /src/test/resources/generic/empty.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typedb-osi/typedb-loader/9a5fc166cbe8c26b4f2934a008b19991fccb62a1/src/test/resources/generic/empty.csv -------------------------------------------------------------------------------- /src/test/resources/generic/entity1.tsv: -------------------------------------------------------------------------------- 1 | entity1-id entity1-name entity1-exp 2 | entity1id0 entity1name0 entity1id0exp0 3 | entity1id1 entity1name1 entity1id1exp11###entity1id1exp12 4 | entity1id2 entity1name2 entity1id2exp21###entity1id2exp22###entity1id2exp23 5 | entity1id3 entity1name3 6 | entity1id4 "entity1name4" 7 | entity1id5 entity1name5 8 | entity1id6 entity1name6 9 | entity1id7 entity1name7 ### 10 | entity1id8 entity1name8 ### 11 | entity1id9 entity1name9 ### 12 | entity1id10 entity1name10 ### 13 | entity1id11 entity1name11### ### 14 | entity1id12 entity1name12### ### 15 | entity1id13 entity1name13### ### 16 | entity1id14 entity1name14### 17 | entity1id15 entity1name15### 18 | entity1id16 entity1name16###entity1name16-2 19 | entity1id17 entity1name17 20 | entity1id18 entity1name18 21 | entity1id19 entity1name19 22 | entity1name18 23 | entity1name19 24 | entity1name20 25 | entity1id21 26 | entity1id22 27 | entity1id23 28 | entity1id24 29 | entity1id25 ### 30 | entity1id26 ### 31 | entity1id27 ### 32 | entity1id28 ### 33 | -------------------------------------------------------------------------------- /src/test/resources/generic/entity2.tsv: -------------------------------------------------------------------------------- 1 | entity2-id entity2-bool entity2-double 2 | entity2id0 true 0.0 3 | entity2id1 false 1.1###11.11 4 | entity2id2 True 2.2### 5 | entity2id3 False ###-3.3 6 | entity2id4 4 7 | entity2id5 string 8 | entity2id6 somethingelse 9 | entity2id7 10 | entity2id8 ### 11 | entity2id9 ### 12 | entity2id10 ### 13 | 14 | -------------------------------------------------------------------------------- /src/test/resources/generic/entity3.tsv: -------------------------------------------------------------------------------- 1 | entity3-id entity3-int 2 | entity3id0 0 3 | entity3id1 1###11 4 | entity3id2 2### 5 | entity3id3 ###-3 6 | entity3id4 string 7 | entity3id5 0.1 8 | entity3id6 9 | entity3id7 10 | entity3id8 ### 11 | entity3id9 ### 12 | entity3id10 ### 13 | 14 | -------------------------------------------------------------------------------- /src/test/resources/generic/rel1.tsv: -------------------------------------------------------------------------------- 1 | entity1-id entity2-id entity3-id relAt-1 relAt-2 2 | entity1id1 entity2id1 entity3id1 att0###explosion0 opt0 3 | entity1id1 entity2id1 entity3id1 att1###explosion1###explo1 opt1 4 | entity1id1 entity2id1 entity3id1 att2 opt2 5 | entity1id1 entity2id1 entity3id1 att3### opt3 6 | entity1id1 entity2id1 entity3id1 att4### opt4 7 | entity1id1 entity2id1 entity3id1 att5### opt5 8 | entity1id1 entity2id1 entity3id1 ###att6 opt6 9 | entity1id1 entity2id1 entity3id1 ###att7 opt7 10 | entity1id1 entity2id1 entity3id1 ###att8 opt8 11 | entity1id1 entity2id1 entity3id1 ######att9 opt9 12 | entity1id1 entity2id1 entity3id1 att10 13 | entity1id1 entity2id1 att19 14 | entity1id1 entity2id1 att20 opt20 15 | entity1id1 entity2id1 att21###explosion21 optional21 16 | entity1id1 entity2id1 entity3id1 att22 17 | entity1id1 entity2id1 entity3id1 opt25 18 | entity1id1 entity2id1 entity3id1 19 | entity2id1 entity3id1 att34 opt33 20 | entity1id1 entity3id1 att37 opt36 21 | entity1id1###entity1id2 entity2id1 entity3id1 att39 opt39 22 | entity1id1###entity1id2 entity2id1 att40 23 | ### entity2id1 entity3id1 att41 opt41 24 | 25 | -------------------------------------------------------------------------------- /src/test/resources/generic/schema.gql: -------------------------------------------------------------------------------- 1 | define 2 | 3 | entity1 sub entity, 4 | owns entity1-id, 5 | owns entity1-name, 6 | owns entity1-exp, 7 | plays rel1:player-one; 8 | entity1-id sub attribute, value string; 9 | entity1-name sub attribute, value string; 10 | entity1-exp sub attribute, value string; 11 | 12 | entity2 sub entity, 13 | owns entity2-id, 14 | owns entity2-bool, 15 | owns entity2-double, 16 | plays rel1:player-two; 17 | entity2-id sub attribute, value string; 18 | entity2-bool sub attribute, value boolean; 19 | entity2-double sub attribute, value double; 20 | 21 | entity3 sub entity, 22 | owns entity3-id, 23 | owns entity3-int, 24 | plays rel1:player-optional; 25 | entity3-id sub attribute, value string; 26 | entity3-int sub attribute, value long; 27 | 28 | rel1 sub relation, 29 | relates player-one, 30 | relates player-two, 31 | relates player-optional, 32 | owns relAt-1, 33 | owns relAt-2; 34 | relAt-1 sub attribute, value string; 35 | relAt-2 sub attribute, value string; 36 | 37 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/append-call-rating.csv: -------------------------------------------------------------------------------- 1 | started_at,call_rating 2 | 2018-09-19T01:00:38,5 3 | 2018-09-24T03:16:48,5 4 | 2018-09-26T19:47:20,1 5 | 2018-09-26T23:47:19,2 6 | 2018-09-18T04:54:04,4 7 | 2018-09-19T02:11:53, 8 | ,4 9 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/append-fb-preprocessed.csv: -------------------------------------------------------------------------------- 1 | phone_number,fb 2 | +36 318 105 5629,https://www.fakebook.com/personOne/ 3 | +63 808 497 1769,https://www.fakebook.com/person-Two/ 4 | +62 533 266 3426,https://www.fakebook.com/person_three/ 5 | +62 533 266 3426,insertedWithoutAppliedRegex 6 | ,@notinserted 7 | , 8 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/append-twitter-nickname.csv: -------------------------------------------------------------------------------- 1 | phone_number,twitter,nick_name 2 | +7 171 898 0853,@jojo,another 3 | +263 498 495 0617,@hui###@bui,yetanoter 4 | +370 351 224 5176,@lalulix, one more 5 | +81 308 988 7153,@go34, 6 | +54 398 559 0423,@hadaaa, 7 | +7 690 597 4443,,not inserted 8 | +63 815 962 6097, @kuka ###, 9 | +81 746 154 2598,###, 10 | ,@notinserted, 11 | ,, 12 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/call.csv: -------------------------------------------------------------------------------- 1 | caller_id,callee_id,started_at,duration 2 | +54 398 559 0423,+48 195 624 2025,2018-09-16T22:24:19,122 3 | +54 398 559 0423,+48 195 624 2025,2018-09-17T22:24:19,123 4 | +54 398 559 0423,+48 195 624 2025,2018-09-18T22:24:19,124 5 | +54 398 559 0423,+48 195 624 2025,2018-09-19T22:24:19,125 6 | +54 398 559 0423,+48 195 624 2025,2018-09-20T22:24:19,126 7 | +263 498 495 0617,+48 195 624 2025,2018-09-18T01:34:48,514 8 | +81 308 988 7153,+33 614 339 0298,2018-09-21T20:21:17,120 9 | +263 498 495 0617,+33 614 339 0298,2018-09-11T22:10:34,144 10 | +263 498 495 0617,+33 614 339 0298,2018-09-12T22:10:34,145 11 | +263 498 495 0617,+33 614 339 0298,2018-09-13T22:10:34,146 12 | +263 498 495 0617,+33 614 339 0298,2018-09-14T22:10:34,147 13 | +263 498 495 0617,+33 614 339 0298,2018-09-15T22:10:34,148 14 | +263 498 495 0617,+33 614 339 0298,2018-09-16T22:10:34,149 15 | +54 398 559 0423,+7 552 196 4096,2018-09-25T20:24:59,556 16 | +81 308 988 7153,+351 515 605 7915,2018-09-23T22:23:25,336 17 | +81 308 988 7153,+351 515 605 7915,2018-09-24T22:23:25,332 18 | +81 308 988 7153,+351 515 605 7915,2018-09-25T22:23:25,331 19 | +261 860 539 4754,+351 272 414 6570,2018-09-26T05:34:19,5115 20 | +62 107 530 7500,+263 498 495 0617,2018-09-25T22:58:02,5665 21 | +54 398 559 0423,+86 892 682 0628,2018-09-23T08:55:18,822 22 | +7 690 597 4443,+54 398 559 0423,2018-09-25T09:10:25,8494 23 | +263 498 495 0617,+7 414 625 3019,2018-09-19T20:31:39,12 24 | +7 171 898 0853,+57 629 420 5680,2018-09-17T10:47:21,29 25 | +7 171 898 0853,+57 629 420 5680,2018-09-18T10:47:21,29 26 | +7 171 898 0853,+57 629 420 5680,2018-09-19T10:47:21,29 27 | +81 308 988 7153,+261 860 539 4754,2018-09-25T06:21:55,2851 28 | +263 498 495 0617,+48 195 624 2025,2018-09-26T03:37:06,573 29 | +81 308 988 7153,+1 254 875 4647,2018-09-19T08:19:36,66 30 | +370 351 224 5176,+63 815 962 6097,2018-09-25T19:44:03,3682 31 | +81 746 154 2598,+351 515 605 7915,2018-09-20T13:27:42,32 32 | +370 351 224 5176,+86 921 547 9004,2018-09-20T00:56:31,1434 33 | +81 746 154 2598,+7 414 625 3019,2018-09-18T18:47:17,166 34 | +62 107 530 7500,+86 202 257 8619,2018-09-22T17:27:52,112 35 | +81 308 988 7153,+351 515 605 7915,2018-09-16T03:38:09,1142 36 | +7 171 898 0853,+86 202 257 8619,2018-09-23T14:57:25,1665 37 | +81 308 988 7153,+86 922 760 0418,2018-09-27T05:08:53,365 38 | +263 498 495 0617,+36 318 105 5629,2018-09-16T01:44:31,96 39 | +7 690 597 4443,+63 808 497 1769,2018-09-20T12:27:48,766 40 | +81 746 154 2598,+27 117 258 4149,2018-09-27T11:28:11,710 41 | +261 860 539 4754,+48 195 624 2025,2018-09-18T23:24:30,151 42 | +7 171 898 0853,+86 892 682 0628,2018-09-26T14:04:33,5710 43 | +81 746 154 2598,+81 308 988 7153,2018-09-24T04:12:07,9923 44 | +81 308 988 7153,+261 860 539 4754,2018-09-21T22:54:31,4264 45 | +263 498 495 0617,+48 195 624 2025,2018-09-22T21:17:48,202 46 | +7 690 597 4443,+48 195 624 2025,2018-09-17T22:55:06,151 47 | +81 308 988 7153,+86 922 760 0418,2018-09-24T10:16:51,2895 48 | +7 690 597 4443,+63 808 497 1769,2018-09-23T15:37:45,251 49 | +54 398 559 0423,+86 921 547 9004,2018-09-25T11:34:35,139 50 | +370 351 224 5176,+351 272 414 6570,2018-09-22T03:13:47,140 51 | +261 860 539 4754,+62 107 530 7500,2018-09-16T19:18:32,3660 52 | +7 690 597 4443,+30 419 575 7546,2018-09-21T21:42:00,582 53 | +7 690 597 4443,+351 515 605 7915,2018-09-19T01:00:38,141 54 | +81 308 988 7153,+63 808 497 1769,2018-09-24T03:16:48,89 55 | +261 860 539 4754,+86 892 682 0628,2018-09-26T19:47:20,21 56 | +7 690 597 4443,+48 697 447 6933,2018-09-26T23:47:19,144 57 | +81 746 154 2598,+86 922 760 0418,2018-09-18T04:54:04,163 58 | +370 351 224 5176,+62 533 266 3426,2018-09-19T02:11:53,2681 59 | +63 815 962 6097,+36 318 105 5629,2018-09-23T14:14:42,492 60 | +62 107 530 7500,+351 272 414 6570,2018-09-27T04:00:59,384 61 | +81 308 988 7153,+7 414 625 3019,2018-09-17T05:58:16,2575 62 | +62 107 530 7500,+86 921 547 9004,2018-09-27T18:02:22,546 63 | +54 398 559 0423,+1 254 875 4647,2018-09-17T18:41:52,869 64 | +62 107 530 7500,+36 318 105 5629,2018-09-16T04:41:12,139 65 | +263 498 495 0617,+7 552 196 4096,2018-09-21T06:44:17,53 66 | +81 308 988 7153,+86 892 682 0628,2018-09-27T12:32:32,457 67 | +7 690 597 4443,+351 515 605 7915,2018-09-18T03:42:30,157 68 | +54 398 559 0423,+48 195 624 2025,2018-09-21T21:20:56,207 69 | +54 398 559 0423,+7 552 196 4096,2018-09-25T15:32:57,500 70 | +261 860 539 4754,+86 892 682 0628,2018-09-17T23:45:04,30 71 | +263 498 495 0617,+7 552 196 4096,2018-09-22T19:17:54,161 72 | +62 107 530 7500,+351 515 605 7915,2018-09-22T02:01:08,306 73 | +81 308 988 7153,+57 629 420 5680,2018-09-19T21:03:04,129 74 | +261 860 539 4754,+7 552 196 4096,2018-09-22T07:55:23,594 75 | +54 398 559 0423,+63 808 497 1769,2018-09-23T02:24:36,125 76 | +7 690 597 4443,+62 533 266 3426,2018-09-26T09:21:22,100 77 | +54 398 559 0423,+36 318 105 5629,2018-09-21T13:00:15,172 78 | +7 690 597 4443,+63 815 962 6097,2018-09-16T23:11:52,6789 79 | +62 107 530 7500,+48 894 777 5173,2018-09-19T07:41:23,66 80 | +62 107 530 7500,+7 552 196 4096,2018-09-22T17:26:29,950 81 | +63 815 962 6097,+7 414 625 3019,2018-09-16T23:28:04,144 82 | +62 107 530 7500,+7 171 898 0853,2018-09-27T17:33:06,4868 83 | +63 815 962 6097,+7 171 898 0853,2018-09-15T10:03:34,6298 84 | +370 351 224 5176,+63 808 497 1769,2018-09-16T20:26:23,2606 85 | +263 498 495 0617,+86 921 547 9004,2018-09-28T05:06:44,886 86 | +81 308 988 7153,+54 398 559 0423,2018-09-23T04:48:41,3458 87 | +261 860 539 4754,+27 117 258 4149,2018-09-26T22:32:19,609 88 | +263 498 495 0617,+351 515 605 7915,2018-09-25T03:50:51,68 89 | +81 308 988 7153,+351 272 414 6570,2018-09-23T09:20:33,212 90 | +370 351 224 5176,+86 892 682 0628,2018-09-17T16:52:29,156 91 | +7 690 597 4443,+351 515 605 7915,2018-09-18T09:21:38,60 92 | +261 860 539 4754,+63 808 497 1769,2018-09-16T22:11:35,681 93 | +63 815 962 6097,+7 552 196 4096,2018-09-21T15:12:44,124 94 | +63 815 962 6097,+86 892 682 0628,2018-09-22T05:39:03,124 95 | +54 398 559 0423,+48 195 624 2025,2018-09-23T18:08:42,163 96 | +370 351 224 5176,+1 254 875 4647,2018-09-25T13:06:40,45 97 | +63 815 962 6097,+62 107 530 7500,2018-09-24T05:05:43,3924 98 | +370 351 224 5176,+7 552 196 4096,2018-09-17T21:40:47,79 99 | +62 107 530 7500,+48 894 777 5173,2018-09-19T04:16:21,79 100 | +62 107 530 7500,+63 808 497 1769,2018-09-16T03:22:12,988 101 | +263 498 495 0617,+86 202 257 8619,2018-09-26T00:41:44,164 102 | +370 351 224 5176,+48 697 447 6933,2018-09-20T18:55:03,212 103 | +370 351 224 5176,+30 419 575 7546,2018-09-25T15:20:43,283 104 | +54 398 559 0423,+7 690 597 4443,2018-09-26T00:06:19,2357 105 | +81 746 154 2598,+351 272 414 6570,2018-09-22T13:27:01,157 106 | +81 308 988 7153,+1 254 875 4647,2018-09-18T07:53:09,295 107 | +7 171 898 0853,+81 746 154 2598,2018-09-17T07:13:25,9460 108 | +62 107 530 7500,+86 892 682 0628,2018-09-15T22:08:23,2308 109 | +370 351 224 5176,+86 892 682 0628,2018-09-23T02:24:15,1018 110 | +7 690 597 4443,+263 498 495 0617,2018-09-26T17:22:34,10499 111 | +54 398 559 0423,+48 195 624 2025,2018-09-16T18:17:49,47 112 | +63 815 962 6097,+63 808 497 1769,2018-09-17T07:29:13,1036 113 | +54 398 559 0423,+81 308 988 7153,2018-09-27T08:22:32,6468 114 | +370 351 224 5176,+33 614 339 0298,2018-09-17T01:58:04,77 115 | +63 815 962 6097,+263 498 495 0617,2018-09-19T23:16:49,lulu 116 | +81 746 154 2598,+1 254 875 4647,2018-09-15T07:02:32,914 117 | +62 107 530 7500,+81 308 988 7153,2018-09-22T04:57:20,4455 118 | +54 398 559 0423,+62 107 530 7500,2018-09-28T01:57:20,5272 119 | +261 860 539 4754,+36 318 105 5629,2018-09-14T17:18:49,1111 120 | +81 308 988 7153,+86 922 760 0418,2018-09-23T21:11:41,290 121 | +263 498 495 0617,+351 515 605 7915,2018-09-25T22:57:07,169 122 | +263 498 495 0617,+86 825 153 5518,2018-09-20T19:43:24,276 123 | +370 351 224 5176,+48 894 777 5173,2018-09-17T14:43:15,4 124 | +7 171 898 0853,+86 892 682 0628,2018-09-20T22:48:50,655 125 | +63 815 962 6097,+48 697 447 6933,2018-09-17T13:54:54,146 126 | +63 815 962 6097,+30 419 575 7546,2018-09-25T20:34:51,22 127 | +81 746 154 2598,+86 892 682 0628,2018-09-24T21:14:56,1177 128 | +62 107 530 7500,+7 690 597 4443,2018-09-20T21:01:36,6227 129 | +81 308 988 7153,+63 808 497 1769,2018-09-23T11:04:36,67 130 | +63 815 962 6097,+1 254 875 4647,2018-09-15T22:31:03,71 131 | +7 171 898 0853,+86 202 257 8619,2018-09-19T02:01:36,60 132 | +7 690 597 4443,+86 922 760 0418,2018-09-25T05:22:05,2544 133 | +7 690 597 4443,+62 107 530 7500,2018-09-17T18:51:01,7444 134 | +62 107 530 7500,+81 308 988 7153,2018-09-26T03:49:07,1696 135 | +81 308 988 7153,+54 398 559 0423,2018-09-22T15:58:58,9465 136 | +81 308 988 7153,+30 419 575 7546,2018-09-21T17:02:25,442 137 | +81 308 988 7153,+81 746 154 2598,2018-09-21T16:46:05,3928 138 | +263 498 495 0617,+7 690 597 4443,2018-09-24T08:11:36,10309 139 | +261 860 539 4754,+62 533 266 3426,2018-09-24T15:41:04,76 140 | +263 498 495 0617,+48 195 624 2025,2018-09-20T01:13:18,997 141 | +63 815 962 6097,+36 318 105 5629,2018-09-24T08:53:14,90 142 | +63 815 962 6097,+63 808 497 1769,2018-09-25T11:47:59,124 143 | +261 860 539 4754,+48 894 777 5173,2018-09-15T21:44:00,1122 144 | +63 815 962 6097,+86 892 682 0628,2018-09-21T17:24:29,520 145 | +7 171 898 0853,+62 107 530 7500,2018-09-23T12:10:49,3251 146 | +261 860 539 4754,+48 697 447 6933,2018-09-28T08:52:06,599 147 | +7 690 597 4443,+86 921 547 9004,2018-09-18T06:03:28,160 148 | +370 351 224 5176,+7 552 196 4096,2018-09-17T19:51:51,79 149 | +62 107 530 7500,+7 171 898 0853,2018-09-21T12:08:28,6538 150 | +81 308 988 7153,+86 921 547 9004,2018-09-21T02:19:22,508 151 | +62 107 530 7500,+7 552 196 4096,2018-09-23T00:35:41,809 152 | +54 398 559 0423,+86 892 682 0628,2018-09-16T15:16:47,2457 153 | +62 107 530 7500,+33 614 339 0298,2018-09-17T09:48:54,93 154 | +7 171 898 0853,+54 398 559 0423,2018-09-23T03:19:40,1259 155 | +263 498 495 0617,+81 746 154 2598,2018-09-26T08:08:28,6475 156 | +261 860 539 4754,+7 552 196 4096,2018-09-14T22:01:01,547 157 | +81 308 988 7153,+81 746 154 2598,2018-09-23T20:04:51,4941 158 | +263 498 495 0617,+7 552 196 4096,2018-09-16T03:35:49,134 159 | +81 308 988 7153,+7 171 898 0853,2018-09-20T11:38:53,9705 160 | +62 107 530 7500,+81 746 154 2598,2018-09-28T00:37:16,6945 161 | +62 107 530 7500,+86 921 547 9004,2018-09-18T23:25:20,3225 162 | +81 746 154 2598,+62 533 266 3426,2018-09-14T23:33:20,27 163 | +62 107 530 7500,+30 419 575 7546,2018-09-19T07:52:06,174 164 | +7 690 597 4443,+63 808 497 1769,2018-09-16T09:42:58,143 165 | +81 308 988 7153,+63 815 962 6097,2018-09-19T02:17:35,5413 166 | +62 107 530 7500,+81 308 988 7153,2018-09-26T06:19:03,3552 167 | +7 171 898 0853,+86 202 257 8619,2018-09-20T08:28:51,170 168 | +81 746 154 2598,+86 921 547 9004,2018-09-24T06:33:20,108 169 | +62 107 530 7500,+86 921 547 9004,2018-09-25T10:37:27,556 170 | +7 690 597 4443,+7 414 625 3019,2018-09-26T11:17:09,1051 171 | +54 398 559 0423,+86 921 547 9004,2018-09-21T21:49:34,802 172 | +7 690 597 4443,+62 533 266 3426,2018-09-23T11:41:10,1903 173 | +54 398 559 0423,+7 414 625 3019,2018-09-21T15:26:33,175 174 | +63 815 962 6097,+36 318 105 5629,2018-09-24T06:23:45,166 175 | +7 690 597 4443,+1 254 875 4647,2018-09-25T04:08:43,501 176 | +63 815 962 6097,+86 921 547 9004,2018-09-20T23:50:37,52 177 | +7 690 597 4443,+30 419 575 7546,2018-09-24T01:11:29,1959 178 | +63 815 962 6097,+48 697 447 6933,2018-09-17T18:43:42,29 179 | +62 107 530 7500,+86 922 760 0418,2018-09-19T19:07:10,3196 180 | +7 171 898 0853,+86 202 257 8619,2018-09-25T06:05:16,4362 181 | +62 107 530 7500,+63 815 962 6097,2018-09-17T16:57:41,4016 182 | +261 860 539 4754,+86 202 257 8619,2018-09-24T19:45:03,422 183 | +81 308 988 7153,+48 894 777 5173,2018-09-23T20:00:30,720 184 | +81 746 154 2598,+86 922 760 0418,2018-09-16T21:33:26,402 185 | +81 746 154 2598,+63 808 497 1769,2018-09-22T06:56:03,44 186 | +63 815 962 6097,+86 922 760 0418,2018-09-28T09:45:40,78 187 | +63 815 962 6097,+351 515 605 7915,2018-09-27T13:01:03,113 188 | +7 171 898 0853,+57 629 420 5680,2018-09-17T12:20:01,128 189 | +63 815 962 6097,+36 318 105 5629,2018-09-26T08:07:59,1171 190 | +81 308 988 7153,+48 697 447 6933,2018-09-20T19:41:35,126 191 | +7 690 597 4443,+7 552 196 4096,2018-09-20T01:48:26,176 192 | +261 860 539 4754,+7 690 597 4443,2018-09-14T23:40:40,4389 193 | +54 398 559 0423,+7 414 625 3019,2018-09-17T23:24:18,2391 194 | +63 815 962 6097,+86 922 760 0418,2018-09-15T04:32:48,44 195 | +81 308 988 7153,+86 825 153 5518,2018-09-28T14:57:53,160 196 | +81 308 988 7153,+63 815 962 6097,2018-09-16T07:34:42,9303 197 | +7 171 898 0853,+86 892 682 0628,2018-09-23T01:37:19,141 198 | +81 308 988 7153,+63 808 497 1769,2018-09-22T08:48:48,83 199 | +370 351 224 5176,+62 533 266 3426,2018-09-15T12:12:59,426 200 | +81 308 988 7153,+351 515 605 7915,2018-09-20T14:12:40,51 201 | +370 351 224 5176,+54 398 559 0423,2018-09-21T22:45:47,8861 202 | +370 351 224 5176,+33 614 339 0298,2018-09-23T22:18:08,162 203 | +81 308 988 7153,+27 117 258 4149,2018-09-25T00:26:05,69 204 | +263 498 495 0617,+62 533 266 3426,2018-09-20T03:03:46,12 205 | +62 107 530 7500,+86 892 682 0628,2018-09-26T14:12:03,68 206 | +81 746 154 2598,+48 195 624 2025,2018-09-26T21:20:35,136 207 | +54 398 559 0423,+86 921 547 9004,2018-09-24T09:38:52,80 208 | +63 815 962 6097,+1 254 875 4647,2018-09-20T12:02:37,9 209 | +81 308 988 7153,+48 894 777 5173,2018-09-28T06:12:49,156 210 | +370 351 224 5176,+351 515 605 7915,2018-09-16T22:31:25,543 211 | +7 690 597 4443,+48 697 447 6933,2018-09-19T00:12:47,132 212 | +81 746 154 2598,+351 515 605 7915,2018-09-19T08:10:14,76 213 | +54 398 559 0423,+81 746 154 2598,2018-09-18T22:47:52,5356 214 | +63 815 962 6097,+7 552 196 4096,2018-09-23T01:14:56,lala 215 | +63 815 962 6097,+7 552 196 4096,2018-09-23T01:14:56, 216 | +63 815 962 6097,+7 552 196 4096,,53 217 | +63 815 962 6097,,2018-09-23T01:14:56,53 218 | 219 | ,+7 552 196 4096,2018-09-23T01:14:56,53 220 | 221 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/communication-channel-pm.csv: -------------------------------------------------------------------------------- 1 | peer_1,peer_2 2 | +81 308 988 7153,+351 515 605 7915 3 | +7 171 898 0853,+57 629 420 5680 4 | +261 860 539 4754, 5 | , 6 | , 7 | 8 | ,+261 860 539 4754 9 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/communication-channel.csv: -------------------------------------------------------------------------------- 1 | peer_1,peer_2,call_started_at 2 | +54 398 559 0423,+48 195 624 2025,2018-09-16T22:24:19 3 | +263 498 495 0617,+33 614 339 0298,2018-09-11T22:10:34### 2018-09-12T22:10:34###2018-09-13T22:10:34 ###2018-09-14T22:10:34###2018-09-15T22:10:34###2018-09-16T22:10:34 4 | +263 498 495 0617,+33 614 339 0298,2018-09-11T22:10:34 5 | +370 351 224 5176,+62 533 266 3426,2018-09-15T12:12:59 6 | ,+62 533 266 3426,2018-09-15T12:12:59 7 | +370 351 224 5176,,2018-09-15T12:12:59 8 | ,,2018-09-15T12:12:59 9 | +7 690 597 4443,+54 398 559 9999, 10 | +7 690 597 4443,, 11 | ,+54 398 559 9999, 12 | 13 | ,, 14 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/company.csv: -------------------------------------------------------------------------------- 1 | name 2 | Telecom 3 | Unity 4 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/configValidationTest.json: -------------------------------------------------------------------------------- 1 | { 2 | "globalConfig": { 3 | "separator": ',', 4 | "rowsPerCommit": 200, 5 | "schema": "src/test/resources/phoneCalls/schema-updated.gql" 6 | }, 7 | relations: { 8 | 9 | } 10 | } -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/contract.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typedb-osi/typedb-loader/9a5fc166cbe8c26b4f2934a008b19991fccb62a1/src/test/resources/phoneCalls/contract.csv.gz -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/in-use.csv: -------------------------------------------------------------------------------- 1 | pn,in_use 2 | +7 171 898 0853,yes 3 | +370 351 224 5176,no 4 | +81 308 988 7153,yes 5 | +54 398 559 0423,yes 6 | +263 498 495 0617,yes 7 | 8 | +81 746 154 2598,no 9 | +63 815 962 6097,yes 10 | +62 107 530 7500, 11 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/is-in-use.csv: -------------------------------------------------------------------------------- 1 | values 2 | yes 3 | no 4 | 5 | 5 6 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/person-append-or-insert.csv: -------------------------------------------------------------------------------- 1 | first_name,last_name,phone_number,city,age,nick_name 2 | Melli,Winchcum,+7 171 898 0853,London,55,Mel 3 | Sakura,,,Fire Village,13, 4 | Xylina,D'Alesco,+7 690 597 4443,Cambridge,51,Xyl 5 | Sasuke,,+62 107 666 3334,Fire Village,13, 6 | Elenore,Stokey,+62 107 530 7500,Oxford,35,Elen 7 | Naruto,Uzamaki,+62 107 321 3333,Fire Village,12, 8 | Missing,Age,+62 107 321 3333,notinsertcity,,notinsertnickname 9 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/person.csv: -------------------------------------------------------------------------------- 1 | first_name,last_name,phone_number,city,age,nick_name 2 | Melli,Winchcum,+7 171 898 0853,London,55, 3 | Celinda,Bonick,+370 351 224 5176,London,52, 4 | Chryste,Lilywhite,+81 308 988 7153,London,66, 5 | D'arcy,Byfford,+54 398 559 0423,London,19,D 6 | Xylina,D'Alesco,+7 690 597 4443,Cambridge,51, 7 | Roldan,Cometti,+263 498 495 0617,Oxford,59,Rolly;Rolli 8 | Cob,Lafflin,+63 815 962 6097,Cambridge,56, 9 | Olag,Heakey,+81 746 154 2598,London,45, 10 | Mandie,Assender,+261 860 539 4754,London,18, 11 | Elenore,Stokey,+62 107 530 7500,Oxford,35, 12 | ,,+86 921 547 9004,,, 13 | ,,+48 894 777 5173,,, 14 | ,,+86 922 760 0418,,, 15 | ,,+33 614 339 0298,,, 16 | ,,+30 419 575 7546,,, 17 | ,,+7 414 625 3019,,, 18 | ,,+57 629 420 5680,,, 19 | ,,+351 515 605 7915,,, 20 | ,,+36 318 105 5629,,, 21 | ,,+63 808 497 1769,,, 22 | ,,+62 533 266 3426,,, 23 | ,,+351 272 414 6570,,, 24 | ,,+86 825 153 5518,,, 25 | ,,+86 202 257 8619,,, 26 | ,,+27 117 258 4149,,, 27 | ,,+48 697 447 6933,,, 28 | ,,+48 195 624 2025,,, 29 | ,,+1 254 875 4647,,, 30 | ,,+7 552 196 4096,,, 31 | ,,+86 892 682 0628,,, 32 | John,Smith,+62 999 888 7777,London,43,Jack;J 33 | Jane,Smith,+62 999 888 7778,London,43, 34 | 35 | ,,,,23, 36 | ,, ,,23, 37 | ,,,,, 38 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/person1.csv: -------------------------------------------------------------------------------- 1 | first_name,last_name,phone_number,city,age,nick_name 2 | fn_order_1,ln_order_1,+7 171 898 0801,London,1, 3 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/person2.csv: -------------------------------------------------------------------------------- 1 | first_name,last_name,phone_number,city,age,nick_name 2 | fn_order_2,ln_order_2,+7 171 898 0802,London,2, 3 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/person3.csv: -------------------------------------------------------------------------------- 1 | first_name,last_name,phone_number,city,age,nick_name 2 | fn_order_3,ln_order_3,+7 171 898 0803,London,3, 3 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/person_nextfile.csv: -------------------------------------------------------------------------------- 1 | first_name,last_name,phone_number,city,age,nick_name 2 | Jennifer,Mulcrum,+7 171 898 9999,London,51,Jenny 3 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/schema-updated.gql: -------------------------------------------------------------------------------- 1 | define 2 | 3 | name sub attribute, 4 | value string; 5 | started-at sub attribute, 6 | value datetime; 7 | duration sub attribute, 8 | value long; 9 | first-name sub attribute, 10 | value string; 11 | last-name sub attribute, 12 | value string; 13 | phone-number sub attribute, 14 | value string; 15 | city sub attribute, 16 | value string; 17 | age sub attribute, 18 | value long; 19 | nick-name sub attribute, 20 | value string; 21 | twitter-username sub attribute, 22 | value string; 23 | fakebook-link sub attribute, 24 | value string; 25 | call-rating sub attribute, 26 | value long; 27 | added-attribute sub attribute, 28 | value string; 29 | 30 | contract sub relation, 31 | relates provider, 32 | relates customer; 33 | 34 | call sub relation, 35 | relates caller, 36 | relates callee, 37 | owns started-at, 38 | owns duration, 39 | owns call-rating, 40 | plays communication-channel:past-call; 41 | 42 | communication-channel sub relation, 43 | relates peer, 44 | relates past-call; 45 | 46 | company sub entity, 47 | plays contract:provider, 48 | owns name @key; 49 | 50 | person sub entity, 51 | plays contract:customer, 52 | plays call:caller, 53 | plays call:callee, 54 | owns first-name, 55 | owns last-name, 56 | owns phone-number, 57 | owns city, 58 | owns age, 59 | owns nick-name, 60 | owns twitter-username, 61 | owns fakebook-link, 62 | plays communication-channel:peer; 63 | 64 | added-entity sub entity, 65 | owns added-attribute, 66 | plays added-relation:added-role; 67 | 68 | added-relation sub relation, 69 | relates added-role; 70 | -------------------------------------------------------------------------------- /src/test/resources/phoneCalls/schema.gql: -------------------------------------------------------------------------------- 1 | define 2 | 3 | name sub attribute, 4 | value string; 5 | started-at sub attribute, 6 | value datetime; 7 | duration sub attribute, 8 | value long; 9 | first-name sub attribute, 10 | value string; 11 | last-name sub attribute, 12 | value string; 13 | phone-number sub attribute, 14 | value string, 15 | plays in-use:account; 16 | city sub attribute, 17 | value string; 18 | age sub attribute, 19 | value long; 20 | nick-name sub attribute, 21 | value string; 22 | twitter-username sub attribute, 23 | value string; 24 | fakebook-link sub attribute, 25 | value string; 26 | call-rating sub attribute, 27 | value long; 28 | 29 | contract sub relation, 30 | relates provider, 31 | relates customer; 32 | 33 | call sub relation, 34 | relates caller, 35 | relates callee, 36 | owns started-at, 37 | owns duration, 38 | owns call-rating, 39 | plays communication-channel:past-call; 40 | 41 | communication-channel sub relation, 42 | relates peer, 43 | relates past-call; 44 | 45 | company sub entity, 46 | plays contract:provider, 47 | owns name @key; 48 | 49 | person sub entity, 50 | plays contract:customer, 51 | plays call:caller, 52 | plays call:callee, 53 | owns first-name, 54 | owns last-name, 55 | owns phone-number, 56 | owns city, 57 | owns age, 58 | owns nick-name, 59 | owns twitter-username, 60 | owns fakebook-link, 61 | plays communication-channel:peer; 62 | 63 | is-in-use sub attribute, value string, 64 | plays in-use:status; 65 | 66 | in-use sub relation, 67 | relates status, 68 | relates account; 69 | -------------------------------------------------------------------------------- /typedbloader.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/typedb-osi/typedb-loader/9a5fc166cbe8c26b4f2934a008b19991fccb62a1/typedbloader.png --------------------------------------------------------------------------------