├── .circleci └── config.yml ├── .eslintrc.js ├── .gitignore ├── .vscode └── settings.json ├── LICENSE ├── README.md ├── bin └── typesense-mongodb ├── config.js ├── data ├── books.json └── books1.json ├── docker-compose.yml ├── jest.config.js ├── lib ├── ChangeStreams.js ├── ChangeStreams.js.map ├── MongoClient.js ├── MongoClient.js.map ├── TypesenseClient.js ├── TypesenseClient.js.map ├── cli.js ├── cli.js.map ├── defaults.js ├── defaults.js.map ├── interfaces │ ├── config.js │ ├── config.js.map │ ├── node.js │ ├── node.js.map │ ├── schema.js │ └── schema.js.map ├── main.js ├── main.js.map ├── parseArguments.js └── parseArguments.js.map ├── package-lock.json ├── package.json ├── src ├── ChangeStreams.ts ├── MongoClient.ts ├── TypesenseClient.ts ├── cli.ts ├── defaults.ts ├── interfaces │ ├── config.ts │ ├── node.ts │ └── schema.ts ├── main.ts └── parseArguments.ts ├── tests ├── changeStreams.test.ts ├── globalSetup.ts ├── globalTeardown.ts ├── mongo.test.ts ├── setupFilesAfterEnv.ts └── typesense.test.ts └── tsconfig.json /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | jobs: 3 | test: 4 | machine: 5 | image: ubuntu-2004:202101-01 6 | resource_class: medium 7 | steps: 8 | - checkout 9 | - run: 10 | name: Install dependencies 11 | command: | 12 | curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash 13 | source ~/.bash_profile 14 | sudo apt update 15 | nvm install node 16 | - restore_cache: 17 | key: dependency-cache-v1-{{ checksum "package-lock.json" }} 18 | - run: 19 | name: npm install 20 | command: npm install 21 | - save_cache: 22 | key: dependency-cache-v1-{{ checksum "package-lock.json" }} 23 | paths: 24 | - ./node_modules 25 | - run: 26 | name: Docker Images 27 | command: docker-compose up -d 28 | - run: 29 | name: Install dockerize 30 | command: wget https://github.com/jwilder/dockerize/releases/download/$DOCKERIZE_VERSION/dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz && rm dockerize-linux-amd64-$DOCKERIZE_VERSION.tar.gz 31 | environment: 32 | DOCKERIZE_VERSION: v0.6.1 33 | - run: 34 | name: Wait for db 35 | command: dockerize -wait http://localhost:8081 -timeout 1m 36 | - run: 37 | name: Wait for typesense 38 | command: dockerize -wait http://localhost:8108/health -timeout 1m && sleep 10s 39 | - run: 40 | name: npm test 41 | command: npm test -- --ci --reporters=default --reporters=jest-junit 42 | environment: 43 | JEST_JUNIT_OUTPUT_DIR: "reports/junit/js-test-results.xml" 44 | - store_test_results: 45 | path: reports/junit 46 | - store_artifacts: 47 | path: reports/junit 48 | 49 | workflows: 50 | typesense-mongodb: 51 | jobs: 52 | - test 53 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: "@typescript-eslint/parser", 3 | parserOptions: { 4 | ecmaVersion: 2021, 5 | }, 6 | extends: [ 7 | "plugin:@typescript-eslint/recommended", 8 | "prettier", 9 | "plugin:prettier/recommended", 10 | ], 11 | rules: { 12 | semi: ["error", "always"], 13 | "@typescript-eslint/no-namespace": "off", 14 | }, 15 | }; 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | mongo-data 2 | typesense-data 3 | node_modules 4 | .idea 5 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "editor.codeActionsOnSave": { 3 | "source.fixAll.eslint": true 4 | }, 5 | "editor.formatOnSave": false, 6 | "editor.tabSize": 2, 7 | "jestrunner.debugOptions": { 8 | "args": ["--no-cache"] 9 | } 10 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2021-present Typesense, Inc. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Typesense MongoDB Integration 2 | ![CircleCI](https://img.shields.io/circleci/build/github/typesense/typesense-mongodb/master) 3 | 4 | A CLI to sync documents from a MongoDB collection to Typesense. 5 | 6 | ## Installation 7 | 8 | ```bash 9 | npm install -g typesense-mongodb 10 | ``` 11 | ## Usage 12 | 13 | ### Prerequisites 14 | 15 | - Make sure you are running MongoDB instance in a replica set. Or convert your [standalone MongoDB instance to MongoDB replica set](https://docs.mongodb.com/manual/tutorial/convert-standalone-to-replica-set/). 16 | - We highly recommend that you stop writes to your MongoDB collection until you get a message from the process. 17 | - If you are familiar with Typesense, it has an option to mark certain fields as **facets**. If you would like to use this feature, create your own Typesense collection and pass the collection name as an argument. 18 | 19 | ### Example 20 | 21 | ```bash 22 | typesense-mongodb \ 23 | --mongo-collection=collection \ 24 | --mongo-database=database \ 25 | --typesense-collection=collection \ 26 | --mongo-url=mongodb://localhost:27017 \ 27 | --typesense-url=http://localhost:8108 \ 28 | --typesense-api-key=xyz 29 | ``` 30 | 31 | ### Arguments 32 | 33 | | Parameter | Default | Description | 34 | | :--- | :--- |:--- | 35 | | `--mongo-database` | database | MongoDB database name | 36 | | `--mongo-collection` | collection | MongoDB collection name | 37 | | `--mongo-url` | mongodb://localhost:27017 | MongoDB instance URI along with username and passsword | 38 | | `--typesense-collection` | collection | Typesense collection name | 39 | | `--typesense-url` | http://localhost:8108 | Typesense endpoint URL | 40 | | `--typesense-api-key` | xyz | Typesense API key | 41 | 42 | 43 | ## Support 44 | 45 | Please open a Github issue or join our [Slack Community](https://join.slack.com/t/typesense-community/shared_invite/zt-mx4nbsbn-AuOL89O7iBtvkz136egSJg) 46 | -------------------------------------------------------------------------------- /bin/typesense-mongodb: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env node 2 | 3 | // eslint-disable-next-line @typescript-eslint/no-var-requires 4 | require("../lib/cli").cli(process.argv); 5 | -------------------------------------------------------------------------------- /config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | typesenseCollectionName: null, 3 | mongodbDatabaseName: null, 4 | mongodbCollectionName: null, 5 | mongodbURL: null, 6 | typesenseURL: null, 7 | typenseseKey: null, 8 | }; 9 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | mongo: 4 | image: mongo 5 | container_name: mongo 6 | ports: 7 | - "27017:27017" 8 | volumes: 9 | - mongo:/data/db 10 | entrypoint: [ "/usr/bin/mongod", "--bind_ip_all", "--replSet", "rs0" ] 11 | 12 | mongo-express: 13 | image: mongo-express:0.54 14 | ports: 15 | - "8081:8081" 16 | environment: 17 | ME_CONFIG_MONGODB_SERVER: mongo 18 | depends_on: 19 | - mongo 20 | 21 | typesense: 22 | image: typesense/typesense:0.21.0 23 | ports: 24 | - "8108:8108" 25 | environment: 26 | TYPESENSE_API_KEY: xyz 27 | TYPESENSE_DATA_DIR: /data/typesense 28 | TYPESENSE_ENABLE_CORS: "true" 29 | volumes: 30 | - typesense:/data/typesense 31 | 32 | volumes: 33 | mongo: 34 | driver: local 35 | typesense: 36 | driver: local 37 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: "ts-jest", 3 | testEnvironment: "node", 4 | globalSetup: "./tests/globalSetup.ts", 5 | globalTeardown: "./tests/globalTeardown.ts", 6 | setupFilesAfterEnv: ["./tests/setupFilesAfterEnv.ts"], 7 | testTimeout: 60000, 8 | slowTestThreshold: 30, 9 | }; 10 | -------------------------------------------------------------------------------- /lib/ChangeStreams.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | Object.defineProperty(exports, "__esModule", { value: true }); 12 | exports.ChangeStreams = void 0; 13 | var Events; 14 | (function (Events) { 15 | Events["insert"] = "insert"; 16 | Events["update"] = "update"; 17 | Events["replace"] = "replace"; 18 | Events["delete"] = "delete"; 19 | Events["drop"] = "drop"; 20 | Events["rename"] = "rename"; 21 | Events["dropDatabase"] = "dropDatabase"; 22 | Events["invalidate"] = "invalidate"; 23 | })(Events || (Events = {})); 24 | class ChangeStreams { 25 | constructor(mongo, typesense, databaseName, collectionName, typesenseCollectionName) { 26 | this.mongo = mongo; 27 | this.typesense = typesense; 28 | this.mongoDatabaseName = databaseName; 29 | this.mongoCollectionName = collectionName; 30 | this.typesenseCollectionName = typesenseCollectionName; 31 | this.changeStream = this.mongo.changeStreams(this.mongoDatabaseName, this.mongoCollectionName); 32 | this.eventMapper(); 33 | } 34 | eventMapper() { 35 | this.changeStream.on("change", (response) => __awaiter(this, void 0, void 0, function* () { 36 | console.log(response.operationType); 37 | if (response.operationType === Events.insert) { 38 | yield this.insert(response); 39 | } 40 | if (response.operationType === Events.update) { 41 | yield this.update(response); 42 | } 43 | if (response.operationType === Events.replace) { 44 | yield this.replace(response); 45 | } 46 | if (response.operationType === Events.delete) { 47 | yield this.delete(response); 48 | } 49 | if (response.operationType === Events.drop) { 50 | yield this.drop(); 51 | } 52 | if (response.operationType === Events.rename) { 53 | yield this.rename(response); 54 | } 55 | })); 56 | } 57 | closeChangeStream() { 58 | return __awaiter(this, void 0, void 0, function* () { 59 | this.changeStream.close(); 60 | }); 61 | } 62 | insert(response) { 63 | return __awaiter(this, void 0, void 0, function* () { 64 | const data = response.fullDocument; 65 | Object.assign(data, { 66 | id: String(response.documentKey._id), 67 | }); 68 | delete data._id; 69 | yield this.typesense.insertDocument(this.typesenseCollectionName, data); 70 | }); 71 | } 72 | update(response) { 73 | return __awaiter(this, void 0, void 0, function* () { 74 | const data = response.fullDocument; 75 | Object.assign(data, { 76 | id: String(response.documentKey._id), 77 | }); 78 | delete data._id; 79 | yield this.typesense.updateDocument(this.typesenseCollectionName, data); 80 | }); 81 | } 82 | replace(response) { 83 | return __awaiter(this, void 0, void 0, function* () { 84 | const id = String(response.documentKey._id); 85 | const data = response.fullDocument; 86 | Object.assign(data, { 87 | id: id, 88 | }); 89 | delete data._id; 90 | yield this.typesense.replaceDocument(this.typesenseCollectionName, id, data); 91 | }); 92 | } 93 | delete(response) { 94 | return __awaiter(this, void 0, void 0, function* () { 95 | const id = String(response.documentKey._id); 96 | yield this.typesense.deleteDocument(this.typesenseCollectionName, id); 97 | }); 98 | } 99 | drop() { 100 | return __awaiter(this, void 0, void 0, function* () { 101 | yield this.typesense.dropCollection(this.typesenseCollectionName); 102 | yield this.closeChangeStream(); 103 | }); 104 | } 105 | rename(response) { 106 | return __awaiter(this, void 0, void 0, function* () { 107 | yield this.typesense.renameCollection(this.typesenseCollectionName, `${this.mongoDatabaseName}_${response.to.coll}`); 108 | this.mongoCollectionName = response.to.coll; 109 | this.typesenseCollectionName = `${this.mongoDatabaseName}_${response.to.coll}`; 110 | this.closeChangeStream(); 111 | this.changeStream = this.mongo.changeStreams(this.mongoDatabaseName, this.mongoCollectionName); 112 | this.eventMapper(); 113 | }); 114 | } 115 | } 116 | exports.ChangeStreams = ChangeStreams; 117 | //# sourceMappingURL=ChangeStreams.js.map -------------------------------------------------------------------------------- /lib/ChangeStreams.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"ChangeStreams.js","sourceRoot":"","sources":["../src/ChangeStreams.ts"],"names":[],"mappings":";;;;;;;;;;;;AAWA,IAAK,MASJ;AATD,WAAK,MAAM;IACT,2BAAiB,CAAA;IACjB,2BAAiB,CAAA;IACjB,6BAAmB,CAAA;IACnB,2BAAiB,CAAA;IACjB,uBAAa,CAAA;IACb,2BAAiB,CAAA;IACjB,uCAA6B,CAAA;IAC7B,mCAAyB,CAAA;AAC3B,CAAC,EATI,MAAM,KAAN,MAAM,QASV;AAED,MAAa,aAAa;IAQxB,YACE,KAAkB,EAClB,SAA0B,EAC1B,YAAoB,EACpB,cAAsB,EACtB,uBAA+B;QAE/B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,iBAAiB,GAAG,YAAY,CAAC;QACtC,IAAI,CAAC,mBAAmB,GAAG,cAAc,CAAC;QAC1C,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAC1C,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,mBAAmB,CACzB,CAAC;QACF,IAAI,CAAC,WAAW,EAAE,CAAC;IACrB,CAAC;IAED,WAAW;QACT,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAO,QAAQ,EAAE,EAAE;YAChD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YACpC,IAAI,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,MAAM,EAAE;gBAC5C,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;aAC7B;YACD,IAAI,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,MAAM,EAAE;gBAC5C,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;aAC7B;YACD,IAAI,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,OAAO,EAAE;gBAC7C,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;aAC9B;YACD,IAAI,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,MAAM,EAAE;gBAC5C,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;aAC7B;YACD,IAAI,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,IAAI,EAAE;gBAC1C,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;aACnB;YACD,IAAI,QAAQ,CAAC,aAAa,KAAK,MAAM,CAAC,MAAM,EAAE;gBAC5C,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;aAC7B;QACH,CAAC,CAAA,CAAC,CAAC;IACL,CAAC;IAEK,iBAAiB;;YACrB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;KAAA;IAEK,MAAM,CAAC,QAA4B;;YACvC,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;gBAClB,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC;aACrC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC,GAAG,CAAC;YAChB,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;QAC1E,CAAC;KAAA;IAEK,MAAM,CAAC,QAAgC;;YAC3C,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;gBAClB,EAAE,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC;aACrC,CAAC,CAAC;YACH,OAAO,IAAI,CAAC,GAAG,CAAC;YAChB,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,uBAAuB,EAAE,IAAI,CAAC,CAAC;QAC1E,CAAC;KAAA;IAEK,OAAO,CAAC,QAA4B;;YACxC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,YAAY,CAAC;YACnC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE;gBAClB,EAAE,EAAE,EAAE;aACP,CAAC,CAAC;YACH,OAAO,IAAI,CAAC,GAAG,CAAC;YAChB,MAAM,IAAI,CAAC,SAAS,CAAC,eAAe,CAClC,IAAI,CAAC,uBAAuB,EAC5B,EAAE,EACF,IAAI,CACL,CAAC;QACJ,CAAC;KAAA;IAEK,MAAM,CAAC,QAAgC;;YAC3C,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YAC5C,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;QACxE,CAAC;KAAA;IAEK,IAAI;;YACR,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;YAClE,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACjC,CAAC;KAAA;IAEK,MAAM,CAAC,QAAgC;;YAC3C,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB,CACnC,IAAI,CAAC,uBAAuB,EAC5B,GAAG,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAChD,CAAC;YACF,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC;YAC5C,IAAI,CAAC,uBAAuB,GAAG,GAAG,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC;YAC/E,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAC1C,IAAI,CAAC,iBAAiB,EACtB,IAAI,CAAC,mBAAmB,CACzB,CAAC;YACF,IAAI,CAAC,WAAW,EAAE,CAAC;QACrB,CAAC;KAAA;CACF;AA/GD,sCA+GC"} -------------------------------------------------------------------------------- /lib/MongoClient.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | Object.defineProperty(exports, "__esModule", { value: true }); 12 | exports.MongoClient = void 0; 13 | const mongodb_1 = require("mongodb"); 14 | class MongoClient { 15 | constructor(url) { 16 | this.mongoOptions = { 17 | useNewUrlParser: true, 18 | useUnifiedTopology: true, 19 | }; 20 | this.client = new mongodb_1.MongoClient(url, this.mongoOptions); 21 | this.url = url; 22 | } 23 | listAllDatabases() { 24 | return __awaiter(this, void 0, void 0, function* () { 25 | let dbList = yield this.client.db().admin().listDatabases(); 26 | dbList = dbList.databases.map((database) => { 27 | return database.name; 28 | }); 29 | return dbList; 30 | }); 31 | } 32 | connectMongo() { 33 | return __awaiter(this, void 0, void 0, function* () { 34 | try { 35 | yield this.client.connect(); 36 | } 37 | catch (e) { 38 | console.error(e); 39 | } 40 | }); 41 | } 42 | closeMongo() { 43 | return __awaiter(this, void 0, void 0, function* () { 44 | try { 45 | yield this.client.close(); 46 | } 47 | catch (e) { 48 | console.error(e); 49 | } 50 | }); 51 | } 52 | listCollections(databaseName) { 53 | return __awaiter(this, void 0, void 0, function* () { 54 | let collectionList = yield this.client 55 | .db(databaseName) 56 | .listCollections() 57 | .toArray(); 58 | collectionList = collectionList.map((collection) => { 59 | return collection.name; 60 | }); 61 | return collectionList; 62 | }); 63 | } 64 | insertDocuments(databaseName, collectionName) { 65 | return __awaiter(this, void 0, void 0, function* () { 66 | const books = yield require("../data/books.json"); 67 | const sample_data = books.slice(0, 5000); 68 | const db = this.client.db(databaseName); 69 | yield db.collection(collectionName).insertMany(sample_data); 70 | }); 71 | } 72 | readDocuments(databaseName, collectionName) { 73 | return __awaiter(this, void 0, void 0, function* () { 74 | const db = this.client.db(databaseName); 75 | const result = yield db 76 | .collection(collectionName) 77 | .find() 78 | .toArray(); 79 | result.forEach((document) => { 80 | document.id = String(document._id); 81 | delete document._id; 82 | }); 83 | return result; 84 | }); 85 | } 86 | changeStreams(databaseName, collectionName) { 87 | const collection = this.client.db(databaseName).collection(collectionName); 88 | const changeStream = collection.watch([], { fullDocument: "updateLookup" }); 89 | return changeStream; 90 | } 91 | } 92 | exports.MongoClient = MongoClient; 93 | //# sourceMappingURL=MongoClient.js.map -------------------------------------------------------------------------------- /lib/MongoClient.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"MongoClient.js","sourceRoot":"","sources":["../src/MongoClient.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,qCAA8D;AAC9D,MAAa,WAAW;IA8EtB,YAAY,GAAW;QACrB,IAAI,CAAC,YAAY,GAAG;YAClB,eAAe,EAAE,IAAI;YACrB,kBAAkB,EAAE,IAAI;SACzB,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,IAAI,qBAAM,CAAC,GAAG,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC;QACjD,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IA7EK,gBAAgB;;YACpB,IAAI,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,KAAK,EAAE,CAAC,aAAa,EAAE,CAAC;YAC5D,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACzC,OAAO,QAAQ,CAAC,IAAI,CAAC;YACvB,CAAC,CAAC,CAAC;YACH,OAAO,MAAM,CAAC;QAChB,CAAC;KAAA;IAEK,YAAY;;YAChB,IAAI;gBACF,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC7B;YAAC,OAAO,CAAC,EAAE;gBACV,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;QACH,CAAC;KAAA;IAEK,UAAU;;YACd,IAAI;gBACF,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;aAC3B;YAAC,OAAO,CAAC,EAAE;gBACV,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;aAClB;QACH,CAAC;KAAA;IAEK,eAAe,CAAC,YAAoB;;YACxC,IAAI,cAAc,GAAG,MAAM,IAAI,CAAC,MAAM;iBACnC,EAAE,CAAC,YAAY,CAAC;iBAChB,eAAe,EAAE;iBACjB,OAAO,EAAE,CAAC;YACb,cAAc,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;gBACjD,OAAO,UAAU,CAAC,IAAI,CAAC;YACzB,CAAC,CAAC,CAAC;YACH,OAAO,cAAc,CAAC;QACxB,CAAC;KAAA;IAEK,eAAe,CACnB,YAAoB,EACpB,cAAsB;;YAEtB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,oBAAoB,CAAC,CAAC;YAClD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YACzC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;YACxC,MAAM,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QAC9D,CAAC;KAAA;IAEK,aAAa,CACjB,YAAoB,EACpB,cAAsB;;YAEtB,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;YACxC,MAAM,MAAM,GAA8B,MAAM,EAAE;iBAC/C,UAAU,CAAC,cAAc,CAAC;iBAC1B,IAAI,EAAE;iBACN,OAAO,EAAE,CAAC;YACb,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;gBAC1B,QAAQ,CAAC,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;gBACnC,OAAO,QAAQ,CAAC,GAAG,CAAC;YACtB,CAAC,CAAC,CAAC;YACH,OAAO,MAAM,CAAC;QAChB,CAAC;KAAA;IAED,aAAa,CACX,YAAoB,EACpB,cAAsB;QAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAC3E,MAAM,YAAY,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC;QAC5E,OAAO,YAAY,CAAC;IACtB,CAAC;CAUF;AAtFD,kCAsFC"} -------------------------------------------------------------------------------- /lib/TypesenseClient.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | Object.defineProperty(exports, "__esModule", { value: true }); 12 | exports.TypesenseClient = void 0; 13 | const typesense_1 = require("typesense"); 14 | class TypesenseClient { 15 | constructor(apiKey, nodes, nearestNode, connectionTimeout) { 16 | this.client = new typesense_1.Client({ 17 | nodes: nodes, 18 | nearestNode: nearestNode, 19 | apiKey: apiKey, 20 | connectionTimoutSeconds: connectionTimeout, 21 | }); 22 | } 23 | createCollection(collectionName) { 24 | return __awaiter(this, void 0, void 0, function* () { 25 | const autoSchema = { 26 | name: collectionName, 27 | fields: [{ name: ".*", type: "auto" }], 28 | }; 29 | yield this.client.collections().create(autoSchema); 30 | }); 31 | } 32 | importDocuments(collectionName, documents) { 33 | return __awaiter(this, void 0, void 0, function* () { 34 | yield this.client 35 | .collections(collectionName) 36 | .documents() 37 | .import(documents, { action: "create" }); 38 | }); 39 | } 40 | insertDocument(collectionName, document) { 41 | return __awaiter(this, void 0, void 0, function* () { 42 | yield this.client.collections(collectionName).documents().create(document); 43 | }); 44 | } 45 | updateDocument(collectionName, updatedDocument) { 46 | return __awaiter(this, void 0, void 0, function* () { 47 | yield this.client 48 | .collections(collectionName) 49 | .documents() 50 | .upsert(updatedDocument); 51 | }); 52 | } 53 | deleteDocument(collectionName, id) { 54 | return __awaiter(this, void 0, void 0, function* () { 55 | yield this.client.collections(collectionName).documents(id).delete(); 56 | }); 57 | } 58 | replaceDocument(collectionName, id, document) { 59 | return __awaiter(this, void 0, void 0, function* () { 60 | try { 61 | yield this.client.collections(collectionName).documents(id).delete(); 62 | } 63 | catch (err) { 64 | if (err instanceof typesense_1.Errors.ObjectNotFound) { 65 | return; 66 | } 67 | throw err; 68 | } 69 | yield this.client.collections(collectionName).documents().create(document); 70 | }); 71 | } 72 | dropCollection(collectionName) { 73 | return __awaiter(this, void 0, void 0, function* () { 74 | yield this.client.collections(collectionName).delete(); 75 | }); 76 | } 77 | renameCollection(collectionName, newCollectionName) { 78 | return __awaiter(this, void 0, void 0, function* () { 79 | const aliased_collection = { 80 | collection_name: collectionName, 81 | }; 82 | yield this.client.aliases().upsert(newCollectionName, aliased_collection); 83 | }); 84 | } 85 | checkCollection(collectionName) { 86 | return __awaiter(this, void 0, void 0, function* () { 87 | try { 88 | const result = yield this.client.collections(collectionName).retrieve(); 89 | return result.num_documents; 90 | } 91 | catch (err) { 92 | if (err instanceof typesense_1.Errors.ObjectNotFound) { 93 | return undefined; 94 | } 95 | throw err; 96 | } 97 | }); 98 | } 99 | checkServer() { 100 | return __awaiter(this, void 0, void 0, function* () { 101 | return yield this.client.health.retrieve(); 102 | }); 103 | } 104 | } 105 | exports.TypesenseClient = TypesenseClient; 106 | //# sourceMappingURL=TypesenseClient.js.map -------------------------------------------------------------------------------- /lib/TypesenseClient.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"TypesenseClient.js","sourceRoot":"","sources":["../src/TypesenseClient.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,yCAA2C;AAI3C,MAAa,eAAe;IAE1B,YACE,MAAc,EACd,KAAa,EACb,WAAkB,EAClB,iBAA0B;QAE1B,IAAI,CAAC,MAAM,GAAG,IAAI,kBAAM,CAAC;YACvB,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,WAAW;YACxB,MAAM,EAAE,MAAM;YACd,uBAAuB,EAAE,iBAAiB;SAC3C,CAAC,CAAC;IACL,CAAC;IAEK,gBAAgB,CAAC,cAAsB;;YAC3C,MAAM,UAAU,GAAW;gBACzB,IAAI,EAAE,cAAc;gBACpB,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;aACvC,CAAC;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACrD,CAAC;KAAA;IAEK,eAAe,CACnB,cAAsB,EACtB,SAAoC;;YAEpC,MAAM,IAAI,CAAC,MAAM;iBACd,WAAW,CAAC,cAAc,CAAC;iBAC3B,SAAS,EAAE;iBACX,MAAM,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC7C,CAAC;KAAA;IAEK,cAAc,CAClB,cAAsB,EACtB,QAAiC;;YAEjC,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7E,CAAC;KAAA;IAEK,cAAc,CAClB,cAAsB,EACtB,eAAwC;;YAExC,MAAM,IAAI,CAAC,MAAM;iBACd,WAAW,CAAC,cAAc,CAAC;iBAC3B,SAAS,EAAE;iBACX,MAAM,CAAC,eAAe,CAAC,CAAC;QAC7B,CAAC;KAAA;IAEK,cAAc,CAAC,cAAsB,EAAE,EAAU;;YACrD,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;QACvE,CAAC;KAAA;IAEK,eAAe,CACnB,cAAsB,EACtB,EAAU,EACV,QAAiC;;YAEjC,IAAI;gBACF,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;aACtE;YAAC,OAAO,GAAG,EAAE;gBACZ,IAAI,GAAG,YAAY,kBAAM,CAAC,cAAc,EAAE;oBACxC,OAAO;iBACR;gBACD,MAAM,GAAG,CAAC;aACX;YACD,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7E,CAAC;KAAA;IAEK,cAAc,CAAC,cAAsB;;YACzC,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,MAAM,EAAE,CAAC;QACzD,CAAC;KAAA;IAEK,gBAAgB,CACpB,cAAsB,EACtB,iBAAyB;;YAEzB,MAAM,kBAAkB,GAAG;gBACzB,eAAe,EAAE,cAAc;aAChC,CAAC;YACF,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;QAC5E,CAAC;KAAA;IAEK,eAAe,CAAC,cAAsB;;YAC1C,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,QAAQ,EAAE,CAAC;gBACxE,OAAO,MAAM,CAAC,aAAa,CAAC;aAC7B;YAAC,OAAO,GAAG,EAAE;gBACZ,IAAI,GAAG,YAAY,kBAAM,CAAC,cAAc,EAAE;oBACxC,OAAO,SAAS,CAAC;iBAClB;gBACD,MAAM,GAAG,CAAC;aACX;QACH,CAAC;KAAA;IAEK,WAAW;;YACf,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;QAC7C,CAAC;KAAA;CACF;AApGD,0CAoGC"} -------------------------------------------------------------------------------- /lib/cli.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | Object.defineProperty(exports, "__esModule", { value: true }); 12 | exports.cli = void 0; 13 | const main_1 = require("./main"); 14 | const parseArguments_1 = require("./parseArguments"); 15 | function cli(args) { 16 | return __awaiter(this, void 0, void 0, function* () { 17 | const parsed = parseArguments_1.parseArguments(args); 18 | yield main_1.Main(parsed); 19 | }); 20 | } 21 | exports.cli = cli; 22 | //# sourceMappingURL=cli.js.map -------------------------------------------------------------------------------- /lib/cli.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";;;;;;;;;;;;AACA,iCAA8B;AAC9B,qDAAkD;AAElD,SAAsB,GAAG,CAAC,IAAc;;QACtC,MAAM,MAAM,GAAW,+BAAc,CAAC,IAAI,CAAC,CAAC;QAC5C,MAAM,WAAI,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;CAAA;AAHD,kBAGC"} -------------------------------------------------------------------------------- /lib/defaults.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | const defaults = { 4 | mongodbDatabaseName: "database", 5 | mongodbCollectionName: "collection", 6 | typesenseCollectionName: "collection", 7 | mongodbURL: "mongodb://localhost:27017", 8 | typesenseURL: "http://localhost:8108", 9 | typesenseKey: "xyz", 10 | }; 11 | exports.default = defaults; 12 | //# sourceMappingURL=defaults.js.map -------------------------------------------------------------------------------- /lib/defaults.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"defaults.js","sourceRoot":"","sources":["../src/defaults.ts"],"names":[],"mappings":";;AAEA,MAAM,QAAQ,GAAW;IACvB,mBAAmB,EAAE,UAAU;IAC/B,qBAAqB,EAAE,YAAY;IACnC,uBAAuB,EAAE,YAAY;IACrC,UAAU,EAAE,2BAA2B;IACvC,YAAY,EAAE,uBAAuB;IACrC,YAAY,EAAE,KAAK;CACpB,CAAC;AAEF,kBAAe,QAAQ,CAAC"} -------------------------------------------------------------------------------- /lib/interfaces/config.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | //# sourceMappingURL=config.js.map -------------------------------------------------------------------------------- /lib/interfaces/config.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/interfaces/config.ts"],"names":[],"mappings":""} -------------------------------------------------------------------------------- /lib/interfaces/node.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | //# sourceMappingURL=node.js.map -------------------------------------------------------------------------------- /lib/interfaces/node.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"node.js","sourceRoot":"","sources":["../../src/interfaces/node.ts"],"names":[],"mappings":""} -------------------------------------------------------------------------------- /lib/interfaces/schema.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | Object.defineProperty(exports, "__esModule", { value: true }); 3 | //# sourceMappingURL=schema.js.map -------------------------------------------------------------------------------- /lib/interfaces/schema.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/interfaces/schema.ts"],"names":[],"mappings":""} -------------------------------------------------------------------------------- /lib/main.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { 3 | function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } 4 | return new (P || (P = Promise))(function (resolve, reject) { 5 | function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } 6 | function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } 7 | function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } 8 | step((generator = generator.apply(thisArg, _arguments || [])).next()); 9 | }); 10 | }; 11 | var __importDefault = (this && this.__importDefault) || function (mod) { 12 | return (mod && mod.__esModule) ? mod : { "default": mod }; 13 | }; 14 | Object.defineProperty(exports, "__esModule", { value: true }); 15 | exports.Main = void 0; 16 | const defaults_1 = __importDefault(require("./defaults")); 17 | const MongoClient_1 = require("./MongoClient"); 18 | const TypesenseClient_1 = require("./TypesenseClient"); 19 | const listr_1 = __importDefault(require("listr")); 20 | const chalk_1 = __importDefault(require("chalk")); 21 | const ChangeStreams_1 = require("./ChangeStreams"); 22 | let typesense; 23 | let mongo; 24 | let need; 25 | function typesenseURLParser(url) { 26 | const splits = url.split(":"); 27 | return { 28 | host: splits[1].slice(2, splits[1].length), 29 | port: splits[splits.length - 1], 30 | protocol: splits[0], 31 | }; 32 | } 33 | function initializeTypesenseClient(options) { 34 | return __awaiter(this, void 0, void 0, function* () { 35 | typesense = new TypesenseClient_1.TypesenseClient(options.typesenseKey, [ 36 | typesenseURLParser(options.typesenseURL), 37 | ]); 38 | try { 39 | yield typesense.checkServer(); 40 | } 41 | catch (err) { 42 | console.error(err); 43 | } 44 | return typesense; 45 | }); 46 | } 47 | function initializeMongoClient(options) { 48 | return __awaiter(this, void 0, void 0, function* () { 49 | mongo = new MongoClient_1.MongoClient(options.mongodbURL); 50 | try { 51 | yield mongo.connectMongo(); 52 | } 53 | catch (err) { 54 | console.error(err); 55 | } 56 | return mongo; 57 | }); 58 | } 59 | function checkForExistingCollection(typesense, options) { 60 | return __awaiter(this, void 0, void 0, function* () { 61 | need = yield typesense.checkCollection(options.typesenseCollectionName); 62 | return need; 63 | }); 64 | } 65 | function indexExistingDocuments(typesense, mongo, options) { 66 | return __awaiter(this, void 0, void 0, function* () { 67 | const document = yield mongo.readDocuments(options.mongodbDatabaseName, options.mongodbCollectionName); 68 | yield typesense.importDocuments(options.typesenseCollectionName, document); 69 | }); 70 | } 71 | function enableChangeStreams(typesense, mongo, options) { 72 | new ChangeStreams_1.ChangeStreams(mongo, typesense, options.mongodbDatabaseName, options.mongodbCollectionName, options.typesenseCollectionName); 73 | } 74 | function Main(parsed) { 75 | return __awaiter(this, void 0, void 0, function* () { 76 | const options = { 77 | mongodbDatabaseName: parsed.mongodbDatabaseName || defaults_1.default.mongodbDatabaseName, 78 | mongodbCollectionName: parsed.mongodbCollectionName || defaults_1.default.mongodbCollectionName, 79 | mongodbURL: parsed.mongodbURL || defaults_1.default.mongodbURL, 80 | typesenseCollectionName: parsed.typesenseCollectionName || defaults_1.default.typesenseCollectionName, 81 | typesenseKey: parsed.typesenseKey || defaults_1.default.typesenseKey, 82 | typesenseURL: parsed.typesenseURL || defaults_1.default.typesenseURL, 83 | }; 84 | const tasks = new listr_1.default([ 85 | { 86 | title: "Initialize Typesense Client", 87 | task: () => initializeTypesenseClient(options), 88 | }, 89 | { 90 | title: "Initialize Mongo Client", 91 | task: () => initializeMongoClient(options), 92 | }, 93 | { 94 | title: "Check for an existing typesense collection", 95 | task: () => checkForExistingCollection(typesense, options), 96 | }, 97 | { 98 | title: "Create a new Typesense Collection", 99 | task: () => typesense.createCollection(options.typesenseCollectionName), 100 | skip: () => need 101 | ? "Found an existing collection skipping create collection" 102 | : undefined, 103 | }, 104 | { 105 | title: "Index existing documents", 106 | task: () => indexExistingDocuments(typesense, mongo, options), 107 | }, 108 | { 109 | title: "Open Change Stream", 110 | task: () => enableChangeStreams(typesense, mongo, options), 111 | }, 112 | ]); 113 | yield tasks.run(); 114 | console.log("%s Watching for changes..", chalk_1.default.green("DONE")); 115 | }); 116 | } 117 | exports.Main = Main; 118 | //# sourceMappingURL=main.js.map -------------------------------------------------------------------------------- /lib/main.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"main.js","sourceRoot":"","sources":["../src/main.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,0DAAkC;AAGlC,+CAA4C;AAC5C,uDAAoD;AACpD,kDAA0B;AAC1B,kDAA0B;AAC1B,mDAAgD;AAEhD,IAAI,SAA0B,CAAC;AAC/B,IAAI,KAAkB,CAAC;AACvB,IAAI,IAAY,CAAC;AAEjB,SAAS,kBAAkB,CAAC,GAAW;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAE9B,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1C,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAC/B,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;KACpB,CAAC;AACJ,CAAC;AAED,SAAe,yBAAyB,CACtC,OAAe;;QAEf,SAAS,GAAG,IAAI,iCAAe,CAAC,OAAO,CAAC,YAAY,EAAE;YACpD,kBAAkB,CAAC,OAAO,CAAC,YAAY,CAAC;SACzC,CAAC,CAAC;QACH,IAAI;YACF,MAAM,SAAS,CAAC,WAAW,EAAE,CAAC;SAC/B;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACpB;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;CAAA;AAED,SAAe,qBAAqB,CAAC,OAAe;;QAClD,KAAK,GAAG,IAAI,yBAAW,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI;YACF,MAAM,KAAK,CAAC,YAAY,EAAE,CAAC;SAC5B;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;SACpB;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CAAA;AAED,SAAe,0BAA0B,CACvC,SAA0B,EAC1B,OAAe;;QAEf,IAAI,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;CAAA;AAED,SAAe,sBAAsB,CACnC,SAA0B,EAC1B,KAAkB,EAClB,OAAe;;QAEf,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,aAAa,CACxC,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,qBAAqB,CAC9B,CAAC;QACF,MAAM,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;IAC7E,CAAC;CAAA;AAED,SAAS,mBAAmB,CAC1B,SAA0B,EAC1B,KAAkB,EAClB,OAAe;IAEf,IAAI,6BAAa,CACf,KAAK,EACL,SAAS,EACT,OAAO,CAAC,mBAAmB,EAC3B,OAAO,CAAC,qBAAqB,EAC7B,OAAO,CAAC,uBAAuB,CAChC,CAAC;AACJ,CAAC;AAED,SAAsB,IAAI,CAAC,MAAc;;QACvC,MAAM,OAAO,GAAW;YACtB,mBAAmB,EACjB,MAAM,CAAC,mBAAmB,IAAI,kBAAQ,CAAC,mBAAmB;YAC5D,qBAAqB,EACnB,MAAM,CAAC,qBAAqB,IAAI,kBAAQ,CAAC,qBAAqB;YAChE,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,kBAAQ,CAAC,UAAU;YACpD,uBAAuB,EACrB,MAAM,CAAC,uBAAuB,IAAI,kBAAQ,CAAC,uBAAuB;YACpE,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,kBAAQ,CAAC,YAAY;YAC1D,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,kBAAQ,CAAC,YAAY;SAC3D,CAAC;QAEF,MAAM,KAAK,GAAG,IAAI,eAAK,CAAC;YACtB;gBACE,KAAK,EAAE,6BAA6B;gBACpC,IAAI,EAAE,GAAG,EAAE,CAAC,yBAAyB,CAAC,OAAO,CAAC;aAC/C;YACD;gBACE,KAAK,EAAE,yBAAyB;gBAChC,IAAI,EAAE,GAAG,EAAE,CAAC,qBAAqB,CAAC,OAAO,CAAC;aAC3C;YACD;gBACE,KAAK,EAAE,4CAA4C;gBACnD,IAAI,EAAE,GAAG,EAAE,CAAC,0BAA0B,CAAC,SAAS,EAAE,OAAO,CAAC;aAC3D;YACD;gBACE,KAAK,EAAE,mCAAmC;gBAC1C,IAAI,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,gBAAgB,CAAC,OAAO,CAAC,uBAAuB,CAAC;gBACvE,IAAI,EAAE,GAAG,EAAE,CACT,IAAI;oBACF,CAAC,CAAC,yDAAyD;oBAC3D,CAAC,CAAC,SAAS;aAChB;YACD;gBACE,KAAK,EAAE,0BAA0B;gBACjC,IAAI,EAAE,GAAG,EAAE,CAAC,sBAAsB,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC;aAC9D;YACD;gBACE,KAAK,EAAE,oBAAoB;gBAC3B,IAAI,EAAE,GAAG,EAAE,CAAC,mBAAmB,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC;aAC3D;SACF,CAAC,CAAC;QAEH,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC;QAClB,OAAO,CAAC,GAAG,CAAC,2BAA2B,EAAE,eAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAChE,CAAC;CAAA;AA9CD,oBA8CC"} -------------------------------------------------------------------------------- /lib/parseArguments.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | var __importDefault = (this && this.__importDefault) || function (mod) { 3 | return (mod && mod.__esModule) ? mod : { "default": mod }; 4 | }; 5 | Object.defineProperty(exports, "__esModule", { value: true }); 6 | exports.parseArguments = void 0; 7 | const arg_1 = __importDefault(require("arg")); 8 | function parseArguments(rawArgs) { 9 | const args = arg_1.default({ 10 | "--mongo-database": String, 11 | "--mongo-collection": String, 12 | "--typesense-collection": String, 13 | "--typesense-api-key": String, 14 | "--typesense-url": String, 15 | "--mongo-url": String, 16 | }, { 17 | argv: rawArgs.slice(2), 18 | }); 19 | return { 20 | mongodbDatabaseName: args["--mongo-database"], 21 | mongodbCollectionName: args["--mongo-collection"], 22 | typesenseCollectionName: args["--typesense-collection"], 23 | mongodbURL: args["--mongo-url"], 24 | typesenseURL: args["--typesense-url"], 25 | typesenseKey: args["--typesense-api-key"], 26 | }; 27 | } 28 | exports.parseArguments = parseArguments; 29 | /* 30 | typesense-mongodb \ 31 | --mongo-database=database \ 32 | --mongo-collection=collection \ 33 | --typesense-collection=collection \ 34 | --mongo-url=mongodb://localhost:27017 \ 35 | --typesense-url=http://localhost:8108 \ 36 | --typesense-api-key=xyz 37 | */ 38 | // --mongo-database=database --mongo-collection=collection --typesense-collection=collection --mongo-url=mongodb://localhost:27017 --typesense-url=http://localhost:8108 --typesense-api-key=xyz 39 | //# sourceMappingURL=parseArguments.js.map -------------------------------------------------------------------------------- /lib/parseArguments.js.map: -------------------------------------------------------------------------------- 1 | {"version":3,"file":"parseArguments.js","sourceRoot":"","sources":["../src/parseArguments.ts"],"names":[],"mappings":";;;;;;AAAA,8CAAsB;AAGtB,SAAgB,cAAc,CAAC,OAAiB;IAC9C,MAAM,IAAI,GAAG,aAAG,CACd;QACE,kBAAkB,EAAE,MAAM;QAC1B,oBAAoB,EAAE,MAAM;QAC5B,wBAAwB,EAAE,MAAM;QAChC,qBAAqB,EAAE,MAAM;QAC7B,iBAAiB,EAAE,MAAM;QACzB,aAAa,EAAE,MAAM;KACtB,EACD;QACE,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;KACvB,CACF,CAAC;IACF,OAAO;QACL,mBAAmB,EAAE,IAAI,CAAC,kBAAkB,CAAC;QAC7C,qBAAqB,EAAE,IAAI,CAAC,oBAAoB,CAAC;QACjD,uBAAuB,EAAE,IAAI,CAAC,wBAAwB,CAAC;QACvD,UAAU,EAAE,IAAI,CAAC,aAAa,CAAC;QAC/B,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC;QACrC,YAAY,EAAE,IAAI,CAAC,qBAAqB,CAAC;KAC1C,CAAC;AACJ,CAAC;AAtBD,wCAsBC;AAED;;;;;;;;EAQE;AAEF,gMAAgM"} -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "typesense-mongodb", 3 | "version": "0.0.2", 4 | "description": "A CLI process to sync data from MongoDB to Typesense", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "jest --no-cache --max-workers=1 --verbose --silent", 8 | "lint": "./node_modules/.bin/eslint -c .eslintrc.js", 9 | "build": "tsc" 10 | }, 11 | "bin": { 12 | "typesense-mongodb": "bin/typesense-mongodb" 13 | }, 14 | "repository": { 15 | "type": "git", 16 | "url": "https://github.com/typesense/typesense-mongodb" 17 | }, 18 | "keywords": [ 19 | "mongodb", 20 | "typesense" 21 | ], 22 | "author": "Typesense, Inc.", 23 | "license": "Apache-2.0", 24 | "bugs": { 25 | "url": "https://github.com/typesense/typesense-mongodb/issues" 26 | }, 27 | "files": [ 28 | "src", 29 | "lib", 30 | "bin", 31 | "config.js" 32 | ], 33 | "publishConfig": { 34 | "access": "public" 35 | }, 36 | "dependencies": { 37 | "@babel/runtime": "^7.14.5", 38 | "@types/jest": "^26.0.23", 39 | "@types/mongodb": "^3.6.17", 40 | "@types/node": "^15.12.2", 41 | "arg": "^5.0.0", 42 | "chalk": "^4.1.1", 43 | "child_process": "^1.0.2", 44 | "execa": "^5.1.1", 45 | "inquirer": "^8.1.0", 46 | "jest": "^27.0.4", 47 | "jest-cli": "^27.0.4", 48 | "jest-dev-server": "^5.0.3", 49 | "jest-junit": "^12.2.0", 50 | "listr": "^0.14.3", 51 | "mongodb": "^3.6.9", 52 | "ncp": "^2.0.0", 53 | "pkg-install": "^1.0.0", 54 | "ts-jest": "^27.0.3", 55 | "typesense": "^0.12.1" 56 | }, 57 | "devDependencies": { 58 | "@types/listr": "^0.14.3", 59 | "@typescript-eslint/eslint-plugin": "^4.26.0", 60 | "@typescript-eslint/parser": "^4.26.0", 61 | "eslint": "^7.28.0", 62 | "eslint-config-prettier": "^8.3.0", 63 | "eslint-plugin-prettier": "^3.4.0", 64 | "prettier": "^2.3.1", 65 | "typescript": "^4.3.2" 66 | }, 67 | "homepage": "https://github.com/typesense/typesense-mongodb" 68 | } 69 | -------------------------------------------------------------------------------- /src/ChangeStreams.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-explicit-any */ 2 | import { 3 | ChangeEventCR, 4 | ChangeEventDelete, 5 | ChangeEventRename, 6 | ChangeEventUpdate, 7 | ChangeStream, 8 | } from "mongodb"; 9 | import { MongoClient } from "./MongoClient"; 10 | import { TypesenseClient } from "./TypesenseClient"; 11 | 12 | enum Events { 13 | insert = "insert", 14 | update = "update", 15 | replace = "replace", 16 | delete = "delete", 17 | drop = "drop", 18 | rename = "rename", 19 | dropDatabase = "dropDatabase", 20 | invalidate = "invalidate", 21 | } 22 | 23 | export class ChangeStreams { 24 | private mongo: MongoClient; 25 | private typesense: TypesenseClient; 26 | private mongoDatabaseName: string; 27 | private mongoCollectionName: string; 28 | private typesenseCollectionName: string; 29 | private changeStream: ChangeStream; 30 | 31 | constructor( 32 | mongo: MongoClient, 33 | typesense: TypesenseClient, 34 | databaseName: string, 35 | collectionName: string, 36 | typesenseCollectionName: string 37 | ) { 38 | this.mongo = mongo; 39 | this.typesense = typesense; 40 | this.mongoDatabaseName = databaseName; 41 | this.mongoCollectionName = collectionName; 42 | this.typesenseCollectionName = typesenseCollectionName; 43 | this.changeStream = this.mongo.changeStreams( 44 | this.mongoDatabaseName, 45 | this.mongoCollectionName 46 | ); 47 | this.eventMapper(); 48 | } 49 | 50 | eventMapper(): void { 51 | this.changeStream.on("change", async (response) => { 52 | console.log(response.operationType); 53 | if (response.operationType === Events.insert) { 54 | await this.insert(response); 55 | } 56 | if (response.operationType === Events.update) { 57 | await this.update(response); 58 | } 59 | if (response.operationType === Events.replace) { 60 | await this.replace(response); 61 | } 62 | if (response.operationType === Events.delete) { 63 | await this.delete(response); 64 | } 65 | if (response.operationType === Events.drop) { 66 | await this.drop(); 67 | } 68 | if (response.operationType === Events.rename) { 69 | await this.rename(response); 70 | } 71 | }); 72 | } 73 | 74 | async closeChangeStream(): Promise { 75 | this.changeStream.close(); 76 | } 77 | 78 | async insert(response: ChangeEventCR): Promise { 79 | const data = response.fullDocument; 80 | Object.assign(data, { 81 | id: String(response.documentKey._id), 82 | }); 83 | delete data._id; 84 | await this.typesense.insertDocument(this.typesenseCollectionName, data); 85 | } 86 | 87 | async update(response: ChangeEventUpdate): Promise { 88 | const data = response.fullDocument; 89 | Object.assign(data, { 90 | id: String(response.documentKey._id), 91 | }); 92 | delete data._id; 93 | await this.typesense.updateDocument(this.typesenseCollectionName, data); 94 | } 95 | 96 | async replace(response: ChangeEventCR): Promise { 97 | const id = String(response.documentKey._id); 98 | const data = response.fullDocument; 99 | Object.assign(data, { 100 | id: id, 101 | }); 102 | delete data._id; 103 | await this.typesense.replaceDocument( 104 | this.typesenseCollectionName, 105 | id, 106 | data 107 | ); 108 | } 109 | 110 | async delete(response: ChangeEventDelete): Promise { 111 | const id = String(response.documentKey._id); 112 | await this.typesense.deleteDocument(this.typesenseCollectionName, id); 113 | } 114 | 115 | async drop(): Promise { 116 | await this.typesense.dropCollection(this.typesenseCollectionName); 117 | await this.closeChangeStream(); 118 | } 119 | 120 | async rename(response: ChangeEventRename): Promise { 121 | await this.typesense.renameCollection( 122 | this.typesenseCollectionName, 123 | `${this.mongoDatabaseName}_${response.to.coll}` 124 | ); 125 | this.mongoCollectionName = response.to.coll; 126 | this.typesenseCollectionName = `${this.mongoDatabaseName}_${response.to.coll}`; 127 | this.closeChangeStream(); 128 | this.changeStream = this.mongo.changeStreams( 129 | this.mongoDatabaseName, 130 | this.mongoCollectionName 131 | ); 132 | this.eventMapper(); 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/MongoClient.ts: -------------------------------------------------------------------------------- 1 | import { ChangeStream, MongoClient as Client } from "mongodb"; 2 | export class MongoClient { 3 | private client: Client; 4 | private url: string; 5 | private mongoOptions: { 6 | useNewUrlParser: boolean; 7 | useUnifiedTopology: boolean; 8 | }; 9 | 10 | async listAllDatabases(): Promise { 11 | let dbList = await this.client.db().admin().listDatabases(); 12 | dbList = dbList.databases.map((database) => { 13 | return database.name; 14 | }); 15 | return dbList; 16 | } 17 | 18 | async connectMongo(): Promise { 19 | try { 20 | await this.client.connect(); 21 | } catch (e) { 22 | console.error(e); 23 | } 24 | } 25 | 26 | async closeMongo(): Promise { 27 | try { 28 | await this.client.close(); 29 | } catch (e) { 30 | console.error(e); 31 | } 32 | } 33 | 34 | async listCollections(databaseName: string): Promise { 35 | let collectionList = await this.client 36 | .db(databaseName) 37 | .listCollections() 38 | .toArray(); 39 | collectionList = collectionList.map((collection) => { 40 | return collection.name; 41 | }); 42 | return collectionList; 43 | } 44 | 45 | async insertDocuments( 46 | databaseName: string, 47 | collectionName: string 48 | ): Promise { 49 | const books = await require("../data/books.json"); 50 | const sample_data = books.slice(0, 5000); 51 | const db = this.client.db(databaseName); 52 | await db.collection(collectionName).insertMany(sample_data); 53 | } 54 | 55 | async readDocuments( 56 | databaseName: string, 57 | collectionName: string 58 | ): Promise[]> { 59 | const db = this.client.db(databaseName); 60 | const result: Record[] = await db 61 | .collection(collectionName) 62 | .find() 63 | .toArray(); 64 | result.forEach((document) => { 65 | document.id = String(document._id); 66 | delete document._id; 67 | }); 68 | return result; 69 | } 70 | 71 | changeStreams( 72 | databaseName: string, 73 | collectionName: string 74 | ): ChangeStream { 75 | const collection = this.client.db(databaseName).collection(collectionName); 76 | const changeStream = collection.watch([], { fullDocument: "updateLookup" }); 77 | return changeStream; 78 | } 79 | 80 | constructor(url: string) { 81 | this.mongoOptions = { 82 | useNewUrlParser: true, 83 | useUnifiedTopology: true, 84 | }; 85 | this.client = new Client(url, this.mongoOptions); 86 | this.url = url; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/TypesenseClient.ts: -------------------------------------------------------------------------------- 1 | import { Client, Errors } from "typesense"; 2 | import { node } from "./interfaces/node"; 3 | import { schema } from "./interfaces/schema"; 4 | 5 | export class TypesenseClient { 6 | private client: Client; 7 | constructor( 8 | apiKey: string, 9 | nodes: node[], 10 | nearestNode?: node, 11 | connectionTimeout?: number 12 | ) { 13 | this.client = new Client({ 14 | nodes: nodes, 15 | nearestNode: nearestNode, 16 | apiKey: apiKey, 17 | connectionTimoutSeconds: connectionTimeout, 18 | }); 19 | } 20 | 21 | async createCollection(collectionName: string): Promise { 22 | const autoSchema: schema = { 23 | name: collectionName, 24 | fields: [{ name: ".*", type: "auto" }], 25 | }; 26 | await this.client.collections().create(autoSchema); 27 | } 28 | 29 | async importDocuments( 30 | collectionName: string, 31 | documents: Record[] 32 | ): Promise { 33 | await this.client 34 | .collections(collectionName) 35 | .documents() 36 | .import(documents, { action: "create" }); 37 | } 38 | 39 | async insertDocument( 40 | collectionName: string, 41 | document: Record 42 | ): Promise { 43 | await this.client.collections(collectionName).documents().create(document); 44 | } 45 | 46 | async updateDocument( 47 | collectionName: string, 48 | updatedDocument: Record 49 | ): Promise { 50 | await this.client 51 | .collections(collectionName) 52 | .documents() 53 | .upsert(updatedDocument); 54 | } 55 | 56 | async deleteDocument(collectionName: string, id: string): Promise { 57 | await this.client.collections(collectionName).documents(id).delete(); 58 | } 59 | 60 | async replaceDocument( 61 | collectionName: string, 62 | id: string, 63 | document: Record 64 | ): Promise { 65 | try { 66 | await this.client.collections(collectionName).documents(id).delete(); 67 | } catch (err) { 68 | if (err instanceof Errors.ObjectNotFound) { 69 | return; 70 | } 71 | throw err; 72 | } 73 | await this.client.collections(collectionName).documents().create(document); 74 | } 75 | 76 | async dropCollection(collectionName: string): Promise { 77 | await this.client.collections(collectionName).delete(); 78 | } 79 | 80 | async renameCollection( 81 | collectionName: string, 82 | newCollectionName: string 83 | ): Promise { 84 | const aliased_collection = { 85 | collection_name: collectionName, 86 | }; 87 | await this.client.aliases().upsert(newCollectionName, aliased_collection); 88 | } 89 | 90 | async checkCollection(collectionName: string): Promise { 91 | try { 92 | const result = await this.client.collections(collectionName).retrieve(); 93 | return result.num_documents; 94 | } catch (err) { 95 | if (err instanceof Errors.ObjectNotFound) { 96 | return undefined; 97 | } 98 | throw err; 99 | } 100 | } 101 | 102 | async checkServer(): Promise { 103 | return await this.client.health.retrieve(); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- 1 | import { config } from "./interfaces/config"; 2 | import { Main } from "./main"; 3 | import { parseArguments } from "./parseArguments"; 4 | 5 | export async function cli(args: string[]): Promise { 6 | const parsed: config = parseArguments(args); 7 | await Main(parsed); 8 | } 9 | -------------------------------------------------------------------------------- /src/defaults.ts: -------------------------------------------------------------------------------- 1 | import { config } from "./interfaces/config"; 2 | 3 | const defaults: config = { 4 | mongodbDatabaseName: "database", 5 | mongodbCollectionName: "collection", 6 | typesenseCollectionName: "collection", 7 | mongodbURL: "mongodb://localhost:27017", 8 | typesenseURL: "http://localhost:8108", 9 | typesenseKey: "xyz", 10 | }; 11 | 12 | export default defaults; 13 | -------------------------------------------------------------------------------- /src/interfaces/config.ts: -------------------------------------------------------------------------------- 1 | export interface config { 2 | mongodbDatabaseName: string; 3 | mongodbCollectionName: string; 4 | typesenseCollectionName: string; 5 | mongodbURL: string; 6 | typesenseURL: string; 7 | typesenseKey: string; 8 | } 9 | -------------------------------------------------------------------------------- /src/interfaces/node.ts: -------------------------------------------------------------------------------- 1 | export interface node { 2 | host: string; 3 | port: string; 4 | protocol: string; 5 | } 6 | -------------------------------------------------------------------------------- /src/interfaces/schema.ts: -------------------------------------------------------------------------------- 1 | export interface schema { 2 | name: string; 3 | fields: field[]; 4 | default_sorting_field?: string; 5 | } 6 | 7 | interface field { 8 | name: string; 9 | type: string; 10 | facet?: boolean; 11 | index?: boolean; 12 | optional?: boolean; 13 | } 14 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import defaults from "./defaults"; 2 | import { config } from "./interfaces/config"; 3 | import { node } from "./interfaces/node"; 4 | import { MongoClient } from "./MongoClient"; 5 | import { TypesenseClient } from "./TypesenseClient"; 6 | import Listr from "listr"; 7 | import chalk from "chalk"; 8 | import { ChangeStreams } from "./ChangeStreams"; 9 | 10 | let typesense: TypesenseClient; 11 | let mongo: MongoClient; 12 | let need: number; 13 | 14 | function typesenseURLParser(url: string): node { 15 | const splits = url.split(":"); 16 | 17 | return { 18 | host: splits[1].slice(2, splits[1].length), 19 | port: splits[splits.length - 1], 20 | protocol: splits[0], 21 | }; 22 | } 23 | 24 | async function initializeTypesenseClient( 25 | options: config 26 | ): Promise { 27 | typesense = new TypesenseClient(options.typesenseKey, [ 28 | typesenseURLParser(options.typesenseURL), 29 | ]); 30 | try { 31 | await typesense.checkServer(); 32 | } catch (err) { 33 | console.error(err); 34 | } 35 | return typesense; 36 | } 37 | 38 | async function initializeMongoClient(options: config): Promise { 39 | mongo = new MongoClient(options.mongodbURL); 40 | try { 41 | await mongo.connectMongo(); 42 | } catch (err) { 43 | console.error(err); 44 | } 45 | return mongo; 46 | } 47 | 48 | async function checkForExistingCollection( 49 | typesense: TypesenseClient, 50 | options: config 51 | ): Promise { 52 | need = await typesense.checkCollection(options.typesenseCollectionName); 53 | return need; 54 | } 55 | 56 | async function indexExistingDocuments( 57 | typesense: TypesenseClient, 58 | mongo: MongoClient, 59 | options: config 60 | ): Promise { 61 | const document = await mongo.readDocuments( 62 | options.mongodbDatabaseName, 63 | options.mongodbCollectionName 64 | ); 65 | await typesense.importDocuments(options.typesenseCollectionName, document); 66 | } 67 | 68 | function enableChangeStreams( 69 | typesense: TypesenseClient, 70 | mongo: MongoClient, 71 | options: config 72 | ): void { 73 | new ChangeStreams( 74 | mongo, 75 | typesense, 76 | options.mongodbDatabaseName, 77 | options.mongodbCollectionName, 78 | options.typesenseCollectionName 79 | ); 80 | } 81 | 82 | export async function Main(parsed: config): Promise { 83 | const options: config = { 84 | mongodbDatabaseName: 85 | parsed.mongodbDatabaseName || defaults.mongodbDatabaseName, 86 | mongodbCollectionName: 87 | parsed.mongodbCollectionName || defaults.mongodbCollectionName, 88 | mongodbURL: parsed.mongodbURL || defaults.mongodbURL, 89 | typesenseCollectionName: 90 | parsed.typesenseCollectionName || defaults.typesenseCollectionName, 91 | typesenseKey: parsed.typesenseKey || defaults.typesenseKey, 92 | typesenseURL: parsed.typesenseURL || defaults.typesenseURL, 93 | }; 94 | 95 | const tasks = new Listr([ 96 | { 97 | title: "Initialize Typesense Client", 98 | task: () => initializeTypesenseClient(options), 99 | }, 100 | { 101 | title: "Initialize Mongo Client", 102 | task: () => initializeMongoClient(options), 103 | }, 104 | { 105 | title: "Check for an existing typesense collection", 106 | task: () => checkForExistingCollection(typesense, options), 107 | }, 108 | { 109 | title: "Create a new Typesense Collection", 110 | task: () => typesense.createCollection(options.typesenseCollectionName), 111 | skip: () => 112 | need 113 | ? "Found an existing collection skipping create collection" 114 | : undefined, 115 | }, 116 | { 117 | title: "Index existing documents", 118 | task: () => indexExistingDocuments(typesense, mongo, options), 119 | }, 120 | { 121 | title: "Open Change Stream", 122 | task: () => enableChangeStreams(typesense, mongo, options), 123 | }, 124 | ]); 125 | 126 | await tasks.run(); 127 | console.log("%s Watching for changes..", chalk.green("DONE")); 128 | } 129 | -------------------------------------------------------------------------------- /src/parseArguments.ts: -------------------------------------------------------------------------------- 1 | import arg from "arg"; 2 | import { config } from "./interfaces/config"; 3 | 4 | export function parseArguments(rawArgs: string[]): config { 5 | const args = arg( 6 | { 7 | "--mongo-database": String, 8 | "--mongo-collection": String, 9 | "--typesense-collection": String, 10 | "--typesense-api-key": String, 11 | "--typesense-url": String, 12 | "--mongo-url": String, 13 | }, 14 | { 15 | argv: rawArgs.slice(2), 16 | } 17 | ); 18 | return { 19 | mongodbDatabaseName: args["--mongo-database"], 20 | mongodbCollectionName: args["--mongo-collection"], 21 | typesenseCollectionName: args["--typesense-collection"], 22 | mongodbURL: args["--mongo-url"], 23 | typesenseURL: args["--typesense-url"], 24 | typesenseKey: args["--typesense-api-key"], 25 | }; 26 | } 27 | 28 | /* 29 | typesense-mongodb \ 30 | --mongo-database=database \ 31 | --mongo-collection=collection \ 32 | --typesense-collection=collection \ 33 | --mongo-url=mongodb://localhost:27017 \ 34 | --typesense-url=http://localhost:8108 \ 35 | --typesense-api-key=xyz 36 | */ 37 | 38 | // --mongo-database=database --mongo-collection=collection --typesense-collection=collection --mongo-url=mongodb://localhost:27017 --typesense-url=http://localhost:8108 --typesense-api-key=xyz 39 | -------------------------------------------------------------------------------- /tests/changeStreams.test.ts: -------------------------------------------------------------------------------- 1 | import { ChangeStreams } from "../src/ChangeStreams"; 2 | import { MongoClient } from "mongodb"; 3 | import { Client } from "typesense"; 4 | import { MongoClient as TestClient } from "../src/MongoClient"; 5 | import { TypesenseClient } from "../src/TypesenseClient"; 6 | import { schema } from "../src/interfaces/schema"; 7 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 8 | // @ts-ignore 9 | import { book } from "./globalSetup"; 10 | 11 | declare global { 12 | namespace NodeJS { 13 | interface Global { 14 | mongoUrl: string; 15 | mongo: MongoClient; 16 | testMongo: TestClient; 17 | typesense: typeof Client; 18 | testTypesense: TypesenseClient; 19 | autoSchema: schema; 20 | books: book[]; 21 | } 22 | } 23 | namespace jest { 24 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 25 | interface Matchers { 26 | toBeIn(expected: string): CustomMatcherResult; 27 | } 28 | } 29 | } 30 | let changeStream: ChangeStreams; 31 | const databaseName = "database"; 32 | const collectionName = "books"; 33 | const typesenseCollectionName = "database_books"; 34 | 35 | describe("ChangeStreams functions", () => { 36 | beforeEach(async () => { 37 | let data = JSON.parse(JSON.stringify(global.books.slice(0, 40))); 38 | data = data.map((obj) => { 39 | return { ...obj, _id: obj.id }; 40 | }); 41 | await global.mongo 42 | .db(databaseName) 43 | .collection(collectionName) 44 | .insertMany(data); 45 | await global.typesense 46 | .collections() 47 | .create({ ...global.autoSchema, name: typesenseCollectionName }); 48 | await global.typesense 49 | .collections(typesenseCollectionName) 50 | .documents() 51 | .import(global.books.slice(0, 40), { action: "create" }); 52 | changeStream = new ChangeStreams( 53 | global.testMongo, 54 | global.testTypesense, 55 | databaseName, 56 | collectionName, 57 | typesenseCollectionName 58 | ); 59 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 60 | }); 61 | 62 | afterEach(async () => { 63 | changeStream.closeChangeStream(); 64 | }); 65 | it("insert()", async () => { 66 | await global.mongo 67 | .db(databaseName) 68 | .collection(collectionName) 69 | .insertOne({ 70 | ...global.books[60], 71 | _id: "61", 72 | }); 73 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 74 | const result = await global.typesense 75 | .collections(typesenseCollectionName) 76 | .documents("61") 77 | .retrieve(); 78 | expect(global.books[60]).toEqual({ 79 | ...result, 80 | }); 81 | }); 82 | 83 | it("update()", async () => { 84 | const query = { 85 | _id: "1", 86 | }; 87 | const data = JSON.parse(JSON.stringify(global.books[100])); 88 | const update = { 89 | title: data.title, 90 | }; 91 | await global.mongo 92 | .db(databaseName) 93 | .collection(collectionName) 94 | .updateOne(query, { 95 | $set: update, 96 | }); 97 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 98 | const result = await global.typesense 99 | .collections(typesenseCollectionName) 100 | .documents("1") 101 | .retrieve(); 102 | expect(global.books[100].title).toEqual(result.title); 103 | expect(global.books[0].publication_year).toEqual(result.publication_year); 104 | expect(global.books[0].id).toEqual(result.id); 105 | }); 106 | 107 | it("replace()", async () => { 108 | const data = JSON.parse(JSON.stringify(global.books[100])); 109 | const query = { 110 | _id: "1", 111 | }; 112 | await global.mongo 113 | .db(databaseName) 114 | .collection(collectionName) 115 | .replaceOne(query, data, { 116 | upsert: true, 117 | }); 118 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 119 | const result = await global.typesense 120 | .collections(typesenseCollectionName) 121 | .documents("1") 122 | .retrieve(); 123 | expect(global.books[100].title).toEqual(result.title); 124 | expect(global.books[100].publication_year).toEqual(result.publication_year); 125 | expect(global.books[0].id).toEqual(result.id); 126 | }); 127 | 128 | it("delete()", async () => { 129 | const id = "1"; 130 | await global.mongo.db(databaseName).collection(collectionName).deleteOne({ 131 | _id: id, 132 | }); 133 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 134 | await expect( 135 | global.typesense 136 | .collections(typesenseCollectionName) 137 | .documents(id) 138 | .retrieve() 139 | ).rejects.toThrow("404"); 140 | }); 141 | 142 | it("drop()", async () => { 143 | await global.mongo.db(databaseName).collection(collectionName).drop(); 144 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 145 | await expect( 146 | global.typesense.collections(typesenseCollectionName).retrieve() 147 | ).rejects.toThrow("404"); 148 | }); 149 | 150 | it("rename()", async () => { 151 | await global.mongo 152 | .db(databaseName) 153 | .collection(collectionName) 154 | .rename("books_1"); 155 | const newCollectionName = "books_1"; 156 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 157 | let { num_documents } = await global.typesense 158 | .collections(`${databaseName}_books_1`) 159 | .retrieve(); 160 | expect(num_documents).toEqual(40); 161 | await global.mongo 162 | .db(databaseName) 163 | .collection(newCollectionName) 164 | .deleteOne({ 165 | _id: "10", 166 | }); 167 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 168 | ({ num_documents } = await global.typesense 169 | .collections(`${databaseName}_books_1`) 170 | .retrieve()); 171 | expect(num_documents).toEqual(39); 172 | }); 173 | }); 174 | -------------------------------------------------------------------------------- /tests/globalSetup.ts: -------------------------------------------------------------------------------- 1 | import { MongoClient } from "mongodb"; 2 | import { Client } from "typesense"; 3 | import { schema } from "../src/interfaces/schema"; 4 | import { TypesenseClient } from "../src/TypesenseClient"; 5 | import { MongoClient as TestClient } from "../src/MongoClient"; 6 | import { setup as setupDevServer } from "jest-dev-server"; 7 | import { exec } from "child_process"; 8 | import * as util from "util"; 9 | 10 | export interface book { 11 | id: string; 12 | title: string; 13 | publication_year: number; 14 | ratings_count: number; 15 | average_rating: number; 16 | authors: string[]; 17 | } 18 | declare global { 19 | namespace NodeJS { 20 | interface Global { 21 | mongoUrl: string; 22 | mongo: MongoClient; 23 | testMongo: TestClient; 24 | typesense: typeof Client; 25 | testTypesense: TypesenseClient; 26 | autoSchema: schema; 27 | books: book[]; 28 | } 29 | } 30 | namespace jest { 31 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 32 | interface Matchers { 33 | toBeIn(expected: string): CustomMatcherResult; 34 | } 35 | } 36 | } 37 | 38 | export default async function globalSetup(): Promise { 39 | await setupDevServer([ 40 | { 41 | command: "docker-compose up", 42 | port: 27017, 43 | host: "0.0.0.0", 44 | usedPortAction: "ignore", 45 | launchTimeout: 5000, 46 | }, 47 | ]); 48 | await new Promise((resolve) => setTimeout(resolve, 5000)); 49 | const newExec = util.promisify(exec); 50 | await newExec('docker exec -i mongo mongo --eval "rs.initiate()"'); 51 | } 52 | -------------------------------------------------------------------------------- /tests/globalTeardown.ts: -------------------------------------------------------------------------------- 1 | import { teardown as teardownDevServer } from "jest-dev-server"; 2 | 3 | export default async function globalTeardown(): Promise { 4 | await teardownDevServer(); 5 | } 6 | -------------------------------------------------------------------------------- /tests/mongo.test.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 2 | // @ts-ignore 3 | import { book } from "./globalSetup"; 4 | 5 | describe("MongoClient functions", () => { 6 | it("listAllDatabases()", async () => { 7 | const expectedDatabasesList = await global.mongo 8 | .db() 9 | .admin() 10 | .listDatabases(); 11 | const received = await global.testMongo.listAllDatabases(); 12 | expectedDatabasesList.databases.map((database) => { 13 | expect(received).toBeIn(database.name); 14 | }); 15 | }); 16 | 17 | it("listCollections()", async () => { 18 | const databaseName = "database"; 19 | const collectionName = "books"; 20 | const document = { 21 | name: "sample", 22 | title: "Hello", 23 | }; 24 | await global.mongo 25 | .db(databaseName) 26 | .collection(collectionName) 27 | .insertOne(document); 28 | const recieved = await global.testMongo.listCollections(databaseName); 29 | expect(recieved).toBeIn(collectionName); 30 | }); 31 | 32 | it("insertDocuments()", async () => { 33 | const databaseName = "database"; 34 | const collectionName = "collection"; 35 | await global.testMongo.insertDocuments("database", "collection"); 36 | const result = await global.mongo 37 | .db(databaseName) 38 | .collection(collectionName) 39 | .countDocuments(); 40 | expect(result).toEqual(5000); 41 | }); 42 | 43 | it("readDocuments()", async () => { 44 | const databaseName = "database"; 45 | const collectionName = "collection"; 46 | await global.mongo 47 | .db(databaseName) 48 | .collection(collectionName) 49 | .insertMany(global.books.slice(5)); 50 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 51 | const sample_data: book[] = JSON.parse( 52 | JSON.stringify(global.books.slice(5)) 53 | ); 54 | const titles: string[] = sample_data.map((book) => { 55 | return book.title; 56 | }); 57 | const result = await global.testMongo.readDocuments( 58 | databaseName, 59 | collectionName 60 | ); 61 | const recieved = result.map((book) => { 62 | return book.title; 63 | }); 64 | titles.forEach((title) => { 65 | expect(recieved).toBeIn(title); 66 | }); 67 | }); 68 | 69 | it("changeStreams()", async () => { 70 | const databaseName = "database"; 71 | const collectionName = "collection"; 72 | const document = { 73 | name: "sample", 74 | title: "Hello", 75 | }; 76 | await global.mongo 77 | .db(databaseName) 78 | .collection(collectionName) 79 | .insertOne(document); 80 | const changeStream = global.testMongo.changeStreams( 81 | databaseName, 82 | collectionName 83 | ); 84 | changeStream.on("change", (response) => { 85 | expect(response.operationType).toEqual("insert"); 86 | }); 87 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 88 | await global.mongo.db(databaseName).collection(collectionName).insertOne({ 89 | name: "36000", 90 | title: "weird", 91 | }); 92 | await Promise.resolve(new Promise((resolve) => setTimeout(resolve, 1000))); 93 | changeStream.close(); 94 | }); 95 | }); 96 | -------------------------------------------------------------------------------- /tests/setupFilesAfterEnv.ts: -------------------------------------------------------------------------------- 1 | import { MongoClient } from "mongodb"; 2 | import { MongoClient as TestClient } from "../src/MongoClient"; 3 | import { TypesenseClient } from "../src/TypesenseClient"; 4 | import { Client } from "typesense"; 5 | import { schema } from "../src/interfaces/schema"; 6 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 7 | // @ts-ignore 8 | import { book } from "./globalSetup"; 9 | import { type } from "os"; 10 | 11 | declare global { 12 | namespace NodeJS { 13 | interface Global { 14 | mongoUrl: string; 15 | mongo: MongoClient; 16 | testMongo: TestClient; 17 | typesense: typeof Client; 18 | testTypesense: TypesenseClient; 19 | autoSchema: schema; 20 | books: book[]; 21 | } 22 | } 23 | namespace jest { 24 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 25 | interface Matchers { 26 | toBeIn(expected: string): CustomMatcherResult; 27 | } 28 | } 29 | } 30 | beforeEach(async () => { 31 | expect.extend({ 32 | toBeIn(received: string[], name: string) { 33 | if (received.includes(name)) { 34 | return { 35 | pass: true, 36 | message: () => "Success", 37 | }; 38 | } else { 39 | return { 40 | pass: false, 41 | message: () => `${name} not in ${received}`, 42 | }; 43 | } 44 | }, 45 | }); 46 | global.books = await require("../data/books.json"); 47 | global.mongoUrl = "mongodb://localhost:27017"; 48 | const mongoOptions = { 49 | useNewUrlParser: true, 50 | useUnifiedTopology: true, 51 | }; 52 | const mongo = new MongoClient(global.mongoUrl, mongoOptions); 53 | global.mongo = mongo; 54 | global.testMongo = new TestClient(global.mongoUrl); 55 | try { 56 | await mongo.connect(); 57 | } catch (e) { 58 | console.error(e); 59 | } 60 | await global.testMongo.connectMongo(); 61 | const typesense = new Client({ 62 | nodes: [ 63 | { 64 | host: "localhost", 65 | port: "8108", 66 | protocol: "http", 67 | }, 68 | ], 69 | apiKey: "xyz", 70 | }); 71 | global.typesense = new Client({ 72 | nodes: [ 73 | { 74 | host: "localhost", 75 | port: "8108", 76 | protocol: "http", 77 | }, 78 | ], 79 | apiKey: "xyz", 80 | }); 81 | global.autoSchema = { 82 | name: "books", 83 | fields: [ 84 | { 85 | name: ".*", 86 | type: "auto", 87 | }, 88 | ], 89 | }; 90 | global.testTypesense = new TypesenseClient("xyz", [ 91 | { 92 | host: "localhost", 93 | port: "8108", 94 | protocol: "http", 95 | }, 96 | ]); 97 | const typesenseCollections = await typesense.collections().retrieve(); 98 | const typesenseAliases = await typesense.aliases().retrieve(); 99 | await Promise.all( 100 | typesenseAliases.aliases.map(async (c) => 101 | typesense.aliases(c.name).delete() 102 | ) 103 | ); 104 | await Promise.all( 105 | typesenseCollections.map( 106 | async (c) => await typesense.collections(c.name).delete() 107 | ) 108 | ); 109 | const mongoDatabases = await global.mongo.db().admin().listDatabases(); 110 | await Promise.all( 111 | mongoDatabases.databases.map(async (database) => { 112 | if (!["admin", "local", "config"].includes(database.name)) { 113 | await global.mongo.db(database.name).dropDatabase(); 114 | } 115 | }) 116 | ); 117 | }); 118 | 119 | afterEach(async () => { 120 | const mongoDatabases = await global.mongo.db().admin().listDatabases(); 121 | await Promise.all( 122 | mongoDatabases.databases.map(async (database) => { 123 | if (!["admin", "local", "config"].includes(database.name)) { 124 | await global.mongo.db(database.name).dropDatabase(); 125 | } 126 | }) 127 | ); 128 | 129 | const typesenseCollections = await global.typesense.collections().retrieve(); 130 | await Promise.all( 131 | typesenseCollections.map( 132 | async (c) => await global.typesense.collections(c.name).delete() 133 | ) 134 | ); 135 | 136 | await global.mongo.close(); 137 | await global.testMongo.closeMongo(); 138 | }); 139 | -------------------------------------------------------------------------------- /tests/typesense.test.ts: -------------------------------------------------------------------------------- 1 | import { MongoClient } from "mongodb"; 2 | import { Client } from "typesense"; 3 | import { MongoClient as TestClient } from "../src/MongoClient"; 4 | import { TypesenseClient } from "../src/TypesenseClient"; 5 | import { schema } from "../src/interfaces/schema"; 6 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 7 | // @ts-ignore 8 | import { book } from "./globalSetup"; 9 | 10 | declare global { 11 | namespace NodeJS { 12 | interface Global { 13 | mongoUrl: string; 14 | mongo: MongoClient; 15 | testMongo: TestClient; 16 | typesense: typeof Client; 17 | testTypesense: TypesenseClient; 18 | autoSchema: schema; 19 | books: book[]; 20 | } 21 | } 22 | namespace jest { 23 | // eslint-disable-next-line @typescript-eslint/no-unused-vars 24 | interface Matchers { 25 | toBeIn(expected: string): CustomMatcherResult; 26 | } 27 | } 28 | } 29 | 30 | describe("TypesenseClient functions", () => { 31 | it("createCollection()", async () => { 32 | const collectionName = "books"; 33 | await global.testTypesense.createCollection(collectionName); 34 | const result = await global.typesense 35 | .collections(collectionName) 36 | .retrieve(); 37 | expect(result.name).toEqual(collectionName); 38 | }); 39 | 40 | it("importDocuments()", async () => { 41 | const collectionName = "books"; 42 | await global.typesense.collections().create(global.autoSchema); 43 | const sample_data = JSON.parse(JSON.stringify(global.books.slice(0, 40))); 44 | await global.testTypesense.importDocuments(collectionName, sample_data); 45 | const result = await global.typesense 46 | .collections(collectionName) 47 | .retrieve(); 48 | expect(result.num_documents).toEqual(40); 49 | }); 50 | 51 | it("insertDocument()", async () => { 52 | const collectionName = "books"; 53 | const sample_document = JSON.parse(JSON.stringify(global.books[0])); 54 | await global.typesense.collections().create(global.autoSchema); 55 | await global.testTypesense.insertDocument(collectionName, sample_document); 56 | const result = await global.typesense 57 | .collections(collectionName) 58 | .documents(sample_document.id) 59 | .retrieve(); 60 | expect(result).toEqual(sample_document); 61 | }); 62 | 63 | it("updateDocument()", async () => { 64 | const collectionName = "books"; 65 | const sample_document = JSON.parse(JSON.stringify(global.books[0])); 66 | await global.typesense.collections().create(global.autoSchema); 67 | await global.typesense 68 | .collections(collectionName) 69 | .documents() 70 | .create(sample_document); 71 | sample_document.title = "test_name"; 72 | await global.testTypesense.updateDocument(collectionName, sample_document); 73 | const result = await global.typesense 74 | .collections(collectionName) 75 | .documents(sample_document.id) 76 | .retrieve(); 77 | expect(result).toEqual(sample_document); 78 | }); 79 | 80 | it("deleteDocument()", async () => { 81 | const collectionName = "books"; 82 | const sample_document = JSON.parse( 83 | JSON.stringify(global.books.slice(0, 100)) 84 | ); 85 | await global.typesense.collections().create(global.autoSchema); 86 | await global.typesense 87 | .collections(collectionName) 88 | .documents() 89 | .import(sample_document, { action: "create" }); 90 | await global.testTypesense.deleteDocument(collectionName, "10"); 91 | await expect( 92 | global.typesense.collections(collectionName).documents("10").retrieve() 93 | ).rejects.toThrow("404"); 94 | }); 95 | 96 | it("replaceDocument()", async () => { 97 | const collectionName = "books"; 98 | const sample_document = JSON.parse( 99 | JSON.stringify(global.books.slice(0, 100)) 100 | ); 101 | await global.typesense.collections().create(global.autoSchema); 102 | await global.typesense 103 | .collections(collectionName) 104 | .documents() 105 | .import(sample_document, { action: "create" }); 106 | const replaced_document = JSON.parse(JSON.stringify(global.books[40])); 107 | replaced_document.id = "10"; 108 | await global.testTypesense.replaceDocument( 109 | collectionName, 110 | replaced_document.id, 111 | replaced_document 112 | ); 113 | const result = await global.typesense 114 | .collections(collectionName) 115 | .documents(replaced_document.id) 116 | .retrieve(); 117 | expect(result).toEqual(replaced_document); 118 | }); 119 | 120 | it("renameCollection()", async () => { 121 | const collectionName = "books"; 122 | await global.typesense.collections().create(global.autoSchema); 123 | await global.testTypesense.renameCollection(collectionName, "books_1"); 124 | await expect( 125 | global.typesense.collections("books_1").retrieve() 126 | ).resolves.toBeDefined(); 127 | }); 128 | 129 | it("checkCollection()", async () => { 130 | const collectionName = "books"; 131 | await global.typesense.collections().create(global.autoSchema); 132 | let result: number = await global.testTypesense.checkCollection( 133 | collectionName 134 | ); 135 | expect(result).not.toBeUndefined(); 136 | result = await global.testTypesense.checkCollection("books_1"); 137 | expect(result).toBeUndefined(); 138 | }); 139 | }); 140 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "rootDir": "./src", 5 | "esModuleInterop": true, 6 | "target": "es6", 7 | "moduleResolution": "node", 8 | "sourceMap": true, 9 | "outDir": "./lib", 10 | "noImplicitAny": false, 11 | "resolveJsonModule": true, 12 | }, 13 | "include": [ 14 | "src/**/*", 15 | ], 16 | "exclude": ["tests"], 17 | "lib": ["es2015"], 18 | } 19 | --------------------------------------------------------------------------------