├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | **/node_modules/* 3 | **/.firebaserc 4 | */npm-debug.log 5 | *~ 6 | **/package-lock.json 7 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '8' 4 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | We love contributions! But we'd like you to follow these guidelines to make the contribution process 4 | easy for everyone involved: 5 | 6 | ## Issue Tracker 7 | 8 | The issue tracker is our main channel to report a bug in the source code or a mistake in the 9 | documentation, and to request a new feature. 10 | 11 | Before submitting your issue, please search the archive. Maybe your question was already answered, 12 | your bug has already been reported or your feature has already been requested. 13 | 14 | Providing the following information will increase the chances of your issue being dealt with 15 | quickly: 16 | 17 | * **Bug reports** - if an error is being thrown, please include a stack trace and the steps to 18 | reproduce the error. If there is no error, please explain why do you consider it a bug. 19 | 20 | * **Feature Requests** - please make it clear whether you're willing to write the code for it or you 21 | need someone else to do it. 22 | 23 | * **Related Issues** - if you found a similar issue that has been reported before, be sure to 24 | mention it. 25 | 26 | ## Pull Requests 27 | 28 | Before making any changes, consider following these steps: 29 | 30 | 1. Search for an open or closed Pull Request related to your changes. 31 | 32 | 2. Search the issue tracker for issues related to your changes. 33 | 34 | 3. Open a [new issue](github.com/rosariopfernandes/tocha/issues/new) to discuss your changes 35 | with the project owners. If they approve it, send the Pull Request. 36 | 37 | ### Sending Pull Requests 38 | If your change has been approved, follow this process: 39 | 40 | 1. [Fork](http://help.github.com/fork-a-repo/) the project, clone your fork and configure the 41 | remotes: 42 | 43 | ```bash 44 | # Clone your fork into the current directory 45 | git clone https://github.com// 46 | # Navigate to the newly cloned directory 47 | cd 48 | # Assign the original repo to a remote called "upstream" 49 | git remote add upstream https://github.com/rosariopfernandes/Tocha 50 | ``` 51 | 52 | 2. Make your changes in a new branch: 53 | 54 | ```bash 55 | git checkout -b my-fix-branch master 56 | ``` 57 | 58 | 3. Commit the changes using a descriptive commit message 59 | 60 | ```bash 61 | git commit -a 62 | ``` 63 | 64 | Note: the optional commit `-a` command line option will automatically "add" and "rm" edited files. 65 | 66 | 4. Push your branch to GitHub: 67 | 68 | ```bash 69 | git push origin my-fix-branch 70 | ``` 71 | 72 | 5. In GitHub, [send a Pull Request](https://help.github.com/articles/using-pull-requests/) with a 73 | clear title and description. 74 | 75 | * If we suggest changes then: 76 | * Make the required changes; 77 | * Rebase your branch and force push to your GitHub repository (this updates your Pull Request): 78 | ```bash 79 | # Rebase the branch 80 | git rebase master -i 81 | # Update the Pull Request 82 | git push origin my-fix-branch -f 83 | ``` 84 | That's it! Thank you for you contribution! 85 | 86 | ### After your Pull Request is merged 87 | 88 | You can delete your branch and pull changes from the original 89 | (upstream) repository: 90 | 91 | 1. Delete the remote branch on GitHub either through the GitHub UI or your local shell as follows: 92 | 93 | ```bash 94 | git push origin --delete my-fix-branch 95 | ``` 96 | 97 | 2. Check out the master branch: 98 | 99 | ```bash 100 | git checkout master -f 101 | ``` 102 | 103 | 3. Delete the local branch: 104 | 105 | ```bash 106 | git branch -D my-fix-branch 107 | ``` 108 | 109 | 4. Update your master with the latest upstream version: 110 | 111 | ```bash 112 | git pull --ff upstream master 113 | ``` 114 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Rosário Pereira Fernandes 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/rosariopfernandes/Tocha.svg?branch=master)](https://travis-ci.org/rosariopfernandes/Tocha) 2 | 3 | # Tocha [DEPRECATED] 4 | 5 | **TOCHA STARTED AS A PERSONAL PROJECT AND WAS VERY USEFUL TO BUILD PROTOTYPES AND SMALL APPS 6 | WHILE KEEPING FIREBASE PROJECTS ON THE SPARK PLAN.** 7 | 8 | **STARTING AUGUST 17 2020, CLOUD FUNCTIONS FOR FIREBASE WILL DEPRECATE THEIR NODE 8 RUNTIME AND 9 | DEPLOYING NODE 10 FUNCTIONS WILL REQUIRE BILLING, WHICH MEANS CLOUD FUNCTIONS WILL NO LONGER BE 10 | AVAILABLE ON THE SPARK PLAN. BECAUSE TOCHA RELIES ON CLOUD FUNCTIONS, THIS MEANS THIS PROJECT WILL NO LONGER BE ABLE TO SERVE ITS PURPOSE.** 11 | 12 | **I'VE DECIDED THAT THE BEST THING TO DO IS TO DEPRECATE THE PROJECT. CONSIDER `ARRAY-CONTAINS` 13 | AS A FREE WORKAROUND TO IMPLEMENT TEXT SEARCH IN FIRESTORE (SEE THIS [BLOG POST](https://medium.com/@ken11zer01/firebase-firestore-text-search-and-pagination-91a0df8131ef)).** 14 | 15 | **THANK YOU FOR EVERYONE WHO CONTRIBUTED, SUPPORTED OR USED TOCHA.** 16 | 17 | Full-Text Search for Firebase Projects using the [Spark (Free) plan](https://firebase.google.com/pricing/) 18 | (no billing enabled). 19 | 20 | Use Tocha if you want to implement full-text search on: 21 | - Android/iOS App Prototypes; 22 | - Small Android/iOS Apps using the Firebase Spark Plan. 23 | 24 | If your app has scaled and you're using a plan with billing enabled (Flame or Blaze), then this library is 25 | **not for you**. Instead, prefer using the 26 | [solution recommended on the Firebase Documentation](https://firebase.google.com/docs/firestore/solutions/search). 27 | 28 | Using Tocha on web (JavaScript) applications is also **not recommended**, as it might be redundant. If you want to 29 | implement full-text search on your js app, consider using [Lunr.js](https://github.com/olivernn/lunr.js). 30 | 31 | ## Getting Started 32 | 33 | ### Prerequisites 34 | 35 | To install and deploy Tocha, you'll need: 36 | - [Node.js](https://nodejs.org/en/download/), which comes with the Node Package Manager (npm); 37 | - The [Firebase CLI](https://github.com/firebase/firebase-tools) which can be installed using 38 | `npm install -g firebase-tools`. See full installation details on 39 | [the Documentation](https://firebase.google.com/docs/cli/). 40 | 41 | ### Installing 42 | 43 | Tocha runs on Cloud Functions, so you'll need to install the Firebase CLI (as instructed on the 44 | [Prerequisites](#Prerequisites)) in order to setup Cloud Functions for your Firebase Project. 45 | 46 | _(If you know how to deploy Cloud Functions to Firebase and have already done so, you may skip to step 4.)_ 47 | 48 | 1. Create a new directory on your local machine for the project and navigate into it: 49 | ```bash 50 | mkdir my_tocha_project 51 | cd my_tocha_project 52 | ``` 53 | 54 | 2. If you haven't already, login to Firebase: 55 | ```bash 56 | firebase login 57 | ``` 58 | This will launch a web page for you to authenticate your Firebase Account. 59 | 60 | 3. Initialize and configure your Firebase Project. In this step, you'll be asked what project you'll be using and what 61 | features you would like to setup (be sure to select `functions` at least). 62 | ```bash 63 | firebase init 64 | ``` 65 | If successful, this should create 2 files under your project directory: `package.json` and `index.js`. 66 | 67 | 4. Open the `package.json` file on your favorite text editor and make sure you have set node version to 8: 68 | ```json5 69 | { 70 | // ... name, description, dependencies, etc 71 | "engines": { 72 | "node": "8" 73 | } 74 | } 75 | ``` 76 | 77 | 5. Install Tocha using: 78 | ```bash 79 | npm install tocha 80 | ``` 81 | 82 | 6. Open the `index.js` file on the text editor, import Tocha and create the functions you need: 83 | ```js 84 | const functions = require('firebase-functions'); 85 | // ... you may have more imports here ... 86 | const tocha = require('tocha'); 87 | 88 | // Add this line to enable Full-Text Search for Cloud Firestore 89 | exports.searchFirestore = tocha.searchFirestore; 90 | 91 | // Add this line to enable Full-Text Search for the Realtime Database 92 | exports.searchRTDB = tocha.searchRTDB; 93 | 94 | // ... you may have more cloud functions here ... 95 | ``` 96 | 97 | 98 | ## Deployment 99 | 100 | To deploy your functions to Firebase, you can either: 101 | - deploy the cloud functions and all the other tools you have enabled for that project: 102 | ```bash 103 | firebase deploy 104 | ``` 105 | - or deploy the cloud functions only 106 | ```bash 107 | firebase deploy --only functions 108 | ``` 109 | 110 | ## Example 111 | Let's say we have a Firestore Collection named "notes" with the following Documents: 112 | ```json 113 | { 114 | "note1": { 115 | "title": "Remember to buy butter", 116 | "description": "Valeria asked me to get some butter at the supermarket on my way home." 117 | }, 118 | "note2": { 119 | "title": "Eta's birthday coming up", 120 | "description": "Eta is turning 28 this Friday. Don't forget to call her wishing HBD." 121 | } 122 | } 123 | ``` 124 | 125 | ### Performing a Simple Search 126 | In order to search the collection, you'll need to create a new collection named "tocha_searches" and add a new document 127 | to it. This document should contain the following fields: 128 | - `collectionName` - the name of the collection to be searched; 129 | - `fields` - array of fields to search on. 130 | - `query` - the word/expression you're looking for. 131 | 132 | **Example 1:** Let's run an exact search for notes with the word "butter". Our document would look like this: 133 | ```json 134 | { 135 | "collectionName": "notes", 136 | "fields": ["title"], 137 | "query": "butter" 138 | } 139 | ``` 140 | 141 | **Example 2:** Sometimes you may need an inexact search. Let's look for the note about Eta's birthday: 142 | ```json 143 | { 144 | "collectionName": "notes", 145 | "fields": ["title", "description"], 146 | "query": "Eta*" 147 | } 148 | ``` 149 | 150 | Notice that on the last example we've used the wildcard `*`. You can find the 151 | [list of all possible wildcards, boosts and fuzzy matchings here](https://lunrjs.com/guides/searching.html). 152 | 153 | --- 154 | 155 | Adding that document to the collection should trigger our Cloud Function which adds a `response` field to it. 156 | This field is an array of matches. Each match contains the following fields: 157 | - `id` - the id of the document that matches our query; 158 | - `score` - the relevance of the document, calculated using the [BM25](https://en.wikipedia.org/wiki/Okapi_BM25) 159 | algorithm. Find out more [here](https://lunrjs.com/guides/searching.html#scoring). 160 | - `data` - the actual document returned by our query. 161 | 162 | So our document from Example 1 would become: 163 | ```json 164 | { 165 | "collectionName": "notes", 166 | "fields": ["title"], 167 | "query": "butter", 168 | "response": { 169 | "result": [ 170 | { 171 | "id": "note1", 172 | "score": 0.856, 173 | "data": { 174 | "title": "Remember to buy butter", 175 | "description": "Valeria asked me to get some butter at the supermarket on my way home." 176 | } 177 | } 178 | ], 179 | "isSuccessful": true 180 | } 181 | } 182 | ``` 183 | 184 | ### Advanced Search on Cloud Firestore 185 | Although running a text-search in the whole collection is great, sometimes you may need to filter this collection before 186 | running the search. And to do that, you can use these optional parameters: 187 | 188 | #### `where` 189 | An array of map values where you can perform simple or compound queries for firestore. Each map in this array must 190 | contain the following fields: 191 | - `field` - the field to filter on. 192 | - `operator` - a query operator (can be `<`, `<=`, `==`, `>`, `>=`, `array-contains`, `in`, or `array-contains-any`). 193 | - `val` or `value` - the value to filter on. 194 | 195 | **Example:** Suppose our notes had one more field named `ownerUID`, which tells us which user created the note. 196 | 197 | We might want to query only on the notes created by a specific user (`uid: randomUserUID`). To do that, we can use: 198 | ```json5 199 | { 200 | // ... Other Fields (collectionName, fields, query, etc) 201 | "where": [ 202 | { 203 | "field": "ownerUID", 204 | "operator": "==", 205 | "value": "randomUserUID" 206 | } 207 | // Optionally, you can add more filter maps here. 208 | ] 209 | } 210 | ``` 211 | 212 | See a full list of valid filters and query limitations on the 213 | [Firebase Documentation](https://firebase.google.com/docs/firestore/query-data/queries). 214 | 215 | #### `orderBy`, `limit` and `limitToLast` 216 | The `orderBy` field allows you to order the result of your query. This field is an array of map objects with 2 fields: 217 | - `field` - the field to sort on. 218 | - `direction` (optional) - `asc` for ascending order or `desc` for descending. If you omit this field, it will use 219 | ascending order. 220 | 221 | **Note:** If you want to order by multiple fields you might need to 222 | [Create an Index on Firestore](https://firebase.google.com/docs/firestore/query-data/indexing). 223 | 224 | The `limit` field allows you to get only the first `n` documents retrieved, where `n` is the positive number 225 | you pass as value of the field. 226 | 227 | **Example:** Let's order our notes by `title` and get the first 5: 228 | ```json5 229 | { 230 | // ... Other Fields (collectionName, fields, query, etc) 231 | "orderBy": [ 232 | { 233 | "field": "title", 234 | "direction": "desc" 235 | } 236 | // Optionally, you can add more orderBy maps here. 237 | ], 238 | "limit": 5 239 | } 240 | ``` 241 | 242 | The `limitToLast` field allows you to get only the last `n` documents retrieved, where `n` is the positive number 243 | you pass as value of the field. 244 | 245 | Please **note** that you need at least one `orderBy` field to use `limitToLast`, otherwise it will return an exception. 246 | 247 | ### Advanced Search on The Realtime Database 248 | If you need to sort/filter your data before performing a search, you can add these optional parameters to your query: 249 | 250 | #### orderChild, orderValue, orderKey 251 | Those are equivalent to `orderByChild()`, `orderByValue()` and `orderByKey()`. You can find more about it on the 252 | [documentation](https://firebase.google.com/docs/database/web/lists-of-data#sort_data). 253 | 254 | **Example usage:** 255 | ```json5 256 | { 257 | // ... Other Fields (collectionName, fields, query, etc) 258 | "orderValue": true, 259 | "orderKey": true, 260 | "orderChild": "birthday" // ordering by the birthday child 261 | } 262 | ``` 263 | 264 | Please note that you can only use **one** order-by method at a time. Calling an order-by method multiple times in the 265 | same query throws an error. 266 | 267 | #### limitFirst, limitLast, startAtBound, endAtBound, equalToBound 268 | Equivalent to `limitToFirst()`, `limitToLast()`, `startAt()`, `endAt()` and `equalTo()`. See the 269 | [documentation](https://firebase.google.com/docs/database/web/lists-of-data#filtering_data) for more details. 270 | 271 | **Example usage:** 272 | ```json5 273 | { 274 | // ... Other Fields (collectionName, fields, query, etc) 275 | "limitFirst": 10, 276 | "startAtBound": 5, 277 | "equalToBound": "john.doe@tocha.com" 278 | } 279 | ``` 280 | 281 | ## Contributing 282 | 283 | Anyone and everyone is welcome to contribute. Please take a moment to review the 284 | [Contributing Guidelines](CONTRIBUTING.md). 285 | 286 | ## License 287 | 288 | This project is licensed under the [MIT LICENSE](LICENSE). 289 | 290 | ## Acknowledgments 291 | 292 | - This project makes use of the [Lunr.js](https://github.com/olivernn/lunr.js) library by 293 | [Oliver Nightingale](https://github.com/olivernn). 294 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const functions = require('firebase-functions'); 2 | const admin = require('firebase-admin'); 3 | admin.initializeApp(); 4 | 5 | const firestore = admin.firestore(); 6 | firestore.settings({timestampsInSnapshots: true}); 7 | 8 | const lunr = require('lunr'); 9 | const tochaCollection = 'tocha_searches'; 10 | 11 | // Full Text-Search on Cloud Firestore 12 | exports.searchFirestore = functions.firestore 13 | .document(tochaCollection + '/{searchId}') 14 | .onCreate(async (snap, context) => { 15 | let responseResults = []; 16 | let response = { 17 | result: responseResults 18 | }; 19 | try { 20 | // Obtain the request parameters 21 | const req = snap.data(); 22 | const collectionName = req.collectionName; 23 | const fields = req.fields; 24 | const query = req.query; 25 | const queryRef = req.queryRef; 26 | const where = req.where; // array containing all the extra queries 27 | const orderBy = req.orderBy; // object containing field and direction 28 | const limit = req.limit; 29 | const limitToLast = req.limitToLast; 30 | 31 | // Construct the query to the collection being searched 32 | let userCollection = firestore.collection(collectionName); 33 | if (where) { 34 | where.forEach(function(subquery) { 35 | if (subquery.val && subquery.field && subquery.operator) { 36 | userCollection = userCollection.where(subquery.field, subquery.operator, subquery.val); 37 | } else if (subquery.value && subquery.field && subquery.operator) { 38 | userCollection = userCollection.where(subquery.field, subquery.operator, subquery.value); 39 | } 40 | }); 41 | } 42 | if (orderBy) { 43 | orderBy.forEach(function (sortOrder) { 44 | if (sortOrder.direction && sortOrder.field) { 45 | userCollection = userCollection.orderBy(sortOrder.field, sortOrder.direction); 46 | } else if (sortOrder.field) { 47 | userCollection = userCollection.orderBy(sortOrder.field); 48 | } 49 | }); 50 | } 51 | if (limit) { 52 | userCollection = userCollection.limit(limit); 53 | } else if (limitToLast) { 54 | userCollection = userCollection.limitToLast(limitToLast); 55 | } 56 | 57 | // Read all the documents from the collection to be searched 58 | const querySnapshot = await userCollection.get(); 59 | let documents = []; 60 | let lunrIndex = lunr(function() { 61 | if (queryRef) { 62 | this.ref(queryRef); 63 | } else { 64 | this.ref('key'); 65 | } 66 | for (let i in fields) { 67 | this.field(fields[i]); 68 | } 69 | querySnapshot.forEach(function (docSnapshot) { 70 | let snapshotData = docSnapshot.data(); 71 | documents[docSnapshot.id] = docSnapshot.data(); 72 | snapshotData.key = docSnapshot.id; 73 | this.add(snapshotData); 74 | }, this); 75 | }); 76 | const results = lunrIndex.search(query); 77 | results.forEach(function(result) { 78 | responseResults.push({ 79 | id: result.ref, 80 | score: result.score, 81 | data: documents[result.ref] 82 | }) 83 | }); 84 | response.isSuccessful = true; 85 | } catch (e) { 86 | console.log(e); 87 | response.isSuccessful = false; 88 | response.errorMessage = e.toString(); 89 | } 90 | 91 | 92 | return firestore.collection(tochaCollection).doc(context.params.searchId) 93 | .update({ 94 | response: response, 95 | responseTimestamp: admin.firestore.FieldValue.serverTimestamp() 96 | }); 97 | }); 98 | 99 | // Full Text-Search on the Realtime Database 100 | exports.searchRTDB = functions.database 101 | .ref(tochaCollection + '/{searchId}') 102 | .onCreate((snap, context) => { 103 | const responseResults = []; 104 | let response = { 105 | result: responseResults 106 | }; 107 | const database = admin.database(); 108 | try { 109 | // Obtain the request parameters 110 | const req = snap.val(); 111 | const nodeName = req.collectionName; 112 | const fields = req.fields; 113 | const query = req.query; 114 | const queryRef = req.queryRef; 115 | 116 | // Read everything from the node to be searched 117 | return database.ref(nodeName) 118 | .once('value', function(dataSnapshot) { 119 | try { 120 | let documents = new Map(); 121 | dataSnapshot.forEach(function (snapshot) { 122 | let snapshotVal = snapshot.val(); 123 | snapshotVal.key = snapshot.key; 124 | documents.set(snapshot.key, snapshotVal); 125 | }); 126 | let lunrIndex = lunr(function () { 127 | if (queryRef) { this.ref(queryRef); } else { this.ref('key'); } 128 | 129 | for (let i in fields) { this.field(fields[i]); } 130 | 131 | documents.forEach(function (value) { this.add(value); }, this); 132 | }); 133 | const results = lunrIndex.search(query); 134 | results.forEach(function (result) { 135 | responseResults.push({ 136 | id: result.ref, 137 | score: result.score, 138 | data: documents.get(result.ref) 139 | }); 140 | }); 141 | response.isSuccessful = true; 142 | } catch (e) { 143 | response.isSuccessful = false; 144 | response.errorMessage = e.toString(); 145 | } 146 | database.ref(tochaCollection).child(context.params.searchId) 147 | .update({ 148 | response: response, 149 | responseTimestamp: admin.database.ServerValue.TIMESTAMP 150 | }); 151 | }) 152 | } catch (e) { 153 | response.isSuccessful = false; 154 | response.errorMessage = e.toString(); 155 | return database.ref(tochaCollection).child(context.params.searchId) 156 | .update({ 157 | response: response, 158 | responseTimestamp: admin.database.ServerValue.TIMESTAMP 159 | }); 160 | } 161 | 162 | }); 163 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tocha", 3 | "version": "0.0.3", 4 | "description": "Full-text Search for Firebase through Cloud Functions", 5 | "homepage": "https://github.com/rosariopfernandes/Tocha", 6 | "bugs": "https://github.com/rosariopfernandes/Tocha/issues", 7 | "license": "MIT", 8 | "repository": { 9 | "type": "git", 10 | "url": "https://github.com/rosariopfernandes/Tocha.git" 11 | }, 12 | "dependencies": { 13 | "firebase-admin": "~8.10.0", 14 | "firebase-functions": "^3.6.0", 15 | "lunr": "^2.3.5" 16 | }, 17 | "scripts": { 18 | "test": "node --check *.js" 19 | }, 20 | "engines": { 21 | "node": "8" 22 | } 23 | } 24 | --------------------------------------------------------------------------------