├── .dockerignore ├── .gitignore ├── Dockerfile ├── Procfile ├── README.md ├── app.js ├── config.example.js ├── docker-compose.yml ├── example ├── README.md ├── example.js ├── index.html ├── seed │ ├── data.json │ └── security_rules.json └── style.css ├── lib ├── DynamicPathMonitor.js ├── PathMonitor.js ├── SearchQueue.js └── fbutil.js └── package.json /.dockerignore: -------------------------------------------------------------------------------- 1 | .*? -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store/ 2 | /.idea 3 | /node_modules 4 | service-account.json 5 | config.js 6 | app.yaml -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:6.9.2-alpine 2 | 3 | ARG APP_DIR=/opt/app 4 | WORKDIR $APP_DIR 5 | 6 | COPY package.json $APP_DIR/ 7 | RUN npm install 8 | 9 | COPY lib/ $APP_DIR/lib/ 10 | COPY app.js $APP_DIR/ 11 | COPY config.example.js $APP_DIR/config.js 12 | 13 | CMD ["node", "./app.js"] 14 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | worker: node ./app.js -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Status: Archived 2 | This repository has been archived and is no longer maintained. 3 | 4 | ![status: inactive](https://img.shields.io/badge/status-inactive-red.svg) 5 | 6 | Flashlight 7 | ========== 8 | 9 | A pluggable integration with ElasticSearch to provide advanced content searches in Firebase. 10 | 11 | This script can: 12 | - monitor multiple Firebase paths and index data in real time 13 | - communicates with client completely via Firebase (client pushes search terms to `search/request` and reads results from `search/result`) 14 | - clean up old, outdated requests 15 | 16 | Getting Started 17 | =============== 18 | 19 | - Install and run [ElasticSearch](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html) or add [Bonsai service](https://addons.heroku.com/bonsai#starter) via Heroku 20 | - `git clone https://github.com/firebase/flashlight` 21 | - `npm install` 22 | - edit config.js (see comments at the top, you must set FB_URL and FB_SERVICEACCOUNT at a minimum) 23 | - `node app.js` (run the app) 24 | 25 | Check out the recommended security rules in example/seed/security_rules.json. 26 | See example/README.md to seed and run an example client app. 27 | 28 | If you experience errors like `{"error":"IndexMissingException[[firebase] missing]","status":404}`, you may need 29 | to manually create the index referenced in each path: 30 | 31 | curl -X POST http://localhost:9200/firebase 32 | 33 | To read more about setting up a Firebase service account and configuring FB_SERVICEACCOUNT, [click here](https://firebase.google.com/docs/database/server/start). 34 | 35 | Client Implementations 36 | ====================== 37 | 38 | Read `example/index.html` and `example/example.js` for a client implementation. It works like this: 39 | 40 | - Push an object to `/search/request` which has the following keys: `index`, `type`, and `q` (or `body` for advanced queries) 41 | - Listen on `/search/response` for the reply from the server 42 | 43 | The `body` object can be any valid ElasticSearch DSL structure (see [Building ElasticSearch Queries](https://github.com/firebase/flashlight#building-elasticsearch-queries)). 44 | 45 | Deploy to Heroku 46 | ================ 47 | 48 | - `cd flashlight` 49 | - `heroku login` 50 | - `heroku create` (add heroku to project) 51 | - `heroku addons:add bonsai` (install bonsai) 52 | - `heroku config` (check bonsai instance info and copy your new BONSAI_URL - you will need it later) 53 | - `heroku config:set FB_NAME= FB_TOKEN=""` (declare environment variables) 54 | - `git add config.js` (update) 55 | - `git commit -m "configure bonsai"` 56 | - `git push heroku master` (deploy to heroku) 57 | - `heroku ps:scale worker=1` (start dyno worker) 58 | 59 | ### Setup Initial Index with Bonsai 60 | 61 | After you've deployed to Heroku, you need to create your initial index name to prevent IndexMissingException error from Bonsai. Create an index called "firebase" via curl using the BONSAI_URL that you copied during Heroku deployment. 62 | 63 | - `curl -X POST /firebase` (ex: https://user:pass@yourbonsai.bonsai.io/firebase) 64 | 65 | Migration 66 | ========= 67 | 68 | 0.2.0 -> 0.3.0 69 | ----------------- 70 | 71 | Flashlight now returns the direct output of ElasticSearch, instead of just returning the _hits_ part. This change is required to support _aggregations_ and include richer information. You must change how you read the reponse accordingly. You can see example responses of Flashlight below: 72 | 73 | ### Before, in 0.2.0 74 | ``` 75 | "total" : 1000, 76 | "max_score" : null, 77 | "hits" : [ 78 | .. 79 | ] 80 | ``` 81 | ### After, in 0.3.0 82 | ``` 83 | { 84 | "took" : 63, 85 | "timed_out" : false, 86 | "_shards" : { 87 | "total" : 5, 88 | "successful" : 5, 89 | "failed" : 0 90 | }, 91 | "hits" : { 92 | "total" : 1000, 93 | "max_score" : null, 94 | "hits" : [ 95 | .. 96 | ] 97 | }, 98 | "aggregations" : { 99 | .. 100 | } 101 | } 102 | ``` 103 | 104 | Advanced Topics 105 | =============== 106 | 107 | Parsing and filtering indexed data 108 | ---------------------------------- 109 | The `paths` specified in `config.js` can include the special `filter` 110 | and `parse` functions to manipulate the contents of the index. For 111 | example, if I had a messaging app, but I didn't want to index any 112 | system-generated messages, I could add the following filter to my 113 | messages path: 114 | 115 | ``` 116 | filter: function(data) { return data.name !== 'system'; } 117 | ``` 118 | 119 | Here, data represents the JSON snapshot obtained from the database. If 120 | this method does not return true, that record will not be indexed. Note 121 | that the `filter` method is applied before `parse`. 122 | 123 | If I want to remove or alter data getting indexed, that is done using the 124 | `parse` function. For example, assume I wanted to index user records, but 125 | remove any private information from the index. I could add a parse 126 | function to do this: 127 | 128 | ``` 129 | parse: function(data) { 130 | return { 131 | first_name: data.first_name, 132 | last_name: data.last_name, 133 | birthday: new Date(data.birthday_as_number).toISOString() 134 | }; 135 | } 136 | ``` 137 | 138 | Building ElasticSearch Queries 139 | ------------------------------ 140 | 141 | The full ElasticSearch API is supported. Check out [this great tutorial](http://okfnlabs.org/blog/2013/07/01/elasticsearch-query-tutorial.html) on querying ElasticSearch. And be sure to read the [ElasticSearch API Reference](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/). 142 | 143 | ### Example: Simple text search 144 | 145 | ``` 146 | { 147 | "q": "foo*" 148 | } 149 | ``` 150 | 151 | ### Example: Paginate 152 | 153 | You can control the number of matches (defaults to 10) and initial offset for paginating search results: 154 | 155 | ``` 156 | { 157 | "from" : 0, 158 | "size" : 50, 159 | "body": { 160 | "query": { 161 | "match": { 162 | "_all": "foo" 163 | } 164 | } 165 | } 166 | }; 167 | ``` 168 | 169 | #### Example: Search for multiple tags or categories 170 | 171 | ``` 172 | { 173 | "body": { 174 | "query": { 175 | { "tag": [ "foo", "bar" ] } 176 | } 177 | } 178 | } 179 | ``` 180 | 181 | [read more](https://www.elastic.co/guide/en/elasticsearch/guide/current/complex-core-fields.html) 182 | 183 | ### Example: Search only specific fields 184 | ``` 185 | { 186 | "body": { 187 | "query": { 188 | "match": { 189 | "field": "foo", 190 | } 191 | } 192 | } 193 | } 194 | ``` 195 | 196 | ### Example: Give more weight to specific fields 197 | ``` 198 | { 199 | "body": { 200 | "query": { 201 | "multi_match": { 202 | "query": "foo", 203 | "type": "most_fields", 204 | "fields": [ 205 | "important_field^10", // adding ^10 makes this field relatively more important 206 | "trivial_field" 207 | ] 208 | } 209 | } 210 | } 211 | } 212 | ``` 213 | 214 | [read more](https://www.elastic.co/guide/en/elasticsearch/guide/current/most-fields.html) 215 | 216 | #### Helpful section of ES docs 217 | 218 | [Search lite (simple text searches with `q`)](https://www.elastic.co/guide/en/elasticsearch/guide/current/search-lite.html) 219 | [Finding exact values](https://www.elastic.co/guide/en/elasticsearch/guide/current/_finding_exact_values.html) 220 | [Sorting and relevance](https://www.elastic.co/guide/en/elasticsearch/guide/current/sorting.html) 221 | [Partial matching](https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-matching.html) 222 | [Wildcards and regexp](https://www.elastic.co/guide/en/elasticsearch/guide/current/_wildcard_and_regexp_queries.html) 223 | [Proximity matching](https://www.elastic.co/guide/en/elasticsearch/guide/current/proximity-matching.html) 224 | [Dealing with human language](https://www.elastic.co/guide/en/elasticsearch/guide/current/languages.html) 225 | 226 | Operating at massive scale 227 | -------------------------- 228 | Is Flashlight designed to work at millions or requests per second? 229 | No. It's designed to be a template for implementing your production services. 230 | Some assembly required. 231 | 232 | Here are a couple quick optimizations you can make to improve scale: 233 | * Separate the indexing worker and the query worker (this could be 234 | as simple as creating two Flashlight workers, opening `app.js` in each, 235 | and commenting out SearchQueue.init() or PathMonitor.process() respectively. 236 | * When your service restarts, all data is re-indexed. To prevent this, 237 | you can use refBuilder as described in the next section. 238 | * With a bit of work, both PathMonitor and SearchQueue could be adapted 239 | to function as a Service Worker for 240 | [firebase-queue](https://github.com/firebase/firebase-queue), 241 | allowing multiple workers and potentially hundreds of thousands of 242 | writes per second (with minor degredation and no losses at even higher throughput). 243 | 244 | Use refBuilder to improve indexing efficiency 245 | --------------------------------------------- 246 | In `config.js`, each entry in `paths` can be assigned a `refBuilder` 247 | function. This can construct a query for determining which records 248 | get indexed. 249 | 250 | This can be utilized to improve efficiency by preventing all data from 251 | being re-indexed any time the Flashlight service is restarted, and generally 252 | by preventing a large backlog from being read into memory at once. 253 | 254 | For example, if I were indexing chat messages, and they 255 | had a timestamp field, I could use the following to never look back 256 | more than a day during a server restart: 257 | 258 | ``` 259 | exports.paths = [ 260 | { 261 | path : "chat/messages", 262 | index : "firebase", 263 | type : "message", 264 | fields: ['message_body', 'tags'], 265 | refBuilder: function(ref, path) { 266 | return ref.orderByChild('timestamp').startAt(Date.now()); 267 | } 268 | } 269 | ]; 270 | ``` 271 | 272 | Loading paths to index from the database instead of config file 273 | --------------------------------------------------------------- 274 | 275 | Paths to be indexed can be loaded dynamically from the database by 276 | providing a path string instead of the `paths` array. For example, 277 | the paths given in config.example.js could be replaced with `dynamic_paths` 278 | and then those paths could be stored in the database, similar to 279 | [this](https://kato-flashlight-dev.firebaseio.com/dynamic_paths.json?print=pretty). 280 | 281 | Any updates to the database paths are handled by Flashlight (new paths are 282 | indexed when they are added, old paths stop being indexed when they 283 | are removed). 284 | 285 | Unfortunately, since JSON data stored in Firebase can't contain functions, 286 | the `filter`, `parser`, and `refBuilder` options can't be used with this 287 | approach. 288 | 289 | Support 290 | ======= 291 | Submit questions or bugs using the [issue tracker](https://github.com/firebase/flashlight). 292 | 293 | For Firebase-releated questions, try the [mailing list](https://groups.google.com/forum/#!forum/firebase-talk). 294 | 295 | License 296 | ======= 297 | 298 | [MIT LICENSE](http://firebase.mit-license.org/) 299 | Copyright © 2013 Firebase 300 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /* 4 | * @version 0.3, 3 June 2014 5 | */ 6 | 7 | var elasticsearch = require('elasticsearch'), 8 | conf = require('./config'), 9 | fbutil = require('./lib/fbutil'), 10 | PathMonitor = require('./lib/PathMonitor'), 11 | SearchQueue = require('./lib/SearchQueue'); 12 | 13 | var escOptions = { 14 | hosts: [{ 15 | host: conf.ES_HOST, 16 | port: conf.ES_PORT, 17 | auth: (conf.ES_USER && conf.ES_PASS) ? conf.ES_USER + ':' + conf.ES_PASS : null 18 | }] 19 | }; 20 | 21 | for (var attrname in conf.ES_OPTS) { 22 | if( conf.ES_OPTS.hasOwnProperty(attrname) ) { 23 | escOptions[attrname] = conf.ES_OPTS[attrname]; 24 | } 25 | } 26 | 27 | // connect to ElasticSearch 28 | var esc = new elasticsearch.Client(escOptions); 29 | 30 | console.log('Connecting to ElasticSearch host %s:%s'.grey, conf.ES_HOST, conf.ES_PORT); 31 | 32 | var timeoutObj = setInterval(function() { 33 | esc.ping() 34 | .then(function() { 35 | console.log('Connected to ElasticSearch host %s:%s'.grey, conf.ES_HOST, conf.ES_PORT); 36 | clearInterval(timeoutObj); 37 | initFlashlight(); 38 | }); 39 | }, 5000); 40 | 41 | function initFlashlight() { 42 | console.log('Connecting to Firebase %s'.grey, conf.FB_URL); 43 | fbutil.init(conf.FB_URL, conf.FB_SERVICEACCOUNT); 44 | PathMonitor.process(esc, conf.paths, conf.FB_PATH); 45 | SearchQueue.init(esc, conf.FB_REQ, conf.FB_RES, conf.CLEANUP_INTERVAL); 46 | } -------------------------------------------------------------------------------- /config.example.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This config file is provided as a convenience for development. You can either 3 | * set the environment variables on your server or modify the values here. 4 | * 5 | * At a minimum, you must set FB_URL and Paths to Monitor. Everything else is optional, assuming your 6 | * ElasticSearch server is at localhost:9200. 7 | */ 8 | 9 | /** Firebase Settings 10 | ***************************************************/ 11 | 12 | // Your Firebase instance where we will listen and write search results 13 | exports.FB_URL = process.env.FB_URL || 'https://.firebaseio.com'; 14 | 15 | // The path in your Firebase where clients will write search requests 16 | exports.FB_REQ = process.env.FB_REQ || 'search/request'; 17 | 18 | // The path in your Firebase where this app will write the results 19 | exports.FB_RES = process.env.FB_RES || 'search/response'; 20 | 21 | // See https://firebase.google.com/docs/server/setup for instructions 22 | // to auto-generate the service-account.json file 23 | exports.FB_SERVICEACCOUNT = process.env.FB_ACC || 'service-account.json'; 24 | 25 | /** ElasticSearch Settings 26 | *********************************************/ 27 | 28 | if( process.env.BONSAI_URL ) { 29 | processBonsaiUrl(exports, process.env.BONSAI_URL); 30 | } 31 | else { 32 | // ElasticSearch server's host URL 33 | exports.ES_HOST = process.env.ES_HOST || 'localhost'; 34 | 35 | // ElasticSearch server's host port 36 | exports.ES_PORT = process.env.ES_PORT || '9200'; 37 | 38 | // ElasticSearch username for http auth 39 | exports.ES_USER = process.env.ES_USER || null; 40 | 41 | // ElasticSearch password for http auth 42 | exports.ES_PASS = process.env.ES_PASS || null; 43 | } 44 | 45 | /** Paths to Monitor 46 | * 47 | * Each path can have these keys: 48 | * {string} path: [required] the Firebase path to be monitored, for example, `users/profiles` 49 | * would monitor https://.firebaseio.com/users/profiles 50 | * {string} index: [required] the name of the ES index to write data into 51 | * {string} type: [required] name of the ES object type this document will be stored as 52 | * {Array} fields: list of fields to be monitored and indexed (defaults to all fields, ignored if "parser" is specified) 53 | * {Array} omit: list of fields that should not be indexed in ES (ignored if "parser" is specified) 54 | * {Function} filter: if provided, only records that return true are indexed 55 | * {Function} parser: if provided, the results of this function are passed to ES, rather than the raw data (fields is ignored if this is used) 56 | * {Function} refBuilder: see README 57 | * 58 | * To store your paths dynamically, rather than specifying them all here, you can store them in Firebase. 59 | * Format each path object with the same keys described above, and store the array of paths at whatever 60 | * location you specified in the FB_PATHS variable. Be sure to restrict that data in your Security Rules. 61 | ****************************************************/ 62 | exports.paths = [ 63 | { 64 | path : "users", 65 | index: "firebase", 66 | type : "user" 67 | }, 68 | { 69 | path : "messages", 70 | index : "firebase", 71 | type : "message", 72 | fields: ['msg', 'name'], 73 | filter: function(data) { return data.name !== 'system'; } 74 | // see readme 75 | //, parser: function(data) { data.msg = data.msg.toLowerCase(); return data; } 76 | // see readme 77 | //, refBuilder: function(ref, path) { return ref.orderBy(path.sortField).startAt(Date.now()); } 78 | } 79 | ]; 80 | 81 | // Paths can also be stored in Firebase! See README for details. 82 | //exports.paths = process.env.FB_PATHS || null; 83 | 84 | // Additional options for ElasticSearch client 85 | exports.ES_OPTS = { 86 | //requestTimeout: 60000, maxSockets: 100, log: 'error' 87 | }; 88 | 89 | /** Config Options 90 | ***************************************************/ 91 | 92 | // How often should the script remove unclaimed search results? probably just leave this alone 93 | exports.CLEANUP_INTERVAL = 94 | process.env.NODE_ENV === 'production' ? 95 | 3600 * 1000 /* once an hour */ : 96 | 60 * 1000 /* once a minute */; 97 | 98 | function processBonsaiUrl(exports, url) { 99 | var matches = url.match(/^https?:\/\/([^:]+):([^@]+)@([^/]+)\/?$/); 100 | exports.ES_HOST = matches[3]; 101 | exports.ES_PORT = 80; 102 | exports.ES_USER = matches[1]; 103 | exports.ES_PASS = matches[2]; 104 | console.log('Configured using BONSAI_URL environment variable', url, exports); 105 | } -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | 3 | services: 4 | flashlight: 5 | build: . 6 | image: flashlight:dev 7 | environment: 8 | FB_URL: https://my-project.firebaseio.com 9 | ES_HOST: elasticsearch 10 | NODE_ENV: production 11 | volumes: 12 | - ./service-account.json:/opt/app/service-account.json 13 | 14 | elasticsearch: 15 | image: elasticsearch:5.1.1-alpine 16 | -------------------------------------------------------------------------------- /example/README.md: -------------------------------------------------------------------------------- 1 | # Running this Example 2 | 3 | - Create a dev [Firebase instance](https://www.firebase.com/account) 4 | - Import example/seed/data.json into your dev instance 5 | - [Start ElasticSearch](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/setup.html) 6 | - [Download ElasticSearch](http://www.elasticsearch.org/download) 7 | - Unzip/Untar 8 | - `./bin/elasticsearch` 9 | - Configure `config.js` with your settings (you can use defaults if you set up ElasticSearch locally) 10 | - `node app.js` 11 | - Edit `example/example.js` to point to your Firebase URL 12 | - Start a local web server 13 | - `npm install -g serve` 14 | - `cd example/` 15 | - `serve` 16 | - Open [example/index.html in your browser](http://localhost:3000) 17 | -------------------------------------------------------------------------------- /example/example.js: -------------------------------------------------------------------------------- 1 | (function ($) { 2 | "use strict"; 3 | 4 | /**====== SET ME =====**/ 5 | /**====== SET ME =====**/ 6 | /**====== SET ME =====**/ 7 | // Set the configuration for your app 8 | // TODO: Replace with your project's config object 9 | var config = { 10 | databaseURL: "https://kato-flashlight.firebaseio.com" 11 | }; 12 | 13 | // TODO: Replace this with the path to your ElasticSearch queue 14 | // TODO: This is monitored by your app.js node script on the server 15 | // TODO: And this should match your seed/security_rules.json 16 | var PATH = "search"; 17 | /**====== /SET ME =====**/ 18 | /**====== /SET ME =====**/ 19 | /**====== /SET ME =====**/ 20 | 21 | // Initialize connection using our project credentials 22 | firebase.initializeApp(config); 23 | 24 | // Get a reference to the database service 25 | var database = firebase.database(); 26 | 27 | // handle form submits and conduct a search 28 | // this is mostly DOM manipulation and not very 29 | // interesting; you're probably interested in 30 | // doSearch() and buildQuery() 31 | $('form').on('submit', function(e) { 32 | e.preventDefault(); 33 | var $form = $(this); 34 | $('#results').text(''); 35 | $('#total').text(''); 36 | $('#query').text(''); 37 | if( $form.find('[name=term]').val() ) { 38 | doSearch(buildQuery($form)); 39 | } 40 | }); 41 | 42 | function buildQuery($form) { 43 | // this just gets data out of the form 44 | var index = $form.find('[name=index]').val(); 45 | var type = $form.find('[name="type"]:checked').val(); 46 | var term = $form.find('[name="term"]').val(); 47 | var matchWholePhrase = $form.find('[name="exact"]').is(':checked'); 48 | var size = parseInt($form.find('[name="size"]').val()); 49 | var from = parseInt($form.find('[name="from"]').val()); 50 | 51 | // skeleton of the JSON object we will write to DB 52 | var query = { 53 | index: index, 54 | type: type 55 | }; 56 | 57 | // size and from are used for pagination 58 | if( !isNaN(size) ) { query.size = size; } 59 | if( !isNaN(from) ) { query.from = from; } 60 | 61 | buildQueryBody(query, term, matchWholePhrase); 62 | 63 | return query; 64 | } 65 | 66 | function buildQueryBody(query, term, matchWholePhrase) { 67 | if( matchWholePhrase ) { 68 | var body = query.body = {}; 69 | body.query = { 70 | // match_phrase matches the phrase exactly instead of breaking it 71 | // into individual words 72 | "match_phrase": { 73 | // this is the field name, _all is a meta indicating any field 74 | "_all": term 75 | } 76 | /** 77 | * Match breaks up individual words and matches any 78 | * This is the equivalent of the `q` string below 79 | "match": { 80 | "_all": term 81 | } 82 | */ 83 | } 84 | } 85 | else { 86 | query.q = term; 87 | } 88 | } 89 | 90 | // conduct a search by writing it to the search/request path 91 | function doSearch(query) { 92 | var ref = database.ref().child(PATH); 93 | var key = ref.child('request').push(query).key; 94 | 95 | console.log('search', key, query); 96 | $('#query').text(JSON.stringify(query, null, 2)); 97 | ref.child('response/'+key).on('value', showResults); 98 | } 99 | 100 | // when results are written to the database, read them and display 101 | function showResults(snap) { 102 | if( !snap.exists() ) { return; } // wait until we get data 103 | var dat = snap.val().hits; 104 | 105 | // when a value arrives from the database, stop listening 106 | // and remove the temporary data from the database 107 | snap.ref.off('value', showResults); 108 | snap.ref.remove(); 109 | 110 | // the rest of this just displays data in our demo and probably 111 | // isn't very interesting 112 | var totalText = dat.total; 113 | if( dat.hits && dat.hits.length !== dat.total ) { 114 | totalText = dat.hits.length + ' of ' + dat.total; 115 | } 116 | $('#total').text('(' + totalText + ')'); 117 | 118 | var $pair = $('#results') 119 | .text(JSON.stringify(dat, null, 2)) 120 | .removeClass('error zero'); 121 | if( dat.error ) { 122 | $pair.addClass('error'); 123 | } 124 | else if( dat.total < 1 ) { 125 | $pair.addClass('zero'); 126 | } 127 | } 128 | 129 | // display raw data for reference, this is just for the demo 130 | // and probably not very interesting 131 | database.ref().on('value', setRawData); 132 | function setRawData(snap) { 133 | $('#raw').text(JSON.stringify(snap.val(), null, 2)); 134 | } 135 | })(jQuery); 136 | -------------------------------------------------------------------------------- /example/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 21 | 22 | 23 | 24 | 25 | Flashlight: A search library using Firebase and ElasticSearch 26 | 27 | 28 | 29 |
30 |

Flashlight: Search library built on Firebase and ElasticSearch

31 |
32 |
33 |
34 | 35 | 36 |
37 | 38 | 39 | 40 | 41 |
42 |
    43 |
    • 44 |
    • 45 |
  • 46 |
    • 47 |
48 |
    49 |
  • Pagination:
    • 50 |
    • 51 |
    • 52 |
53 |

You may use * or ? for wild cards in your search.

54 |
55 |
56 |
57 |
58 |
59 |

Results 0

60 | 
(enter a search term)
61 |

The query used

62 | 

63 |         

64 |         

65 |            
Raw data

66 |             

67 |         

68 |     

69 |     

70 |         
MIT License | Firebase

71 |     

72 | 

73 | 
74 | Fork me on GitHub
75 | 
76 | 
77 | 
78 | 
79 | 
80 | 
81 | 
82 | 
83 | 
84 | 
85 | 
86 | 


--------------------------------------------------------------------------------
/example/seed/data.json:
--------------------------------------------------------------------------------
  1 | {
  2 |    "users": {
  3 |       "kato": {
  4 |          "name": "Michael Wulf",
  5 |          "nick": "Kato",
  6 |          "martial art": "Kung Fu"
  7 |       },
  8 |       "bruce": {
  9 |          "name": "Bruce Lee",
 10 |          "nick": "Little Phoenix",
 11 |          "martial art": "Jeet Kune Do"
 12 |       },
 13 |       "chuck": {
 14 |          "name": "Carlos Ray Norris",
 15 |          "nick": "Chuck",
 16 |          "martial art": "Chun Kuk Do"
 17 |       },
 18 |       "musashi": {
 19 |          "name": "Miyamoto Musashi",
 20 |          "nick": "Musashi",
 21 |          "martial art": "Hyōhō Niten Ichi-ryū"
 22 |       },
 23 |       "chan": {
 24 |          "name": "Jackie Chan",
 25 |          "nick": "Cannonball",
 26 |          "martial art": "Kung Fu"
 27 |       },
 28 |       "kaiso": {
 29 |          "name": "Morihei Ueshiba",
 30 |          "nick": "Kaiso",
 31 |          "martial art": "Aikido"
 32 |       }
 33 |    },
 34 |    "messages": {
 35 |       "-J3cHinvzvlUPjx4j3or" : {
 36 |          "msg" : "Whether I am or not, you've been more than fair.",
 37 |          "name" : "Man in Black",
 38 |          "time" : 1379360825251
 39 |       },
 40 |       "-J3cHim6gLsDpdkw4gOP" : {
 41 |          "msg" : "Stop saying that!",
 42 |          "name" : "Count Rugen",
 43 |          "time" : 1379360506466
 44 |       },
 45 |       "-J3cHimJJr-jM-FcYhGS" : {
 46 |          "msg" : "You keep using that word. I do not think it means what you think it means.",
 47 |          "name" : "Inigo Montoya",
 48 |          "time" : 1379360531877
 49 |       },
 50 |       "-J3cHin3nNHyGkdDoVJU" : {
 51 |          "msg" : "I... am not left-handed.",
 52 |          "name" : "Inigo Montoya",
 53 |          "time" : 1379360663613
 54 |       },
 55 |       "-J3cHimybK8ztWAf2rUz" : {
 56 |          "msg" : "And what is that?",
 57 |          "name" : "Man in Black",
 58 |          "time" : 1379360649865
 59 |       },
 60 |       "-J3cHio2kk27UwU5dOwD" : {
 61 |          "msg" : "You seem a decent fellow... I hate to die.",
 62 |          "name" : "Man in Black",
 63 |          "time" : 1379360876086
 64 |       },
 65 |       "-J3cHin_pu54kajCnD7X" : {
 66 |          "msg" : "Sonny, true love is the greatest thing, in the world-except for a nice MLT - mutton, lettuce and tomato sandwich, where the mutton is nice and lean and the tomato is ripe",
 67 |          "name" : "Miracle Max",
 68 |          "time" : 1379360755785
 69 |       },
 70 |       "-J3cHinCUCwcv5X0sg8V" : {
 71 |          "msg" : "You are amazing.",
 72 |          "name" : "Man in Black",
 73 |          "time" : 1379360668763
 74 |       },
 75 |       "-J3cHim0Pip2vYB5ih9N" : {
 76 |          "msg" : "Hello. My name is Inigo Montoya. You killed my father. Prepare to die.",
 77 |          "name" : "Inigo Montoya",
 78 |          "time" : 1379360499777
 79 |       },
 80 |       "-J3cHinoEs1yOSKzOZf0" : {
 81 |          "msg" : "You are ready then?",
 82 |          "name" : "Inigo Montoya",
 83 |          "time" : 1379360824329
 84 |       },
 85 |       "-J3cHimP84vQ8Kid9qvn" : {
 86 |          "msg" : "I'm not a witch, I'm your wife. But after what you just said, I'm not even sure I want to be that any more.",
 87 |          "name" : "Valerie",
 88 |          "time" : 1379360550246
 89 |       },
 90 |       "-J3cHio0I7xkDynubSBQ" : {
 91 |          "msg" : "[drawing his sword] You seem a decent fellow... I hate to kill you.",
 92 |          "name" : "Inigo Montoya",
 93 |          "time" : 1379360851322
 94 |       },
 95 |       "-J3cHimkVeoVCEqfSNQq" : {
 96 |          "msg" : "I admit it, you are better than I am.",
 97 |          "name" : "Inigo Montoya",
 98 |          "time" : 1379360596961
 99 |       },
100 |       "-J3cHineYGIfa_X-d1l2" : {
101 |          "msg" : "More pursue than study lately. You see, I cannot find him... it's been twenty years now and I'm starting to lose confidence. I just work for Vizzini to pay the bills. There's not a lot of money in revenge.",
102 |          "name" : "Inigo Montoya",
103 |          "time" : 1379360796489
104 |       },
105 |       "-J3cHimWMUaFAIGTNYjU" : {
106 |          "msg" : "You are wonderful.",
107 |          "name" : "Inigo Montoya",
108 |          "time" : 1379360552187
109 |       },
110 |       "-J3cHinJS4j_wJs0ALUv" : {
111 |          "msg" : "Oh, there's something I ought to tell you.",
112 |          "name" : "Man in Black",
113 |          "time" : 1379360694591
114 |       },
115 |       "-J3cHimCtV0WEjiIalFr" : {
116 |          "msg" : "HE DIDN'T FALL? INCONCEIVABLE.",
117 |          "name" : "Vizzini",
118 |          "time" : 1379360523517
119 |       },
120 |       "-J3cHio67-CVCLSGp4tM" : {
121 |          "msg" : "[confidently] Begin.",
122 |          "name" : "Inigo Montoya",
123 |          "time" : 1379360877263
124 |       },
125 |       "-J3cHimMYswXnVfEBJ24" : {
126 |          "msg" : "Get back, witch.",
127 |          "name" : "Miracle Max",
128 |          "time" : 1379360533431
129 |       },
130 |       "-J3cHinFvlXY0QkvdPyM" : {
131 |          "msg" : "I ought to be, after 20 years.",
132 |          "name" : "Inigo Montoya",
133 |          "time" : 1379360671667
134 |       },
135 |       "-J3cHimsbDWM-aC7-c8H" : {
136 |          "msg" : "Because I know something you don't know.",
137 |          "name" : "Inigo Montoya",
138 |          "time" : 1379360626360
139 |       },
140 |       "-J3cHinjLpY0yt7fPFjK" : {
141 |          "msg" : "Well I certainly hope you find him someday.",
142 |          "name" : "Man in Black",
143 |          "time" : 1379360797458
144 |       },
145 |       "-J3cHinMtaSYErZTqY2Y" : {
146 |          "msg" : "Tell me.",
147 |          "name" : "Inigo Montoya",
148 |          "time" : 1379360703671
149 |       },
150 |       "-J3cHinPJ9YibEycsgM1" : {
151 |          "msg" : "I'm not left-handed either.",
152 |          "name" : "Man in Black",
153 |          "time" : 1379360727162
154 |       },
155 |       "-J3cHimouQdcYuFJ5tLA" : {
156 |          "msg" : "Then why are you smiling?",
157 |          "name" : "Man in Black",
158 |          "time" : 1379360621832
159 |       },
160 |       "-J3cHim_vmfIN--0G_Iu" : {
161 |          "msg" : "Thank you; I've worked hard to become so.",
162 |          "name" : "Man in Black",
163 |          "time" : 1379360569143
164 |       },
165 |       "-J3cHincoaMtXsYy4MEt" : {
166 |          "msg" : "[intrigued] You've done nothing but study sword-play?",
167 |          "name" : "Man in Black",
168 |          "time" : 1379360767716
169 |       }
170 |    }
171 | }


--------------------------------------------------------------------------------
/example/seed/security_rules.json:
--------------------------------------------------------------------------------
 1 | {
 2 |    "rules": {
 3 |      ".read": false,
 4 |      ".write": false,
 5 |      "search": {
 6 |          "request": {
 7 |             "$recid": {
 8 |                // I can only read records assigned to me
 9 |                ".read": "auth.id === data.child('id').val() || auth.uid === data.child('id').val()",
10 |                // I can only write new records that don't exist yet
11 |                ".write": "!data.exists() && (newData.child('id').val() === auth.id || newData.child('id').val() === auth.uid)",
12 |                ".validate": "newData.hasChildren(['index', 'type']) && (newData.hasChild('q') || newData.hasChild('query') || newData.hasChild('body'))",
13 |                "index": {
14 |                   // accepts arrays or strings
15 |                   ".validate": "(newData.isString() && newData.val().length < 1000) || newData.hasChildren()",
16 |                   "$child": {
17 |                      ".validate": "newData.isString() && newData.val().length < 1000"
18 |                   }
19 |                },
20 |                "type": {
21 |                   // accepts arrays or strings
22 |                   ".validate": "(newData.isString() && newData.val().length < 1000) || newData.hasChildren()",
23 |                   "$child": {
24 |                      ".validate": "newData.isString() && newData.val().length < 1000"
25 |                   }
26 |                },
27 |                "query": {
28 |                   // lucene formatted string, such as "title:search_term" or a `body` attribute
29 |                   // see https://www.elastic.co/guide/en/elasticsearch/guide/current/query-dsl-intro.html
30 |                   ".validate": "newData.isString() || newData.hasChildren()"
31 |                },
32 |                "body": {
33 |                   // The `body` object of an ES search, such as { size: 25, from: 0, query: "*foo*" }, see
34 |                   // https://www.elastic.co/guide/en/elasticsearch/guide/current/query-dsl-intro.html
35 |                   ".validate": "newData.hasChildren()"
36 |                },
37 |                "q": {
38 |                   // lucene formatted 'lite' search string, such as "*foo*" or "+name:(mary john) +date:>2014-09-10", see
39 |                   // https://www.elastic.co/guide/en/elasticsearch/guide/current/search-lite.html
40 |                   ".validate": "newData.isString()"
41 |                },
42 |                "size": {
43 |                   ".validate": "newData.isNumber() && newData.val() >= 0"
44 |                },
45 |                "from": {
46 |                  ".validate": "newData.isNumber() && newData.val() >= 0"
47 |                },
48 |                "$other": {
49 |                   ".validate": false
50 |                }
51 |             }
52 |          },
53 |          "response": {
54 |            ".indexOn": "timestamp",
55 |             "$recid": {
56 |               // I can only read/write records assigned to me
57 |               ".read": "auth.uid === data.child('id').val()",
58 |               // delete only
59 |               ".write": "auth.uid === data.child('id').val() && !newData.exists()"
60 |             }
61 |          }
62 |       }
63 |    }
64 | }
65 | 


--------------------------------------------------------------------------------
/example/style.css:
--------------------------------------------------------------------------------
 1 | 
 2 | h2 span {
 3 |    padding: 0 5px;
 4 |    background-color: #eaeaea;
 5 |    border-radius: 5px;
 6 | }
 7 | 
 8 | h2 span.zero, .zero {
 9 |    color: #999;
10 | }
11 | 
12 | h2 span.error, .error {
13 |    color: red;
14 | }
15 | 
16 | #fork_on_github {
17 |    position: absolute;
18 |    top: 0;
19 |    right: 0;
20 |    border: 0;
21 | }
22 | 
23 | .list-inline label {
24 |    display: inline-block;
25 |    padding: 3px;
26 |    cursor: pointer;
27 | }
28 | 
29 | .list-inline li.divider {
30 |    padding: 3px 20px;
31 | }
32 | 
33 | .list-inline label:hover {
34 |    background-color: #fafafa;
35 | }
36 | 
37 | @media (max-width: 767px) {
38 |    #fork_on_github { display: none }
39 | }
40 | 
41 | .paginate-fields [type=number] {
42 |    width: 4em;
43 | }


--------------------------------------------------------------------------------
/lib/DynamicPathMonitor.js:
--------------------------------------------------------------------------------
 1 | var fbutil = require('./fbutil');
 2 | 
 3 | function DynamicPathMonitor(ref, factory) {
 4 |   console.log('Dynamic monitoring started', ref.toString());
 5 |   this.factory = factory;
 6 |   this.paths = {}; // store instance of monitor, so we can unset it if the value changes
 7 |   ref.on('child_added', this._add.bind(this));
 8 |   ref.on('child_changed', this._change.bind(this));
 9 |   ref.on('child_removed', this._remove.bind(this));
10 | }
11 | 
12 | DynamicPathMonitor.prototype = {
13 |   _add: function(snap) {
14 |     var name = snap.key;
15 |     var pathProps = snap.val();
16 |     if ( isValidPath(pathProps) ) {
17 |       this.paths[name] = this.factory(pathProps);
18 |       console.log('Monitoring dynamic index %s (%s/%s at path %s)'.blue, name, pathProps.index, pathProps.type, pathProps.path);
19 |     }
20 |     else {
21 |       console.error('Invalid dynamic path fetched from db. Most be an object with index, type, and path attributes.', name, pathProps);
22 |     }
23 |   },
24 |   _remove: function(snap) {
25 |     this._purge(snap.key);
26 |   },
27 |   _change: function(snap) {
28 |     var name = snap.key;
29 |     this._purge(name);
30 |     this._add(snap);
31 |   },
32 |   _purge: function(name) {
33 |     // kill old monitor
34 |     if (this.paths[name]) {
35 |       var path = this.paths[name];
36 |       this.paths[name]._stop();
37 |       this.paths[name] = null;
38 |       console.log('Stopped monitoring dynamic index %s (%s/%s at path %s)'.blue, name, path.index, path.type, fbutil.pathName(path.ref));
39 |     }
40 |   }
41 | };
42 | 
43 | function isValidPath(props) {
44 |   return props && typeof(props) === 'object' && props.index && props.path && props.type;
45 | }
46 | 
47 | module.exports = DynamicPathMonitor;
48 | 


--------------------------------------------------------------------------------
/lib/PathMonitor.js:
--------------------------------------------------------------------------------
  1 | 
  2 | var fbutil = require('./fbutil');
  3 | var DynamicPathMonitor = require('./DynamicPathMonitor');
  4 | require('colors');
  5 | 
  6 | function PathMonitor(esc, path) {
  7 |    this.ref = fbutil.fbRef(path.path);
  8 |    if( fbutil.isFunction(path.refBuilder) ) {
  9 |      this.ref = path.refBuilder(this.ref, path);
 10 |    }
 11 |    console.log('Indexing %s/%s from DB "%s"'.grey, path.index, path.type, fbutil.pathName(this.ref));
 12 |    this.esc = esc;
 13 | 
 14 |    this.index = path.index;
 15 |    this.type  = path.type;
 16 |    this.filter = path.filter || function() { return true; };
 17 |    this.parse  = path.parser || function(data) { return parseKeys(data, path.fields, path.omit) };
 18 | 
 19 |    this._init();
 20 | }
 21 | 
 22 | PathMonitor.prototype = {
 23 |    _init: function() {
 24 |       this.addMonitor = this.ref.on('child_added', this._process.bind(this, this._childAdded));
 25 |       this.changeMonitor = this.ref.on('child_changed', this._process.bind(this, this._childChanged));
 26 |       this.removeMonitor = this.ref.on('child_removed', this._process.bind(this, this._childRemoved));
 27 |    },
 28 | 
 29 |    _stop: function() {
 30 |       this.ref.off('child_added', this.addMonitor);
 31 |       this.ref.off('child_changed', this.changeMonitor);
 32 |       this.ref.off('child_removed', this.removeMonitor);
 33 |    },
 34 | 
 35 |    _process: function(fn, snap) {
 36 |       var dat = snap.val();
 37 |       if( this.filter(dat) ) {
 38 |          fn.call(this, snap.key, this.parse(dat));
 39 |       }
 40 |    },
 41 |    
 42 |    _index: function (key, data, callback) {
 43 |      this.esc.index({
 44 |       index: this.index,
 45 |       type: this.type,
 46 |       id: key,
 47 |       body: data
 48 |     }, function (error, response) {
 49 |       if (callback) {
 50 |         callback(error, response);    
 51 |       }
 52 |     }.bind(this));
 53 |   },
 54 | 
 55 |    _childAdded: function(key, data) {
 56 |       var name = nameFor(this, key);
 57 |       this._index(key, data, function (error, response) {
 58 |         if (error) {
 59 |           console.error('failed to index %s: %s'.red, name, error);
 60 |         } else {
 61 |           console.log('indexed'.green, name);
 62 |         }
 63 |       }.bind(this));
 64 |    },
 65 | 
 66 |    _childChanged: function(key, data) {
 67 |       var name = nameFor(this, key);
 68 |       this._index(key, data, function (error, response) {
 69 |         if (error) {
 70 |           console.error('failed to update %s: %s'.red, name, error);
 71 |         } else {
 72 |           console.log('updated'.green, name);
 73 |         }
 74 |       }.bind(this));
 75 |    },
 76 | 
 77 |    _childRemoved: function(key, data) {
 78 |       var name = nameFor(this, key);
 79 |       this.esc.delete({
 80 |         index: this.index, 
 81 |         type: this.type, 
 82 |         id: key
 83 |       }, function(error, data) {
 84 |          if( error ) {
 85 |             console.error('failed to delete %s: %s'.red, name, error);
 86 |          }
 87 |          else {
 88 |             console.log('deleted'.cyan, name);
 89 |          }
 90 |       }.bind(this));
 91 |    }
 92 | };
 93 | 
 94 | function nameFor(path, key) {
 95 |    return path.index + '/' + path.type + '/' + key;
 96 | }
 97 | 
 98 | function parseKeys(data, fields, omit) {
 99 |   if (!data || typeof(data)!=='object') {
100 |     return data;
101 |   }
102 |   var out = data;
103 |   // restrict to specified fields list
104 |   if( Array.isArray(fields) && fields.length) {
105 |     out = {};
106 |     fields.forEach(function(f) {
107 |       if( data.hasOwnProperty(f) ) {
108 |         out[f] = data[f];
109 |       }
110 |     })
111 |   }
112 |   // remove omitted fields
113 |   if( Array.isArray(omit) && omit.length) {
114 |     omit.forEach(function(f) {
115 |       if( out.hasOwnProperty(f) ) {
116 |         delete out[f];
117 |       }
118 |     })
119 |   }
120 |   return out;
121 | }
122 | 
123 | exports.process = function(esc, paths) {
124 |   if( fbutil.isString(paths) ) {
125 |     new DynamicPathMonitor(fbutil.fbRef(paths), function(pathProps) {
126 |       return new PathMonitor(esc, pathProps);
127 |     });
128 |   }
129 |   else if( fbutil.isObject(paths) && paths.length ) {
130 |     paths.forEach(function(pathProps) {
131 |       new PathMonitor(esc, pathProps);
132 |     });
133 |   }
134 |   else {
135 |     console.warn("No paths have been specified to index".yellow);
136 |   }
137 | };
138 | 


--------------------------------------------------------------------------------
/lib/SearchQueue.js:
--------------------------------------------------------------------------------
  1 | var fbutil = require('./fbutil');
  2 | 
  3 | function SearchQueue(esc, reqRef, resRef, cleanupInterval) {
  4 |    this.esc = esc;
  5 |    this.inRef = reqRef;
  6 |    this.outRef = resRef;
  7 |    this.cleanupInterval = cleanupInterval;
  8 |    console.log('Queue started, IN: "%s", OUT: "%s"'.grey, fbutil.pathName(this.inRef), fbutil.pathName(this.outRef));
  9 |    setTimeout(function() {
 10 |       this.inRef.on('child_added', this._process, this);
 11 |    }.bind(this), 1000);
 12 |    this._nextInterval();
 13 | }
 14 | 
 15 | SearchQueue.prototype = {
 16 |   _process: function(snap) {
 17 |      var dat = snap.val();
 18 |      var key = snap.key;
 19 | 
 20 |     console.log('processing query request'.grey, key);
 21 | 
 22 |      var query = this._buildQuery(key, dat);
 23 |      if( query === null ) { return; }
 24 | 
 25 |      console.log('built query'.grey, query);
 26 |      this.esc.search(query, function(error, response) {
 27 |        if (error) {
 28 |          this._replyError(key, error);
 29 |        } else {
 30 |          this._reply(key, response);
 31 |        }
 32 |      }.bind(this));
 33 |   },
 34 | 
 35 |    _reply: function(key, results) {
 36 |       if( results.error ) {
 37 |          this._replyError(key, results.error);
 38 |       }
 39 |       else {
 40 |          console.log('query result %s: %d hits'.yellow, key, results.hits.total);
 41 |          this._send(key, results);
 42 |       }
 43 |    },
 44 | 
 45 |    _replyError: function(key, err) {
 46 |      this._send(key, { total: 0, error: fbutil.unwrapError(err), timestamp: Date.now() })
 47 |    },
 48 | 
 49 |    _send: function(key, data) {
 50 |       this.inRef.child(key).remove().catch(this._logErrors.bind(this, 'Unable to remove queue item!'));
 51 |       this.outRef.child(key).set(data).catch(this._logErrors.bind(this, 'Unable to send reply!'));
 52 |    },
 53 | 
 54 |    _logErrors: function(message, err) {
 55 |       if( err ) {
 56 |          console.error(message.red);
 57 |          console.error(err.red);
 58 |       }
 59 |    },
 60 | 
 61 |    _housekeeping: function() {
 62 |       var self = this;
 63 |      // remove all responses which are older than CHECK_INTERVAL
 64 |      this.outRef.orderByChild('timestamp')
 65 |        .endAt(new Date().valueOf() - self.cleanupInterval)
 66 |        .once('value', function(snap) {
 67 |          var count = snap.numChildren();
 68 |          if( count ) {
 69 |            console.warn('housekeeping: found %d outbound orphans (removing them now) %s'.red, count, new Date());
 70 |            snap.forEach(function(ss) { ss.ref.remove(); });
 71 |          }
 72 |          self._nextInterval();
 73 |        });
 74 |    },
 75 | 
 76 |    _nextInterval: function() {
 77 |       var interval = this.cleanupInterval > 60000? 'minutes' : 'seconds';
 78 |       console.log('Next cleanup in %d %s'.grey, Math.round(this.cleanupInterval/(interval==='seconds'? 1000 : 60000)), interval);
 79 |       setTimeout(this._housekeeping.bind(this), this.cleanupInterval);
 80 |    },
 81 | 
 82 |    _buildQuery: function(key, queryData) {
 83 |      if( !this._assertValidSearch(key, queryData) ) {
 84 |        return null;
 85 |      }
 86 | 
 87 |      // legacy support: q and body were merged on the client as `query`
 88 |      // in previous versions; this makes sure they still work
 89 |      if( fbutil.isString(queryData.query) ) {
 90 |        queryData.q = queryData.query;
 91 |      }
 92 |      else if( fbutil.isObject(queryData.query) ) {
 93 |        queryData.body = queryData.query;
 94 |      }
 95 | 
 96 |      if( fbutil.isString(queryData.body) ) {
 97 |        queryData.body = this._getJSON(queryData.body);
 98 |        if( queryData.body === null ) {
 99 |          this._replyError(key, 'Search body was a string but did not contain a valid JSON object. It must be an object or a JSON parsable string.');
100 |          return null;
101 |        }
102 |      }
103 | 
104 |      var query = {};
105 | 
106 |      Object.keys(queryData).filter(function(k) {
107 |        return k !== 'query';
108 |      }).forEach(function(k) {
109 |        query[k] = queryData[k];
110 |      });
111 | 
112 |      return query;
113 |    },
114 | 
115 |     _assertValidSearch: function(key, props) {
116 |       if( !fbutil.isObject(props) ||
117 |         !fbutil.isString(props.index) ||
118 |         !fbutil.isString(props.type)
119 |       ) {
120 |         this._replyError(key, 'Search request must be a valid object with keys index, type, and one of q or body.');
121 |         return false;
122 |       }
123 | 
124 |       if( !fbutil.isString(props.query) && fbutil.isObject(props.query) &&
125 |         !fbutil.isString(props.q) &&
126 |         !fbutil.isObject(props.body) &&
127 |         !fbutil.isString(props.body)
128 |       ) {
129 |         this._replyError(key, 'Search must contain one of (string)q or (object)body. (Legacy `query` is deprecated but still works)');
130 |         return false;
131 |       }
132 | 
133 |       return true;
134 |     },
135 | 
136 | 
137 |   _getJSON: function(str) {
138 |        try {
139 |            return JSON.parse(str);
140 |        } catch (e) {
141 |            console.log('Error parsing JSON body', e);
142 |            return null;
143 |        }
144 |    }
145 | };
146 | 
147 | exports.init = function(esc, reqPath, resPath, matchWholeWords, cleanupInterval) {
148 |    new SearchQueue(esc, fbutil.fbRef(reqPath), fbutil.fbRef(resPath), matchWholeWords, cleanupInterval);
149 | };
150 | 


--------------------------------------------------------------------------------
/lib/fbutil.js:
--------------------------------------------------------------------------------
 1 | 'use strict';
 2 | 
 3 | var admin = require('firebase-admin');
 4 | require('colors');
 5 | 
 6 | exports.init = function(databaseURL, serviceAccount) {
 7 |    var config = {
 8 |      databaseURL: databaseURL,
 9 |      credential: admin.credential.cert(serviceAccount)
10 |    };
11 |    admin.initializeApp(config)
12 | };
13 | 
14 | exports.fbRef = function(path) {
15 |    return admin.database().ref().child(path);
16 | };
17 | 
18 | exports.pathName = function(ref) {
19 |    var p = ref.parent.key;
20 |    return (p? p+'/' : '')+ref.key;
21 | };
22 | 
23 | exports.isString = function(s) {
24 |   return typeof s === 'string';
25 | };
26 | 
27 | exports.isObject = function(o) {
28 |   return o && typeof o === 'object';
29 | };
30 | 
31 | exports.unwrapError = function(err) {
32 |   if( err && typeof err === 'object' ) {
33 |     return err.toString();
34 |   }
35 |   return err;
36 | };
37 | 
38 | exports.isFunction = function(f) {
39 |   return typeof f === 'function';
40 | };
41 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "Flashlight",
 3 |   "version": "0.3.0",
 4 |   "description": "A search utility for Firebase",
 5 |   "homepage": "https://github.com/firebase/flashlight",
 6 |   "scripts": {
 7 |     "start": "node app.js",
 8 |     "monitor": "nodemon app.js",
 9 |     "deploy": "gcloud app deploy"
10 |   },
11 |   "main": "app.js",
12 |   "devDependencies": {},
13 |   "dependencies": {
14 |     "JQDeferred": "~1.9.1",
15 |     "colors": "~0.6.2",
16 |     "elasticsearch": "^11.0.1",
17 |     "firebase-admin": "^4.0.4"
18 |   }
19 | }
20 | 


--------------------------------------------------------------------------------