├── .jshintignore
├── lib
├── database
│ ├── datasource.js
│ ├── main_database_adapter.js
│ └── elasticsearch_adapter.js
├── systemMessage.js
├── message_queue
│ ├── messaging_client.js
│ ├── kafka_queue.js
│ ├── azure_servicebus_queue.js
│ └── amqp_queue.js
├── logger
│ └── logger.js
├── Admin.js
├── TelepatIndexedLists.js
├── Context.js
├── Delta.js
├── Channel.js
├── User.js
├── TelepatError.js
├── ConfigurationManager.js
├── Application.js
├── Subscription.js
└── Model.js
├── LICENSE
├── .gitignore
├── .npmignore
├── .jshintrc
├── utils
├── profiling.js
├── filterbuilder.js
└── utils.js
├── package.json
├── index.js
├── README.md
└── CHANGELOG.md
/.jshintignore:
--------------------------------------------------------------------------------
1 | /client/
2 | /node_modules/
3 |
--------------------------------------------------------------------------------
/lib/database/datasource.js:
--------------------------------------------------------------------------------
1 | function Datasource() {
2 | /**
3 | *
4 | * @type {Main_Database_Adapter}
5 | */
6 | this.dataStorage = null;
7 | this.cacheStorage = null;
8 | };
9 |
10 | /**
11 | *
12 | * @param {Main_Database_Adapter} database
13 | */
14 | Datasource.prototype.setMainDatabase = function(database) {
15 | this.dataStorage = database;
16 | };
17 |
18 | Datasource.prototype.setCacheDatabase = function(database) {
19 | this.cacheStorage = database;
20 | };
21 |
22 | module.exports = Datasource;
23 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright 2015 Telepat
2 |
3 | Licensed under the Apache License, Version 2.0 (the "License");
4 | you may not use this file except in compliance with the License.
5 | You may obtain a copy of the License at
6 |
7 | http://www.apache.org/licenses/LICENSE-2.0
8 |
9 | Unless required by applicable law or agreed to in writing, software
10 | distributed under the License is distributed on an "AS IS" BASIS,
11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | See the License for the specific language governing permissions and
13 | limitations under the License.
14 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 |
5 | # Runtime data
6 | pids
7 | *.pid
8 | *.seed
9 |
10 | # Directory for instrumented libs generated by jscoverage/JSCover
11 | lib-cov
12 |
13 | # Coverage directory used by tools like istanbul
14 | coverage
15 |
16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
17 | .grunt
18 |
19 | # node-waf configuration
20 | .lock-wscript
21 |
22 | # Compiled binary addons (http://nodejs.org/api/addons.html)
23 | build/Release
24 |
25 | # Dependency directory
26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
27 | node_modules
28 |
29 |
30 |
--------------------------------------------------------------------------------
/.npmignore:
--------------------------------------------------------------------------------
1 | # Logs
2 | logs
3 | *.log
4 |
5 | # Runtime data
6 | pids
7 | *.pid
8 | *.seed
9 |
10 | # Directory for instrumented libs generated by jscoverage/JSCover
11 | lib-cov
12 |
13 | # Coverage directory used by tools like istanbul
14 | coverage
15 |
16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
17 | .grunt
18 |
19 | # node-waf configuration
20 | .lock-wscript
21 |
22 | # Compiled binary addons (http://nodejs.org/api/addons.html)
23 | build/Release
24 |
25 | # Dependency directory
26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
27 | node_modules
28 |
29 |
30 |
--------------------------------------------------------------------------------
/.jshintrc:
--------------------------------------------------------------------------------
1 | {
2 | "node": true,
3 | "esnext": true,
4 | "bitwise": false,
5 | "camelcase": true,
6 | "curly": true,
7 | "eqeqeq": true,
8 | "forin": false,
9 | "eqnull": true,
10 | "immed": true,
11 | "indent": 4,
12 | "latedef": "nofunc",
13 | "newcap": true,
14 | "nonew": true,
15 | "noarg": true,
16 | "quotmark": "single",
17 | "undef": true,
18 | "unused": true,
19 | "trailing": true,
20 | "sub": true,
21 | "maxlen": 120,
22 | "devel": true,
23 | "mocha": true,
24 | "freeze": true,
25 | "funcscope": true,
26 | "globals": {
27 | "app": false,
28 | "it": false,
29 | "describe": false
30 | },
31 | "shadow": "outer"
32 | }
33 |
--------------------------------------------------------------------------------
/lib/systemMessage.js:
--------------------------------------------------------------------------------
1 | var Models = require('telepat-models');
2 |
3 | var SystemMessageProcessor = {
4 | process: function(message) {
5 | if (message.to == '_all' || message.to == SystemMessageProcessor.identity) {
6 | switch(message.action) {
7 | case 'update_app': {
8 | SystemMessageProcessor.updateApp(message.content.appId, message.content.appObject);
9 |
10 | break;
11 | }
12 | case 'delete_app': {
13 | SystemMessageProcessor.deleteApp(message.content.id);
14 | }
15 | }
16 | }
17 | },
18 | updateApp: function(appId, app) {
19 | Models.Application.loadedAppModels[appId] = app;
20 | },
21 | deleteApp: function(appId) {
22 | delete Models.Application.loadedAppModels[appId];
23 | }
24 | };
25 |
26 | SystemMessageProcessor.identity = null;
27 |
28 | module.exports = SystemMessageProcessor;
29 |
--------------------------------------------------------------------------------
/utils/profiling.js:
--------------------------------------------------------------------------------
1 | var async = require('async');
2 |
3 | var ProfilingContext = function() {
4 | this.timestamps = [];
5 | this.timerCollection = [];
6 | this.functions = [];
7 | this.initialTimestamp = null;
8 | };
9 |
10 | ProfilingContext.prototype.initial = function() {
11 | this.initialTimestamp = Math.floor(parseInt(process.hrtime().join(''))/1000);
12 | };
13 |
14 | ProfilingContext.prototype.addMark = function(name) {
15 | var timestamp = Math.floor(parseInt(process.hrtime().join(''))/1000);
16 | if (!this.timestamps.length)
17 | this.timerCollection.push(timestamp - this.initialTimestamp);
18 | else
19 | this.timerCollection.push(timestamp - this.timestamps[this.timestamps.length-1]);
20 | this.timestamps.push(timestamp);
21 | this.functions.push(name);
22 | };
23 |
24 | ProfilingContext.prototype.show = function() {
25 | var self = this;
26 | async.reduce(this.timerCollection, 0, function(memo, item, c) {
27 | c(null, memo+item);
28 | }, function(err, totalTime) {
29 | console.log('Total time: '+totalTime+' μs');
30 | self.functions.forEach(function(item, index) {
31 | console.log('['+item+']: '+self.timerCollection[index]+' μs ('+((self.timerCollection[index]/totalTime*100).toPrecision(3)+'%')+')');
32 | });
33 | });
34 | };
35 |
36 | module.exports = ProfilingContext;
37 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "telepat-models",
3 | "version": "0.4.4",
4 | "description": "Telepat lib used by the API and the workers",
5 | "main": "index.js",
6 | "repository": {
7 | "type": "git",
8 | "url": "git@github.com:telepat-io/telepat-models.git"
9 | },
10 | "keywords": [
11 | "telepat",
12 | "api",
13 | "models"
14 | ],
15 | "bugs": {
16 | "url": "https://github.com/telepat-io/telepat-models/issues"
17 | },
18 | "author": {
19 | "name": "Răzvan Botea",
20 | "email": "razvan@telepat.io"
21 | },
22 | "contributors": [
23 | {
24 | "name": "Răzvan Botea",
25 | "email": "razvan@telepat.io"
26 | },
27 | {
28 | "name": "Gabi Dobocan",
29 | "email": "gabi@telepat.io"
30 | }
31 | ],
32 | "directories": {
33 | "lib": "lib/"
34 | },
35 | "license": "Apache-2.0",
36 | "dependencies": {
37 | "agentkeepalive": "2.0.3",
38 | "amqplib": "0.4.1",
39 | "async": "1.5.2",
40 | "azure": "0.10.6",
41 | "clone": "1.0.2",
42 | "dateformat": "1.0.12",
43 | "dot-object": "1.5.1",
44 | "elasticsearch": "10.1.3",
45 | "kafka-node": "0.3.2",
46 | "lz4": "0.5.2",
47 | "object-merge": "2.5.1",
48 | "sprintf-js": "1.0.3",
49 | "uuid": "2.0.1",
50 | "validator": "6.2.0",
51 | "winston": "2.1.1"
52 | },
53 | "optionalDependencies": {
54 | "winston-syslog": "1.2.1"
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | var fs = require('fs');
2 |
3 | module.exports.Application = require('./lib/Application');
4 |
5 | module.exports.Context = require('./lib/Context');
6 |
7 | module.exports.Model = require('./lib/Model');
8 |
9 | module.exports.Subscription = require('./lib/Subscription');
10 |
11 | module.exports.Admin = require('./lib/Admin');
12 |
13 | module.exports.User = require('./lib/User');
14 |
15 | module.exports.utils = require('./utils/utils');
16 | module.exports.Builders = require('./utils/filterbuilder');
17 |
18 | module.exports.Channel = require('./lib/Channel');
19 | module.exports.Delta = require('./lib/Delta');
20 |
21 | module.exports.ProfilingContext = require('./utils/profiling');
22 |
23 | module.exports.TelepatError = require('./lib/TelepatError');
24 |
25 | module.exports.Datasource = require('./lib/database/datasource');
26 | module.exports.ElasticSearch = require('./lib/database/elasticsearch_adapter');
27 |
28 | module.exports.TelepatLogger = require('./lib/logger/logger');
29 |
30 | module.exports.TelepatIndexedList = require('./lib/TelepatIndexedLists');
31 |
32 | module.exports.ConfigurationManager = require('./lib/ConfigurationManager');
33 |
34 | module.exports.SystemMessageProcessor = require('./lib/systemMessage');
35 |
36 | fs.readdirSync(__dirname+'/lib/message_queue').forEach(function(filename) {
37 | var filenameParts = filename.split('_');
38 |
39 | if (filenameParts.pop() == 'queue.js') {
40 | module.exports[filenameParts.join('_')] = require('./lib/message_queue/'+filename);
41 | }
42 | });
43 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Telepat Models Library
2 |
3 | This package is used by the [Telepat API](https://github.com/telepat-io/telepat-api) and the [Telepat Worker](https://github.com/telepat-io/telepat-worker).
4 |
5 | This package contains the libraries for using various telepat resources:
6 |
7 | * Databases:
8 | * At the moment only elasticSearch support is implemented (version 1.7.x). Configuration variables:
9 | * `TP_ES_HOST`: Elasticsearch server:port. This option will use autodiscovery.
10 | * `TP_ES_HOSTS`: server1:port,server2:port. This option will not use autodiscovery.
11 | * `TP_ES_INDEX`: Elasticsearch index
12 | * `TP_ES_SUBSCRIBE_LIMIT` (optional): How many results the modelSearch method (used in subscriptions) should return (paginated). Default 64.
13 | * `TP_ES_GET_LIMIT` (optional): How many resutls every other search methods should return (not paginated, fixed). Default 384.
14 | * The state database doesn't use the adapter model yet because it's locked to Redis. Only `Subscription.js` uses it.
15 | * `TP_REDIS_HOST`: Redis server
16 | * `TP_REDIS_PORT`: Redis server port
17 |
18 | * Messaging Systems:
19 | * Apache Kafka
20 | * `TP_KFK_HOST`: Kafka (zooekeeper) server
21 | * `TP_KFK_PORT`: Kafka (zooekeeper) server port
22 | * Azure ServiceBus
23 | * `TP_AZURESB_CONNECTION_STRING`: Azure SB connection string
24 | * `TP_AZURESB_MSG_POLLING`: How fast should the messaging server be polled (in milliseconds)
25 | * AMQP: we've tested it with RabbitMQ 3.5.5
26 | * `TP_AMQP_HOST`: AMQP server host
27 | * `TP_AMQP_USER`: AMQP user used by Telepat
28 | * `TP_AMQP_PASSWORD`: The password for the user
29 |
30 | * Loggers:
31 | * We use winston logger: [Winston](https://github.com/winstonjs/winston)
32 | * `TP_LOGGER`: the name of the Winston logger (eg.: Console)
33 | * `TP_LOG_LEVEL`: logging level
34 |
--------------------------------------------------------------------------------
/lib/message_queue/messaging_client.js:
--------------------------------------------------------------------------------
1 | var MessagingClient = function(config, name, channel) {
2 | this.broadcast = config.broadcast ? true : false;
3 | delete config.broadcast;
4 |
5 | this.config = config || {};
6 | this.channel = config.exclusive ? name : channel;
7 | this.name = name;
8 | this.connectionClient = null;
9 | this.onReadyFunc = null;
10 | };
11 |
12 | /**
13 | * @callback onMessageCb
14 | * @param {string} data
15 | */
16 | /**
17 | * @abstract
18 | * @param {onMessageCb} callback
19 | */
20 | MessagingClient.prototype.onMessage = function(callback) {
21 | throw new Error('Unimplemented onMessage function.');
22 | };
23 |
24 | /**
25 | * @abstract
26 | * @param {onMessageCb} callback
27 | */
28 | MessagingClient.prototype.onSystemMessage = function(callback) {
29 | throw new Error('Unimplemented onSystemMessage function.');
30 | };
31 |
32 | MessagingClient.prototype.onReady = function(callback) {
33 | this.onReadyFunc = callback;
34 | };
35 |
36 | /**
37 | * @abstract
38 | * @param {string[]} messages
39 | * @param {string} channel
40 | * @param callback
41 | */
42 | MessagingClient.prototype.send = function(message, channel, callback) {
43 | throw new Error('Unimplemented send function.');
44 | };
45 |
46 | /**
47 | * @abstract
48 | * @param message
49 | * @param callback
50 | */
51 | MessagingClient.prototype.sendSystemMessages = function(message, callback) {
52 | throw new Error('Unimplemented sendSystemMessages function.');
53 | };
54 |
55 | /**
56 | * @abstract
57 | * @param {string[]} messages
58 | * @param {string} channel
59 | * @param callback
60 | */
61 | MessagingClient.prototype.publish = function(message, channel, callback) {
62 | throw new Error('Unimplemented publish function.');
63 | };
64 |
65 | MessagingClient.prototype.shutdown = function(callback) {
66 | throw new Error('Unimplemented shutdown function.');
67 | };
68 |
69 | MessagingClient.prototype.on = function(event, callback) {
70 | if (typeof this.connectionClient.on == 'function')
71 | this.connectionClient.on(event, callback);
72 | };
73 |
74 | module.exports = MessagingClient;
75 |
--------------------------------------------------------------------------------
/lib/logger/logger.js:
--------------------------------------------------------------------------------
1 | var winston = require('winston');
2 | var dateformat = require('dateformat');
3 |
4 | var TelepatLogger = function(options) {
5 | this.options = options;
6 |
7 | TelepatLogger.loggers = {
8 | Syslog: 'winston-syslog'
9 | };
10 |
11 | if (options.type != 'Console') {
12 | try {
13 | require(TelepatLogger.loggers[options.type])[options.type];
14 | winston.add(winston.transports[options.type], options.settings);
15 | winston.remove(winston.transports.Console);
16 | } catch (e) {
17 | console.log('Could not load winston logger: '+e);
18 | }
19 | } else {
20 | winston.colorize = true;
21 | }
22 |
23 | winston.setLevels(winston.config.syslog.levels);
24 | winston.level = options.settings.level || 'info';
25 | };
26 |
27 | /**
28 | *
29 | * @param {string} level
30 | * @param {string} message
31 | */
32 | TelepatLogger.prototype.log = function(level, message) {
33 | var timestamp = dateformat(new Date(), 'yyyy-mm-dd HH:MM:ss.l');
34 |
35 | message = '['+timestamp+']['+this.options.name+'] '+message;
36 | winston.log(level, message);
37 | };
38 |
39 | /**
40 | *
41 | * @param {string} message
42 | */
43 | TelepatLogger.prototype.debug = function(message) {
44 | this.log('debug', message);
45 | };
46 |
47 | /**
48 | *
49 | * @param {string} message
50 | */
51 | TelepatLogger.prototype.info = function(message) {
52 | this.log('info', message);
53 | };
54 |
55 | /**
56 | *
57 | * @param {string} message
58 | */
59 | TelepatLogger.prototype.notice = function(message) {
60 | this.log('notice', message);
61 | };
62 |
63 | /**
64 | *
65 | * @param {string} message
66 | */
67 | TelepatLogger.prototype.warning = function(message) {
68 | this.log('warn', message);
69 | };
70 |
71 | /**
72 | *
73 | * @param {string} message
74 | */
75 | TelepatLogger.prototype.error = function(message) {
76 | this.log('error', message);
77 | };
78 |
79 | /**
80 | *
81 | * @param {string} message
82 | */
83 | TelepatLogger.prototype.critical = function(message) {
84 | this.log('crit', message);
85 | };
86 |
87 | /**
88 | *
89 | * @param {string} message
90 | */
91 | TelepatLogger.prototype.alert = function(message) {
92 | this.log('alert', message);
93 | };
94 |
95 | /**
96 | *
97 | * @param {string} message
98 | */
99 | TelepatLogger.prototype.emergency = function(message) {
100 | this.log('emerg', message);
101 | };
102 |
103 | module.exports = TelepatLogger;
104 |
--------------------------------------------------------------------------------
/lib/message_queue/kafka_queue.js:
--------------------------------------------------------------------------------
1 | var MessagingClient = require('./messaging_client');
2 | var Application = require('../Application');
3 | var kafka = require('kafka-node');
4 | var async = require('async');
5 |
6 | var KafkaClient = function(config, name, channel){
7 | MessagingClient.call(this, config, name, channel);
8 |
9 | this.config = config;
10 | var self = this;
11 |
12 | async.series([
13 | function(callback) {
14 | self.connectionClient = kafka.Client(self.config.host+':'+self.config.port+'/', self.name);
15 | self.connectionClient.on('ready', function() {
16 | Application.logger.info('Client connected to Zookeeper & Kafka Messaging Client.');
17 | callback();
18 | });
19 | self.connectionClient.on('error', function() {
20 | Application.logger.error('Kafka broker not available. Trying to reconnect.');
21 | });
22 | },
23 | function(callback) {
24 | var groupId = self.broadcast ? self.name : channel;
25 | if (channel) {
26 | self.kafkaConsumer = new kafka.HighLevelConsumer(self.connectionClient, [{topic: channel}], {groupId: groupId});
27 | self.kafkaConsumer.on('error', function(err) {
28 | console.log(err);
29 | });
30 | }
31 | callback();
32 | },
33 | function(callback) {
34 | self.kafkaProducer = new kafka.HighLevelProducer(self.connectionClient);
35 | self.kafkaProducer.on('error', function() {});
36 | callback();
37 | }
38 | ], function(err) {
39 | if (err) {
40 | Application.logger.emergency('Kafka Queue: '+err.toString());
41 | process.exit(-1);
42 | } else {
43 | if(typeof self.onReadyFunc == 'function') {
44 | self.onReadyFunc();
45 | }
46 | }
47 | });
48 | };
49 |
50 | KafkaClient.prototype = Object.create(MessagingClient.prototype);
51 |
52 | KafkaClient.prototype.send = function(messages, channel, callback) {
53 | this.kafkaProducer.send([{
54 | topic: channel,
55 | messages: messages
56 | }], function(err) {
57 | callback(err);
58 | });
59 | };
60 |
61 | KafkaClient.prototype.publish = KafkaClient.prototype.send;
62 |
63 | KafkaClient.prototype.onMessage = function(callback) {
64 | if (this.kafkaConsumer) {
65 | this.kafkaConsumer.on('message', function(message) {
66 | callback(message.value);
67 | });
68 | }
69 | };
70 |
71 | KafkaClient.prototype.shutdown = function(callback) {
72 | this.connectionClient.close(callback);
73 | };
74 |
75 | KafkaClient.prototype.consumerOn = function(event, callback) {
76 | if (this.kafkaConsumer)
77 | this.kafkaConsumer.on(event, callback);
78 | };
79 |
80 | KafkaClient.prototype.producerOn = function(event, callback) {
81 | this.kafkaProducer.on(event, callback);
82 | };
83 |
84 | module.exports = KafkaClient;
85 |
--------------------------------------------------------------------------------
/lib/Admin.js:
--------------------------------------------------------------------------------
1 | var Application = require('./Application');
2 | var User = require('./User');
3 | var async = require('async');
4 | var FilterBuilder = require('../utils/filterbuilder').FilterBuilder;
5 | var guid = require('uuid');
6 | var TelepatError = require('./TelepatError');
7 |
8 | /**
9 | * @callback adminCb
10 | * @param {TelepatError|Error|null} err
11 | * @param {Object] admin
12 | */
13 | /**
14 | * Retrieves an admin by email address or id.
15 | * @param {Object} admin
16 | * @param {Object} [admin.id]
17 | * @param {Object} [admin.email]
18 | * @param {adminCb} callback
19 | * @constructor
20 | */
21 | function Admin(admin, callback) {
22 | if (admin.id)
23 | Application.datasource.dataStorage.getObjects([admin.id], function(errs, results) {
24 | if (errs.length)
25 | return callback(errs[0]);
26 | callback(null, results[0]);
27 | });
28 | else if (admin.email) {
29 | var filter = new FilterBuilder();
30 | filter.addFilter('is', {email: admin.email});
31 |
32 | Application.datasource.dataStorage.searchObjects({modelName: 'admin', filters: filter}, function(err, results) {
33 | if (err)
34 | return callback(err);
35 | if (!results.length)
36 | return callback(new TelepatError(TelepatError.errors.AdminNotFound));
37 |
38 | callback(null, results[0]);
39 | });
40 | }
41 | }
42 |
43 | /**
44 | * Creates a new admin.
45 | * @param {string} email The email address of the admin
46 | * @param {Object} props Properties of the admin
47 | * @param {adminCb} callback
48 | */
49 | Admin.create = function(email, props, callback) {
50 | props.email = email;
51 | props.id = guid.v4();
52 | props.created = Math.floor((new Date()).getTime()/1000);
53 | props.modified = props.created;
54 | props.type = 'admin';
55 | Application.datasource.dataStorage.createObjects([props], function(err) {
56 | callback(err, props);
57 | });
58 | }
59 |
60 | /**
61 | * Updates an admin object
62 | * @param {Object[]} patches
63 | * @param {adminCb} callback
64 | */
65 | Admin.update = function(patches, callback) {
66 | Application.datasource.dataStorage.updateObjects(patches, function(errs) {
67 | callback(errs && errs.length ? errs[0] : null);
68 | });
69 | }
70 |
71 | /**
72 | *
73 | * @param {string} email
74 | * @param {adminCb} callback
75 | */
76 | Admin.delete = function(admin, callback) {
77 | async.waterfall([
78 | function get(callback1) {
79 | new Admin(admin, callback1);
80 | },
81 | function deleteAdmin(admin, callback1) {
82 | var adminToDelete = {};
83 | adminToDelete[admin.id] = 'admin';
84 | Application.datasource.dataStorage.deleteObjects(adminToDelete, function(errs) {
85 | callback1(errs && errs.length ? errs[0] : null);
86 | });
87 | }
88 | ], callback);
89 | }
90 |
91 | module.exports = Admin;
92 |
--------------------------------------------------------------------------------
/lib/database/main_database_adapter.js:
--------------------------------------------------------------------------------
1 | var Main_Database_Adapter = function(connection) {
2 | this.connection = connection;
3 | /**
4 | *
5 | * @type {Function|null}
6 | */
7 | this.onReadyCallback = null;
8 | };
9 |
10 | /**
11 | *
12 | * @param {Function} callback Called after the database has finished setting up
13 | */
14 | Main_Database_Adapter.prototype.onReady = function(callback) {
15 | this.onReadyCallback = callback;
16 | };
17 |
18 | /**
19 | * @callback returnObjectsCb
20 | * @param {TelepatError[]} err
21 | * @param {Object[]} results
22 | */
23 | /**
24 | * @abstract
25 | * @param {String[]} ids
26 | * @param {returnObjectsCb} callback
27 | */
28 | Main_Database_Adapter.prototype.getObjects = function(ids, callback) {
29 | throw new Error('Database adapter "getObjects" not implemented');
30 | };
31 |
32 | /**
33 | * @callback ScanCallback
34 | * @param {Object[]} results
35 | */
36 | /**
37 | * @abstract
38 | * @param {Object} options
39 | * @param {string} {options.modelName}
40 | * @param {FilterBuilder} [options.filters]
41 | * @param {Object} [options.sort]
42 | * @param {Number} [options.offset]
43 | * @param {Number} [options.limit]
44 | * @param {string[]} [options.fields]
45 | * @param {ScanCallback} [options.scanFunction]
46 | * @param {returnObjectsCb} callback
47 | */
48 | Main_Database_Adapter.prototype.searchObjects = function(options, callback) {
49 | throw new Error('Database adapter "searchObjects" not implemented');
50 | };
51 |
52 | /**
53 | * @callback countObjectsCB
54 | * @param {Object} err
55 | * @param {Object} result
56 | * @param {Number} result.count
57 | * @param {Number} [result.aggregation]
58 | */
59 | /**
60 | *
61 | * @abstract
62 | * @param {Object} options
63 | * @param {string} options.modelName
64 | * @param {FilterBuilder} [options.filters]
65 | * @param {Object} [options.aggregation]
66 | * @param {countObjectsCB} callback
67 | */
68 | Main_Database_Adapter.prototype.countObjects = function(options, callback) {
69 | throw new Error('Database adapter "countObjects" not implemented');
70 | };
71 |
72 | /**
73 | * @callback CUDObjectsCb
74 | * @param {TelepatError[]|null} err
75 | */
76 | /**
77 | * @abstract
78 | * @param {Object[]} objects
79 | * @param {CUDObjectsCb} callback
80 | */
81 | Main_Database_Adapter.prototype.createObjects = function(objects, callback) {
82 | throw new Error('Database adapter "createObjects" not implemented');
83 | };
84 |
85 | /**
86 | * @abstract
87 | * @param {Object[]} patches
88 | * @param {CUDObjectsCb} callback
89 | */
90 | Main_Database_Adapter.prototype.updateObjects = function(patches, callback) {
91 | throw new Error('Database adapter "updateObjects" not implemented');
92 | };
93 |
94 | /**
95 | * @abstract
96 | * @param {Object[]} ids {ID => modelName}
97 | * @param {CUDObjectsCb} callback
98 | */
99 | Main_Database_Adapter.prototype.deleteObjects = function(ids, callback) {
100 | throw new Error('Database adapter "deleteObjects" not implemented');
101 | };
102 |
103 | module.exports = Main_Database_Adapter;
104 |
--------------------------------------------------------------------------------
/lib/TelepatIndexedLists.js:
--------------------------------------------------------------------------------
1 | var Application = require('./Application');
2 | var TelepatError = require('./TelepatError');
3 | var utils = require('../utils/utils');
4 | var async = require('async');
5 |
6 | var TelepatIndexedList = {
7 | /**
8 | * @callback appendCb
9 | * @param {Error|null} [err]
10 | */
11 |
12 | /**
13 | * @callback removeCb
14 | * @param {Error|null} [err]
15 | * @param {Boolean} [removed]
16 | */
17 |
18 | /**
19 | * @callback getCb
20 | * @param {Error|null} [err]
21 | * @param {Object[]} [results]
22 | */
23 |
24 | /**
25 | *
26 | * @param {string} listName Name of the list
27 | * @param {string} indexedProperty The property that's being indexed by
28 | * @param {Object} object The key of this object is the member that will be inserted with its value
29 | * @param {appendCb} callback
30 | */
31 | append: function(listName, indexedProperty, object , callback) {
32 | var commandArgs = ['blg:til:' + listName + ':' + indexedProperty + ':' + object[indexedProperty]];
33 |
34 | for(var prop in object) {
35 | if (typeof object[prop] != 'object') {
36 | commandArgs.push(prop, object[prop]);
37 | }
38 | }
39 |
40 | Application.redisCacheClient.hmset(commandArgs, function(err) {
41 | if (err) return callback(err);
42 | else callback();
43 | });
44 | },
45 |
46 | /**
47 | *
48 | * @param {string} listName Name of the list
49 | * @param {string} indexedProperty The property that's being indexed by
50 | * @param {string[]} members Array of memembers to check for
51 | * @param {getCb} callback
52 | */
53 | get: function(listName, indexedProperty, members, callback) {
54 | var baseKey = 'blg:til:'+listName+':'+indexedProperty;
55 | var tranzaction = Application.redisCacheClient.multi();
56 |
57 | members.forEach(function(member) {
58 | tranzaction.hgetall([baseKey + ':' + member]);
59 | });
60 |
61 | tranzaction.exec(function(err, replies) {
62 | if (err) return callback(err);
63 | var memebershipResults = {};
64 |
65 | async.forEachOf(replies, function(result, index, c) {
66 | memebershipResults[members[index]] = result || false;
67 | c();
68 | }, function() {
69 | callback(null, memebershipResults);
70 | });
71 | });
72 | },
73 |
74 | /**
75 | *
76 | * @param {string} listName Name of the list to remove
77 | * @param {removeCb} callback
78 | */
79 | removeList: function(listName, callback) {
80 | var keyPattern = 'blg:til:'+listName+':*';
81 | var self = this;
82 |
83 | utils.scanRedisKeysPattern(keyPattern, Application.redisCacheClient, function(err, results) {
84 | if (err) return callback(err);
85 |
86 | if (!results.length)
87 | return callback(new TelepatError(TelepatError.errors.TilNotFound, [listName]));
88 |
89 | Application.redisCacheClient.del(results, function(err, removed) {
90 | if (err) return callback(err);
91 |
92 | callback(null, new Boolean(removed));
93 | });
94 | })
95 | },
96 |
97 | /**
98 | *
99 | * @param {string} listName Name of the list
100 | * @param {string} indexedProperty The property that's being indexed by
101 | * @param {string[]} members Array of memembers to remove
102 | * @param {removeCb} callback
103 | */
104 | removeMember: function(listName, indexedProperty, members, callback) {
105 | var baseKey = 'blg:til:'+listName+':'+indexedProperty;
106 | var existingMembers = {}; //to avoid duplicate members
107 | var delArguments = [];
108 |
109 | members.forEach(function(member) {
110 | if (!existingMembers[member]) {
111 | delArguments.push([baseKey + ':' + member]);
112 | existingMembers[member] = true;
113 | }
114 | });
115 |
116 | Application.redisCacheClient.del(delArguments, function(err, reply) {
117 | callback(err, reply);
118 | });
119 | }
120 | };
121 |
122 | module.exports = TelepatIndexedList;
123 |
--------------------------------------------------------------------------------
/lib/Context.js:
--------------------------------------------------------------------------------
1 | var Application = require('./Application');
2 | var async = require('async');
3 | var FilterBuilder = require('../utils/filterbuilder').FilterBuilder;
4 | var guid = require('uuid');
5 |
6 | /**
7 | * Gets a context object.
8 | * @param _id number Context ID
9 | * @param callback
10 | * @constructor
11 | */
12 | function Context(_id, callback) {
13 | Application.datasource.dataStorage.getObjects([_id], function(errs, results) {
14 | if (errs.length) return callback(errs[0]);
15 | callback(null, results[0]);
16 | });
17 | }
18 |
19 | /**
20 | * Loads the configuration spec file. Automatically called at module require.
21 | */
22 | Context.load = function() {
23 | Context._model = require('../models/context.json');
24 |
25 | if (!Context._model) {
26 | Application.logger.emergency('Model \'context\' spec file does not exist.');
27 | process.exit(-1);
28 | }
29 | };
30 |
31 | /**
32 | * Get all contexts.
33 | * @param [by_app] integer Gets the context only from this application.
34 | * @param callback
35 | */
36 | Context.getAll = function(by_app, offset, limit, callback) {
37 | var filter = null;
38 | offset = offset || 0;
39 | limit = limit || Application.datasource.dataStorage.config.get_limit;
40 |
41 | if (by_app)
42 | filter = (new FilterBuilder('and')).addFilter('is', {application_id: by_app});
43 |
44 | Application.datasource.dataStorage.searchObjects({modelName: 'context', filters: filter, offset: offset, limit: limit}, callback);
45 | }
46 |
47 | /**
48 | * Gets the curent index of the contexts.
49 | * @param callback
50 | */
51 | Context.count = function(callback) {
52 | Application.datasource.dataStorage.countObjects({modelName: 'context'}, callback);
53 | }
54 |
55 |
56 | /**
57 | * Creates a new context
58 | * @param props Object properties of the context
59 | * @param callback
60 | */
61 | Context.create = function(props, callback) {
62 | props.type = 'context';
63 | props.id = guid.v4();
64 | props.created = Math.floor((new Date()).getTime()/1000);
65 | props.modified = props.created;
66 |
67 | Application.datasource.dataStorage.createObjects([props], function(err) {
68 | if (err) return callback(err);
69 | callback(null, props);
70 | });
71 | }
72 |
73 | /**
74 | * Updates a context.
75 | * @param id integer Context ID
76 | * @param patches[] Object The new properties of the context
77 | * @param callback
78 | */
79 | Context.update = function(patches, callback) {
80 | Application.datasource.dataStorage.updateObjects(patches, function(errs) {
81 | callback(errs.length ? errs[0] : null);
82 | });
83 | }
84 |
85 | /**
86 | * Deletes a context and all of its objects and subscriptions
87 | * @param id integer Context ID
88 | * @param callback
89 | */
90 | Context.delete = function(id, callback) {
91 | var delObj = {};
92 | delObj[id] = 'context';
93 |
94 | async.series([
95 | function(callback1) {
96 | Application.datasource.dataStorage.deleteObjects(delObj, function(errs) {
97 | if (errs) return callback1(errs[0]);
98 | callback1();
99 | });
100 | },
101 | function(callback1) {
102 | var deleteContextObjects = function(obj) {
103 | var deleteObjects = {};
104 | async.each(obj, function(o, c) {
105 | deleteObjects[o.id] = o.type;
106 | c();
107 | }, function() {
108 | Application.datasource.dataStorage.deleteObjects(deleteObjects, function(errs) {
109 | if (errs && errs.length > 1) {
110 | Application.logger.warning('Failed to delete '+errs.length+' context objects.');
111 | }
112 | });
113 | });
114 | };
115 | var filter = (new FilterBuilder()).addFilter('is', {context_id: id});
116 | Application.datasource.dataStorage.searchObjects({filters: filter, fields: ['id', 'type'], scanFunction: deleteContextObjects}, callback1);
117 | }
118 | ], callback);
119 | }
120 |
121 | module.exports = Context;
122 |
--------------------------------------------------------------------------------
/utils/filterbuilder.js:
--------------------------------------------------------------------------------
1 | var TelepatError = require('../lib/TelepatError');
2 |
3 | var BuilderNode = function(name) {
4 | if (BuilderNode.CONNECTORS.indexOf(name) === -1)
5 | throw new TelepatError(TelepatError.errors.QueryError, ['unsupported query connector "'+name+'"']);
6 |
7 | this.parent = null;
8 | /**
9 | *
10 | * @type {BuilderNode[]|Object[]}
11 | */
12 | this.children = [];
13 | this.name = name;
14 | };
15 |
16 | BuilderNode.CONNECTORS = [
17 | 'and',
18 | 'or'
19 | ];
20 |
21 | BuilderNode.FILTERS = [
22 | 'is',
23 | 'not',
24 | 'exists',
25 | 'range',
26 | 'in_array',
27 | 'like'
28 | ];
29 |
30 | BuilderNode.prototype.addFilter = function(name, value) {
31 | if (BuilderNode.FILTERS.indexOf(name) !== -1) {
32 | var filter = {};
33 | filter[name] = value;
34 | this.children.push(filter);
35 | } else
36 | throw new TelepatError(TelepatError.errors.QueryError, ['invalid filter "'+name+'"']);
37 | };
38 |
39 | /**
40 | *
41 | * @param {BuilderNode} node
42 | */
43 | BuilderNode.prototype.addNode = function(node) {
44 | node.parent = this;
45 | this.children.push(node);
46 | };
47 |
48 | /**
49 | *
50 | * @param {BuilderNode} node
51 | */
52 | BuilderNode.prototype.removeNode = function(node) {
53 | var idx = this.children.indexOf(node);
54 |
55 | if (idx !== -1) {
56 | node.parent = null;
57 | return this.children.splice(idx, 1)[0];
58 | } else {
59 | return null;
60 | }
61 | };
62 |
63 | BuilderNode.prototype.toObject = function() {
64 | var obj = {};
65 | obj[this.name] = [];
66 |
67 | this.children.forEach(function(item) {
68 | if (item instanceof BuilderNode)
69 | obj[this.name].push(item.toObject());
70 | else
71 | obj[this.name].push(item);
72 | }, this);
73 |
74 | return obj;
75 | };
76 |
77 | var FilterBuilder = function(initial) {
78 | /**
79 | *
80 | * @type {null|BuilderNode}
81 | */
82 | this.root = null;
83 |
84 | if (initial)
85 | this.root = new BuilderNode(initial);
86 | else
87 | this.root = new BuilderNode('and');
88 |
89 | this.pointer = this.root;
90 | };
91 |
92 | FilterBuilder.prototype.and = function() {
93 | if (this.root === null) {
94 | this.root = new BuilderNode('and');
95 | } else {
96 | var child = new BuilderNode('and');
97 | this.pointer.addNode(child);
98 | this.pointer = child;
99 | }
100 |
101 | return this;
102 | };
103 |
104 | FilterBuilder.prototype.or = function() {
105 | if (this.root === null) {
106 | this.root = new BuilderNode('or');
107 | } else {
108 | var child = new BuilderNode('or');
109 | this.pointer.addNode(child);
110 | this.pointer = child;
111 | }
112 |
113 | return this;
114 | };
115 |
116 | FilterBuilder.prototype.addFilter = function(name, value) {
117 | this.pointer.addFilter(name, value);
118 |
119 | return this;
120 | };
121 |
122 | FilterBuilder.prototype.removeNode = function() {
123 | if (this.root !== this.pointer) {
124 | var nodeToRemove = this.pointer;
125 | this.pointer = this.pointer.parent;
126 |
127 | return this.pointer.removeNode(nodeToRemove);
128 | } else
129 | return null;
130 | };
131 |
132 | FilterBuilder.prototype.isEmpty = function() {
133 | return this.root.children.length ? false : true;
134 | };
135 |
136 | FilterBuilder.prototype.end = function() {
137 | if (this.pointer.parent)
138 | this.pointer = this.pointer.parent;
139 |
140 | return this;
141 | };
142 |
143 | FilterBuilder.prototype.build = function() {
144 | return this.root ? this.root.toObject() : null;
145 | };
146 |
147 | /*var FB = new FilterBuilder('and');
148 | FB.
149 | or().
150 | addFilter('is', {a: 1}).
151 | addFilter('is', {b: 2}).
152 | addFilter('is', {c: 3}).
153 | end().
154 | or().
155 | addFilter('is', {d: 4}).
156 | addFilter('is', {e: 5}).
157 | addFilter('is', {f: 6});*/
158 |
159 | module.exports = {
160 | FilterBuilder: FilterBuilder,
161 | BuilderNode: BuilderNode
162 | };
163 |
--------------------------------------------------------------------------------
/lib/message_queue/azure_servicebus_queue.js:
--------------------------------------------------------------------------------
1 | var MessagingClient = require('./messaging_client');
2 | var Application = require('../Application');
3 | var azure = require('azure');
4 | var async = require('async');
5 |
6 | var AzureServiceBus = function(config, name, channel){
7 | MessagingClient.call(this, config, name, channel);
8 |
9 | var self = this;
10 |
11 | var envVariables = {
12 | TP_AZURESB_CONNECTION_STRING: process.env.TP_AZURESB_CONNECTION_STRING
13 | };
14 |
15 | if (channel) {
16 | envVariables.TP_AZURESB_MSG_POLLING = process.env.TP_AZURESB_MSG_POLLING;
17 | }
18 |
19 | var validEnvVariables = true;
20 |
21 | for(var varName in envVariables) {
22 | if (envVariables[varName] === undefined) {
23 | Application.logger.notice('Missing environment variable "'+varName+'". Trying configuration file.');
24 |
25 | if (!this.config || !Object.getOwnPropertyNames(this.config).length) {
26 | Application.logger.emergency('Configuration file is missing configuration for Azure ServiceBus ' +
27 | 'messaging client.');
28 | process.exit(-1);
29 | }
30 |
31 | validEnvVariables = false;
32 | break;
33 | }
34 | }
35 |
36 | if (validEnvVariables) {
37 | this.config.connection_string = process.env.TP_AZURESB_CONNECTION_STRING;
38 | this.config.polling_interval =process.env.TP_AZURESB_MSG_POLLING;
39 | }
40 |
41 | /**
42 | *
43 | * @type {ServiceBusService|*}
44 | */
45 | this.connectionClient = azure.createServiceBusService(config.connection_string);
46 | this.isSubscribed = false;
47 | //this can be undefined if receiving messages from topics is not desired
48 | this.subscription = channel;
49 |
50 | async.series([
51 | function Connect(callback) {
52 | if (channel) {
53 | self.connectionClient.getSubscription(channel, name, function(err) {
54 | if (err && err.statusCode == 404) {
55 | self.connectionClient.createSubscription(channel, name, function(err) {
56 | if (err) return callback(err);
57 | self.isSubscribed = true;
58 | callback();
59 | });
60 | } else if (err) {
61 | console.log(err);
62 | Application.logger.error('Failed connecting to Azure ServiceBus ('+
63 | err.toString()+'). Reconnecting... ');
64 | setTimeout(function() {
65 | Connect(callback);
66 | }, 1000);
67 | } else {
68 | self.isSubscribed = true;
69 | callback()
70 | }
71 | });
72 | } else
73 | callback();
74 | }
75 | ], function(err) {
76 | if (err) {
77 | console.log(err);
78 | console.log('Aborting...'.red);
79 | process.exit(-1);
80 | } else {
81 | if(typeof self.onReadyFunc == 'function') {
82 | Application.logger.info('Connected to Azure ServiceBus Messaging Queue');
83 | self.onReadyFunc();
84 | }
85 | }
86 | });
87 | };
88 |
89 | AzureServiceBus.prototype = Object.create(MessagingClient.prototype);
90 |
91 | AzureServiceBus.prototype.send = function(messages, channel, callback) {
92 | var self = this;
93 | async.each(messages, function(message, c) {
94 | self.connectionClient.sendTopicMessage(channel, message, c);
95 | }, callback);
96 | };
97 |
98 | AzureServiceBus.prototype.publish = AzureServiceBus.prototype.send;
99 |
100 | AzureServiceBus.prototype.onMessage = function(callback) {
101 | var self = this;
102 |
103 | if (this.isSubscribed) {
104 | setInterval(function() {
105 | self.connectionClient.receiveSubscriptionMessage(self.subscription, self.name, {timeoutIntervalInS: 30}, function(err, serverMessage) {
106 | if (err && err == 'No messages to receive') {
107 | return ;
108 | } else if (err) {
109 | Application.logger.error('Error receiving Subscription Message from Azure ServiceBus ('+err.toString()+')');
110 | } else {
111 | callback(serverMessage.body);
112 | }
113 | });
114 | }, this.config.polling_interval < 50 ? 50 : this.config.polling_interval);
115 | }
116 | };
117 |
118 | AzureServiceBus.prototype.shutdown = function(callback) {
119 | callback();
120 | };
121 |
122 | module.exports = AzureServiceBus;
123 |
--------------------------------------------------------------------------------
/lib/Delta.js:
--------------------------------------------------------------------------------
1 | var guid = require('uuid');
2 | var Application = require('./Application');
3 | var cloneObject = require('clone');
4 |
5 | /**
6 | * @typedef {{
7 | * op: Delta.PATCH_OP,
8 | * path: string,
9 | * value: any,
10 | * [timestamp]: Number
11 | * }} Patch
12 | */
13 |
14 | /**
15 | *
16 | * @param {Object} fields
17 | * @param {Delta.OP} fields.op
18 | * @param {Object} fields.object
19 | * @param {Patch[]} fields.patch
20 | * @param {string} fields.application_id
21 | * @param {string} fields.timestamp
22 | * @param {string} [fields.instant]
23 | * @param {Channel[]} subscriptions
24 | * @constructor
25 | */
26 | var Delta = function(fields, subscriptions) {
27 | this.op = fields.op;
28 | this.object = fields.object;
29 | this.patch = fields.patch;
30 | this.application_id = fields.application_id;
31 | this.timestamp = fields.timestamp;
32 | if (this.instant)
33 | this.instant = fields.instant;
34 | this.subscriptions = subscriptions || [];
35 | };
36 |
37 | /**
38 | * @enum {string}
39 | * @type {{ADD: string, UPDATE: string, DELETE: string}}
40 | */
41 | Delta.OP = {
42 | ADD: 'add',
43 | UPDATE: 'update',
44 | DELETE: 'delete'
45 | };
46 |
47 | /**
48 | * @enum {string}
49 | * @type {{APPEND: string, INCREMENT: string, REPLACE: string, REMOVE: string}}
50 | */
51 | Delta.PATCH_OP = {
52 | APPEND: 'append',
53 | INCREMENT: 'increment',
54 | REPLACE: 'replace',
55 | REMOVE: 'remove'
56 | };
57 |
58 |
59 | /*Delta.prototype.clone = function() {
60 | var d = new Delta(this.op, this.value, this.path, this.channel, this.guid, this.ts);
61 | d.subscription = this.subscription;
62 | d.application_id = this.application_id;
63 |
64 | if (this.context)
65 | d.context = this.context;
66 |
67 | if (this.username)
68 | d.username = this.username;
69 |
70 | if (this.instant)
71 | d.instant = this.instant;
72 |
73 | return d;
74 | };*/
75 |
76 | Delta.prototype.toObject = function() {
77 | var obj = {
78 | op: this.op,
79 | object: this.object,
80 | subscriptions: this.subscriptions,
81 | application_id: this.application_id,
82 | timestamp: this.timestamp
83 | };
84 |
85 | if (this.op == 'update')
86 | obj.patch = this.patch;
87 |
88 | if (this.instant)
89 | obj.instant = true;
90 |
91 | return obj;
92 | };
93 |
94 | Delta.formPatch = function(object, op, property) {
95 | var patch = {};
96 |
97 | if (op)
98 | patch.op = op;
99 |
100 | if (property) {
101 | var prop = Object.keys(property)[0];
102 | patch.path = object.type+'/'+object.id+'/'+prop;
103 | patch.value = property[prop];
104 | } else if (object.id) {
105 | patch.path = object.type+'/'+object.id;
106 | }
107 |
108 | return patch;
109 | };
110 |
111 | Delta.processObject = function(patches, object) {
112 | for (var i in patches) {
113 | var objectField = patches[i].path.split('/')[2];
114 |
115 | if (patches.hasOwnProperty(i) && ['id', 'type', 'created', 'modified', 'application_id', 'context_id'].indexOf(objectField) == -1) {
116 | switch (patches[i].op) {
117 | case 'replace': {
118 | object[objectField] = patches[i].value;
119 |
120 | break;
121 | }
122 |
123 | case 'increment': {
124 | object[objectField] += patches[i].value;
125 |
126 | break;
127 | }
128 |
129 | case 'append': {
130 | if (Array.isArray(object[objectField])) {
131 | object[objectField].push(patches[i].value);
132 | } else if (typeof object[objectField] == 'string') {
133 | object[objectField] += patches[i].value;
134 | } else if (object[objectField] === undefined) {
135 | object[objectField] = [patches[i].value];
136 | }
137 |
138 | break;
139 | }
140 |
141 | case 'remove': {
142 | if (Array.isArray(object[objectField])) {
143 | var idx = object[objectField].indexOf(patches[i].value);
144 | if (idx !== -1)
145 | object[objectField].splice(idx, 1);
146 | }
147 |
148 | break;
149 | }
150 | }
151 | }
152 | }
153 |
154 | object.modified = Math.floor((new Date()).getTime()/1000);
155 |
156 | return object;
157 | };
158 |
159 | module.exports = Delta;
160 |
--------------------------------------------------------------------------------
/lib/message_queue/amqp_queue.js:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | * @type {MessagingClient|exports|module.exports}
4 | */
5 | var MessagingClient = require('./messaging_client');
6 | var amqplib = require('amqplib/callback_api');
7 | var async = require('async');
8 | var Application = require('../Application');
9 | var lz4 = require('../../utils/utils').lz4;
10 |
11 | var AMQPClient = function(config, name, channel) {
12 | MessagingClient.call(this, config, name, channel);
13 |
14 | this.config = config;
15 | var self = this;
16 | this.amqpChannel = null;
17 | this.broadcastQueue = null;
18 | this.assertedQueues = {};
19 |
20 | async.series([
21 | function TryConnecting(callback) {
22 | amqplib.connect('amqp://'+self.config.user+':'+self.config.password+'@'+self.config.host, function(err, conn) {
23 | if (err) {
24 | Application.logger.error('Failed connecting to AMQP messaging queue ('+
25 | err.toString()+'). Retrying... ');
26 | setTimeout(function() {
27 | TryConnecting(callback);
28 | }, 1000);
29 | } else {
30 | self.connectionClient = conn;
31 | callback();
32 | }
33 | });
34 | },
35 | function TryChannel(callback) {
36 | self.connectionClient.createChannel(function(err, ch) {
37 | if (err) {
38 | Application.logger.error('Failed creating channel on the AMQP messaging queue ('+
39 | err.toString()+'). Retrying... ');
40 | setTimeout(function() {
41 | TryChannel(callback);
42 | }, 1000);
43 | } else {
44 | self.amqpChannel = ch;
45 | //create queue or exchange if it doesnt exist; used for consumers
46 | if (self.broadcast) {
47 | self.amqpChannel.assertExchange(self.channel+'-exchange', 'fanout', {}, function(err) {
48 | if (err) return callback(err);
49 | self.amqpChannel.assertQueue(self.name, {durable: false, autoDelete: true}, function(err1, result) {
50 | if (err1) return callback(err1);
51 |
52 | self.broadcastQueue = result.queue;
53 | self.amqpChannel.bindQueue(self.broadcastQueue, self.channel+'-exchange', '', {}, callback);
54 | });
55 | });
56 | } else {
57 | //we only need to assert the queue if the sending of messages is needed
58 | if (self.channel)
59 | self.amqpChannel.assertQueue(self.channel, {}, callback);
60 | else
61 | callback();
62 | }
63 | }
64 | });
65 | }
66 | ], function(err) {
67 | if (err) {
68 | Application.logger.emergency('AMQP Queue: '+err.toString());
69 | process.exit(1);
70 | }
71 | self.amqpChannel.bindQueue(self.broadcastQueue || self.channel, 'amq.fanout', '', {}, function(err) {
72 | if (err) {
73 | Application.logger.error('Failed to bind AMQP queue to amq.fanout');
74 | self.failedBind = true;
75 | }
76 |
77 | if(typeof self.onReadyFunc == 'function') {
78 | Application.logger.info('Connected to AMQP Messaging Queue');
79 | self.onReadyFunc();
80 | }
81 | });
82 | });
83 | };
84 |
85 | AMQPClient.prototype = Object.create(MessagingClient.prototype);
86 |
87 | AMQPClient.prototype.onMessage = function(callback) {
88 | var fromWhere = this.broadcast ? this.broadcastQueue : this.channel;
89 |
90 | this.amqpChannel.consume(fromWhere, function(message) {
91 | if (message !== null) {
92 | lz4.decompress(message.content, function(data) {
93 | callback(data.toString());
94 | });
95 | }
96 | }, {noAck: true});
97 | };
98 |
99 | AMQPClient.prototype.send = function(messages, channel, callback) {
100 | var self = this;
101 |
102 | if (this.assertedQueues[channel]) {
103 | async.each(messages, function(message, c) {
104 | lz4.compress(message, function(compressed) {
105 | self.amqpChannel.sendToQueue(channel, compressed);
106 | c();
107 | });
108 | }, callback);
109 | } else {
110 | this.amqpChannel.checkQueue(channel, function(err) {
111 | if (err) return callback(err);
112 |
113 | self.assertedQueues[channel] = true;
114 |
115 | async.each(messages, function(message, c) {
116 | lz4.compress(message, function(compressed) {
117 | self.amqpChannel.sendToQueue(channel, compressed);
118 | c();
119 | });
120 | }, callback);
121 | });
122 | }
123 | };
124 |
125 | AMQPClient.prototype.sendSystemMessages = function(to, action, messages, callback) {
126 | if (this.failedBind)
127 | return callback();
128 |
129 | var self = this;
130 |
131 | async.each(messages, function(message, c) {
132 | var messagePayload = {
133 | _systemMessage: true,
134 | to: to,
135 | action: action,
136 | content: message
137 | };
138 |
139 | lz4.compress(JSON.stringify(messagePayload), function(compressed) {
140 | self.amqpChannel.publish('amq.fanout', '', compressed);
141 | c();
142 | });
143 | }, callback);
144 | };
145 |
146 | AMQPClient.prototype.publish = function(messages, channel, callback) {
147 | var self = this;
148 |
149 | if (this.assertedQueues[channel+'-exchange']) {
150 | async.each(messages, function(message, c) {
151 | lz4.compress(message, function(compressed) {
152 | self.amqpChannel.publish(channel+'-exchange', '', compressed);
153 | c();
154 | });
155 | }, callback);
156 | } else {
157 | this.amqpChannel.assertExchange(channel+'-exchange', 'fanout', {}, function(err) {
158 | if (err) return callback(err);
159 |
160 | self.assertedQueues[channel+'-exchange'] = true;
161 |
162 | async.each(messages, function(message, c) {
163 | lz4.compress(message, function(compressed) {
164 | self.amqpChannel.publish(channel+'-exchange', '', compressed);
165 | c();
166 | });
167 | }, callback);
168 | });
169 | }
170 | };
171 |
172 | AMQPClient.prototype.shutdown = function(callback) {
173 | this.amqpChannel.close(callback);
174 | };
175 |
176 | module.exports = AMQPClient;
177 | 34
178 |
--------------------------------------------------------------------------------
/lib/Channel.js:
--------------------------------------------------------------------------------
1 | var utils = require('../utils/utils');
2 | var Application = require('./Application');
3 | var objectClone = require('clone');
4 |
5 | var Channel = function Channel(appId, props) {
6 | this.props = {};
7 | this.filter = null;
8 | this.mask = 0;
9 | this.forceInvalid = false;
10 | this.errorMessage = '';
11 |
12 | if (!appId)
13 | throw new Error('Must supply application ID to channel constructor');
14 | else
15 | this.appId = appId;
16 |
17 | if (props)
18 | this.props = props;
19 | }
20 |
21 | Channel.keyPrefix = "blg";
22 |
23 | Channel.MASKS = {
24 | CONTEXT: 1,
25 | USER: 2,
26 | MODEL: 4,
27 | PARENT: 8,
28 | ID: 16,
29 | };
30 |
31 | Channel.validChannels = {
32 | 4: Channel.keyPrefix+":{appId}:{model}", //channel used for built-in models (users, contexts)
33 | 5: Channel.keyPrefix+":{appId}:context:{context}:{model}", //the Channel of all objects from a context
34 | 7: Channel.keyPrefix+":{appId}:context:{context}:users:{user_id}:{model}", //the Channel of all objects from a context from an user
35 | 12: Channel.keyPrefix+":{appId}:{parent_model}:{parent_id}:{model}", //the Channel of all objects belong to a parent
36 | 14: Channel.keyPrefix+":{appId}:users:{user_id}:{parent_model}:{parent_id}:{model}",//the Channel of all comments from event 1 from user 16
37 | 20: Channel.keyPrefix+":{appId}:{model}:{id}", //the Channel of one item
38 | };
39 |
40 | Channel.builtInModels = ['user', 'context'];
41 |
42 | Channel.prototype.model = function(model, id) {
43 | if (model) {
44 | if (Application.loadedAppModels[this.appId] !== undefined && Application.loadedAppModels[this.appId].schema !== undefined && Application.loadedAppModels[this.appId].schema[model]) {
45 | this.props.model = model;
46 | } else if (Channel.builtInModels.indexOf(model) !== -1)
47 | this.props.model = model;
48 | else
49 | throw new Error('Model "'+model+'" is not a valid model name');
50 |
51 | this.mask |= Channel.MASKS.MODEL;
52 | }
53 |
54 | if (id) {
55 | this.props.modelId = id;
56 | this.mask |= Channel.MASKS.ID;
57 | }
58 |
59 |
60 | if (!model && !id)
61 | this.forceInvalid = true;
62 |
63 | return this;
64 | };
65 |
66 | Channel.prototype.parent = function(parent) {
67 | if (parent && parent.model && parent.id) {
68 | this.mask |= Channel.MASKS.PARENT;
69 | this.props.parent = parent;
70 | } else
71 | this.forceInvalid = true;
72 |
73 | return this;
74 | }
75 |
76 | Channel.prototype.context = function(context) {
77 | if (context) {
78 | this.mask |= Channel.MASKS.CONTEXT;
79 | this.props.context = context;
80 | } else
81 | this.forceInvalid = true;
82 |
83 | return this;
84 | };
85 |
86 | Channel.prototype.user = function(user) {
87 | if (user) {
88 | this.mask |= Channel.MASKS.USER;
89 | this.props.user = user;
90 | } else
91 | this.forceInvalid = true;
92 |
93 | return this;
94 | };
95 |
96 | Channel.prototype.setFilter = function(filter) {
97 | if (filter)
98 | this.filter = filter;
99 | else
100 | this.forceInvalid = true;
101 |
102 | return this;
103 | };
104 |
105 | Channel.prototype.isValid = function() {
106 | var mask = Channel.validChannels[this.mask];
107 |
108 | //only built in models can have a subscription on all without a context
109 | if (this.mask === 4 && Channel.builtInModels.indexOf(this.props.model) === -1) {
110 | this.errorMessage = 'Only builtin models (user,context) can be subscribed to all without a context';
111 | return false;
112 | }
113 |
114 | var result = !this.forceInvalid && (mask !== undefined);
115 |
116 | if (!result)
117 | this.errorMessage = 'Invalid channel "'+this.mask+'".';
118 |
119 | return result;
120 | };
121 |
122 | Channel.prototype.get = function(options) {
123 | if (!this.isValid())
124 | throw new Error('Invalid channel with mask "'+this.mask+'"');
125 |
126 | var validChannel = Channel.validChannels[this.mask];
127 |
128 | validChannel = validChannel.replace('{appId}', this.appId);
129 |
130 | switch(this.mask) {
131 | case 4: {
132 | if (Channel.builtInModels.indexOf(this.props.model) === -1)
133 | throw new Error('Channel with mask "4" can only be used with built in models');
134 |
135 | validChannel = validChannel.replace('{model}', this.props.model);
136 |
137 | break;
138 | }
139 | case 5: { // MASKS.CONTEXT | MASKS.MODEL
140 | validChannel = validChannel.replace('{context}', this.props.context);
141 | validChannel = validChannel.replace('{model}', this.props.model);
142 |
143 | break;
144 | }
145 |
146 | case 7: { // MASKS.CONTEXT | MASKS.USER | MASKS.MODEL
147 | validChannel = validChannel.replace('{context}', this.props.context);
148 | validChannel = validChannel.replace('{user_id}', this.props.user);
149 | validChannel = validChannel.replace('{model}', this.props.model);
150 |
151 | break;
152 | }
153 |
154 | case 12: { // MASKS.MODEL | MASKS.PARENT
155 | validChannel = validChannel.replace('{parent_model}', this.props.parent.model);
156 | validChannel = validChannel.replace('{parent_id}', this.props.parent.id);
157 | validChannel = validChannel.replace('{model}', this.props.model);
158 |
159 | break;
160 | }
161 |
162 | case 14: { // MASKS.USER | MASKS.MODEL | MASKS.PARENT
163 | validChannel = validChannel.replace('{parent_model}', this.props.parent.model);
164 | validChannel = validChannel.replace('{parent_id}', this.props.parent.id);
165 | validChannel = validChannel.replace('{model}', this.props.model);
166 | validChannel = validChannel.replace('{user_id}', this.props.user);
167 |
168 | break;
169 | }
170 |
171 | case 20: { // MASKS.MODEL | MASKS.ID
172 | validChannel = validChannel.replace('{model}', this.props.model);
173 | validChannel = validChannel.replace('{id}', this.props.modelId);
174 |
175 | break;
176 | }
177 | }
178 |
179 | if (this.filter)
180 | validChannel += ':filter:'+(new Buffer(JSON.stringify(this.filter))).toString('base64');
181 |
182 | if (options) {
183 | if (options.deltas)
184 | validChannel += ':deltas';
185 | }
186 |
187 |
188 | return validChannel;
189 | };
190 |
191 | /**
192 | *
193 | * @param {Channel} channel
194 | * @returns {Channel}
195 | */
196 | Channel.cloneFrom = function(channel) {
197 | var c = new Channel(channel.appId, channel.props);
198 | c.mask = channel.mask;
199 |
200 | return c;
201 | };
202 |
203 | Channel.prototype.clone = function() {
204 | var c = new Channel(this.appId, this.props);
205 | c.mask = this.mask;
206 | c.filter = this.filter;
207 |
208 | return c;
209 | }
210 |
211 | module.exports = Channel;
212 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # 0.4.4
2 |
3 | * Updated `apn` package which uses the new HTTP/2 protocol which should make much more reliable and faster
4 | * The array containing the devices' subscription in the device object have been moved to a dedicated redis key
5 | * Implemented configuration manager used for validating the config file
6 | * Subscription elements in the redis key now contain the device ID
7 | * Implemented `Subscription.removeAllSubscriptionsFromDevice`
8 | * `Subscription.getAllDevices` device identifier changed to "device.id|device.token"
9 | * `Subscription.remove` accepts token as an optional argument
10 | * `User.update` now returns the updated user object
11 | * Subscription methods should take `device.volatile.active` into account when adding/removing subscriptions
12 | * BUGFIX: `utils.testObject` has been rewritten and should be bug free now
13 | * BUGFIX: fixed incomplete filters when a complex filter contained the AND connector
14 | * BUGFIX: fixed object updates if an object field's contain " or ' characters
15 | * BUGFIX: fixed `Subscription.add` when `device.persistent` is missing
16 | * BUGFIX: Removed count LOCK key in redis if transaction fails
17 |
18 | # 0.4.3.
19 |
20 | * ES adapter updateObjects implemented with retry-on-conflict strategy
21 | * Fixed bug when updating multiple fields
22 |
23 | # 0.4.2
24 |
25 | * Fixed ES 'like' filter
26 | * Fixed ES geolocation sorting
27 | * Implemented System messages
28 | * Support for multi-sort
29 | * fixed model.GetFiltersFromChannel when 'and' 'or' operators are direct parents/children
30 |
31 | # 0.4.1
32 |
33 | * added default subscribe limit to Model.search
34 | * Bugfix: error handling when device is invalid
35 | * Implemented **like** filter
36 | * TelepatIndexedLists.get now returns the object if it's found
37 | * Bugfix: search no longer fails when an object has no mapping in ES
38 | * Bugfix: User() filters
39 | * ExpiredAuthorizationToken error status code should be 401
40 |
41 | # 0.4.0
42 |
43 | * Major performance improvements
44 | * AMQP messages are now compressed using LZ4
45 | * TelepatIndexedLists is a data structure on redis used to rapidly search
46 | for members
47 | * Simpliefied the database adapter, now they can be used in bulk operations
48 | * Device IDs are no longer stored in subscriptions, instead their tokens
49 | are stored in order to avoid retrieving the device object everytime
50 | * Filter builder is used by database adapters in order to form filters
51 | for querying the database
52 | * Moved some of the logic from the old adapter's functions to functions
53 | in Admin/Context/Model/etc
54 |
55 | # 0.3.0
56 |
57 | * Performance fix: AMQP queues/exchanges are only asserted once
58 | * Devices are created in an inactive keyspace. They should be activated
59 | when a client connects to a volatile transport
60 | * Implemented methods to activate/deactivate devices
61 | * Performance fix: incresed maxSockets for ElasticSearch
62 | * Performance fix: used agentKeepAlive for elasticSearch SDK
63 | * Ability to connect to multiple ES nodes, or just one if you want to
64 | use auto-discovery
65 | * Messaging clients has the exclusive flag making them have one queue
66 | per client
67 | * Implemented offset/limit for some methods
68 | * Implemented userMetadata use only by their respectiv owners
69 | * Replaced syslog_logger with Winston
70 | * Fixed a bug where updating objects with a field of type object would
71 | make the operation to merge the fields instead of replacing the whole
72 | object
73 | * Implemented caching for count queries
74 |
75 | # 0.2.8
76 |
77 | * Bugfix: Channel.isValid when mask is invalid
78 | * Bugfix: Channel when used with parent objects
79 | * Bugfix: `applicationHasContext` in elasticSearch adapter returns false instead of an error in callback
80 | * Bugfix: `modelCountByChannel` in elasticSearch adapter, context is optional
81 | * Bugfix: `modelCountByChannel` should now work with filters
82 | * `LoginProviderNotConfigured` error transformed into a more generic error `ServerNotConfigured`
83 | * Added support for aggregating results in `modelCountByChannel`
84 | * Added supoprt for object sort in `modelSearch`
85 |
86 | # 0.2.7
87 |
88 | * Fixed some bugs and crashes
89 | * Redis keys are being scanned untill cursor returns 0
90 | * Changed **email** property of user to **username**
91 | * Added `ExpiredAuthorizationToken` error
92 |
93 | # 0.2.6
94 |
95 | * User.get can now be called with an object containing ID or email
96 | * Fixed bug in AMQP message client which created unintended queues
97 | * `applicationDeleteModelSchema` now deletes all model siblings
98 | * Implemented `removeAllSubscriptionsFromDevice`
99 | * Fixed handling of errors on Application.loadAllApllications
100 | * Correct stack trace generation on TelepatError
101 | * Subscription and Device keys now contain the applicationID in them
102 | * Implemented `removeDevice`
103 | * Implemented `TelepatLogger` used for logging purposes
104 | * `syslogger`
105 | * `console_logger`
106 | * Fixed kafka/AzureSB crash
107 |
108 | # 0.2.5
109 |
110 | * Fixed various bugs
111 | * 2 New message queue clients: **Azure ServiceBus** and **AMQP (RabbitMQ)**
112 | * ElasticSearch should refresh its index on create/delete
113 | * Configuration variables for ElasticSearch adapter get/subscribe result limit + paginated results for subscribe
114 | * Model.countByChannel fixed
115 | * Rewored how loadedAppModels work: every package should populate this at boot up. App.get loads from redis (cached)
116 | * All message queue clients now have a broadcast method which broadcasts messages to all consumers on a channel/topic
117 |
118 | # 0.2.4
119 |
120 | * Standardized Errors with TelepatError object
121 | * Implemented Delta.formPatches to more easily form patches from objects
122 | * Moved messaging client from telepat-worker to telepat-models to be reusable by other components
123 | * Added 'type' field to application, context and admin objects
124 |
125 | # 0.2.3
126 |
127 | * Replaced couchbase with elasticsearch through adapters
128 | * Fixed many bugs
129 | * Added email field to Deltas (used by user operations)
130 | * Admin create throws 409 error when admin with that email address already exists
131 | * Implemented admin.delete
132 | * getDevices returns the corect error message when missing
133 | * Implemented Delta.processObject which can be used by all update operations
134 |
135 | # 0.2.2
136 |
137 | * Release full of bug fixes
138 |
139 | # 0.2.1
140 |
141 | * Important performance issue fixed: all Models operations require context id when getting the object from database
142 |
143 | # 0.2.0
144 |
145 | * Implemented Channel and Delta classes to further separate code logic
146 | * Added password field to user objects
147 | * Fixed subscribe.remove and .add
148 | * Fixed application schema keys
149 | * Fixed device persistent udid key
150 | * Return 404 error when unsubscribing with an invalid subscription
151 |
152 | # 0.1.2
153 |
154 | * Added LICENSE and README files
155 | * get All Models and get All Contexts now return an array in the callback instead of hash map
156 |
157 | # 0.1.0
158 |
159 | * Initial Release
160 |
--------------------------------------------------------------------------------
/lib/User.js:
--------------------------------------------------------------------------------
1 | var Application = require('./Application');
2 | var async = require('async');
3 | var FilterBuilder = require('../utils/filterbuilder').FilterBuilder;
4 | var guid = require('uuid');
5 | var TelepatError = require('./TelepatError');
6 |
7 | /**
8 | * Gets an user by email address
9 | * @param user Object object containing the id or email address of the user
10 | * @param callback
11 | * @constructor
12 | */
13 | function User(user, appId, callback) {
14 | if (user.id) {
15 | Application.datasource.dataStorage.getObjects([user.id], function(errs, results) {
16 | if (errs && errs.length > 0) return callback(errs[0]);
17 | callback(null, results[0]);
18 | });
19 | } else if (user.username) {
20 | var filters = (new FilterBuilder('and')).addFilter('is', {application_id: appId}).addFilter('is', {username: user.username});
21 | Application.datasource.dataStorage.searchObjects({modelName: 'user', filters: filters}, function(err, results) {
22 | if (err)
23 | return callback(err);
24 | if (!results.length)
25 | return callback(new TelepatError(TelepatError.errors.UserNotFound));
26 | callback(null, results[0]);
27 | });
28 | }
29 | }
30 |
31 | /**
32 | * Loads the configuration spec file. Automatically loaded at module require.
33 | */
34 | User.load = function() {
35 | User._model = require('../models/user.json');
36 |
37 | if (!User._model) {
38 | Application.logger.emergency('Model \'user\' spec file does not exist.');
39 | process.exit(-1);
40 | }
41 | };
42 |
43 | /**
44 | * Creates a user
45 | * @param props Object Properties of the user.
46 | * @param callback
47 | */
48 | User.create = function(props, appId, callback) {
49 | var self = this;
50 | props.id = props.id || guid.v4();
51 | props.application_id = appId;
52 | props.created = Math.floor((new Date()).getTime()/1000);
53 | props.modified = props.created;
54 | props.type = 'user';
55 |
56 | if (!props.hasOwnProperty('friends'))
57 | props.friends = [];
58 |
59 | if (!props.hasOwnProperty('devices'))
60 | props.devices = [];
61 |
62 | var userMetadata = {
63 | id: guid.v4(),
64 | user_id: props.id,
65 | application_id: appId,
66 | type: 'user_metadata'
67 | };
68 |
69 | User({username: props.username}, appId, function(err, result) {
70 | if (err && err.status == 404) {
71 | Application.datasource.dataStorage.createObjects([props, userMetadata], function(errs) {
72 | if (errs && errs.length) {
73 | errs.forEach(function(error) {
74 | Application.logger.error(error.message);
75 | });
76 | return callback(new TelepatError(TelepatError.errors.ServerFailure, ['failed to create user.']));
77 | }
78 |
79 | callback(null, props);
80 | });
81 | } else {
82 | callback(new TelepatError(TelepatError.errors.UserAlreadyExists));
83 | }
84 | });
85 | }
86 |
87 | User.count = function(appId, callback) {
88 | var filters = null;
89 | if (appId)
90 | filters = (new FilterBuilder()).addFilter('is', {application_id: appId});
91 |
92 | Application.datasource.dataStorage.countObjects({modelName: 'user', filters: filters}, callback);
93 | }
94 |
95 | /**
96 | * Updates a user
97 | * @param patches Object[] The new/updated properties of the user.
98 | * @param callback
99 | */
100 | User.update = function(patches, callback) {
101 | Application.datasource.dataStorage.updateObjects(patches, function(errs, dbObjects) {
102 | if (errs.length) {
103 | return callback(errs[0]);
104 | }
105 |
106 | var objId = Object.keys(dbObjects)[0];
107 |
108 | callback(null, dbObjects[objId]);
109 | });
110 | }
111 |
112 | /**
113 | * Deletes a user.
114 | * @param id string Email address of the user.
115 | * @param callback
116 | */
117 | User.delete = function(id, appId, callback) {
118 | var user = null;
119 |
120 | async.series([
121 | function(callback1) {
122 | User({id: id}, appId, function(err, result) {
123 | if (err) return callback1(err);
124 |
125 | user = result;
126 | callback1();
127 | });
128 | },
129 | function deleteSubscriptions(callback1) {
130 | async.each(user.devices, function(deviceId, c1) {
131 | Application.redisClient.get('blg:devices:'+deviceId, function(err, response) {
132 | if (err) return c1(err);
133 |
134 | if (response) {
135 | var device = JSON.parse(response);
136 | if (device.subscriptions) {
137 | var transaction = Application.redisClient.multi();
138 |
139 | device.subscriptions.each(function(sub) {
140 | transaction.srem([sub, deviceId]);
141 | });
142 |
143 | transaction.del('blg:devices:'+deviceId);
144 |
145 | transaction.exec(function(err, res) {
146 | if (err) Application.logger.warning('Failed removing device from subscriptions: '+err.message);
147 | });
148 | }
149 | }
150 | c1();
151 | });
152 | });
153 | callback1();
154 | },
155 | function(callback1) {
156 | var usrObj = {};
157 | usrObj[id] = 'user';
158 | Application.datasource.dataStorage.deleteObjects(usrObj, function(errs) {
159 | callback1(errs && errs.length > 1 ? errs[0] : null);
160 | });
161 | },
162 | function(callback1) {
163 | var deleteUserObjects = function(obj) {
164 | var deleteObjects = {};
165 | async.each(obj, function(o, c) {
166 | deleteObjects[o.id] = o.type;
167 | c();
168 | }, function() {
169 | Application.datasource.dataStorage.deleteObjects(deleteObjects, function(errs) {
170 | if (errs && errs.length > 1) {
171 | Application.logger.warning('Failed to delete '+errs.length+' user objects.');
172 | }
173 | });
174 | });
175 | };
176 | var filter = (new FilterBuilder()).addFilter('is', {user_id: id});
177 | Application.datasource.dataStorage.searchObjects({filters: filter, fields: ['id', 'type'], scanFunction: deleteUserObjects}, callback1);
178 | }
179 | ], callback);
180 | };
181 |
182 | User.getAll = function(appId, offset, limit, callback) {
183 | var filters = (new FilterBuilder()).addFilter('is', {application_id: appId});
184 | Application.datasource.dataStorage.searchObjects({modelName: 'user', filters: filters, offset: offset, limit: limit}, callback);
185 | };
186 |
187 | User.search = function(appId, filters, offset, limit, callback) {
188 | var filterBuilder = (new FilterBuilder()).addFilter('is', {application_id: appId});
189 |
190 | Object.keys(filters).forEach(function (field) {
191 | var fieldObject = {};
192 | fieldObject[field] = filters[field];
193 | filterBuilder.addFilter('like', fieldObject);
194 | });
195 |
196 | Application.datasource.dataStorage.searchObjects({modelName: 'user', filters: filterBuilder, offset: offset, limit: limit}, callback);
197 | };
198 |
199 | User.getMetadata = function(userId, callback) {
200 | var filters = (new FilterBuilder()).addFilter('is', {user_id: userId});
201 | Application.datasource.dataStorage.searchObjects({modelName: 'user_metadata', filters: filters}, function(err, results) {
202 | if (err) return callback(err);
203 | callback(null, results[0]);
204 | });
205 | };
206 |
207 | User.updateMetadata = function(userId, patches, callback) {
208 | Application.datasource.dataStorage.updateObjects(patches, function(errs) {
209 | callback(errs && errs.length ? errs[0] : null);
210 | });
211 | };
212 |
213 | module.exports = User;
214 |
--------------------------------------------------------------------------------
/lib/TelepatError.js:
--------------------------------------------------------------------------------
1 | var sprintf = require('sprintf-js').vsprintf;
2 |
3 | Error.stackTraceLimit = Infinity;
4 |
5 | var TelepatError = function(error, placeholders) {
6 | Error.captureStackTrace(this, this);
7 | this.name = Object.keys(error)[0];
8 | this.code = error.code;
9 | this.message = placeholders ? sprintf(error.message, placeholders) : error.message;
10 | this.args = placeholders;
11 | this.status = error.status;
12 | };
13 |
14 | TelepatError.prototype = Object.create(Error.prototype);
15 |
16 | TelepatError.errors = {
17 | ServerNotAvailable: {
18 | code: '001',
19 | message: 'The API server is unable to fulfil your request. Try again later',
20 | status: 503
21 | },
22 | ServerFailure: {
23 | code: '002',
24 | message: 'API internal server error: %s',
25 | status: 500
26 | },
27 | NoRouteAvailable: {
28 | code: '003',
29 | message: 'There is no route with this URL path',
30 | status: 404
31 | },
32 | MissingRequiredField: {
33 | code: '004',
34 | message: 'Request body is missing a required field: %s',
35 | status: 400
36 | },
37 | RequestBodyEmpty: {
38 | code: '005',
39 | message: 'Required request body is empty',
40 | status: 400,
41 | },
42 | InvalidContentType: {
43 | code: '006',
44 | message: 'Request content type must be application/json',
45 | status: 415
46 | },
47 | ApiKeySignatureMissing: {
48 | code: '007',
49 | message: 'API key is missing from the request headers',
50 | status: 400
51 | },
52 | InvalidApikey: {
53 | code: '008',
54 | message: 'API key is not valid for this application',
55 | status: 401
56 | },
57 | DeviceIdMissing: {
58 | code: '009',
59 | message: 'Required device ID header is missing',
60 | status: 400
61 | },
62 | ApplicationIdMissing: {
63 | code: '010',
64 | message: 'Required application ID header is missing',
65 | status: 400
66 | },
67 | ApplicationNotFound: {
68 | code: '011',
69 | message: 'Requested application with ID "%s" does not exist',
70 | status: 404
71 | },
72 | ApplicationForbidden: {
73 | code: '012',
74 | message: 'This application does not belong to you',
75 | status: 401
76 | },
77 | AuthorizationMissing: {
78 | code: '013',
79 | message: 'Authorization header is not present',
80 | status: 401
81 | },
82 | InvalidAuthorization: {
83 | code: '014',
84 | message: 'Invalid authorization: %s',
85 | status: 401
86 | },
87 | OperationNotAllowed: {
88 | code: '015',
89 | message: 'You don\'t have the necessary privileges for this operation',
90 | status: 403
91 | },
92 | AdminBadLogin: {
93 | code: '016',
94 | message: 'Wrong user email address or password',
95 | status: 401
96 | },
97 | AdminAlreadyAuthorized: {
98 | code: '017',
99 | message: 'Admin with that email address is already authorized in this application',
100 | status: 409
101 | },
102 | AdminDeauthorizeLastAdmin: {
103 | code: '018',
104 | message: 'Cannot remove yourself from the application because you\'re the only authorized admin',
105 | status: 409
106 | },
107 | AdminNotFoundInApplication: {
108 | code: '019',
109 | message: 'Admin with email address %s does not belong to this application',
110 | status: 404
111 | },
112 | ContextNotFound: {
113 | code: '020',
114 | message: 'Context not found',
115 | status: 404
116 | },
117 | ContextNotAllowed: {
118 | code: '021',
119 | message: 'This context doesn\'t belong to you',
120 | status: 403
121 | },
122 | ApplicationSchemaModelNotFound: {
123 | code: '022',
124 | message: 'Application with ID %s does not have a model named %s',
125 | status: 404
126 | },
127 | UserNotFound: {
128 | code: '023',
129 | message: 'User not found',
130 | status: 404
131 | },
132 | InvalidApplicationUser: {
133 | code: '024',
134 | message: 'User does not belong to this application',
135 | status: 404
136 | },
137 | DeviceNotFound: {
138 | code: '025',
139 | message: 'Device with ID %s not found',
140 | status: 404
141 | },
142 | InvalidContext: {
143 | code: '026',
144 | message: 'Context with id %s does not belong to app with id %s',
145 | status: 403
146 | },
147 | InvalidChannel: {
148 | code: '027',
149 | message: 'Channel is invalid: %s',
150 | status: 400
151 | },
152 | InsufficientFacebookPermissions: {
153 | code: '028',
154 | message: 'Insufficient facebook permissions: %s ',
155 | status: 400
156 | },
157 | UserAlreadyExists: {
158 | code: '029',
159 | message: 'User already exists',
160 | status: 409
161 | },
162 | AdminAlreadyExists: {
163 | code: '030',
164 | message: 'Admin already exists',
165 | status: 409
166 | },
167 | UserBadLogin: {
168 | code: '031',
169 | message: 'User email address or password do not match',
170 | status: 401
171 | },
172 | UnspecifiedError: {
173 | code: '032',
174 | message: 'Unspecified error',
175 | status: 500
176 | },
177 | AdminNotFound: {
178 | code: '033',
179 | message: 'Admin not found',
180 | status: 404
181 | },
182 | ObjectNotFound: {
183 | code: '034',
184 | message: 'Object model %s with ID %s not found',
185 | status: 404
186 | },
187 | ParentObjectNotFound: {
188 | code: '035',
189 | message: 'Unable to create: parent "%s" with ID "%s" does not exist',
190 | status: 404
191 | },
192 | InvalidObjectRelationKey: {
193 | code: '036',
194 | message: 'Unable to create: parent relation key "%s" is not valid. Must be at most %s',
195 | status: 400
196 | },
197 | SubscriptionNotFound: {
198 | code: '037',
199 | message: 'Subscription not found',
200 | status: 404
201 | },
202 | InvalidFieldValue: {
203 | code: '038',
204 | message: 'Invalid field value: %s',
205 | status: 400
206 | },
207 | ClientBadRequest: {
208 | code: '039',
209 | message: 'Generic bad request error: %s',
210 | status: 400
211 | },
212 | MalformedAuthorizationToken: {
213 | code: '040',
214 | message: 'Malformed authorization token',
215 | status: 400
216 | },
217 | InvalidAdmin: {
218 | code: '041',
219 | message: 'Invalid admin',
220 | status: 401
221 | },
222 | InvalidPatch: {
223 | code: '042',
224 | message: 'Invalid patch: %s',
225 | status: 400
226 | },
227 | ApplicationHasNoSchema: {
228 | code: '043',
229 | message: 'Could not fulfill request because application has no schema defined',
230 | status: 501
231 | },
232 | InvalidLoginProvider: {
233 | code: '044',
234 | message: 'Invalid login provider. Possible choices: %s',
235 | status: 400
236 | },
237 | ServerNotConfigured: {
238 | code: '045',
239 | message: 'Unable to fullfill request because the server has not been configured: "%s"',
240 | status: 501
241 | },
242 | ExpiredAuthorizationToken: {
243 | code: '046',
244 | message: 'Expired authorization token',
245 | status: 401
246 | },
247 | UnconfirmedAccount: {
248 | code: '047',
249 | message: 'This user account has not been confirmed',
250 | status: 403
251 | },
252 | QueryError: {
253 | code: '048',
254 | message: 'Failed to parse query filter: %s',
255 | status: 400
256 | },
257 | TilNotFound: {
258 | code: '049',
259 | message: 'TelepatIndexedList with name "%s" does not exist',
260 | status: 404
261 | },
262 | DeviceInvalid: {
263 | code: '050',
264 | message: 'Device with ID %s is invalid: %s',
265 | status: 400
266 | },
267 | ServerConfigurationFailure: {
268 | code: '051',
269 | message: 'Server configuration failure: %s',
270 | status: 500
271 | }
272 | }
273 |
274 | module.exports = TelepatError;
275 |
--------------------------------------------------------------------------------
/utils/utils.js:
--------------------------------------------------------------------------------
1 | var lz4Module = require('lz4');
2 | var stream = require('stream');
3 | var async = require('async');
4 |
5 | /**
6 | * Transform the object that is sent in the request body in the subscribe endpoint so its compatible with
7 | * the elasticsearch query object.
8 | * @param filterObject Object
9 | * @example
10 | * {
11 | "or": [
12 | {
13 | "and": [
14 | {
15 | "is": {
16 | "gender": "male",
17 | "age": 23
18 | }
19 | },
20 | {
21 | "range": {
22 | "experience": {
23 | "gte": 1,
24 | "lte": 6
25 | }
26 | }
27 | }
28 | ]
29 | },
30 | {
31 | "and": [
32 | {
33 | "like": {
34 | "image_url": "png",
35 | "website": "png"
36 | }
37 | }
38 | ]
39 | }
40 | ]
41 | }
42 | */
43 | var parseQueryObject = function(filterObject) {
44 | var objectKey = Object.keys(filterObject)[0];
45 | var result = {};
46 | result[objectKey] = [];
47 |
48 | for(var f in filterObject[objectKey]) {
49 | var filterObjectKey = Object.keys(filterObject[objectKey][f])[0];
50 | var filterType = null;
51 |
52 | if (filterObjectKey == 'and' || filterObjectKey == 'or') {
53 | result[objectKey].push(parseQueryObject(filterObject[objectKey][f]));
54 | continue;
55 | }
56 |
57 | switch(filterObjectKey) {
58 | case 'is': {
59 | filterType = 'term';
60 | break;
61 | }
62 | case 'like': {
63 | filterType = 'text';
64 | break;
65 | }
66 | default: {
67 | var otherFilters = {};
68 | otherFilters[filterObjectKey] = {};
69 |
70 | for(var prop in filterObject[objectKey][f][filterObjectKey]) {
71 | otherFilters[filterObjectKey][prop] = filterObject[objectKey][f][filterObjectKey][prop];
72 | }
73 |
74 | result[objectKey].push(otherFilters);
75 | continue;
76 | }
77 | }
78 |
79 | for(var prop in filterObject[objectKey][f][filterObjectKey]) {
80 | var p = {};
81 | p[filterType] = {};
82 | p[filterType][prop] = filterObject[objectKey][f][filterObjectKey][prop];
83 | result[objectKey].push(p);
84 | }
85 | }
86 |
87 | return result;
88 | };
89 |
90 | /**
91 | * Tests an object against a query object.
92 | * @param Object object Database item
93 | * @param Object query The simplified query object (not the elasticsearch one).
94 | * @returns {boolean}
95 | */
96 | function testObject(object, query) {
97 | if (typeof object != 'object')
98 | return false;
99 |
100 | if (typeof query != 'object')
101 | return false;
102 |
103 | var mainOperator = Object.keys(query)[0];
104 |
105 | if (mainOperator != 'and' && mainOperator != 'or')
106 | return false;
107 |
108 | var result = null;
109 | var partialResult = null
110 |
111 | function updateResult(result, partial) {
112 | //if result is not initialised, use the value of the operation
113 | //otherwise if it had a value from previous operations, combine the previous result with result from
114 | // the current operation
115 | return result === null ? partialResult :(mainOperator == 'and') ? result && partialResult :
116 | result || partialResult;
117 | }
118 |
119 | for(var i in query[mainOperator]) {
120 | if (typeof query[mainOperator][i] != 'object')
121 | continue;
122 |
123 | var operation = Object.keys(query[mainOperator][i])[0];
124 |
125 | operationsLoop:
126 | for(var property in query[mainOperator][i][operation]) {
127 | switch(operation) {
128 | case 'is': {
129 | partialResult = object[property] == query[mainOperator][i][operation][property];
130 |
131 | break;
132 | }
133 |
134 | case 'like': {
135 | partialResult = object[property].toString().search(query[mainOperator][i][operation][property]) !== -1;
136 |
137 | break;
138 | }
139 |
140 | case 'range': {
141 | if (typeof query[mainOperator][i][operation][operation][property] != 'object')
142 | continue;
143 |
144 | rangeQueryLoop:
145 | for(var rangeOperator in query[mainOperator][i][operation][property]) {
146 | var objectPropValue = parseInt(object[property]);
147 | var queryPropValue = parseInt(query[mainOperator][i][operation][property][rangeOperator]);
148 |
149 | switch(rangeOperator) {
150 | case 'gte': {
151 | partialResult = objectPropValue >= queryPropValue;
152 |
153 | break;
154 | }
155 |
156 | case 'gt': {
157 | partialResult = objectPropValue > queryPropValue;
158 |
159 | break;
160 | }
161 |
162 | case 'lte': {
163 | partialResult = objectPropValue <= queryPropValue;
164 |
165 | break;
166 | }
167 |
168 | case 'lt': {
169 | partialResult = objectPropValue < queryPropValue;
170 |
171 | break;
172 | }
173 |
174 | default: {
175 | continue rangeQueryLoop;
176 | }
177 | }
178 |
179 | result = updateResult(result, partialResult);
180 | }
181 |
182 | break;
183 | }
184 |
185 | case 'or':
186 | case 'and': {
187 | //console.log(query[mainOperator][i]);
188 | partialResult = testObject(object, query[mainOperator][i]);
189 |
190 | break;
191 | }
192 |
193 | default: {
194 | continue operationsLoop;
195 | }
196 | }
197 |
198 | result = updateResult(result, partialResult);
199 | }
200 | }
201 |
202 | return !!result;
203 | }
204 |
205 | var lz4 = (function() {
206 |
207 | /**
208 | * @callback lz4ResultCb
209 | * @param {Buffer} result The result of compression/decompression
210 | */
211 | /**
212 | * Only used internally to avoid code dupe
213 | * @param {string|Buffer} data
214 | * @param {int} operation 0 for compression, 1 for decompression
215 | * @param {lz4ResultCb} callback
216 | */
217 | var doWork = function(data, operation, callback) {
218 | var lz4Stream = null;
219 |
220 | if (operation == 0)
221 | lz4Stream = lz4Module.createEncoderStream();
222 | else if (operation == 1)
223 | lz4Stream = lz4Module.createDecoderStream();
224 |
225 | var outputStream = new stream.Writable();
226 | var result = new Buffer('');
227 |
228 | outputStream._write = function(chunk, encoding, callback1) {
229 | result = Buffer.concat([result, chunk]);
230 | callback1();
231 | };
232 |
233 | outputStream.on('finish', function() {
234 | callback(result);
235 | });
236 |
237 | var inputStream = new stream.Readable();
238 | inputStream.push(data);
239 | inputStream.push(null);
240 |
241 | inputStream.pipe(lz4Stream).pipe(outputStream);
242 | }
243 |
244 | return {
245 | /**
246 | * LZ4 compress a string
247 | * @param {string} string
248 | * @param {lz4ResultCb} callback
249 | */
250 | compress: function(string, callback) {
251 | doWork(string, 0, callback);
252 | },
253 | /**
254 | * LZ4 decompress a string
255 | * @param {Buffer} buffer
256 | * @param {lz4ResultCb} callback
257 | */
258 | decompress: function(buffer, callback) {
259 | doWork(buffer, 1, callback);
260 | }
261 | };
262 | })();
263 |
264 | var scanRedisKeysPattern = function(pattern, redisInstance, callback) {
265 | var redisScanCursor = -1;
266 | var results = [];
267 |
268 | var scanAndGet = function(callback1) {
269 | redisInstance.scan([redisScanCursor == -1 ? 0 : redisScanCursor,
270 | 'MATCH', pattern, 'COUNT', 100000], function(err, partialResults) {
271 | if (err) return callback1(err);
272 |
273 | redisScanCursor = partialResults[0];
274 | results = results.concat(partialResults[1]);
275 |
276 | callback1();
277 | });
278 | };
279 |
280 | async.during(
281 | function(callback1) {
282 | callback1(null, redisScanCursor != 0);
283 | },
284 | scanAndGet,
285 | function(err) {
286 | callback(err, results);
287 | }
288 | );
289 | };
290 |
291 | //console.log(JSON.stringify(getQueryKey(JSON.parse('{"or":[{"and":[{"is":{"gender":"male","age":23}},{"range":{"experience":{"gte":1,"lte":6}}}]},{"and":[{"like":{"image_url":"png","website":"png"}}]}]}'))));
292 | //console.log(parseQueryObject(JSON.parse('{"or":[{"and":[{"is":{"gender":"male","age":23}},{"range":{"experience":{"gte":1,"lte":6}}}]},{"and":[{"like":{"image_url":"png","website":"png"}}]}]}')));
293 |
294 | module.exports = {
295 | parseQueryObject: parseQueryObject,
296 | testObject: testObject,
297 | scanRedisKeysPattern: scanRedisKeysPattern,
298 | lz4: lz4
299 | };
300 |
--------------------------------------------------------------------------------
/lib/ConfigurationManager.js:
--------------------------------------------------------------------------------
1 | var dot = require('dot-object');
2 | var validator = require('validator');
3 | var TelepatError = require('./TelepatError');
4 | var async = require('async');
5 | var fs = require('fs');
6 | var clone = require('clone');
7 | var Application = require('./Application');
8 |
9 | var ConfigurationManager = function(specFile, configFile) {
10 | this.specFile = specFile;
11 | this.configFile = configFile;
12 |
13 | /**
14 | * This object holds the collection of all groups containing the grouped variables
15 | * @type {{string}}
16 | */
17 | this.exclusivityGroups = {};
18 |
19 | /**
20 | *
21 | * @type {{string[]}}
22 | */
23 | this.foundExclusiveVariables = {};
24 | };
25 |
26 | ConfigurationManager.prototype.load = function(callback) {
27 | var self = this;
28 |
29 | async.series([
30 | function(callback1) {
31 | fs.readFile(self.specFile, {encoding: 'utf8'}, function(err, contents) {
32 | if (err) {
33 | callback1(TelepatError(TelepatError.errors.ServerConfigurationFailure, [err.message]));
34 | } else {
35 | self.spec = JSON.parse(contents);
36 | callback1();
37 | }
38 | });
39 | },
40 | function(callback1) {
41 | fs.readFile(self.configFile, {encoding: 'utf8'}, function(err, contents) {
42 | if (err) {
43 | callback1(TelepatError(TelepatError.errors.ServerConfigurationFailure, [err.message]));
44 | } else {
45 | self.config = JSON.parse(contents);
46 | callback1();
47 | }
48 | });
49 | }
50 | ], function(err) {
51 | if (err)
52 | throw err;
53 | else {
54 | self._loadExclusivityGroups();
55 | callback();
56 | }
57 | });
58 | };
59 |
60 | ConfigurationManager.prototype._loadExclusivityGroups = function(spec) {
61 | spec = spec || this.spec.root;
62 |
63 | for(var i in spec) {
64 | if (spec[i].exclusive_group) {
65 | if (!this.exclusivityGroups[spec[i].exclusive_group])
66 | this.exclusivityGroups[spec[i].exclusive_group] = [spec[i].name];
67 | else
68 | this.exclusivityGroups[spec[i].exclusive_group].push(spec[i].name);
69 | }
70 | if (spec[i].root) {
71 | this._loadExclusivityGroups(spec[i].root);
72 | }
73 | }
74 | };
75 |
76 | /**
77 | *
78 | * @param {object} [spec]
79 | * @param {object} [config]
80 | * @returns {boolean|TelepatError}
81 | */
82 | ConfigurationManager.prototype._validate = function(spec, config, rootName) {
83 | var result = true;
84 | spec = spec || this.spec.root;
85 | config = config || this.config;
86 | rootName = rootName || '';
87 |
88 | for(var s in spec) {
89 | var varName = spec[s].name;
90 |
91 | if (spec[s].exclusive_group && config[spec[s].name]) {
92 | if (!this.foundExclusiveVariables[spec[s].exclusive_group])
93 | this.foundExclusiveVariables[spec[s].exclusive_group] = [varName];
94 | else
95 | this.foundExclusiveVariables[spec[s].exclusive_group].push(varName);
96 | }
97 |
98 | var varValue = config[spec[s].name];
99 |
100 | if (spec[s].env_var && process.env[spec[s].env_var] && !spec[s].root) {
101 | var theEnvVar = process.env[spec[s].env_var];
102 |
103 | if (spec[s].type == 'array') {
104 | varValue = theEnvVar ? theEnvVar.split(' ') : undefined;
105 | } else if (spec[s].type == 'bool') {
106 | varValue = !!theEnvVar;
107 | } else {
108 | varValue = theEnvVar
109 | }
110 | config[spec[s].name] = varValue;
111 | } else if (spec[s].root && !spec[s].optional && !varValue) {
112 | config[spec[s].name] = varValue = {};
113 | }
114 |
115 | result = result && this.verifyVariable(varValue, spec[s], rootName);
116 |
117 | if (result instanceof TelepatError)
118 | return result;
119 | }
120 |
121 | return result;
122 | };
123 |
124 | ConfigurationManager.prototype.test = function() {
125 | return this._validate();
126 | }
127 |
128 | /**
129 | *
130 | * @param {*} variable
131 | * @param {Object} specVariable
132 | * @param {string} specVariable.name Name of the variable
133 | * @param {string} specVariable.env_var Name of the environment variable to check for, if it's not present in the file
134 | * @param {string} specVariable.type The type of the variable (int, float, array, object, string, bool)
135 | * @param {string} specVariable.array_type The type of the array's elements
136 | * @param {string} specVariable.optional The test passes if the variable is not set, null or empty
137 | * @param {string} specVariable.exclusive_group Exclusivity group for a config variable. Only 1 variable can be in this
138 | * group.
139 | * @param {array} specVariable.enum This variable can only have these values
140 | * @param {Object} specVariable.required_by This variable is verified only when a specific variable has a certain value
141 | * @param {array} specVariable.root Allows for nested objects
142 | * @return {boolean|TelepatError} true if variable passed or an error describing the problem if it didn't pass
143 | */
144 | ConfigurationManager.prototype.verifyVariable = function(variable, specVariable, rootName) {
145 | if (!specVariable.name) {
146 | console.log('Spec file ' + this.specFile + ' has a variable which is missing the "name"' +
147 | ' property');
148 | return true;
149 | }
150 |
151 | var fullVarName = rootName + '.' + specVariable.name;
152 |
153 | if (specVariable.required_by) {
154 | var requiredVarName = Object.keys(specVariable.required_by)[0];
155 | var requiredVarValue = specVariable.required_by[requiredVarName];
156 |
157 | if (dot.pick(requiredVarName, this.config) != requiredVarValue)
158 | return true;
159 | }
160 |
161 | // because nested objects don't have environment variables (only their children) we need to simmulate an empty object
162 | // in the loaded configuration
163 | if (specVariable.root && variable instanceof Object && !Object.keys(variable).length)
164 | return this._validate(specVariable.root, dot.pick(fullVarName.slice(1), this.config), fullVarName);
165 |
166 | if (specVariable.optional)
167 | return true;
168 | //if the value in the config file doen't exist and it also doesn't belong in a exclusive group, it means that it's
169 | //a mandatory config var which is missing
170 | else if (!variable && !specVariable.exclusive_group) {
171 | return new TelepatError(TelepatError.errors.ServerConfigurationFailure, [fullVarName +
172 | ' is mising from the configuration']);
173 | } else if (!variable && specVariable.exclusive_group)
174 | return true;
175 |
176 | if (specVariable.enum && !this.enumVerifier(variable, specVariable.enum)) {
177 | return new TelepatError(TelepatError.errors.ServerConfigurationFailure, [fullVarName + ' can only have these ' +
178 | 'values: "'+specVariable.enum.join(' ')+'"']);
179 | }
180 |
181 | if (!this.typeVerifier(variable, specVariable.type)) {
182 | return new TelepatError(TelepatError.errors.ServerConfigurationFailure, ['Invalid type for variable '
183 | + fullVarName + ', must be "'+specVariable.type+'"']);
184 | }
185 |
186 | if (specVariable.array_type && !this.arrayTypeVerifier(variable, specVariable.array_type))
187 | return new TelepatError(TelepatError.errors.ServerConfigurationFailure, ['Invalid type for variable '
188 | + fullVarName + ' or array has wrong type for its elements']);
189 |
190 | if (specVariable.exclusive_group) {
191 | // !(if the value doesn't exist in the config file but it's a part of an exclusive group)
192 | if (!(!variable && this.foundExclusiveVariables[specVariable.exclusive_group]) && !this.exclusivityVerifier(specVariable.exclusive_group))
193 | return new TelepatError(TelepatError.errors.ServerConfigurationFailure, ['At most one of these variables "'
194 | + this.exclusivityGroups[specVariable.exclusive_group].join(' ') + '" can be present']);
195 | }
196 |
197 | return specVariable.root ? this._validate(specVariable.root, dot.pick(fullVarName.slice(1), this.config), fullVarName) : true;
198 | };
199 |
200 | /**
201 | *
202 | * @param {*} variable
203 | * @param {string} type
204 | * @returns {boolean}
205 | */
206 | ConfigurationManager.prototype.typeVerifier = function(variable, type) {
207 | switch(type) {
208 | case 'int': {
209 | return validator.isInt('' + variable);
210 | }
211 |
212 | case 'float': {
213 | return validator.isFloat('' + variable);
214 | }
215 |
216 | case 'array': {
217 | return Array.isArray(variable);
218 | }
219 |
220 | case 'object': {
221 | return (variable instanceof Object);
222 | }
223 |
224 | case 'string': {
225 | return (typeof variable) === 'string';
226 | }
227 |
228 | case 'bool': {
229 | return (typeof variable) === 'boolean';
230 | }
231 |
232 | default:
233 | return true;
234 | }
235 | };
236 |
237 | /**
238 | * Checks the elements of an array if the are of the same specified type
239 | * @param {*} array
240 | * @param {string} type
241 | * @returns {boolean} True of all elements pass the type test
242 | */
243 | ConfigurationManager.prototype.arrayTypeVerifier = function(array, type) {
244 | if (!Array.isArray(array))
245 | return false;
246 |
247 | for(var i in array) {
248 | if (!this.typeVerifier(array[i], type))
249 | return false;
250 | }
251 | };
252 |
253 | /**
254 | * Checks if the value is found in the enum array
255 | * @param {*} value
256 | * @param {Array} array
257 | * @returns {boolean} True of all elements pass the type test
258 | */
259 | ConfigurationManager.prototype.enumVerifier = function(value, array) {
260 | return array.indexOf(value) !== -1;
261 | };
262 |
263 | /**
264 | * Verifies if the group hasn't been already created. Only one variable in this group should exist.
265 | * @param {string} group
266 | * @returns {boolean} returns true if the group is empty
267 | */
268 | ConfigurationManager.prototype.exclusivityVerifier = function(group) {
269 | //we use .length == 1 because in _validate() we insert this variable here before calling this function
270 | return this.foundExclusiveVariables[group] && this.foundExclusiveVariables[group].length === 1;
271 | };
272 |
273 | module.exports = ConfigurationManager;
--------------------------------------------------------------------------------
/lib/Application.js:
--------------------------------------------------------------------------------
1 | var async = require('async');
2 | var FilterBuilder = require('../utils/filterbuilder').FilterBuilder;
3 | var guid = require('uuid');
4 | var TelepatError = require('./TelepatError');
5 |
6 | /**
7 | * @typedef {{
8 | * relationType: string,
9 | * parentModel: string
10 | * }} Relation
11 | */
12 |
13 | /**
14 | * @typedef {{
15 | * meta_read_acl: Number,
16 | * read_acl: Number,
17 | * write_acl: Number,
18 | * properties: Object,
19 | * belongsTo: Relation[],
20 | * hasSome: string[],
21 | * hasMany: string[],
22 | * hasSome_property: string,
23 | * ios_push_field: string,
24 | * author_fields: string[]
25 | * }} Model
26 | */
27 |
28 | /**
29 | * @typedef {{
30 | * name: string,
31 | * keys: string[],
32 | * admins: string[],
33 | * type: "application",
34 | * id: "id",
35 | * created: 0,
36 | * modified: Number,
37 | * email_confirmation: Boolean,
38 | * from_email: string,
39 | * password_reset: Object,
40 | * password_reset.android_app_link: string,
41 | * password_reset.app_link: string,
42 | * password_reset.web_link: string,
43 | * schema: Object.,
44 | * apn_key: string,
45 | * apn_key_id: string,
46 | * apn_team_id: string,
47 | * apn_topic: string,
48 | * gcm_api_key: string,
49 | * email_templates: {weblink: string, confirm_account: string, after_confirm: string, reset_password: string}
50 | * }} App
51 | */
52 |
53 | /**
54 | * Gets an application by id.
55 | * @param _id integer Application id
56 | * @param callback
57 | * @constructor
58 | * @property bucket Bucket Couchbase data bucket
59 | * @property stateBucket Bucket Couchbase state bucket
60 | * @property {Object} loadedAppModels
61 | * @property {Object.} loadedAppModels.
62 | */
63 | function Application(id, callback) {
64 | async.waterfall([
65 | //get from cache
66 | function(callback1) {
67 | Application.redisClient.get(['blg:application:'+id], function(err, result) {
68 | if (err) return callback1(err);
69 |
70 | if (!result)
71 | return callback1(null, false);
72 |
73 | callback1(null, JSON.parse(result));
74 | });
75 | },
76 | //if application is found send it right away
77 | //otherwise get it from dataStorage
78 | function(application, callback1) {
79 | if(application)
80 | return callback1(null, application);
81 |
82 | Application.datasource.dataStorage.getObjects([id], function(err, results) {
83 | if (err & err.length) return callback1(err[0]);
84 | callback1(null, results[0]);
85 | });
86 | },
87 | //add to cache with expiration time of 7 weeks
88 | //all update/delete operation will invalidate this key (redis 'del')
89 | function(application, callback1) {
90 | Application.redisClient.set('blg:application:'+id, JSON.stringify(application), 'EX', '604800', function(err) {
91 | if (err) return callback1(err);
92 | callback1(null, application);
93 | });
94 | }
95 | ], callback);
96 | }
97 |
98 | Application.loadedAppModels = {};
99 |
100 | /**
101 | *
102 | * @type {RedisClient}
103 | */
104 | Application.redisClient = null;
105 |
106 | /**
107 | *
108 | * @type {RedisClient}
109 | */
110 | Application.redisCacheClient = null;
111 |
112 | /**
113 | *
114 | * @type {Datasource}
115 | */
116 | Application.datasource = null;
117 |
118 | /**
119 | *
120 | * @type {TelepatLogger}
121 | */
122 | Application.logger = null;
123 |
124 | /**
125 | * Gets all aplications
126 | * @param callback
127 | */
128 | Application.loadAllApplications = function(offset, limit, callback) {
129 | offset = offset || 0;
130 | limit = limit || Application.datasource.dataStorage.config.get_limit;
131 |
132 | Application.datasource.dataStorage.searchObjects({modelName: 'application', offset: offset, limit: limit}, function(err, applications) {
133 | if (err) return callback(err);
134 |
135 | applications.forEach(function(app) {
136 | Application.loadedAppModels[app.id] = app;
137 | });
138 | callback();
139 | });
140 | }
141 |
142 | /**
143 | * Gets the current application index used for IDs.
144 | * @param callback
145 | */
146 | Application.count = function(callback) {
147 | Application.datasource.dataStorage.countObjects({modelName: 'application'}, callback);
148 | };
149 |
150 | /**
151 | * Creates a new application
152 | * @param props Object properties
153 | * @param callback
154 | */
155 | Application.create = function(props, callback) {
156 | props.id = guid.v4();
157 | props.keys = props.keys || [];
158 | props.created = Math.floor((new Date()).getTime()/1000);
159 | props.modified = props.created;
160 | props.type = 'application';
161 | Application.datasource.dataStorage.createObjects([props], function(errs) {
162 | if (errs) return callback(errs[0]);
163 |
164 | Application.loadedAppModels[props.id] = props;
165 |
166 | callback(null, props);
167 | });
168 | };
169 |
170 | /**
171 | * Updates an application
172 | * @param id integer Application ID
173 | * @param props Object new properties for the application
174 | * @param callback
175 | */
176 | Application.update = function(id, patches, callback) {
177 | async.waterfall([
178 | function(callback1) {
179 | Application.datasource.dataStorage.updateObjects(patches, function(errs, apps) {
180 | if (errs && errs.length) return callback1(errs[0]);
181 |
182 | Application.loadedAppModels[id] = apps[id];
183 | callback1(null, apps[id]);
184 | });
185 | },
186 | function(application, callback1) {
187 | Application.redisClient.del('blg:application:'+id, function(err) {
188 | if (err) return callback1(err);
189 | callback1(null, application);
190 | });
191 | }
192 | ], callback);
193 | }
194 |
195 | /**
196 | * Deletes an application and all of its contexts.
197 | * @param id integer Application ID.
198 | * @param callback
199 | */
200 | Application.delete = function (id, callback) {
201 | async.series([
202 | function(callback1) {
203 | var appObj = {};
204 | appObj[id] = 'application';
205 | Application.datasource.dataStorage.deleteObjects(appObj, callback1);
206 | },
207 | function(callback1) {
208 | delete Application.loadedAppModels[id];
209 | var deleteAppObjects = function(obj) {
210 | var deleteObjects = {};
211 | async.each(obj, function(o, c) {
212 | deleteObjects[o.id] = o.type;
213 | c();
214 | }, function() {
215 | Application.datasource.dataStorage.deleteObjects(deleteObjects, function(errs) {
216 | if (errs && errs.length > 1) {
217 | Application.logger.warning('Failed to delete '+errs.length+' application objects.');
218 | }
219 | });
220 | });
221 | };
222 |
223 | var filter = (new FilterBuilder()).addFilter('is', {application_id: id});
224 | Application.datasource.dataStorage.searchObjects({filters: filter, fields: ['id', 'type'], scanFunction: deleteAppObjects}, callback1);
225 | },
226 | function(callback1) {
227 | Application.redisClient.del('blg:application:'+id, callback1);
228 | }
229 | ], callback);
230 | }
231 |
232 | /**
233 | * Gets the model schema of an app from database.
234 | * @param appId ingeger Application ID.
235 | * @param callback
236 | */
237 | Application.getAppSchema = function(appId, callback) {
238 | if (!Application.loadedAppModels[appId])
239 | return callback(new TelepatError(TelepatError.errors.ApplicationNotFound, [appId]));
240 | else if (!Application.loadedAppModels[appId].schema)
241 | return callback(new TelepatError(TelepatError.errors.ApplicationHasNoSchema, [appId]))
242 |
243 | callback(null, Application.loadedAppModels[appId].schema);
244 | }
245 |
246 | /**
247 | * Updates the model schema of an app
248 | * @param appId integer Application ID
249 | * @param schema Object The schema object with updated values.
250 | * @param callback
251 | */
252 | Application.updateSchema = function(appId, schema, callback) {
253 | async.series([
254 | function(callback1) {
255 | Application.datasource.dataStorage.updateObjects([
256 | {
257 | op: 'replace',
258 | path: 'application/'+appId+'/schema',
259 | value: schema
260 | }
261 | ], function(errs) {
262 | callback1(errs && errs.length ? errs[0] : null);
263 | });
264 | },
265 | function(callback1) {
266 | Application.redisClient.del('blg:application:'+appId, function(err, result) {
267 | if (err) return callback1(err);
268 | callback1(null, schema);
269 | });
270 | }
271 | ], callback);
272 | }
273 |
274 | /**
275 | * Deletes a model schema along with its items.
276 | * @param appId integer Application ID
277 | * @param modelName string The model name to delete
278 | * @param callback
279 | */
280 | Application.deleteModel = function(appId, modelName, callback) {
281 | if (!Application.loadedAppModels[appId])
282 | return callback(new TelepatError(TelepatError.errors.ApplicationNotFound, [appId]));
283 | else if (!Application.loadedAppModels[appId].schema)
284 | return callback(new TelepatError(TelepatError.errors.ApplicationHasNoSchema, [appId]))
285 | else if (!Application.loadedAppModels[appId].schema[modelName])
286 | return callback(new TelepatError(TelepatError.errors.ApplicationSchemaModelNotFound, [appId, modelName]))
287 |
288 | async.series([
289 | function(callback1) {
290 | delete Application.loadedAppModels[appId].schema[modelName];
291 | Application.datasource.dataStorage.updateObjects([
292 | {
293 | op: 'replace',
294 | path: 'application/'+appId+'/schema',
295 | value: Application.loadedAppModels[appId].schema
296 | }
297 | ], function(errs, results) {
298 | if (errs && errs.length)
299 | return callback1(errs[0]);
300 | callback1();
301 | });
302 | },
303 | function(callback1) {
304 | Application.redisClient.del('blg:application:'+appId, callback1);
305 | }
306 | ], callback);
307 | }
308 |
309 | Application.hasContext = function(appId, contextId, callback) {
310 | Application.datasource.dataStorage.getObjects([contextId], function(err, results) {
311 | if (err && err[0] && err[0].status == 404)
312 | return callback(new TelepatError(TelepatError.errors.ContextNotFound));
313 |
314 | callback(null, results[0].application_id == appId);
315 | });
316 | }
317 |
318 | module.exports = Application;
319 |
--------------------------------------------------------------------------------
/lib/Subscription.js:
--------------------------------------------------------------------------------
1 | var Application = require('./Application');
2 | var async = require('async');
3 | var User = require('./User');
4 | var utils = require('../utils/utils');
5 | var TelepatError = require('./TelepatError');
6 | var objectMerge = require('object-merge');
7 |
8 | function Subscription() {};
9 |
10 | /**
11 | * Get the subscribed devices of a filtered channel
12 | * @param channel {Channel} object that contains model name and id
13 | * @param callback
14 | */
15 | Subscription.getSubscribedDevices = function(channel, callback) {
16 | Application.redisClient.smembers(channel.get(), callback);
17 | };
18 |
19 | /**
20 | * Adds a subscription
21 | * @param deviceId string Device ID
22 | * @param channel {Channel} object that contains model name and id, user and parent {id, model}
23 | * @param callback
24 | */
25 | Subscription.add = function(appId, deviceId, channel, callback) {
26 | var device = null;
27 |
28 | async.series([
29 | function(callback1) {
30 | Subscription.getDevice(appId, deviceId, function(err, result) {
31 | if (err) return callback(result);
32 |
33 | device = result;
34 | callback1();
35 | });
36 | },
37 | function(callback1) {
38 | var transportType = '';
39 | var token = '';
40 |
41 | if (device.volatile && device.volatile.active) {
42 | transportType = device.volatile.server_name;
43 | token = device.volatile.token;
44 |
45 | if (!transportType || !token)
46 | return callback1(new TelepatError(TelepatError.errors.DeviceInvalid, [deviceId, 'volatile server_name or token is missing']));
47 |
48 | } else {
49 | if (!device.persistent || !device.persistent.type || !device.persistent.token)
50 | return callback1(new TelepatError(TelepatError.errors.DeviceInvalid, [deviceId, 'persistent type and/or token is missing']));
51 | transportType = device.persistent.type+'_transport';
52 | token = device.persistent.token;
53 | }
54 |
55 | Application.redisClient.sadd([channel.get(), transportType+'|'+deviceId+'|'+token+'|'+appId], callback1);
56 | },
57 | function(callback1) {
58 | var deviceSubscriptionsKey = 'blg:' + appId + ':device:' + deviceId + ':subscriptions';
59 |
60 | Application.redisClient.sadd([deviceSubscriptionsKey, channel.get()], callback1);
61 | }
62 | ], callback);
63 | };
64 |
65 | /**
66 | * Adds a new inactive device.
67 | * @param device Object The device properties
68 | * @param callback
69 | */
70 | Subscription.addDevice = function(device, callback) {
71 | var key = 'blg:'+device.application_id+':devices:'+device.id;
72 |
73 | async.parallel([
74 | function(callback1) {
75 | if (device.info && device.info.udid) {
76 | var udidKey = 'blg:'+device.application_id+':devices:udid:'+device.info.udid;
77 | Application.redisClient.set(udidKey, device.id, callback1);
78 | } else
79 | callback1();
80 |
81 | },
82 | function(callback1) {
83 | Application.redisClient.set(key, JSON.stringify(device), callback1);
84 | }
85 | ], callback);
86 | };
87 |
88 | /**
89 | * Removes the subscription
90 | * @param context integer Context ID
91 | * @param deviceId string Device ID
92 | * @param channel Object object that contains model name and id
93 | * @param filters Object filter that contains user ID and and an object of parent (model & id)
94 | * @param {string|undefined} [token]
95 | * @param callback
96 | */
97 | Subscription.remove = function(appId, deviceId, channel, token, callback){
98 | if (typeof channel != 'string') {
99 | channel = channel.get();
100 | }
101 |
102 | if (typeof token === 'function') {
103 | callback = token;
104 | token = undefined;
105 | }
106 |
107 | var removed = 0;
108 | var device = null;
109 |
110 | async.series([
111 | function(callback1) {
112 | Subscription.getDevice(appId, deviceId, function(err, result) {
113 | if (err) return callback1(err);
114 |
115 | device = result;
116 | callback1();
117 | });
118 | },
119 | function(callback1) {
120 | var deviceSubscriptionsKey = 'blg:' + appId + ':device:' + deviceId + ':subscriptions';
121 |
122 | Application.redisClient.srem([deviceSubscriptionsKey, channel], callback1);
123 | },
124 | function(callback1) {
125 | var transportType = '';
126 |
127 | if (!token) {
128 | if (device.volatile && device.volatile.active) {
129 | token = device.volatile.token;
130 | } else {
131 | token = device.persistent.token;
132 | }
133 | }
134 |
135 | if (device.volatile && device.volatile.active) {
136 | transportType = device.volatile.server_name;
137 | } else {
138 | transportType = device.persistent.type + '_transport';
139 | }
140 |
141 | Application.redisClient.srem([channel, transportType+'|'+deviceId+'|'+token+'|'+appId], function(err, result) {
142 | if (err) return callback1(err);
143 |
144 | removed = result;
145 | callback1();
146 | });
147 | },
148 | function(callback1) {
149 | if (removed == 0) {
150 | return callback1(new TelepatError(TelepatError.errors.SubscriptionNotFound));
151 | }
152 | callback1();
153 | }
154 | ], callback);
155 | };
156 |
157 | Subscription.removeAllSubscriptionsFromDevice = function(appId, deviceId, token, transport, callback) {
158 | var device = null,
159 | subscriptions = [];
160 |
161 | if (!transport || typeof transport !== 'string') {
162 | return callback(new TelepatError(TelepatError.errors.UnspecifiedError, ['removeAllSubscriptionsFromDevice: need to specify transport']));
163 | }
164 |
165 | async.series([
166 | function(callback1) {
167 | Subscription.getDeviceSubscriptions(appId, deviceId, function(err, results) {
168 | if (err) {
169 | return callback1(err);
170 | }
171 |
172 | subscriptions = results;
173 | callback1();
174 | });
175 | },
176 | function(callback1) {
177 | if (!subscriptions.length) {
178 | return callback1();
179 | }
180 |
181 | var transaction = Application.redisClient.multi();
182 |
183 | subscriptions.forEach(function(subscription) {
184 | transaction.srem([subscription, transport+'|'+deviceId+'|'+token+'|'+appId]);
185 | });
186 |
187 | transaction.exec(callback1);
188 | }
189 | ], function(err) {
190 | callback(err, subscriptions);
191 | });
192 | };
193 |
194 | /**
195 | * Gets a device.
196 | * @param id string Device ID
197 | * @param callback
198 | */
199 | Subscription.getDevice = function(appId, id, callback) {
200 | var key = 'blg:'+appId+':devices:'+id;
201 |
202 | Application.redisClient.get(key, function(err, result) {
203 | if (err) return callback(err);
204 |
205 | if (result === null) {
206 | return callback(new TelepatError(TelepatError.errors.DeviceNotFound, [id]));
207 | }
208 |
209 | callback(null, JSON.parse(result));
210 | });
211 | };
212 |
213 | Subscription.getDeviceSubscriptions = function(appId, deviceId, callback) {
214 | var deviceSubscriptionsKey = 'blg:' + appId + ':device:' + deviceId + ':subscriptions';
215 |
216 | Application.redisClient.smembers([deviceSubscriptionsKey], callback);
217 | };
218 |
219 | Subscription.removeDevice = function(appId, id, callback) {
220 | var keys = ['blg:'+appId+':devices:'+id];
221 | keys.push('blg:'+appId+':device:'+ id +':subscriptions');
222 |
223 | Application.redisClient.del(keys, function(err, result) {
224 | if (err) return callback(err);
225 |
226 | if (result === null || result === 0) {
227 | return callback(new TelepatError(TelepatError.errors.DeviceNotFound, [id]));
228 | }
229 |
230 | callback();
231 | });
232 | };
233 |
234 | Subscription.findDeviceByUdid = function(appId, udid, callback) {
235 | var udidkey = 'blg:'+appId+':devices:udid:'+udid;
236 | Application.redisClient.get(udidkey, callback);
237 | };
238 |
239 | /**
240 | * Gets all the devices.
241 | * @param callback
242 | */
243 | Subscription.getAllDevices = function(appId, callback) {
244 | utils.scanRedisKeysPattern('blg:'+appId+':devices:[^udid]*', Application.redisClient, function(err, results) {
245 | if (err) return callback(err);
246 |
247 | Application.redisClient.mget(results, function(err, results) {
248 | var devices = {};
249 |
250 | async.each(results, function(result, c) {
251 | if (result) {
252 | var parsedDevice = JSON.parse(result);
253 |
254 | if (parsedDevice.volatile && parsedDevice.volatile.active) {
255 | if (!devices[parsedDevice.volatile.server_name])
256 | devices[parsedDevice.volatile.server_name] = [parsedDevice.id + '|' +parsedDevice.volatile.token];
257 | else
258 | devices[parsedDevice.volatile.server_name].push(parsedDevice.id + '|' +parsedDevice.volatile.token);
259 |
260 | } else if(parsedDevice.persistent) {
261 | var queueName = parsedDevice.persistent.type+'_transport';
262 |
263 | if (!devices[queueName])
264 | devices[queueName] = [parsedDevice.id + '|' +parsedDevice.persistent.token];
265 | else
266 | devices[queueName].push(parsedDevice.id + '|' +parsedDevice.persistent.token);
267 | }
268 | }
269 | c();
270 | }, function() {
271 | callback(null, devices);
272 | });
273 | });
274 | });
275 | }
276 |
277 | /**
278 | * Updates a device
279 | * @param deviceUpdates Object Device properties (must include 'id').
280 | * @param callback
281 | */
282 | Subscription.updateDevice = function(appId, device, props, callback) {
283 | var key = 'blg:'+appId+':devices:'+device;
284 |
285 | Subscription.getDevice(appId, device, function(err, dev) {
286 | if (err) return callback(err);
287 |
288 | var newDevice = objectMerge(dev, props);
289 |
290 | Application.redisClient.set([key, JSON.stringify(newDevice), 'XX'], callback);
291 | });
292 | }
293 |
294 | /**
295 | *
296 | * @param {Channel} channel
297 | * @param calllback
298 | */
299 | Subscription.getSubscriptionKeysWithFilters = function(channel, callback) {
300 | var filterChannels = [];
301 | utils.scanRedisKeysPattern(channel.get()+':filter:*[^:count_cache:LOCK]', Application.redisClient, function(err, results) {
302 | if (err) return callback(err);
303 |
304 | for(var k in results) {
305 | //last part of the key is the base64-encoded filter object
306 | var lastKeyPart = results[k].split(':').pop();
307 |
308 | //the base64 encoded filter object is at the end of the key name, after ':filter:'
309 | var queryObject = JSON.parse((new Buffer(lastKeyPart, 'base64')).toString('utf-8'));
310 |
311 | filterChannels.push(channel.clone().setFilter(queryObject));
312 | }
313 | callback(null, filterChannels);
314 | });
315 | };
316 |
317 | module.exports = Subscription;
318 |
--------------------------------------------------------------------------------
/lib/Model.js:
--------------------------------------------------------------------------------
1 | var Application = require('./Application');
2 | var User = require('./User');
3 | var Context = require('./Context');
4 | var TelepatError = require('./TelepatError');
5 | var async = require('async');
6 | var utils = require('../utils/utils');
7 | var FilterBuilder = require('../utils/filterbuilder').FilterBuilder;
8 | var guid = require('uuid');
9 | /**
10 | * Retrieves a single item
11 | * @param _id ID of the item
12 | * @param callback
13 | * @constructor
14 | */
15 | function Model(_id, callback) {
16 | Application.datasource.dataStorage.getObjects([_id], function(errs, results) {
17 | if (errs.length) return callback(errs[0]);
18 | callback(null, results[0]);
19 | });
20 | }
21 |
22 | Model.delete = function(objects, callback) {
23 | var childrenFilter = new FilterBuilder('or');
24 |
25 | var appModels = {};
26 |
27 | async.series([
28 | function(callback1) {
29 | async.forEachOfSeries(objects, function(modelName, id, c) {
30 | if (modelName == 'context')
31 | Context.delete(id, function() {});
32 | else
33 | appModels[id] = modelName;
34 | c();
35 | }, callback1);
36 | },
37 | function(callback1) {
38 | Application.datasource.dataStorage.deleteObjects(appModels, function(errs) {
39 | if (errs && errs.length >= 1) {
40 | async.each(errs, function(error, c) {
41 | if (error.status == 404) {
42 | Application.logger.notice('Model "'+error.args[0]+'" with ID "'+error.args[1]+'" not found.');
43 | delete appModels[error.args[1]];
44 | c();
45 | } else {
46 | c(error);
47 | }
48 | }, callback1);
49 | } else
50 | callback1();
51 | });
52 | },
53 | function(callback1) {
54 | async.each(Object.keys(appModels), function(id, c) {
55 | var modelName = appModels[id];
56 | var filterObj = {};
57 |
58 | filterObj[modelName+'_id'] = id;
59 | childrenFilter.addFilter('is', filterObj);
60 | c();
61 | }, callback1);
62 | },
63 | function(callback1) {
64 | if (!childrenFilter.isEmpty()) {
65 | var deleteChildObjects = function(obj) {
66 | var deleteObjects = {};
67 |
68 | async.each(obj, function(o, c) {
69 | deleteObjects[o.id] = o.type;
70 | c();
71 | }, function() {
72 | Application.datasource.dataStorage.deleteObjects(deleteObjects, function(errs) {});
73 | });
74 | };
75 | Application.datasource.dataStorage.searchObjects({filters: childrenFilter, fields: ['id', 'type'], scanFunction: deleteChildObjects}, callback1);
76 | }
77 | else
78 | callback1();
79 | }
80 | ]);
81 | callback();
82 | };
83 |
84 | /**
85 | * Used for unique IDs
86 | * @param modelName
87 | * @param callback
88 | */
89 | Model.count = function(modelName, appId, callback) {
90 | var filter = new FilterBuilder();
91 | filter.addFilter('is', {application_id: appId});
92 | Application.datasource.dataStorage.countObjects({modelName: modelName, filters: filter}, callback);
93 | }
94 |
95 | Model.create = function(deltas, callback, returnedObjectsCb) {
96 | var curatedDeltas = [];
97 | var objParentInfo = {};
98 |
99 | var builtinModels = ['application', 'admin', 'user', 'user_metadata', 'context'];
100 |
101 | async.series([
102 | function(callback1) {
103 | async.forEachOf(deltas, function(d, i, c) {
104 | var object = deltas[i].object;
105 | object.id = guid.v4();
106 | object.created = Math.floor((new Date()).getTime()/1000);
107 | object.modified = object.created;
108 |
109 | var modelName = object.type;
110 |
111 | if (!Application.loadedAppModels[object.application_id] || !Application.loadedAppModels[object.application_id].schema)
112 | return c();
113 | var appModels = Application.loadedAppModels[object.application_id].schema;
114 |
115 | if (builtinModels.indexOf(modelName) !== -1)
116 | return c();
117 |
118 | for (var r in appModels[modelName].belongsTo) {
119 | if (object[appModels[modelName].belongsTo[r].parentModel + '_id']) {
120 | var parent = {
121 | model: appModels[modelName].belongsTo[r].parentModel,
122 | id: object[appModels[modelName].belongsTo[r].parentModel + '_id']
123 | };
124 | var relationType = appModels[modelName].belongsTo[r].relationType;
125 | }
126 | }
127 |
128 | if (!parent)
129 | return c();
130 |
131 | if (relationType == 'hasSome') {
132 | var parentRelationKey = object[appModels[parent.model].hasSome_property+'_index'];
133 | }
134 |
135 | objParentInfo[parent.id] = {
136 | model: parent.model,
137 | parentRelationKey: parentRelationKey
138 | };
139 |
140 | c();
141 | }, callback1);
142 | }, function(callback1) {
143 | if (!Object.keys(objParentInfo).length) {
144 | curatedDeltas = deltas;
145 | callback1();
146 | } else {
147 | Application.datasource.dataStorage.getObjects(Object.keys(objParentInfo), function(errs, results) {
148 | if (errs && errs.length >= 1) {
149 | errs.forEach(function(error) {
150 | if (error && error.status == 404) {
151 | Application.logger.warning((new TelepatError(TelepatError.errors.ParentObjectNotFound, [error.args[0], error.args[1]])).message);
152 | delete objParentInfo[error.args[1]];
153 | } else {
154 | return callback1(error);
155 | }
156 | });
157 | }
158 | if (results && results.length >= 1) {
159 | results.forEach(function(result) {
160 | if (!Application.loadedAppModels[result.application_id] || !Application.loadedAppModels[result.application_id].schema)
161 | return ;
162 | var appModels = Application.loadedAppModels[result.application_id].schema;
163 | if (objParentInfo[result.id].parentRelationKey)
164 | objParentInfo[result.id].relationKeyLength = result[appModels[result.type].hasSome_property].length;
165 | });
166 | deltas.forEach(function(delta, i) {
167 | var obj = delta.object;
168 | var appModels = Application.loadedAppModels[obj.application_id].schema;
169 |
170 | if (!Application.loadedAppModels[obj.application_id] || !Application.loadedAppModels[obj.application_id].schema)
171 | return ;
172 |
173 | if (builtinModels.indexOf(obj.type) !== -1) {
174 | curatedDeltas.push(delta);
175 | return;
176 | }
177 |
178 | for (var r in appModels[obj.type].belongsTo) {
179 | if (appModels[obj.type].belongsTo[r].relationType == 'hasSome' && obj[appModels[obj.type].belongsTo[r].parentModel + '_id']) {
180 | var parent = {
181 | model: appModels[obj.type].belongsTo[r].parentModel,
182 | id: obj[appModels[obj.type].belongsTo[r].parentModel + '_id']
183 | };
184 | }
185 | }
186 |
187 | if (!parent) {
188 | curatedDeltas.push(delta);
189 | return;
190 | }
191 |
192 | if (obj[appModels[obj.type].hasSome_property] &&
193 | obj[appModels[obj.type].hasSome_property].length <= objParentInfo[obj.id].relationKeyLength) {
194 |
195 | Application.logger.warning((new TelepatError(TelepatError.errors.InvalidObjectRelationKey,
196 | [
197 | objParentInfo[obj.id].parentRelationKey,
198 | objParentInfo[obj.id].relationKeyLength
199 | ])).message);
200 |
201 | } else
202 | curatedDeltas.push(delta);
203 | });
204 | }
205 | callback1();
206 | });
207 | }
208 | }, function(callback1) {
209 | returnedObjectsCb(curatedDeltas);
210 |
211 | var dbItems = [];
212 |
213 | curatedDeltas.forEach(function(d) {
214 | dbItems.push(d.object);
215 | });
216 |
217 | var appModelsObjects = [];
218 |
219 | async.eachSeries(dbItems, function(item, c) {
220 | if (item.type == 'user') {
221 | User.create(item, item.application_id, function(){});
222 | }
223 | else if (item.type == 'context')
224 | Context.create(item, function(){});
225 | else
226 | appModelsObjects.push(item);
227 | c();
228 | }, function() {
229 | if (appModelsObjects.length > 0)
230 | Application.datasource.dataStorage.createObjects(appModelsObjects, function(errs, result) {});
231 |
232 | callback1();
233 | });
234 | }
235 | ], callback);
236 | };
237 |
238 | /**
239 | * Updates and item
240 | * @param props changed properties of the model
241 | * @param user_id author of the model
242 | * @param parent parent of the model ({name, id} object)
243 | * @param cb
244 | */
245 | Model.update = function(patches, callback) {
246 | var appModelsPatches = [];
247 | var userPatches = [];
248 | var contextPatches = [];
249 |
250 | async.eachSeries(patches, function(p, c) {
251 | if (p.path.split('/')[0] == 'user')
252 | userPatches.push(p);
253 | else if (p.path.split('/')[0] == 'context')
254 | contextPatches.push(p);
255 | else
256 | appModelsPatches.push(p);
257 | c();
258 | }, function() {
259 | if (appModelsPatches.length) {
260 | Application.datasource.dataStorage.updateObjects(appModelsPatches, function(errs) {
261 | if (errs && errs.length >= 1) {}
262 | errs.forEach(function(error) {
263 | if (error.status == 404)
264 | Application.logger.notice(error.message);
265 | else
266 | Application.logger.error(error.toString());
267 | });
268 | });
269 | }
270 |
271 | if (userPatches.length) {
272 | User.update(userPatches, function(err){
273 | if (err)
274 | Application.logger.error(err);
275 | });
276 | }
277 |
278 | if (contextPatches.length) {
279 | Context.update(contextPatches, function(err) {
280 | if (err)
281 | Application.logger.error(err);
282 | });
283 | }
284 |
285 | callback();
286 | });
287 | };
288 |
289 | Model.getFilterFromChannel = function(channel) {
290 | var searchFilters = new FilterBuilder();
291 |
292 | searchFilters.addFilter('is', {application_id: channel.appId});
293 |
294 | if (channel.props.context)
295 | searchFilters.addFilter('is', {context_id: channel.props.context});
296 |
297 | if (channel.props.user) {
298 | searchFilters.addFilter('is', {user_id: channel.props.user});
299 | }
300 |
301 | if(channel.props.parent) {
302 | var filterObj = {};
303 | filterObj[channel.props.parent.model+'_id'] = channel.props.parent.id;
304 | searchFilters.addFilter('is', filterObj);
305 | }
306 |
307 | if (channel.filter) {
308 | (function AddFilters(filterObject) {
309 | var filterKey = Object.keys(filterObject);
310 | if (filterKey == 'or')
311 | searchFilters.or();
312 | else if (filterKey == 'and')
313 | searchFilters.and();
314 |
315 | filterObject[filterKey].forEach(function(filters, key) {
316 | if (key == 'and' || key == 'or')
317 | AddFilters(filterObject[filterKey]);
318 | else {
319 | for(var key2 in filters) {
320 | if (key2 == 'and' || key2 == 'or') {
321 | AddFilters(filters);
322 | } else {
323 | searchFilters.addFilter(key2, filters[key2]);
324 | }
325 | }
326 | }
327 | });
328 | searchFilters.end();
329 | })(channel.filter);
330 | }
331 |
332 | return searchFilters;
333 | };
334 |
335 | Model.search = function(channel, sort, offset, limit, callback) {
336 | var searchFilters = Model.getFilterFromChannel(channel);
337 |
338 | Application.datasource.dataStorage.searchObjects({
339 | filters: searchFilters,
340 | modelName: channel.props.model,
341 | sort: sort,
342 | offset: offset,
343 | limit: limit || Application.datasource.dataStorage.config.subscribe_limit
344 | }, callback);
345 | };
346 |
347 | Model.modelCountByChannel = function(channel, aggregation, callback) {
348 | var cachingKey = channel.get();
349 |
350 | if (aggregation)
351 | cachingKey += (new Buffer(JSON.stringify(aggregation))).toString('base64');
352 |
353 | cachingKey += ':count_cache';
354 |
355 | var filters = Model.getFilterFromChannel(channel);
356 |
357 | var waterfall = function(waterfallCallback) {
358 | async.waterfall([
359 | function(callback1) {
360 | Application.redisCacheClient.get(cachingKey, callback1);
361 | },
362 | function(result, callback1) {
363 | if (result) {
364 | callback1(null, JSON.parse(result));
365 | } else {
366 | //incearca sa creeze 2nd key
367 | //if (OK) get from ES & (unset)set redis key
368 | //else (null, adica nu exista 2nd redis key) retry
369 | Application.redisCacheClient.set([cachingKey+':LOCK', '1', 'NX'], function(err, result) {
370 | if (err) return callback1(err);
371 |
372 | if(result !== null) {
373 | var countFilters = Model.getFilterFromChannel(channel);
374 | Application.datasource.dataStorage.countObjects({modelName: channel.props.model, filters: filters, aggregation: aggregation}, function(err1, count) {
375 | if (err1) return callback1(err1);
376 |
377 | var tranzaction = Application.redisCacheClient.multi();
378 | tranzaction.set([cachingKey, JSON.stringify(count), 'EX', 300]);
379 | tranzaction.del(cachingKey+':LOCK');
380 |
381 | tranzaction.exec(function(err2) {
382 | if (err2) {
383 | Application.redisCacheClient.del([cachingKey+':LOCK'], function() {
384 | callback1(null, count);
385 | });
386 | } else
387 | callback1(err2, count);
388 | });
389 | });
390 | } else {
391 | setTimeout(waterfall, 100);
392 | }
393 | });
394 | }
395 | }
396 | ], waterfallCallback);
397 | };
398 |
399 | waterfall(callback);
400 | };
401 |
402 | module.exports = Model;
403 |
--------------------------------------------------------------------------------
/lib/database/elasticsearch_adapter.js:
--------------------------------------------------------------------------------
1 | var Main_Database_Adapter = require('./main_database_adapter');
2 | var Application = require('../Application');
3 | var elasticsearch = require('elasticsearch');
4 | var guid = require('uuid');
5 | var async = require('async');
6 | var utils = require('../../utils/utils');
7 | var Delta = require('../Delta');
8 | var TelepatError = require('../TelepatError');
9 | var AgentKeepAlive = require('agentkeepalive');
10 | var cloneObject = require('clone');
11 | var FilterBuilder = require('../../utils/filterbuilder').FilterBuilder;
12 | var BuilderNode = require('../../utils/filterbuilder').BuilderNode;
13 |
14 | require('colors');
15 |
16 | var ElasticSearchDB = function(config) {
17 | var self = this;
18 | this.config = config;
19 |
20 | this.config.subscribe_limit = this.config.subscribe_limit || 64;
21 | this.config.get_limit = this.config.get_limit || 384;
22 |
23 | var esConfig = {
24 | apiVersion: '1.7',
25 | keepAlive: true,
26 | maxSockets: 300,
27 | createNodeAgent: function(connection, config) {
28 | return new AgentKeepAlive(connection.makeAgentConfig(config));
29 | }
30 | };
31 |
32 | if (this.config.hosts) {
33 | esConfig.hosts = this.config.hosts;
34 | } else if (this.config.host) {
35 | esConfig.host = this.config.host;
36 | esConfig.port = this.config.port;
37 | esConfig.sniffOnStart = true;
38 | esConfig.sniffInterval = 30000;
39 | esConfig.sniffOnConnectionFault = true;
40 | }
41 |
42 | Main_Database_Adapter.call(this, new elasticsearch.Client(esConfig));
43 |
44 | var retryConnection = (function() {
45 | //we had to copy paste the config variable because the es sdk doesn't allow to reuse the config object
46 | var esConfig = {
47 | apiVersion: '1.7',
48 | keepAlive: true,
49 | maxSockets: 300,
50 | createNodeAgent: function(connection, config) {
51 | return new AgentKeepAlive(connection.makeAgentConfig(config));
52 | }
53 | };
54 |
55 | if (this.config.hosts)
56 | esConfig.hosts = this.config.hosts;
57 | else if (this.config.host) {
58 | esConfig.host = this.config.host;
59 | esConfig.port = this.config.port;
60 | esConfig.sniffOnStart = true;
61 | esConfig.sniffInterval = 30000;
62 | esConfig.sniffOnConnectionFault = true;
63 | }
64 |
65 | this.connection = new elasticsearch.Client(esConfig);
66 | }).bind(this);
67 |
68 | this.connection.ping({
69 | requestTimeout: Infinity
70 | }, function(err) {
71 | if (err) {
72 | var d = new Date();
73 | Application.logger.error('Failed connecting to Elasticsearch "'+self.config.host+'": '
74 | +err.message+'. Retrying...');
75 | setTimeout(function () {
76 | retryConnection();
77 | }, 1000);
78 | } else {
79 | Application.logger.info('Connected to ElasticSearch MainDatabase');
80 | self.onReadyCallback(self);
81 | }
82 | });
83 | };
84 |
85 | function escapeRegExp(str) {
86 | return str.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, "\\$&");
87 | }
88 |
89 | ElasticSearchDB.prototype = Object.create(Main_Database_Adapter.prototype);
90 |
91 | /**
92 | *
93 | * @param {FilterBuilder} builder
94 | * @return {Object} The result of builder.build() but with a few translations for ES
95 | */
96 | ElasticSearchDB.prototype.getQueryObject = function(builder) {
97 | var translationMappings = {
98 | is: 'term',
99 | not: 'not',
100 | exists: 'exists',
101 | range: 'range',
102 | in_array: 'terms',
103 | like: 'regexp'
104 | }
105 |
106 | function Translate(node) {
107 | node.children.forEach(function(child) {
108 | if (child instanceof BuilderNode) {
109 | Translate(child);
110 | } else {
111 | var replaced = Object.keys(child)[0];
112 | if (translationMappings[replaced]) {
113 | //'not' contains a filter name
114 | if (replaced == 'not') {
115 | var secondReplaced = Object.keys(child[replaced])[0];
116 |
117 | if (translationMappings[secondReplaced] !== secondReplaced) {
118 | child[replaced][translationMappings[secondReplaced]] = cloneObject(child[replaced][secondReplaced]);
119 | delete child[replaced][secondReplaced];
120 | }
121 | } else if (replaced == 'like') {
122 | child[translationMappings[replaced]] = cloneObject(child[replaced]);
123 |
124 | var fieldObj = {};
125 | Object.keys(child[translationMappings[replaced]]).forEach(function (field) {
126 | fieldObj[field] = '.*'+escapeRegExp(child[translationMappings[replaced]][field])+'.*';
127 | });
128 | child[translationMappings[replaced]] = fieldObj;
129 | delete child[replaced];
130 | } else if (translationMappings[replaced] !== replaced) {
131 | child[translationMappings[replaced]] = cloneObject(child[replaced]);
132 | delete child[replaced];
133 | }
134 | }
135 | }
136 | });
137 | };
138 |
139 | Translate(builder.root);
140 |
141 | return builder.build();
142 | };
143 |
144 | ElasticSearchDB.prototype.getObjects = function(ids, callback) {
145 | ids = ids.map(function(id) {
146 | return {_id: id};
147 | }, this);
148 |
149 | this.connection.mget({
150 | index: this.config.index,
151 | body: {
152 | docs: ids
153 | }
154 | }, function(err, results) {
155 | if (err) return callback([err]);
156 |
157 | var notFoundErrors = [];
158 | var objects = [];
159 | var versions = {};
160 |
161 | async.each(results.docs, function(result, c) {
162 | if (result.found) {
163 | objects.push(result._source);
164 | versions[result._id] = result._version;
165 | }
166 | else
167 | notFoundErrors.push(new TelepatError(TelepatError.errors.ObjectNotFound, [result._type, result._id]));
168 | c();
169 | }, function() {
170 | callback(notFoundErrors, objects, versions);
171 | });
172 | });
173 | };
174 |
175 | ElasticSearchDB.prototype.searchObjects = function(options, callback) {
176 | var reqBody = {
177 | query: {
178 | filtered: {
179 | filter: {}
180 | }
181 | }
182 | };
183 | var self = this;
184 |
185 | if (options.filters && !options.filters.isEmpty())
186 | reqBody.query.filtered.filter = this.getQueryObject(options.filters);
187 |
188 | if (options.fields) {
189 | if (!(options.scanFunction instanceof Function))
190 | return callback(new TelepatError(TelepatError.errors.ServerFailure, ['searchObjects was provided with fields but no scanFunction']));
191 |
192 | var hitsCollected = 0;
193 |
194 | this.connection.search({
195 | index: this.config.index,
196 | type: options.modelName ? options.modelName : '',
197 | body: reqBody,
198 | scroll: '10s',
199 | searchType: 'scan',
200 | fields: options.fields,
201 | size: 1024
202 | }, function getMore(err, response) {
203 | if (err) return callback(err);
204 |
205 | if (response.hits.total !== hitsCollected) {
206 | var objects = [];
207 |
208 | hitsCollected += response.hits.hits.length;
209 |
210 | async.each(response.hits.hits, function(hit, c) {
211 | var obj = {};
212 | async.forEachOf(hit.fields, function(value, f, c1) {
213 | obj[f] = value[0];
214 | c1();
215 | }, function() {
216 | objects.push(obj);
217 | c();
218 | });
219 | }, function() {
220 | if (response.hits.hits.length)
221 | options.scanFunction(objects);
222 |
223 | self.connection.scroll({
224 | scrollId: response._scroll_id,
225 | scroll: '10s'
226 | }, getMore);
227 | });
228 | } else {
229 | callback();
230 | }
231 | });
232 | } else {
233 | if (options.sort) {
234 | reqBody.sort = [];
235 |
236 | var sortFieldName = Object.keys(options.sort)[0];
237 | //old sort method
238 | if (typeof options.sort[sortFieldName] == 'string') {
239 | var sortObject = {};
240 |
241 | sortObject[sortFieldName] = {order: options.sort[sortFieldName], unmapped_type : "long"};
242 | reqBody.sort = [sortObject];
243 | } else {
244 | Object.keys(options.sort).forEach(function(field) {
245 | var sortObjectField = {};
246 |
247 | if (!options.sort[field].type) {
248 | sortObjectField[field] = {order: options.sort[field].order, unmapped_type : "long"};
249 | } else if (options.sort[field].type == 'geo') {
250 | sortObjectField._geo_distance = {};
251 | sortObjectField._geo_distance[field] = {lat: options.sort[field].poi.lat || 0.0, lon: options.sort[field].poi.long || 0.0};
252 | sortObjectField._geo_distance.order = options.sort[field].order;
253 | }
254 |
255 | reqBody.sort.push(sortObjectField);
256 | });
257 | }
258 | }
259 |
260 | this.connection.search({
261 | index: this.config.index,
262 | type: options.modelName,
263 | body: reqBody,
264 | from: options.offset,
265 | size: options.limit
266 | }, function(err, results) {
267 | if (err) return callback(err);
268 |
269 | var objects = [];
270 |
271 | results.hits.hits.forEach(function(object) {
272 | objects.push(object._source);
273 | })
274 |
275 | callback(null, objects);
276 | });
277 | }
278 | };
279 |
280 | ElasticSearchDB.prototype.countObjects = function(options, callback) {
281 | var reqBody = {
282 | query: {
283 | filtered: {
284 | filter: {}
285 | }
286 | }
287 | };
288 |
289 | if (options.filters && !options.filters.isEmpty())
290 | reqBody.query.filtered.filter = this.getQueryObject(options.filters);
291 |
292 | if (options.aggregation) {
293 | reqBody.aggs = {aggregation: options.aggregation};
294 |
295 | this.connection.search({
296 | index: this.config.index,
297 | type: options.modelName,
298 | body: reqBody,
299 | search_type: 'count',
300 | queryCache: true
301 | }, function(err, results) {
302 | if (err) return callback(err);
303 |
304 | var countResult = {count: result.hits.total};
305 |
306 | countResult.aggregation = result.aggregations.aggregation.value;
307 |
308 | callback(null, countResult);
309 | });
310 | } else {
311 | this.connection.count({
312 | index: this.config.index,
313 | type: options.modelName,
314 | body: reqBody
315 | }, function(err, result) {
316 | if (err) return callback(err);
317 |
318 | var countResult = {count: result.count};
319 |
320 | callback(null, countResult);
321 | });
322 | }
323 | };
324 |
325 | ElasticSearchDB.prototype.createObjects = function(objects, callback) {
326 | var bulk = [];
327 | var builtinModels = ['application', 'admin', 'user', 'user_metadata', 'context'];
328 | var builtinDetected = false;
329 |
330 | objects.forEach(function(obj) {
331 | var modelName = obj.type;
332 | if (builtinModels.indexOf(modelName) !== -1)
333 | builtinDetected = true;
334 |
335 | bulk.push({index: {_type: modelName, _id: obj.id}});
336 | bulk.push(obj);
337 | }, this);
338 |
339 | this.connection.bulk({
340 | index: this.config.index,
341 | body: bulk,
342 | refresh: builtinDetected
343 | }, function(err, res) {
344 | if (res.errors) {
345 | res.items.forEach(function(error) {
346 | Application.logger.error('Error creating '+error.index._type+': '+error.index.error);
347 | });
348 | }
349 |
350 | callback(err, res);
351 | });
352 | };
353 |
354 | ElasticSearchDB.prototype.updateObjects = function(patches, callback) {
355 | var ids = {};
356 | var dbObjects = {};
357 | var totalErrors = [];
358 | var builtinModels = ['application', 'admin', 'user', 'user_metadata', 'context'];
359 | var builtinDetected = false;
360 | var self = this;
361 |
362 | patches.forEach(function(patch) {
363 | var id = patch.path.split('/')[1];
364 | if (!ids[id])
365 | ids[id] = [patch];
366 | else
367 | ids[id].push(patch);
368 | });
369 |
370 | function getAndUpdate(objectIds, callback2) {
371 | var dbObjectVersions = {};
372 | var conflictedObjectIds = {};
373 | var bulk = [];
374 |
375 | async.series([
376 | function getObjects(callback1) {
377 | self.getObjects(Object.keys(objectIds), function(err, results, versions) {
378 | if (err && err.length == 1) return callback1(err[0]);
379 |
380 | totalErrors = err;
381 | results.forEach(function(object) {
382 | dbObjects[object.id] = object;
383 | dbObjectVersions[object.id] = versions[object.id];
384 | });
385 | callback1();
386 | });
387 | },
388 | function updateBulk(callback1) {
389 | async.forEachOf(objectIds, function(patches, id, c) {
390 | var objectModel = null;
391 |
392 | objectModel = patches[0].path.split('/')[0];
393 |
394 | if (builtinModels.indexOf(objectModel) !== -1)
395 | builtinDetected = true;
396 |
397 | dbObjects[id] = Delta.processObject(patches, dbObjects[id]);
398 |
399 | var script = 'def jsonSlurper = new groovy.json.JsonSlurper();'+
400 | 'def parsed = jsonSlurper.parseText(\''+JSON.stringify(dbObjects[id]).replace(/'/g, "\\'").replace(/"/g, "\\\"")+'\');'+
401 | 'ctx._source = parsed;';
402 |
403 | bulk.push({update: {_type: objectModel, _id: id, _version: dbObjectVersions[id]}});
404 | bulk.push({script: script});
405 | c();
406 | }, function() {
407 | self.connection.bulk({
408 | index: self.config.index,
409 | body: bulk,
410 | refresh: builtinDetected
411 | }, function(err, res) {
412 | if (err)
413 | return callback1(err);
414 |
415 | var temporaryErrors = [];
416 | if (res.errors)
417 | res.items.forEach(function(error) {
418 | if (error.update.status === 409)
419 | conflictedObjectIds[error.update._id] = ids[error.update._id];
420 | else
421 | totalErrors.push(new Error('Failed to update '+error.update._type+' with ID '+error.update._id+': '+error.update.error));
422 | });
423 |
424 | callback1();
425 | });
426 | });
427 | }
428 | ], function(err) {
429 | if (err)
430 | return callback2(err);
431 |
432 | if (Object.keys(conflictedObjectIds).length)
433 | return getAndUpdate(conflictedObjectIds, callback2);
434 |
435 | callback2();
436 | });
437 | }
438 |
439 | getAndUpdate(ids, function(err) {
440 | if (err)
441 | callback([err]);
442 | else
443 | callback(totalErrors, dbObjects);
444 | });
445 | };
446 |
447 | ElasticSearchDB.prototype.deleteObjects = function(ids, callback) {
448 | var self = this;
449 | var bulk = [];
450 | var builtinModels = ['application', 'admin', 'user', 'user_metadata', 'context'];
451 | var builtinDetected = false;
452 |
453 | async.each(Object.keys(ids), function(id, c) {
454 | if (builtinModels.indexOf(ids[id]) !== -1)
455 | builtinDetected = true;
456 |
457 | bulk.push({delete: {_type: ids[id], _id: id}});
458 | c();
459 | }, function(err) {
460 | self.connection.bulk({
461 | index: self.config.index,
462 | body: bulk,
463 | refresh: builtinDetected
464 | }, function(err, results) {
465 | if (err) return callback([err]);
466 | var notFoundErrors = [];
467 |
468 | async.each(results.docs, function(result, c) {
469 | if (!result.found)
470 | notFoundErrors.push(new TelepatError(TelepatError.errors.ObjectNotFound, [result._type, result._id]));
471 | c();
472 | }, function() {
473 | callback(notFoundErrors.length ? notFoundErrors : null);
474 | });
475 | });
476 | });
477 | };
478 |
479 | module.exports = ElasticSearchDB;
480 |
--------------------------------------------------------------------------------