├── test ├── mocha.opts ├── sample.small.json ├── text.test.js ├── db.test.js ├── transaction.test.js └── sample.json ├── .gitignore ├── lib ├── finder.js ├── finders │ ├── indexed.js │ ├── simple.js │ └── scan.js ├── backup.js ├── s3.js ├── capacity.js ├── indexer.js ├── parser.js ├── indexes │ ├── cloud-search.js │ └── fat.js ├── dyn.tx.js └── refiner.js ├── package.json ├── LICENSE └── cli.js /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --reporter spec 2 | --timeout 30000 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | node_modules 14 | 15 | npm-debug.log 16 | -------------------------------------------------------------------------------- /test/sample.small.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Minerva Stewart", 4 | "company": "Ultricies Sem Magna Institute", 5 | "phone": "04 66 53 61 19", 6 | "about": "eu," 7 | }, 8 | { 9 | "name": "Hedley Booth", 10 | "company": "Lobortis Company", 11 | "phone": "08 32 96 63 10", 12 | "about": "non, luctus sit amet, faucibus" 13 | }, 14 | { 15 | "name": "Duncan Wall", 16 | "company": "Adipiscing Fringilla Inc.", 17 | "phone": "06 84 05 84 87", 18 | "about": "iaculis odio. Nam interdum enim non" 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /lib/finder.js: -------------------------------------------------------------------------------- 1 | var _finders= [require('./finders/simple'), 2 | require('./finders/indexed'), 3 | require('./finders/scan')]; 4 | 5 | module.exports= function (dyn) 6 | { 7 | var finder= {}; 8 | 9 | finder.find= function (query) 10 | { 11 | var canFind= false, 12 | p= dyn.promise(['results','count','end'],'notfound','consumed'); 13 | 14 | process.nextTick(function () 15 | { 16 | 17 | _finders.every(function (f) 18 | { 19 | var _finder= f(dyn); 20 | 21 | if (_finder.canFind(query)) 22 | { 23 | canFind= true; 24 | _finder.find(query) 25 | .chain(p); 26 | return false; 27 | } 28 | 29 | return true; 30 | }); 31 | 32 | if (!canFind) 33 | p.trigger.error(new Error('Cannot handle that query yet: '+JSON.stringify(query.cond))); 34 | 35 | }); 36 | 37 | return p; 38 | }; 39 | 40 | return finder; 41 | }; 42 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dyngodb2", 3 | "description": "An experiment (alpha) to get a MongoDB like interface in front of DynamoDB", 4 | "version": "0.1.30", 5 | "main": "Gruntfile.js", 6 | "bin": { 7 | "dyngodb2": "./cli.js" 8 | }, 9 | "engines": { 10 | "node": ">= 0.8.0" 11 | }, 12 | "keywords": [ 13 | "mongodb", 14 | "dynamodb" 15 | ], 16 | "dependencies": { 17 | "async": "^0.2.10", 18 | "aws-sdk": "~2.5.5", 19 | "carrier": "~0.1.13", 20 | "circularclone": "~0.1.7", 21 | "coffee-script": "^1.7.1", 22 | "colors": "~0.6.1", 23 | "csv": "~0.3.6", 24 | "deep-diff": "~0.1.7", 25 | "gson": "~0.1.3", 26 | "knox": "^0.9.0", 27 | "knox-copy": "^0.2.2", 28 | "knox-mpu": "^0.1.6", 29 | "node-uuid": "~1.4.0", 30 | "node-zip": "~1.0.1", 31 | "optimist": "~0.6.0", 32 | "querystring": "~0.2.0", 33 | "ret": "~0.1.6", 34 | "underscore": "~1.5.1" 35 | }, 36 | "repository": { 37 | "type": "git", 38 | "url": "http://github.com/aaaristo/dyngodb.git" 39 | }, 40 | "preferGlobal": true, 41 | "devDependencies": { 42 | "chai": "~1.8.1", 43 | "chai-spies": "^0.5.1" 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /lib/finders/indexed.js: -------------------------------------------------------------------------------- 1 | var async= require('async'), 2 | _= require('underscore'), 3 | colors= require('colors'); 4 | 5 | module.exports= function (dyn) 6 | { 7 | var finder= {}; 8 | 9 | finder.canFind= function (query) 10 | { 11 | query.table.indexes.some(function (ind) 12 | { 13 | if (ind.usable(query)) 14 | { 15 | query.index= ind; 16 | return true; 17 | } 18 | }); 19 | 20 | return !!query.index; 21 | }; 22 | 23 | finder.find= function (query) 24 | { 25 | if (query.opts.hints) 26 | console.log(('Query INDEX ('+query.index.name+')').yellow); 27 | 28 | var p= dyn.promise(['results','count','end'],null,'consumed'), 29 | cursor= query.index.find(query); 30 | 31 | if (query.count) 32 | cursor.chain(p); 33 | else 34 | cursor.results(function (items) 35 | { 36 | items.forEach(function (item, idx) { items[idx]= { _ref: item }; }); 37 | p.trigger.results(items); 38 | }) 39 | .error(p.trigger.error) 40 | .consumed(p.trigger.consumed) 41 | .end(p.trigger.end); 42 | 43 | return p; 44 | 45 | }; 46 | 47 | return finder; 48 | }; 49 | -------------------------------------------------------------------------------- /lib/backup.js: -------------------------------------------------------------------------------- 1 | var _= require('underscore'), 2 | carrier= require('carrier'), 3 | async= require('async'); 4 | 5 | module.exports= function (dyn,dbopts) 6 | { 7 | var backup= {}; 8 | 9 | backup.backup= function (table) 10 | { 11 | return function (opts) 12 | { 13 | var p= dyn.promise(), 14 | s3= require('./s3')(_.extend(opts,dbopts)); 15 | 16 | s3.write(table+'/'+(new Date().toISOString())+'.dbk', 17 | function (wstream) 18 | { 19 | var rstream= dyn.stream(table).scan({ limit: opts.limit }); 20 | 21 | rstream 22 | .on('data',function (items) 23 | { 24 | rstream.pause(); 25 | 26 | async.forEach(items, 27 | function (item,done) 28 | { 29 | if (!wstream.write(new Buffer(JSON.stringify(item)+'\n','utf8'))) 30 | wstream.on('drain',_.once(done)); 31 | else 32 | done(); 33 | }, 34 | function (err) 35 | { 36 | rstream.resume(); 37 | }); 38 | }) 39 | .on('end',function () 40 | { 41 | wstream.end(); 42 | }); 43 | 44 | wstream.on('close',p.trigger.success); 45 | }); 46 | 47 | return p; 48 | }; 49 | }; 50 | 51 | backup.restore= function (table) 52 | { 53 | return function (opts) 54 | { 55 | var p= dyn.promise(), 56 | s3= require('./s3')(_.extend(opts,dbopts)), 57 | wstream= dyn.stream(table).mput('put'); 58 | 59 | s3.read(opts.file,function (rstream) 60 | { 61 | carrier.carry(rstream, function (line) 62 | { 63 | wstream.write([JSON.parse(line)]); 64 | },'utf8'); 65 | 66 | rstream.on('end',p.trigger.success); 67 | }); 68 | 69 | return p; 70 | }; 71 | }; 72 | 73 | return backup; 74 | }; 75 | -------------------------------------------------------------------------------- /lib/finders/simple.js: -------------------------------------------------------------------------------- 1 | var _= require('underscore'); 2 | 3 | module.exports= function (dyn) 4 | { 5 | var finder= {}; 6 | 7 | finder.canFind= function (query) 8 | { 9 | return !!query.cond._id&&!(query.cond._id instanceof RegExp); 10 | }; 11 | 12 | finder.find= function (query) 13 | { 14 | if (query.opts.hints) 15 | console.log(('PK on '+query.table.name+' for '+JSON.stringify(query.cond,null,2)).green); 16 | 17 | if (query.cond._id) 18 | delete query.$filter._id; 19 | 20 | if (query.cond._pos!==undefined) 21 | delete query.$filter._pos; 22 | 23 | var p= dyn.promise(['results','count','end'],'notfound','consumed'), 24 | _triggerError= query.count ? function (err) 25 | { 26 | if (err.code=='notfound') 27 | p.trigger.count(0); 28 | else 29 | p.trigger.error(err); 30 | } : p.trigger.error; 31 | 32 | if (query.cond._pos!==undefined) 33 | dyn.table(query.table.name) 34 | .hash('_id',query.cond._id) 35 | .range('_pos',query.cond._pos) 36 | .get(function (item) 37 | { 38 | if (query.count) 39 | p.trigger.count(1); 40 | else 41 | { 42 | p.trigger.results([item]); 43 | p.trigger.end(); 44 | } 45 | },{ attrs: query.finderProjection(), 46 | consistent: query.$consistent }) 47 | .consumed(p.trigger.consumed) 48 | .error(_triggerError); 49 | else 50 | if (query.limit==1) 51 | dyn.table(query.table.name) 52 | .hash('_id',query.cond._id) 53 | .range('_pos',0) 54 | .query(query.count ? p.trigger.count : p.trigger.results, 55 | { attrs: query.finderProjection(), count: query.count, consistent: query.$consistent }) 56 | .error(_triggerError) 57 | .consumed(p.trigger.consumed) 58 | .end(p.trigger.end); 59 | else 60 | dyn.table(query.table.name) 61 | .hash('_id',query.cond._id) 62 | .query(query.count ? p.trigger.count : p.trigger.results, 63 | { attrs: query.finderProjection(), limit: query.limit, count: query.count, consistent: query.$consistent }) 64 | .error(_triggerError) 65 | .consumed(p.trigger.consumed) 66 | .end(p.trigger.end); 67 | 68 | return p; 69 | }; 70 | 71 | return finder; 72 | }; 73 | -------------------------------------------------------------------------------- /lib/s3.js: -------------------------------------------------------------------------------- 1 | var copy= require('knox-copy'), 2 | MultiPartUpload = require('knox-mpu'), 3 | stream = require('stream'), 4 | async = require('async'), 5 | _ = require('underscore'), 6 | fs= require('fs'); 7 | 8 | module.exports= function (opts) 9 | { 10 | opts= _.defaults(opts || {}, 11 | { 12 | accessKeyId: process.env.AWS_ACCESS_KEY_ID, 13 | secretAccessKey: process.env.AWS_SECRET_KEY || process.env.AWS_SECRET_ACCESS_KEY, 14 | region: process.env.AWS_REGION 15 | }); 16 | 17 | var s3 = copy.createClient 18 | ({ 19 | key: opts.accessKeyId 20 | , secret: opts.secretAccessKey 21 | , bucket: opts.bucket 22 | , region: opts.region 23 | }), 24 | file= {}; 25 | 26 | file.write= function (path,cb) 27 | { 28 | var wstream = new stream.Stream() 29 | wstream.writable = true 30 | var rstream = new stream.Stream() 31 | rstream.readable = true; 32 | 33 | wstream.write = function (data) 34 | { 35 | rstream.emit('data',data); 36 | return true; // true means 'yes i am ready for more data now' 37 | // OR return false and emit('drain') when ready later 38 | }; 39 | 40 | wstream.end = function (data) 41 | { 42 | if (data) rstream.emit('data',data); 43 | rstream.emit('end'); 44 | }; 45 | 46 | new MultiPartUpload 47 | ({ 48 | client: s3, 49 | objectName: path, 50 | stream: rstream 51 | }, 52 | function(err, res) 53 | { 54 | if (err) 55 | console.log('file.s3.write',err); 56 | 57 | wstream.emit('close'); 58 | }); 59 | 60 | cb(wstream); 61 | }; 62 | 63 | file.read= function (path,cb) 64 | { 65 | s3.getFile(path, function(err,res) 66 | { 67 | cb(res); 68 | }); 69 | }; 70 | 71 | file.delete= function (path,cb) 72 | { 73 | s3.deleteFile(path, function(err, res) 74 | { 75 | if (err) throw err; 76 | cb(); 77 | }); 78 | }; 79 | 80 | file.size= function (path, cb) 81 | { 82 | s3.headFile(path, function(err, res) 83 | { 84 | if (err) throw err; 85 | cb(parseInt(res.headers['content-length'])); 86 | }); 87 | }; 88 | 89 | file.copyDir= function (src,dest,cb) 90 | { 91 | var queue= async.queue(function (key,done) 92 | { 93 | s3.copyFile(key,key.replace(src,dest),done); 94 | },10); 95 | 96 | queue.drain= cb; 97 | 98 | var found= false; 99 | 100 | s3.streamKeys({ prefix: src }) 101 | .on('data', function (key) 102 | { 103 | found= true; 104 | queue.push(key,function (err) { console.log(err); }); 105 | }) 106 | .on('end', function () 107 | { 108 | if (!found) cb(); 109 | }); 110 | }; 111 | 112 | return file; 113 | }; 114 | -------------------------------------------------------------------------------- /lib/finders/scan.js: -------------------------------------------------------------------------------- 1 | var colors= require('colors'), 2 | _= require('underscore'), 3 | async= require('async'); 4 | 5 | const M= 1048576; 6 | 7 | module.exports= function (dyn) 8 | { 9 | var finder= {}; 10 | 11 | finder.canFind= function (query) 12 | { 13 | return query.$supported; 14 | }; 15 | 16 | finder.find= function (query) 17 | { 18 | if (query.opts.hints) 19 | console.log(('SCAN on '+query.table.name+' for '+JSON.stringify(query.cond,null,2)).red); 20 | 21 | var p= dyn.promise(['results','count','end'],null,'consumed'), 22 | avgItemSize= Math.ceil(query.table._dynamo.TableSizeBytes/query.table._dynamo.ItemCount), 23 | perWorker= (M/avgItemSize)*80/100, 24 | workers= Math.ceil(query.table._dynamo.ItemCount/perWorker); 25 | 26 | if (!workers) workers= 1; 27 | else 28 | if (workers>query.table._dynamo.ProvisionedThroughput.ReadCapacityUnits) 29 | workers= query.table._dynamo.ProvisionedThroughput.ReadCapacityUnits; 30 | 31 | // FIXME: implement opts.maxworkers ? better divide & conquer algorithm 32 | 33 | var filter= {}; 34 | 35 | Object.keys(query.$filter).forEach(function (fieldName) 36 | { 37 | var field= query.$filter[fieldName]; 38 | 39 | if (field.op!='REGEXP') 40 | { 41 | filter[fieldName]= field; 42 | delete query.$filter[fieldName]; 43 | query.$filtered.push(fieldName); 44 | } 45 | }); 46 | 47 | query.counted= query.canCount()&&query.count; 48 | 49 | if (query.counted) 50 | { 51 | 52 | var count= 0, 53 | progress= [], 54 | _progress= function (segment, pcount) 55 | { 56 | progress[segment]= pcount; 57 | 58 | process.stdout.write(('\r'+_.reduce(progress,function (memo,num) { return memo+num; },0)).yellow); 59 | }; 60 | 61 | async.forEach(_.range(workers), 62 | function (segment,done) 63 | { 64 | var sp= dyn.table(query.table.name) 65 | .scan(function (wcount) 66 | { 67 | count+=wcount; 68 | done(); 69 | }, 70 | { 71 | filter: filter, 72 | attrs: query.finderProjection(), 73 | limit: query.window, 74 | count: query.count, 75 | segment: { no: segment, of: workers } 76 | }) 77 | .consumed(p.trigger.consumed) 78 | .error(done); 79 | 80 | if (dyn.iscli()) 81 | sp.progress(function (pcount) { _progress(segment,pcount); }) 82 | }, 83 | function (err) 84 | { 85 | if (err) 86 | p.trigger.error(err); 87 | else 88 | p.trigger.count(count); 89 | }); 90 | } 91 | else 92 | { 93 | async.forEach(_.range(workers), 94 | function (segment,done) 95 | { 96 | dyn.table(query.table.name) 97 | .scan(p.trigger.results, 98 | { 99 | filter: filter, 100 | attrs: query.finderProjection(), 101 | segment: { no: segment, of: workers }, 102 | limit: query.window 103 | }) 104 | .consumed(p.trigger.consumed) 105 | .end(done) 106 | .error(done); 107 | }, 108 | function (err) 109 | { 110 | if (err) 111 | p.trigger.error(err); 112 | else 113 | p.trigger.end(); 114 | }); 115 | } 116 | 117 | return p; 118 | }; 119 | 120 | return finder; 121 | }; 122 | -------------------------------------------------------------------------------- /lib/capacity.js: -------------------------------------------------------------------------------- 1 | var _= require('underscore'), 2 | async= require('async'); 3 | 4 | var __steps= function (rcurrent, rtarget, wcurrent, wtarget) 5 | { 6 | var _steps= function (current, target) 7 | { 8 | var diff= target-current, 9 | steps= diff>0 ? (diff > current ? [] : [target]) : [target]; 10 | 11 | if (!steps.length) 12 | { 13 | while ((current=current*2)0) 19 | steps.push(target); 20 | } 21 | 22 | return steps; 23 | }, 24 | _fill= function (arr,len) 25 | { 26 | var diff= len-arr.length, 27 | last= arr[arr.length-1]; 28 | 29 | _(diff).times(function () { arr.push(last); }); 30 | }; 31 | 32 | var rsteps= _steps(rcurrent,rtarget), 33 | wsteps= _steps(wcurrent,wtarget); 34 | 35 | if (rsteps.length>wsteps.length) 36 | _fill(wsteps,rsteps.length); 37 | else 38 | if (wsteps.length>rsteps.length) 39 | _fill(rsteps,wsteps.length); 40 | 41 | return _.zip(rsteps,wsteps); 42 | } 43 | 44 | module.exports= function (dyn,table,read,write) 45 | { 46 | var p= dyn.promise(), 47 | _check= function (cb) 48 | { 49 | dyn.describeTable(table, 50 | function (err,data) 51 | { 52 | if (err) 53 | p.trigger.error(err); 54 | else 55 | if (data.Table.TableStatus=='UPDATING') 56 | setTimeout(_check,5000,cb); 57 | else 58 | { 59 | table._dynamo= data.Table; 60 | cb(); 61 | } 62 | }); 63 | }; 64 | 65 | if (!read||!write) 66 | process.nextTick(function () 67 | { p.trigger.error(new Error('You should specify read and write ProvisionedThroughput')) }); 68 | else 69 | dyn.describeTable(table, 70 | function (err,data) 71 | { 72 | if (err) 73 | p.trigger.error(err); 74 | else 75 | { 76 | var current= data.Table.ProvisionedThroughput; 77 | 78 | if (current.ReadCapacityUnits==read 79 | &¤t.WriteCapacityUnits==write) 80 | p.trigger.success(); 81 | else 82 | { 83 | console.log('This may take a while...'.yellow); 84 | 85 | var steps= __steps(current.ReadCapacityUnits, 86 | read, 87 | current.WriteCapacityUnits, 88 | write); 89 | 90 | async.forEachSeries(steps,function (step,done) 91 | { 92 | var sread= step[0], swrite= step[1]; 93 | 94 | dyn.updateTable(table,sread,swrite, 95 | function (err,data) 96 | { 97 | if (err) 98 | done(err); 99 | else 100 | setTimeout(_check,5000,function () 101 | { 102 | if (sread==read&&swrite==write) 103 | console.log(('current capacity: '+sread+' read '+swrite+' write') 104 | .green); 105 | else 106 | console.log(('current capacity: '+sread+' read '+swrite+' write') 107 | .yellow); 108 | 109 | done(); 110 | }); 111 | }); 112 | }, 113 | p.should('success')); 114 | } 115 | } 116 | }); 117 | 118 | return p; 119 | }; 120 | -------------------------------------------------------------------------------- /test/text.test.js: -------------------------------------------------------------------------------- 1 | var should= require('chai').should(), 2 | assert= require('chai').assert, 3 | AWS = require('aws-sdk'), 4 | _= require('underscore'), 5 | fs= require('fs'), 6 | dyngo= require('../index.js'); 7 | 8 | const _noerr= function (done) 9 | { 10 | return function (err) 11 | { 12 | if (err) console.log(err,err.stack); 13 | should.not.exist(err); 14 | done(); 15 | }; 16 | }, 17 | accept= function (code,done) 18 | { 19 | return function (err) 20 | { 21 | if (err.code==code) 22 | done(); 23 | else 24 | done(err); 25 | }; 26 | }, 27 | profiles= function () 28 | { 29 | return JSON.parse(fs.readFileSync('test/sample.small.json','utf8')); 30 | }; 31 | 32 | 33 | describe('text',function () 34 | { 35 | var db; 36 | 37 | before(function (done) 38 | { 39 | dyngo({ dynamo: { endpoint: new AWS.Endpoint('http://localhost:8000') }, hints: false }, 40 | function (err,_db) 41 | { 42 | db= _db; 43 | 44 | db.test.remove().success(function () 45 | { 46 | var ensure= function () 47 | { 48 | return db.test.ensureIndex 49 | ({ 50 | name: 'S', 51 | $text: function (item) 52 | { 53 | return _.pick(item,['name','company','about']); 54 | } 55 | }) 56 | }; 57 | 58 | ensure().success(function () 59 | { 60 | db.test.indexes[0].drop().success(function () 61 | { 62 | ensure().success(function () 63 | { 64 | var p= profiles(); 65 | 66 | db.test.save(p) 67 | .success(done) 68 | .error(done); 69 | }).error(done); 70 | }).error(done); 71 | }).error(_noerr(done)); 72 | }) 73 | .error(_noerr(done)); 74 | 75 | }); 76 | }); 77 | 78 | it('Can do bloodhound like full text searches', function (done) 79 | { 80 | db.test.findOne({ $text: 'interdum adi' }) // inspired by bloodhound search with whitespace tokenizer 81 | .result(function (obj) 82 | { 83 | obj.name.should.equal('Duncan Wall'); 84 | done(); 85 | }) 86 | .error(_noerr(done)); 87 | }); 88 | 89 | it('Can chain full text searches with normal fields', function (done) 90 | { 91 | db.test.findOne({ name: 'Duncan Wall', $text: 'interdum adi' }) 92 | .result(function (obj) 93 | { 94 | obj.name.should.equal('Duncan Wall'); 95 | 96 | db.test.findOne({ name: 'Hedley Booth', $text: 'interdum adi' }) 97 | .result(function (obj) 98 | { 99 | should.not.exist(obj); 100 | done(); 101 | }) 102 | .error(accept('notfound',done)); 103 | }) 104 | .error(_noerr(done)); 105 | }); 106 | 107 | it('Should not find any result', function (done) 108 | { 109 | db.test.findOne({ $text: 'Xinterdum adi' }) // inspired by bloodhound search with whitespace tokenizer 110 | .result(function (obj) 111 | { 112 | should.not.exist(obj); 113 | done(); 114 | }) 115 | .error(accept('notfound',done)); 116 | }); 117 | }); 118 | -------------------------------------------------------------------------------- /lib/indexer.js: -------------------------------------------------------------------------------- 1 | var _= require('underscore'), 2 | async= require('async'), 3 | _modify= require('./capacity'); 4 | 5 | var _indexes= [require('./indexes/fat'), 6 | require('./indexes/cloud-search')]; 7 | 8 | const _oa = function(o, s) 9 | { 10 | s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties 11 | s = s.replace(/^\./, ''); // strip a leading dot 12 | var a = s.split('.'); 13 | while (a.length) { 14 | var n = a.shift(); 15 | if (n in o) { 16 | o = o[n]; 17 | } else { 18 | return; 19 | } 20 | } 21 | return o; 22 | }; 23 | 24 | module.exports= function (dyn,table,fields,dbopts) 25 | { 26 | var index= { dbopts: dbopts }; 27 | 28 | index.exists= function (done) 29 | { 30 | dyn.describeTable(index.name,function (err,data) 31 | { 32 | if (err) 33 | { 34 | if (err.code=='ResourceNotFoundException') 35 | done(null,false); 36 | else 37 | done(err); 38 | } 39 | else 40 | { 41 | index._dynamo= data.Table; 42 | done(null,true); 43 | } 44 | }); 45 | }; 46 | 47 | index.drop= function (done) 48 | { 49 | var _check= function (p) 50 | { 51 | dyn.describeTable(index.name, 52 | function (err,data) 53 | { 54 | if (err) 55 | { 56 | if (err.code=='ResourceNotFoundException') 57 | p ? p.trigger.success() : done(); 58 | else 59 | p ? p.trigger.error(err) : done(err); 60 | } 61 | else 62 | setTimeout(_check,5000,p); 63 | }); 64 | }; 65 | 66 | if (index.dbopts.hints) console.log('This may take a while...'.yellow); 67 | 68 | if (done) 69 | dyn.deleteTable(index.name,function (err) 70 | { 71 | if (err) 72 | done(err); 73 | else 74 | setTimeout(_check,5000); 75 | }); 76 | else 77 | { 78 | var p= dyn.promise(); 79 | 80 | dyn.deleteTable(index.name,function (err) 81 | { 82 | if (err) 83 | p.trigger.error(err); 84 | else 85 | setTimeout(_check,5000,p); 86 | }); 87 | 88 | return p; 89 | } 90 | }; 91 | 92 | index.rebuild= function (window) 93 | { 94 | var p= dyn.promise(), 95 | sindex= dyn.stream(index.name), 96 | dcnt= 0, 97 | pcnt= 0, 98 | del= sindex.mput('del'), 99 | mput= sindex.mput('put'), 100 | limit= window || (index._dynamo ? index._dynamo.ProvisionedThroughput.WriteCapacityUnits : 25); 101 | 102 | sindex.scan({ attrs: ['_hash','_range'], limit: limit }) 103 | .on('data',function (items) { process.stdout.write(('\r'+(dcnt+=items.length)).red); }) 104 | .pipe(del) 105 | .on('finish',function () 106 | { 107 | console.log(); 108 | 109 | dyn.stream(table._dynamo.TableName) 110 | .scan({ limit: limit }) 111 | .pipe(index.streamElements()) 112 | .on('data',function (items) { process.stdout.write(('\r'+(pcnt+=items.length)).yellow); }) 113 | .on('error',p.trigger.error) 114 | .pipe(mput) 115 | .on('error',p.trigger.error) 116 | .on('finish',_.compose(p.trigger.success,console.log)); 117 | }); 118 | 119 | return p; 120 | }; 121 | 122 | index.empty= function (window) 123 | { 124 | var p= dyn.promise(), 125 | sindex= dyn.stream(index.name), 126 | dcnt= 0, 127 | del= sindex.mput('del'), 128 | limit= window || (index._dynamo ? index._dynamo.ProvisionedThroughput.WriteCapacityUnits : 25); 129 | 130 | sindex.scan({ attrs: ['_hash','_range'], limit: limit }) 131 | .on('data',function (items) { process.stdout.write(('\r'+(dcnt+=items.length)).red); }) 132 | .pipe(del) 133 | .on('finish',_.compose(p.trigger.success,console.log)); 134 | 135 | return p; 136 | }; 137 | 138 | index.tstream= function (fn) 139 | { 140 | return dyn.stream(index.name).transform(fn); 141 | }; 142 | 143 | index.ensure= function (done) 144 | { 145 | index.exists(function (err, exists) 146 | { 147 | if (err) 148 | done(err); 149 | else 150 | if (exists) 151 | done(); 152 | else 153 | index.create(done); 154 | }); 155 | }; 156 | 157 | index.modify= function (read,write) 158 | { 159 | return _modify(dyn,index.name,read,write); 160 | }; 161 | 162 | index.put= function (item,done) 163 | { 164 | if (index.indexable(item)) 165 | { 166 | var elem= index.makeElement(item); 167 | 168 | dyn.table(index.name) 169 | .hash('_hash',elem._hash) 170 | .range('_range',elem._range) 171 | .put(elem,done) 172 | .error(done); 173 | } 174 | else 175 | done(); 176 | }; 177 | 178 | index.update= function (item,op) 179 | { 180 | var iops= {}, 181 | ops= iops[index.name]= []; 182 | 183 | if (index.indexable(item)) 184 | { 185 | var elem= index.makeElement(item); 186 | 187 | if (index.indexable(item._old)) 188 | { 189 | var old= index.makeElement(item._old); 190 | 191 | if (elem._hash!=old._hash 192 | ||elem._range!=old._range) 193 | ops.push({ op: 'del', item: _.pick(old,['_hash','_range']) }); 194 | } 195 | 196 | ops.push({ op: op, item: elem }); 197 | } 198 | 199 | if (ops.length) 200 | return iops; 201 | else 202 | return undefined; 203 | }; 204 | 205 | index.remove= function (item,done) 206 | { 207 | if (index.indexable(item)) 208 | dyn.table(index.name) 209 | .hash('_hash',index.makeHash(item)) 210 | .range('_range',index.makeRange(item)) 211 | .delete(done) 212 | .error(function (err) 213 | { 214 | if (err.code=='notfound') 215 | done(); 216 | else 217 | done(err); 218 | }); 219 | else 220 | done(); 221 | }; 222 | 223 | _indexes.every(function (canIndex) 224 | { 225 | if (ind=canIndex(dyn,table,fields)) 226 | { 227 | _.extend(ind,_.extend(index,ind)); 228 | return false; 229 | } 230 | else 231 | return true; 232 | }); 233 | 234 | if (index.create) 235 | return index; 236 | else 237 | return false; 238 | }; 239 | 240 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and 10 | distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright 13 | owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities 16 | that control, are controlled by, or are under common control with that entity. 17 | For the purposes of this definition, "control" means (i) the power, direct or 18 | indirect, to cause the direction or management of such entity, whether by 19 | contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the 20 | outstanding shares, or (iii) beneficial ownership of such entity. 21 | 22 | "You" (or "Your") shall mean an individual or Legal Entity exercising 23 | permissions granted by this License. 24 | 25 | "Source" form shall mean the preferred form for making modifications, including 26 | but not limited to software source code, documentation source, and configuration 27 | files. 28 | 29 | "Object" form shall mean any form resulting from mechanical transformation or 30 | translation of a Source form, including but not limited to compiled object code, 31 | generated documentation, and conversions to other media types. 32 | 33 | "Work" shall mean the work of authorship, whether in Source or Object form, made 34 | available under the License, as indicated by a copyright notice that is included 35 | in or attached to the work (an example is provided in the Appendix below). 36 | 37 | "Derivative Works" shall mean any work, whether in Source or Object form, that 38 | is based on (or derived from) the Work and for which the editorial revisions, 39 | annotations, elaborations, or other modifications represent, as a whole, an 40 | original work of authorship. For the purposes of this License, Derivative Works 41 | shall not include works that remain separable from, or merely link (or bind by 42 | name) to the interfaces of, the Work and Derivative Works thereof. 43 | 44 | "Contribution" shall mean any work of authorship, including the original version 45 | of the Work and any modifications or additions to that Work or Derivative Works 46 | thereof, that is intentionally submitted to Licensor for inclusion in the Work 47 | by the copyright owner or by an individual or Legal Entity authorized to submit 48 | on behalf of the copyright owner. For the purposes of this definition, 49 | "submitted" means any form of electronic, verbal, or written communication sent 50 | to the Licensor or its representatives, including but not limited to 51 | communication on electronic mailing lists, source code control systems, and 52 | issue tracking systems that are managed by, or on behalf of, the Licensor for 53 | the purpose of discussing and improving the Work, but excluding communication 54 | that is conspicuously marked or otherwise designated in writing by the copyright 55 | owner as "Not a Contribution." 56 | 57 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf 58 | of whom a Contribution has been received by Licensor and subsequently 59 | incorporated within the Work. 60 | 61 | 2. Grant of Copyright License. 62 | 63 | Subject to the terms and conditions of this License, each Contributor hereby 64 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 65 | irrevocable copyright license to reproduce, prepare Derivative Works of, 66 | publicly display, publicly perform, sublicense, and distribute the Work and such 67 | Derivative Works in Source or Object form. 68 | 69 | 3. Grant of Patent License. 70 | 71 | Subject to the terms and conditions of this License, each Contributor hereby 72 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 73 | irrevocable (except as stated in this section) patent license to make, have 74 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where 75 | such license applies only to those patent claims licensable by such Contributor 76 | that are necessarily infringed by their Contribution(s) alone or by combination 77 | of their Contribution(s) with the Work to which such Contribution(s) was 78 | submitted. If You institute patent litigation against any entity (including a 79 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 80 | Contribution incorporated within the Work constitutes direct or contributory 81 | patent infringement, then any patent licenses granted to You under this License 82 | for that Work shall terminate as of the date such litigation is filed. 83 | 84 | 4. Redistribution. 85 | 86 | You may reproduce and distribute copies of the Work or Derivative Works thereof 87 | in any medium, with or without modifications, and in Source or Object form, 88 | provided that You meet the following conditions: 89 | 90 | You must give any other recipients of the Work or Derivative Works a copy of 91 | this License; and 92 | You must cause any modified files to carry prominent notices stating that You 93 | changed the files; and 94 | You must retain, in the Source form of any Derivative Works that You distribute, 95 | all copyright, patent, trademark, and attribution notices from the Source form 96 | of the Work, excluding those notices that do not pertain to any part of the 97 | Derivative Works; and 98 | If the Work includes a "NOTICE" text file as part of its distribution, then any 99 | Derivative Works that You distribute must include a readable copy of the 100 | attribution notices contained within such NOTICE file, excluding those notices 101 | that do not pertain to any part of the Derivative Works, in at least one of the 102 | following places: within a NOTICE text file distributed as part of the 103 | Derivative Works; within the Source form or documentation, if provided along 104 | with the Derivative Works; or, within a display generated by the Derivative 105 | Works, if and wherever such third-party notices normally appear. The contents of 106 | the NOTICE file are for informational purposes only and do not modify the 107 | License. You may add Your own attribution notices within Derivative Works that 108 | You distribute, alongside or as an addendum to the NOTICE text from the Work, 109 | provided that such additional attribution notices cannot be construed as 110 | modifying the License. 111 | You may add Your own copyright statement to Your modifications and may provide 112 | additional or different license terms and conditions for use, reproduction, or 113 | distribution of Your modifications, or for any such Derivative Works as a whole, 114 | provided Your use, reproduction, and distribution of the Work otherwise complies 115 | with the conditions stated in this License. 116 | 117 | 5. Submission of Contributions. 118 | 119 | Unless You explicitly state otherwise, any Contribution intentionally submitted 120 | for inclusion in the Work by You to the Licensor shall be under the terms and 121 | conditions of this License, without any additional terms or conditions. 122 | Notwithstanding the above, nothing herein shall supersede or modify the terms of 123 | any separate license agreement you may have executed with Licensor regarding 124 | such Contributions. 125 | 126 | 6. Trademarks. 127 | 128 | This License does not grant permission to use the trade names, trademarks, 129 | service marks, or product names of the Licensor, except as required for 130 | reasonable and customary use in describing the origin of the Work and 131 | reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. 134 | 135 | Unless required by applicable law or agreed to in writing, Licensor provides the 136 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, 137 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, 138 | including, without limitation, any warranties or conditions of TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are 140 | solely responsible for determining the appropriateness of using or 141 | redistributing the Work and assume any risks associated with Your exercise of 142 | permissions under this License. 143 | 144 | 8. Limitation of Liability. 145 | 146 | In no event and under no legal theory, whether in tort (including negligence), 147 | contract, or otherwise, unless required by applicable law (such as deliberate 148 | and grossly negligent acts) or agreed to in writing, shall any Contributor be 149 | liable to You for damages, including any direct, indirect, special, incidental, 150 | or consequential damages of any character arising as a result of this License or 151 | out of the use or inability to use the Work (including but not limited to 152 | damages for loss of goodwill, work stoppage, computer failure or malfunction, or 153 | any and all other commercial damages or losses), even if such Contributor has 154 | been advised of the possibility of such damages. 155 | 156 | 9. Accepting Warranty or Additional Liability. 157 | 158 | While redistributing the Work or Derivative Works thereof, You may choose to 159 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or 160 | other liability obligations and/or rights consistent with this License. However, 161 | in accepting such obligations, You may act only on Your own behalf and on Your 162 | sole responsibility, not on behalf of any other Contributor, and only if You 163 | agree to indemnify, defend, and hold each Contributor harmless for any liability 164 | incurred by, or claims asserted against, such Contributor by reason of your 165 | accepting any such warranty or additional liability. 166 | 167 | END OF TERMS AND CONDITIONS 168 | 169 | APPENDIX: How to apply the Apache License to your work 170 | 171 | To apply the Apache License to your work, attach the following boilerplate 172 | notice, with the fields enclosed by brackets "[]" replaced with your own 173 | identifying information. (Don't include the brackets!) The text should be 174 | enclosed in the appropriate comment syntax for the file format. We also 175 | recommend that a file or class name and description of purpose be included on 176 | the same "printed page" as the copyright notice for easier identification within 177 | third-party archives. 178 | 179 | Copyright [yyyy] [name of copyright owner] 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. 192 | -------------------------------------------------------------------------------- /lib/parser.js: -------------------------------------------------------------------------------- 1 | var _= require('underscore'), 2 | ret= require('ret'); 3 | 4 | const _soa = function (o, s, v) 5 | { 6 | s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties 7 | s = s.replace(/^\./, ''); // strip a leading dot 8 | var a = s.split('.'), 9 | prop= a.pop(); 10 | 11 | while (a.length) 12 | { 13 | var n = a.shift(); 14 | o = o[n] || (o[n]={}); 15 | } 16 | 17 | o[prop]= v; 18 | }, 19 | _oa = function(o, s) 20 | { 21 | s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties 22 | s = s.replace(/^\./, ''); // strip a leading dot 23 | var a = s.split('.'); 24 | while (a.length) { 25 | var n = a.shift(); 26 | if (n in o) { 27 | o = o[n]; 28 | } else { 29 | return; 30 | } 31 | } 32 | return o; 33 | }, 34 | _buildFilter= function (query) 35 | { 36 | var cond= query.cond, 37 | filter= query.$filter= {}, 38 | canFind= true; 39 | 40 | if (cond.$text) 41 | { 42 | query.$text= cond.$text; 43 | cond= query.cond= _.omit(query.cond,'$text'); 44 | } 45 | 46 | Object.keys(cond).every(function (field) 47 | { 48 | var val= cond[field], type= typeof val, 49 | _field= function (field,val,op) 50 | { 51 | filter[field]= { values: val===undefined ? [] : (Array.isArray(val) ? val : [ val ]), 52 | op: op }; 53 | }, 54 | _regexp= function (field,val) 55 | { 56 | var tks= ret(val.source), 57 | _chars= function (start) 58 | { 59 | return _.collect(tks.stack.slice(start), 60 | function (tk) { return String.fromCharCode(tk.value); }) 61 | .join(''); 62 | }; 63 | 64 | if (tks.stack 65 | &&tks.stack[0] 66 | &&tks.stack[0].type==ret.types.POSITION 67 | &&tks.stack[0].value=='^' 68 | &&!_.filter(tks.stack.slice(1),function (tk) { return tk.type!=ret.types.CHAR; }).length) 69 | { 70 | var val= _chars(1); 71 | if (val!='') 72 | _field(field,val,'BEGINS_WITH'); 73 | } 74 | else 75 | if (tks.stack 76 | &&!_.filter(tks.stack,function (tk) { return tk.type!=ret.types.CHAR; }).length) 77 | { 78 | var val= _chars(0); 79 | if (val!='') 80 | _field(field,val,'CONTAINS'); 81 | } 82 | else 83 | _field(field,val,'REGEXP'); 84 | }; 85 | 86 | if (type=='object') 87 | { 88 | if (Array.isArray(val)) 89 | return canFind= false; 90 | else 91 | { 92 | if (val instanceof RegExp) 93 | _regexp(field,val); 94 | else 95 | if (val.$gte!==undefined&&val.$lte!==undefined) 96 | _field(field,[val.$gte,val.$lte],'BETWEEN'); 97 | else 98 | if (val.$ne!==undefined) 99 | _field(field,val.$ne,'NE'); 100 | else 101 | if (val.$gt!==undefined) 102 | _field(field,val.$gt,'GT'); 103 | else 104 | if (val.$lt!==undefined) 105 | _field(field,val.$lt,'LT'); 106 | else 107 | if (val.$gte!==undefined) 108 | _field(field,val.$gte,'GE'); 109 | else 110 | if (val.$lte!==undefined) 111 | _field(field,val.$lte,'LE'); 112 | else 113 | if (val.$in!==undefined) 114 | _field(field,val.$in,'IN'); 115 | else 116 | if (val.$all!==undefined) 117 | _field(field,val.$all,'ALL'); 118 | else 119 | if (val.$exists!==undefined) 120 | _field(field,undefined, val.$exists ? 'NOT_NULL' : 'NULL'); 121 | else 122 | return canFind= false; 123 | } 124 | } 125 | else 126 | _field(field,val,'EQ'); 127 | 128 | return true; 129 | }); 130 | 131 | return canFind; 132 | }; 133 | 134 | 135 | module.exports= function (dyn,opts) 136 | { 137 | var parser= {}; 138 | 139 | parser.parse= function (table,modifiers,cond,projection,identity) 140 | { 141 | var p= dyn.promise('parsed'), 142 | query= { table: table, cond: cond || {}, projection: { root: {} }, opts: opts }, 143 | _projection= function (root) 144 | { 145 | var proj= { include: ['_id','_pos','_ref'], exclude: []}; 146 | 147 | _.keys(root).forEach(function (attr) 148 | { 149 | if (root[attr].$include) 150 | { 151 | proj.include.push(attr); 152 | proj.include.push('__'+attr); 153 | proj.include.push('___'+attr); 154 | } 155 | else 156 | if (root[attr].$exclude) 157 | { 158 | proj.exclude.push(attr); 159 | proj.exclude.push('__'+attr); 160 | proj.exclude.push('___'+attr); 161 | } 162 | }); 163 | 164 | proj.include= _.uniq(proj.include); 165 | proj.exclude= _.uniq(proj.exclude); 166 | 167 | if (proj.include.length==3) 168 | proj.include= undefined; 169 | 170 | return proj; 171 | }; 172 | 173 | table.name= table._dynamo.TableName; 174 | table.indexes= table.indexes || []; 175 | 176 | query.project= _projection; 177 | query.soa= _soa; 178 | query.oa= _oa; 179 | 180 | if (modifiers.orderby) 181 | { 182 | var fields= []; 183 | 184 | Object.keys(modifiers.orderby).forEach(function (name) 185 | { 186 | fields.push({ name: name, dir: modifiers.orderby[name] }); 187 | }); 188 | 189 | query.$orderby= fields; 190 | } 191 | 192 | query.toprojection= function (root) 193 | { 194 | var proj= {}; 195 | 196 | _.keys(root || {}).forEach(function (attr) 197 | { 198 | if (root[attr].$include) 199 | proj[attr]= 1; 200 | else 201 | if (root[attr].$exclude) 202 | proj[attr]= -1; 203 | }); 204 | 205 | return proj; 206 | }; 207 | 208 | if (projection&&_.keys(projection).length) 209 | { 210 | query.identity= {}; 211 | 212 | query.identity.get= function (_id, _pos, done) 213 | { 214 | done(); 215 | }; 216 | 217 | query.identity.set= function (item) 218 | { 219 | }; 220 | } 221 | else 222 | { 223 | if (identity&&identity.map) 224 | query.identity= identity; 225 | else 226 | { 227 | query.identity= {}; 228 | query.identity.map= {}; 229 | 230 | query.identity.get= function (_id, _pos, cb) 231 | { 232 | var item= query.identity.map[_id+':'+_pos]; 233 | 234 | if (!item) 235 | { 236 | query.identity.map[_id+':'+_pos]= true; 237 | cb(); 238 | } 239 | else 240 | if (item===true) 241 | setTimeout(query.identity.get,100,_id,_pos,cb); 242 | else 243 | cb(item); 244 | } 245 | 246 | query.identity.set= function (_id, _pos, item) 247 | { 248 | query.identity.map[_id+':'+_pos]= item; 249 | } 250 | } 251 | } 252 | 253 | process.nextTick(function () 254 | { 255 | 256 | _.extend(query,modifiers); 257 | 258 | if (projection) 259 | { 260 | _.keys(projection).every(function (attr) 261 | { 262 | if (projection[attr]==1) 263 | _soa(query.projection.root,attr,{ $include: true }); 264 | else 265 | if (projection[attr]==-1) 266 | _soa(query.projection.root,attr,{ $exclude: true }); 267 | else 268 | { 269 | p.trigger.error(new Error('unknown projection value '+JSON.stringify(projection[attr]))); 270 | return false; 271 | } 272 | 273 | return true; 274 | }); 275 | 276 | _.extend(query.projection,query.project(query.projection.root)); 277 | } 278 | 279 | query.$supported= _buildFilter(query); 280 | query.$returned= 0; 281 | query.$filtered= []; 282 | query.window= query.window || 50; 283 | 284 | query.filterComplete= function () 285 | { 286 | return Object.keys(query.$filter).length==0; 287 | }; 288 | 289 | query.sortComplete= function () 290 | { 291 | return !(query.orderby&&!query.sorted); 292 | }; 293 | 294 | query.limitComplete= function () 295 | { 296 | return !(query.limit!==undefined&&!query.limited); 297 | }; 298 | 299 | query.skipComplete= function () 300 | { 301 | return !(query.skip!==undefined&&!query.skipped); 302 | }; 303 | 304 | query.canLimit= function () 305 | { 306 | return !query.limited 307 | &&query.sortComplete() 308 | &&query.filterComplete(); 309 | }; 310 | 311 | query.canSkip= function () 312 | { 313 | return !query.skipped 314 | &&query.sortComplete() 315 | &&query.filterComplete(); 316 | }; 317 | 318 | query.canCount= function () 319 | { 320 | return query.filterComplete() 321 | &&query.limitComplete() 322 | &&query.skipComplete(); 323 | }; 324 | 325 | query.finderProjection= function () 326 | { 327 | if (query.projection.include) 328 | { 329 | // if i cannot filter them i project them to the refiner for client-side filtering 330 | var toproject= _.difference(_.keys(query.$filter),query.projection.include) || []; 331 | return _.union(query.projection.include,toproject); 332 | } 333 | else 334 | return undefined; 335 | }; 336 | 337 | p.trigger.parsed(query); 338 | 339 | }); 340 | 341 | return p; 342 | }; 343 | 344 | return parser; 345 | }; 346 | -------------------------------------------------------------------------------- /lib/indexes/cloud-search.js: -------------------------------------------------------------------------------- 1 | var _= require('underscore'), 2 | ret= require('ret'), 3 | http= require('http'), 4 | url= require('url'), 5 | querystring= require('querystring'), 6 | colors= require('colors'), 7 | async= require('async'), 8 | Stream= require('stream').Stream, 9 | AWS = require('aws-sdk'); 10 | 11 | const IDRE= /^[a-z0-9][a-z0-9_]*$/; 12 | 13 | module.exports= function (dyn,table,fields) 14 | { 15 | var index= {}, domain= {}, fieldNames= _.keys(fields); 16 | 17 | if (fieldNames[0]!='$search') return false; 18 | 19 | var CS= new AWS.CloudSearch(); 20 | 21 | domain= fields[fieldNames[0]]; 22 | 23 | if (!domain.domain||!domain.lang) return; 24 | 25 | index.name= 'CloudSearch-'+(domain.name= domain.domain); 26 | delete domain.domain; 27 | 28 | var postQueue= []; 29 | 30 | setInterval(function poster() 31 | { 32 | var _post= function (elems) 33 | { 34 | var post_data= JSON.stringify(_.pluck(elems,'elem')), 35 | post_options = { 36 | hostname: domain.aws.DocService.Endpoint, 37 | port: 80, 38 | path: '/2011-02-01/documents/batch', 39 | method: 'POST', 40 | headers: { 41 | 'Accept': 'application/json', 42 | 'Content-Type': 'application/json; charset=UTF-8', 43 | 'Content-Length': Buffer.byteLength(post_data) 44 | } 45 | }; 46 | 47 | var post_req = http.request(post_options, function(res) 48 | { 49 | res.setEncoding('utf8'); 50 | var json= ''; 51 | res.on('data', function (chunk) 52 | { 53 | json+=chunk; 54 | }) 55 | .on('end', function () 56 | { 57 | var res= JSON.parse(json); 58 | elems.forEach(function (e) { e.done(null,res); }); 59 | }); 60 | }) 61 | .on('error', function (err) 62 | { 63 | elems.forEach(function (e) { e.done(err); }); 64 | }); 65 | 66 | post_req.write(post_data); 67 | post_req.end(); 68 | }; 69 | 70 | var cnt=0, elems= []; 71 | 72 | while (cnt++<1000&&postQueue.length) 73 | elems.push(postQueue.shift()); 74 | 75 | if (elems.length) 76 | _post(elems); 77 | 78 | },1000); 79 | 80 | domain.post= function (elem,done) 81 | { 82 | postQueue.push({ elem: elem, done: done }); 83 | }; 84 | 85 | domain.stream= function () 86 | { 87 | var wstream= new Stream(), 88 | emit= { 89 | drain: _.bind(wstream.emit,wstream,'drain'), 90 | error: _.bind(wstream.emit,wstream,'error'), 91 | finish: _.bind(wstream.emit,wstream,'finish') 92 | }, 93 | _ops= function (items) 94 | { 95 | return _.filter(_.collect(items,function (item) 96 | { 97 | var elem= index.makeElement(item); 98 | 99 | if (elem) 100 | { 101 | elem.type='add'; 102 | return elem; 103 | } 104 | else 105 | return undefined; 106 | 107 | }),function (item) { return !!item; }); 108 | }, 109 | write= function (items,emit) 110 | { 111 | async.forEach(_ops(items),domain.post, 112 | function (err) 113 | { 114 | if (err) 115 | emit.error(err); 116 | else 117 | emit.done(); 118 | }); 119 | }; 120 | 121 | wstream.writeable= true; 122 | 123 | wstream.write= function (items) 124 | { 125 | if (items&&items.length) 126 | write(items,{ done: emit.drain, error: emit.error }); 127 | else 128 | emit.drain(); 129 | 130 | return false; 131 | }; 132 | 133 | wstream.end= function (items) 134 | { 135 | if (items&&items.length) 136 | write(items,{ done: emit.finish, error: emit.error },true); 137 | else 138 | emit.finish(); 139 | }; 140 | 141 | wstream.wemit= emit; 142 | 143 | return wstream; 144 | }; 145 | 146 | domain.get= function (query,done) 147 | { 148 | var _query= 'http://'+domain.aws.SearchService.Endpoint+'/2011-02-01/search?' 149 | +querystring.stringify(query); 150 | 151 | //console.log(_query); 152 | 153 | http.get(_query, 154 | function(res) 155 | { 156 | res.setEncoding('utf8'); 157 | 158 | var json= ''; 159 | res.on('data', function (chunk) 160 | { 161 | json+=chunk; 162 | }) 163 | .on('end', function () 164 | { 165 | // 2xx status codes indicate that the request was processed successfully. 166 | if ((res.statusCode+'').indexOf('2')==0) 167 | done(null,JSON.parse(json)); 168 | else 169 | if (res.statusCode==404) 170 | done(new Error('not found')); 171 | else 172 | if (res.statusCode==405) 173 | done(new Error('Invalid HTTP Method')); 174 | else 175 | if (res.statusCode==408) 176 | done(new Error('Request Timeout')); 177 | else 178 | if (res.statusCode==500) 179 | done(new Error('Internal Server Error')); 180 | else 181 | if (res.statusCode==502) 182 | done(new Error('Search service is overloaded')); 183 | else 184 | if (res.statusCode==504) 185 | done(new Error('Search service is overloaded, retry later')); 186 | else 187 | if (res.statusCode==507) 188 | done(new Error('Insufficient Storage')); 189 | else 190 | if (res.statusCode==509) 191 | done(new Error('Bandwidth Limit Exceeded')); 192 | else 193 | if ((res.statusCode+'').indexOf('4')==0) 194 | done(new Error('Malformed request: '+res.statusCode)); 195 | else 196 | if ((res.statusCode+'').indexOf('5')==0) 197 | done(new Error('CloudSearch is experiencing problems: '+res.statusCode)); 198 | else 199 | done(new Error('Unknown status code from CloudSearch: '+res.statusCode)); 200 | }); 201 | }) 202 | .on('error', done); 203 | }; 204 | 205 | index.exists= function (done) 206 | { 207 | CS.describeDomains({ DomainNames: [domain.name] },function(err, data) 208 | { 209 | if (err) done(err); 210 | else 211 | done(null,!!_.filter(data.DomainStatusList, 212 | function (d) 213 | { 214 | if (d.DomainName==domain.name&&d.Created&&!d.Deleted) 215 | { 216 | domain.aws= d; 217 | return true; 218 | } 219 | }).length); 220 | }); 221 | }; 222 | 223 | index.drop= function (done) 224 | { 225 | CS.deleteDomain({ DomainName: domain.name }, 226 | function (err,data) 227 | { 228 | if (err) done(err); 229 | else 230 | (function check() 231 | { 232 | index.exists(function (err,exists) 233 | { 234 | if (err) 235 | done(err); 236 | else 237 | if (exists) 238 | setTimeout(check,5000); 239 | else 240 | done(); 241 | }); 242 | })(); 243 | }); 244 | }; 245 | 246 | index.create= function (done) 247 | { 248 | console.log('this operation may take several minutes'.yellow); 249 | 250 | CS.createDomain({ DomainName: domain.name }, 251 | function (err,data) 252 | { 253 | if (err) done(err); 254 | else 255 | (function check() 256 | { 257 | index.exists(function (err,exists) 258 | { 259 | if (err) 260 | done(err); 261 | else 262 | if (!exists||domain.aws.Processing) 263 | { 264 | if (domain.aws.Processing) 265 | console.log(('CloudSearch is initializing domain: '+domain.name).yellow); 266 | 267 | setTimeout(check,5000); 268 | } 269 | else 270 | table.find().results(function (items) 271 | { 272 | async.forEach(items,index.put,done); 273 | }) 274 | .error(done); 275 | }); 276 | })(); 277 | }); 278 | }; 279 | 280 | index.ensure= function (done) 281 | { 282 | index.exists(function (err, exists) 283 | { 284 | if (err) 285 | done(err); 286 | else 287 | if (exists) 288 | { 289 | /*if (domain.aws.Processing) 290 | { 291 | console.log(('CloudSearch is configuring domain: '+domain.name).yellow); 292 | setTimeout(function () { index.ensure(done); },5000); 293 | } 294 | else*/ 295 | done(); 296 | } 297 | else 298 | index.create(done); 299 | }); 300 | }; 301 | 302 | index.put= function (item,done) 303 | { 304 | if (index.indexable(item)) 305 | { 306 | var elem= index.makeElement(item); 307 | 308 | if (elem) 309 | { 310 | elem.type='add'; 311 | domain.post(elem,done); 312 | } 313 | else 314 | done(); 315 | } 316 | else 317 | done(); 318 | }; 319 | 320 | index.update= function (item,op) // @FIXME: should give errors to the indexer? 321 | { 322 | _.bind(op=='put' ? index.put : index.remove,index)(item,function (err, res) 323 | { 324 | if (err) 325 | console.log((err+'').red,err.stack); 326 | else 327 | if (res) 328 | { 329 | if (res.status=='error') 330 | res.errors.forEach(function (err) { console.log(err.message.red); }); 331 | else 332 | if (res.warnings) 333 | res.warnings.forEach(function (warn) { console.log(warn.message.yellow); }); 334 | } 335 | }); 336 | 337 | return undefined; 338 | }; 339 | 340 | index.remove= function (item,done) 341 | { 342 | var p= done ? { trigger: { success: done } } : dyn.promise(null,null,'consumed'); 343 | 344 | if (index.indexable(item)) 345 | { 346 | var elem= index.makeElement(item); 347 | 348 | if (elem) 349 | { 350 | elem.type='del'; 351 | domain.post(_.omit(elem,'fields'),p.trigger.success); 352 | } 353 | else 354 | process.nextTick(p.trigger.success); 355 | } 356 | else 357 | process.nextTick(p.trigger.success); 358 | 359 | return p; 360 | }; 361 | 362 | index.indexable= function (item) 363 | { 364 | return _.some(_.keys(item),function (field) { return field.indexOf('_')!=0 }); 365 | }; 366 | 367 | index.usable= function (query) 368 | { 369 | return !!query.cond.$search; 370 | }; 371 | 372 | index.makeElement= function (_item) 373 | { 374 | var item= domain.transform ? domain.transform(_item) : _item, 375 | elem= { id: item._id.replace(/\./g,'_dot_').replace(/@/g,'_at_').replace(/-/g,'_')+'__'+item._pos, version: item._rev, fields: {} }; 376 | 377 | elem.lang= item._lang || domain.lang; 378 | 379 | _.keys(item).forEach(function (field) 380 | { 381 | var val= item[field]; 382 | 383 | if (field.indexOf('_')!=0&&typeof val!='object'&&field!='__jsogObjectId') 384 | elem.fields[field.toLowerCase()]= val; 385 | }); 386 | 387 | if (_.keys(elem.fields).length&&IDRE.exec(elem.id)) 388 | return elem; 389 | else 390 | return undefined; 391 | }; 392 | 393 | index.rebuild= function (window) 394 | { 395 | var p= dyn.promise(), 396 | sindex= dyn.stream(index.name), 397 | pcnt= 0; 398 | 399 | dyn.stream(table._dynamo.TableName) 400 | .scan({ limit: window }) 401 | .on('data',function (items) { process.stdout.write(('\r'+(pcnt+=items.length)).yellow); }) 402 | .pipe(domain.stream()) 403 | .on('error',p.trigger.error) 404 | .on('finish',_.compose(p.trigger.success,console.log)); 405 | 406 | return p; 407 | }; 408 | 409 | index.find= function (query) 410 | { 411 | var p= dyn.promise(['results','count','end'],null,'consumed'); 412 | 413 | query.limited= query.canLimit(); 414 | 415 | if (query.limited) 416 | { 417 | if (query.limit) 418 | query.cond.$search.size= query.limit; 419 | 420 | if (query.skip) 421 | query.cond.$search.start= query.skip; 422 | } 423 | 424 | domain.get(query.cond.$search, function (err, res) 425 | { 426 | if (err) p.trigger.error(err); 427 | else 428 | { 429 | delete query.$filter['$search']; 430 | 431 | query.counted= query.canCount(); 432 | 433 | if (query.count&&query.counted) 434 | p.trigger.count(res.hits.hit.length); 435 | else 436 | { 437 | p.trigger.results(_.collect(res.hits.hit, 438 | function (hit) 439 | { 440 | var id= hit.id.split('__'); 441 | return { _id: id[0].replace(/_dot_/,'.').replace(/_at_/,'@').replace(/_/g,'-'), _pos: parseInt(id[1]) }; 442 | })); 443 | p.trigger.end(); 444 | } 445 | } 446 | }); 447 | 448 | return p; 449 | }; 450 | 451 | return index; 452 | }; 453 | 454 | -------------------------------------------------------------------------------- /test/db.test.js: -------------------------------------------------------------------------------- 1 | var should= require('chai').should(), 2 | assert= require('chai').assert, 3 | AWS = require('aws-sdk'), 4 | _= require('underscore'), 5 | dyngo= require('../index.js'); 6 | 7 | const noerr= function (done) 8 | { 9 | return function (err) 10 | { 11 | should.not.exist(err); 12 | done(); 13 | }; 14 | }, 15 | accept= function (code,done) 16 | { 17 | return function (err) 18 | { 19 | if (err.code==code) 20 | done(); 21 | else 22 | done(err); 23 | }; 24 | }; 25 | 26 | 27 | describe('database',function () 28 | { 29 | var db; 30 | 31 | before(function (done) 32 | { 33 | dyngo({ dynamo: { endpoint: new AWS.Endpoint('http://localhost:8000') }, hints: false }, 34 | function (err,_db) 35 | { 36 | db= _db; 37 | done(); 38 | }); 39 | }); 40 | 41 | beforeEach(function (done) 42 | { 43 | db.test.remove().success(done) 44 | .error(done); 45 | }); 46 | 47 | it('Can connect', function (done) 48 | { 49 | should.exist(db); 50 | done(); 51 | }); 52 | 53 | it('test table should be empty', function (done) 54 | { 55 | db.test.find().results(function (items) 56 | { 57 | items.length.should.equal(0); 58 | done(); 59 | }) 60 | .error(noerr(done)); 61 | }); 62 | 63 | describe('save',function () 64 | { 65 | it('Can insert a new object, and then find it', function (done) 66 | { 67 | var _noerr= noerr(done); 68 | 69 | db.test.save({ somedata: 'ok' }) 70 | .success(function () 71 | { 72 | db.test.findOne({ somedata: 'ok' }) 73 | .result(function (obj) 74 | { 75 | obj.somedata.should.equal('ok'); 76 | done(); 77 | }) 78 | .error(_noerr); 79 | }) 80 | .error(_noerr); 81 | }); 82 | 83 | it('Can insert an object with a child object, and have it back', function (done) 84 | { 85 | var _noerr= noerr(done); 86 | 87 | db.test.save({ somedata: 'parent', child: { somedata: 'child' } }) 88 | .success(function () 89 | { 90 | db.test.findOne({ somedata: 'parent' }) 91 | .result(function (obj) 92 | { 93 | obj.somedata.should.equal('parent'); 94 | obj.child.somedata.should.equal('child'); 95 | done(); 96 | }) 97 | .error(_noerr); 98 | }) 99 | .error(_noerr); 100 | }); 101 | 102 | it('Can insert an object with a child object, and then remove the child', function (done) 103 | { 104 | var _noerr= noerr(done); 105 | 106 | var par= { somedata: 'parentrchild', child: { somedata: 'child' } }; 107 | 108 | db.test.save(par) 109 | .success(function () 110 | { 111 | delete par.child; 112 | 113 | db.test.save(par) 114 | .success(function () 115 | { 116 | db.test.findOne({ _id: par._id }) 117 | .result(function (obj) 118 | { 119 | obj.somedata.should.equal('parentrchild'); 120 | should.not.exist(obj.child); 121 | done(); 122 | }) 123 | .error(_noerr); 124 | }) 125 | .error(_noerr); 126 | }) 127 | .error(_noerr); 128 | }); 129 | 130 | it('Can insert an object with a child object, and then change the type of the child', function (done) 131 | { 132 | var _noerr= noerr(done); 133 | 134 | var par= { somedata: 'parentrchild', child: { somedata: 'child' } }; 135 | 136 | db.test.save(par) 137 | .success(function () 138 | { 139 | par.child= 'child'; 140 | 141 | db.test.save(par) 142 | .success(function () 143 | { 144 | db.test.findOne({ _id: par._id }) 145 | .result(function (obj) 146 | { 147 | obj.somedata.should.equal('parentrchild'); 148 | obj.child.should.equal('child'); 149 | done(); 150 | }) 151 | .error(_noerr); 152 | }) 153 | .error(_noerr); 154 | }) 155 | .error(_noerr); 156 | }); 157 | 158 | it('Can insert an object with a child object array, and have it back', function (done) 159 | { 160 | var _noerr= noerr(done); 161 | 162 | db.test.save({ somedata: 'parentarr', childs: [{ somedata: 'child1' },{ somedata: 'child2' },{ somedata: 'child3' }] }) 163 | .success(function () 164 | { 165 | db.test.findOne({ somedata: 'parentarr' }) 166 | .result(function (obj) 167 | { 168 | obj.somedata.should.equal('parentarr'); 169 | obj.childs.length.should.equal(3); 170 | _.pluck(obj.childs,'somedata').should.contain('child1'); 171 | _.pluck(obj.childs,'somedata').should.contain('child2'); 172 | _.pluck(obj.childs,'somedata').should.contain('child3'); 173 | done(); 174 | }) 175 | .error(_noerr); 176 | }) 177 | .error(_noerr); 178 | }); 179 | 180 | it('Can insert an object with a child object array, and then update the parent without loosing the child object array', function (done) 181 | { 182 | var _noerr= noerr(done); 183 | 184 | var par= { somedata: 'parentrarrupd', childs: [{ somedata: 'child1' },{ somedata: 'child2' },{ somedata: 'child3' }] }; 185 | 186 | db.test.save(par) 187 | .success(function () 188 | { 189 | db.test.save(par) 190 | .success(function () 191 | { 192 | db.test.findOne({ _id: par._id }) 193 | .result(function (obj) 194 | { 195 | obj.somedata.should.equal('parentrarrupd'); 196 | should.exist(obj.childs); 197 | obj.childs.length.should.equal(3); 198 | _.pluck(obj.childs,'somedata').should.contain('child1'); 199 | _.pluck(obj.childs,'somedata').should.contain('child2'); 200 | _.pluck(obj.childs,'somedata').should.contain('child3'); 201 | done(); 202 | }) 203 | .error(_noerr); 204 | }) 205 | .error(_noerr); 206 | }) 207 | .error(_noerr); 208 | }); 209 | 210 | it('Can insert an object with a child object array, and then remove the child object array', function (done) 211 | { 212 | var _noerr= noerr(done); 213 | 214 | var par= { somedata: 'parentrarrrm', childs: [{ somedata: 'child1' },{ somedata: 'child2' },{ somedata: 'child3' }] }; 215 | 216 | db.test.save(par) 217 | .success(function () 218 | { 219 | delete par.childs; 220 | 221 | db.test.save(par) 222 | .success(function () 223 | { 224 | db.test.findOne({ _id: par._id }) 225 | .result(function (obj) 226 | { 227 | obj.somedata.should.equal('parentrarrrm'); 228 | should.not.exist(obj.childs); 229 | done(); 230 | }) 231 | .error(_noerr); 232 | }) 233 | .error(_noerr); 234 | }) 235 | .error(_noerr); 236 | }); 237 | 238 | it('Can insert an object with a child object array, and then change the type of the child array', function (done) 239 | { 240 | var _noerr= noerr(done); 241 | 242 | var par= { somedata: 'parentcarr', childs: [{ somedata: 'child1' },{ somedata: 'child2' },{ somedata: 'child3' }] }; 243 | 244 | db.test.save(par) 245 | .success(function () 246 | { 247 | par.childs= ['child1','child2','child3']; 248 | 249 | db.test.save(par) 250 | .success(function () 251 | { 252 | db.test.findOne({ _id: par._id }) 253 | .result(function (obj) 254 | { 255 | obj.somedata.should.equal('parentcarr'); 256 | obj.childs.length.should.equal(3); 257 | obj.childs.should.contain('child1'); 258 | obj.childs.should.contain('child2'); 259 | obj.childs.should.contain('child3'); 260 | done(); 261 | }) 262 | .error(_noerr); 263 | }) 264 | .error(_noerr); 265 | }) 266 | .error(_noerr); 267 | }); 268 | 269 | it('Can insert an object with a child object array, and remove an element from the array', function (done) 270 | { 271 | var _noerr= noerr(done); 272 | 273 | var par= { somedata: 'parentcarr', childs: [{ somedata: 'child1' },{ somedata: 'child2' },{ somedata: 'child3' }] }; 274 | 275 | db.test.save(par) 276 | .success(function () 277 | { 278 | par.childs.splice(2,1); 279 | 280 | db.test.save(par) 281 | .success(function () 282 | { 283 | db.test.findOne({ _id: par._id }) 284 | .result(function (obj) 285 | { 286 | obj.somedata.should.equal('parentcarr'); 287 | obj.childs.length.should.equal(2); 288 | _.pluck(obj.childs,'somedata').should.contain('child1'); 289 | _.pluck(obj.childs,'somedata').should.contain('child2'); 290 | done(); 291 | }) 292 | .error(_noerr); 293 | }) 294 | .error(_noerr); 295 | }) 296 | .error(_noerr); 297 | }); 298 | 299 | it('Can insert an object with an independent child object array, and remove an element from the array', function (done) 300 | { 301 | var _noerr= noerr(done); 302 | 303 | var par= { somedata: 'parentcarr', childs: [{ somedata: 'child1' },{ somedata: 'child2' },{ somedata: 'child3' }] }; 304 | 305 | db.test.save(par.childs) 306 | .success(function () 307 | { 308 | db.test.save(par) 309 | .success(function () 310 | { 311 | par.childs.splice(1,1); 312 | 313 | db.test.save(par) 314 | .success(function () 315 | { 316 | db.test.findOne({ _id: par._id }) 317 | .result(function (obj) 318 | { 319 | obj.somedata.should.equal('parentcarr'); 320 | obj.childs.length.should.equal(2); 321 | _.pluck(obj.childs,'somedata').should.contain('child1'); 322 | _.pluck(obj.childs,'somedata').should.contain('child3'); 323 | done(); 324 | }) 325 | .error(_noerr); 326 | }) 327 | .error(_noerr); 328 | }) 329 | .error(_noerr); 330 | }) 331 | .error(_noerr); 332 | }); 333 | 334 | it('supports Date objects', function (done) 335 | { 336 | var _noerr= noerr(done), d= new Date(); 337 | 338 | var par= { val: d }; 339 | 340 | db.test.save(par) 341 | .success(function () 342 | { 343 | db.test.findOne({ _id: par._id }) 344 | .result(function (obj) 345 | { 346 | should.exist(obj.val); 347 | obj.val.should.equal(d.toISOString()); 348 | done(); 349 | }) 350 | .error(_noerr); 351 | }) 352 | .error(_noerr); 353 | }); 354 | }); 355 | 356 | describe('remove',function () 357 | { 358 | it('Can delete objects found by _id, with client side filtering', function (done) 359 | { 360 | // if filter attributes are not included in projection by the finder 361 | // when it can't filter them, this test should fail 362 | db.test.save({ _id: 'toremove', uid: 'andrea' }) 363 | .success(function () 364 | { 365 | db.test.remove({ _id: 'toremove', uid: 'andrea' }) 366 | .success(function () 367 | { 368 | db.test.findOne({ _id: 'toremove', uid: 'andrea' }) 369 | .result(should.not.exist) 370 | .error(accept('notfound',done)); 371 | }) 372 | .error(done); 373 | }) 374 | .error(done); 375 | }); 376 | }); 377 | }); 378 | -------------------------------------------------------------------------------- /lib/dyn.tx.js: -------------------------------------------------------------------------------- 1 | var _= require('underscore'), 2 | debug = require('optimist').argv['dyn-debug'], 3 | async= require('async'); 4 | 5 | const BECONSISTENT= { consistent: true }, 6 | GETSTATE= _.extend({ attrs: ['state'] },BECONSISTENT); 7 | 8 | module.exports= function configureTransaction(dyn,tx) 9 | { 10 | dyn.putNT= dyn.put; 11 | dyn.updateItemNT= dyn.updateItem; 12 | dyn.deleteNT= dyn.delete; 13 | dyn.getNT= dyn.get; 14 | dyn.queryNT= dyn.query; 15 | dyn.scanNT= dyn.scan; 16 | 17 | var _ctxId= function (ctx,item) 18 | { 19 | if (!ctx.hash) 20 | { 21 | if (item._hash!==undefined) 22 | ctx.hash= { attr: '_hash', value: item._hash }; 23 | else 24 | if (item._id!==undefined) 25 | ctx.hash= { attr: '_id', value: item._id }; 26 | else 27 | console.log('dyn.tx.js: unknown hash'.red,item); // :( 28 | } 29 | 30 | if (!ctx.range) 31 | { 32 | if (item._range!==undefined) 33 | ctx.range= { attr: '_range', value: item._range }; 34 | else 35 | if (item._pos!==undefined) 36 | ctx.range= { attr: '_pos', value: item._pos }; 37 | else 38 | console.log('dyn.tx.js: unknown range'.red,item); // :( 39 | } 40 | 41 | var _id= function (prefix) 42 | { 43 | return [prefix, 44 | ctx.table, 45 | ctx.hash.attr, 46 | ctx.hash.value, 47 | ctx.range.attr, 48 | ctx.range.value].join('::'); 49 | }; 50 | 51 | _id.ctx= ctx; 52 | 53 | return _id; 54 | }, 55 | _perform= function (dynI,ctx,p,obj,op,_apply) 56 | { 57 | var _id= _ctxId(ctx), 58 | txR= function () 59 | { 60 | return dyn.table(tx.txTable._dynamo.TableName) 61 | .hash('_id',tx._id) 62 | .range('_item','_'); 63 | }, 64 | dynT= function () 65 | { 66 | return dyn.table(tx.txTable._dynamo.TableName) 67 | .hash('_id',tx._id) 68 | .range('_item',_id('target')); 69 | }, 70 | dynC= function () 71 | { 72 | return dyn.table(tx.txTable._dynamo.TableName) 73 | .hash('_id',tx._id) 74 | .range('_item',_id('copy')); 75 | }, 76 | _decide= function (txId,cb) 77 | { 78 | tx.transaction(txId) 79 | .transaction(function (competing) 80 | { 81 | if (competing.state=='pending') 82 | competing.rollback().rolledback(cb).error(p.trigger.error).consumed(p.trigger.consumed); 83 | else 84 | competing.commit().committed(cb).error(p.trigger.error).consumed(p.trigger.consumed); 85 | }) 86 | .error(function (err) 87 | { 88 | if (err.code=='notfound') 89 | dynI() 90 | .updateItemNT({ update: { _txTransient: { action: 'DELETE' }, 91 | _txApplied: { action: 'DELETE' }, 92 | _txDeleted: { action: 'DELETE' }, 93 | _txLocked: { action: 'DELETE' }, 94 | _tx: { action: 'DELETE' } } }, 95 | cb) 96 | .chain(p); 97 | else 98 | p.trigger.error(err); 99 | }) 100 | .consumed(p.trigger.consumed); 101 | }, 102 | _lock= function (p,_locked) 103 | { 104 | dynI().getNT(function (item) 105 | { 106 | if (!item._tx) // item lock free 107 | { 108 | item._tx= tx._id; 109 | item._txLocked= new Date().toISOString(); 110 | 111 | // set lock 112 | dynI().putNT(item,function () 113 | { 114 | _locked(item); 115 | },{ exists: true, expected: { _tx: false } }) 116 | .consumed(p.trigger.consumed) 117 | .error(p.trigger.error); 118 | } 119 | else // the item is locked 120 | { 121 | if (item._txTransient) 122 | obj['_txTransient']= true; 123 | 124 | if (item._tx==tx._id) // ok: lock already acquired 125 | _locked(item); 126 | else 127 | if (item._tx!=tx._id) // mm: lock acquired by another transaction 128 | _decide(item._tx,function () 129 | { 130 | _lock(p,_locked); 131 | }); 132 | } 133 | },BECONSISTENT) 134 | .consumed(p.trigger.consumed) 135 | .error(function (err) 136 | { 137 | if (err.code=='notfound'&&op=='put') // insert a transient item to acquire the lock 138 | { 139 | var item= { _tx: tx._id }; 140 | item[ctx.hash.attr]= ctx.hash.value; 141 | item[ctx.range.attr]= ctx.range.value; 142 | item['_txTransient']= true; 143 | obj['_txTransient']= true; 144 | 145 | dynI().putNT(item,function () 146 | { 147 | _locked(item,true); 148 | },{ exists: false }) 149 | .consumed(p.trigger.consumed) 150 | .error(function () 151 | { 152 | if (err.code=='found') 153 | p.trigger.error(new Error('Someone inserted an item while we were trying to acquire a lock')); 154 | else 155 | p.trigger.error(err); 156 | }); 157 | } 158 | else 159 | p.trigger.error(err); 160 | }); 161 | 162 | }, 163 | _save= function (p,item,cb) 164 | { 165 | var t= dynC(); 166 | t.putNT(_.extend({},item,{ _id: dyn.ctx.hash.value, _item: dyn.ctx.range.value }),cb,{ exists: false }) 167 | .consumed(p.trigger.consumed) 168 | .error(function (err) 169 | { 170 | if (err.code!='found') 171 | p.trigger.error(err); 172 | else 173 | cb(); 174 | }); 175 | }, 176 | _add= function (p,cb) 177 | { 178 | txR().getNT(function (_txR) 179 | { 180 | if (_txR.state=='pending') 181 | { 182 | var t= dynT(); 183 | 184 | t.putNT(_.extend({ _txOp: op },{ update: JSON.stringify(obj) },{ _id: dyn.ctx.hash.value, _item: dyn.ctx.range.value }),cb) 185 | .consumed(p.trigger.consumed) 186 | .error(p.trigger.error); 187 | } 188 | else 189 | p.trigger.rolledback(true); 190 | },GETSTATE) 191 | .consumed(p.trigger.consumed) 192 | .error(p.trigger.error); 193 | }, 194 | _verify= function (p,cb) 195 | { 196 | txR().getNT(function (_txR) 197 | { 198 | if (_txR.state=='pending') 199 | cb(); 200 | else 201 | p.trigger.rolledback(true); 202 | },GETSTATE) 203 | .consumed(p.trigger.consumed) 204 | .error(p.trigger.error); 205 | }; 206 | 207 | 208 | // @see https://github.com/awslabs/dynamodb-transactions/blob/master/DESIGN.md#no-contention 209 | _add(p,function () 210 | { 211 | _lock(p,function (item,skipSave) 212 | { 213 | if (skipSave||op=='get') 214 | _verify(p,function () 215 | { 216 | _apply(item); 217 | }); 218 | else 219 | _save(p,item,function () 220 | { 221 | _verify(p,_apply); 222 | }); 223 | }); 224 | }); 225 | }, 226 | _selectTxAttrs= function (opts) 227 | { 228 | opts.consistent= true; 229 | 230 | if (opts.attrs) 231 | { 232 | 233 | if (!_.contains(opts.attrs,'_tx')) 234 | opts.attrs.push('_tx'); 235 | 236 | if (!_.contains(opts.attrs,'_txApplied')) 237 | opts.attrs.push('_txApplied'); 238 | 239 | if (!_.contains(opts.attrs,'_txDeleted')) 240 | opts.attrs.push('_txDeleted'); 241 | 242 | if (!_.contains(opts.attrs,'_txTransient')) 243 | opts.attrs.push('_txTransient'); 244 | 245 | } 246 | }, 247 | _getTxItem= function (item,_id,p,cb) 248 | { 249 | if (item._tx) 250 | { 251 | if (tx._id!=item._tx) 252 | { 253 | if (item._txTransient) 254 | cb(null); 255 | else 256 | if (item._txApplied) 257 | { 258 | var ctx= _id.ctx; 259 | 260 | dyn.table(tx.txTable._dynamo.TableName) 261 | .hash('_id',item._tx) 262 | .range('_item',_id('copy')) 263 | .get(function (copy) 264 | { 265 | delete copy['_id']; 266 | delete copy['_item']; 267 | 268 | copy[ctx.hash.attr]= ctx.hash.value; 269 | copy[ctx.range.attr]= ctx.range.value; 270 | 271 | cb(null,copy); 272 | }) 273 | .consumed(p.trigger.consumed) 274 | .error(cb); 275 | } 276 | else 277 | cb(null,item); 278 | } 279 | else 280 | if (item._txDeleted) 281 | cb(null); 282 | else 283 | cb(null,item); 284 | } 285 | else 286 | cb(null,item); 287 | }, 288 | _results= function (fn) 289 | { 290 | return function (cb,opts) 291 | { 292 | opts= opts || {}; 293 | 294 | var p= dyn.promise('end',null,['progress','consumed']), 295 | ctx= JSON.parse(JSON.stringify(dyn.ctx)), 296 | sync= dyn.syncResults(function (err) 297 | { 298 | if (err) 299 | p.trigger.error(err); 300 | else 301 | p.trigger.end(); 302 | }); 303 | 304 | _selectTxAttrs(opts); 305 | 306 | fn(sync.results(function (items,done) 307 | { 308 | async.forEach(_.keys(items), 309 | function (key,done) 310 | { 311 | var idx= +key, item= items[idx]; 312 | 313 | _getTxItem(item,_ctxId(ctx,item),p,function (err,item) 314 | { 315 | if (err) 316 | done(err); 317 | else 318 | { 319 | items[idx]= item; 320 | done(); 321 | } 322 | }); 323 | }, 324 | function (err) 325 | { 326 | if (err) 327 | done(err); 328 | else 329 | { 330 | cb(_.extend(_.filter(items,function (i) { return !!i; }),{ next: items.next })); 331 | done(); 332 | } 333 | }); 334 | }),opts) 335 | .consumed(p.trigger.consumed) 336 | .error(p.trigger.error) 337 | .end(sync.end); 338 | 339 | return p; 340 | }; 341 | }; 342 | 343 | dyn.put= function (obj,cb,opts) 344 | { 345 | var p= dyn.promise(null,['found','notfound','rolledback'],'consumed'), 346 | ctx= JSON.parse(JSON.stringify(dyn.ctx)), 347 | dynI= function () 348 | { 349 | return dyn.table(ctx.table) 350 | .hash(ctx.hash.attr,ctx.hash.value) 351 | .range(ctx.range.attr,ctx.range.value); 352 | }; 353 | 354 | _perform(dynI,ctx,p,obj,'put',function () 355 | { 356 | obj['_txApplied']= true; 357 | obj['_tx']= tx._id; 358 | opts.expected= opts.expected || {}; 359 | opts.expected['_tx']= tx._id; 360 | 361 | dynI() 362 | .putNT(obj,cb,opts) 363 | .chain(p); 364 | }); 365 | 366 | return p; 367 | }; 368 | 369 | dyn.updateItem= function (opts,cb) 370 | { 371 | var p= dyn.promise(null,['rolledback'],'consumed'), 372 | ctx= JSON.parse(JSON.stringify(dyn.ctx)), 373 | dynI= function () 374 | { 375 | return dyn.table(ctx.table) 376 | .hash(ctx.hash.attr,ctx.hash.value) 377 | .range(ctx.range.attr,ctx.range.value); 378 | }; 379 | 380 | _perform(dynI,ctx,p,opts.update,'updateItem',function () 381 | { 382 | opts.update['_txApplied']= { action: 'PUT', value: true }; 383 | opts.expected= opts.expected || {}; 384 | opts.expected['_tx']= tx._id; 385 | 386 | dynI() 387 | .updateItemNT(opts,cb) 388 | .chain(p); 389 | }); 390 | 391 | return p; 392 | }; 393 | 394 | dyn.delete= function (cb,opts) 395 | { 396 | var p= dyn.promise(null,['found','notfound','rolledback'],'consumed'), 397 | ctx= JSON.parse(JSON.stringify(dyn.ctx)), 398 | dynI= function () 399 | { 400 | return dyn.table(ctx.table) 401 | .hash(ctx.hash.attr,ctx.hash.value) 402 | .range(ctx.range.attr,ctx.range.value); 403 | }; 404 | 405 | _perform(dynI,ctx,p,{},'delete',function () 406 | { 407 | opts= opts || {}; 408 | opts.expected= opts.expected || {}; 409 | opts.expected['_tx']= tx._id; 410 | 411 | dynI() 412 | .updateItemNT({ update: { _txDeleted: { action: 'PUT', value: true } } }, 413 | function () { cb(); }) 414 | .chain(p); 415 | }); 416 | 417 | return p; 418 | }; 419 | 420 | dyn.get= function (cb,opts) 421 | { 422 | opts= opts || {}; 423 | 424 | var p= dyn.promise(null,['found','notfound','rolledback'],'consumed'), 425 | ctx= JSON.parse(JSON.stringify(dyn.ctx)), 426 | _id= _ctxId(ctx), 427 | dynI= function () 428 | { 429 | return dyn.table(ctx.table) 430 | .hash(ctx.hash.attr,ctx.hash.value) 431 | .range(ctx.range.attr,ctx.range.value); 432 | }; 433 | 434 | _selectTxAttrs(opts); 435 | 436 | if (opts.lock) 437 | _perform(dynI,ctx,p,{},'get',function (item) 438 | { 439 | cb(item); 440 | }); 441 | else 442 | dynI() 443 | .getNT(function (item) 444 | { 445 | _getTxItem(item,_id,p,function (err,item) 446 | { 447 | if (err) 448 | p.trigger.error(err); 449 | else 450 | if (item==undefined) 451 | p.trigger.notfound(); 452 | else 453 | cb(item); 454 | }); 455 | }, 456 | BECONSISTENT) 457 | .chain(p); 458 | 459 | return p; 460 | }; 461 | 462 | dyn.query= _results(_.bind(dyn.queryNT,dyn)); 463 | 464 | dyn.scan= _results(_.bind(dyn.scanNT,dyn)); 465 | 466 | }; 467 | -------------------------------------------------------------------------------- /test/transaction.test.js: -------------------------------------------------------------------------------- 1 | var chai = require('chai'), 2 | spies = require('chai-spies'), 3 | AWS = require('aws-sdk'), 4 | _= require('underscore'), 5 | dyngo= require('../index.js'); 6 | 7 | chai.use(spies); 8 | 9 | var should= chai.should(), 10 | assert= chai.assert; 11 | 12 | const noerr= function (done) 13 | { 14 | return function (err) 15 | { 16 | should.not.exist(err); 17 | done(); 18 | }; 19 | }, 20 | accept= function (code,done) 21 | { 22 | return function (err) 23 | { 24 | if (err.code==code) 25 | done(); 26 | else 27 | done(err); 28 | }; 29 | }; 30 | 31 | describe('transactions',function () 32 | { 33 | var db; 34 | 35 | before(function (done) 36 | { 37 | dyngo({ dynamo: { endpoint: new AWS.Endpoint('http://localhost:8000') }, hints: false }, 38 | function (err,_db) 39 | { 40 | db= _db; 41 | 42 | db.ensureTransactionTable().success(done).error(done); 43 | }); 44 | }); 45 | 46 | beforeEach(function (done) 47 | { 48 | db.test.remove().success(done) 49 | .error(done); 50 | }); 51 | 52 | describe('insert',function () 53 | { 54 | 55 | it('If transaction A inserts a new object transaction B cannot see it while not committed (transient)', 56 | function (done) 57 | { 58 | db.transaction().transaction(function (A) 59 | { 60 | db.transaction().transaction(function (B) 61 | { 62 | A.test.save({ _id: 'transient' }).success(function () 63 | { 64 | B.test.findOne({ _id: 'transient' }) 65 | .result(should.not.exist) 66 | .error(accept('notfound',done)); 67 | }).error(done); 68 | }).error(done); 69 | }).error(done); 70 | }); 71 | 72 | it('If transaction A inserts a new object, while there was a transient object already present, transaction B cannot see it while not committed (transient)', 73 | function (done) 74 | { 75 | db.transaction().transaction(function (A) 76 | { 77 | db.transaction().transaction(function (B) 78 | { 79 | A.test.save({ _id: 'transient2' }).success(function () 80 | { 81 | A.test.save({ _id: 'transient2' }).success(function () 82 | { 83 | B.test.findOne({ _id: 'transient2' }) 84 | .result(should.not.exist) 85 | .error(accept('notfound',done)); 86 | }).error(done); 87 | }).error(done); 88 | }).error(done); 89 | }).error(done); 90 | }); 91 | 92 | it('If transaction A inserts a new object, and commit, transaction B can see the new object', 93 | function (done) 94 | { 95 | db.transaction().transaction(function (A) 96 | { 97 | db.transaction().transaction(function (B) 98 | { 99 | A.test.save({ _id: 'transient3' }).success(function () 100 | { 101 | A.commit().committed(function () 102 | { 103 | B.test.findOne({ _id: 'transient3' }) 104 | .result(function (r) 105 | { 106 | should.exist(r); 107 | r._id.should.equal('transient3'); 108 | done(); 109 | }) 110 | .error(done); 111 | }).error(done); 112 | }).error(done); 113 | }).error(done); 114 | }).error(done); 115 | }); 116 | 117 | }); 118 | 119 | describe('delete',function () 120 | { 121 | 122 | it('If transaction A deletes an object and does not commit, transaction B can see the object, but transaction A can\'t', 123 | function (done) 124 | { 125 | db.test.save({ _id: 'delete1' }).success(function () 126 | { 127 | db.transaction().transaction(function (A) 128 | { 129 | db.transaction().transaction(function (B) 130 | { 131 | A.test.remove({ _id: 'delete1' }).success(function () 132 | { 133 | B.test.findOne({ _id: 'delete1' }) 134 | .result(function (r) 135 | { 136 | should.exist(r); 137 | r._id.should.equal('delete1'); 138 | 139 | A.test.findOne({ _id: 'delete1' }) 140 | .result(should.not.exist) 141 | .error(accept('notfound',done)); 142 | }) 143 | .error(done); 144 | 145 | }).error(done); 146 | }).error(done); 147 | }).error(done); 148 | }).error(done); 149 | }); 150 | 151 | it('If transaction A deletes an object and commit, transaction B cannot see the object', 152 | function (done) 153 | { 154 | db.test.save({ _id: 'delete2' }).success(function () 155 | { 156 | db.transaction().transaction(function (A) 157 | { 158 | db.transaction().transaction(function (B) 159 | { 160 | A.test.remove({ _id: 'delete2' }).success(function () 161 | { 162 | A.commit().committed(function () 163 | { 164 | B.test.findOne({ _id: 'delete2' }) 165 | .result(should.not.exist) 166 | .error(accept('notfound',done)); 167 | }).error(done); 168 | }).error(done); 169 | }).error(done); 170 | }).error(done); 171 | }).error(done); 172 | }); 173 | 174 | }); 175 | 176 | describe('update',function () 177 | { 178 | 179 | it('If transaction A updates an object and does not commit, transaction B still see the old version of the object, when A commits, B sees the new version', 180 | function (done) 181 | { 182 | var obj= { _id: 'update1', n: 0 }; 183 | 184 | db.test.save(obj).success(function () 185 | { 186 | db.transaction().transaction(function (A) 187 | { 188 | db.transaction().transaction(function (B) 189 | { 190 | obj.name= 'Update2'; 191 | obj.n++; 192 | 193 | A.test.save(obj).success(function () 194 | { 195 | B.test.findOne({ _id: 'update1' }) 196 | .result(function (copy) 197 | { 198 | should.not.exist(copy.name); 199 | copy.n.should.equal(0); 200 | 201 | A.test.findOne({ _id: 'update1' }) 202 | .result(function (copy) 203 | { 204 | copy.name.should.equal('Update2'); 205 | copy.n.should.equal(1); 206 | 207 | A.commit().committed(function () 208 | { 209 | B.test.findOne({ _id: 'update1' }) 210 | .result(function (copy) 211 | { 212 | copy.name.should.equal('Update2'); 213 | copy.n.should.equal(1); 214 | 215 | done(); 216 | }) 217 | .error(done); 218 | }) 219 | .error(done); 220 | }) 221 | .error(done); 222 | }) 223 | .error(done); 224 | }).error(done); 225 | }).error(done); 226 | }).error(done); 227 | }).error(done); 228 | }); 229 | 230 | it('If transaction A incs an object and does not commit, transaction B still see the old version of the object, when A commits, B sees the new version', 231 | function (done) 232 | { 233 | var obj= { _id: 'update2', n: 0 }; 234 | 235 | db.test.save(obj).success(function () 236 | { 237 | db.transaction().transaction(function (A) 238 | { 239 | db.transaction().transaction(function (B) 240 | { 241 | A.test.update({},{ $inc: { n: 1 } }).success(function () 242 | { 243 | B.test.findOne({ _id: 'update2' }) 244 | .result(function (copy) 245 | { 246 | copy.n.should.equal(0); 247 | 248 | A.test.findOne({ _id: 'update2' }) 249 | .result(function (copy) 250 | { 251 | copy.n.should.equal(1); 252 | 253 | A.commit().committed(function () 254 | { 255 | B.test.findOne({ _id: 'update2' }) 256 | .result(function (copy) 257 | { 258 | copy.n.should.equal(1); 259 | 260 | done(); 261 | }) 262 | .error(done); 263 | }) 264 | .error(done); 265 | }) 266 | .error(done); 267 | }) 268 | .error(done); 269 | }).error(done); 270 | }).error(done); 271 | }).error(done); 272 | }).error(done); 273 | }); 274 | 275 | it('When some object is updated more than one time the original copy is rolled back',function (done) 276 | { 277 | db.test.save({ _id: 'update-orig', n: 1 }) 278 | .success(function () 279 | { 280 | db.transaction().transaction(function (tx) 281 | { 282 | tx.test.findOne({ _id: 'update-orig' }) 283 | .result(function (obj) 284 | { 285 | obj.n++; 286 | tx.test.save(obj) 287 | .success(function () 288 | { 289 | obj.n++; 290 | tx.test.save(obj) 291 | .success(function () 292 | { 293 | tx.rollback().rolledback(function () 294 | { 295 | db.test.findOne({ _id: 'update-orig' }) 296 | .result(function (obj) 297 | { 298 | obj.n.should.equal(1); 299 | done(); 300 | }) 301 | .error(done); 302 | }) 303 | .error(done); 304 | }) 305 | .error(done); 306 | }) 307 | .error(done); 308 | }) 309 | .error(done); 310 | }) 311 | .error(done); 312 | }) 313 | .error(done); 314 | }); 315 | }); 316 | 317 | describe('query',function () 318 | { 319 | it('If transaction A inserts an item without committing, transaction B should not see the inserted item until commited', 320 | function (done) 321 | { 322 | var items= _.collect(_.range(10),function (n) { return { _id: 'item'+n, n: n+1 } }); 323 | 324 | db.test.save(items).success(function () 325 | { 326 | db.transaction().transaction(function (A) 327 | { 328 | db.transaction().transaction(function (B) 329 | { 330 | A.test.save({ _id: 'itemS', n: 0 }).success(function () 331 | { 332 | B.test.find().sort({ n: 1 }).limit(3) 333 | .results(function (objs) 334 | { 335 | objs[0].n.should.equal(1); 336 | objs.length.should.equal(3); 337 | }) 338 | .error(done) 339 | .end(function () 340 | { 341 | A.test.find().sort({ n: 1 }).limit(3) 342 | .results(function (objs) 343 | { 344 | objs[0].n.should.equal(0); 345 | objs.length.should.equal(3); 346 | }) 347 | .error(done) 348 | .end(function () 349 | { 350 | A.commit().committed(function () 351 | { 352 | B.test.find().sort({ n: 1 }).limit(3) 353 | .results(function (objs) 354 | { 355 | objs[0].n.should.equal(0); 356 | objs.length.should.equal(3); 357 | }) 358 | .error(done) 359 | .end(done); 360 | }); 361 | }); 362 | }); 363 | }).error(done); 364 | }); 365 | }); 366 | }); 367 | }); 368 | }); 369 | 370 | describe('concurrency',function () 371 | { 372 | it('rollsback competing transaction', 373 | function (done) 374 | { 375 | db.test.save({ _id: 'hot', name: 'Hot' }).success(function () 376 | { 377 | db.transaction().transaction(function (A) 378 | { 379 | db.transaction().transaction(function (B) 380 | { 381 | A.test.findOne({ _id: 'hot' }) 382 | .result(function (hotA) 383 | { 384 | hotA.name= 'HotA'; 385 | 386 | B.test.findOne({ _id: 'hot' }) 387 | .result(function (hotB) 388 | { 389 | hotB.name= 'HotB'; 390 | 391 | A.test.save(hotA).success(function () 392 | { 393 | B.test.save(hotB).success(function () 394 | { 395 | var committedA= chai.spy(); 396 | 397 | A.commit() 398 | .committed(committedA) 399 | .error(function (err) 400 | { 401 | if (err.code=='rolledback') 402 | B.commit().committed(function () 403 | { 404 | db.test.findOne({ _id: 'hot' }) 405 | .result(function (obj) 406 | { 407 | committedA.should.not.have.been.called(); 408 | obj.name.should.equal('HotB'); 409 | done(); 410 | }) 411 | .error(done); 412 | }) 413 | .error(done); 414 | else 415 | done(err); 416 | }); 417 | }) 418 | .error(done); 419 | }) 420 | .error(done); 421 | }) 422 | .error(done); 423 | }) 424 | .error(done); 425 | 426 | }).error(done); 427 | }).error(done); 428 | }).error(done); 429 | }); 430 | }); 431 | }); 432 | -------------------------------------------------------------------------------- /test/sample.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "Minerva Stewart", 4 | "company": "Ultricies Sem Magna Institute", 5 | "phone": "04 66 53 61 19", 6 | "about": "eu," 7 | }, 8 | { 9 | "name": "Hedley Booth", 10 | "company": "Lobortis Company", 11 | "phone": "08 32 96 63 10", 12 | "about": "non, luctus sit amet, faucibus" 13 | }, 14 | { 15 | "name": "Duncan Wall", 16 | "company": "Adipiscing Fringilla Inc.", 17 | "phone": "06 84 05 84 87", 18 | "about": "iaculis odio. Nam interdum enim non" 19 | }, 20 | { 21 | "name": "Charity Sweeney", 22 | "company": "Mollis Nec Foundation", 23 | "phone": "03 14 06 72 83", 24 | "about": "augue, eu tempor erat neque" 25 | }, 26 | { 27 | "name": "Evangeline Sweet", 28 | "company": "Leo Cras Vehicula Company", 29 | "phone": "09 52 86 44 49", 30 | "about": "mauris erat eget ipsum. Suspendisse sagittis. Nullam vitae diam. Proin" 31 | }, 32 | { 33 | "name": "Stone Barker", 34 | "company": "Justo Nec Incorporated", 35 | "phone": "05 18 90 14 73", 36 | "about": "Nam consequat dolor vitae dolor. Donec fringilla. Donec feugiat" 37 | }, 38 | { 39 | "name": "Minerva Pena", 40 | "company": "Ridiculus Mus Incorporated", 41 | "phone": "02 31 12 32 65", 42 | "about": "pellentesque massa lobortis" 43 | }, 44 | { 45 | "name": "Buckminster Holmes", 46 | "company": "Magna Cras LLC", 47 | "phone": "08 39 09 36 81", 48 | "about": "ut, sem. Nulla" 49 | }, 50 | { 51 | "name": "Yeo Becker", 52 | "company": "Dignissim Magna Industries", 53 | "phone": "04 03 04 55 43", 54 | "about": "ac turpis egestas. Fusce aliquet magna a" 55 | }, 56 | { 57 | "name": "Jonas Meyers", 58 | "company": "Est LLC", 59 | "phone": "06 07 39 64 38", 60 | "about": "adipiscing elit. Curabitur sed tortor." 61 | }, 62 | { 63 | "name": "Ariel Kaufman", 64 | "company": "In Hendrerit Consectetuer Associates", 65 | "phone": "09 62 42 28 35", 66 | "about": "dui. Cras pellentesque." 67 | }, 68 | { 69 | "name": "Pascale Aguirre", 70 | "company": "Erat Institute", 71 | "phone": "08 69 15 33 69", 72 | "about": "sit amet diam" 73 | }, 74 | { 75 | "name": "Adrian Stein", 76 | "company": "Lorem Ac Risus Industries", 77 | "phone": "01 59 89 07 06", 78 | "about": "sollicitudin commodo ipsum. Suspendisse non leo. Vivamus nibh dolor, nonummy" 79 | }, 80 | { 81 | "name": "Theodore Frazier", 82 | "company": "Mus Aenean Industries", 83 | "phone": "03 61 89 79 28", 84 | "about": "auctor" 85 | }, 86 | { 87 | "name": "Shelby Thompson", 88 | "company": "Egestas A Dui PC", 89 | "phone": "05 89 98 02 49", 90 | "about": "id" 91 | }, 92 | { 93 | "name": "Kirestin Morris", 94 | "company": "Egestas Industries", 95 | "phone": "09 28 77 41 04", 96 | "about": "magna. Phasellus dolor elit, pellentesque a, facilisis non, bibendum sed," 97 | }, 98 | { 99 | "name": "Rhonda Wright", 100 | "company": "Dolor Nulla Semper Institute", 101 | "phone": "03 47 16 23 64", 102 | "about": "at arcu. Vestibulum ante ipsum" 103 | }, 104 | { 105 | "name": "Whitney Hayden", 106 | "company": "Vivamus Non LLC", 107 | "phone": "05 74 16 03 72", 108 | "about": "porttitor scelerisque neque." 109 | }, 110 | { 111 | "name": "Kelsey Chandler", 112 | "company": "Vivamus Euismod Urna Corp.", 113 | "phone": "03 95 34 56 17", 114 | "about": "nunc. In" 115 | }, 116 | { 117 | "name": "Alexa Macdonald", 118 | "company": "Nunc Ullamcorper Velit Inc.", 119 | "phone": "03 86 42 90 91", 120 | "about": "lobortis ultrices." 121 | }, 122 | { 123 | "name": "Jemima Hess", 124 | "company": "Luctus Corp.", 125 | "phone": "05 68 88 28 06", 126 | "about": "quam" 127 | }, 128 | { 129 | "name": "Veda Hughes", 130 | "company": "Urna Vivamus Corporation", 131 | "phone": "02 30 03 31 77", 132 | "about": "libero est, congue a, aliquet vel, vulputate" 133 | }, 134 | { 135 | "name": "Calista May", 136 | "company": "Mauris Eu Consulting", 137 | "phone": "07 01 94 35 98", 138 | "about": "malesuada vel, venenatis vel," 139 | }, 140 | { 141 | "name": "Lana Tucker", 142 | "company": "Elit Sed Consequat Corporation", 143 | "phone": "05 98 29 15 47", 144 | "about": "Duis sit amet diam eu dolor egestas rhoncus. Proin" 145 | }, 146 | { 147 | "name": "Savannah Pena", 148 | "company": "Consequat Auctor Nunc Foundation", 149 | "phone": "04 30 57 65 16", 150 | "about": "diam nunc, ullamcorper eu," 151 | }, 152 | { 153 | "name": "Knox Case", 154 | "company": "Vel Nisl LLP", 155 | "phone": "02 35 18 80 86", 156 | "about": "metus eu erat semper" 157 | }, 158 | { 159 | "name": "Gannon Spencer", 160 | "company": "Proin Nisl LLP", 161 | "phone": "05 04 40 28 38", 162 | "about": "arcu iaculis enim, sit amet ornare lectus justo eu arcu." 163 | }, 164 | { 165 | "name": "Orson Rivers", 166 | "company": "Fusce Dolor Incorporated", 167 | "phone": "07 87 65 19 17", 168 | "about": "sed, facilisis vitae," 169 | }, 170 | { 171 | "name": "Emi Vaughn", 172 | "company": "Sed Diam Lorem Foundation", 173 | "phone": "09 72 58 91 84", 174 | "about": "Cum sociis natoque penatibus et magnis dis parturient montes, nascetur" 175 | }, 176 | { 177 | "name": "Skyler Lindsay", 178 | "company": "Rutrum LLC", 179 | "phone": "02 33 76 61 95", 180 | "about": "felis ullamcorper viverra. Maecenas iaculis" 181 | }, 182 | { 183 | "name": "Maxwell Ramos", 184 | "company": "Erat Corporation", 185 | "phone": "04 81 46 30 39", 186 | "about": "aliquam, enim nec tempus scelerisque, lorem ipsum" 187 | }, 188 | { 189 | "name": "Galena Pope", 190 | "company": "Vitae Sodales LLC", 191 | "phone": "08 01 28 08 39", 192 | "about": "non," 193 | }, 194 | { 195 | "name": "Armando Peck", 196 | "company": "Quisque Corp.", 197 | "phone": "08 95 97 29 98", 198 | "about": "Mauris eu turpis." 199 | }, 200 | { 201 | "name": "Forrest Wade", 202 | "company": "Lectus Inc.", 203 | "phone": "01 09 38 69 85", 204 | "about": "felis. Nulla tempor augue ac ipsum. Phasellus vitae mauris sit" 205 | }, 206 | { 207 | "name": "Orson Shaffer", 208 | "company": "Eu Euismod LLP", 209 | "phone": "08 27 05 19 37", 210 | "about": "tincidunt tempus" 211 | }, 212 | { 213 | "name": "Kendall Sears", 214 | "company": "Nunc Sed Pede Foundation", 215 | "phone": "09 37 02 03 65", 216 | "about": "Nullam nisl. Maecenas malesuada fringilla" 217 | }, 218 | { 219 | "name": "Germane Morse", 220 | "company": "Primis In Faucibus Corporation", 221 | "phone": "07 21 95 52 92", 222 | "about": "ultrices. Duis volutpat nunc sit amet metus. Aliquam erat" 223 | }, 224 | { 225 | "name": "Lynn Oneill", 226 | "company": "At Auctor Ullamcorper Foundation", 227 | "phone": "02 69 84 81 14", 228 | "about": "nisl elementum purus, accumsan interdum" 229 | }, 230 | { 231 | "name": "Sean Sexton", 232 | "company": "Ornare Libero Corp.", 233 | "phone": "01 21 20 56 06", 234 | "about": "mauris. Integer" 235 | }, 236 | { 237 | "name": "Meghan Alvarez", 238 | "company": "Tristique Senectus Industries", 239 | "phone": "05 61 24 79 46", 240 | "about": "eros nec tellus. Nunc lectus pede, ultrices a, auctor non," 241 | }, 242 | { 243 | "name": "Katelyn Ramsey", 244 | "company": "Ut Nisi Limited", 245 | "phone": "01 55 58 96 21", 246 | "about": "nisi magna sed dui. Fusce aliquam," 247 | }, 248 | { 249 | "name": "Sylvester Schwartz", 250 | "company": "Sem Nulla Interdum LLC", 251 | "phone": "08 25 94 75 30", 252 | "about": "sagittis. Duis gravida. Praesent eu nulla at sem molestie sodales." 253 | }, 254 | { 255 | "name": "Zelenia Flynn", 256 | "company": "Leo Company", 257 | "phone": "07 15 14 07 84", 258 | "about": "massa. Integer vitae nibh. Donec est mauris, rhoncus" 259 | }, 260 | { 261 | "name": "Jenette Fuller", 262 | "company": "Eleifend LLP", 263 | "phone": "08 28 71 34 61", 264 | "about": "nulla. Integer vulputate, risus a ultricies adipiscing," 265 | }, 266 | { 267 | "name": "Dillon Robertson", 268 | "company": "Nec Tempus Mauris LLC", 269 | "phone": "07 18 04 05 62", 270 | "about": "dapibus ligula. Aliquam erat volutpat. Nulla dignissim. Maecenas ornare" 271 | }, 272 | { 273 | "name": "Ila Mcintyre", 274 | "company": "Donec Corporation", 275 | "phone": "09 96 16 68 68", 276 | "about": "enim nec tempus scelerisque, lorem ipsum" 277 | }, 278 | { 279 | "name": "Petra Stein", 280 | "company": "Malesuada Augue Ut Limited", 281 | "phone": "09 18 80 91 53", 282 | "about": "rhoncus." 283 | }, 284 | { 285 | "name": "Burton Nolan", 286 | "company": "Sed Limited", 287 | "phone": "01 96 59 42 06", 288 | "about": "nunc sit amet metus. Aliquam erat volutpat. Nulla" 289 | }, 290 | { 291 | "name": "Hamish Castaneda", 292 | "company": "Mauris Id Sapien Consulting", 293 | "phone": "04 03 87 43 84", 294 | "about": "ornare, lectus ante dictum" 295 | }, 296 | { 297 | "name": "Eric Mullen", 298 | "company": "Tempus Mauris Erat LLC", 299 | "phone": "06 21 33 85 75", 300 | "about": "Suspendisse commodo tincidunt nibh. Phasellus nulla." 301 | }, 302 | { 303 | "name": "Kirk Kirk", 304 | "company": "Sociis Natoque LLC", 305 | "phone": "03 72 07 61 84", 306 | "about": "viverra. Maecenas iaculis aliquet diam. Sed diam" 307 | }, 308 | { 309 | "name": "Colby Christensen", 310 | "company": "In Ornare Sagittis Corporation", 311 | "phone": "06 96 96 77 67", 312 | "about": "cursus" 313 | }, 314 | { 315 | "name": "Ursa Gilliam", 316 | "company": "Tortor Integer Aliquam Ltd", 317 | "phone": "01 77 54 43 17", 318 | "about": "iaculis enim, sit amet ornare" 319 | }, 320 | { 321 | "name": "Karina Little", 322 | "company": "Vivamus Corp.", 323 | "phone": "07 66 14 34 05", 324 | "about": "id, ante. Nunc mauris sapien, cursus" 325 | }, 326 | { 327 | "name": "Jorden Kennedy", 328 | "company": "Sit Ltd", 329 | "phone": "05 93 80 26 29", 330 | "about": "tristique ac, eleifend vitae, erat. Vivamus nisi. Mauris" 331 | }, 332 | { 333 | "name": "Grace Park", 334 | "company": "Aliquam Foundation", 335 | "phone": "05 55 91 64 70", 336 | "about": "et" 337 | }, 338 | { 339 | "name": "Indira Dawson", 340 | "company": "Gravida Foundation", 341 | "phone": "09 09 67 28 20", 342 | "about": "egestas ligula. Nullam feugiat placerat velit." 343 | }, 344 | { 345 | "name": "Wayne David", 346 | "company": "Amet Diam Eu Incorporated", 347 | "phone": "03 40 68 80 76", 348 | "about": "et nunc. Quisque ornare tortor at risus. Nunc ac sem" 349 | }, 350 | { 351 | "name": "Whitney Lane", 352 | "company": "Nec Ligula Consectetuer LLP", 353 | "phone": "03 13 16 98 85", 354 | "about": "hendrerit consectetuer," 355 | }, 356 | { 357 | "name": "Todd Lott", 358 | "company": "Cras Convallis Corp.", 359 | "phone": "05 32 65 89 21", 360 | "about": "a purus." 361 | }, 362 | { 363 | "name": "James Sharpe", 364 | "company": "Adipiscing Lobortis Risus Institute", 365 | "phone": "01 94 29 95 23", 366 | "about": "sodales at, velit. Pellentesque ultricies dignissim lacus. Aliquam rutrum lorem" 367 | }, 368 | { 369 | "name": "Wylie Sherman", 370 | "company": "Et LLP", 371 | "phone": "08 43 19 79 69", 372 | "about": "vestibulum massa rutrum magna. Cras" 373 | }, 374 | { 375 | "name": "Tate Vaughn", 376 | "company": "Sed Eu Eros Company", 377 | "phone": "01 72 03 05 35", 378 | "about": "sapien. Cras dolor dolor, tempus non," 379 | }, 380 | { 381 | "name": "Noah Barber", 382 | "company": "Quis Corporation", 383 | "phone": "03 01 36 70 82", 384 | "about": "mollis dui, in sodales elit erat vitae risus." 385 | }, 386 | { 387 | "name": "Rudyard Hammond", 388 | "company": "Et Netus Et Corp.", 389 | "phone": "03 24 03 38 49", 390 | "about": "felis. Nulla tempor" 391 | }, 392 | { 393 | "name": "Cara Hanson", 394 | "company": "Sapien Gravida Non Institute", 395 | "phone": "08 56 72 67 77", 396 | "about": "arcu. Sed et" 397 | }, 398 | { 399 | "name": "Yoshio Valentine", 400 | "company": "Eu Erat Consulting", 401 | "phone": "09 20 36 44 22", 402 | "about": "cubilia Curae; Donec tincidunt. Donec vitae erat vel" 403 | }, 404 | { 405 | "name": "Shelly Garza", 406 | "company": "Orci Corp.", 407 | "phone": "07 79 23 63 21", 408 | "about": "urna, nec luctus felis purus ac tellus. Suspendisse" 409 | }, 410 | { 411 | "name": "Mia Guy", 412 | "company": "Sagittis Felis Donec Incorporated", 413 | "phone": "06 21 99 45 71", 414 | "about": "semper egestas, urna justo faucibus lectus, a sollicitudin orci sem" 415 | }, 416 | { 417 | "name": "Faith Rivers", 418 | "company": "Quis Massa Industries", 419 | "phone": "04 24 64 12 32", 420 | "about": "vitae" 421 | }, 422 | { 423 | "name": "Whitney Mclaughlin", 424 | "company": "Nulla Consulting", 425 | "phone": "05 68 91 73 69", 426 | "about": "metus urna" 427 | }, 428 | { 429 | "name": "Darius Graham", 430 | "company": "Ut Incorporated", 431 | "phone": "07 07 73 29 75", 432 | "about": "lacus pede sagittis augue, eu tempor erat neque" 433 | }, 434 | { 435 | "name": "Alfreda Joyner", 436 | "company": "Neque Nullam Corporation", 437 | "phone": "01 63 84 66 24", 438 | "about": "elit, dictum eu, eleifend nec, malesuada ut, sem. Nulla" 439 | }, 440 | { 441 | "name": "Alexander Abbott", 442 | "company": "Fringilla Ltd", 443 | "phone": "06 93 95 28 92", 444 | "about": "semper pretium neque. Morbi quis urna. Nunc quis arcu" 445 | }, 446 | { 447 | "name": "Alice Valencia", 448 | "company": "Amet Nulla Foundation", 449 | "phone": "03 60 71 75 34", 450 | "about": "tortor," 451 | }, 452 | { 453 | "name": "Mechelle Walker", 454 | "company": "Non Cursus Non Inc.", 455 | "phone": "08 36 76 58 28", 456 | "about": "sem, consequat nec, mollis" 457 | }, 458 | { 459 | "name": "Chiquita Holder", 460 | "company": "Eget Lacus LLC", 461 | "phone": "07 74 64 28 48", 462 | "about": "lacinia vitae, sodales at, velit. Pellentesque ultricies dignissim" 463 | }, 464 | { 465 | "name": "Merrill Duncan", 466 | "company": "Blandit Enim Ltd", 467 | "phone": "06 41 77 84 46", 468 | "about": "ac" 469 | }, 470 | { 471 | "name": "Germane Hutchinson", 472 | "company": "Tristique Pellentesque Tellus LLC", 473 | "phone": "02 16 63 50 14", 474 | "about": "pede. Cum sociis natoque penatibus et" 475 | }, 476 | { 477 | "name": "Mufutau Powers", 478 | "company": "A Auctor Non Corporation", 479 | "phone": "07 33 35 86 61", 480 | "about": "auctor odio a" 481 | }, 482 | { 483 | "name": "Hamilton Weiss", 484 | "company": "Ac Turpis Egestas Incorporated", 485 | "phone": "06 44 73 35 50", 486 | "about": "ut" 487 | }, 488 | { 489 | "name": "Baxter Bright", 490 | "company": "Turpis Nec Associates", 491 | "phone": "09 39 36 62 84", 492 | "about": "diam lorem, auctor quis, tristique ac, eleifend vitae," 493 | }, 494 | { 495 | "name": "Madeson Brennan", 496 | "company": "Commodo Ltd", 497 | "phone": "01 15 84 57 69", 498 | "about": "at lacus. Quisque purus sapien, gravida non, sollicitudin a, malesuada" 499 | }, 500 | { 501 | "name": "Constance Palmer", 502 | "company": "Eu Sem Pellentesque Corporation", 503 | "phone": "01 11 35 02 60", 504 | "about": "arcu. Vestibulum ante" 505 | }, 506 | { 507 | "name": "Regan Wolfe", 508 | "company": "Consectetuer Rhoncus Nullam LLC", 509 | "phone": "03 74 59 98 72", 510 | "about": "eget laoreet posuere, enim nisl elementum" 511 | }, 512 | { 513 | "name": "Maggie Waller", 514 | "company": "Eu Incorporated", 515 | "phone": "05 06 50 58 86", 516 | "about": "Cras dictum ultricies ligula. Nullam enim. Sed nulla" 517 | }, 518 | { 519 | "name": "Sydney Horton", 520 | "company": "Est LLC", 521 | "phone": "02 30 14 48 85", 522 | "about": "mauris sapien, cursus in, hendrerit consectetuer," 523 | }, 524 | { 525 | "name": "Coby Lara", 526 | "company": "Duis Gravida Corporation", 527 | "phone": "06 11 06 51 48", 528 | "about": "Curabitur massa. Vestibulum accumsan neque et nunc." 529 | }, 530 | { 531 | "name": "Yeo Chang", 532 | "company": "Tincidunt Nunc Ac LLP", 533 | "phone": "09 97 21 89 28", 534 | "about": "diam nunc, ullamcorper eu, euismod" 535 | }, 536 | { 537 | "name": "Medge Espinoza", 538 | "company": "Commodo At Industries", 539 | "phone": "07 83 60 59 06", 540 | "about": "nonummy ac, feugiat" 541 | }, 542 | { 543 | "name": "Callum Fry", 544 | "company": "Luctus Industries", 545 | "phone": "01 16 76 76 45", 546 | "about": "felis, adipiscing fringilla, porttitor" 547 | }, 548 | { 549 | "name": "Jena Owens", 550 | "company": "Mauris Integer Sem Company", 551 | "phone": "02 14 66 53 60", 552 | "about": "cursus luctus, ipsum leo elementum sem, vitae aliquam" 553 | }, 554 | { 555 | "name": "Edan Frost", 556 | "company": "Sem Molestie Sodales LLP", 557 | "phone": "07 08 89 91 24", 558 | "about": "Donec est mauris," 559 | }, 560 | { 561 | "name": "Naomi Davidson", 562 | "company": "Sed LLP", 563 | "phone": "06 09 46 83 97", 564 | "about": "ante blandit viverra. Donec tempus, lorem fringilla ornare" 565 | }, 566 | { 567 | "name": "Brenna Michael", 568 | "company": "Accumsan Neque Et Corp.", 569 | "phone": "04 81 58 31 88", 570 | "about": "vehicula. Pellentesque tincidunt tempus" 571 | }, 572 | { 573 | "name": "Hermione Rowe", 574 | "company": "Enim Nec Company", 575 | "phone": "03 99 78 61 07", 576 | "about": "interdum. Sed auctor odio" 577 | }, 578 | { 579 | "name": "Amena Ortega", 580 | "company": "Fusce Feugiat Lorem Associates", 581 | "phone": "02 84 50 84 15", 582 | "about": "dui augue eu tellus. Phasellus elit pede, malesuada vel, venenatis" 583 | }, 584 | { 585 | "name": "Clinton Jacobs", 586 | "company": "Nascetur Ridiculus Company", 587 | "phone": "03 25 10 93 71", 588 | "about": "dignissim lacus. Aliquam rutrum" 589 | }, 590 | { 591 | "name": "Hyatt Little", 592 | "company": "Cras Interdum Company", 593 | "phone": "06 33 89 72 81", 594 | "about": "mauris. Morbi non sapien molestie orci" 595 | }, 596 | { 597 | "name": "Noelle Slater", 598 | "company": "Penatibus LLP", 599 | "phone": "02 87 14 82 00", 600 | "about": "nonummy. Fusce fermentum" 601 | } 602 | ] 603 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/node 2 | 3 | const NOPE= function (){}; 4 | 5 | var dyngo= require('./index'), 6 | async= require('async'), 7 | fs= require('fs'), 8 | csv = require('csv'), 9 | xlsx = require('./lib/xlsx'), 10 | carrier= require('carrier'), 11 | util= require('util'), 12 | readline= require('readline'), 13 | _= require('underscore'), 14 | path= require('path').join, 15 | colors = require('colors'), 16 | GSON = require('gson'), 17 | coffee= require('coffee-script'), 18 | AWS = require('aws-sdk'); 19 | 20 | var argv = require('optimist').argv; 21 | 22 | var _history= []; 23 | 24 | const _json= function (path,content) 25 | { 26 | try 27 | { 28 | if (!content) 29 | return JSON.parse(fs.readFileSync(path,'utf8')); 30 | else 31 | { 32 | fs.writeFileSync(path,JSON.stringify(content,null,2),'utf8') 33 | return { success: function (fn) { process.nextTick(fn); } }; 34 | } 35 | } 36 | catch (ex) 37 | { 38 | console.log((ex+'').red); 39 | } 40 | }, 41 | _gson= function (path,content) 42 | { 43 | try 44 | { 45 | if (!content) 46 | return GSON.parse(fs.readFileSync(path,'utf8')); 47 | else 48 | { 49 | fs.writeFileSync(path,GSON.stringify(content),'utf8') 50 | return { success: function (fn) { process.nextTick(fn); } }; 51 | } 52 | } 53 | catch (ex) 54 | { 55 | console.log((ex+'').red); 56 | } 57 | }, 58 | _toJSON= function (fields,_transformFnc) 59 | { 60 | var r= []; 61 | 62 | this.forEach(function (row,idx) 63 | { 64 | var obj= {}, 65 | _val= function (field,value) 66 | { 67 | var r, 68 | path= field.split('.'), 69 | current= obj; 70 | 71 | for (var i=0;i '+cmd+')')(global,db,tx,fs,last,_,_json,_gson,__csv,_xlsx,argv); 198 | else 199 | throw ex; 200 | } 201 | }, 202 | _dobatch= function (db,lines,done) 203 | { 204 | var last, tx; 205 | 206 | return function () 207 | { 208 | async.forEachSeries(lines, 209 | function (cmd,done) 210 | { 211 | console.log(cmd); 212 | 213 | var promise= _eval(cmd,db,last,tx); 214 | 215 | if (promise==undefined||!(promise.result||promise.success||promise.error||promise.notfound)) 216 | done(); 217 | else 218 | { 219 | if (promise.notfound) 220 | promise.notfound(function () 221 | { 222 | last= undefined; 223 | done(); 224 | }); 225 | else 226 | if (promise.error) 227 | promise.error(function (err) 228 | { 229 | console.log((err+'').red,err.stack); 230 | done(); 231 | }); 232 | 233 | if (promise.transaction) 234 | promise.transaction(function (_tx) 235 | { 236 | tx= _tx; 237 | console.log(('tx: '+tx._id).green); 238 | done(); 239 | }); 240 | else 241 | if (promise.result) 242 | promise.result(function (res) 243 | { 244 | last= res; 245 | done(); 246 | }); 247 | else 248 | if (promise.success) 249 | promise.success(function () 250 | { 251 | console.log('done!'.green); 252 | done(); 253 | }); 254 | } 255 | 256 | 257 | },done); 258 | }; 259 | }, 260 | _doinput= function (db,cb) 261 | { 262 | var _lines= []; 263 | 264 | carrier.carry(process.stdin, function (line) 265 | { 266 | _lines.push(line); 267 | },'utf8'); 268 | 269 | process.stdin.on('end',function () 270 | { 271 | process.nextTick(function () 272 | { 273 | _dobatch(db,_lines, 274 | function (err) 275 | { 276 | if (err) 277 | { 278 | console.log(err.message.red,err.stack); 279 | process.exit(1); 280 | } 281 | else 282 | process.exit(0); 283 | })(); 284 | }); 285 | }); 286 | 287 | process.stdin.resume(); 288 | 289 | // if we have no input go to interactive mode 290 | setTimeout(function () { if (_lines.length==0) cb(); },100); 291 | }, 292 | _dorc= function (db,cb) 293 | { 294 | var rcFile= path(getUserHome(),'.dyngorc'), 295 | localRcFile= '.dyngorc', 296 | _file= function (f, done) 297 | { 298 | var _lines= []; 299 | 300 | if (fs.existsSync(f)) 301 | { 302 | console.log(('executing '+f+'...').green); 303 | var rstream= fs.createReadStream(f, { encoding: 'utf8' }); 304 | 305 | rstream.on('end',function () 306 | { 307 | process.nextTick(function () 308 | { 309 | _dobatch(db,_lines,done)(); 310 | }); 311 | }); 312 | 313 | carrier.carry(rstream,function (line) 314 | { 315 | _lines.push(line); 316 | }); 317 | } 318 | else 319 | done(); 320 | }; 321 | 322 | _file(rcFile,function (err) 323 | { 324 | if (err) 325 | { 326 | console.log(err.message.red,err.stack); 327 | process.exit(1); 328 | } 329 | else 330 | if (localRcFile!=rcFile) 331 | _file(localRcFile,function (err) 332 | { 333 | if (err) 334 | { 335 | console.log(err.message.red,err.stack); 336 | process.exit(1); 337 | } 338 | else 339 | cb(); 340 | }); 341 | else 342 | cb(); 343 | }); 344 | }, 345 | getUserHome= function() 346 | { 347 | return process.env[(process.platform == 'win32') ? 'USERPROFILE' : 'HOME']; 348 | }, 349 | getHistory= function() 350 | { 351 | var historyFile= path(getUserHome(),'.dyngodb_history'); 352 | 353 | try 354 | { 355 | 356 | if (fs.existsSync(historyFile)) 357 | _history.push 358 | .apply(_history,JSON.parse(fs.readFileSync(historyFile,'utf8'))); 359 | 360 | } 361 | catch(ex) 362 | {} 363 | 364 | return _history; 365 | }, 366 | saveHistory= function () 367 | { 368 | var historyFile= path(getUserHome(),'.dyngodb_history'); 369 | 370 | if (_history&&_history.length>0) 371 | fs.writeFileSync(historyFile,JSON.stringify(_history),'utf8'); 372 | }, 373 | _collect= function (consume) 374 | { 375 | return function (cons) 376 | { 377 | _.keys(cons).forEach(function (table) 378 | { 379 | var c, tcons= cons[table]; 380 | 381 | if (!(c=consume[table])) 382 | c= consume[table]= { read: 0, write: 0 }; 383 | 384 | c.read+= tcons.read; 385 | c.write+= tcons.write; 386 | }); 387 | }; 388 | }; 389 | 390 | process.on('exit', saveHistory); 391 | process.on('SIGINT', function () { saveHistory(); process.exit(0); }); 392 | process.stdin.pause(); 393 | 394 | var args= [function (err,db) 395 | { 396 | var last, tx; 397 | 398 | if (err) 399 | console.log(err); 400 | else 401 | { 402 | 403 | _dorc(db,function () 404 | { 405 | _doinput(db,function() 406 | { 407 | var rl = readline.createInterface 408 | ({ 409 | input: process.stdin, 410 | output: process.stdout, 411 | completer: function (linePartial, cb) 412 | { 413 | if (linePartial.indexOf('db.')==0) 414 | { 415 | var tables= _.collect(_.filter(_.keys(db), 416 | function (key) { return key.indexOf(linePartial.replace('db.',''))==0; }), 417 | function (res) { return 'db.'+res; }); 418 | cb(null,[tables, linePartial]); 419 | } 420 | else 421 | cb(null,[[], linePartial]); 422 | } 423 | }); 424 | 425 | rl.history= getHistory(); 426 | 427 | (function ask() 428 | { 429 | var _ask= function (fn) 430 | { 431 | return function () 432 | { 433 | var args= arguments; 434 | fn.apply(null,args); 435 | ask(); 436 | }; 437 | }, 438 | _print= function (obj,cb) 439 | { 440 | if (obj._old||(obj[0]&&obj[0]._old)) 441 | db.cleanup(obj).clean(function (obj) 442 | { 443 | console.log(util.inspect(obj,{ depth: null })); 444 | cb(); 445 | }); 446 | else 447 | { 448 | console.log(util.inspect(obj,{ depth: null })); 449 | cb(); 450 | } 451 | }; 452 | 453 | rl.question('> ', function (answer) 454 | { 455 | 456 | if (!answer) { ask(); return; }; 457 | 458 | if (answer.indexOf('clean ')==0) 459 | { 460 | var target= answer.substring(6); 461 | 462 | answer= 'db.cleanup('+target+')'; 463 | } 464 | 465 | if (answer.indexOf('show collections') > -1) 466 | { 467 | _.filter(_.keys(db),function (key) { return !!db[key]&&!!db[key].find; }).forEach(function (c) { console.log(c); }); 468 | ask(); 469 | return; 470 | } 471 | else 472 | if (answer=='clear') 473 | { 474 | process.stdout.write('\u001B[2J\u001B[0;0f'); 475 | ask(); 476 | return; 477 | } 478 | else 479 | if (answer=='exit') 480 | { 481 | process.exit(0); 482 | return; 483 | } 484 | 485 | try 486 | { 487 | var time= process.hrtime(), 488 | promise= _eval(answer,db,last,tx), 489 | end, 490 | printed, 491 | chunks= 0, 492 | consume= { read: 0, write: 0 }, 493 | _doneres= function () { elapsed(); ask(); }, 494 | doneres= _.wrap(_doneres,function (done) { if (printed&&end) done(); }), 495 | elapsed= function () 496 | { 497 | var diff= process.hrtime(time), 498 | secs= (diff[0]*1e9+diff[1])/1e9; 499 | 500 | _.keys(consume).forEach(function (table) 501 | { 502 | var tcons= consume[table], s= secs<1 ? 1 : secs; 503 | 504 | if (tcons.read) 505 | console.log(('consumed read capacity['+table+']: '+tcons.read+' ('+(tcons.read/s)+' read/sec)').green); 506 | 507 | if (tcons.write) 508 | console.log(('consumed write capacity['+table+']: '+tcons.write+' ('+(tcons.write/s)+' write/sec)').green); 509 | }); 510 | 511 | if (chunks) console.log((chunks+' roundtrips').green); 512 | console.log((secs+' secs').green); 513 | }; 514 | 515 | if (promise==_||promise===false||promise===undefined||promise.createCollection) 516 | { 517 | _ask(function () { console.log(promise); })(); 518 | return; 519 | } 520 | 521 | promise= promise || {}; 522 | 523 | if (promise.consumed) 524 | promise.consumed(_collect(consume)); 525 | 526 | if (promise.error) 527 | promise.error(_ask(function (err) 528 | { 529 | if (!err) return; 530 | 531 | if (err.code=='notfound') 532 | console.log('no data found'.yellow); 533 | else 534 | if (err.code=='exists') 535 | console.log('The item already exists'.red); 536 | else 537 | if (err.code=='updatedsinceread') 538 | console.log('The item is changed since you read it'.red); 539 | else 540 | console.log((err+'').red,err.stack); 541 | })); 542 | 543 | if (promise.end) 544 | promise.end(function () { end= true; doneres(); }); 545 | 546 | if (promise.count) 547 | promise.count(_ask(function (count) { console.log(('\r'+count).green); elapsed(); })); 548 | 549 | if (promise.transaction) 550 | promise.transaction(function (_tx) { tx= _tx; console.log(('tx: '+tx._id).green); elapsed(); ask(); }); 551 | else 552 | if (promise.committed) 553 | promise.committed(function () { tx= undefined; console.log('transaction committed'.green); elapsed(); ask(); }); 554 | else 555 | if (promise.rolledback) 556 | promise.rolledback(function (competing) { tx= undefined; if (competing) console.log('transaction rolled back'.red); else console.log('transaction rolled back'.green); elapsed(); ask(); }); 557 | else 558 | if (promise.clean) 559 | promise.clean(function (obj) { console.log(util.inspect(obj,{ depth: null })); ask(); }); 560 | else 561 | if (promise.result) 562 | { 563 | last= undefined; 564 | promise.result(function (obj) { last= obj; _print(obj,function () { elapsed(); ask(); }); }); 565 | } 566 | else 567 | if (promise.results) 568 | { 569 | last= []; 570 | promise.results(function (items) 571 | { 572 | chunks++; 573 | printed= false; 574 | 575 | last.push.apply(last,items); 576 | 577 | _print(items,function () 578 | { 579 | printed= true; 580 | doneres(); 581 | }); 582 | }); 583 | } 584 | else 585 | if (promise.success) 586 | promise.success(_ask(function () { console.log('done!'.green); elapsed(); })); 587 | else 588 | _ask(function () { console.log(util.inspect(promise,{ depth: null })); })(); 589 | } 590 | catch (ex) 591 | { 592 | console.log('unknown command'.red,ex,ex.stack); 593 | ask(); 594 | } 595 | 596 | //rl.close(); 597 | }); 598 | })(); 599 | }); 600 | }); 601 | } 602 | }]; 603 | 604 | if (argv.local) 605 | args.unshift({ dynamo: { endpoint: new AWS.Endpoint('http://localhost:8000') } }); 606 | 607 | dyngo.apply(null,args); 608 | -------------------------------------------------------------------------------- /lib/indexes/fat.js: -------------------------------------------------------------------------------- 1 | var _= require('underscore'), 2 | ret= require('ret'), 3 | async= require('async'); 4 | 5 | const _oa = function(o, s) 6 | { 7 | s = s.replace(/\[(\w+)\]/g, '.$1'); // convert indexes to properties 8 | s = s.replace(/^\./, ''); // strip a leading dot 9 | var a = s.split('.'); 10 | while (a.length) { 11 | var n = a.shift(); 12 | if (n in o) { 13 | o = o[n]; 14 | } else { 15 | return []; 16 | } 17 | } 18 | 19 | if (!Array.isArray(o)) o= [o]; 20 | 21 | return o; 22 | }, 23 | _cast= function (v) 24 | { 25 | if (typeof v=='boolean') 26 | return v ? 1 : 0; 27 | else 28 | return v; 29 | }, 30 | _cartesian= function() // cartesian product of N array 31 | { 32 | return _.reduce(arguments, function(a, b) { 33 | return _.flatten(_.map(a, function(x) { 34 | return _.map(b, function(y) { 35 | return x.concat([y]); 36 | }); 37 | }), true); 38 | }, [ [] ]); 39 | }, 40 | combine = function(a, min) { 41 | var fn = function(n, src, got, all) { 42 | if (n == 0) { 43 | if (got.length > 0) { 44 | all[all.length] = got.sort(); 45 | } 46 | return; 47 | } 48 | for (var j = 0; j < src.length; j++) { 49 | fn(n - 1, src.slice(j + 1), got.concat([src[j]]), all); 50 | } 51 | return; 52 | } 53 | var all = []; 54 | for (var i = min; i < a.length; i++) { 55 | fn(i, a, [], all); 56 | } 57 | all.push(a.sort()); 58 | return all; 59 | }, 60 | NONWORDRE= /[^\w, ]+/, 61 | ngrams = function(value, gramSize) 62 | { 63 | gramSize = gramSize || 3; 64 | 65 | var simplified = '-' + value.toLowerCase().replace(NONWORDRE, '') + '-', 66 | lenDiff = gramSize - simplified.length, 67 | results = []; 68 | 69 | if (lenDiff > 0) 70 | for (var i = 0; i < lenDiff; ++i) 71 | value += '-'; 72 | 73 | for (var i = 0; i < simplified.length - gramSize + 1; ++i) 74 | results.push(simplified.slice(i, i + gramSize)); 75 | 76 | return results; 77 | }, 78 | join= function (dyn,queries) 79 | { 80 | var p= dyn.promise(['results','end'],null,'consumed'), 81 | leading= queries.shift(); // @TODO: get an hint 82 | sync= dyn.syncResults(function (err) 83 | { 84 | if (err) 85 | p.trigger.error(err); 86 | else 87 | p.trigger.end(); 88 | }); 89 | 90 | leading 91 | ({ 92 | results: sync.results(function (results,done) 93 | { 94 | if (!results.length) 95 | { 96 | var r= []; 97 | r.next= function () { if (results.next) results.next(); done(); }; 98 | p.trigger.results(r); 99 | return; 100 | } 101 | 102 | var ranges= _.pluck(results,'_range').sort(), 103 | btw= [_.first(ranges),_.last(ranges)], 104 | idx= {}, 105 | _vote= function (v) 106 | { 107 | var r= idx[v._range]; 108 | if (r) r._voted++; 109 | }; 110 | 111 | results.forEach(function (r) 112 | { 113 | idx[r._range]= r; 114 | r._voted= 0; 115 | }); 116 | 117 | async.forEachSeries(queries, 118 | function (qry,done) 119 | { 120 | var first, last; 121 | 122 | if (btw) 123 | qry 124 | ({ 125 | results: function (res) 126 | { 127 | res.forEach(_vote); 128 | if (!first) first= _.first(res); 129 | last= _.last(res); 130 | if (res.next) res.next(); // if leading query is ordered btw ids may be distant 131 | }, 132 | error: done, 133 | consumed: p.trigger.consumed, 134 | end: function () 135 | { 136 | if (first) 137 | { 138 | btw[0]= first._range; 139 | btw[1]= last._range; 140 | } 141 | else 142 | btw= undefined; 143 | 144 | done(); 145 | } 146 | },btw); 147 | else 148 | done(); 149 | }, 150 | function (err) 151 | { 152 | if (err) 153 | done(err); 154 | else 155 | { 156 | var voted= _.where(results,{ _voted: queries.length }); 157 | joined= _.collect(voted,function (item) 158 | { 159 | return _.pick(item,['_id','_pos']); 160 | }); 161 | 162 | joined.next= function () { if (results.next) results.next(); done(); }; 163 | 164 | p.trigger.results(joined); 165 | } 166 | }); 167 | }), 168 | error: p.trigger.error, 169 | consumed: p.trigger.consumed, 170 | end: sync.end 171 | }); 172 | 173 | return p; 174 | }; 175 | 176 | 177 | module.exports= function (dyn,table,fields) 178 | { 179 | var $text= fields.$text, $combine= fields.$combine; 180 | 181 | fields= _.omit(fields,['$text','$combine']); 182 | 183 | var index= {}, 184 | fieldNames= Object.keys(fields), 185 | lookupNames= [], 186 | _fields= _.collect(fieldNames, 187 | function (fieldName) 188 | { 189 | var type= fields[fieldName], pos= 0; 190 | 191 | if ((pos=fieldName.indexOf('.'))>-1) 192 | lookupNames.push(fieldName.substring(0,pos)); 193 | 194 | return { name: fieldName, 195 | type: type }; 196 | }); 197 | 198 | index.name= 'fat-'+table._dynamo.TableName+'--'+fieldNames.join('-').replace(/\$/g,'_')+( $text ? '-text' : ''); 199 | 200 | var unhandled= false; 201 | 202 | _fields.every(function (field) 203 | { 204 | if (!_.contains(['S','N','SS'],field.type)) 205 | return !(unhandled=true); 206 | 207 | return true; 208 | }); 209 | 210 | if (unhandled) return false; 211 | 212 | index.create= function (done) 213 | { 214 | var _secondary= function () 215 | { 216 | return _.collect(_.filter(_fields,function (field){ return field.type.length==1; }), // not sets 217 | function (field) 218 | { 219 | return { name: field.name.replace(/\$/g,'_'), 220 | key: { name: field.name, 221 | type: field.type }, 222 | projection: ['_id','_pos','_rev'] }; 223 | }); 224 | }; 225 | 226 | if (index.dbopts.hints) console.log('This may take a while...'.yellow); 227 | 228 | dyn.table(index.name) 229 | .hash('_hash','S') 230 | .range('_range','S') 231 | .create(function indexItems() 232 | { 233 | dyn.table(index.name) 234 | .hash('_hash','xx') 235 | .query(function () 236 | { 237 | index.rebuild().error(done).success(done); 238 | }) 239 | .error(function (err) 240 | { 241 | if (err.code=='ResourceNotFoundException') 242 | setTimeout(indexItems,5000); 243 | else 244 | done(err); 245 | }); 246 | },{ secondary: _secondary() }) 247 | .error(function (err) 248 | { 249 | if (err.code=='ResourceInUseException') 250 | setTimeout(function () { index.ensure(done) },5000); 251 | else 252 | done(err); 253 | }); 254 | }; 255 | 256 | index.makeRange= function (item) 257 | { 258 | return item._id+':'+item._pos; 259 | }; 260 | 261 | index.makeKeys= function (item) 262 | { 263 | var keys= [], keyFieldNames= [], keyFieldValues= {}; 264 | 265 | keys.push 266 | ({ 267 | '_hash': 'VALUES', 268 | '_range': index.makeRange(item) 269 | }); 270 | 271 | fieldNames.some(function (fieldName) 272 | { 273 | keyFieldNames.push(fieldName); 274 | 275 | var val= keyFieldValues[fieldName]= _oa(item,fieldName), 276 | _combine= $combine ? function (val) { return $combine(item,val,combine); } : combine; 277 | 278 | if (val.length) 279 | _cartesian.apply(null,_.collect(_.values(keyFieldValues),function (val) { return _combine(val.sort(),1); })).forEach(function (hv) 280 | { 281 | keys.push 282 | ({ 283 | '_hash': _.collect(hv, 284 | function (vls) 285 | { 286 | return JSON.stringify(_.collect(vls,_cast)); 287 | }) 288 | .join(':'), 289 | '_range': index.makeRange(item) 290 | }); 291 | }); 292 | else 293 | return true; 294 | }); 295 | 296 | if ($text) 297 | { 298 | var textItem= $text(item); 299 | 300 | _.keys(textItem).forEach(function (key) 301 | { 302 | var val= textItem[key], 303 | words= (val+'').split(/\s+/); 304 | 305 | words.forEach(function (w) 306 | { 307 | var tgs= _.uniq(ngrams(w,3)); 308 | 309 | tgs.forEach(function (tg) 310 | { 311 | keys.push 312 | ({ 313 | '_hash': '$text$'+tg, 314 | '_range': index.makeRange(item) 315 | }); 316 | }); 317 | }); 318 | }); 319 | 320 | } 321 | 322 | return keys; 323 | }; 324 | 325 | index.makeFilterHash= function (query) 326 | { 327 | var filter= query.$filter, 328 | values= []; 329 | 330 | fieldNames.some(function (fieldName) 331 | { 332 | var field= filter[fieldName]; 333 | 334 | if (field&&_.contains(['ALL','EQ'],field.op)) 335 | { 336 | values.push(JSON.stringify(_.collect(field.values,_cast).sort())); 337 | delete filter[fieldName]; 338 | query.$filtered.push(fieldName); 339 | } 340 | else 341 | return true; 342 | }); 343 | 344 | return values.length ? values.join(':') : 'VALUES'; 345 | }; 346 | 347 | index.indexable= function (item) 348 | { 349 | return item&&($text||!!_oa(item,fieldNames[0]).length); 350 | }; 351 | 352 | index.usable= function (query) 353 | { 354 | return query.cond&&(query.$text||!!query.cond[fieldNames[0]]); 355 | }; 356 | 357 | index.makeElements= function (item) 358 | { 359 | return _.collect(index.makeKeys(item), 360 | function (key) 361 | { 362 | return _.extend(_.pick(item,_.union(['_id','_pos','_rev'],fieldNames)),key); 363 | }); 364 | }; 365 | 366 | index.put= function (item,done) 367 | { 368 | if (index.indexable(item)) 369 | { 370 | var elems= index.makeElements(item); 371 | 372 | async.forEach(elems,function (elem, done) 373 | { 374 | dyn.table(index.name) 375 | .hash('_hash',elem._hash) 376 | .range('_range',elem._range) 377 | .put(elem,done) 378 | .error(done); 379 | },done); 380 | } 381 | else 382 | done(); 383 | }; 384 | 385 | index.streamElements= function () 386 | { 387 | var _transform= function (items) 388 | { 389 | return _.flatten(_.collect(items,index.makeElements)); 390 | }, 391 | transform= _transform; 392 | 393 | if (lookupNames.length) 394 | { 395 | transform= function (items, cb) 396 | { 397 | async.forEach(_.range(items.length), 398 | function (idx,done) 399 | { 400 | var item= items[idx]; 401 | 402 | async.forEach(lookupNames,function (name,done) 403 | { 404 | var attr= item['__'+name]; 405 | 406 | if (attr) 407 | { 408 | var parts= attr.split('$:$'); 409 | table.findOne({ _id: parts[0], _pos: (+parts[1]||0) }) 410 | .result(function (value) 411 | { 412 | item[name]= value; 413 | done(); 414 | }) 415 | .error(done); 416 | } 417 | else 418 | done(); 419 | }, 420 | done); 421 | }, 422 | function (err) 423 | { 424 | if (err) 425 | cb(err); 426 | else 427 | cb(null,_transform(items)); 428 | }); 429 | }; 430 | 431 | transform.async= true; 432 | } 433 | 434 | return index.tstream(transform); 435 | }; 436 | 437 | index.update= function (item,op) 438 | { 439 | var iops= {}, 440 | ops= iops[index.name]= []; 441 | 442 | if (index.indexable(item)) 443 | { 444 | var elems= index.makeElements(item); 445 | 446 | if (index.indexable(item._old)) 447 | { 448 | var oldKeys= index.makeKeys(item._old); 449 | 450 | oldKeys.forEach(function (key) 451 | { 452 | if (!_.findWhere(elems,key)) 453 | ops.push({ op: 'del', item: key }); 454 | }); 455 | } 456 | 457 | elems.forEach(function (elem) 458 | { 459 | ops.push({ op: op, item: elem }); 460 | }); 461 | } 462 | 463 | if (ops.length) 464 | return iops; 465 | else 466 | return undefined; 467 | }; 468 | 469 | index.remove= function (item) 470 | { 471 | var p= dyn.promise(null,null,'consumed'); 472 | 473 | if (index.indexable(item)) 474 | { 475 | var keys= index.makeKeys(item); 476 | 477 | async.forEach(keys, 478 | function (key,done) 479 | { 480 | dyn.table(index.name) 481 | .hash('_hash',key._hash) 482 | .range('_range',key._range) 483 | .delete(done) 484 | .consumed(p.trigger.consumed) 485 | .error(function (err) 486 | { 487 | if (err.code=='notfound') 488 | done(); 489 | else 490 | done(err); 491 | }); 492 | }, 493 | p.should('success')); 494 | } 495 | else 496 | process.nextTick(p.trigger.success); 497 | 498 | return p; 499 | }; 500 | 501 | index.find= function (query) 502 | { 503 | var p= dyn.promise(['results','count','end'],null,'consumed'), 504 | hash= index.makeFilterHash(query), 505 | tab= dyn.table(index.name) 506 | .hash('_hash',hash), 507 | _index, 508 | sort, 509 | secondaryFieldName= _.first(_.intersection(_.difference(fieldNames,query.$filtered),Object.keys(query.$filter))); 510 | 511 | if (secondaryFieldName!==undefined) 512 | { 513 | var field= query.$filter[secondaryFieldName]; 514 | 515 | if (field&&!_.contains(['IN','CONTAINS'],field.op)) 516 | { 517 | delete query.$filter[secondaryFieldName]; 518 | tab.range(secondaryFieldName,field.values,field.op) 519 | .index(_index=secondaryFieldName.replace(/\$/g,'_')); 520 | } 521 | } 522 | 523 | if (query.$orderby) 524 | { 525 | var field= query.$orderby[0]; 526 | 527 | if (_index) 528 | { 529 | if (_index==field.name) 530 | sort= field; 531 | } 532 | else 533 | if (_.contains(fieldNames,field.name)) 534 | { 535 | tab.index(field.name.replace(/\$/g,'_')); 536 | sort= field; 537 | } 538 | 539 | query.sorted= (!!sort)&&query.$orderby.length==1; 540 | } 541 | 542 | if (query.$text) 543 | { 544 | var words= query.$text.split(/\s+/), tgs= []; 545 | 546 | words.forEach(function (w) 547 | { 548 | tgs.push(_.uniq(ngrams(w,3))); 549 | }); 550 | 551 | words= _.collect(words,function (w) { return w.toLowerCase().replace(NONWORDRE, ''); }); 552 | 553 | tgs= _.filter(_.uniq(_.union.apply(_,tgs)),function (tg) { return tg[2]!='-'; }); 554 | 555 | var queries= []; 556 | 557 | if (query.$filtered.length) 558 | queries.push(function (qry) 559 | { 560 | tab.query(qry.results, 561 | { attrs: ['_range','_id','_pos'], 562 | desc: sort&&(sort.dir==-1), 563 | consistent: query.$consistent, 564 | limit: query.window, 565 | count: undefined }) 566 | .error(qry.error) 567 | .consumed(qry.consumed) 568 | .end(qry.end); 569 | }); 570 | 571 | tgs.forEach(function (tg) 572 | { 573 | var _forTg= function (fn) { fn.tg=tg; return fn; }; 574 | 575 | queries.push(_forTg(function (qry,btw) 576 | { 577 | var t= dyn.table(index.name) 578 | .hash('_hash', '$text$'+tg); 579 | 580 | if (btw) 581 | t.range('_range',btw,'BETWEEN'); 582 | 583 | t.query(qry.results, 584 | { attrs: ['_range','_id','_pos'], 585 | consistent: query.$consistent, 586 | limit: query.window }) 587 | .error(qry.error) 588 | .consumed(qry.consumed) 589 | .end(qry.end); 590 | })); 591 | }); 592 | 593 | join(dyn,queries) 594 | .results(function (items) 595 | { 596 | items.refine= function (items) 597 | { 598 | var refined= _.filter(items, 599 | function (item) 600 | { 601 | var textItem= $text(item), 602 | missing= words.slice(); 603 | 604 | _.keys(textItem).every(function (key) 605 | { 606 | var val= textItem[key], 607 | vwords= _.collect((val+'').split(/\s+/), 608 | function (w) { return w.toLowerCase() 609 | .replace(NONWORDRE, ''); }); 610 | 611 | missing= _.filter(missing,function (w) 612 | { 613 | return !vwords.some(function (vw) 614 | { 615 | return vw.indexOf(w)==0; 616 | }); 617 | }); 618 | 619 | return !!missing.length; 620 | }); 621 | 622 | return !missing.length; 623 | }); 624 | 625 | refined.next= items.next; 626 | 627 | return refined; 628 | }; 629 | 630 | p.trigger.results(items); 631 | }) 632 | .error(p.trigger.error) 633 | .consumed(p.trigger.consumed) 634 | .end(p.trigger.end); 635 | } 636 | else 637 | { 638 | query.counted= query.canCount(); 639 | 640 | tab.query(query.count&&query.canCount() ? p.trigger.count : p.trigger.results, 641 | { attrs: ['_id','_pos'], 642 | desc: sort&&(sort.dir==-1), 643 | consistent: query.$consistent, 644 | limit: query.counted ? undefined : query.window, 645 | count: query.counted ? query.count : undefined }) 646 | .error(p.trigger.error) 647 | .consumed(p.trigger.consumed) 648 | .end(p.trigger.end); 649 | } 650 | 651 | return p; 652 | 653 | }; 654 | 655 | return index; 656 | }; 657 | 658 | -------------------------------------------------------------------------------- /lib/refiner.js: -------------------------------------------------------------------------------- 1 | var _= require('underscore'), 2 | cclone= require('circularclone'), 3 | diff = require('deep-diff').diff, 4 | async= require('async'); 5 | 6 | const _compare= function (x, y) 7 | { 8 | if (x===y) 9 | return 0; 10 | 11 | return x > y ? 1 : -1; 12 | }, 13 | _collect= function (consume) 14 | { 15 | return function (cons) 16 | { 17 | _.keys(cons).forEach(function (table) 18 | { 19 | var c, tcons= cons[table]; 20 | 21 | if (!(c=consume[table])) 22 | c= consume[table]= { read: 0, write: 0 }; 23 | 24 | c.read+= tcons.read; 25 | c.write+= tcons.write; 26 | }); 27 | }; 28 | }, 29 | _ignoreNotFound= function (done,_id,_pos,query) 30 | { 31 | return function (err) 32 | { 33 | if (err.code=='notfound') 34 | { 35 | query.identity.set(_id,_pos,undefined); 36 | done(); 37 | } 38 | else 39 | done(err); 40 | }; 41 | }, 42 | _limit= function (items,query) 43 | { 44 | if (query.limit&&(items.length+query.$returned>query.limit)&&!query.limited) 45 | { 46 | if (!query.count&&query.opts.hints) console.log('client side limit'.red); 47 | items= items.slice(0,query.limit-query.$returned); 48 | query.limited= true; 49 | } 50 | 51 | if (query.skip&&!query.skipped) 52 | { 53 | if (query.opts.hints) console.log('client side skip'.red); 54 | 55 | items= items.slice(query.skip); 56 | query.skipped= true; 57 | } 58 | 59 | return items; 60 | }, 61 | _oa = function(o, s) 62 | { 63 | s = s.replace(/\[(\w+)\]/g, '.$1'); 64 | s = s.replace(/^\./, ''); 65 | var a = s.split('.'); 66 | while (a.length) { 67 | var n = a.shift(); 68 | if (n in o) { 69 | o = o[n]; 70 | } else { 71 | return; 72 | } 73 | } 74 | return o; 75 | }, 76 | _operator= function (cmp) 77 | { 78 | return function (fieldName,vals) 79 | { 80 | return function (item) 81 | { 82 | return cmp(_oa(item,fieldName),vals); 83 | }; 84 | }; 85 | }, 86 | _operators= { 87 | EQ: _operator(function (itemVal,vals) 88 | { 89 | return itemVal===vals[0]; 90 | }), 91 | 92 | NE: _operator(function (itemVal,vals) 93 | { 94 | return itemVal!==vals[0]; 95 | }), 96 | 97 | GT: _operator(function (itemVal,vals) 98 | { 99 | return itemVal>vals[0]; 100 | }), 101 | 102 | GE: _operator(function (itemVal,vals) 103 | { 104 | return itemVal>=vals[0]; 105 | }), 106 | 107 | LT: _operator(function (itemVal,vals) 108 | { 109 | return itemVal-1; 130 | }), 131 | 132 | REGEXP: _operator(function (itemVal,vals) 133 | { 134 | return !!vals[0].exec(itemVal); 135 | }), 136 | 137 | BETWEEN: _operator(function (itemVal,vals) 138 | { 139 | return itemVal>=vals[0]&&itemVal<=vals[1]; 140 | }), 141 | 142 | NULL: _operator(function (itemVal,vals) 143 | { 144 | return itemVal==undefined || itemVal==null; 145 | }), 146 | 147 | NOT_NULL: _operator(function (itemVal,vals) 148 | { 149 | return !(itemVal==undefined || itemVal==null); 150 | }), 151 | 152 | ALL: _operator(function (itemVal,vals) 153 | { 154 | return !_.difference(vals,itemVal).length; 155 | }) 156 | }, 157 | _filter= function (items,query) 158 | { 159 | 160 | var fieldNames= Object.keys(query.$filter), next= items.next; 161 | 162 | if (fieldNames.length) 163 | { 164 | if (query.opts.hints) console.log('client side filter'.red); 165 | 166 | fieldNames.forEach(function (fieldName) 167 | { 168 | var field= query.$filter[fieldName]; 169 | delete query.$filter[fieldName]; 170 | query.$filtered.push(fieldName); 171 | items= _.filter(items,_operators[field.op](fieldName,field.values)); 172 | }); 173 | 174 | } 175 | 176 | items.next= next; 177 | 178 | return items; 179 | }, 180 | _sort= function (items,query) 181 | { 182 | if (query.orderby&&!query.sorted) 183 | { 184 | if (query.opts.hints) console.log('client side sort'.red); 185 | 186 | var fields= query.$orderby; 187 | 188 | items.sort(function (x, y) 189 | { 190 | var retval; 191 | 192 | fields.some(function (field) 193 | { 194 | var fx= query.oa(x,field.name), 195 | fy= query.oa(y,field.name); 196 | 197 | if (fx!=fy) 198 | { 199 | retval= _compare(fx,fy)*field.dir; 200 | return true; 201 | } 202 | }); 203 | 204 | return retval; 205 | }); 206 | } 207 | 208 | return items; 209 | }, 210 | _notfound= function (items) 211 | { 212 | var ret= _.filter(items,function (i) { return !!i._id; }); 213 | 214 | ret.next= items.next; 215 | 216 | return ret; 217 | }, 218 | _modifiers= function (items,query) 219 | { 220 | if (items.refine) 221 | items= items.refine(items); 222 | 223 | items= _notfound(items); 224 | items= _filter(items,query); 225 | items= _sort(items,query); 226 | items= _limit(items,query); 227 | 228 | return items; 229 | }, 230 | _refineItems= function (dyn,trigger,items,query,db) 231 | { 232 | if (query.canLimit()) 233 | items= _limit(items,query); 234 | 235 | var p= dyn.promise(null,null,'consumed'), 236 | _load= function (key,proot,done) 237 | { 238 | var item= items[key]; 239 | 240 | query.identity.set(item._id,0,item); 241 | 242 | async.forEach(Object.keys(item), 243 | function (field,done) 244 | { 245 | if (field.indexOf('___')==0) 246 | { 247 | var attr= field.substring(3), 248 | _id= item[field]; 249 | 250 | query.identity.get(_id,'_',function (items) 251 | { 252 | if (items) 253 | { 254 | item[attr]= items; 255 | done(); 256 | } 257 | else 258 | query.table.find({ _id: _id }, 259 | query.toprojection(proot[attr]), 260 | query.identity) 261 | .results(function (values) 262 | { 263 | item[attr]= values; 264 | query.identity.set(_id,'_',values); 265 | done(); 266 | }) 267 | .consumed(p.trigger.consumed) 268 | .error(_ignoreNotFound(done,_id,'_',query)); 269 | }); 270 | } 271 | else 272 | if (field.indexOf('__')==0) 273 | { 274 | var attr= field.substring(2), 275 | ptr= dyn.deref(item[field],query.table._dynamo.TableName); 276 | 277 | query.identity.get(ptr._id,ptr._pos,function (loaded) 278 | { 279 | if (loaded) 280 | { 281 | item[attr]= loaded; 282 | done(); 283 | } 284 | else 285 | db[ptr._table].findOne({ _id: ptr._id, _pos: ptr._pos }, 286 | query.toprojection(proot[attr]), 287 | query.identity) 288 | .result(function (value) 289 | { 290 | item[attr]= value; 291 | query.identity.set(ptr._id,ptr._pos,value); 292 | done(); 293 | }) 294 | .consumed(p.trigger.consumed) 295 | .error(_ignoreNotFound(done,ptr._id,ptr._pos,query)); 296 | }); 297 | } 298 | else 299 | if (field=='_ref') 300 | { 301 | var ptr; 302 | 303 | if (query.noderef) 304 | { 305 | if (!item._id) 306 | item= _.extend(item,dyn.deref(item._ref,query.table._dynamo.TableName)); 307 | 308 | delete item['_ref']; 309 | done(); 310 | return; 311 | } 312 | 313 | ptr= dyn.deref(item._ref,query.table._dynamo.TableName); 314 | 315 | var _diff= diff(query.toprojection(proot),{ _id: 1, _pos: 1 }); 316 | 317 | if (_diff) 318 | query.identity.get(ptr._id,ptr._pos,function (loaded) 319 | { 320 | if (loaded) 321 | { 322 | item= loaded; 323 | done(); 324 | } 325 | else 326 | db[ptr._table].findOne({ _id: ptr._id, _pos: ptr._pos }, 327 | query.toprojection(proot), 328 | query.identity) 329 | .result(function (loaded) 330 | { 331 | item= loaded; 332 | query.identity.set(ptr._id,ptr._pos,loaded); 333 | done(); 334 | }) 335 | .consumed(p.trigger.consumed) 336 | .error(_ignoreNotFound(done,ptr._id,ptr._pos,query)); 337 | }); 338 | else 339 | { 340 | item= { _id: ptr._id, _pos: ptr._pos }; 341 | done(); 342 | } 343 | } 344 | else 345 | done(); 346 | }, 347 | function (err) 348 | { 349 | if (err) done(err); 350 | else 351 | { 352 | items[key]= item; 353 | done(); 354 | } 355 | }); 356 | }, 357 | _refine= function (key, done) 358 | { 359 | _load(key, query.projection.root, 360 | function (err) 361 | { 362 | var item= items[key]; 363 | query.identity.set(item._id,0,item); 364 | 365 | if (err) 366 | done(err); 367 | else 368 | done(); 369 | }); 370 | 371 | }; 372 | 373 | if ((query.projection.exclude || []).length>0) 374 | _refine= _.wrap(_refine,function (wrapped,key,done) 375 | { 376 | var item= items[key]; 377 | items[key]= item._ref ? item : _.omit(item,query.projection.exclude); 378 | 379 | wrapped(key, 380 | function (err) 381 | { 382 | if (err) done(err); 383 | else 384 | { 385 | var item= items[key]; 386 | items[key]= item._ref ? _.omit(item,query.projection.exclude) : item; 387 | done(); 388 | } 389 | }); 390 | }); 391 | 392 | _refine= _.wrap(_refine,function (wrapped,key,done) 393 | { 394 | wrapped(key, 395 | function (err) 396 | { 397 | if (err) done(err) 398 | else 399 | { 400 | var item= items[key], cir= [], 401 | queue= [], 402 | set= function (known,clone,attr,filter) 403 | { 404 | var val= _.findWhere(known,filter); 405 | 406 | if (val) 407 | return val; 408 | else 409 | queue.push(arguments); 410 | }; 411 | 412 | item._table= query.table._dynamo.TableName; 413 | 414 | item._old= cclone(item,function (field,value,clone,node,origValue,known) 415 | { 416 | if (typeof field!='string') return value; 417 | 418 | if (field.indexOf('___')==0) 419 | { 420 | var attr= field.substring(3), 421 | parts= value.split('$:$'), 422 | _id= parts[0]; 423 | 424 | if (!clone[attr]) 425 | clone[attr]= set(known,clone,attr,{ _id: _id }); 426 | else 427 | if (Array.isArray(clone[attr])) 428 | clone[attr]._id= _id; 429 | } 430 | else 431 | if (field.indexOf('__')==0) 432 | { 433 | var attr= field.substring(2), 434 | parts= value.split('$:$'), 435 | _id= parts[0]; 436 | 437 | if (!clone[attr]) 438 | clone[attr]= set(known,clone,attr,{ _id: _id, _pos: 0 }); 439 | } 440 | else 441 | if (field=='_ref') 442 | { 443 | var _id, _pos= 0; 444 | 445 | if (typeof value=='string') 446 | { 447 | var parts= value.split('$:$'); 448 | _id= parts[0]; 449 | 450 | if (parts[1]) 451 | _pos= +parts[1]; 452 | } 453 | else 454 | { 455 | _id= value._id; 456 | _pos= value._pos; 457 | } 458 | 459 | if (!clone[attr]) 460 | clone[attr]= set(known,clone,attr,{ _id: _id, _pos: _pos }); 461 | } 462 | 463 | return value; 464 | }); 465 | 466 | queue.forEach(function (args) 467 | { 468 | set.apply(null,args); 469 | }); 470 | 471 | done(); 472 | } 473 | }); 474 | }); 475 | 476 | async.forEach(_.range(items.length), 477 | _refine, 478 | function (err) 479 | { 480 | if (err) 481 | refiner.trigger.error(err); 482 | else 483 | trigger(_modifiers(items,query)); 484 | }); 485 | 486 | return p; 487 | }; 488 | 489 | module.exports= function (dyn, query, db) 490 | { 491 | var promises= ['results','end']; 492 | 493 | if (query.count) promises.push('count'); 494 | 495 | var refiner= dyn.promise(promises,null,'consumed'), fended, ended, _end= refiner.trigger.end, _results, _items, consume= {}, _consumed= refiner.trigger.consumed, 496 | _stop= function () 497 | { 498 | ended= true; 499 | }; 500 | 501 | refiner.trigger.end= _.wrap(refiner.trigger.end,function (trigger) 502 | { 503 | fended= true; 504 | }); 505 | 506 | refiner.trigger.consumed= _.wrap(refiner.trigger.consumed,function (trigger,cons) 507 | { 508 | var c; 509 | 510 | if (!(c=consume[cons.table])) 511 | c= consume[cons.table]= { read: 0, write: 0 }; 512 | 513 | c.read+= cons.read; 514 | c.write+= cons.write; 515 | }); 516 | 517 | refiner.trigger.results= _.wrap(refiner.trigger.results,function (trigger,items) 518 | { 519 | _results= trigger; 520 | 521 | if (!ended) 522 | process.nextTick(function () // so we can call _end 523 | { 524 | _refineItems(dyn,function (items) 525 | { 526 | if (items.next&&!query.limited) 527 | items.next(); 528 | 529 | ended= query.limited || fended; 530 | 531 | delete items.next; 532 | 533 | query.$returned+= items.length; 534 | 535 | if (query.count) 536 | { 537 | process.stdout.write(('\r'+query.$returned).yellow); 538 | 539 | if (query.canCount()) 540 | { 541 | _consumed(consume); 542 | refiner.trigger.count(query.$returned); 543 | } 544 | } 545 | else 546 | if (items.length>0) 547 | _results(items,_stop); 548 | 549 | if (ended) 550 | { 551 | if (query.$returned==0) 552 | _results([]); 553 | 554 | _consumed(consume); 555 | _end(); 556 | } 557 | }, 558 | items,query,db).consumed(_collect(consume)); 559 | }); 560 | }); 561 | 562 | return refiner; 563 | }; 564 | --------------------------------------------------------------------------------