├── .gitignore ├── .npmignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── bin └── c3d2.js ├── docs └── plan.md ├── graphics ├── lemon.jpg ├── logo.fla ├── logo.png └── logosmall.png ├── package.json ├── src ├── index.js └── tools.js └── test ├── mocha.opts └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | 10 | # Packages # 11 | ############ 12 | # it's better to unpack these files and commit the raw source 13 | # git has its own built in compression methods 14 | *.7z 15 | *.dmg 16 | *.gz 17 | *.iso 18 | *.jar 19 | *.rar 20 | *.tar 21 | *.zip 22 | 23 | # Logs and databases # 24 | ###################### 25 | *.log 26 | *.sql 27 | *.sqlite 28 | 29 | # OS generated files # 30 | ###################### 31 | .DS_Store 32 | .DS_Store? 33 | ._* 34 | .Spotlight-V100 35 | .Trashes 36 | Icon? 37 | ehthumbs.db 38 | Thumbs.db 39 | 40 | # Vagrant # 41 | ########### 42 | .vagrant 43 | 44 | # nodejs # 45 | ########### 46 | lib-cov 47 | *.seed 48 | *.log 49 | *.csv 50 | *.dat 51 | *.out 52 | *.pid 53 | *.gz 54 | 55 | dist 56 | pids 57 | logs 58 | results 59 | 60 | npm-debug.log 61 | node_modules 62 | build 63 | 64 | test/tempdb 65 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # Vagrant 2 | .vagrant 3 | 4 | # Windows 5 | Thumbs.db 6 | Desktop.ini 7 | 8 | # Vim 9 | .*.sw[a-z] 10 | *.un~i 11 | tags 12 | 13 | # OsX 14 | .DS_Store 15 | Icon? 16 | ._* 17 | .Spotlight-V100 18 | .Trashes 19 | 20 | # nodejs npm modules 21 | /node_modules 22 | build 23 | test/tempdb -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "5.4" -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Kai Davenport 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: install 2 | @NODE_ENV=test ./node_modules/.bin/mocha \ 3 | --reporter spec \ 4 | --timeout 300 \ 5 | --require should \ 6 | --growl \ 7 | test/test.js 8 | 9 | install: 10 | npm install 11 | 12 | .PHONY: test -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | lem 2 | === 3 | 4 | ![lem logo](https://github.com/binocarlos/lem/raw/master/graphics/logosmall.png "Lem Logo") 5 | 6 | ![Build status](https://travis-ci.org/binocarlos/lem.svg?branch=master) 7 | 8 | database for time-series data using LevelDB and node.js 9 | 10 | ## installation 11 | 12 | ``` 13 | $ npm install lem 14 | ``` 15 | 16 | ## usage 17 | 18 | ```js 19 | var lem = require('lem'); 20 | var level = require('level'); 21 | 22 | // create a new leveldb - this can also be a sub-level 23 | var leveldb = level('/tmp/lemtest'); 24 | 25 | // create a new lem store using the leveldb 26 | var lemdb = lem(leveldb); 27 | 28 | // when nodes are indexed 29 | lemdb.on('index', function(key, meta){ 30 | 31 | }) 32 | 33 | // a live stream from the database 34 | lemdb.on('data', function(data){ 35 | 36 | }) 37 | 38 | // nodes are represented by keys 39 | var key = 'myhouse.kitchen.fridge.temperature'; 40 | 41 | // index a node with some meta data 42 | lemdb.index(key, 'My Fridge Temp'); 43 | 44 | // create a recorder which will write data to the node 45 | var temp = lemdb.recorder(key); 46 | 47 | // write a value every second 48 | setInterval(function(){ 49 | temp(Math.random()*100); 50 | }, 1000) 51 | 52 | ``` 53 | 54 | ## timestamps 55 | 56 | When values are written to recorders - they are timestamped. Sometimes - more acurate timestamping (like a GPS source) is used - you can provide the timestamp to the recorder: 57 | 58 | ```js 59 | var temp = lemdb.recorder('timestamp.test'); 60 | setInterval(function(){ 61 | // get a custom timestamp from somewhere - the current time is the default 62 | var timestamp = new Date().getTime(); 63 | temp(Math.random()*100, timestamp); 64 | }, 1000) 65 | ``` 66 | 67 | ## index 68 | 69 | You can read the index from any point in the tree - it returns a ReadStream of the keys that have been indexed: 70 | 71 | ```js 72 | ... 73 | var through = require('through'); 74 | 75 | // index a key into the tree 76 | lemdb.index('cars.red5.speed', 'The speed of the car', function(){ 77 | var keysfound = {}; 78 | 79 | // keys returns a readstream of objects each with a 'key' and 'data' property 80 | lemdb.keys('cars.red5').pipe(through(function(data){ 81 | keysfound[data.key] = data.value; 82 | }, function(){ 83 | console.log('Meta: ' + keysfound.speed); 84 | }) 85 | }) 86 | ``` 87 | 88 | This will log: 89 | 90 | ``` 91 | Meta: The speed of the car 92 | ``` 93 | 94 | ## valuestream 95 | 96 | Create a ReadStream of telemetry values for a node - you can specify start and end keys to view windows in time: 97 | 98 | ```js 99 | 100 | // create a range - this can be a 'session' to make meaningful groups within lem 101 | var sessionstart = new Date('04/05/2013 12:34:43'); 102 | var sessionend = new Date('04/05/2013 12:48:10'); 103 | var counter = 0; 104 | var total = 0; 105 | 106 | var secs = (sessionend.getTime() - sessionstart.getTime()) / 1000; 107 | 108 | lemdb.valuestream('cars.red5.speed', { 109 | start:sessionstart.getTime(), 110 | end:sessionend.getTime() 111 | }).pipe(through(function(data){ 112 | 113 | // this is the timestamp of the value 114 | var key = data.key; 115 | 116 | // this is the actual value 117 | var value = data.value; 118 | 119 | // map-reduce beginnings 120 | total += value; 121 | counter++; 122 | }, function(){ 123 | 124 | var avg = 0; 125 | 126 | if(counter>0){ 127 | avg = total / counter; 128 | } 129 | 130 | console.log('average speed of: ' + avg); 131 | console.log('data points: ' + total); 132 | console.log('time period: ' + secs + ' secs'); 133 | 134 | })) 135 | ``` 136 | 137 | ## api 138 | 139 | #### `var lemdb = lem(leveldb);` 140 | 141 | Create a new lem database from the provided [leveldb](https://github.com/rvagg/node-levelup). This can be a [level-sublevel](https://github.com/dominictarr/level-sublevel) so you can partition lem into an existing database. 142 | 143 | ```js 144 | var lem = require('lem'); 145 | var level = require('level'); 146 | 147 | var leveldb = level('/tmp/mylem'); 148 | var lemdb = lem(leveldb); 149 | ``` 150 | 151 | #### `lemdb.index(path, meta, [done])` 152 | 153 | Write a node and some meta data to the index. 154 | 155 | The index is used to build a tree of key-values that exist without having to traverse the time-stamped keys. 156 | 157 | The stream returned can be used to build any kind of data structure you want (list, tree, etc). 158 | 159 | The meta data for each node is saved as a string - you can use your own encoding (e.g. JSON). 160 | 161 | Create some indexes: 162 | 163 | ```js 164 | lemdb.index('myhouse.kitchen.fridge.temperature', '{"title":"Fridge Temp","owner":344}'); 165 | lemdb.index('myhouse.kitchen.thermostat.temperature', '{"title":"Stat Temp","owner":344}'); 166 | ``` 167 | 168 | #### `lemdb.keys(path)` 169 | 170 | keys returns a ReadStream of all keys in the index beneath the key you provide. 171 | 172 | For example - convert the stream into a tree representing all nodes in the kitchen: 173 | 174 | ```js 175 | ... 176 | var through = require('through'); 177 | var tree = {}; 178 | lemdb.keys('myhouse.kitchen').pipe(through(function(data){ 179 | tree[data.key] = data.value; 180 | }, function(){ 181 | console.dir(tree); 182 | })) 183 | ``` 184 | 185 | This outputs: 186 | 187 | ```js 188 | { 189 | "fridge.temperature":'{"title":"Fridge Temp","owner":344}', 190 | "thermostat.temperature":'{"title":"Stat Temp","owner":344}' 191 | } 192 | ``` 193 | 194 | #### `lemdb.recorder(path)` 195 | 196 | A recorder is used to write time-series data to a node. 197 | 198 | You create it with the path of the node: 199 | 200 | ```js 201 | var recorder = lemdb.recorder('myhouse.kitchen.fridge.temperature'); 202 | ``` 203 | 204 | #### `recorder(value, [timestamp], [done])` 205 | 206 | The recorder itself is a function that you run with a value and optional timestamp and callback. 207 | 208 | If no timestamp is provided a default is created: 209 | 210 | ```js 211 | var timestamp = new Date().getTime(); 212 | ``` 213 | 214 | The callback is run once the value has been committed to disk: 215 | 216 | ```js 217 | 218 | // a function to get an accurate time-stamp from somewhere 219 | function getProperTime(){ 220 | return ...; 221 | } 222 | 223 | // a function to return the current value of an external sensor 224 | function getSensorValue(){ 225 | return ...; 226 | } 227 | var recorder = lemdb.recorder('myhouse.kitchen.fridge.temperature'); 228 | 229 | // sample the value every second 230 | setInterval(function(){ 231 | var value = getSensorValue(); 232 | var timestamp = getProperTime(); 233 | recorder(value, timestamp, function(){ 234 | console.log(timestamp + ':' + value); 235 | }) 236 | }, 1000) 237 | 238 | ``` 239 | 240 | ## events 241 | 242 | #### `lemdb.on('index', function(key, meta){})` 243 | 244 | the 'index' event is emitted when a node is added to the index: 245 | 246 | ```js 247 | lemdb.on('index', function(key, meta){ 248 | console.log('the key is: ' + key); 249 | 250 | // the meta is a string 251 | var obj = JSON.parse(meta); 252 | console.dir(obj); 253 | }) 254 | ``` 255 | 256 | #### `lemdb.on('data', function(key, value){})` 257 | 258 | This is a livestream from leveldb and so contains a full description of the operation: 259 | 260 | ```js 261 | lemdb.on('index', function(data){ 262 | console.dir(data); 263 | }) 264 | ``` 265 | 266 | This would log: 267 | 268 | ```js 269 | { type: 'put', 270 | key: 'values~cars~red5~speed~1394886656496', 271 | value: '85' 272 | } 273 | ``` 274 | 275 | ## license 276 | 277 | MIT 278 | -------------------------------------------------------------------------------- /bin/c3d2.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /** 4 | * Module dependencies. 5 | */ 6 | 7 | var program = require('commander'); 8 | var version = require('../package.json').version; 9 | var C3D2 = require('../src'); 10 | var async = require('async'); 11 | var prompt = require('cli-prompt'); 12 | var path = require('path'); 13 | var fs = require('fs'); 14 | 15 | program 16 | .option('-d, --dir ', 'where to create the android app', '.') 17 | .option('-a, --assets ', 'where to copy the HTML5 app from') 18 | .option('-n, --name ', 'the folder name of the app') 19 | .option('-t, --title ', 'the title of the app') 20 | .option('-p, --package ', 'the package name of the app') 21 | .version(version) 22 | 23 | 24 | 25 | program 26 | .command('create [dir]') 27 | .description('convert the big images to small ones') 28 | .action(function(dir){ 29 | 30 | function get_setting(name, eg, done){ 31 | var value = program[name]; 32 | 33 | if(value){ 34 | done(null, value); 35 | } 36 | else{ 37 | prompt(name + ' (e.g. ' + eg + '): ', function (val) { 38 | done(null, val); 39 | }) 40 | } 41 | } 42 | 43 | function get_dir(st){ 44 | if(st.indexOf('/')!=0){ 45 | st = process.cwd() + '/' + st; 46 | } 47 | st = path.normalize(st); 48 | return st; 49 | } 50 | 51 | async.series({ 52 | dir:function(next){ 53 | dir = get_dir(dir || program.dir); 54 | next(null, dir); 55 | }, 56 | assets:function(next){ 57 | if(program.assets){ 58 | next(null, get_dir(program.assets)); 59 | } 60 | else{ 61 | next(); 62 | } 63 | }, 64 | name:function(next){ 65 | get_setting('name', 'MyApp', next); 66 | }, 67 | title:function(next){ 68 | get_setting('title', 'My App', next); 69 | }, 70 | package:function(next){ 71 | get_setting('package', 'com.me.myapp', next); 72 | } 73 | }, function(error, settings){ 74 | 75 | var maker = new C3D2(settings); 76 | 77 | function finish(){ 78 | console.log('-------------------------------------------'); 79 | console.log('android application built: '); 80 | console.log(settings.dir); 81 | } 82 | 83 | function inject_assets(done){ 84 | maker.inject_assets(settings.assets, done); 85 | } 86 | 87 | maker.create(function(){ 88 | if(settings.assets){ 89 | inject_assets(finish); 90 | } 91 | else{ 92 | finish(); 93 | } 94 | }) 95 | 96 | }) 97 | }) 98 | 99 | program 100 | .command('*') 101 | .action(function(command){ 102 | console.log('command: "%s" not found', command); 103 | }) 104 | 105 | program.parse(process.argv); -------------------------------------------------------------------------------- /docs/plan.md: -------------------------------------------------------------------------------- 1 | lem 2 | === 3 | 4 | telemetry database for time-series data using LevelDB and node.js 5 | 6 | ## plan 7 | 8 | A 'node' is an entity in a property tree that has a history of values changing over time. 9 | 10 | Nodes are identified by pathname - either dot or slash delimeted: 11 | 12 | ``` 13 | tracktube.cars.red5.speed 14 | ``` 15 | 16 | and 17 | 18 | ``` 19 | tracktube/cars/red5/speed 20 | ``` 21 | 22 | are the same nodes - i.e. a variable that has a history of values. 23 | 24 | LevelDB is perfect for time-series data because it saves to disk in key order. 25 | 26 | The keys for actual values will include the timestamp for the value. 27 | 28 | The timestamp is either sent with the value or added on insert. 29 | 30 | An example of a full key for the above node: 31 | 32 | ``` 33 | tracktube/cars/red5/speed/123456 34 | ``` 35 | 36 | This is one value and can be read as 'the speed for tracktube.cars.red5 at timestamp: 123456' 37 | 38 | One limitation of lem is one value per node per millisecond. 39 | 40 | If you need a telemetry system with more resolution than 1/1000th of a second - lem is probably not for you : ) 41 | 42 | 43 | ### POST / 44 | 45 | Add/update a node at the given key - you can save meta data to nodes this way: 46 | 47 | ``` 48 | $ curl -X POST -d '{"meta":"apples"}' http://lem.digger.io/tracktube/cars/red5/speed 49 | ``` 50 | 51 | ### GET / 52 | 53 | Returns the data for a node at a key: 54 | 55 | ``` 56 | $ curl http://lem.digger.io/tracktube/cars/red5/speed 57 | ``` 58 | 59 | returns: 60 | 61 | ```json 62 | { 63 | "id":"tracktube.cars.red5.speed", 64 | "meta":"apples", 65 | "count":1242, 66 | "modified":23238282 67 | } 68 | ``` 69 | 70 | Three extra fields are added to the results for each node: 71 | 72 | * id 73 | * count 74 | * modified 75 | 76 | id is the node id - count represents how many values are in the nodes history - modified is the timestamp of the most recent value. 77 | 78 | ### POST /history 79 | 80 | Add a new value for a node - if the request body is JSON the object should contain: 81 | 82 | * value 83 | * timestamp 84 | 85 | ``` 86 | $ curl -X POST -d '{"value":76,"timestamp":123460}' http://lem.digger.io/tracktube/cars/red5/speed/history 87 | ``` 88 | 89 | if the request body is not JSON - the body is used as the value and a timestamp is added from the servers local time: 90 | 91 | ``` 92 | $ curl -X POST -d '76' http://lem.digger.io/tracktube/cars/red5/speed/history 93 | ``` 94 | 95 | ### GET /history 96 | 97 | Returns an array of values for a node: 98 | 99 | ``` 100 | $ curl http://lem.digger.io/tracktube/cars/red5/speed/history 101 | ``` 102 | 103 | returns: 104 | 105 | ```json 106 | { 107 | "id":"tracktube.cars.red5.speed", 108 | "results":[[ 109 | 123456,70 110 | ],[ 111 | 123457,72 112 | ],[ 113 | 123458,74 114 | ],[ 115 | 123459,75 116 | ],[ 117 | 123460,76 118 | ] 119 | } 120 | ``` 121 | 122 | ### get /history?from=&to= 123 | 124 | You can control the time-period that results are returned for using the from and to query parameters 125 | 126 | ``` 127 | $ curl http://lem.digger.io/tracktube/cars/red5/speed/history?from=123458&to=123459 128 | ``` 129 | 130 | returns: 131 | 132 | ```json 133 | { 134 | "id":"tracktube.cars.red5.speed", 135 | "results":[[ 136 | 123458,74 137 | ],[ 138 | 123459,75 139 | ] 140 | } 141 | ``` 142 | 143 | ## license 144 | 145 | MIT 146 | 147 | -------------------------------------------------------------------------------- /graphics/lemon.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binocarlos/lem/379c55421ab659f8ccce34bb4187fb517930fa2a/graphics/lemon.jpg -------------------------------------------------------------------------------- /graphics/logo.fla: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binocarlos/lem/379c55421ab659f8ccce34bb4187fb517930fa2a/graphics/logo.fla -------------------------------------------------------------------------------- /graphics/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binocarlos/lem/379c55421ab659f8ccce34bb4187fb517930fa2a/graphics/logo.png -------------------------------------------------------------------------------- /graphics/logosmall.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binocarlos/lem/379c55421ab659f8ccce34bb4187fb517930fa2a/graphics/logosmall.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lem", 3 | "version": "1.0.0", 4 | "description": "telemetry database for time-series data using LevelDB and node.js", 5 | "main": "src/index.js", 6 | "scripts": { 7 | "test": "make test" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/binocarlos/lem.git" 12 | }, 13 | "keywords": [ 14 | "telemetry", 15 | "database", 16 | "time", 17 | "series", 18 | "leveldb" 19 | ], 20 | "devDependencies": { 21 | "async": "^2.0.1", 22 | "level-test": "^2.0.2", 23 | "mocha": "^3.0.0", 24 | "should": "^10.0.0" 25 | }, 26 | "author": "Kai Davenport", 27 | "license": "MIT", 28 | "bugs": { 29 | "url": "https://github.com/binocarlos/lem/issues" 30 | }, 31 | "homepage": "https://github.com/binocarlos/lem", 32 | "dependencies": { 33 | "level-live-stream": "^1.4.11", 34 | "through": "^2.3.8" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | var EventEmitter = require('events').EventEmitter 2 | var util = require('util') 3 | var liveStream = require('level-live-stream') 4 | var through = require('through') 5 | var tools = require('./tools') 6 | 7 | module.exports = function(db, options){ 8 | 9 | if(!db){ 10 | throw new Error('db required') 11 | } 12 | 13 | options = options || {} 14 | 15 | return new Lem(db, options) 16 | } 17 | 18 | function Lem(db, options){ 19 | var self = this 20 | 21 | EventEmitter.call(this) 22 | 23 | this._db = db 24 | this._options = options 25 | 26 | this._livestream = liveStream(this._db) 27 | this._livestream.on('data', function(data){ 28 | self.emit('data', data) 29 | }) 30 | } 31 | 32 | util.inherits(Lem, EventEmitter) 33 | 34 | Lem.prototype.index = function(key, meta, done){ 35 | if(!key || !meta){ 36 | this.emit('error', 'key and value must be supplied to lem.index()') 37 | return 38 | } 39 | this.emit('index', key, meta) 40 | this._db.put('keys.' + key, meta, done) 41 | } 42 | 43 | Lem.prototype.remove = function(key, done){ 44 | // tbc 45 | throw new Error('not done yet') 46 | } 47 | 48 | Lem.prototype.recorder = function(path){ 49 | var self = this 50 | path = 'values.' + (path || '') 51 | return function(value, timestamp, done){ 52 | if(arguments.length<=2){ 53 | done = timestamp 54 | timestamp = new Date().getTime() 55 | } 56 | var valpath = path + '.' + timestamp 57 | self._db.put(valpath, value.toString(), done) 58 | } 59 | } 60 | 61 | Lem.prototype.valuestream = function(path, query){ 62 | query = query || {} 63 | var dotpath = 'values.' + (path || '') 64 | var range = tools.querykeys(dotpath, query.start, query.end) 65 | 66 | return this._db.createReadStream(range) 67 | .pipe(through(function(data){ 68 | 69 | var parts = data.key.toString().split('.') 70 | 71 | data.key = parseInt(parts[parts.length-1]) 72 | data.value = parseFloat(data.value.toString()) 73 | 74 | this.queue(data) 75 | })) 76 | } 77 | 78 | Lem.prototype.keys = function(path){ 79 | var self = this 80 | var dotpath = 'keys.' + (path || '') 81 | var range = tools.querykeys(dotpath) 82 | return this._db.createReadStream(range) 83 | .pipe(through(function(data){ 84 | data.key = data.key.toString().substr(dotpath.length+1) 85 | data.value = data.value.toString() 86 | if(data.value.charAt(0)=='{'){ 87 | data.value = JSON.parse(data.value) 88 | } 89 | this.queue(data) 90 | })) 91 | } 92 | 93 | -------------------------------------------------------------------------------- /src/tools.js: -------------------------------------------------------------------------------- 1 | function levelrange(start, end){ 2 | return { 3 | start:start, 4 | end:end + '\xff' 5 | } 6 | } 7 | 8 | function querykeys(path, starttime, endtime){ 9 | var start = path; 10 | var end = path; 11 | if(starttime){ 12 | start += '.' + starttime; 13 | } 14 | if(endtime){ 15 | end += '.' + endtime; 16 | } 17 | return levelrange(start, end); 18 | } 19 | 20 | module.exports = { 21 | querykeys:querykeys, 22 | levelrange:levelrange 23 | } 24 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --reporter spec 2 | --timeout 300 3 | --require should 4 | --growl 5 | --ui bdd -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | var lem = require('../src/index') 2 | var level = require('level-test')({mem:true}) 3 | var through = require('through') 4 | var async = require('async') 5 | 6 | describe('lem', function(){ 7 | 8 | var leveldb; 9 | 10 | beforeEach(function(done){ 11 | this.timeout(1000) 12 | leveldb = level('lemtest') 13 | done() 14 | }) 15 | 16 | afterEach(function(done){ 17 | leveldb.close(done) 18 | }) 19 | 20 | describe('constructor', function(){ 21 | 22 | it('should be a function', function(){ 23 | lem.should.be.type('function'); 24 | }) 25 | 26 | it('should throw if no leveldb or options', function(){ 27 | (function(){ 28 | var lemdb = lem(); 29 | }).should.throw('db required'); 30 | }) 31 | 32 | it('should create a lem server which should be an event emitter', function(done){ 33 | var lemdb = lem(leveldb); 34 | 35 | lemdb.on('apples', done); 36 | lemdb.emit('apples'); 37 | }) 38 | 39 | }) 40 | 41 | describe('keys', function(){ 42 | 43 | 44 | it('should not throw if no callback is supplied', function(done){ 45 | var lemdb = lem(leveldb); 46 | 47 | (function(){ 48 | lemdb.index('cars.red5.speed', 'apples'); 49 | }).should.not.throw() 50 | 51 | done() 52 | 53 | }) 54 | 55 | it('should emit an error if no meta value supplied', function(done){ 56 | var lemdb = lem(leveldb); 57 | 58 | lemdb.on('error', function(err){ 59 | err.should.equal('key and value must be supplied to lem.index()') 60 | done() 61 | }) 62 | 63 | 64 | lemdb.index('cars.red5.speed', null); 65 | 66 | }) 67 | 68 | it('should list all the nodes that have been indexed', function(done){ 69 | var lemdb = lem(leveldb); 70 | 71 | 72 | 73 | async.series([ 74 | function(next){ 75 | lemdb.index('cars.red5.speed', 10, next); 76 | }, 77 | 78 | function(next){ 79 | lemdb.index('cars.red5.address.postcode', 'sw10', next); 80 | }, 81 | 82 | function(next){ 83 | lemdb.index('cars.red5.height', 11, next); 84 | }, 85 | 86 | function(next){ 87 | lemdb.index('cars.red5.weight', 12, next); 88 | }, 89 | 90 | function(next){ 91 | var nodes = {}; 92 | lemdb.keys('cars.red5').pipe(through(function(data){ 93 | nodes[data.key] = data.value; 94 | }, function(){ 95 | nodes['speed'].should.equal('10'); 96 | nodes['height'].should.equal('11'); 97 | nodes['weight'].should.equal('12'); 98 | nodes['address.postcode'].should.equal('sw10'); 99 | Object.keys(nodes).length.should.equal(4); 100 | done(); 101 | })) 102 | } 103 | ], done) 104 | 105 | }) 106 | 107 | 108 | 109 | it('should split the keys properly in a valuestream', function(done){ 110 | var lemdb = lem(leveldb); 111 | 112 | var recorder = lemdb.recorder('cars.red4'); 113 | 114 | 115 | async.series([ 116 | function(next){ 117 | 118 | recorder(10, next) 119 | 120 | }, 121 | 122 | function(next){ 123 | 124 | setTimeout(next, 100) 125 | 126 | }, 127 | 128 | function(next){ 129 | lemdb.valuestream('cars.red4', { 130 | 131 | }).pipe(through(function(data){ 132 | 133 | var t = typeof(data.key) 134 | t.should.equal('number') 135 | data.value.should.equal(10) 136 | done() 137 | 138 | })) 139 | } 140 | ], done) 141 | 142 | }) 143 | }) 144 | 145 | describe('events', function(){ 146 | 147 | 148 | it('should emit events as data is written', function(done){ 149 | var lemdb = lem(leveldb); 150 | 151 | var hit = {}; 152 | var index = {}; 153 | 154 | lemdb.on('data', function(data){ 155 | hit[data.key] = data; 156 | }) 157 | 158 | lemdb.on('index', function(key, data){ 159 | index[key] = data; 160 | }) 161 | 162 | function delayNext(next){ 163 | setTimeout(next, 10) 164 | } 165 | 166 | async.series([ 167 | function(next){ 168 | lemdb.index('cars.red5.speed', 10, delayNext(next)); 169 | }, 170 | 171 | function(next){ 172 | lemdb.index('cars.red5.address.postcode', 'sw10', delayNext(next)); 173 | }, 174 | 175 | function(next){ 176 | lemdb.index('cars.red5.height', 11, delayNext(next)); 177 | }, 178 | 179 | function(next){ 180 | lemdb.index('cars.red5.weight', 12, delayNext(next)); 181 | }, 182 | 183 | function(next){ 184 | hit['keys.cars.red5.speed'].value.should.equal(10); 185 | hit['keys.cars.red5.address.postcode'].value.should.equal('sw10'); 186 | hit['keys.cars.red5.height'].value.should.equal(11); 187 | hit['keys.cars.red5.weight'].value.should.equal(12); 188 | index['cars.red5.speed'].should.equal(10); 189 | next(); 190 | } 191 | ], done) 192 | 193 | }) 194 | 195 | }) 196 | 197 | describe('recorder', function(){ 198 | 199 | this.timeout(5000); 200 | 201 | it('should save and load values', function(done){ 202 | var lemdb = lem(leveldb); 203 | 204 | var recorder = lemdb.recorder('cars.red5.speed'); 205 | 206 | var counter = 0; 207 | var total = 0; 208 | var midtotal = 0; 209 | 210 | var midtime = null; 211 | var endtime = null; 212 | 213 | function docheckrange(){ 214 | var hitc = 0; 215 | var hitt = 0; 216 | lemdb.valuestream('cars.red5.speed', { 217 | start:midtime, 218 | end:endtime 219 | }).pipe(through(function(data){ 220 | hitc++; 221 | hitt += data.value; 222 | }, function(){ 223 | hitc.should.equal(5); 224 | hitt.should.equal(midtotal); 225 | done(); 226 | })) 227 | } 228 | 229 | function docheckall(){ 230 | var hitc = 0; 231 | var hitt = 0; 232 | lemdb.valuestream('cars.red5.speed', { 233 | 234 | }).pipe(through(function(data){ 235 | hitc++; 236 | hitt += data.value; 237 | }, function(){ 238 | hitc.should.equal(10); 239 | hitt.should.equal(total); 240 | docheckrange(); 241 | })) 242 | } 243 | 244 | function dorecord(){ 245 | if(counter>=10){ 246 | endtime = new Date().getTime(); 247 | docheckall(); 248 | return; 249 | } 250 | var speed = 50 + Math.round(Math.random()*50); 251 | total += speed; 252 | counter++; 253 | if(counter==6){ 254 | midtime = new Date().getTime(); 255 | } 256 | if(counter>=6){ 257 | midtotal += speed; 258 | } 259 | recorder(speed, function(){ 260 | setTimeout(dorecord, 100); 261 | }) 262 | 263 | } 264 | 265 | dorecord(); 266 | 267 | }) 268 | 269 | }) 270 | 271 | 272 | }) 273 | 274 | 275 | --------------------------------------------------------------------------------