├── .gitignore ├── .npmignore ├── LICENCE ├── README.md ├── bench.js ├── chaos.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | benchtest 2 | databasetest 3 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | benchtest 2 | databasetest 3 | -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010 George Stagakis 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | chaos - a node.js database 2 | ========================== 3 | 4 | Why chaos? Because we exploit the sha1 chaotic randomness to store the keys evenly in the filesystem. 5 | 6 | ## NEW VERSION WARNING 7 | 8 | This version is incompatible with previous. No longer creates directories, as it turns out 9 | it gets really slow and has no real gain over storing everything in the same directory. 10 | New commands `jset` and `jget` are to be used when you want to store a lot of small values, 11 | they perform better than regular keys and hkeys. 12 | 13 | ## Installation 14 | 15 | npm install chaos 16 | 17 | ## Usage 18 | 19 | var db = require('chaos')('your database path') 20 | 21 | ## Commands 22 | 23 | ### db.set(key, val, function(err) {}) 24 | 25 | Sets a key value pair. 26 | 27 | ### db.get(key, function(err, val) {}) 28 | 29 | Gets the value of a key. 30 | 31 | ### db.del(key, function(err) {}) 32 | 33 | Deletes a key. 34 | 35 | ### db.incr(key, function(err, new_number) {}) 36 | ### db.decr(key, function(err, new_number) {}) 37 | 38 | Increment or decrement a key value by 1 and return the new number. If a key doesn't exist or its value isn't a number it will be created starting from 0. Therefore will return 1 or -1 respectively. 39 | 40 | ### db.getset(key, val, function(err, old_val) {}) 41 | Get a key value and set another afterwards. 42 | 43 | ### db.getdel(key, function(err, val) {}) 44 | Get a key value and delete it afterwards. 45 | 46 | ### db.getorsetget(key, default_value, function(err, val) {}) 47 | Get a key's value or if it doesn't exist, set the value and get it afterwards. 48 | 49 | ### db.hset(hkey, field, val, function(err) {}) 50 | Set a hkey field value. 51 | 52 | ### db.hget(hkey, field, function(err, val) {}) 53 | Get the value of a hkey field. 54 | 55 | ### db.hdel(hkey, field, function(err) {}) 56 | Delete a hkey field. 57 | 58 | _Warning:_ This deletes only a field, to delete the hkey itself, use `db.del`. 59 | 60 | ### db.hgetall(hkey, function(err, field_value_object) {}) 61 | Get all field value pairs from a hkey. Returns an object with fields as keys and their values. 62 | 63 | ### db.hkeys(hkey, function(err, fields_array) {}) 64 | Get all field names from a hkey. Returns an unsorted array with the field names. 65 | 66 | ### db.hvals(hkey, function(err, values_array) {}) 67 | Get all field values from a hkey. Returns an unsorted array with the field values. 68 | 69 | ### db.jset(key, val, function(err) {}) 70 | Set a jkey value pair. 71 | 72 | ### db.jget(key, function(err, val) {}) 73 | Gets the value of a jkey. 74 | 75 | ### db.watch(key, [options,] function(err, val) {}) 76 | Watch a key for changes and attempt a `db.get`. If options is provided, it is passed on to 77 | `fs.watchFile` and it's an object. The default is `{persistent: true, interval: 0}`. 78 | 79 | _Notes:_ This maps to fs.watchFile and thus is not very reliable as to when it's going to fire the callback. Don't trust it will fire on every key change. 80 | Also on hkeys, it fires _only_ when a field is added or removed from the hkey, not when fields change. 81 | 82 | ### db.unwatch(key) 83 | Stop watching a key. 84 | 85 | ## Future 86 | 87 | * More commands 88 | * Better tests 89 | * Optimizations 90 | * Who knows? 91 | 92 | Contributions are welcome! :) 93 | 94 | ## Disclaimer 95 | 96 | It's still just a proof of concept, no real life tests are done. 97 | 98 | ## Questions? 99 | 100 | Find me on Twitter @stagas and IRC freenode.net #node.js as stagas 101 | -------------------------------------------------------------------------------- /bench.js: -------------------------------------------------------------------------------- 1 | // bench for chaos 2 | 3 | var db = require('./chaos')('benchtest') 4 | , assert = require('assert') 5 | 6 | var best = {writes: 0, hwrites:0, jwrites:0, reads: 0, hreads: 0, jreads: 0 } 7 | , avg = {writes: 0, writesCnt: 0, hwrites: 0, hwritesCnt: 0, jwrites: 0, jwritesCnt: 0 8 | , reads: 0, readsCnt: 0, hreads: 0, hreadsCnt: 0, jreads:0, jreadsCnt: 0 9 | } 10 | 11 | db.maxOpenFiles = 30 12 | 13 | var big = [] 14 | for (var i=1000; i--; ) { 15 | big.push(i) 16 | } 17 | 18 | var objects = [ 19 | JSON.stringify('tiny') 20 | 21 | , JSON.stringify('hello I am a medium sized string') 22 | 23 | , JSON.stringify({ 24 | there: 'is' 25 | , only: 'chaos' 26 | , butterfly: ['says', ['there', 'is', 'only', 'chaos']] 27 | , pi: Math.PI 28 | }) 29 | 30 | , JSON.stringify({ 31 | there: 'is' 32 | , only: 'chaos' 33 | , butterfly: ['says', ['there', 'is', 'only', 'chaos']] 34 | , pi: Math.PI 35 | , big: big 36 | }) 37 | ] 38 | 39 | function bench(obj, what, num, cb) { 40 | console.log(' obj length:', obj.length) 41 | console.log(' operations:', num) 42 | console.log('-------------------') 43 | 44 | switch (what) { 45 | case 'all': 46 | sets(obj, num, function() { 47 | gets(obj, num, function() { 48 | hsets(obj, num, function() { 49 | hgets(obj, num, function() { 50 | jsets(JSON.parse(obj), num, function() { 51 | jgets(JSON.parse(obj), num, function() { 52 | console.log('') 53 | cb() 54 | }) 55 | }) 56 | }) 57 | }) 58 | }) 59 | }) 60 | break 61 | case 'sets': 62 | sets(obj, num, function() { 63 | console.log('') 64 | cb() 65 | }) 66 | break 67 | case 'gets': 68 | gets(obj, num, function() { 69 | console.log('') 70 | cb() 71 | }) 72 | break 73 | case 'hsets': 74 | hsets(obj, num, function() { 75 | console.log('') 76 | cb() 77 | }) 78 | break 79 | case 'hgets': 80 | hgets(obj, num, function() { 81 | console.log('') 82 | cb() 83 | }) 84 | break 85 | case 'jsets': 86 | jsets(JSON.parse(obj), num, function() { 87 | console.log('') 88 | cb() 89 | }) 90 | break 91 | case 'jgets': 92 | jgets(JSON.parse(obj), num, function() { 93 | console.log('') 94 | cb() 95 | }) 96 | break 97 | default: 98 | cb() 99 | break 100 | } 101 | } 102 | 103 | function sets(obj, num, cb) { 104 | var done = 0 105 | , clients = 0 106 | , timer = new Date() 107 | 108 | for (var i=num; i--; ) { 109 | db.set(i, obj, function(err) { 110 | done++ 111 | if (done === num) { 112 | var result = ( (num) / ((new Date() - timer) / 1000)) 113 | if (result > best.writes) best.writes = result 114 | avg.writes += result 115 | avg.writesCnt += 1 116 | console.log('writes:', result.toFixed(2) + '/s') 117 | cb() 118 | } 119 | }) 120 | } 121 | } 122 | 123 | function gets(obj, num, cb) { 124 | var done = 0 125 | , clients = 0 126 | , timer = new Date() 127 | 128 | for (var i=num; i--; ) { 129 | db.get(i, function(err, data) { 130 | done++ 131 | if (done === num) { 132 | var result = ( (num) / ((new Date() - timer) / 1000)) 133 | if (result > best.reads) best.reads = result 134 | avg.reads += result 135 | avg.readsCnt += 1 136 | console.log('reads:', result.toFixed(2) + '/s') 137 | cb() 138 | } 139 | }) 140 | } 141 | } 142 | 143 | function hsets(obj, num, cb) { 144 | var done = 0 145 | , clients = 0 146 | , timer = new Date() 147 | 148 | for (var i=num; i--; ) { 149 | db.hset('hkey', i, obj, function(err) { 150 | done++ 151 | if (done === num) { 152 | var result = ( (num) / ((new Date() - timer) / 1000)) 153 | if (result > best.hwrites) best.hwrites = result 154 | avg.hwrites += result 155 | avg.hwritesCnt += 1 156 | console.log('hkey writes:', result.toFixed(2) + '/s') 157 | cb() 158 | } 159 | }) 160 | } 161 | } 162 | 163 | function hgets(obj, num, cb) { 164 | var done = 0 165 | , clients = 0 166 | , timer = new Date() 167 | 168 | for (var i=num; i--; ) { 169 | db.hget('hkey', i, function(err, data) { 170 | done++ 171 | if (done === num) { 172 | var result = ( (num) / ((new Date() - timer) / 1000)) 173 | if (result > best.hreads) best.hreads = result 174 | avg.hreads += result 175 | avg.hreadsCnt += 1 176 | console.log('hkey reads:', result.toFixed(2) + '/s') 177 | cb() 178 | } 179 | }) 180 | } 181 | } 182 | 183 | function jsets(obj, num, cb) { 184 | var done = 0 185 | , clients = 0 186 | , timer = new Date() 187 | 188 | for (var i=num; i--; ) { 189 | db.jset(i, obj, function(err) { 190 | done++ 191 | if (done === num) { 192 | var result = ( (num) / ((new Date() - timer) / 1000)) 193 | if (result > best.jwrites) best.jwrites = result 194 | avg.jwrites += result 195 | avg.jwritesCnt += 1 196 | console.log('jkey writes:', result.toFixed(2) + '/s') 197 | cb() 198 | } 199 | }) 200 | } 201 | } 202 | 203 | function jgets(obj, num, cb) { 204 | var done = 0 205 | , clients = 0 206 | , timer = new Date() 207 | 208 | for (var i=num; i--; ) { 209 | db.jget(i, function(err, data) { 210 | done++ 211 | if (done === num) { 212 | var result = ( (num) / ((new Date() - timer) / 1000)) 213 | if (result > best.jreads) best.jreads = result 214 | avg.jreads += result 215 | avg.jreadsCnt += 1 216 | console.log('jkey reads:', result.toFixed(2) + '/s') 217 | cb() 218 | } 219 | }) 220 | } 221 | } 222 | 223 | var scenario = [ 224 | 225 | ['all', 2000] 226 | , ['all', 2000] 227 | //, ['all', 1000] 228 | //, ['all', 2000] 229 | //, ['sets', 5000] 230 | //, ['sets', 10000] 231 | 232 | //, ['gets', 5000] 233 | //, ['gets', 10000] 234 | 235 | ] 236 | 237 | var scenarioLen = scenario.length 238 | 239 | var next = function(i, o) { 240 | if (i < scenarioLen) { 241 | bench(objects[o], scenario[i][0], scenario[i][1], function() { 242 | setTimeout(function() { 243 | next(++i, o) 244 | }, scenario[i][1] / 3) // give some time for the hd to breath 245 | }) 246 | } else { 247 | o++ 248 | if (o===objects.length) { 249 | console.log('---------------------') 250 | console.log('version:', db.version) 251 | console.log('max open files:', db.maxOpenFiles) 252 | console.log('') 253 | console.log('best writes:', best.writes.toFixed(2) + '/s') 254 | console.log('best hkey writes:', best.hwrites.toFixed(2) + '/s') 255 | console.log('best jkey writes:', best.jwrites.toFixed(2) + '/s') 256 | console.log('best reads:', best.reads.toFixed(2) + '/s') 257 | console.log('best hkey reads:', best.hreads.toFixed(2) + '/s') 258 | console.log('best jkey reads:', best.jreads.toFixed(2) + '/s') 259 | console.log('avg writes:', (avg.writes / avg.writesCnt).toFixed(2) + '/s') 260 | console.log('avg hwrites:', (avg.hwrites / avg.hwritesCnt).toFixed(2) + '/s') 261 | console.log('avg jwrites:', (avg.jwrites / avg.jwritesCnt).toFixed(2) + '/s') 262 | console.log('avg reads:', (avg.reads / avg.readsCnt).toFixed(2) + '/s') 263 | console.log('avg hreads:', (avg.hreads / avg.hreadsCnt).toFixed(2) + '/s') 264 | console.log('avg jreads:', (avg.jreads / avg.jreadsCnt).toFixed(2) + '/s') 265 | console.log('---------------------') 266 | console.log('') 267 | console.log('all done!') 268 | } else { 269 | next(0, o) 270 | } 271 | } 272 | } 273 | 274 | var consistency = function(cb) { 275 | var done = 0 276 | , num = 100 277 | 278 | console.log('writes...') 279 | for (var i=num; i--; ) { 280 | db.set(i, '1234567890', function(err) { 281 | done++ 282 | if (done===num) { 283 | done = 0 284 | console.log('reads...') 285 | for (var i=num; i--; ) { 286 | db.get(i, function(err, data) { 287 | done++ 288 | assert.equal(data, '1234567890', 'Consistency error!') 289 | if (done===num) { 290 | doesitwork(cb) 291 | } 292 | }) 293 | } 294 | } 295 | }) 296 | } 297 | } 298 | 299 | var doesitwork = function(cb) { 300 | var cnt = 0 301 | , max = 6 302 | 303 | var cntincr = 0 304 | db.del('incr', function(err) { 305 | for (var i=50; i--; ) { 306 | db.incr('incr', function(err, number) { 307 | cntincr++ 308 | if (cntincr == 50) { 309 | console.log('incr test:', number) 310 | cnt++ 311 | if (cnt === max) cb() 312 | } 313 | }) 314 | } 315 | }) 316 | var cntdecr = 0 317 | db.del('decr', function(err) { 318 | for (var i=50; i--; ) { 319 | db.decr('decr', function(err, number) { 320 | cntdecr++ 321 | if (cntdecr == 50) { 322 | console.log('decr test:', number) 323 | cnt++ 324 | if (cnt === max) cb() 325 | } 326 | }) 327 | } 328 | }) 329 | db.getorsetget('getorsetget', 'ok', function(err, data) { 330 | cnt++ 331 | console.log('getorsetget:', 'ok') 332 | if (cnt === max) cb() 333 | }) 334 | db.set('getset', 'hello', function(err) { 335 | db.getset('getset', 'world', function(err, data) { 336 | cnt++ 337 | console.log('getset:', data) 338 | if (cnt === max) cb() 339 | }) 340 | }) 341 | db.set('getdel', 'hello', function(err) { 342 | db.getdel('getdel', function(err, data) { 343 | db.get('getdel', function(err, data2) { 344 | cnt++ 345 | console.log('getdel:', data, data2) 346 | if (cnt === max) cb() 347 | }) 348 | }) 349 | }) 350 | 351 | var cntmass = 0 352 | for (var i=50; i--;) { 353 | db.set('mass', 'writes', function(err) { 354 | cntmass++ 355 | if (err) throw err 356 | if (cntmass===50) { 357 | db.get('mass', function(err, data) { 358 | console.log('mass:', data) 359 | cnt++ 360 | if (cnt === max) cb() 361 | }) 362 | } 363 | }) 364 | } 365 | } 366 | 367 | var start = function() { 368 | if (!db.ready) { 369 | setTimeout(function() { start() }, 1000) 370 | } else { 371 | console.log('checking consistency...') 372 | consistency(function() { 373 | console.log('done.') 374 | console.log('=====================') 375 | console.log('benchmark starting...') 376 | console.log('') 377 | next(0, 0) 378 | }) 379 | } 380 | } 381 | 382 | start() 383 | -------------------------------------------------------------------------------- /chaos.js: -------------------------------------------------------------------------------- 1 | /* 2 | * chaos v0.2.0 3 | * 4 | * by stagas 5 | * 6 | */ 7 | 8 | var sys 9 | , fs = require('fs') 10 | , path = require('path') 11 | , crypto = require('crypto') 12 | , EventEmitter = require('events').EventEmitter 13 | 14 | try { 15 | sys = require('util') 16 | } catch (err) { 17 | sys = require('sys') 18 | } 19 | 20 | var VALID_FILENAME = new RegExp('([^a-zA-Z0-9 \-\_\.])', 'g') 21 | 22 | // to_array from mranney / node_redis 23 | function to_array(args) { 24 | var len = args.length, 25 | arr = new Array(len), i; 26 | 27 | for (i = 0; i < len; i += 1) { 28 | arr[i] = args[i]; 29 | } 30 | 31 | return arr; 32 | } 33 | 34 | // creationix's fast Queue 35 | var Queue = function() { 36 | this.tail = []; 37 | this.head = to_array(arguments); 38 | this.offset = 0; 39 | // Lock the object down 40 | Object.seal(this); 41 | } 42 | 43 | Queue.prototype = { 44 | shift: function shift() { 45 | if (this.offset === this.head.length) { 46 | var tmp = this.head; 47 | tmp.length = 0; 48 | this.head = this.tail; 49 | this.tail = tmp; 50 | this.offset = 0; 51 | if (this.head.length === 0) return; 52 | } 53 | return this.head[this.offset++]; 54 | }, 55 | push: function push(item) { 56 | return this.tail.push(item); 57 | }, 58 | get length() { 59 | return this.head.length - this.offset + this.tail.length; 60 | } 61 | } 62 | 63 | var Chaos = exports.Chaos = function(dbName) { 64 | if (!(this instanceof Chaos)) return new Chaos(dbName) 65 | 66 | EventEmitter.call(this) 67 | 68 | var self = this 69 | this.version = 'v0.2.0' 70 | this.dbName = dbName 71 | this.ready = false 72 | 73 | path.exists(this.dbName, function(exists) { 74 | if (!exists) { 75 | self.__createDB(self.dbName) 76 | } else { 77 | self.ready = true 78 | } 79 | }) 80 | 81 | this.__hashAlgo = 'sha1' 82 | this.__hashEnc = 'hex' 83 | 84 | this.maxOpenFiles = 30 85 | this.__openFiles = 0 86 | 87 | this.__queue__ = new Queue() 88 | this.__queued = false 89 | 90 | this.__busy__ = {} 91 | 92 | this.on('queue', function() { 93 | if (!self.__queued) { 94 | self.__queued = true 95 | self.__flush() 96 | } 97 | }) 98 | } 99 | 100 | sys.inherits(Chaos, EventEmitter) 101 | Chaos.Chaos = Chaos 102 | module.exports = Chaos 103 | 104 | Chaos.prototype.__busy = function(key) { 105 | this.__openFiles++ 106 | this.__busy__[key] = true 107 | } 108 | 109 | Chaos.prototype.__free = function(key) { 110 | this.__openFiles-- 111 | delete this.__busy__[key] 112 | } 113 | 114 | Chaos.prototype.__createDB = function(dir) { 115 | var self = this 116 | 117 | fs.mkdirSync(dir, 0755) 118 | self.ready = true 119 | } 120 | 121 | Chaos.prototype.__hash = function(key) { 122 | return crypto.createHash(this.__hashAlgo).update(key.toString()).digest(this.__hashEnc) 123 | } 124 | 125 | Chaos.prototype.__queue = function(a, b) { 126 | this.__queue__.push([a, b]) 127 | 128 | this.emit('queue') 129 | } 130 | 131 | Chaos.prototype.__flush = function() { 132 | var oper 133 | 134 | if (this.__queue__.length) { 135 | if (this.ready && this.__openFiles < this.maxOpenFiles) { 136 | oper = this.__queue__.shift() 137 | this[oper[0]].apply(this, oper[1]) 138 | } 139 | 140 | var self = this 141 | 142 | process.nextTick(function() { 143 | self.__flush() 144 | }) 145 | } else { 146 | this.__queued = false 147 | } 148 | } 149 | 150 | function rmdir (dirname, cb) { 151 | fs.readdir(dirname, function(err, files) { 152 | if (err) { 153 | if (cb) cb(err) 154 | return 155 | } 156 | 157 | var counter = files.length 158 | if (!counter) { 159 | fs.rmdir(dirname, cb) 160 | return 161 | } 162 | 163 | for (var i=files.length; i--; ) { 164 | ;(function(file) { 165 | fs.unlink(dirname + '/' + file, function(err) { 166 | if (!--counter && cb) { 167 | fs.rmdir(dirname, cb) 168 | } 169 | }) 170 | }(files[i])) 171 | } 172 | }) 173 | } 174 | 175 | function prepare (val) { 176 | switch (typeof val) { 177 | case 'string': 178 | break 179 | case 'number': 180 | case 'function': 181 | val = val.toString() 182 | break 183 | default: 184 | val = JSON.stringify(val) 185 | break 186 | } 187 | return val 188 | } 189 | 190 | function parse (s) { 191 | var data 192 | try { 193 | data = JSON.parse(s) 194 | } catch(e) { 195 | data = s 196 | } 197 | return data 198 | } 199 | 200 | // COMMANDS 201 | 202 | Chaos.prototype._set = function(key, val, cb) { 203 | if (typeof this.__busy__[key] != 'undefined') return this.set(key, val, cb) 204 | 205 | var self = this 206 | , filename = self.dbName +'/'+ this.__hash(key) 207 | 208 | val = prepare(val) 209 | 210 | this.__busy(key) 211 | 212 | fs.writeFile(filename, val, 'utf8', function(err) { 213 | self.__free(key) 214 | 215 | if (cb) cb(err) 216 | }) 217 | } 218 | 219 | Chaos.prototype._get = function(key, cb) { 220 | if (typeof this.__busy__[key] != 'undefined') return this.get(key, cb) 221 | 222 | var self = this 223 | , filename = self.dbName +'/'+ this.__hash(key) 224 | 225 | this.__busy(key) 226 | 227 | fs.readFile(filename, 'utf8', function(err, data) { 228 | self.__free(key) 229 | data = parse(data) 230 | if (cb) cb(err, data) 231 | }) 232 | } 233 | 234 | Chaos.prototype._del = function(key, cb) { 235 | if (typeof this.__busy__[key] != 'undefined') return this.del(key, cb) 236 | 237 | var self = this 238 | , filename = self.dbName +'/'+ this.__hash(key) 239 | 240 | this.__busy(key) 241 | 242 | fs.stat(filename, function(err, stats) { 243 | if (err) { 244 | self.__free(key) 245 | if (cb) cb(err) 246 | return 247 | } 248 | if (stats.isFile()) { 249 | fs.unlink(filename, function(err) { 250 | self.__free(key) 251 | if (cb) cb(err) 252 | }) 253 | } else if (stats.isDirectory()) { 254 | rmdir(filename, function(err) { 255 | self.__free(key) 256 | if (cb) cb(err) 257 | }) 258 | } else { 259 | self.__free(key) 260 | if (cb) cb(err) 261 | } 262 | }) 263 | } 264 | 265 | Chaos.prototype._getset = function(key, val, cb) { 266 | if (typeof this.__busy__[key] != 'undefined') return this.getset(key, val, cb) 267 | 268 | var self = this 269 | , filename = self.dbName +'/'+ this.__hash(key) 270 | 271 | this.__busy(key) 272 | 273 | fs.readFile(filename, 'utf8', function(err, data) { 274 | val = prepare(val) 275 | 276 | fs.writeFile(filename, val, 'utf8', function(err) { 277 | self.__free(key) 278 | cb && cb(err, parse(data)) 279 | }) 280 | }) 281 | } 282 | 283 | Chaos.prototype._getdel = function(key, cb) { 284 | if (typeof this.__busy__[key] != 'undefined') return this.getdel(key, cb) 285 | 286 | var self = this 287 | , filename = self.dbName +'/'+ this.__hash(key) 288 | 289 | this.__busy(key) 290 | 291 | fs.readFile(filename, 'utf8', function(err, data) { 292 | fs.unlink(filename, function(err) { 293 | self.__free(key) 294 | cb && cb(err, parse(data)) 295 | }) 296 | }) 297 | } 298 | 299 | Chaos.prototype._getorsetget = function(key, val, cb) { 300 | if (typeof this.__busy__[key] != 'undefined') return this.getorsetget(key, val, cb) 301 | 302 | var self = this 303 | , filename = self.dbName +'/'+ this.__hash(key) 304 | 305 | this.__busy(key) 306 | 307 | fs.readFile(filename, 'utf8', function(err, data) { 308 | if (err) { 309 | val = prepare(val) 310 | fs.writeFile(filename, val, 'utf8', function(err) { 311 | self.__free(key) 312 | cb && cb(err, parse(val)) 313 | }) 314 | } else { 315 | self.__free(key) 316 | cb && cb(err, parse(data)) 317 | } 318 | }) 319 | } 320 | 321 | Chaos.prototype._incr = function(key, cb) { 322 | if (typeof this.__busy__[key] != 'undefined') return this.incr(key, cb) 323 | 324 | var self = this 325 | , filename = self.dbName +'/'+ this.__hash(key) 326 | 327 | this.__busy(key) 328 | 329 | var num = 0 330 | fs.readFile(filename, 'utf8', function(err, data) { 331 | if (!err) { 332 | num = parseInt(data, 10) 333 | if (isNaN(num)) num = 0 334 | } 335 | 336 | num++ 337 | 338 | fs.writeFile(filename, num.toString(), 'utf8', function(err) { 339 | self.__free(key) 340 | cb && cb(err, num) 341 | }) 342 | }) 343 | } 344 | 345 | Chaos.prototype._decr = function(key, cb) { 346 | if (typeof this.__busy__[key] != 'undefined') return this.decr(key, cb) 347 | 348 | var self = this 349 | , filename = self.dbName +'/'+ this.__hash(key) 350 | 351 | this.__busy(key) 352 | 353 | var num = 0 354 | fs.readFile(filename, 'utf8', function(err, data) { 355 | if (!err) { 356 | num = parseInt(data, 10) 357 | if (isNaN(num)) num = 0 358 | } 359 | 360 | num-- 361 | 362 | fs.writeFile(filename, num.toString(), 'utf8', function(err) { 363 | self.__free(key) 364 | cb && cb(err, num) 365 | }) 366 | }) 367 | } 368 | 369 | Chaos.prototype._hset = function(key, field, val, cb) { 370 | if (typeof this.__busy__[key] != 'undefined') return this.hset(key, field, val, cb) 371 | 372 | if (typeof field != 'string') field = field.toString() 373 | 374 | var self = this 375 | , dirname = self.dbName +'/'+ this.__hash(key) 376 | , filename = field.toString().replace(VALID_FILENAME, '') 377 | 378 | if (filename.length == 0) { 379 | if (cb) cb(new Error('Invalid field name (must be [a-zA-Z0-9 ]): ' + field)) 380 | return 381 | } 382 | 383 | val = prepare(val) 384 | 385 | filename = dirname +'/'+ filename 386 | 387 | this.__busy(key) 388 | 389 | fs.mkdir(dirname, 0755, function(err) { 390 | fs.writeFile(filename, val, 'utf8', function(err) { 391 | self.__free(key) 392 | cb && cb(err) 393 | }) 394 | }) 395 | } 396 | 397 | Chaos.prototype._hget = function(key, field, cb) { 398 | if (typeof this.__busy__[key] != 'undefined') return this.hget(key, field, cb) 399 | 400 | var self = this 401 | , dirname = this.dbName +'/'+ this.__hash(key) 402 | , filename = field.toString().replace(VALID_FILENAME, '') 403 | 404 | if (filename.length == 0) { 405 | return cb && cb(new Error('Invalid field name (must be [a-zA-Z0-9 -_]): ' + field)) 406 | } 407 | 408 | filename = dirname +'/'+ filename 409 | 410 | this.__busy(key) 411 | 412 | fs.readFile(filename, 'utf8', function(err, data) { 413 | self.__free(key) 414 | cb && cb(err, parse(data)) 415 | }) 416 | } 417 | 418 | Chaos.prototype._hdel = function(key, field, cb) { 419 | if (typeof this.__busy__[key] != 'undefined') return this.hdel(key, field, cb) 420 | 421 | var self = this 422 | , dirname = self.dbName +'/'+ this.__hash(key) 423 | , filename = field.toString().replace(VALID_FILENAME, '') 424 | 425 | if (filename.length == 0) { 426 | return cb && cb(new Error('Invalid field name (must be [a-zA-Z0-9 ]): ' + field)) 427 | } 428 | 429 | filename = dirname +'/'+ filename 430 | 431 | this.__busy(key) 432 | 433 | fs.unlink(filename, function(err) { 434 | self.__free(key) 435 | if (cb) cb(err) 436 | }) 437 | } 438 | 439 | Chaos.prototype._hgetall = function(key, cb) { 440 | if (typeof this.__busy__[key] != 'undefined') return this.hgetall(key, cb) 441 | 442 | var self = this 443 | , dirname = self.dbName +'/'+ this.__hash(key) 444 | 445 | this.__busy(key) 446 | 447 | fs.readdir(dirname, function(err, files) { 448 | if (err) { 449 | self.__free(key) 450 | return cb && cb(err) 451 | } 452 | 453 | var counter = files.length 454 | , keyvals = {} 455 | 456 | if (!counter) { 457 | self.__free(key) 458 | return cb && cb(null, keyvals) 459 | } 460 | 461 | dirname += '/' 462 | 463 | for (var i=files.length; i--; ) { 464 | ;(function(file) { 465 | fs.readFile(dirname + file, 'utf8', function(err, data) { 466 | if (!err) keyvals[file] = parse(data) 467 | if (!--counter) { 468 | self.__free(key) 469 | cb && cb(null, keyvals) 470 | } 471 | }) 472 | }(files[i])) 473 | } 474 | }) 475 | } 476 | 477 | Chaos.prototype._hkeys = function(key, cb) { 478 | if (typeof this.__busy__[key] != 'undefined') return this.hkeys(key, cb) 479 | 480 | var self = this 481 | , dirname = self.dbName +'/'+ this.__hash(key) 482 | 483 | this.__busy(key) 484 | 485 | fs.readdir(dirname, function(err, files) { 486 | self.__free(key) 487 | cb && cb(err, files) 488 | }) 489 | } 490 | 491 | Chaos.prototype._hrand = function(key, cb) { 492 | if (typeof this.__busy__[key] != 'undefined') return this.hrand(key, cb) 493 | 494 | var self = this 495 | , dirname = self.dbName +'/'+ this.__hash(key) 496 | 497 | this.__busy(key) 498 | 499 | fs.readdir(dirname, function(err, files) { 500 | self.__free(key) 501 | 502 | if (files.length) { 503 | var f = files[ Math.floor(Math.random() * files.length) ] 504 | return self.hget(key, f, cb) 505 | } 506 | 507 | cb && cb(err, null) 508 | }) 509 | } 510 | 511 | Chaos.prototype._hvals = function(key, cb) { 512 | if (typeof this.__busy__[key] != 'undefined') return this.hvals(key, cb) 513 | 514 | var self = this 515 | , dirname = self.dbName +'/'+ this.__hash(key) 516 | 517 | this.__busy(key) 518 | 519 | fs.readdir(dirname, function(err, files) { 520 | if (err) { 521 | self.__free(key) 522 | return cb && cb(err) 523 | } 524 | 525 | var counter = files.length 526 | , vals = [] 527 | 528 | dirname += '/' 529 | 530 | for (var i=files.length; i--; ) { 531 | ;(function(file) { 532 | fs.readFile(dirname + file, 'utf8', function(err, data) { 533 | if (!err) vals.push(parse(data)) 534 | if (!--counter && cb) { 535 | self.__free(key) 536 | cb(null, vals) 537 | } 538 | }) 539 | }(files[i])) 540 | } 541 | }) 542 | } 543 | 544 | Chaos.prototype.__append = function(filename, key, val, cb) { 545 | var buf = new Buffer(key + '\t' + JSON.stringify(val) + '\n') 546 | fs.open(filename, 'a+', 0644, function(err, fd) { 547 | fs.write(fd, buf, 0, buf.length, null, function(err, written) { 548 | fs.close(fd, cb) 549 | }) 550 | }) 551 | } 552 | 553 | Chaos.prototype._jset = function(key, val, cb) { 554 | if (typeof this.__busy__[key] != 'undefined') return this.jset(key, val, cb) 555 | 556 | var self = this 557 | , hash = this.__hash(key) 558 | , filename = self.dbName +'/'+ hash.substr(0, 3) 559 | 560 | this.__busy(key) 561 | 562 | this.__append(filename, key, val, function(err) { 563 | self.__free(key) 564 | 565 | if (cb) cb(err) 566 | }) 567 | } 568 | 569 | Chaos.prototype._jget = function(key, cb) { 570 | if (typeof this.__busy__[key] != 'undefined') return this.jget(key, cb) 571 | 572 | var self = this 573 | , hash = this.__hash(key) 574 | , filename = self.dbName +'/'+ hash.substr(0, 3) 575 | 576 | this.__busy(key) 577 | 578 | var found = false 579 | , val = null 580 | , buffer = '' 581 | 582 | // this is loosely based on felixge's node-dirty (MIT LICENCED) 583 | 584 | var rs = fs.createReadStream(filename, { 585 | encoding: 'utf8' 586 | , flags: 'r' 587 | }) 588 | 589 | rs.on('error', function(err) { 590 | self.__free(key) 591 | rs.destroy() 592 | if (cb) cb(err) 593 | }) 594 | 595 | rs.on('data', function(chunk) { 596 | buffer += chunk 597 | buffer = buffer.replace(/([^\n]+)\n/g, function(m, rowStr) { 598 | var tabIndex = rowStr.indexOf('\t') 599 | , rowKey = rowStr.substring(0, tabIndex) 600 | 601 | if (rowKey != key) return '' 602 | 603 | var rowJson = rowStr.substring(tabIndex + 1) 604 | , rowVal 605 | 606 | try { 607 | val = JSON.parse(rowJson) 608 | } catch(err) { 609 | val = null 610 | return '' 611 | } 612 | 613 | found = true 614 | 615 | return '' 616 | }) 617 | }) 618 | 619 | rs.on('end', function() { 620 | self.__free(key) 621 | cb(null, val) 622 | }) 623 | } 624 | 625 | Chaos.prototype._watch = function(key, opts, cb) { 626 | if (typeof opts == 'function') cb = opts, opts = {} 627 | 628 | var self = this 629 | , filename = self.dbName +'/'+ this.__hash(key) 630 | 631 | fs.watchFile(filename, opts, function(curr, prev) { 632 | self.get(key, cb) 633 | }) 634 | } 635 | 636 | Chaos.prototype._unwatch = function(key) { 637 | var self = this 638 | , filename = self.dbName +'/'+ this.__hash(key) 639 | 640 | fs.unwatchFile(filename) 641 | } 642 | 643 | Chaos.prototype.mount = function(key) { 644 | var self = this 645 | return { 646 | 'get': function(field, cb) { 647 | self.hget(key, field, cb) 648 | } 649 | , 'set': function(field, val, cb) { 650 | self.hset(key, field, val, cb) 651 | } 652 | , 'all': function(cb) { 653 | self.hgetall(key, cb) 654 | } 655 | , 'remove': function(field, cb) { 656 | self.hdel(key, field, cb) 657 | } 658 | } 659 | } 660 | 661 | ;[ 'get', 'set', 'del' 662 | , 'getset', 'getdel', 'getorsetget' 663 | , 'incr', 'decr' 664 | , 'hset', 'hget', 'hdel', 'hrand', 'hgetall', 'hkeys', 'hvals' 665 | , 'jset', 'jget', 666 | , 'watch', 'unwatch' 667 | ].forEach(function(command) { 668 | Chaos.prototype[command] = function() { 669 | this.__queue('_' + command, to_array(arguments)) 670 | } 671 | }) 672 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { "name" : "chaos" 2 | , "description" : "chaos is a node.js database" 3 | , "version" : "0.2.0" 4 | , "main" : "chaos.js" 5 | , "homepage" : "http://github.com/stagas/chaos" 6 | , "author" : "George Stagas (http://stagas.com/)" 7 | , "repository" : 8 | { "type" : "git" 9 | , "url" : "http://github.com/stagas/chaos.git" 10 | } 11 | , "bugs" : { "web" : "http://github.com/stagas/chaos/issues" } 12 | , "engines" : { "node" : ">=0.2.4" } 13 | , "licenses" : 14 | [ { "type" : "MIT" 15 | , "url" : "http://github.com/stagas/chaos/raw/master/LICENSE" 16 | } 17 | ] 18 | } -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | // tests for chaos 2 | 3 | var assert = require('assert') 4 | , db = require('./chaos')('databasetest') 5 | 6 | var test = {} 7 | 8 | test.simple = [ 9 | function(next) { 10 | db.set('foo', { bar: 'bar' }, function(err) { 11 | db.get('foo', function(err, val) { 12 | assert.deepEqual({ bar: 'bar' }, val) 13 | next() 14 | }) 15 | }) 16 | } 17 | ] 18 | 19 | test.hkeys = [ 20 | function (next) { 21 | db.del('john', function(err) { 22 | db.hset('john', 'last name', 'doe', function(err) { 23 | db.hget('john', 'last name', function(err, val) { 24 | assert.equal(null, err) 25 | assert.equal('doe', val) 26 | next() 27 | }) 28 | }) 29 | }) 30 | } 31 | , function (next) { 32 | db.del('json', function(err) { 33 | db.hset('json', 'foo-_', { foo: 'bar' }, function(err) { 34 | db.hget('json', 'foo-_', function(err, val) { 35 | assert.equal(null, err) 36 | assert.deepEqual({ foo: 'bar' }, val) 37 | next() 38 | }) 39 | }) 40 | }) 41 | } 42 | , function (next) { 43 | var store = db.mount('storage') 44 | store.set('foo', ['bar'], function(err) { 45 | store.set('bar', 'foo', function(err) { 46 | store.all(function(err, kv) { 47 | assert.deepEqual({ foo: ['bar'], bar: 'foo' }, kv) 48 | next() 49 | }) 50 | }) 51 | }) 52 | } 53 | , function (next) { 54 | var test_many = { 55 | 'john': 'doe' 56 | , 'mary': 'loo' 57 | } 58 | var test_many_keys = [] 59 | for (var k in test_many) { 60 | test_many_keys.push(k) 61 | } 62 | var test_many_vals = [] 63 | for (var k in test_many) { 64 | test_many_vals.push(test_many[k]) 65 | } 66 | 67 | db.del('persons', function(err) { 68 | db.hset('persons', 'john', 'doe', function(err) { 69 | db.hset('persons', 'mary', 'loo', function(err) { 70 | db.hgetall('persons', function(err, data) { 71 | assert.equal(null, err) 72 | assert.deepEqual(data, test_many) 73 | db.hkeys('persons', function(err, data) { 74 | assert.equal(null, err) 75 | assert.deepEqual(data.sort(), test_many_keys.sort()) 76 | db.hvals('persons', function(err, data) { 77 | assert.equal(null, err) 78 | assert.deepEqual(data.sort(), test_many_vals.sort()) 79 | next() 80 | }) 81 | }) 82 | }) 83 | }) 84 | }) 85 | }) 86 | } 87 | , function (next) { 88 | db.del('will', function(err) { 89 | db.hset('will', 'delete', 'now', function(err) { 90 | db.hget('will', 'delete', function(err, data) { 91 | assert.equal(null, err) 92 | db.hdel('will', 'delete', function(err) { 93 | db.hget('will', 'delete', function(err, data) { 94 | assert.notEqual(null, err) 95 | db.del('will', function(err) { 96 | assert.equal(null, err) 97 | db.hgetall('will', function(err, data) { 98 | assert.notEqual(null, err) 99 | }) 100 | }) 101 | }) 102 | }) 103 | }) 104 | }) 105 | }) 106 | } 107 | ] 108 | 109 | test.watch = [ 110 | function (next) { 111 | db.watch('foo', function(err, data) { 112 | assert.equal('bar', data) 113 | db.unwatch('foo') 114 | db.set('foo', 'hey', function() { 115 | next() 116 | }) 117 | }) 118 | db.set('foo', 'bar') 119 | } 120 | , function (next) { 121 | db.del('foobar', function(err) { 122 | db.watch('foobar', function(err, data) { 123 | //assert.equal(null, err) 124 | //assert.equal('', data) 125 | db.unwatch('foobar') 126 | db.hgetall('foobar', function(err, data) { 127 | assert.equal(null, err) 128 | assert.deepEqual({bar: 'bar'}, data) 129 | next() 130 | }) 131 | }) 132 | db.hset('foobar', 'bar', 'bar', function(err) { 133 | assert.equal(null, err) 134 | }) 135 | }) 136 | } 137 | ] 138 | 139 | test.jkeys = [ 140 | function(next) { 141 | db.jset('hello', { foo: 'bar' }, function(err) { 142 | db.jget('hello', function(err, data) { 143 | assert.deepEqual(data, { foo: 'bar' }) 144 | next() 145 | }) 146 | }) 147 | } 148 | ] 149 | 150 | var tests = [] 151 | for (var k in test) { 152 | test[k].forEach(function (t) { 153 | tests.push(t) 154 | }) 155 | } 156 | 157 | var handle = function () {} 158 | 159 | tests.forEach(function (layer) { 160 | var child = handle 161 | handle = function () { 162 | layer(function() { 163 | child() 164 | }) 165 | } 166 | }) 167 | 168 | handle() 169 | --------------------------------------------------------------------------------