├── .gitignore ├── .jshintrc ├── .travis.yml ├── README.md ├── encoding.js ├── index.js ├── iter-stream.js ├── iterator.js ├── license.md ├── package.json └── test ├── index.html ├── test.js ├── testBrowser.js └── testCommon.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | *~ 4 | coverage 5 | Thumbs.db 6 | .bak 7 | .tmp 8 | 9 | lib-cov 10 | *.seed 11 | *.log 12 | *.dat 13 | *.out 14 | *.pid 15 | *.gz 16 | 17 | pids 18 | logs 19 | results 20 | 21 | npm-debug.log 22 | temp.js 23 | test/test-bundle.js 24 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "node" : true, 3 | "unused": true, 4 | "curly": true, 5 | "eqeqeq": true, 6 | "immed": true, 7 | "newcap": true, 8 | "noarg": true, 9 | "sub": true, 10 | "undef": "nofunc", 11 | "strict": true, 12 | "white": true, 13 | "indent": 2, 14 | "trailing": true, 15 | "quotmark": "single" 16 | } -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: false 3 | node_js: 4 | - "0.10" 5 | - "0.12" 6 | 7 | before_script: 8 | - sh -c "if [ '$DB' = 'postgres' ]; then psql -c 'DROP DATABASE IF EXISTS sqldown;' -U postgres; fi" 9 | - sh -c "if [ '$DB' = 'mysql' ]; then mysql -e 'DROP DATABASE IF EXISTS sqldown;'; fi" 10 | - sh -c "if [ '$DB' = 'postgres' ]; then psql -c 'create database sqldown;' -U postgres; fi" 11 | - sh -c "if [ '$DB' = 'mysql' ]; then mysql -e 'create database sqldown;'; fi" 12 | 13 | env: 14 | - COMMAND=test 15 | - DB=mysql COMMAND=test 16 | - DB=postgres COMMAND=test 17 | 18 | script: npm run $COMMAND 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | SQLdown [![Build Status](https://travis-ci.org/calvinmetcalf/SQLdown.svg)](https://travis-ci.org/calvinmetcalf/SQLdown) 2 | ==== 3 | 4 | A levelup backend with knex (sqlite3, postgres, and mysql tested and websql possible). 5 | 6 | ```bash 7 | npm install --save sqldown 8 | ``` 9 | 10 | Also it doesn't come with any of the database backends so you need to install those yourself, one of 11 | 12 | ```bash 13 | npm install --save sqlite3 14 | npm install --save pg pg-query-stream 15 | npm install --save mysql 16 | ``` 17 | 18 | In node locations should be connection strings (e.g. `postgres://username:password@localhost/database`), if it doesn't start with a 'dbType://' it is assumed to be the path for a local sqlite3 database. Table defaults to sqldown but can be overridden either by passing an `table` option or setting a query param (`pg://localhost/database?table=tablename`). 19 | 20 | In the browser location will always be the table name. 21 | 22 | Test setup and much else taken from [level-js](https://github.com/maxogden/level.js). 23 | 24 | ## Setup 25 | 26 | To get around the fact that postgres does not feature upserts instead of a simple table with 2 columns, `key` and `value` with `key` being the primary and unique key, instead we have a more complex setup with 3 columns `id`, `key` and `value` with `id` being an auto-incremented integer. When we `get` we query for the value with the given key which has the highest id. 27 | 28 | This could lead to much excess data if you were to update the same key a bunch so it's set to periodically (by default every 25 puts) clean up any entries that aren't the max id for a given key. 29 | 30 | Databases that support indexes on arbitrarily long fields have the `key` field index. If you know your keys or values are going to be shorter then a given length you may specify `keyLength` or `valueLength` option to limit it to that length, this is a prerequisite for mysql indexes. 31 | -------------------------------------------------------------------------------- /encoding.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | 4 | function encode(value, isValue) { 5 | if (isValue && !value) { 6 | value = new Buffer(''); 7 | } 8 | if (!Buffer.isBuffer(value) && typeof value !== 'string') { 9 | value = String(value); 10 | } 11 | if (!Buffer.isBuffer(value)) { 12 | value = new Buffer(value); 13 | } 14 | return value; 15 | } 16 | 17 | function decode(value, asBuffer) { 18 | if (asBuffer) { 19 | return value; 20 | } else { 21 | return value.toString(); 22 | } 23 | } 24 | if (process.browser) { 25 | exports.encode = function (value, isValue) { 26 | var out = encode(value, isValue); 27 | return out.toString(isValue ? 'base64' : 'hex'); 28 | }; 29 | exports.decode = function (value, asBuffer, isValue) { 30 | return decode(new Buffer(value, isValue ? 'base64' : 'hex'), asBuffer); 31 | }; 32 | } else { 33 | exports.encode = encode; 34 | exports.decode = decode; 35 | } 36 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var inherits = require('inherits'); 3 | var knex = require('knex'); 4 | var AbstractLevelDOWN = require('abstract-leveldown').AbstractLevelDOWN; 5 | var Iter = require('./iterator'); 6 | var fs = require('fs'); 7 | var Promise = require('bluebird'); 8 | var url = require('url'); 9 | var TABLENAME = 'sqldown'; 10 | var util = require('./encoding'); 11 | var debug = require('debug')('sqldown:main'); 12 | module.exports = SQLdown; 13 | function parseConnectionString(string) { 14 | if (process.browser) { 15 | return { 16 | client: 'websql' 17 | }; 18 | } 19 | var parsed = url.parse(string); 20 | var protocol = parsed.protocol; 21 | if(protocol === null) { 22 | return { 23 | client: 'sqlite3', 24 | connection: { 25 | filename: string 26 | } 27 | }; 28 | } 29 | if (protocol.slice(-1) === ':') { 30 | protocol = protocol.slice(0, -1); 31 | } 32 | return { 33 | client: protocol, 34 | connection: fixDB(parsed) 35 | }; 36 | } 37 | function fixDB(parsed) { 38 | var out = {}; 39 | var db = parsed.pathname; 40 | if (db[0] === '/') { 41 | db = db.slice(1); 42 | } 43 | out.database = db; 44 | if (parsed.hostname) { 45 | out.host = parsed.hostname; 46 | } 47 | if (parsed.port) { 48 | out.port = parsed.port; 49 | } 50 | if (parsed.auth) { 51 | var idx = parsed.auth.indexOf(':'); 52 | if (~idx) { 53 | out.user = parsed.auth.slice(0, idx); 54 | if (idx < parsed.auth.length - 1) { 55 | out.password = parsed.auth.slice(idx + 1); 56 | } 57 | } 58 | } 59 | return out; 60 | } 61 | function getTableName (location, options) { 62 | if (process.browser) { 63 | return location; 64 | } 65 | var parsed = url.parse(location, true).query; 66 | return parsed.table || options.table || TABLENAME; 67 | } 68 | // constructor, passes through the 'location' argument to the AbstractLevelDOWN constructor 69 | // our new prototype inherits from AbstractLevelDOWN 70 | inherits(SQLdown, AbstractLevelDOWN); 71 | 72 | function SQLdown(location) { 73 | if (!(this instanceof SQLdown)) { 74 | return new SQLdown(location); 75 | } 76 | AbstractLevelDOWN.call(this, location); 77 | this.knexDb = this.counter = this.dbType = this.compactFreq = this.tablename = void 0; 78 | this._paused = 0; 79 | this._compactable = true; 80 | } 81 | SQLdown.destroy = function (location, options, callback) { 82 | if (typeof options === 'function') { 83 | callback = options; 84 | options = {}; 85 | } 86 | var conn = parseConnectionString(location); 87 | if (conn.client === 'sqlite3') { 88 | fs.unlink(location, callback); 89 | return; 90 | } 91 | var db = knex(conn); 92 | db.schema.dropTableIfExists(getTableName(location, options)).then(function () { 93 | return db.destroy(); 94 | }).asCallback(callback); 95 | }; 96 | 97 | 98 | SQLdown.prototype._open = function (options, callback) { 99 | var self = this; 100 | var conn = parseConnectionString(this.location); 101 | this.dbType = conn.client; 102 | this.knexDb = knex(conn); 103 | this.tablename = getTableName(this.location, options); 104 | this.compactFreq = options.compactFrequency || 25; 105 | this.counter = 0; 106 | function createTable() { 107 | return self.knexDb.schema.createTable(self.tablename, function (table) { 108 | table.increments('id').primary(); 109 | if (process.browser) { 110 | if (typeof options.keySize === 'number') { 111 | table.string('key', options.keySize).index(); 112 | } else { 113 | table.text('key').index(); 114 | } 115 | } else if(self.dbType === 'mysql') { 116 | if (typeof options.keySize === 'number') { 117 | table.binary('key', options.keySize).index(); 118 | } else { 119 | table.binary('key'); 120 | } 121 | } else { 122 | table.binary('key').index(); 123 | } 124 | if (process.browser) { 125 | if (typeof options.valueSize === 'number') { 126 | table.string('value', options.valueSize); 127 | } else { 128 | table.text('value'); 129 | } 130 | } else if(self.dbType === 'mysql' && typeof options.valueSize === 'number') { 131 | table.binary('value', options.valueSize); 132 | } else { 133 | table.binary('value'); 134 | } 135 | 136 | }); 137 | } 138 | if (process.browser){ 139 | this.knexDb.select('id').from(this.tablename).limit(1).catch(function (){ 140 | return createTable(); 141 | }).nodeify(callback); 142 | } else { 143 | self.knexDb.schema.hasTable(self.tablename).then(function (has) { 144 | if (!has) { 145 | return createTable(); 146 | } 147 | }).nodeify(callback); 148 | } 149 | }; 150 | 151 | SQLdown.prototype._get = function (key, options, cb) { 152 | var self = this; 153 | var asBuffer = true; 154 | if (options.asBuffer === false) { 155 | asBuffer = false; 156 | } 157 | if (options.raw) { 158 | asBuffer = false; 159 | } 160 | key = util.encode(key); 161 | this.knexDb.select('value').from(this.tablename).whereIn('id', function (){ 162 | this.max('id').from(self.tablename).where({ key: key}); 163 | }).asCallback(function (err, res) { 164 | if (err) { 165 | return cb(err.stack); 166 | } 167 | if (!res.length) { 168 | return cb(new Error('NotFound')); 169 | } 170 | try { 171 | var value = res[0].value; 172 | if (value === undefined || value === null) { 173 | return cb(new Error('NotFound')); 174 | } 175 | cb(null, util.decode(value, asBuffer, true)); 176 | } catch (e) { 177 | cb(new Error('NotFound')); 178 | } 179 | }); 180 | }; 181 | SQLdown.prototype._put = function (key, value, opt, cb) { 182 | var self = this; 183 | value = util.encode(value, true); 184 | key = util.encode(key); 185 | 186 | self.pause(function () { 187 | self.knexDb(self.tablename).insert({ 188 | key: key, 189 | value: value 190 | }).then(function () { 191 | return self.maybeCompact(); 192 | }).nodeify(cb); 193 | }); 194 | }; 195 | SQLdown.prototype._del = function (key, opt, cb) { 196 | var self = this; 197 | key = util.encode(key); 198 | debug('before del pause'); 199 | this.pause(function () { 200 | debug('after del pause'); 201 | self.knexDb(self.tablename).insert({key: key}).then(function () { 202 | return self.maybeCompact(); 203 | }).nodeify(cb); 204 | }); 205 | }; 206 | function unique(array) { 207 | var things = {}; 208 | array.forEach(function (item) { 209 | things[item.key] = item; 210 | }); 211 | return Object.keys(things).map(function (key) { 212 | return things[key]; 213 | }); 214 | } 215 | SQLdown.prototype._batch = function (array, options, callback) { 216 | var self = this; 217 | var inserts = 0; 218 | this.pause(function () { 219 | self.knexDb.transaction(function (trx) { 220 | return Promise.all(unique(array).map(function (item) { 221 | var key = util.encode(item.key); 222 | 223 | if (item.type === 'del') { 224 | return trx.insert({ 225 | key: key 226 | }).into(self.tablename); 227 | } else { 228 | var value = util.encode(item.value, true); 229 | inserts++; 230 | return trx.insert({ 231 | key: key, 232 | value: value 233 | }).into(self.tablename); 234 | } 235 | })); 236 | }).then(function () { 237 | return self.maybeCompact(inserts); 238 | }).asCallback(callback); 239 | }); 240 | }; 241 | SQLdown.prototype.compact = function () { 242 | var self = this; 243 | return this.knexDb(this.tablename).select('key', 'value').not.whereIn('id', function () { 244 | this.select('id').from(function () { 245 | this.select(self.knexDb.raw('max(id) as id')).from(self.tablename).groupBy('key').as('__tmp__table'); 246 | }); 247 | }).delete(); 248 | }; 249 | 250 | SQLdown.prototype.maybeCompact = function (inserts) { 251 | if (!this._compactable) { 252 | return Promise.resolve(); 253 | } 254 | if (inserts + this.counter > this.compactFreq) { 255 | this.counter += inserts; 256 | this.counter %= this.compactFreq; 257 | return this.compact(); 258 | } 259 | this.counter++; 260 | this.counter %= this.compactFreq; 261 | if (this.counter) { 262 | return Promise.resolve(); 263 | } else { 264 | return this.compact(); 265 | } 266 | }; 267 | 268 | SQLdown.prototype._close = function (callback) { 269 | var self = this; 270 | process.nextTick(function () { 271 | self.knexDb.destroy().asCallback(callback); 272 | }); 273 | }; 274 | SQLdown.prototype.pause = function (cb) { 275 | if (this.dbType !== 'mysql' || !this._paused) { 276 | cb(); 277 | } else { 278 | this.knexDb.once('unpaused', cb); 279 | } 280 | }; 281 | SQLdown.prototype.iterator = function (options) { 282 | var self = this; 283 | if (this.dbType === 'mysql') { 284 | debug('pausing iterator'); 285 | this._paused++; 286 | } 287 | this._compactable = false; 288 | return new Iter(this, options, function () { 289 | self._compactable = true; 290 | self.maybeCompact(); 291 | }); 292 | }; 293 | -------------------------------------------------------------------------------- /iter-stream.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var Queue = require('double-ended-queue'); 3 | var through = require('through2').obj; 4 | var EE = require('events').EventEmitter; 5 | var inherits = require('inherits'); 6 | var debug = require('debug')('sqldown:iter-stream'); 7 | 8 | module.exports = IterStream; 9 | inherits(IterStream, EE); 10 | function IterStream(_stream, db) { 11 | if (!(this instanceof IterStream)) { 12 | return new IterStream(_stream, db); 13 | } 14 | var self = this; 15 | EE.call(self); 16 | this.stream = null; 17 | this.queue = new Queue(); 18 | var outStream = through(function (chunk, _, next) { 19 | debug('transform'); 20 | if (self.queue.isEmpty()) { 21 | self.once('callback', function () { 22 | self.queue.shift()(null, chunk); 23 | next(); 24 | }); 25 | } else { 26 | self.queue.shift()(null, chunk); 27 | next(); 28 | } 29 | }, function (next) { 30 | debug('flush'); 31 | while(!self.queue.isEmpty()) { 32 | self.queue.shift()(new Error('ended')); 33 | } 34 | self.on('callback', function () { 35 | while(!self.queue.isEmpty()) { 36 | self.queue.shift()(new Error('ended')); 37 | } 38 | }); 39 | next(); 40 | }); 41 | if (db) { 42 | _stream.then(function (query) { 43 | var stream = query[0].stream(); 44 | self.stream = stream; 45 | db._paused--; 46 | debug(db._paused); 47 | if (!db._paused) { 48 | debug('unpause'); 49 | db.knexDb.emit('unpaused'); 50 | } 51 | stream.pipe(outStream); 52 | }).catch(function (e) { 53 | db._paused--; 54 | debug(db._paused); 55 | if (!db._paused) { 56 | debug('unpause after error'); 57 | db.knexDb.emit('unpaused'); 58 | } 59 | if (self.queue.isEmpty()) { 60 | self.once('callback', function () { 61 | self.queue.shift()(e); 62 | }); 63 | } else { 64 | self.queue.shift()(e); 65 | } 66 | }); 67 | } else { 68 | this.stream = _stream.stream(); 69 | this.stream.pipe(outStream); 70 | } 71 | } 72 | IterStream.prototype.next = function (callback) { 73 | this.queue.push(callback); 74 | this.emit('callback'); 75 | }; 76 | -------------------------------------------------------------------------------- /iterator.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var inherits = require('inherits'); 3 | var AbstractIterator = require('abstract-leveldown/abstract-iterator'); 4 | var IterStream = require('./iter-stream'); 5 | var util = require('./encoding'); 6 | var debug = require('debug')('sqldown:iterator'); 7 | 8 | function goodOptions(opts, name) { 9 | if (!(name in opts)) { 10 | return; 11 | } 12 | var thing = opts[name]; 13 | if (thing === null) { 14 | delete opts[name]; 15 | return; 16 | } 17 | if (Buffer.isBuffer(thing) || typeof thing === 'string') { 18 | if (!thing.length) { 19 | delete opts[name]; 20 | return; 21 | } 22 | 23 | opts[name] = util.encode(thing); 24 | } 25 | 26 | } 27 | inherits(Iterator, AbstractIterator); 28 | module.exports = Iterator; 29 | var names = [ 30 | 'start', 31 | 'end', 32 | 'gt', 33 | 'gte', 34 | 'lt', 35 | 'lte' 36 | ]; 37 | function Iterator(db, options, cb) { 38 | AbstractIterator.call(this, db); 39 | this._db = db.knexDb; 40 | options = options || {}; 41 | this._order = !options.reverse; 42 | this._options = options; 43 | names.forEach(function (i) { 44 | goodOptions(options, i); 45 | }); 46 | this._count = 0; 47 | var self = this; 48 | if ('limit' in options) { 49 | this._limit = options.limit; 50 | } else { 51 | this._limit = -1; 52 | } 53 | 54 | if ('keyAsBuffer' in options) { 55 | this._keyAsBuffer = options.keyAsBuffer; 56 | } else { 57 | this._keyAsBuffer = true; 58 | } 59 | if ('valueAsBuffer' in options) { 60 | this._valueAsBuffer = options.valueAsBuffer; 61 | } else { 62 | this._valueAsBuffer = true; 63 | } 64 | 65 | var makeSql; 66 | if (db.dbType === 'mysql') { 67 | makeSql = this.getCurrentId().then(function (key) { 68 | return [self.buildSQL(key)]; 69 | }); 70 | } else { 71 | makeSql = this.buildSQL(); 72 | } 73 | if (this._limit === 0) { 74 | this._next = function (cb) { 75 | process.nextTick(cb); 76 | }; 77 | } else { 78 | if (db.dbType === 'mysql') { 79 | this._sql = new IterStream(makeSql, this.db); 80 | } else { 81 | this._sql = new IterStream(makeSql); 82 | } 83 | this.__value = null; 84 | this.__cb = null; 85 | this.___cb = cb; 86 | this._next(function (err, key, value) { 87 | if (typeof self.__cb === 'function') { 88 | self.__value = null; 89 | if (self._ended || (err === void 0 && key === void 0 && value === void 0)) { 90 | return self.__cb(); 91 | } 92 | self.__cb(err, key, value); 93 | self.__cb = null; 94 | } else { 95 | self.__value = [err, key, value]; 96 | } 97 | if (typeof self.___cb === 'function') { 98 | self.___cb(); 99 | self.___cb = null; 100 | } 101 | }); 102 | this.__value = 'in progress'; 103 | } 104 | } 105 | 106 | Iterator.prototype._next = function (callback) { 107 | debug('_nexting'); 108 | var self = this; 109 | if (self._ended) { 110 | if (typeof this.___cb === 'function') { 111 | this.___cb(); 112 | this.___cb = null; 113 | } 114 | return callback(); 115 | } 116 | debug(this.__value); 117 | if (this.__value !== null) { 118 | if (this.__value === 'in progress') { 119 | this.__cb = callback; 120 | return; 121 | } else { 122 | return process.nextTick(function () { 123 | var value = self.__value; 124 | self.__value = null; 125 | if (value.every(function (val) { 126 | return val === void 0; 127 | })) { 128 | return callback(); 129 | } 130 | debug(value); 131 | callback(value[0], value[1], value[2]); 132 | }); 133 | } 134 | } 135 | this._sql.next(function (err, resp) { 136 | debug(err, resp); 137 | if (err || !resp || !resp.value) { 138 | return callback(); 139 | } 140 | var key = util.decode(resp.key, self._keyAsBuffer); 141 | var value = util.decode(resp.value, self._valueAsBuffer, true); 142 | 143 | if (!self._keyAsBuffer) { 144 | key = key.toString(); 145 | } 146 | if (!self._valueAsBuffer) { 147 | value = value.toString(); 148 | } 149 | callback(null, key, value); 150 | }); 151 | }; 152 | 153 | Iterator.prototype.buildSQL = function (maxKey) { 154 | debug(maxKey); 155 | var self = this; 156 | var outersql = this._db.select('key', 'value').from(this.db.tablename).whereNotNull('value'); 157 | var innerSQL = this._db.max('id').from(self.db.tablename).groupBy('key'); 158 | if (typeof maxKey !== 'undefined') { 159 | innerSQL.where('id', '<=', maxKey); 160 | } 161 | if (this._order) { 162 | outersql.orderBy('key'); 163 | if ('start' in this._options) { 164 | if (this._options.exclusiveStart) { 165 | if ('start' in this._options) { 166 | this._options.gt = this._options.start; 167 | } 168 | } else { 169 | if ('start' in this._options) { 170 | this._options.gte = this._options.start; 171 | } 172 | } 173 | } 174 | if ('end' in this._options) { 175 | this._options.lte = this._options.end; 176 | } 177 | } else { 178 | outersql.orderBy('key', 'DESC'); 179 | if ('start' in this._options) { 180 | if (this._options.exclusiveStart) { 181 | if ('start' in this._options) { 182 | this._options.lt = this._options.start; 183 | } 184 | } else { 185 | if ('start' in this._options) { 186 | this._options.lte = this._options.start; 187 | } 188 | } 189 | } 190 | if ('end' in this._options) { 191 | this._options.gte = this._options.end; 192 | } 193 | } 194 | 195 | if ('lt' in this._options) { 196 | innerSQL.where('key', '<', this._options.lt); 197 | } 198 | if ('lte' in this._options) { 199 | innerSQL.where('key', '<=', this._options.lte); 200 | } 201 | if ('gt' in this._options) { 202 | innerSQL.where('key', '>', this._options.gt); 203 | } 204 | if ('gte' in this._options) { 205 | innerSQL.where('key', '>=', this._options.gte); 206 | } 207 | outersql.whereIn('id', innerSQL); 208 | if (this._limit > 0) { 209 | outersql.limit(this._limit); 210 | } 211 | return outersql; 212 | }; 213 | Iterator.prototype.getCurrentId = function () { 214 | return this._db.select(this._db.raw('max(id) as id')).from(this.db.tablename).then(function (resp) { 215 | debug('get id'); 216 | debug(resp); 217 | return resp[0].id; 218 | }); 219 | }; 220 | -------------------------------------------------------------------------------- /license.md: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2014 Applied Geophraphics, Inc. & Calvin Metcalf 2 | Includes code from https://github.com/maxogden/level.js by Max Ogden 3 | 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | **THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.** -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqldown", 3 | "description": "A sqlite3 implementation of the LevelDOWN API", 4 | "version": "2.1.0", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/calvinmetcalf/sqldown.git" 8 | }, 9 | "homepage": "https://github.com/calvinmetcalf/sqldown", 10 | "keywords": [ 11 | "leveldb", 12 | "leveldown", 13 | "levelup", 14 | "sqlite" 15 | ], 16 | "main": "./index.js", 17 | "scripts": { 18 | "test": "node ./test/test.js -p | tspec", 19 | "browser": "browserify ./test/testBrowser.js > test/test-bundle.js" 20 | }, 21 | "license": "MIT", 22 | "browser": { 23 | "fs": false 24 | }, 25 | "browserify": { 26 | "transform": [ 27 | "es3ify" 28 | ] 29 | }, 30 | "dependencies": { 31 | "abstract-leveldown": "^2.1.0", 32 | "bluebird": "^2.3.11", 33 | "debug": "^2.2.0", 34 | "double-ended-queue": "^2.0.0-0", 35 | "es3ify": "^0.1.3", 36 | "inherits": "^2.0.1", 37 | "knex": "^0.8.3", 38 | "through2": "^0.6.3" 39 | }, 40 | "devDependencies": { 41 | "browserify": "^11.0.0", 42 | "mysql": "^2.5.3", 43 | "pg": "^4.1.1", 44 | "pg-query-stream": "^0.7.0", 45 | "sqlite3": "^3.0.4", 46 | "tap-browser-el": "^2.0.0", 47 | "tap-spec": "^2.1.1", 48 | "tape": "^3.0.3" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /test/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var tape = require('tape') 3 | var leveljs = require('../') 4 | var testCommon = require('./testCommon') 5 | 6 | var testBuffer = new Buffer('foo') 7 | 8 | /*** compatibility with basic LevelDOWN API ***/ 9 | function test(testCommon) { 10 | require('abstract-leveldown/abstract/leveldown-test').args(leveljs, tape, testCommon); 11 | require('abstract-leveldown/abstract/open-test').open(leveljs, tape, testCommon); 12 | require('abstract-leveldown/abstract/put-test').all(leveljs, tape, testCommon); 13 | require('abstract-leveldown/abstract/del-test').all(leveljs, tape, testCommon); 14 | require('abstract-leveldown/abstract/get-test').all(leveljs, tape, testCommon); 15 | require('abstract-leveldown/abstract/put-get-del-test').all(leveljs, tape, testCommon, testBuffer); 16 | require('abstract-leveldown/abstract/batch-test').all(leveljs, tape, testCommon) 17 | require('abstract-leveldown/abstract/chained-batch-test').all(leveljs, tape, testCommon) 18 | require('abstract-leveldown/abstract/close-test').close(leveljs, tape, testCommon); 19 | require('abstract-leveldown/abstract/iterator-test').all(leveljs, tape, testCommon) 20 | require('abstract-leveldown/abstract/ranges-test').all(leveljs, tape, testCommon) 21 | 22 | function custom(leveldown, test, testCommon) { 23 | 24 | var db; 25 | test('setUp common', testCommon.setUp) 26 | test('open close open', function (t) { 27 | db = leveldown(testCommon.location()) 28 | 29 | // default createIfMissing=true, errorIfExists=false 30 | db.open(function (err) { 31 | t.notOk(err, 'no error') 32 | db.close(function (err) { 33 | t.notOk(err, 'no error') 34 | db.open(function (err) { 35 | t.notOk(err, 'no error') 36 | t.end(); 37 | }) 38 | }) 39 | }) 40 | }); 41 | test('close up', function (t) { 42 | db.close(function (err) { 43 | if (err) { 44 | process.exit(1); 45 | } 46 | testCommon.cleanup(function (){ 47 | t.end(); 48 | }); 49 | }); 50 | }); 51 | 52 | test('setUp common', testCommon.setUp) 53 | test('test keySize', function (t) { 54 | db = leveldown(testCommon.location()) 55 | 56 | // default createIfMissing=true, errorIfExists=false 57 | db.open({ 58 | keySize: 150, 59 | valueSize: 150 60 | }, function (err) { 61 | t.notOk(err, 'no error in open') 62 | db.put('foo', 'bar', function (err) { 63 | t.notOk(err, 'no error in put') 64 | db.get('foo', { 65 | asBuffer: false 66 | }, function (err, value) { 67 | t.notOk(err, 'no error in get') 68 | t.equals('bar', value); 69 | t.end(); 70 | }); 71 | }); 72 | }) 73 | }); 74 | test('close up', function (t) { 75 | db.close(function (err) { 76 | if (err) { 77 | process.exit(1); 78 | } 79 | testCommon.cleanup(function (){ 80 | t.end(); 81 | }); 82 | }); 83 | }); 84 | if (process.env.DB === 'mysql') { 85 | test('setUp common', testCommon.setUp) 86 | test('test keySize2', function (t) { 87 | db = leveldown(testCommon.location()) 88 | 89 | // default createIfMissing=true, errorIfExists=false 90 | db.open({keySize: 3}, function (err) { 91 | t.notOk(err, 'no error in open') 92 | db.put('foobar', 'bar', function (err) { 93 | t.not(err, 'error in put') 94 | db.get('foo', { 95 | asBuffer: false 96 | }, function (err) { 97 | t.not(err, 'error in get') 98 | t.end(); 99 | }); 100 | }); 101 | }) 102 | }); 103 | test('close up', function (t) { 104 | db.close(function (err) { 105 | if (err) { 106 | process.exit(1); 107 | } 108 | testCommon.cleanup(function (){ 109 | t.end(); 110 | }); 111 | }); 112 | }); 113 | test('setUp common', testCommon.setUp) 114 | test('test valuesize', function (t) { 115 | db = leveldown(testCommon.location()) 116 | 117 | // default createIfMissing=true, errorIfExists=false 118 | db.open({valueSize: 3}, function (err) { 119 | t.notOk(err, 'no error in open') 120 | db.put('foo', 'barvar', function (err) { 121 | t.not(err, 'error in put') 122 | db.get('foo', { 123 | asBuffer: false 124 | }, function (err) { 125 | t.not(err, 'error in get') 126 | t.end(); 127 | }); 128 | }); 129 | }) 130 | }); 131 | test('close up', function (t) { 132 | db.close(function (err) { 133 | if (err) { 134 | process.exit(1); 135 | } 136 | testCommon.cleanup(function (){ 137 | t.end(); 138 | }); 139 | }); 140 | }); 141 | } 142 | } 143 | custom(leveljs, tape, testCommon) 144 | } 145 | if (process.env.DB === 'postgres') { 146 | test(testCommon('postgres://localhost/sqldown?table=_leveldown_test_db_')); 147 | } else if (process.env.DB === 'mysql') { 148 | test(testCommon('mysql://travis:@localhost/sqldown?table=_leveldown_test_db_')); 149 | } else { 150 | test(testCommon('_leveldown_test_db_')); 151 | } 152 | -------------------------------------------------------------------------------- /test/testBrowser.js: -------------------------------------------------------------------------------- 1 | require('tap-browser-el')(); 2 | require('./test'); 3 | -------------------------------------------------------------------------------- /test/testCommon.js: -------------------------------------------------------------------------------- 1 | module.exports = function (root) { 2 | var dbidx = 0 3 | , leveljs = require('../') 4 | , location = function () { 5 | return root + dbidx++ 6 | } 7 | 8 | , lastLocation = function () { 9 | return root + dbidx 10 | } 11 | 12 | , cleanup = function (callback) { 13 | var list = [] 14 | if (dbidx === 0) return callback() 15 | for (var i = 0; i < dbidx; i++) { 16 | list.push(root + i) 17 | } 18 | 19 | function destroy() { 20 | if (list.length === 0) return callback() 21 | var f = list.pop() 22 | leveljs.destroy(f, destroy) 23 | } 24 | 25 | destroy() 26 | } 27 | 28 | , setUp = function (t) { 29 | cleanup(function (err) { 30 | t.notOk(err, 'cleanup returned an error') 31 | t.end() 32 | }) 33 | } 34 | 35 | , tearDown = function (t) { 36 | setUp(t) // same cleanup! 37 | } 38 | 39 | , collectEntries = function (iterator, callback) { 40 | var data = [] 41 | , next = function () { 42 | iterator.next(function (err, key, value) { 43 | if (err) return callback(err) 44 | if (!arguments.length) { 45 | return iterator.end(function (err) { 46 | callback(err, data) 47 | }) 48 | } 49 | data.push({ key: key, value: value }) 50 | process.nextTick(next) 51 | }) 52 | } 53 | next() 54 | } 55 | 56 | , makeExistingDbTest = function (name, test, leveldown, testFn) { 57 | test(name, function (t) { 58 | cleanup(function () { 59 | var loc = location() 60 | , db = leveldown(loc) 61 | , done = function (close) { 62 | if (close === false) 63 | return cleanup(t.end.bind(t)) 64 | db.close(function (err) { 65 | t.notOk(err, 'no error from close()') 66 | cleanup(t.end.bind(t)) 67 | }) 68 | } 69 | db.open(function (err) { 70 | t.notOk(err, 'no error from open()') 71 | db.batch( 72 | [ 73 | { type: 'put', key: 'one', value: '1' } 74 | , { type: 'put', key: 'two', value: '2' } 75 | , { type: 'put', key: 'three', value: '3' } 76 | ] 77 | , function (err) { 78 | t.notOk(err, 'no error from batch()') 79 | testFn(db, t, done, loc) 80 | } 81 | ) 82 | }) 83 | }) 84 | }) 85 | } 86 | 87 | return { 88 | location : location 89 | , cleanup : cleanup 90 | , lastLocation : lastLocation 91 | , setUp : setUp 92 | , tearDown : tearDown 93 | , collectEntries : collectEntries 94 | , makeExistingDbTest : makeExistingDbTest 95 | } 96 | }; --------------------------------------------------------------------------------