├── .gitignore ├── .npmignore ├── .travis.yml ├── LICENSE ├── README.md ├── dynamo-down.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | index.js 3 | *.log 4 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | dynamo-down.js 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - iojs 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Jed Schmidt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | dynamo-down 2 | =========== 3 | 4 | [![Build Status](https://travis-ci.org/jed/dynamo-down.svg?branch=master)](https://travis-ci.org/jed/dynamo-down) 5 | 6 | A [DynamoDB][] implementation of [leveldown][] for [io.js][]. 7 | 8 | This library uses [abstract-leveldown][] to turn a subsection of a DynamoDB table into a leveldown-compatible store for use with [levelup][]. 9 | 10 | Because the architecture of DynamoDB does not allow for sorted table scans, dynamo-down is implemented using table queries on a given hash key. This means that one DynamoDB table can host many levelup stores, but cannot iterate across them. 11 | 12 | Keep in mind that there are some differences between LevelDB and DynamoDB. For example, unlike LevelDB, DynamoDB does not guarantee batch write atomicity, and does not snapshot reads. 13 | 14 | Installation 15 | ------------ 16 | 17 | npm install dynamo-down 18 | 19 | Example 20 | ------- 21 | 22 | ```javascript 23 | var aws = require("aws-sdk") 24 | var DynamoDOWN = require("dynamo-down") 25 | var levelup = require("levelup") 26 | 27 | var dynamo = new aws.DynamoDB({region: "us-east-1"}) 28 | var dynamoDown = DynamoDOWN(dynamo) 29 | var options = {db: dynamoDown, valueEncoding: "json"} 30 | var db = levelup("my-table/nyc-js-meetups", options) 31 | 32 | db.put("queens_js" , {name: "QueensJS" }) 33 | db.put("jerseyscriptusa" , {name: "JerseyScript"}) 34 | db.put("manhattan_js" , {name: "ManhattanJS" }) 35 | db.put("brooklyn_js" , {name: "BrooklynJS" }) 36 | 37 | db.createReadStream().on("data", console.log) 38 | 39 | // { key: 'brooklyn_js', value: { name: 'BrooklynJS' } } 40 | // { key: 'jerseyscriptusa', value: { name: 'JerseyScript' } } 41 | // { key: 'manhattan_js', value: { name: 'ManhattanJS' } } 42 | // { key: 'queens_js', value: { name: 'QueensJS' } } 43 | ``` 44 | 45 | API 46 | --- 47 | 48 | ### dynamoDown = DynamoDOWN(new aws.DynamoDB) 49 | 50 | `DynamoDOWN` takes a DynamoDB instance created using the [aws-sdk][] library, and returns a leveldown-compatible constructor. 51 | 52 | ### db = levelup("table-name/hash-name", {db: dynamoDown}) 53 | 54 | When instantiating a levelup store, the location passed as the first argument represents the name of the DynamoDB table and the hash key within the table, separated by a `/`. The table must already exist, and have a schema with both hash and range keys. 55 | 56 | ### dynamoDown.destroy("table-name/hash-name", cb) 57 | 58 | This function leaves the backing DynamoDB table in place, but deletes all items with the specified hash name. 59 | 60 | [aws-sdk]: http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/ 61 | [abstract-leveldown]: https://github.com/rvagg/abstract-leveldown 62 | [levelup]: https://github.com/rvagg/node-levelup 63 | [DynamoDB]: http://aws.amazon.com/dynamodb/ 64 | [leveldown]: https://github.com/rvagg/node-leveldown/ 65 | [io.js]: https://iojs.org 66 | -------------------------------------------------------------------------------- /dynamo-down.js: -------------------------------------------------------------------------------- 1 | import {AbstractLevelDOWN, AbstractIterator} from "abstract-leveldown" 2 | 3 | const serialize = function(value) { 4 | if (value == null || value === "") return {NULL: true} 5 | 6 | const type = value.constructor.name 7 | const reduce = function(value) { 8 | return Object.keys(value).reduce(function(acc, key) { 9 | acc[key] = serialize(value[key]) 10 | return acc 11 | }, {}) 12 | } 13 | 14 | switch (type) { 15 | case "String" : return {S: value} 16 | case "Buffer" : return {B: value.toString("base64")} 17 | case "Boolean" : return {BOOL: value} 18 | case "Number" : return {N: String(value)} 19 | case "Array" : return {L: value.map(serialize)} 20 | case "Object" : return {M: reduce(value)} 21 | default : throw new Error(`Cannot serialize ${type}`) 22 | } 23 | } 24 | 25 | const parse = function(val) { 26 | const type = Object.keys(val)[0] 27 | const value = val[type] 28 | const reduce = function(value) { 29 | return Object.keys(value).reduce(function(acc, key) { 30 | acc[key] = parse(value[key]) 31 | return acc 32 | }, {}) 33 | } 34 | 35 | switch (type) { 36 | case "NULL" : return null 37 | case "S" : return value 38 | case "B" : return Buffer(value, "base64") 39 | case "BOOL" : return value 40 | case "N" : return parseFloat(value, 10) 41 | case "L" : return value.map(parse) 42 | case "M" : return reduce(value) 43 | default : throw new Error(`Cannot parse ${type}.`) 44 | } 45 | } 46 | 47 | class DynamoIterator extends AbstractIterator { 48 | constructor(db, options) { 49 | super(db) 50 | 51 | this._limit = Infinity 52 | if (options.limit !== -1) this._limit = options.limit 53 | 54 | this._reverse = false 55 | if (options.reverse === true) this._reverse = true 56 | 57 | if ("gt" in options || "gte" in options) { 58 | this._lowerBound = { 59 | key: options.gt || options.gte, 60 | inclusive: "gte" in options 61 | } 62 | } 63 | 64 | if ("lt" in options || "lte" in options) { 65 | this._upperBound = { 66 | key: options.lt || options.lte, 67 | inclusive: "lte" in options 68 | } 69 | } 70 | 71 | this._params = { 72 | TableName: this.db._table.name, 73 | KeyConditions: {} 74 | } 75 | 76 | if (this._limit !== Infinity) this._params.Limit = this._limit 77 | if (this._reverse) this._params.ScanIndexForward = false 78 | 79 | this._params.KeyConditions[this.db._schema.hash.name] = { 80 | ComparisonOperator: "EQ", 81 | AttributeValueList: [serialize(this.db._schema.hash.value)] 82 | } 83 | 84 | if (this._lowerBound && this._upperBound) { 85 | this._params.KeyConditions[this.db._schema.range.name] = { 86 | ComparisonOperator: "BETWEEN", 87 | AttributeValueList: [ 88 | serialize(this._lowerBound.key), 89 | serialize(this._upperBound.key) 90 | ] 91 | } 92 | } 93 | 94 | else if (this._lowerBound) { 95 | this._params.KeyConditions[this.db._schema.range.name] = { 96 | ComparisonOperator: this._lowerBound.inclusive ? "GE" : "GT", 97 | AttributeValueList: [serialize(this._lowerBound.key)] 98 | } 99 | } 100 | 101 | else if (this._upperBound) { 102 | this._params.KeyConditions[this.db._schema.range.name] = { 103 | ComparisonOperator: this._upperBound.inclusive ? "LE" : "LT", 104 | AttributeValueList: [serialize(this._upperBound.key)] 105 | } 106 | } 107 | 108 | this._items = [] 109 | this._cursor = 0 110 | } 111 | 112 | _next(cb) { 113 | const item = this._items[this._cursor] 114 | 115 | if (item) { 116 | // make sure not excluded from gt/lt key conditions 117 | setImmediate(cb, null, item.key, JSON.stringify(item.value)) 118 | delete this._items[this._cursor] 119 | this._cursor++ 120 | return 121 | } 122 | 123 | if (item === null || this._cursor === this._limit) { 124 | setImmediate(cb) 125 | return 126 | } 127 | 128 | this.db._dynamo.query(this._params, (err, data) => { 129 | if (err) return cb(err) 130 | 131 | const {Items, LastEvaluatedKey} = data 132 | 133 | for (let item of Items) this._items.push(this.db._toKV(item)) 134 | 135 | if (!LastEvaluatedKey) this._items.push(null) 136 | 137 | this._params.ExclusiveStartKey = LastEvaluatedKey 138 | this._next(cb) 139 | }) 140 | } 141 | } 142 | 143 | class DynamoDOWN extends AbstractLevelDOWN { 144 | constructor(dynamo, location) { 145 | super(location) 146 | 147 | const [table, hash] = location.split("/") 148 | 149 | this._dynamo = dynamo 150 | this._table = {name: table} 151 | this._schema = { 152 | hash: {value: hash}, 153 | range: {} 154 | } 155 | } 156 | 157 | _toItem({key, value}) { 158 | const item = value ? JSON.parse(value) : {} 159 | 160 | item[this._schema.hash.name] = this._schema.hash.value 161 | item[this._schema.range.name] = key 162 | 163 | return serialize(item).M 164 | } 165 | 166 | _toKV(item) { 167 | const value = parse({M: item}) 168 | const key = value[this._schema.range.name] 169 | 170 | delete value[this._schema.range.name] 171 | delete value[this._schema.hash.name] 172 | 173 | return {key, value} 174 | } 175 | 176 | _open(options, cb) { 177 | const params = {TableName: this._table.name} 178 | const ontable = (err, data) => { 179 | if (err) return cb(err) 180 | 181 | for (let {KeyType, AttributeName} of data.Table.KeySchema) { 182 | this._schema[KeyType.toLowerCase()].name = AttributeName 183 | } 184 | 185 | cb() 186 | } 187 | 188 | this._dynamo.describeTable(params, ontable) 189 | } 190 | 191 | _get(key, options, cb) { 192 | const TableName = this._table.name 193 | const {valueEncoding} = options 194 | const Key = this._toItem({key}) 195 | const params = {TableName, Key} 196 | 197 | this._dynamo.getItem(params, (err, data) => { 198 | if (err) return cb(err) 199 | 200 | if (!data.Item) return cb(new Error("NotFound")) 201 | 202 | const {value} = this._toKV(data.Item) 203 | const isValue = valueEncoding !== "json" 204 | 205 | let item = isValue ? value.value : JSON.stringify(value) 206 | if (options.asBuffer !== false) item = new Buffer(item) 207 | 208 | cb(null, item) 209 | }) 210 | } 211 | 212 | _put(key, value, options, cb) { 213 | const TableName = this._table.name 214 | const {valueEncoding} = options 215 | 216 | const Item = this._toItem({key, value}) 217 | const params = {TableName, Item} 218 | 219 | this._dynamo.putItem(params, err => cb(err)) 220 | } 221 | 222 | _del(key, options, cb) { 223 | const TableName = this._table.name 224 | const Key = this._toItem({key}) 225 | const params = {TableName, Key} 226 | 227 | this._dynamo.deleteItem(params, err => cb(err)) 228 | } 229 | 230 | _iterator(options) { 231 | return new DynamoIterator(this, options) 232 | } 233 | 234 | _batch(array, options, cb) { 235 | const TableName = this._table.name 236 | 237 | const ops = array.map(({type, key, value}) => ( 238 | type === "del" 239 | ? {DeleteRequest: {Key: this._toItem({key})}} 240 | : {PutRequest: {Item: this._toItem({key, value})}} 241 | )) 242 | 243 | const params = {RequestItems: {}} 244 | 245 | const loop = (err, data) => { 246 | if (err) return cb(err) 247 | 248 | const reqs = [] 249 | 250 | if (data && data.UnprocessedItems && data.UnprocessedItems[TableName]) { 251 | reqs.push(...data.UnprocessedItems[TableName]) 252 | } 253 | 254 | reqs.push(...ops.splice(0, 25 - reqs.length)) 255 | 256 | if (reqs.length === 0) return cb() 257 | 258 | params.RequestItems[TableName] = reqs 259 | 260 | this._dynamo.batchWriteItem(params, loop) 261 | } 262 | 263 | loop() 264 | } 265 | } 266 | 267 | export default function(dynamo) { 268 | const ctor = function(location) { 269 | return new DynamoDOWN(dynamo, location) 270 | } 271 | 272 | ctor.destroy = function(location, cb) { 273 | const dynamoDown = ctor(location) 274 | 275 | dynamoDown.open(err => { 276 | if (err) return cb(err) 277 | 278 | const iterator = dynamoDown.iterator() 279 | const ops = [] 280 | const pull = function(err) { 281 | if (err) return cb(err) 282 | 283 | iterator.next((err, key) => { 284 | if (err) return cb(err) 285 | 286 | if (!key) return flush(cb) 287 | 288 | ops.push({type: "del", key}) 289 | 290 | ops.length < 25 ? pull() : flush(pull) 291 | }) 292 | } 293 | 294 | const flush = function(cb) { 295 | dynamoDown.batch(ops.splice(0), cb) 296 | } 297 | 298 | pull() 299 | }) 300 | } 301 | 302 | return ctor 303 | } 304 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamo-down", 3 | "version": "1.0.4", 4 | "description": "A leveldown API implementation on AWS DynamoDB", 5 | "author": "Jed Schmidt (http://jed.is)", 6 | "license": "MIT", 7 | "scripts": { 8 | "build": "babel dynamo-down.js > index.js", 9 | "test": "babel-node test.js", 10 | "prepublish": "npm run build" 11 | }, 12 | "keywords": [ 13 | "leveldb", 14 | "leveldown", 15 | "levelup", 16 | "dynamodb" 17 | ], 18 | "repository": { 19 | "type": "git", 20 | "url": "https://github.com/jed/dynamo-down.git" 21 | }, 22 | "devDependencies": { 23 | "async": "^0.9.0", 24 | "aws-sdk": "^2.1.10", 25 | "babel": "^4.0.1", 26 | "concat-stream": "^1.4.7", 27 | "dynalite": "^0.5.2", 28 | "levelup": "^0.19.0" 29 | }, 30 | "dependencies": { 31 | "abstract-leveldown": "^2.1.0" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | import url from "url" 2 | import http from "http" 3 | import assert from "assert" 4 | import dynalite from "dynalite" 5 | import async from "async" 6 | import aws from "aws-sdk" 7 | import levelup from "levelup" 8 | import concat from "concat-stream" 9 | import DynamoDOWN from "./dynamo-down" 10 | 11 | let server 12 | let dynamo 13 | let dynamoDown 14 | let db 15 | 16 | const httpStatusCodes = Object 17 | .keys(http.STATUS_CODES) 18 | .map(key => ({key, value: {message: http.STATUS_CODES[key]}})) 19 | 20 | const openDatabase = function(cb) { 21 | server = dynalite({createTableMs: 0}) 22 | 23 | server.listen(function(err) { 24 | if (err) throw err 25 | 26 | const {address, port} = server.address() 27 | const endpoint = url.format({ 28 | protocol: "http", 29 | hostname: address, 30 | port: port 31 | }) 32 | 33 | dynamo = new aws.DynamoDB({ 34 | endpoint: endpoint, 35 | region: "us-east-1", 36 | accessKeyId: "....................", 37 | secretAccessKey: "........................................" 38 | }) 39 | 40 | cb() 41 | }) 42 | } 43 | 44 | const closeDatabase = function(cb) { 45 | server.close(cb) 46 | } 47 | 48 | const createTable = function(cb) { 49 | const params = { 50 | TableName: "test", 51 | KeySchema: [ 52 | { 53 | "AttributeName": "type", 54 | "KeyType": "HASH" 55 | }, 56 | { 57 | "AttributeName": "key", 58 | "KeyType": "RANGE" 59 | }, 60 | ], 61 | AttributeDefinitions: [ 62 | { 63 | AttributeName: "type", 64 | AttributeType: "S" 65 | }, 66 | { 67 | AttributeName: "key", 68 | AttributeType: "S" 69 | }, 70 | ], 71 | ProvisionedThroughput: { 72 | ReadCapacityUnits: 5, 73 | WriteCapacityUnits: 5 74 | } 75 | } 76 | 77 | const oncreated = (err) => { 78 | if (err) throw err 79 | 80 | dynamoDown = DynamoDOWN(dynamo) 81 | db = levelup("test/httpStatusCode", { 82 | db: dynamoDown, 83 | valueEncoding: "json" 84 | }) 85 | 86 | setTimeout(cb, 1000) 87 | } 88 | 89 | dynamo.createTable(params, oncreated) 90 | } 91 | 92 | const deleteTable = function(cb) { 93 | const params = { 94 | TableName: "test" 95 | } 96 | 97 | const ondeleted = (err) => { 98 | if (err) throw err 99 | 100 | setTimeout(cb, 1000) 101 | } 102 | 103 | dynamo.deleteTable(params, ondeleted) 104 | } 105 | 106 | const resetTable = function(cb) { 107 | async.series([deleteTable, createTable], cb) 108 | } 109 | 110 | const write = function(cb) { 111 | const ws = db.createWriteStream() 112 | 113 | for (let code of httpStatusCodes) ws.write(code) 114 | 115 | ws.end() 116 | setTimeout(cb, 1000) 117 | } 118 | 119 | const read = function(cb) { 120 | const rs = db.createReadStream({reverse: true}) 121 | const ws = concat(array => { 122 | assert.deepEqual(array.reverse(), httpStatusCodes) 123 | cb() 124 | }) 125 | 126 | rs.pipe(ws) 127 | } 128 | 129 | const empty = function(cb) { 130 | dynamoDown.destroy("test/httpStatusCode", (err) => { 131 | if (err) return cb(err) 132 | 133 | const rs = db.createReadStream() 134 | const ws = concat(array => { 135 | assert.deepEqual(array, []) 136 | cb() 137 | }) 138 | 139 | rs.pipe(ws) 140 | }) 141 | 142 | } 143 | 144 | async.series([ 145 | openDatabase, 146 | createTable, 147 | write, 148 | read, 149 | empty, 150 | deleteTable, 151 | closeDatabase 152 | ]) 153 | --------------------------------------------------------------------------------