├── .gitignore ├── .travis.yml ├── LICENSE ├── badge.png ├── collaborators.md ├── index.d.ts ├── index.js ├── package.json ├── readme.md └── tests ├── index.js ├── read-write.js ├── remove.js ├── run.js └── string-key.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '0.10' 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Max Ogden 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 5 | 6 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 7 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /badge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/max-mapper/abstract-blob-store/a73456e57d9566953324fe2332822a20aca8d909/badge.png -------------------------------------------------------------------------------- /collaborators.md: -------------------------------------------------------------------------------- 1 | ## Collaborators 2 | 3 | abstract-blob-store is only possible due to the excellent work of the following collaborators: 4 | 5 | 6 | 7 |
maxogdenGitHub/maxogden
mafintoshGitHub/mafintosh
8 | -------------------------------------------------------------------------------- /index.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | 3 | declare class MemBlobs implements MemBlobs.AbstractBlobStore { 4 | data: {[key: string]: any} 5 | createWriteStream(opts: MemBlobs.BlobKey, callback: MemBlobs.CreateCallback): NodeJS.WriteStream 6 | createReadStream(opts: MemBlobs.BlobKey): NodeJS.ReadStream 7 | exists(opts: MemBlobs.BlobKey, callback: MemBlobs.ExistsCallback): void 8 | remove(opts: MemBlobs.BlobKey, callback: MemBlobs.RemoveCallback): void 9 | } 10 | 11 | declare namespace MemBlobs { 12 | type BlobKey = string | {key: string, [name: string]: any} 13 | type BlobMetadata = {key: string, [name: string]: any} 14 | type CreateCallback = (error: Error | null, metadata: BlobMetadata) => void 15 | type ExistsCallback = (error: Error | null, exists: boolean) => void 16 | type RemoveCallback = (error: Error | null) => void 17 | interface AbstractBlobStore { 18 | createWriteStream(opts: BlobKey, callback: CreateCallback): NodeJS.WriteStream 19 | createReadStream(opts: BlobKey): NodeJS.ReadStream 20 | exists(opts: BlobKey, callback: ExistsCallback): void 21 | remove(opts: BlobKey, callback: RemoveCallback): void 22 | } 23 | } 24 | 25 | export = MemBlobs 26 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var from = require('from2-array') 2 | var concat = require('concat-stream') 3 | var crypto = require('crypto') 4 | 5 | module.exports = MemBlobs 6 | 7 | function MemBlobs() { 8 | if (!(this instanceof MemBlobs)) return new MemBlobs() 9 | this.data = {} 10 | } 11 | 12 | MemBlobs.prototype.createWriteStream = function(opts, cb) { 13 | if (typeof opts === 'function') return this.createWriteStream(null, opts) 14 | if (typeof opts === 'string') opts = {key:opts} 15 | if (!opts) opts = {} 16 | if (!cb) cb = noop 17 | 18 | var self = this 19 | return concat(done) 20 | 21 | function done(contents) { 22 | var key = opts.key || crypto.createHash('sha1').update(contents).digest('hex') 23 | self.data[key] = contents 24 | cb(null, {key: key, size: contents.length, name: opts.name}) 25 | } 26 | } 27 | 28 | MemBlobs.prototype.createReadStream = function(opts) { 29 | if (typeof opts === 'string') opts = {key:opts} 30 | 31 | var buff = this.data[opts.key] 32 | var stream 33 | if (!buff) { 34 | stream = from([]) 35 | var error = new Error('Blob not found') 36 | error.notFound = true 37 | stream.destroy(error) 38 | } else { 39 | stream = from([buff]) 40 | } 41 | return stream 42 | } 43 | 44 | MemBlobs.prototype.exists = function(opts, cb) { 45 | if (typeof opts === 'string') opts = {key:opts} 46 | cb(null, !!this.data[opts.key]) 47 | } 48 | 49 | MemBlobs.prototype.remove = function(opts, cb) { 50 | if (!cb) cb = noop 51 | delete this.data[opts.key] 52 | cb() 53 | } 54 | 55 | function noop() {} -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "abstract-blob-store", 3 | "version": "3.3.5", 4 | "description": "A test suite and interface you can use to implement streaming file (blob) storage modules for various storage backends and platforms.", 5 | "main": "index.js", 6 | "typings": "index.d.ts", 7 | "keywords": [ 8 | "hyperdata", 9 | "dat" 10 | ], 11 | "files": [ 12 | "index.js", 13 | "index.d.ts", 14 | "tests/*" 15 | ], 16 | "scripts": { 17 | "test": "node tests/run.js" 18 | }, 19 | "author": "max ogden", 20 | "license": "BSD-2-Clause", 21 | "dependencies": { 22 | "concat-stream": "^1.6.0", 23 | "from2-array": "0.0.4" 24 | }, 25 | "directories": { 26 | "test": "tests" 27 | }, 28 | "devDependencies": { 29 | "tape": "^4.8.0" 30 | }, 31 | "repository": { 32 | "type": "git", 33 | "url": "https://github.com/maxogden/abstract-blob-store.git" 34 | }, 35 | "bugs": { 36 | "url": "https://github.com/maxogden/abstract-blob-store/issues" 37 | }, 38 | "homepage": "https://github.com/maxogden/abstract-blob-store" 39 | } 40 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # abstract-blob-store 2 | 3 | A test suite and interface you can use to implement streaming file ([blob](http://en.wikipedia.org/wiki/Binary_large_object)) storage modules for various storage backends and platforms. 4 | 5 | [![NPM](https://nodei.co/npm/abstract-blob-store.png)](https://nodei.co/npm/abstract-blob-store/) 6 | 7 | [![Build Status](http://img.shields.io/travis/maxogden/abstract-blob-store.svg?style=flat)](https://travis-ci.org/maxogden/abstract-blob-store) 8 | 9 | The goal of this module is to define a de-facto standard streaming file storage/retrieval API. Inspired by the [abstract-leveldown](https://github.com/rvagg/abstract-leveldown) module, which has [a test suite that is usable as a module](https://github.com/rvagg/abstract-leveldown/tree/master/abstract). 10 | 11 | Publishing a test suite as a module lets multiple modules all ensure compatibility since they use the same test suite. For example, [level.js uses abstract-leveldown](https://github.com/maxogden/level.js/blob/master/test/test.js), and so does [memdown](https://github.com/rvagg/memdown/blob/master/test.js) and [leveldown](https://github.com/rvagg/node-leveldown/blob/master/test/close-test.js) and others. 12 | 13 | ## some modules that use this 14 | 15 | - [content-addressable-blob-store](https://github.com/mafintosh/content-addressable-blob-store) 16 | - [fs-blob-store](https://github.com/mafintosh/fs-blob-store) 17 | - [google-cloud-storage](https://github.com/maxogden/google-cloud-storage) 18 | - [google-drive-blobs](https://github.com/maxogden/google-drive-blobs) 19 | - [idb-blob-store](https://github.com/substack/idb-blob-store) 20 | - [idb-content-addressable-blob-store](https://github.com/substack/idb-content-addressable-blob-store) 21 | - [ipfs-blob-store](https://github.com/ipfs/ipfs-blob-store) 22 | - [local-blob-store](https://github.com/maxogden/local-blob-store) (deprecated) 23 | - [local-storage-blob-store](https://github.com/xicombd/local-storage-blob-store) 24 | - [level-blob-store](https://github.com/diasdavid/level-blob-store) 25 | - [level-blobs](https://github.com/mafintosh/level-blobs) (pre abstract-blob-store, but has the same interface) 26 | - [manta-blob-store](https://github.com/klokoy/manta-blob-store) 27 | - [meta-blob-store](https://github.com/bengl/meta-blob-store) 28 | - [postgres-blob-store](https://github.com/finnp/postgres-blob-store) 29 | - [s3-blob-store](https://github.com/jb55/s3-blob-store) 30 | - [torrent-blob-store](https://github.com/mafintosh/torrent-blob-store) 31 | - [azure-blob-store](https://github.com/svnlto/azure-blob-store) 32 | 33 | send a PR adding yours if you write a new one 34 | 35 | ## badge 36 | 37 | Include this badge in your readme if you make a new module that uses the `abstract-blob-store` API 38 | 39 | [![blob-store-compatible](badge.png)](https://github.com/maxogden/abstract-blob-store) 40 | 41 | ## how to use 42 | 43 | To use the test suite from this module you can `require('abstract-blob-store/tests')` 44 | 45 | An example of this can be found in the [google-drive-blobs](https://github.com/maxogden/google-drive-blobs/blob/master/test.js) test suite. There is also an example in `tests/run.js` in this repo. 46 | 47 | You have to implement a setup and teardown function: 48 | 49 | ```js 50 | var common = { 51 | setup: function(t, cb) { 52 | // setup takes a tap/tape compatible test instance in and a callback 53 | // this method should construct a new blob store instance and pass it to the callback: 54 | var store = createMyBlobStore() 55 | cb(null, store) 56 | }, 57 | teardown: function(t, store, blob, cb) { 58 | // teardown takes in the test instance, as well as the store instance and blob metadata 59 | // you can use the store/blob objects to clean up blobs from your blob backend, e.g. 60 | if (blob) store.remove(blob, cb) 61 | else cb() 62 | // be sure to call cb() when you are done with teardown 63 | } 64 | } 65 | ``` 66 | 67 | To run the tests simply pass your test module (`tap` or `tape` or any other compatible modules are supported) and your `common` methods in: 68 | 69 | ```js 70 | var abstractBlobTests = require('abstract-blob-store/tests') 71 | abstractBlobTests(test, common) 72 | ``` 73 | 74 | ## API 75 | 76 | A valid blob store should implement the following APIs. There is a reference in-memory implementation available at `index.js` in this repo. 77 | 78 | ### store.createWriteStream(opts, cb) 79 | 80 | This method should return a writable stream, and call `cb` with `err, metadata` when it finishes writing the data to the underlying blob store. 81 | 82 | If `opts` is a string it should be interpreted as a `key`. 83 | Otherwise `opts` should be an object with any blob metadata you would like to store, e.g. `name` 84 | 85 | the `metadata` passed to `cb` *must* have a `key` property that the user can pass to other methods to get the blob back again. 86 | 87 | You can choose how to store the blob. The recommended way is to hash the contents of the incoming stream and store the blob using that hash as the key (this is known as 'content-addressed storage'). If this is not an option you can choose some other way to store the data. When calling the callback you should return an object that ideally has all of the relevant metadata on it, as this object will be used to later read the blob from the blob store. 88 | 89 | In ths reference implementation the callback gets called with `{key: sha, size: contents.length, name: opts.name}`. 90 | 91 | ### store.createReadStream(opts) 92 | 93 | This method should return a readable stream that emits blob data from the underlying blob store or emits an error if the blob does not exist or if there was some other error during the read. 94 | 95 | If `opts` is a string it should be interpreted as a `key`. 96 | Otherwise `opts` *must* be an object with a `key` property. The `key` is used to find and read the blob. It is recommended where possible to use the hash of the contents of the file as the `key` in order to avoid duplication or finding the wrong file. 97 | 98 | ### store.exists(opts, cb) 99 | 100 | This checks if a blob exists in the store. 101 | 102 | If `opts` is a string it should be interpreted as a `key`. 103 | Otherwise `opts` *must* be an object with a `key` property (the same key that you got back from createReadStream). The `cb` should be called with `err, exists`, where `err` is an error if something went wrong during the exists check, and `exists` is a boolean. 104 | 105 | ### store.remove(opts, cb) 106 | 107 | This method should remove a blob from the store. 108 | 109 | If `opts` is a string is should be interpreted as a `key`. 110 | Otherwise `opts` *must* be an object with a `key` property. If the `cb` is called without an error subsequent calls to `.exists` with the same opts should return `false`. 111 | 112 | ## Background 113 | 114 | An `abstract-blob-store` is a general system for storing and retrieving binary files, utilizing different storage and addressing schemes. A blob is the set of binary data that makes up an entire binary file. 115 | 116 | Blobs are sometimes cut up into chunks so that they can be processed in various ways (see [rabin](https://github.com/maxogden/rabin)). If you are dealing with chunks of individual blobs, you may be looking for [abstract-chunk-store](https://github.com/mafintosh/abstract-chunk-store). 117 | -------------------------------------------------------------------------------- /tests/index.js: -------------------------------------------------------------------------------- 1 | module.exports = function(test, common) { 2 | require('./read-write.js').all(test, common) 3 | require('./string-key.js').all(test, common) 4 | require('./remove.js').all(test ,common) 5 | } 6 | -------------------------------------------------------------------------------- /tests/read-write.js: -------------------------------------------------------------------------------- 1 | var from = require('from2-array') 2 | var concat = require('concat-stream') 3 | 4 | module.exports.blobWriteStream = function(test, common) { 5 | test('piping a blob into a blob write stream', function(t) { 6 | common.setup(test, function(err, store) { 7 | t.notOk(err, 'no setup err') 8 | var ws = store.createWriteStream({name: 'test.js'}, function(err, obj) { 9 | t.error(err) 10 | t.ok(obj.key, 'blob has key') 11 | common.teardown(test, store, obj, function(err) { 12 | t.error(err) 13 | t.end() 14 | }) 15 | }) 16 | from([new Buffer('foo'), new Buffer('bar')]).pipe(ws) 17 | }) 18 | }) 19 | } 20 | 21 | module.exports.blobReadStream = function(test, common) { 22 | test('reading a blob as a stream', function(t) { 23 | common.setup(test, function(err, store) { 24 | t.notOk(err, 'no setup err') 25 | 26 | var ws = store.createWriteStream({name: 'test.js'}, function(err, blob) { 27 | t.notOk(err, 'no blob write err') 28 | t.ok(blob.key, 'blob has key') 29 | 30 | var rs = store.createReadStream(blob) 31 | 32 | rs.on('error', function(e) { 33 | t.false(e, 'no read stream err') 34 | t.end() 35 | }) 36 | 37 | rs.pipe(concat(function(file) { 38 | t.equal(file.length, 6, 'blob length is correct') 39 | common.teardown(test, store, blob, function(err) { 40 | t.error(err) 41 | t.end() 42 | }) 43 | })) 44 | }) 45 | 46 | from([new Buffer('foo'), new Buffer('bar')]).pipe(ws) 47 | }) 48 | }) 49 | } 50 | 51 | module.exports.blobReadError = function(test, common) { 52 | test('reading a blob that does not exist', function(t) { 53 | common.setup(test, function(err, store) { 54 | t.notOk(err, 'no setup err') 55 | 56 | var rs = store.createReadStream({name: 'test.js', key: '8843d7f92416211de9ebb963ff4ce28125932878'}) 57 | 58 | rs.on('error', function(e) { 59 | t.ok(e, 'got a read stream err') 60 | // t.ok(e.notFound, 'error reports not found') 61 | common.teardown(test, store, undefined, function(err) { 62 | t.error(err) 63 | t.end() 64 | }) 65 | }) 66 | }) 67 | 68 | }) 69 | } 70 | 71 | module.exports.blobExists = function(test, common) { 72 | test('check if a blob exists', function(t) { 73 | common.setup(test, function(err, store) { 74 | t.notOk(err, 'no setup err') 75 | var blobMeta = {name: 'test.js', key: '8843d7f92416211de9ebb963ff4ce28125932878'} 76 | store.exists(blobMeta, function(err, exists) { 77 | t.error(err) 78 | t.notOk(exists, 'does not exist') 79 | 80 | var ws = store.createWriteStream({name: 'test.js'}, function(err, obj) { 81 | t.notOk(err, 'no blob write err') 82 | t.ok(obj.key, 'blob has key') 83 | 84 | // on this .exists call use the metadata from the writeStream 85 | store.exists(obj, function(err, exists) { 86 | t.error(err) 87 | t.ok(exists, 'exists') 88 | common.teardown(test, store, obj, function(err) { 89 | t.error(err) 90 | t.end() 91 | }) 92 | }) 93 | }) 94 | 95 | from([new Buffer('foo'), new Buffer('bar')]).pipe(ws) 96 | }) 97 | 98 | }) 99 | }) 100 | } 101 | 102 | module.exports.all = function (test, common) { 103 | module.exports.blobWriteStream(test, common) 104 | module.exports.blobReadStream(test, common) 105 | module.exports.blobReadError(test, common) 106 | module.exports.blobExists(test, common) 107 | } 108 | -------------------------------------------------------------------------------- /tests/remove.js: -------------------------------------------------------------------------------- 1 | var from = require('from2-array') 2 | var concat = require('concat-stream') 3 | 4 | module.exports.remove = function(test, common) { 5 | test('blobs can be removed', function(t) { 6 | common.setup(test, function(err, store) { 7 | t.notOk(err, 'no setup err') 8 | var ws = store.createWriteStream({key: 'test.js'}, function(err, obj) { 9 | t.error(err) 10 | store.remove({key: obj.key}, function(err) { 11 | t.error(err) 12 | store.exists({key: obj.key}, function(err, exists) { 13 | t.error(err) 14 | t.notOk(exists, 'blob is removed') 15 | t.end() 16 | }) 17 | }) 18 | }) 19 | from([new Buffer('foo'), new Buffer('bar')]).pipe(ws) 20 | }) 21 | }) 22 | } 23 | 24 | module.exports.all = function (test, common) { 25 | module.exports.remove(test, common) 26 | } 27 | -------------------------------------------------------------------------------- /tests/run.js: -------------------------------------------------------------------------------- 1 | var tape = require('tape') 2 | var blobs = require('../') 3 | var tests = require('./') 4 | 5 | var common = { 6 | setup: function(t, cb) { 7 | // make a new blobs instance on every test 8 | cb(null, blobs()) 9 | }, 10 | teardown: function(t, store, blob, cb) { 11 | delete store.data 12 | cb() 13 | } 14 | } 15 | 16 | tests(tape, common) 17 | -------------------------------------------------------------------------------- /tests/string-key.js: -------------------------------------------------------------------------------- 1 | var from = require('from2-array') 2 | var concat = require('concat-stream') 3 | 4 | module.exports.blobWriteStream = function(test, common) { 5 | test('piping a blob into a blob write stream with string key', function(t) { 6 | common.setup(test, function(err, store) { 7 | t.notOk(err, 'no setup err') 8 | var ws = store.createWriteStream('hello-world.txt', function(err, obj) { 9 | t.error(err) 10 | t.ok(obj.key, 'blob has key') 11 | common.teardown(test, store, obj, function(err) { 12 | t.error(err) 13 | t.end() 14 | }) 15 | }) 16 | from([new Buffer('foo'), new Buffer('bar')]).pipe(ws) 17 | }) 18 | }) 19 | } 20 | 21 | module.exports.blobReadStream = function(test, common) { 22 | test('reading a blob as a stream with string key', function(t) { 23 | common.setup(test, function(err, store) { 24 | t.notOk(err, 'no setup err') 25 | 26 | var ws = store.createWriteStream('hello-world.txt', function(err, blob) { 27 | t.notOk(err, 'no blob write err') 28 | t.ok(blob.key, 'blob has key') 29 | 30 | var rs = store.createReadStream(blob.key) 31 | 32 | rs.on('error', function(e) { 33 | t.false(e, 'no read stream err') 34 | t.end() 35 | }) 36 | 37 | rs.pipe(concat(function(file) { 38 | t.equal(file.length, 6, 'blob length is correct') 39 | common.teardown(test, store, blob, function(err) { 40 | t.error(err) 41 | t.end() 42 | }) 43 | })) 44 | }) 45 | 46 | from([new Buffer('foo'), new Buffer('bar')]).pipe(ws) 47 | }) 48 | }) 49 | } 50 | 51 | module.exports.blobReadError = function(test, common) { 52 | test('reading a blob that does not exist with string key', function(t) { 53 | common.setup(test, function(err, store) { 54 | t.notOk(err, 'no setup err') 55 | 56 | var rs = store.createReadStream('foobarbaz.txt') 57 | 58 | rs.on('error', function(e) { 59 | t.ok(e, 'got a read stream err') 60 | common.teardown(test, store, undefined, function(err) { 61 | t.error(err) 62 | t.end() 63 | }) 64 | }) 65 | }) 66 | 67 | }) 68 | } 69 | 70 | module.exports.blobExists = function(test, common) { 71 | test('check if a blob exists with string key', function(t) { 72 | common.setup(test, function(err, store) { 73 | t.notOk(err, 'no setup err') 74 | store.exists('hello-world.txt', function(err, exists) { 75 | t.error(err) 76 | t.notOk(exists, 'does not exist') 77 | 78 | var ws = store.createWriteStream('hello-world.txt', function(err, obj) { 79 | t.notOk(err, 'no blob write err') 80 | t.ok(obj.key, 'blob has key') 81 | 82 | // on this .exists call use the metadata from the writeStream 83 | store.exists(obj.key, function(err, exists) { 84 | t.error(err) 85 | t.ok(exists, 'exists') 86 | common.teardown(test, store, obj, function(err) { 87 | t.error(err) 88 | t.end() 89 | }) 90 | }) 91 | }) 92 | 93 | from([new Buffer('foo'), new Buffer('bar')]).pipe(ws) 94 | }) 95 | 96 | }) 97 | }) 98 | } 99 | 100 | module.exports.all = function (test, common) { 101 | module.exports.blobWriteStream(test, common) 102 | module.exports.blobReadStream(test, common) 103 | module.exports.blobReadError(test, common) 104 | module.exports.blobExists(test, common) 105 | } 106 | --------------------------------------------------------------------------------