├── .gitignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── example.js ├── index.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | data 3 | .vscode 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: node_js 3 | node_js: node 4 | dist: xenial 5 | script: 6 | - npm test 7 | - npm run lint 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # content-addressable-blob-store Change Log 2 | All notable changes to this project will be documented in this file. 3 | This project adheres to [Semantic Versioning](http://semver.org/). 4 | 5 | ## 5.1.0 - 2020-06-11 6 | * opts.algo can now be a function. 7 | 8 | ## 4.6.0 - 2017-07-26 9 | * Add tmpdir option allowing you to set the tmpdir 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Mathias Buus 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # content-addressable-blob-store 2 | 3 | Streamable content addressable [blob](http://en.wikipedia.org/wiki/Binary_large_object) object store that is streams2 and implements the blob store interface on top of the fs module. 4 | 5 | Conforms to the [abstract-blob-store](https://github.com/maxogden/abstract-blob-store) API and passes it's test suite. 6 | 7 | ``` js 8 | npm install content-addressable-blob-store 9 | ``` 10 | 11 | [![build status](http://img.shields.io/travis/mafintosh/content-addressable-blob-store.svg?style=flat)](http://travis-ci.org/mafintosh/content-addressable-blob-store) 12 | ![dat](http://img.shields.io/badge/Development%20sponsored%20by-dat-green.svg?style=flat) 13 | 14 | [![blob-store-compatible](https://raw.githubusercontent.com/maxogden/abstract-blob-store/master/badge.png)](https://github.com/maxogden/abstract-blob-store) 15 | 16 | ## Usage 17 | 18 | ``` js 19 | var blobs = require('content-addressable-blob-store') 20 | var store = blobs('./data') 21 | 22 | var w = store.createWriteStream() 23 | 24 | w.write('hello ') 25 | w.write('world\n') 26 | 27 | w.end(function () { 28 | console.log('blob written: '+w.key) 29 | store.createReadStream(w).pipe(process.stdout) 30 | }) 31 | ``` 32 | 33 | ## API 34 | 35 | #### `var store = blobs(opts)` 36 | 37 | Creates a new instance. Opts should have a `path` property to where the blobs should live on the fs. The directory will be created if it doesn't exist. If not supplied it will default to `path.join(process.cwd(), 'blobs')` 38 | 39 | You can also specify a node `crypto` module hashing algorithm to use using the `algo` key in options. The default is `sha256`. 40 | If you pass a string instead of an options map it will be used as the `path` as well. 41 | 42 | The `tmpdir` option can be used to specify the directory where files are stored temporarily during writing. The default is `os.tmpdir()`. 43 | 44 | #### `var readStream = store.createReadStream(opts)` 45 | 46 | Open a read stream to a blob. `opts` must have a `key` key with the hash of the blob you want to read. `opts` can optionally contain a `start` or `end` key if you only want part of the blob. 47 | 48 | #### `var writeStream = store.createWriteStream([cb])` 49 | 50 | Add a new blob to the store. Use `writeStream.key` to get the hash after the `finish` event has fired 51 | or add a callback which will be called with `callback(err, metadata)`. 52 | 53 | #### `store.exists(metadata, cb)` 54 | 55 | Check if an blob exists in the blob store. `metadata` must have a `key` property. Callback is called with `callback(err, exists)` 56 | 57 | 58 | #### `store.remove(metadata, [cb])` 59 | 60 | Remove a blob from the store. `metadata` must have a `key` property. Callback is called with `callback(err, wasDeleted)` 61 | 62 | #### `store.resolve(metadata, cb)` 63 | 64 | Check if an blob exists in the blob store and return its `path` and `stat` object. `metadata` must have a `key` property. Callback is called with `callback(err, path, stat)` where `path` is the path string to the file on disk and `stat` is an instance of [fs.Stats](https://nodejs.org/api/fs.html#fs_class_fs_stats). If the `key` does does not exist in the store, `path` will be `false` and `stat` will be `null`. 65 | 66 | ## License 67 | 68 | MIT 69 | -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var blobs = require('./') 2 | var store = blobs({ path: './data' }) 3 | 4 | var w = store.createWriteStream() 5 | 6 | w.write('hello ') 7 | w.write('world\n') 8 | 9 | w.end(function () { 10 | console.log('blob written: ' + w.key) 11 | store.createReadStream(w).pipe(process.stdout) 12 | }) 13 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | var crypto = require('crypto') 4 | var stream = require('readable-stream') 5 | var util = require('util') 6 | var eos = require('end-of-stream') 7 | var os = require('os') 8 | var mkdirp = require('mkdirp') 9 | var thunky = require('thunky') 10 | 11 | var noop = function () {} 12 | 13 | var SIGNAL_FLUSH = Buffer.from([0]) 14 | 15 | var toPath = function (base, hash) { 16 | return hash ? path.join(base, hash.slice(0, 2), hash.slice(2)) : base 17 | } 18 | 19 | var Writer = function (dir, algo, init) { 20 | this.key = null 21 | this.size = 0 22 | this.destroyed = false 23 | 24 | this._tmp = null 25 | this._ws = null 26 | this._directory = dir 27 | this._digest = typeof algo === 'function' ? algo() : crypto.createHash(algo) 28 | this._init = init 29 | 30 | stream.Writable.call(this) 31 | } 32 | 33 | util.inherits(Writer, stream.Writable) 34 | 35 | Writer.prototype._flush = function (cb) { 36 | var self = this 37 | var hash = this.key = this._digest.digest('hex') 38 | var dir = path.join(this._directory, hash.slice(0, 2)) 39 | 40 | self._ws.end(function () { 41 | fs.mkdir(dir, function () { 42 | fs.rename(self._tmp, toPath(self._directory, hash), cb) 43 | }) 44 | }) 45 | } 46 | 47 | Writer.prototype._setup = function (data, enc, cb) { 48 | var self = this 49 | var destroy = function (err) { 50 | self.destroy(err) 51 | } 52 | 53 | this._init(function (dir) { 54 | if (self.destroyed) return cb(new Error('stream destroyed')) 55 | self._tmp = path.join(dir, Date.now() + '-' + Math.random().toString().slice(2)) 56 | self._ws = fs.createWriteStream(self._tmp) 57 | self._ws.on('error', destroy) 58 | self._write(data, enc, cb) 59 | }) 60 | } 61 | 62 | Writer.prototype.destroy = function (err) { 63 | if (this.destroyed) return 64 | this.destroyed = true 65 | if (this._ws) this._ws.destroy() 66 | if (err) this.emit('error', err) 67 | this.emit('close') 68 | } 69 | 70 | Writer.prototype._write = function (data, enc, cb) { 71 | if (!this._tmp) return this._setup(data, enc, cb) 72 | if (data === SIGNAL_FLUSH) return this._flush(cb) 73 | this.size += data.length 74 | this._digest.update(data) 75 | this._ws.write(data, enc, cb) 76 | } 77 | 78 | Writer.prototype.end = function (data, enc, cb) { 79 | if (typeof data === 'function') return this.end(null, null, data) 80 | if (typeof enc === 'function') return this.end(data, null, enc) 81 | if (data) this.write(data) 82 | this.write(SIGNAL_FLUSH) 83 | stream.Writable.prototype.end.call(this, cb) 84 | } 85 | 86 | module.exports = function (opts) { 87 | if (typeof opts === 'string') opts = { path: opts } 88 | if (!opts) opts = {} 89 | 90 | var algo = opts.algo 91 | if (!algo) algo = 'sha256' 92 | 93 | var dir = opts.dir || opts.path 94 | if (!dir) dir = path.join(process.cwd(), 'blobs') 95 | 96 | var tmpdir = (opts.tmpdir || os.tmpdir()) 97 | 98 | var that = {} 99 | 100 | var init = thunky(function (cb) { 101 | var tmp = path.join(tmpdir, 'cabs') 102 | mkdirp(tmp, function () { 103 | mkdirp(dir, function () { 104 | cb(tmp) 105 | }) 106 | }) 107 | }) 108 | 109 | that.createWriteStream = function (opts, cb) { 110 | if (typeof opts === 'string') opts = { key: opts } 111 | if (typeof opts === 'function') return that.createWriteStream(null, opts) 112 | 113 | var ws = new Writer(dir, algo, init) 114 | if (!cb) return ws 115 | 116 | eos(ws, function (err) { 117 | if (err) return cb(err) 118 | cb(null, { 119 | key: ws.key, 120 | size: ws.size 121 | }) 122 | }) 123 | 124 | return ws 125 | } 126 | 127 | that.createReadStream = function (opts) { 128 | if (typeof opts === 'string') opts = { key: opts } 129 | return fs.createReadStream(toPath(dir, opts.key || opts.hash), opts) 130 | } 131 | 132 | that.exists = function (opts, cb) { 133 | if (typeof opts === 'string') opts = { key: opts } 134 | fs.stat(toPath(dir, opts.key), function (err, stat) { 135 | if (err && err.code === 'ENOENT') return cb(null, false) 136 | if (err) return cb(err) 137 | cb(null, true) 138 | }) 139 | } 140 | 141 | that.remove = function (opts, cb) { 142 | if (!cb) cb = noop 143 | if (typeof opts === 'string') opts = { key: opts } 144 | fs.unlink(toPath(dir, opts.key), function (err) { 145 | if (err && err.code === 'ENOENT') return cb(null, false) 146 | if (err) return cb(err) 147 | cb(null, true) 148 | }) 149 | } 150 | 151 | that.resolve = function (opts, cb) { 152 | if (typeof opts === 'string') opts = { key: opts } 153 | var path = toPath(dir, opts.key) 154 | fs.stat(path, function (err, stat) { 155 | if (err && err.code === 'ENOENT') return cb(null, false, null) 156 | if (err) return cb(err) 157 | cb(null, path, stat) 158 | }) 159 | } 160 | 161 | return that 162 | } 163 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "content-addressable-blob-store", 3 | "version": "5.1.0", 4 | "description": "Streamable content addressable blob object store that is streams2 and implements the blob store interface on top of the fs module", 5 | "main": "index.js", 6 | "dependencies": { 7 | "end-of-stream": "^1.0.0", 8 | "mkdirp": "^0.5.0", 9 | "readable-stream": "^3.4.0", 10 | "thunky": "^1.0.3" 11 | }, 12 | "devDependencies": { 13 | "abstract-blob-store": "^3.3.5", 14 | "concat-stream": "^2.0.0", 15 | "rimraf": "^2.2.8", 16 | "standard": "^13.1.0", 17 | "standard-markdown": "^5.1.0", 18 | "tape": "^4.11.0" 19 | }, 20 | "scripts": { 21 | "test": "tape test.js", 22 | "lint": "standard ; standard-markdown README.md" 23 | }, 24 | "repository": { 25 | "type": "git", 26 | "url": "https://github.com/mafintosh/content-addressable-blob-store" 27 | }, 28 | "keywords": [ 29 | "blob", 30 | "object", 31 | "store", 32 | "content", 33 | "addressable", 34 | "streams2" 35 | ], 36 | "author": "Mathias Buus", 37 | "license": "MIT", 38 | "bugs": { 39 | "url": "https://github.com/mafintosh/content-addressable-blob-store/issues" 40 | }, 41 | "homepage": "https://github.com/mafintosh/content-addressable-blob-store" 42 | } 43 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var os = require('os') 2 | var path = require('path') 3 | var fs = require('fs') 4 | 5 | var test = require('tape') 6 | var rimraf = require('rimraf') 7 | var abstractBlobTests = require('abstract-blob-store/tests') 8 | 9 | var blobs = require('./') 10 | var blobPath = path.join(os.tmpdir(), 'fs-blob-store-tests') 11 | 12 | var common = { 13 | setup: function (t, cb) { 14 | rimraf(blobPath, function () { 15 | var store = blobs({ path: blobPath }) 16 | cb(null, store) 17 | }) 18 | }, 19 | teardown: function (t, store, blob, cb) { 20 | rimraf(blobPath, cb) 21 | } 22 | } 23 | 24 | abstractBlobTests(test, common) 25 | 26 | test('remove file', function (t) { 27 | common.setup(t, function (err, store) { 28 | t.ifError(err, 'no error') 29 | var w = store.createWriteStream() 30 | w.write('hello') 31 | w.write('world') 32 | w.end(function () { 33 | store.remove(w, function (err, deleted) { 34 | t.notOk(err, 'no err') 35 | t.ok(deleted, 'was deleted') 36 | common.teardown(t, null, null, function (err) { 37 | t.ifError(err, 'no error') 38 | t.end() 39 | }) 40 | }) 41 | }) 42 | }) 43 | }) 44 | 45 | test('seek blob', function (t) { 46 | common.setup(t, function (err, store) { 47 | t.ifError(err, 'no error') 48 | var w = store.createWriteStream() 49 | w.write('hello') 50 | w.write('world') 51 | w.end(function () { 52 | var buff = '' 53 | var blob = store.createReadStream({ key: w.key, start: 5 }) 54 | blob.on('data', function (data) { buff += data }) 55 | blob.on('end', function () { 56 | t.equal(buff, 'world') 57 | common.teardown(t, null, null, function (err) { 58 | t.ifError(err, 'no error') 59 | t.end() 60 | }) 61 | }) 62 | }) 63 | }) 64 | }) 65 | 66 | test('resolve blob', function (t) { 67 | common.setup(t, function (err, store) { 68 | t.ifError(err, 'no error') 69 | var w = store.createWriteStream() 70 | w.write('hello') 71 | w.write('world') 72 | w.end(function () { 73 | store.resolve({ key: w.key }, function (err, path, stat) { 74 | t.error(err, 'no error') 75 | t.notEqual(path, false, 'path should not be false') 76 | t.notEqual(stat, null, 'path is not null') 77 | t.true(stat instanceof fs.Stats, 'stat is instanceof Stats') 78 | store.resolve('foo', function (err, path, stat) { 79 | t.ifError(err, 'no error') 80 | t.equal(path, false, 'path should be false for missing key') 81 | t.equal(stat, null, 'path is null for missing key') 82 | common.teardown(t, null, null, function (err) { 83 | t.ifError(err, 'no error') 84 | t.end() 85 | }) 86 | }) 87 | }) 88 | }) 89 | }) 90 | }) 91 | 92 | 93 | test('error handing', function (t) { 94 | common.setup(t, function (err, store) { 95 | t.ifError(err, 'no error') 96 | var w = store.createWriteStream() 97 | w.on('error', function (err) { 98 | t.equal(err.message, 'A fake error') 99 | t.true(w.destroyed === true) 100 | common.teardown(t, null, null, function (err) { 101 | t.ifError(err, 'no error') 102 | t.end() 103 | }) 104 | }) 105 | w.write('hello') 106 | w.write('world') 107 | setTimeout(function () { 108 | w._ws.destroy(new Error('A fake error')) 109 | }, 100) 110 | }) 111 | }) 112 | --------------------------------------------------------------------------------