├── .gitattributes ├── .gitignore ├── .travis.yml ├── README.md ├── fs.js ├── inmemory.js ├── lib └── test-basics.js ├── package.json └── tests ├── test-fs.js └── test-inmemory.js /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | .nyc_output 4 | coverage -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | cache: 3 | directories: 4 | - node_modules 5 | notifications: 6 | email: false 7 | node_js: 8 | - '8' 9 | before_script: 10 | - npm prune 11 | after_success: 12 | - npm run semantic-release 13 | branches: 14 | except: 15 | - /^v\d+\.\d+\.\d+$/ 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lucass (Lightweight Universal Content Addressable Storage Spec) 2 | 3 | [![Coverage Status](https://coveralls.io/repos/github/mikeal/lucass/badge.svg?branch=master)](https://coveralls.io/github/mikeal/lucass?branch=master) 4 | [![Build Status](https://travis-ci.org/mikeal/lucass.svg?branch=master)](https://travis-ci.org/mikeal/lucass) 5 | [![dependencies Status](https://david-dm.org/mikeal/lucass/status.svg)](https://david-dm.org/mikeal/lucass) 6 | 7 | [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) 8 | [![Greenkeeper badge](https://badges.greenkeeper.io/mikeal/lucass.svg)](https://greenkeeper.io/) 9 | [![Commitizen friendly](https://img.shields.io/badge/commitizen-friendly-brightgreen.svg)](http://commitizen.github.io/cz-cli/) 10 | [![JavaScript Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://standardjs.com) 11 | 12 | There are a bunch of content addressable stores out there, and even some abstract store specs. A few things that are different about lucass: 13 | 14 | * There is **NO** requirement that implementations use a specific hashing method, only that the method they use is consistent. 15 | * This means that there can be many more types of implementations but that they aren't compatible by default. Users of each implementation may need to configure the hashing methods. 16 | * Requires support for both Buffers and Streams as values. 17 | 18 | This module contains compliance tests and two reference implementations (filesystem and inmemory). 19 | 20 | ## Spec 21 | 22 | ```javascript 23 | class Store { 24 | async set (value, cb) { 25 | // value is either a Buffer or a Stream, both must be supported. 26 | // cb(Error, Hash) 27 | // Hash must be consistent. Data written with Buffer or Stream should 28 | // be identical. 29 | // Hash must be a String. 30 | } 31 | async get (hash, cb) { 32 | // Hash must be a String. 33 | // returns a buffer. 34 | } 35 | async hash (value, cb) { 36 | // Identical method signature to set but MUST NOT store the value. 37 | } 38 | } 39 | ``` 40 | 41 | There are also optional APIs. These are not required as they may not be 42 | possible on top of certain storage but *may* be required by certain users 43 | of an implementation. 44 | 45 | ```javascript 46 | class Store { 47 | async set (value, ...args) { 48 | // Optional args are sent to the hashing function.. 49 | } 50 | async hash (value, ...args) { 51 | // Optional args are sent to the hashing function. 52 | } 53 | async missing (hashes) { 54 | // Optional array of hashes. Missing hashes will be returned. 55 | } 56 | } 57 | ``` 58 | 59 | ## In-Memory Implementation 60 | 61 | ```javascript 62 | let store = require('lucass/inmemory')() 63 | let hasher = await store.set(Buffer.from('asdf')) 64 | let value = await store.getBuffer(hash) 65 | console.log(value.toString()) // 'asdf' 66 | ``` 67 | 68 | Additionally, all methods in the spec are implemented. 69 | 70 | ## Filesystem Implementation 71 | 72 | ```javascript 73 | let store = require('lucass/fs')('/var/custom-directory') 74 | let hasher = await store.set(Buffer.from('asdf')) 75 | let value = await store.getBuffer(hash) 76 | console.log(value.toString()) // 'asdf' 77 | ``` 78 | 79 | Additionally, all methods in the spec are implemented. 80 | -------------------------------------------------------------------------------- /fs.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const path = require('path') 3 | const promisify = require('util').promisify 4 | const createHasher = require('multihasher') 5 | 6 | const isDirectory = dir => fs.statSync(dir).isDirectory() 7 | const writeFile = promisify(fs.writeFile) 8 | const readFile = promisify(fs.readFile) 9 | 10 | class FSAddressableStorage { 11 | constructor (dir, hasher = createHasher('sha256')) { 12 | /* This statement is tested but because it gets wrapped in a try/catch 13 | the coverage report doesn't notice. */ 14 | /* istanbul ignore if */ 15 | if (!isDirectory(dir)) throw new Error('Not a directory.') 16 | this.dir = dir 17 | this._hasher = hasher 18 | } 19 | async set (value, ...args) { 20 | if (!Buffer.isBuffer(value)) throw new Error('Invalid type.') 21 | let hash = await this.hash(value, ...args) 22 | await writeFile(path.join(this.dir, hash), value) 23 | return hash 24 | } 25 | async hash (value, ...args) { 26 | if (!Buffer.isBuffer(value)) throw new Error('Invalid type.') 27 | return this._hasher(value, ...args) 28 | } 29 | 30 | async get (hash) { 31 | return readFile(path.join(this.dir, hash)) 32 | } 33 | } 34 | 35 | module.exports = (...args) => new FSAddressableStorage(...args) 36 | -------------------------------------------------------------------------------- /inmemory.js: -------------------------------------------------------------------------------- 1 | let createHasher = require('multihasher') 2 | 3 | class InMemoryContentAddressableStorage { 4 | constructor (hasher = createHasher('sha256')) { 5 | this._store = new Map() 6 | this._hasher = hasher 7 | } 8 | 9 | async get (hash, cb) { 10 | let buff = this._store.get(hash) 11 | if (typeof buff === 'undefined') throw new Error('Not found.') 12 | return buff 13 | } 14 | 15 | async hash (value, ...args) { 16 | if (!Buffer.isBuffer(value)) throw new Error('Invalid type.') 17 | return this._hasher(value, ...args) 18 | } 19 | 20 | async set (value, ...args) { 21 | if (!Buffer.isBuffer(value)) throw new Error('Invalid type.') 22 | let hash = await this._hasher(value, ...args) 23 | this._store.set(hash, value) 24 | return hash 25 | } 26 | 27 | async missing (hashes) { 28 | let have = new Set(this._store.keys()) 29 | let diff = new Set(hashes.filter(h => !have.has(h))) 30 | return Array.from(diff) 31 | } 32 | } 33 | 34 | module.exports = hasher => new InMemoryContentAddressableStorage(hasher) 35 | -------------------------------------------------------------------------------- /lib/test-basics.js: -------------------------------------------------------------------------------- 1 | const test = require('tap').test 2 | 3 | module.exports = (name, store) => { 4 | test(`${name}: set Buffer, get Buffer`, async t => { 5 | t.plan(1) 6 | let x = Buffer.from('asdf') 7 | let hash = await store.set(x) 8 | let buff = await store.get(hash) 9 | t.same(x, buff) 10 | }) 11 | 12 | test(`${name}: consistent hashing w/ set API`, async t => { 13 | t.plan(1) 14 | let buff = Buffer.from('asldkfjalskdjflkasdjf') 15 | let hash1 = await store.set(buff) 16 | let hash2 = await store.set(buff) 17 | t.same(hash1, hash2) 18 | }) 19 | 20 | test(`${name}: consistent hashing w/ hash API, Buffer`, async t => { 21 | t.plan(1) 22 | let buff = Buffer.from('asldkfjalskdjfldddkasdjf') 23 | let hash1 = await store.hash(buff) 24 | let hash2 = await store.hash(buff) 25 | t.same(hash1, hash2) 26 | }) 27 | 28 | test(`${name}: get Buffer from key that has not been stored`, async t => { 29 | t.plan(1) 30 | try { 31 | await store.get('notfound') 32 | throw new Error('Got buffer from key not stored.') 33 | } catch (e) { 34 | t.type(e, 'Error') 35 | } 36 | }) 37 | 38 | test(`${name}: set invalid values`, async t => { 39 | t.plan(8) 40 | let tests = [ 41 | async () => store.set({}), 42 | async () => store.set(null), 43 | async () => store.set('asdf'), 44 | async () => store.set(1123454) 45 | ] 46 | for (let _test of tests) { 47 | try { 48 | await _test() 49 | } catch (e) { 50 | t.type(e, 'Error') 51 | t.same(e.message, 'Invalid type.') 52 | } 53 | } 54 | }) 55 | 56 | test(`${name}: hash invalid values`, async t => { 57 | t.plan(8) 58 | let tests = [ 59 | async () => store.hash({}), 60 | async () => store.hash(null), 61 | async () => store.hash('asdf'), 62 | async () => store.hash(1123454) 63 | ] 64 | for (let _test of tests) { 65 | try { 66 | await _test() 67 | } catch (e) { 68 | t.type(e, 'Error') 69 | t.same(e.message, 'Invalid type.') 70 | } 71 | } 72 | }) 73 | } 74 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lucass", 3 | "version": "0.0.0-development", 4 | "description": "", 5 | "main": "fs.js", 6 | "directories": { 7 | "lib": "lib", 8 | "test": "tests" 9 | }, 10 | "scripts": { 11 | "test": "tap tests/*.js --100 && standard", 12 | "coverage": "tap tests/*.js --coverage-report=lcov && codecov", 13 | "precommit": "npm test", 14 | "prepush": "npm test", 15 | "commitmsg": "validate-commit-msg", 16 | "commit": "git-cz", 17 | "semantic-release": "semantic-release pre && npm publish && semantic-release post" 18 | }, 19 | "keywords": [], 20 | "author": "Mikeal Rogers ", 21 | "license": "Apache-2.0", 22 | "dependencies": { 23 | "multihasher": "^1.0.0", 24 | "tap": "^10.7.0" 25 | }, 26 | "devDependencies": { 27 | "codecov": "^2.2.0", 28 | "commitizen": "^2.9.6", 29 | "coveralls": "^2.13.1", 30 | "cracks": "^3.1.2", 31 | "cz-conventional-changelog": "^2.0.0", 32 | "husky": "^0.14.3", 33 | "rimraf": "^2.6.1", 34 | "semantic-release": "^7.0.1", 35 | "standard": "^10.0.2", 36 | "validate-commit-msg": "^2.13.1" 37 | }, 38 | "config": { 39 | "commitizen": { 40 | "path": "./node_modules/cz-conventional-changelog" 41 | } 42 | }, 43 | "repository": { 44 | "type": "git", 45 | "url": "https://github.com/mikeal/lucass.git" 46 | }, 47 | "nyc": { 48 | "exclude": [ 49 | "lib/test-basics.js", 50 | "tests", 51 | "tests/*", 52 | "**/node_modules/**" 53 | ] 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /tests/test-fs.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const fs = require('fs') 3 | const fsStore = require('../fs') 4 | const testdir = fs.mkdtempSync(path.join(__dirname, '.testtmp-')) 5 | 6 | require('../lib/test-basics')('fs', fsStore(testdir)) 7 | 8 | const test = require('tap').test 9 | 10 | test('fs(implementation): directory does not exist', t => { 11 | t.plan(1) 12 | t.throws(() => { 13 | fsStore('falskdjfoaisdfjoadfjodi8f9823') 14 | }) 15 | }) 16 | 17 | test('fs(implementation): hasher args', async t => { 18 | t.plan(2) 19 | const argHasher = async (buff, one, two, three) => { 20 | t.same([one, two, three], [1, 2, 3]) 21 | return 'asdf' 22 | } 23 | let store = fsStore(testdir, argHasher) 24 | await store.set(Buffer.from('asdf'), 1, 2, 3) 25 | await store.hash(Buffer.from('asdf'), 1, 2, 3) 26 | }) 27 | 28 | let rimraf = require('rimraf') 29 | 30 | process.on('beforeExit', () => { 31 | rimraf.sync(testdir) 32 | }) 33 | -------------------------------------------------------------------------------- /tests/test-inmemory.js: -------------------------------------------------------------------------------- 1 | require('../lib/test-basics')('inmemory', require('../inmemory')()) 2 | 3 | const test = require('tap').test 4 | const inmemory = require('../inmemory') 5 | 6 | test('inmemory(implementation): hasher args', async t => { 7 | t.plan(2) 8 | const argHasher = async (buff, one, two, three) => { 9 | t.same([one, two, three], [1, 2, 3]) 10 | return 'asdf' 11 | } 12 | let store = inmemory(argHasher) 13 | await store.set(Buffer.from('asdf'), 1, 2, 3) 14 | await store.hash(Buffer.from('asdf'), 1, 2, 3) 15 | }) 16 | 17 | test('inmemory: missing API', async t => { 18 | t.plan(2) 19 | let store = inmemory() 20 | let key = await store.set(Buffer.from('asdf'), 1, 2, 3) 21 | t.same(await store.missing(['asdf']), ['asdf']) 22 | t.same(await store.missing([key]), []) 23 | }) 24 | --------------------------------------------------------------------------------