├── .npmignore ├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── LICENSE ├── package.json ├── README.md ├── index.js └── test └── basic.js /.npmignore: -------------------------------------------------------------------------------- 1 | test/ 2 | .github/ 3 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: / 5 | schedule: 6 | interval: daily 7 | labels: 8 | - dependency 9 | versioning-strategy: increase-if-necessary 10 | - package-ecosystem: github-actions 11 | directory: / 12 | schedule: 13 | interval: daily 14 | labels: 15 | - dependency 16 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 'on': 3 | - push 4 | - pull_request 5 | jobs: 6 | test: 7 | name: Node ${{ matrix.node }} / ${{ matrix.os }} 8 | runs-on: ${{ matrix.os }} 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | os: 13 | - ubuntu-latest 14 | node: 15 | - '14' 16 | steps: 17 | - uses: actions/checkout@v2 18 | - uses: actions/setup-node@v2 19 | with: 20 | node-version: ${{ matrix.node }} 21 | - run: npm install 22 | - run: npm run build --if-present 23 | - run: npm test 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Feross Aboukhadijeh 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cache-chunk-store", 3 | "description": "In-memory LRU cache for abstract-chunk-store compliant stores", 4 | "version": "3.2.2", 5 | "author": { 6 | "name": "Feross Aboukhadijeh", 7 | "email": "feross@feross.org", 8 | "url": "https://feross.org" 9 | }, 10 | "bugs": { 11 | "url": "https://github.com/feross/cache-chunk-store/issues" 12 | }, 13 | "dependencies": { 14 | "lru": "^3.1.0", 15 | "queue-microtask": "^1.2.3" 16 | }, 17 | "devDependencies": { 18 | "abstract-chunk-store": "^1.5.0", 19 | "fs-chunk-store": "^2.0.3", 20 | "immediate-chunk-store": "^2.2.0", 21 | "memory-chunk-store": "^1.3.5", 22 | "standard": "*", 23 | "tape": "^5.2.2" 24 | }, 25 | "homepage": "https://github.com/feross/cache-chunk-store", 26 | "keywords": [ 27 | "abstract-chunk-store", 28 | "chunk", 29 | "lru", 30 | "least recently used", 31 | "cache", 32 | "store" 33 | ], 34 | "license": "MIT", 35 | "main": "index.js", 36 | "repository": { 37 | "type": "git", 38 | "url": "git://github.com/feross/cache-chunk-store.git" 39 | }, 40 | "scripts": { 41 | "test": "standard && tape test/*.js" 42 | }, 43 | "funding": [ 44 | { 45 | "type": "github", 46 | "url": "https://github.com/sponsors/feross" 47 | }, 48 | { 49 | "type": "patreon", 50 | "url": "https://www.patreon.com/feross" 51 | }, 52 | { 53 | "type": "consulting", 54 | "url": "https://feross.org/support" 55 | } 56 | ] 57 | } 58 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cache-chunk-store [![ci][ci-image]][ci-url] [![npm][npm-image]][npm-url] [![downloads][downloads-image]][downloads-url] [![javascript style guide][standard-image]][standard-url] 2 | 3 | [ci-image]: https://img.shields.io/github/workflow/status/feross/cache-chunk-store/ci/master 4 | [ci-url]: https://github.com/feross/cache-chunk-store/actions 5 | [npm-image]: https://img.shields.io/npm/v/cache-chunk-store.svg 6 | [npm-url]: https://npmjs.org/package/cache-chunk-store 7 | [downloads-image]: https://img.shields.io/npm/dm/cache-chunk-store.svg 8 | [downloads-url]: https://npmjs.org/package/cache-chunk-store 9 | [standard-image]: https://img.shields.io/badge/code_style-standard-brightgreen.svg 10 | [standard-url]: https://standardjs.com 11 | 12 | #### In-memory LRU (least-recently-used) cache for [abstract-chunk-store](https://github.com/mafintosh/abstract-chunk-store) compliant stores 13 | 14 | [![abstract chunk store](https://cdn.rawgit.com/mafintosh/abstract-chunk-store/master/badge.svg)](https://github.com/mafintosh/abstract-chunk-store) 15 | 16 | This caches the results of `store.get()` calls using 17 | [`lru`](https://www.npmjs.com/package/lru). See the `lru` docs for the 18 | full list of configuration options. 19 | 20 | ## Install 21 | 22 | ``` 23 | npm install cache-chunk-store 24 | ``` 25 | 26 | ## Usage 27 | 28 | ``` js 29 | const CacheChunkStore = require('cache-chunk-store') 30 | const FSChunkStore = require('fs-chunk-store') // any chunk store will work 31 | 32 | const store = new CacheChunkStore(new FSChunkStore(10), { 33 | // options are passed through to `lru-cache` 34 | max: 100 // maximum cache size (this is probably the only option you need) 35 | }) 36 | 37 | store.put(0, new Buffer('abc'), err => { 38 | if (err) throw err 39 | 40 | store.get(0, (err, data) => { 41 | if (err) throw err 42 | console.log(data) 43 | 44 | // this will be super fast because it's cached in memory! 45 | store.get(0, (err, data) => { 46 | if (err) throw err 47 | console.log(data) 48 | }) 49 | }) 50 | }) 51 | 52 | ``` 53 | 54 | ## License 55 | 56 | MIT. Copyright (c) [Feross Aboukhadijeh](https://feross.org). 57 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /*! cache-chunk-store. MIT License. Feross Aboukhadijeh */ 2 | const LRU = require('lru') 3 | const queueMicrotask = require('queue-microtask') 4 | 5 | class CacheStore { 6 | constructor (store, opts) { 7 | this.store = store 8 | this.chunkLength = store.chunkLength 9 | this.inProgressGets = new Map() // Map from chunk index to info on callbacks waiting for that chunk 10 | 11 | if (!this.store || !this.store.get || !this.store.put) { 12 | throw new Error('First argument must be abstract-chunk-store compliant') 13 | } 14 | 15 | this.cache = new LRU(opts) 16 | } 17 | 18 | put (index, buf, cb = () => {}) { 19 | if (!this.cache) { 20 | return queueMicrotask(() => cb(new Error('CacheStore closed'))) 21 | } 22 | 23 | this.cache.remove(index) 24 | this.store.put(index, buf, cb) 25 | } 26 | 27 | get (index, opts, cb = () => {}) { 28 | if (typeof opts === 'function') return this.get(index, null, opts) 29 | 30 | if (!this.cache) { 31 | return queueMicrotask(() => cb(new Error('CacheStore closed'))) 32 | } 33 | 34 | if (!opts) opts = {} 35 | 36 | let buf = this.cache.get(index) 37 | if (buf) { 38 | const offset = opts.offset || 0 39 | const len = opts.length || (buf.length - offset) 40 | if (offset !== 0 || len !== buf.length) { 41 | buf = buf.slice(offset, len + offset) 42 | } 43 | return queueMicrotask(() => cb(null, buf)) 44 | } 45 | 46 | // See if a get for this index has already started 47 | let waiters = this.inProgressGets.get(index) 48 | const getAlreadyStarted = !!waiters 49 | if (!waiters) { 50 | waiters = [] 51 | this.inProgressGets.set(index, waiters) 52 | } 53 | 54 | waiters.push({ 55 | opts, 56 | cb 57 | }) 58 | 59 | if (!getAlreadyStarted) { 60 | this.store.get(index, (err, buf) => { 61 | if (!err && this.cache != null) this.cache.set(index, buf) 62 | 63 | const inProgressEntry = this.inProgressGets.get(index) 64 | this.inProgressGets.delete(index) 65 | 66 | for (const { opts, cb } of inProgressEntry) { 67 | if (err) { 68 | cb(err) 69 | } else { 70 | const offset = opts.offset || 0 71 | const len = opts.length || (buf.length - offset) 72 | let slicedBuf = buf 73 | if (offset !== 0 || len !== buf.length) { 74 | slicedBuf = buf.slice(offset, len + offset) 75 | } 76 | cb(null, slicedBuf) 77 | } 78 | } 79 | }) 80 | } 81 | } 82 | 83 | close (cb = () => {}) { 84 | if (!this.cache) { 85 | return queueMicrotask(() => cb(new Error('CacheStore closed'))) 86 | } 87 | 88 | this.cache = null 89 | this.store.close(cb) 90 | } 91 | 92 | destroy (cb = () => {}) { 93 | if (!this.cache) { 94 | return queueMicrotask(() => cb(new Error('CacheStore closed'))) 95 | } 96 | 97 | this.cache = null 98 | this.store.destroy(cb) 99 | } 100 | } 101 | 102 | module.exports = CacheStore 103 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | const abstractTests = require('abstract-chunk-store/tests') 2 | const CacheChunkStore = require('../') 3 | const FSChunkStore = require('fs-chunk-store') 4 | const ImmediateChunkStore = require('immediate-chunk-store') 5 | const MemoryChunkStore = require('memory-chunk-store') 6 | const test = require('tape') 7 | 8 | test('abstract-chunk-store tests', t => { 9 | testAllStores(t, (Store, t) => abstractTests(t.test.bind(t), Store)) 10 | }) 11 | 12 | test('get should be cached', t => { 13 | testAllStores(t, (Store, t) => { 14 | t.plan(6) 15 | 16 | const originalStore = new Store(10) 17 | const store = new CacheChunkStore(originalStore, { max: 10 }) 18 | 19 | store.put(0, Buffer.from('0123456789'), err => { 20 | t.error(err) 21 | store.get(0, (err, data) => { 22 | t.error(err) 23 | t.deepEqual(data, Buffer.from('0123456789')) 24 | 25 | originalStore.get = () => { 26 | t.fail('get should be cached - not called on underlying store') 27 | } 28 | 29 | store.get(0, (err, data) => { 30 | t.error(err) 31 | t.deepEqual(data, Buffer.from('0123456789')) 32 | 33 | store.destroy(err => { 34 | t.error(err) 35 | }) 36 | }) 37 | }) 38 | }) 39 | }) 40 | }) 41 | 42 | test('concurrent gets should only call underlying get once', t => { 43 | testAllStores(t, (Store, t) => { 44 | t.plan(7) 45 | 46 | const originalStore = new Store(10) 47 | const store = new CacheChunkStore(originalStore, { max: 10 }) 48 | 49 | store.put(0, Buffer.from('0123456789'), err => { 50 | t.error(err) 51 | 52 | let numGetCalls = 0 53 | const originalGet = originalStore.get 54 | originalStore.get = (index, opts, cb) => { 55 | // Need to ensure get isn't recursive to count calls properly 56 | if (typeof opts === 'function') return originalStore.get(index, null, opts) 57 | 58 | numGetCalls += 1 59 | t.equal(numGetCalls, 1, 'get should be called exactly once') 60 | 61 | originalGet.call(originalStore, index, opts, cb) 62 | } 63 | 64 | // First get 65 | store.get(0, { offset: 1, length: 2 }, (err, data) => { 66 | t.error(err) 67 | t.deepEqual(data, Buffer.from('12')) 68 | }) 69 | 70 | // Second get 71 | store.get(0, { offset: 4, length: 3 }, (err, data) => { 72 | t.error(err) 73 | t.deepEqual(data, Buffer.from('456')) 74 | 75 | store.destroy(err => { 76 | t.error(err) 77 | }) 78 | }) 79 | }) 80 | }) 81 | }) 82 | 83 | function testAllStores (t, testFn) { 84 | const allStores = [ 85 | { name: 'fs-chunk-store', Store: FSChunkStore }, 86 | { name: 'memory-chunk-store', Store: MemoryChunkStore }, 87 | { 88 | name: 'immediate-chunk-store', 89 | Store: function (chunkLength) { 90 | return new ImmediateChunkStore(new FSChunkStore(chunkLength)) 91 | } 92 | } 93 | ] 94 | 95 | // Run the same test on all three stores 96 | for (const { name, Store } of allStores) { 97 | t.test(name, t => { 98 | testFn(Store, t) 99 | }) 100 | } 101 | } 102 | --------------------------------------------------------------------------------