├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── dexie-batch.js ├── package-lock.json ├── package.json ├── rollup.config.js └── test ├── dexie-batch.spec.js └── helpers └── dexie-batch.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .source.* 3 | .nyc_output/ 4 | dist/ 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | 3 | # use latest LTS Node.js release 4 | node_js: lts/* 5 | 6 | # run `npm ci` instead of `npm install` 7 | install: npm ci 8 | 9 | # run `build` script instead of `test` 10 | script: npm run build 11 | 12 | # keep the npm cache around to speed up installs 13 | cache: 14 | directories: $HOME/.npm 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Raphael von der Grün 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dexie-batch [![Build Status](https://travis-ci.org/raphinesse/dexie-batch.svg?branch=master)](https://travis-ci.org/raphinesse/dexie-batch) 2 | 3 | Fetch IndexedDB entries in batches to improve performance while avoiding errors like *Maximum IPC message size exceeded*. 4 | 5 | ## Installation 6 | 7 | If you are using some kind of module bundler: 8 | ```shell 9 | npm i dexie-batch 10 | ``` 11 | 12 | Alternatively, you can use one of the [pre-built scripts](https://unpkg.com/dexie-batch/dist/) and include it *after* the script for `Dexie`: 13 | ```html 14 | 15 | ``` 16 | This way, `DexieBatch` will be available as a global variable. 17 | 18 | ## Usage 19 | 20 | ```js 21 | import DexieBatch from 'dexie-batch' 22 | import table from './my-awesome-dexie-table' 23 | 24 | const collection = table.toCollection() 25 | 26 | // Will fetch 99 items in batches of size 25 when used 27 | const batchDriver = new DexieBatch({ batchSize: 25, limit: 99 }) 28 | 29 | // You can check if an instance will fetch batches concurrently 30 | if (batchDriver.isParallel()) { // true in this case 31 | console.log('Fetching batches concurrently!') 32 | } 33 | 34 | batchDriver.each(collection, (entry, idx) => { 35 | // Process each item individually 36 | }).then(n => console.log(`Fetched ${n} batches`)) 37 | 38 | batchDriver.eachBatch(collection, (batch, batchIdx) => { 39 | // Process each batch (array of entries) individually 40 | }).then(n => console.log(`Fetched ${n} batches`)) 41 | ``` 42 | 43 | The returned `Dexie.Promise` resolves when all batch operations have finished. If the user callback returns a `Promise` it is waited upon. 44 | 45 | The `batchSize` option is mandatory since a sensible value depends strongly on the individual record size. 46 | 47 | Batches are requested in parallel iff `limit` option is present. 48 | Otherwise we would not know when to stop sending requests. 49 | When no limit is given, batches are requested serially until one request gives an empty result. 50 | -------------------------------------------------------------------------------- /dexie-batch.js: -------------------------------------------------------------------------------- 1 | const { Promise } = require('dexie') 2 | 3 | module.exports = class DexieBatch { 4 | constructor(opts) { 5 | assertValidOptions(opts) 6 | this.opts = opts 7 | } 8 | 9 | isParallel() { 10 | return Boolean(this.opts.limit) 11 | } 12 | 13 | each(collection, callback) { 14 | assertValidMethodArgs(...arguments) 15 | 16 | return this.eachBatch(collection, (batch, batchIdx) => { 17 | const baseIdx = batchIdx * this.opts.batchSize 18 | return Promise.all(batch.map((item, i) => callback(item, baseIdx + i))) 19 | }) 20 | } 21 | 22 | eachBatch(collection, callback) { 23 | assertValidMethodArgs(...arguments) 24 | 25 | const delegate = this.isParallel() ? 'eachBatchParallel' : 'eachBatchSerial' 26 | return this[delegate](collection, callback) 27 | } 28 | 29 | eachBatchParallel(collection, callback) { 30 | assertValidMethodArgs(...arguments) 31 | const { batchSize, limit } = this.opts 32 | if (!limit) { 33 | throw new Error('Option "limit" must be set for parallel operation') 34 | } 35 | 36 | const nextBatch = batchIterator(collection, batchSize) 37 | const numBatches = Math.ceil(limit / batchSize) 38 | const batchPromises = Array.from({ length: numBatches }, (_, idx) => 39 | nextBatch().then(batch => callback(batch, idx)) 40 | ) 41 | 42 | return Promise.all(batchPromises).then(batches => batches.length) 43 | } 44 | 45 | eachBatchSerial(collection, callback) { 46 | assertValidMethodArgs(...arguments) 47 | 48 | const userPromises = [] 49 | const nextBatch = batchIterator(collection, this.opts.batchSize) 50 | 51 | const nextUnlessEmpty = batch => { 52 | if (batch.length === 0) return 53 | userPromises.push(callback(batch, userPromises.length)) 54 | return nextBatch().then(nextUnlessEmpty) 55 | } 56 | 57 | return nextBatch() 58 | .then(nextUnlessEmpty) 59 | .then(() => Promise.all(userPromises)) 60 | .then(() => userPromises.length) 61 | } 62 | } 63 | 64 | // Does not conform to JS iterator requirements 65 | function batchIterator(collection, batchSize) { 66 | const it = collection.clone() 67 | return () => { 68 | const batchPromise = it.clone().limit(batchSize).toArray() 69 | it.offset(batchSize) 70 | return batchPromise 71 | } 72 | } 73 | 74 | function assertValidOptions(opts) { 75 | const batchSize = opts && opts.batchSize 76 | if (!(batchSize && Number.isInteger(batchSize) && batchSize > 0)) { 77 | throw new Error('Mandatory option "batchSize" must be a positive integer') 78 | } 79 | 80 | if ('limit' in opts && !(Number.isInteger(opts.limit) && opts.limit >= 0)) { 81 | throw new Error('Option "limit" must be a non-negative integer') 82 | } 83 | } 84 | 85 | function assertValidMethodArgs(collection, callback) { 86 | if (arguments.length < 2) { 87 | throw new Error('Arguments "collection" and "callback" are mandatory') 88 | } 89 | 90 | if (!isCollectionInstance(collection)) { 91 | throw new Error('"collection" must be of type Collection') 92 | } 93 | 94 | if (!(typeof callback === 'function')) { 95 | throw new TypeError('"callback" must be a function') 96 | } 97 | } 98 | 99 | // We would need the Dexie instance that created the collection to get the 100 | // Collection constructor and do some proper type checking. 101 | // So for now we resort to duck typing 102 | function isCollectionInstance(obj) { 103 | if (!obj) return false 104 | return ['clone', 'offset', 'limit', 'toArray'].every( 105 | name => typeof obj[name] === 'function' 106 | ) 107 | } 108 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dexie-batch", 3 | "version": "0.4.3", 4 | "description": "Fetch DB entries in batches to improve performance while respecting IPC size constraints", 5 | "license": "MIT", 6 | "author": "Raphael von der Grün", 7 | "main": "dist/dexie-batch.js", 8 | "module": "dist/dexie-batch.mjs", 9 | "repository": "raphinesse/dexie-batch", 10 | "scripts": { 11 | "prebuild": "npm test", 12 | "build": "rollup -c", 13 | "postbuild": "TEST_SUBJECT=dist/dexie-batch.js ava", 14 | "format": "prettier --ignore-path .gitignore --write '**/*.js'", 15 | "prepack": "npm run build", 16 | "test": "xo && nyc ava" 17 | }, 18 | "xo": { 19 | "space": 2, 20 | "prettier": true, 21 | "rules": { 22 | "import/no-anonymous-default-export": "off", 23 | "unicorn/no-array-for-each": "off", 24 | "unicorn/prevent-abbreviations": "off" 25 | } 26 | }, 27 | "prettier": { 28 | "arrowParens": "avoid", 29 | "bracketSpacing": true, 30 | "semi": false, 31 | "singleQuote": true, 32 | "trailingComma": "es5" 33 | }, 34 | "files": [ 35 | "dist" 36 | ], 37 | "devDependencies": { 38 | "@babel/core": "^7.4.0", 39 | "@babel/preset-env": "^7.4.2", 40 | "@rollup/plugin-babel": "^5.3.0", 41 | "@rollup/plugin-commonjs": "^17.1.0", 42 | "@rollup/plugin-node-resolve": "^11.2.0", 43 | "ava": "^3.15.0", 44 | "dexie": "^3.0.3", 45 | "fake-indexeddb": "^3.1.2", 46 | "nyc": "^15.1.0", 47 | "rollup": "^2.40.0", 48 | "rollup-plugin-terser": "^7.0.2", 49 | "xo": "^0.38.2" 50 | }, 51 | "peerDependencies": { 52 | "dexie": ">1.3.6" 53 | }, 54 | "keywords": [ 55 | "batch", 56 | "bulk", 57 | "dexie", 58 | "fetch", 59 | "get" 60 | ] 61 | } 62 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import resolve from '@rollup/plugin-node-resolve' 2 | import commonjs from '@rollup/plugin-commonjs' 3 | import babel from '@rollup/plugin-babel' 4 | import { terser } from 'rollup-plugin-terser' 5 | 6 | const pkg = require('./package') 7 | 8 | const banner = (() => { 9 | const id = `${pkg.name} v${pkg.version}` 10 | const homepage = 'github.com/' + pkg.repository 11 | const license = `${pkg.license} License` 12 | return `/*! ${id} | ${homepage} | ${license} */` 13 | })() 14 | 15 | function outputConfig(config) { 16 | const defaultConfig = { 17 | sourcemap: true, 18 | banner, 19 | } 20 | return Object.assign(defaultConfig, config) 21 | } 22 | 23 | function umdConfig(config) { 24 | const defaultConfig = { 25 | name: 'DexieBatch', 26 | format: 'umd', 27 | globals: { dexie: 'Dexie' }, 28 | } 29 | return outputConfig(Object.assign(defaultConfig, config)) 30 | } 31 | 32 | const babelConfig = { 33 | babelHelpers: 'bundled', 34 | exclude: 'node_modules/**', 35 | presets: [['@babel/env', { targets: { browsers: 'defaults' } }]], 36 | } 37 | 38 | export default { 39 | input: 'dexie-batch.js', 40 | output: [ 41 | // Browser-friendly UMD build 42 | umdConfig({ file: pkg.main }), 43 | umdConfig({ 44 | file: pkg.main.replace(/\.js$/, '.min.js'), 45 | plugins: [terser()], 46 | }), 47 | // ECMAScript module build 48 | outputConfig({ file: pkg.module, format: 'es' }), 49 | ], 50 | external: ['dexie'], 51 | plugins: [resolve(), commonjs(), babel(babelConfig)], 52 | } 53 | -------------------------------------------------------------------------------- /test/dexie-batch.spec.js: -------------------------------------------------------------------------------- 1 | // Fake IndexedDB in global scope 2 | // eslint-disable-next-line import/no-unassigned-import 3 | require('fake-indexeddb/auto') 4 | 5 | const test = require('ava') 6 | const Dexie = require('dexie') 7 | const DexieBatch = require('./helpers/dexie-batch') 8 | 9 | const noop = () => {} 10 | // eslint-disable-next-line no-promise-executor-return 11 | const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) 12 | 13 | const numEntries = 42 14 | const batchSize = 10 15 | const expectedBatchCount = 5 16 | const testEntries = Array.from({ length: numEntries }, (_, i) => i) 17 | 18 | const serialBatchDriver = new DexieBatch({ batchSize }) 19 | const parallelBatchDriver = new DexieBatch({ batchSize, limit: numEntries }) 20 | 21 | testBasicOperation(serialBatchDriver) 22 | testBasicOperation(parallelBatchDriver) 23 | 24 | function testBasicOperation(batchDriver) { 25 | const mode = batchDriver.isParallel() ? 'parallel' : 'serial' 26 | testWithCollection(`${mode} driver: basic operation`, (t, collection) => { 27 | const entries = [] 28 | const indices = [] 29 | let resolvedCount = 0 30 | 31 | return batchDriver 32 | .each(collection, (entry, i) => { 33 | entries.push(entry) 34 | indices.push(i) 35 | return delay(10).then(() => resolvedCount++) 36 | }) 37 | .then(batchCount => { 38 | t.deepEqual(indices, entries, 'indices calculated correctly') 39 | 40 | // Parallel batch driver may yield batches out of order 41 | if (batchDriver.isParallel()) { 42 | entries.sort((a, b) => a - b) 43 | } 44 | 45 | t.deepEqual(entries, testEntries, 'entries read correctly') 46 | t.is(resolvedCount, numEntries, 'waited for user promises') 47 | t.is(batchCount, expectedBatchCount, 'correct batch count') 48 | }) 49 | }) 50 | } 51 | 52 | testBatchProperties(serialBatchDriver) 53 | testBatchProperties(parallelBatchDriver) 54 | 55 | function testBatchProperties(batchDriver) { 56 | const mode = batchDriver.isParallel() ? 'parallel' : 'serial' 57 | testWithCollection(`${mode} driver: batch properties`, (t, collection) => { 58 | let batchSizes = new Set() 59 | 60 | return batchDriver 61 | .eachBatch(collection, batch => { 62 | batchSizes.add(batch.length) 63 | }) 64 | .then(() => { 65 | batchSizes = [...batchSizes.values()] 66 | // Parallel batch driver may yield batches out of order 67 | if (batchDriver.isParallel()) { 68 | batchSizes.sort((a, b) => b - a) 69 | } 70 | 71 | t.is(batchSizes[0], batchSize, 'correct batch size') 72 | t.is(batchSizes.length, 2, 'only last batch size different') 73 | }) 74 | }) 75 | } 76 | 77 | test('constructor argument checking', t => { 78 | t.throws(() => new DexieBatch(), { message: /batchSize/ }) 79 | t.throws(() => new DexieBatch(null), { message: /batchSize/ }) 80 | t.throws(() => new DexieBatch(1), { message: /batchSize/ }) 81 | t.throws(() => new DexieBatch('foo'), { message: /batchSize/ }) 82 | t.throws(() => new DexieBatch({}), { message: /batchSize/ }) 83 | t.throws(() => new DexieBatch({ batchSize: 0 }), { message: /batchSize/ }) 84 | 85 | t.throws(() => new DexieBatch({ batchSize, limit: -1 }), { message: /limit/ }) 86 | }) 87 | 88 | testWithCollection('method argument checking', (t, collection) => { 89 | const driver = parallelBatchDriver 90 | ;['each', 'eachBatch', 'eachBatchParallel', 'eachBatchSerial'] 91 | .map(method => driver[method].bind(driver)) 92 | .forEach(method => { 93 | t.throws(() => method(), { message: /mandatory/ }) 94 | 95 | t.throws(() => method(null, noop), { message: /Collection/ }) 96 | t.throws(() => method(1, noop), { message: /Collection/ }) 97 | t.throws(() => method([1, 2], noop), { message: /Collection/ }) 98 | 99 | t.throws(() => method(collection), { message: /mandatory/ }) 100 | t.throws(() => method(collection, null), { message: /function/ }) 101 | t.throws(() => method(collection, 1), { message: /function/ }) 102 | }) 103 | }) 104 | 105 | testWithCollection('no limit, no parallel operation', (t, collection) => { 106 | t.throws(() => serialBatchDriver.eachBatchParallel(collection, noop), { 107 | message: /limit/, 108 | }) 109 | }) 110 | 111 | function testWithCollection(name, f) { 112 | test(name, async t => { 113 | await Dexie.delete(name) 114 | const db = new Dexie(name) 115 | db.version(1).stores({ test: '++' }) 116 | await db.test.bulkAdd(testEntries) 117 | await f(t, db.test.toCollection()) 118 | }) 119 | } 120 | -------------------------------------------------------------------------------- /test/helpers/dexie-batch.js: -------------------------------------------------------------------------------- 1 | const testSubject = process.env.TEST_SUBJECT || 'dexie-batch' 2 | 3 | module.exports = require(`../../${testSubject}`) 4 | --------------------------------------------------------------------------------