├── .editorconfig ├── .gitignore ├── .istanbul.yml ├── .npmignore ├── .travis.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── docs ├── common-patterns.md ├── guide.md └── index.md ├── lib └── index.js ├── package-lock.json ├── package.json ├── tests ├── counts-js-to-batchloader.test.js ├── counts-no-batch-to-batch.test.js ├── get-results-by-key.text.js ├── get-unique-keys.test.js ├── helpers │ └── make-services.js ├── loader-factory.test.js ├── loader-large-populate.test.js ├── loader-no-loader.test.js ├── loader-small-populate.test.js ├── make-services-await.test.js └── make-services.test.js └── types ├── index.d.ts ├── tests.ts └── tsconfig.json /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | 3 | # Logs 4 | logs 5 | *.log 6 | 7 | # Runtime data 8 | pids 9 | *.pid 10 | *.seed 11 | 12 | # Directory for instrumented libs generated by jscoverage/JSCover 13 | lib-cov 14 | 15 | # Coverage directory used by tools like istanbul 16 | coverage 17 | 18 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 19 | .grunt 20 | 21 | # Compiled binary addons (http://nodejs.org/api/addons.html) 22 | build/Release 23 | 24 | # Dependency directory 25 | # Commenting this out is preferred by some people, see 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 27 | node_modules 28 | 29 | # Users Environment Variables 30 | .lock-wscript 31 | 32 | dist/ 33 | .idea/ 34 | -------------------------------------------------------------------------------- /.istanbul.yml: -------------------------------------------------------------------------------- 1 | verbose: false 2 | instrumentation: 3 | root: ./lib/ 4 | include-all-sources: true 5 | reporting: 6 | print: summary 7 | reports: 8 | - html 9 | - text 10 | - lcov 11 | watermarks: 12 | statements: [50, 80] 13 | lines: [50, 80] 14 | functions: [50, 80] 15 | branches: [50, 80] 16 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .editorconfig 2 | .jshintrc 3 | .travis.yml 4 | .istanbul.yml 5 | .babelrc 6 | .idea/ 7 | .vscode/ 8 | test/ 9 | coverage/ 10 | .github/ 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - 'node' 4 | - '6' 5 | addons: 6 | code_climate: 7 | repo_token: 'your repo token' 8 | notifications: 9 | email: false 10 | before_script: 11 | - npm install -g codeclimate-test-reporter 12 | after_script: 13 | - codeclimate-test-reporter < coverage/lcov.info 14 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## [v0.3.5](https://github.com/feathers-plus/batch-loader/tree/v0.3.5) (2019-01-02) 4 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.3.4...v0.3.5) 5 | 6 | **Merged pull requests:** 7 | 8 | - Fixed bug in custom version of makeCallingParams. [\#8](https://github.com/feathers-plus/batch-loader/pull/8) ([eddyystop](https://github.com/eddyystop)) 9 | 10 | ## [v0.3.4](https://github.com/feathers-plus/batch-loader/tree/v0.3.4) (2018-11-24) 11 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.3.3...v0.3.4) 12 | 13 | **Closed issues:** 14 | 15 | - Creating a batchloader for belongsToMany relationships [\#6](https://github.com/feathers-plus/batch-loader/issues/6) 16 | 17 | **Merged pull requests:** 18 | 19 | - Add typings, tests and necessary infrastructure [\#5](https://github.com/feathers-plus/batch-loader/pull/5) ([j2L4e](https://github.com/j2L4e)) 20 | 21 | ## [v0.3.3](https://github.com/feathers-plus/batch-loader/tree/v0.3.3) (2018-04-18) 22 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.3.2...v0.3.3) 23 | 24 | **Implemented enhancements:** 25 | 26 | - Revert old changes and apply new strategy to add pagination support. [\#4](https://github.com/feathers-plus/batch-loader/pull/4) ([otang](https://github.com/otang)) 27 | 28 | ## [v0.3.2](https://github.com/feathers-plus/batch-loader/tree/v0.3.2) (2018-04-13) 29 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.3.1...v0.3.2) 30 | 31 | ## [v0.3.1](https://github.com/feathers-plus/batch-loader/tree/v0.3.1) (2018-04-13) 32 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.3.0...v0.3.1) 33 | 34 | **Fixed bugs:** 35 | 36 | - Fix README.md link to documentation page [\#2](https://github.com/feathers-plus/batch-loader/pull/2) ([leedongwei](https://github.com/leedongwei)) 37 | 38 | **Merged pull requests:** 39 | 40 | - Add support for handling page object responses for services using pagination. [\#3](https://github.com/feathers-plus/batch-loader/pull/3) ([otang](https://github.com/otang)) 41 | 42 | ## [v0.3.0](https://github.com/feathers-plus/batch-loader/tree/v0.3.0) (2017-11-14) 43 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.2.1...v0.3.0) 44 | 45 | ## [v0.2.1](https://github.com/feathers-plus/batch-loader/tree/v0.2.1) (2017-11-14) 46 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.2.0...v0.2.1) 47 | 48 | ## [v0.2.0](https://github.com/feathers-plus/batch-loader/tree/v0.2.0) (2017-11-13) 49 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.1.2...v0.2.0) 50 | 51 | **Closed issues:** 52 | 53 | - batch-loader on the browser [\#1](https://github.com/feathers-plus/batch-loader/issues/1) 54 | 55 | ## [v0.1.2](https://github.com/feathers-plus/batch-loader/tree/v0.1.2) (2017-11-06) 56 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.1.1...v0.1.2) 57 | 58 | ## [v0.1.1](https://github.com/feathers-plus/batch-loader/tree/v0.1.1) (2017-11-06) 59 | [Full Changelog](https://github.com/feathers-plus/batch-loader/compare/v0.1.0...v0.1.1) 60 | 61 | ## [v0.1.0](https://github.com/feathers-plus/batch-loader/tree/v0.1.0) (2017-11-06) 62 | 63 | 64 | \* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Feathers 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # feathers-ecosystem/batch-loader 2 | 3 | 5 | 6 | > Reduce requests to backend services by batching calls and caching records. 7 | 8 | ## Installation 9 | 10 | ``` 11 | npm install @feathers-plus/batch-loader --save 12 | ``` 13 | 14 | ## Documentation 15 | 16 | Please refer to the [batch-loader documentation](./docs/index.md) for more details. 17 | 18 | ## Basic Example 19 | 20 | Use the `loaderFactory` static method to create a basic batch-loader. This is simply syntatic sugar for manually creating a batch-loader. This "Basic Example" and "Complete Example" create the same batch-loader. 21 | 22 | ```js 23 | const BatchLoader = require("@feathers-plus/batch-loader"); 24 | 25 | const usersBatchLoader = BatchLoader.loaderFactory( 26 | app.service("users"), 27 | "id", 28 | false 29 | ); 30 | 31 | app 32 | .service("comments") 33 | .find() 34 | .then((comments) => 35 | Promise.all( 36 | comments.map((comment) => { 37 | // Attach user record 38 | return usersBatchLoader 39 | .load(comment.userId) 40 | .then((user) => (comment.userRecord = user)); 41 | }) 42 | ) 43 | ); 44 | ``` 45 | 46 | ## Complete Example 47 | 48 | Use the `BatchLoader` class to create more complex loaders. These loaders can call other services, call DB's directly, or even call third party services. This example manually implements the same loader created with the `loaderFactory` above. 49 | 50 | ```js 51 | const BatchLoader = require("@feathers-plus/batch-loader"); 52 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 53 | 54 | const usersBatchLoader = new BatchLoader((keys) => 55 | app 56 | .service("users") 57 | .find({ query: { id: { $in: getUniqueKeys(keys) } } }) 58 | .then((result) => getResultsByKey(keys, result, (user) => user.id, "!")) 59 | ); 60 | 61 | app 62 | .service("comments") 63 | .find() 64 | .then((comments) => 65 | Promise.all( 66 | comments.map((comment) => { 67 | // Attach user record 68 | return usersBatchLoader 69 | .load(comment.userId) 70 | .then((user) => (comment.userRecord = user)); 71 | }) 72 | ) 73 | ); 74 | ``` 75 | 76 | ## License 77 | 78 | Copyright (c) 2017 John J. Szwaronek 79 | 80 | Licensed under the [MIT license](LICENSE). 81 | -------------------------------------------------------------------------------- /docs/common-patterns.md: -------------------------------------------------------------------------------- 1 | ## Creating a new batch-loader per Request. 2 | 3 | In many applications, a server using batch-loader serves requests to many different users with different access permissions. It may be dangerous to use one cache across many users, and it is encouraged to create a new batch-loader per request: 4 | 5 | ```js 6 | function createLoaders(authToken) { 7 | return { 8 | users: new BatchLoader((ids) => genUsers(authToken, ids)), 9 | cdnUrls: new BatchLoader((rawUrls) => genCdnUrls(authToken, rawUrls)), 10 | stories: new BatchLoader((keys) => genStories(authToken, keys)), 11 | }; 12 | } 13 | 14 | // When handling an incoming request: 15 | var loaders = createLoaders(request.query.authToken); 16 | 17 | // Then, within application logic: 18 | var user = await loaders.users.load(4); 19 | var pic = await loaders.cdnUrls.load(user.rawPicUrl); 20 | ``` 21 | 22 | ## Loading by alternative keys. 23 | 24 | Occasionally, some kind of value can be accessed in multiple ways. For example, perhaps a "User" type can be loaded not only by an "id" but also by a "username" value. If the same user is loaded by both keys, then it may be useful to fill both caches when a user is loaded from either source: 25 | 26 | ```js 27 | let userByIDLoader = new BatchLoader((ids) => 28 | genUsersByID(ids).then((users) => { 29 | for (let user of users) { 30 | usernameLoader.prime(user.username, user); 31 | } 32 | return users; 33 | }) 34 | ); 35 | 36 | let usernameLoader = new BatchLoader((names) => 37 | genUsernames(names).then((users) => { 38 | for (let user of users) { 39 | userByIDLoader.prime(user.id, user); 40 | } 41 | return users; 42 | }) 43 | ); 44 | ``` 45 | 46 | ## Persistent caches 47 | 48 | By default, batch-loader uses the standard Map which simply grows until the batch-loader is released. A custom cache is provided as a convenience if you want to persist caches for longer periods of time. It implements a **least-recently-used** algorithm and allows you to limit the number of records cached. 49 | 50 | ```js 51 | const BatchLoader = require('@feathers-plus/batch-loader'); 52 | const cache = require('@feathers-plus/cache'); 53 | 54 | const usersLoader = new BatchLoader( 55 | keys => { ... }, 56 | { cacheMap: cache({ max: 100 }) 57 | ); 58 | ``` 59 | 60 |

The default cache is appropriate when requests to your application are short-lived.

61 | 62 | ## Using non-Feathers services 63 | 64 | batch-loader provides a simplified and consistent API over various data sources, when its used as part of your application's data fetching layer. Custom Feathers services can use batch-loaders to natively accesses local and remote resources. 65 | 66 | ### Redis 67 | 68 | Redis is a very simple key-value store which provides the batch load method MGET which makes it very well suited for use with batch-loader. 69 | 70 | ```js 71 | const BatchLoader = require("@feathers-plus/batch-loader"); 72 | const redis = require("redis"); 73 | 74 | const client = redis.createClient(); 75 | 76 | const redisLoader = new BatchLoader( 77 | (keys) => 78 | new Promise((resolve, reject) => { 79 | client.mget(keys, (error, results) => { 80 | if (error) return reject(error); 81 | 82 | resolve( 83 | results.map((result, index) => 84 | result !== null ? result : new Error(`No key: ${keys[index]}`) 85 | ) 86 | ); 87 | }); 88 | }) 89 | ); 90 | ``` 91 | 92 | ### SQLite 93 | 94 | While not a key-value store, SQL offers a natural batch mechanism with SELECT \* WHERE IN statements. While batch-loader is best suited for key-value stores, it is still suited for SQL when queries remain simple. This example requests the entire row at a given id, however your usage may differ. 95 | 96 | This example uses the sqlite3 client which offers a parallelize method to further batch queries together. Another non-caching batch-loader utilizes this method to provide a similar API. batch-loaders can access other batch-loaders. 97 | 98 | ```js 99 | const BatchLoader = require("@feathers-plus/batch-loader"); 100 | const sqlite3 = require("sqlite3"); 101 | 102 | const db = new sqlite3.Database("./to/your/db.sql"); 103 | 104 | // Dispatch a WHERE-IN query, ensuring response has rows in correct order. 105 | const userLoader = new BatchLoader((ids) => { 106 | const params = ids.map((id) => "?").join(); 107 | const query = `SELECT * FROM users WHERE id IN (${params})`; 108 | return queryLoader 109 | .load([query, ids]) 110 | .then((rows) => 111 | ids.map( 112 | (id) => 113 | rows.find((row) => row.id === id) || new Error(`Row not found: ${id}`) 114 | ) 115 | ); 116 | }); 117 | 118 | // Parallelize all queries, but do not cache. 119 | const queryLoader = new BatchLoader( 120 | (queries) => 121 | new Promise((resolve) => { 122 | const waitingOn = queries.length; 123 | const results = []; 124 | db.parallelize(() => { 125 | queries.forEach((query, index) => { 126 | db.all.apply( 127 | db, 128 | query.concat((error, result) => { 129 | results[index] = error || result; 130 | if (--waitingOn === 0) { 131 | resolve(results); 132 | } 133 | }) 134 | ); 135 | }); 136 | }); 137 | }), 138 | { cache: false } 139 | ); 140 | 141 | // Usage 142 | 143 | const promise1 = userLoader.load("1234"); 144 | const promise2 = userLoader.load("5678"); 145 | 146 | Promise.all([promise1, promise2]).then(([user1, user2]) => { 147 | console.log(user1, user2); 148 | }); 149 | ``` 150 | 151 | ### Knex.js 152 | 153 | This example demonstrates how to use batch-loader with SQL databases via Knex.js, which is a SQL query builder and a client for popular databases such as PostgreSQL, MySQL, MariaDB etc. 154 | 155 | ```js 156 | const BatchLoader = require("@feathers-plus/batch-loader"); 157 | const db = require("./db"); // an instance of Knex client 158 | 159 | // The list of batch loaders 160 | 161 | const batchLoader = { 162 | user: new BatchLoader((ids) => 163 | db 164 | .table("users") 165 | .whereIn("id", ids) 166 | .select() 167 | .then((rows) => ids.map((id) => rows.find((x) => x.id === id))) 168 | ), 169 | 170 | story: new BatchLoader((ids) => 171 | db 172 | .table("stories") 173 | .whereIn("id", ids) 174 | .select() 175 | .then((rows) => ids.map((id) => rows.find((x) => x.id === id))) 176 | ), 177 | 178 | storiesByUserId: new BatchLoader((ids) => 179 | db 180 | .table("stories") 181 | .whereIn("author_id", ids) 182 | .select() 183 | .then((rows) => ids.map((id) => rows.filter((x) => x.author_id === id))) 184 | ), 185 | }; 186 | 187 | // Usage 188 | 189 | Promise.all([ 190 | batchLoader.user.load("1234"), 191 | batchLoader.storiesByUserId.load("1234"), 192 | ]).then(([user, stories]) => { 193 | /* ... */ 194 | }); 195 | ``` 196 | 197 | ### RethinkDB 198 | 199 | Full implementation: 200 | 201 | ```js 202 | const BatchLoader = require("@feathers-plus/batch-loader"); 203 | const r = require("rethinkdb"); 204 | const db = await r.connect(); 205 | 206 | const batchLoadFunc = (keys) => 207 | db 208 | .table("example_table") 209 | .getAll(...keys) 210 | .then((res) => res.toArray()) 211 | .then(normalizeRethinkDbResults(keys, "id")); 212 | 213 | const exampleLoader = new BatchLoader(batchLoadFunc); 214 | 215 | await exampleLoader.loadMany([1, 2, 3]); // [{"id": 1, "name": "Document 1"}, {"id": 2, "name": "Document 2"}, Error]; 216 | 217 | await exampleLoader.load(1); // {"id": 1, "name": "Document 1"} 218 | 219 | function indexResults(results, indexField, cacheKeyFn = (key) => key) { 220 | const indexedResults = new Map(); 221 | results.forEach((res) => { 222 | indexedResults.set(cacheKeyFn(res[indexField]), res); 223 | }); 224 | return indexedResults; 225 | } 226 | 227 | function normalizeRethinkDbResults( 228 | keys, 229 | indexField, 230 | cacheKeyFn = (key) => key 231 | ) { 232 | return (results) => { 233 | const indexedResults = indexResults(results, indexField, cacheKeyFn); 234 | return keys.map( 235 | (val) => 236 | indexedResults.get(cacheKeyFn(val)) || 237 | new Error(`Key not found : ${val}`) 238 | ); 239 | }; 240 | } 241 | ``` 242 | -------------------------------------------------------------------------------- /docs/guide.md: -------------------------------------------------------------------------------- 1 | Loading data from database is one of the major tasks for most web applications. The goal of batch-loader is to improve the performance of database queries with two techniques: batching and caching. 2 | 3 | ## Batching 4 | 5 | Batching is batch-loader's primary feature. The reason for batching is to merge multiple similar database queries into one single query when possible. For example: 6 | 7 | ```js 8 | Promise.all([ 9 | posts.find({ query: { id: 1 } }), 10 | posts.find({ query: { id: 2 } }), 11 | posts.find({ query: { id: { $in: [3, 4] } } }), 12 | posts.find({ query: { id: 5 } }), 13 | ]); 14 | ``` 15 | 16 | is slower than 17 | 18 | ```js 19 | posts.find({ query: { id: { $in: [1, 2, 3, 4, 5] } } }); 20 | ``` 21 | 22 | The latter sends only one query to database and retrieves the same 5 records as the former does, and therefore is much more efficient. 23 | 24 | Batch-loader is a tool to help you batch database calls in such a way. First, create a batch-loader by providing a batch loading function which accepts an array of keys and an optional context. It returns a Promise which resolves to an array of values. 25 | 26 | ```js 27 | const BatchLoader = require('@feathers-plus/batch-loader'); 28 | const usersLoader = new BatchLoader((keys, context) => { 29 | return app.service('users').find({ query: { id: { $in: keys } } }) 30 | .then(records => { 31 | recordsByKey = /* recordsByKey[i] is the value for key[i] */; 32 | return recordsByKey; 33 | }); 34 | }, 35 | { context: {} } 36 | ); 37 | ``` 38 | 39 | You can then call the batch-loader with individual keys. It will coalesce all requests made within the current event loop into a single call to the batch-loader function, and return the results to each call. 40 | 41 | ```js 42 | usersLoader.load(1).then((user) => console.log("key 1", user)); 43 | usersLoader.load(2).then((user) => console.log("key 2", user)); 44 | usersLoader 45 | .loadMany([1, 2, 3, 4]) 46 | .then((users) => console.log(users.length, users)); 47 | ``` 48 | 49 | The above will result in one database service call, i.e. `users.find({ query: { id: { $in: [1, 2, 3, 4] } } })`, instead of 6. 50 | 51 |

*"[W]ill coalesce all requests made within the current event loop into a single call"* sounds ominous. Just don't worry about it. Make `usersLoader.load` and `usersLoader.loadMany` calls the same way you would `users.get` and `users.find`. Everything will work as expected while, behind the scenes, batch-loader is making the fewest database calls logically possible.

52 | 53 | ### Batch Function 54 | 55 | The batch loading function accepts an array of keys and an optional context. It returns a Promise which resolves to an array of values. Each index in the returned array of values must correspond to the same index in the array of keys. 56 | 57 | For example, if the `usersLoader` from above is called with `[1, 2, 3, 4, 99]`, we would execute `users.find({ query: { id: { $in: [1, 2, 3, 4, 99] } } })`. The Feathers service could return the results: 58 | 59 | ```js 60 | [ { id: 4, name: 'Aubree' } 61 | { id: 2, name: 'Marshall' }, 62 | { id: 1, name: 'John' }, 63 | { id: 3, name: 'Barbara' } ] 64 | ``` 65 | 66 | Please not that the order of the results will usually differ from the order of the keys and here, in addition, there is no `users` with an `id` of `99`. 67 | 68 | The batch function has to to reorganize the above results and return: 69 | 70 | ```js 71 | [ 72 | { id: 1, name: "John" }, 73 | { id: 2, name: "Marshall" }, 74 | { id: 3, name: "Barbara" }, 75 | { id: 4, name: "Aubree" }, 76 | null, 77 | ]; 78 | ``` 79 | 80 | The `null` indicating there is no record for `user.id === 99`. 81 | 82 | ### Convenience Methods 83 | 84 | Batch-loader provides two convenience functions that will perform this reorganization for you. 85 | 86 | ```js 87 | const BatchLoader = require('@feathers-plus/batch-loader'); 88 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 89 | 90 | const usersLoader = new BatchLoader(keys => 91 | app.service('users').find({ query: { id: { $in: getUniqueKeys(keys) } } }) 92 | .then(records => getResultsByKey(keys, records, user => user.id, '')); 93 | ); 94 | ``` 95 | 96 | **getUniqueKeys** eliminates any duplicate elements in the keys. 97 | 98 | > The array of keys may contain duplicates when the batch-loader's memoization cache is disabled. 99 | 100 | **getResultsByKey** reorganizes the records from the service call into the result expected from the batch function. The `''` parameter indicates each key expects a single record or `null`. Other options are `'!'` when each key requires a single record, and `'[]'` when each key requires an array of 0, 1 or more records. 101 | 102 | ## Caching 103 | 104 | Each batch-loader instance contains a unique memoized cache. Once `load` or `loadMany` is called, the resulting value is cached. This eliminates redundant database requests, relieving pressure on your database. It also creates fewer objects which may relieve memory pressure on your application. 105 | 106 | ```js 107 | Promise.all([userLoader.load(1), userLoader.load(1)]).then((users) => 108 | assert(users[0] === users[1]) 109 | ); 110 | ``` 111 | 112 |

The same object is returned for each of multiple hits on the cache. You should not mutate that object directly as the mutation would be reflected in every reference to the object. Rather you should deep-copy before mutating the copy.

113 | 114 | ### Caching Per Request 115 | 116 | It may be dangerous to use one cache across many users, and it is encouraged to create a new batch-loader per request. Typically batch-loader instances are created when a request begins and are released once the request ends. 117 | 118 | Since the cache exists for a limited time only, the cache contents should not normally grow large enough to cause memory pressure on the application. 119 | 120 | ### Persistent Caches 121 | 122 | A batch-loader can be shared between requests and between users if care is taken. Use caution when used in long-lived applications or those which serve many users with different access permissions. 123 | 124 | The main advantage is having the cache already primed at the start of each request, which could result in fewer initial database requests. 125 | 126 | #### Memory pressure 127 | 128 | There are two concerns though. First the cache could keep filling up with records causing memory pressure. This can be handled with a custom cache. 129 | 130 | **@feathers-plus/cache** is a least-recently-used (LRU) cache which you can inject when initializing the batch-loader. You can specify the maximum number of records to be kept in the cache, and it will retain the least recently used records. 131 | 132 | ```js 133 | const BatchLoader = require('@feathers-plus/batch-loader'); 134 | const cache = require('@feathers-plus/cache'); 135 | 136 | const usersLoader = new BatchLoader( 137 | keys => { ... }, 138 | { cacheMap: cache({ max: 100 }) 139 | ); 140 | ``` 141 | 142 | #### Mutation 143 | 144 | The other concern is a record mutating. You can create a hook which clears a record from its BatchLoaders' caches when it mutates. 145 | 146 | ```js 147 | usersLoader.clear(1); 148 | ``` 149 | 150 | > `@feathers-plus/cache/lib/hooks` contains hooks which clear the keys of mutated records. 151 | 152 | ## Explore Performance Gains 153 | 154 | ### Our Sample Data 155 | 156 | We will be using Feathers database services containing the following data: 157 | 158 | ```js 159 | // app.service('posts') 160 | const postsStore = [ 161 | { id: 1, body: "John post", userId: 101, starIds: [102, 103, 104] }, 162 | { id: 2, body: "Marshall post", userId: 102, starIds: [101, 103, 104] }, 163 | { id: 3, body: "Barbara post", userId: 103 }, 164 | { id: 4, body: "Aubree post", userId: 104 }, 165 | ]; 166 | 167 | // app.service('comments') 168 | const commentsStore = [ 169 | { id: 11, text: "John post Marshall comment 11", postId: 1, userId: 102 }, 170 | { id: 12, text: "John post Marshall comment 12", postId: 1, userId: 102 }, 171 | { id: 13, text: "John post Marshall comment 13", postId: 1, userId: 102 }, 172 | { id: 14, text: "Marshall post John comment 14", postId: 2, userId: 101 }, 173 | { id: 15, text: "Marshall post John comment 15", postId: 2, userId: 101 }, 174 | { id: 16, text: "Barbara post John comment 16", postId: 3, userId: 101 }, 175 | { id: 17, text: "Aubree post Marshall comment 17", postId: 4, userId: 102 }, 176 | ]; 177 | 178 | // app.service('users') 179 | const usersStore = [ 180 | { id: 101, name: "John" }, 181 | { id: 102, name: "Marshall" }, 182 | { id: 103, name: "Barbara" }, 183 | { id: 104, name: "Aubree" }, 184 | ]; 185 | ``` 186 | 187 | We want to see how using batch-loader affects the number of database calls, and we will do that by populating the `posts` records with related information. 188 | 189 | ### Using Plain JavaScript 190 | 191 | First, let's add the related `comments` records to each `posts` record using regular JavaScript, and let's do this using both Promises and async/await. 192 | 193 | ```js 194 | // Populate using Promises. 195 | Promise.resolve(posts.find() 196 | .then(posts => Promise.all(posts.map(post => comments.find({ query: { postId: post.id } }) 197 | .then(comments => { 198 | post.commentRecords = comments; 199 | return post; 200 | }) 201 | ))) 202 | ) 203 | .then(data => ... ); 204 | 205 | // Populate using async/await. 206 | const postRecords = await posts.find(); 207 | const data = await Promise.all(postRecords.map(async post => { 208 | post.commentRecords = await comments.find({ query: { postId: post.id } }); 209 | return post; 210 | })); 211 | ``` 212 | 213 | Both of these make the following database service calls, and both get the following result. 214 | 215 | ```js 216 | ... posts find 217 | ... comments find { postId: 1 } 218 | ... comments find { postId: 2 } 219 | ... comments find { postId: 3 } 220 | ... comments find { postId: 4 } 221 | 222 | [ { id: 1, 223 | body: 'John post', 224 | userId: 101, 225 | starIds: [ 102, 103, 104 ], 226 | commentRecords: [ 227 | { id: 11, text: 'John post Marshall comment 11', postId: 1, userId: 102 }, 228 | { id: 12, text: 'John post Marshall comment 12', postId: 1, userId: 102 }, 229 | { id: 13, text: 'John post Marshall comment 13', postId: 1, userId: 102 } ] }, 230 | { ... } 231 | ] 232 | ``` 233 | 234 | ### Using Neither Batching nor Caching 235 | 236 | The batch-loader function will be called for every `load` and `loadMany` when batching and caching are disabled in the batch-loader. This means it acts just like individual `get` and `find` method calls. Let's rewrite the above example using such a rudimentary batch-loader: 237 | 238 | ```js 239 | const BatchLoader = require('@feathers-plus/batch-loader'); 240 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 241 | 242 | // Populate using Promises. 243 | const commentsLoaderPromises = new BatchLoader( 244 | keys => comments.find({ query: { postId: { $in: getUniqueKeys(keys) } } }) 245 | .then(result => getResultsByKey(keys, result, comment => comment.postId, '[]')), 246 | { batch: false, cache: false } 247 | ); 248 | 249 | Promise.resolve(posts.find() 250 | .then(postRecords => Promise.all(postRecords.map(post => commentsLoaderPromises.load(post.id) 251 | .then(comments => { 252 | post.commentRecords = comments; 253 | return post; 254 | }) 255 | ))) 256 | ) 257 | .then(data => { ... }); 258 | 259 | // Populate using async/await. 260 | const commentsLoaderAwait = new BatchLoader(async keys => { 261 | const postRecords = await comments.find({ query: { postId: { $in: getUniqueKeys(keys) } } }); 262 | return getResultsByKey(keys, postRecords, comment => comment.postId, '[]'); 263 | }, 264 | { batch: false, cache: false } 265 | ); 266 | 267 | const postRecords = await posts.find(); 268 | const data = await Promise.all(postRecords.map(async post => { 269 | post.commentRecords = await commentsLoaderAwait.load(post.id); 270 | return post; 271 | })); 272 | ``` 273 | 274 | Both of these make the same database service calls as did the [plain JavaScript example](#Using-Plain-JavaScript), because batching and caching were both disabled. 275 | 276 | ```text 277 | ... posts find 278 | ... comments find { postId: { '$in': [ 1 ] } } 279 | ... comments find { postId: { '$in': [ 2 ] } } 280 | ... comments find { postId: { '$in': [ 3 ] } } 281 | ... comments find { postId: { '$in': [ 4 ] } } 282 | ``` 283 | 284 | > A batch-loader with neither batching nor caching makes the same database calls as does a plain Javascript implementation. This is a convenient way to debug issues you might have with batch-loader. The _"magic"_ disappears when you disable batching and caching, which makes it simpler to understand what is happening. 285 | 286 | ### Using Batching and Caching 287 | 288 | Batching and caching are enabled when we remove the 2 `{ batch: false, cache: false }` in the above example. A very different performance profile is now produced: 289 | 290 | ```text 291 | ... posts find 292 | ... comments find { postId: { '$in': [ 1, 2, 3, 4 ] } } 293 | ``` 294 | 295 | Only 1 service call was made for the `comments` records, instead of the previous 4. 296 | 297 | ### A Realistic Example 298 | 299 | The more service calls made, the better batch-loader performs. The above example populated the `posts` records with just the `comments` records. Let's see the effect batch-loader has when we fully populate the `posts` records. 300 | 301 | ```js 302 | const { map, parallel } = require('asyncro'); 303 | const BatchLoader = require('@feathers-plus/batch-loader'); 304 | 305 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 306 | 307 | tester({ batch: false, cache: false }) 308 | .then(data => { ... ) 309 | 310 | async function tester (options) { 311 | const commentsLoader = new BatchLoader(async keys => { 312 | const result = await comments.find({ query: { postId: { $in: getUniqueKeys(keys) } } }); 313 | return getResultsByKey(keys, result, comment => comment.postId, '[]'); 314 | }, 315 | options 316 | ); 317 | 318 | const usersLoader = new BatchLoader(async keys => { 319 | const result = await users.find({ query: { id: { $in: getUniqueKeys(keys) } } }); 320 | return getResultsByKey(keys, result, user => user.id, ''); 321 | }, 322 | options 323 | ); 324 | 325 | const postRecords = await posts.find(); 326 | 327 | await map(postRecords, async post => { 328 | await parallel([ 329 | // Join one users record to posts, for post.userId === users.id 330 | async () => { 331 | post.userRecord = await usersLoader.load(post.userId); 332 | }, 333 | // Join 0, 1 or many comments records to posts, where comments.postId === posts.id 334 | async () => { 335 | const commentRecords = await commentsLoader.load(post.id); 336 | post.commentRecords = commentRecords; 337 | 338 | // Join one users record to comments, for comments.userId === users.id 339 | await map(commentRecords, async comment => { 340 | comment.userRecord = await usersLoader.load(comment.userId); 341 | }); 342 | }, 343 | // Join 0, 1 or many users record to posts, where posts.starIds === users.id 344 | async () => { 345 | if (!post.starIds) return null; 346 | 347 | post.starUserRecords = await usersLoader.loadMany(post.starIds); 348 | } 349 | ]); 350 | }); 351 | 352 | return postRecords; 353 | } 354 | ``` 355 | 356 | > Notice `usersLoader` is being called within 3 quite different joins. These joins will share their batching and cache, noticeably improving overall performance. 357 | 358 | This example has batching and caching disabled. These 22 service calls are made when it is run. They are the same calls which a plain JavaScript implementation would have made: 359 | 360 | ```text 361 | ... posts find 362 | ... users find { id: { '$in': [ 101 ] } } 363 | ... comments find { postId: { '$in': [ 1 ] } } 364 | ... users find { id: { '$in': [ 102 ] } } 365 | ... users find { id: { '$in': [ 103 ] } } 366 | ... users find { id: { '$in': [ 104 ] } } 367 | ... users find { id: { '$in': [ 102 ] } } 368 | ... comments find { postId: { '$in': [ 2 ] } } 369 | ... users find { id: { '$in': [ 101 ] } } 370 | ... users find { id: { '$in': [ 103 ] } } 371 | ... users find { id: { '$in': [ 104 ] } } 372 | ... users find { id: { '$in': [ 103 ] } } 373 | ... comments find { postId: { '$in': [ 3 ] } } 374 | ... users find { id: { '$in': [ 104 ] } } 375 | ... comments find { postId: { '$in': [ 4 ] } } 376 | ... users find { id: { '$in': [ 102 ] } } 377 | ... users find { id: { '$in': [ 102 ] } } 378 | ... users find { id: { '$in': [ 102 ] } } 379 | ... users find { id: { '$in': [ 101 ] } } 380 | ... users find { id: { '$in': [ 101 ] } } 381 | ... users find { id: { '$in': [ 101 ] } } 382 | ... users find { id: { '$in': [ 102 ] } } 383 | ``` 384 | 385 | Now let's enable batching and caching by changing `tester({ batch: false, cache: false })` to `tester()`. Only these **three** service calls are now made to obtain the same results: 386 | 387 | ```text 388 | ... posts find 389 | ... users find { id: { '$in': [ 101, 102, 103, 104 ] } } 390 | ... comments find { postId: { '$in': [ 1, 2, 3, 4 ] } } 391 | ``` 392 | 393 | > The 2 BatchLoaders reduced the number of services calls from 22 for a plain implementation, to just 3! 394 | 395 | The final populated result is: 396 | 397 | ```js 398 | [ 399 | { 400 | id: 1, 401 | body: "John post", 402 | userId: 101, 403 | starIds: [102, 103, 104], 404 | userRecord: { id: 101, name: "John" }, 405 | starUserRecords: [ 406 | { id: 102, name: "Marshall" }, 407 | { id: 103, name: "Barbara" }, 408 | { id: 104, name: "Aubree" }, 409 | ], 410 | commentRecords: [ 411 | { 412 | id: 11, 413 | text: "John post Marshall comment 11", 414 | postId: 1, 415 | userId: 102, 416 | userRecord: { id: 102, name: "Marshall" }, 417 | }, 418 | { 419 | id: 12, 420 | text: "John post Marshall comment 12", 421 | postId: 1, 422 | userId: 102, 423 | userRecord: { id: 102, name: "Marshall" }, 424 | }, 425 | { 426 | id: 13, 427 | text: "John post Marshall comment 13", 428 | postId: 1, 429 | userId: 102, 430 | userRecord: { id: 102, name: "Marshall" }, 431 | }, 432 | ], 433 | }, 434 | { 435 | id: 2, 436 | body: "Marshall post", 437 | userId: 102, 438 | starIds: [101, 103, 104], 439 | userRecord: { id: 102, name: "Marshall" }, 440 | starUserRecords: [ 441 | { id: 101, name: "John" }, 442 | { id: 103, name: "Barbara" }, 443 | { id: 104, name: "Aubree" }, 444 | ], 445 | commentRecords: [ 446 | { 447 | id: 14, 448 | text: "Marshall post John comment 14", 449 | postId: 2, 450 | userId: 101, 451 | userRecord: { id: 101, name: "John" }, 452 | }, 453 | { 454 | id: 15, 455 | text: "Marshall post John comment 15", 456 | postId: 2, 457 | userId: 101, 458 | userRecord: { id: 101, name: "John" }, 459 | }, 460 | ], 461 | }, 462 | { 463 | id: 3, 464 | body: "Barbara post", 465 | userId: 103, 466 | userRecord: { id: 103, name: "Barbara" }, 467 | commentRecords: [ 468 | { 469 | id: 16, 470 | text: "Barbara post John comment 16", 471 | postId: 3, 472 | userId: 101, 473 | userRecord: { id: 101, name: "John" }, 474 | }, 475 | ], 476 | }, 477 | { 478 | id: 4, 479 | body: "Aubree post", 480 | userId: 104, 481 | userRecord: { id: 104, name: "Aubree" }, 482 | commentRecords: [ 483 | { 484 | id: 17, 485 | text: "Aubree post Marshall comment 17", 486 | postId: 4, 487 | userId: 102, 488 | userRecord: { id: 102, name: "Marshall" }, 489 | }, 490 | ], 491 | }, 492 | ]; 493 | ``` 494 | 495 | ## See also 496 | 497 | - [facebook/dataloader](https://github.com/facebook/dataloader) from which batch-loader is derived. 498 | -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- 1 | 2 |

Usage

3 | 4 | ```js 5 | npm install --save @feathers-plus/batch-loader 6 | 7 | // JS 8 | const BatchLoader = require('@feathers-plus/batch-loader'); 9 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 10 | 11 | const usersLoader = new BatchLoader(async (keys, context) => { 12 | const usersRecords = await users.find({ query: { id: { $in: getUniqueKeys(keys) } } }); 13 | return getResultsByKey(keys, usersRecords, user => user.id, '') 14 | }, 15 | { context: {} } 16 | ); 17 | 18 | const user = await usersLoader.load(key); 19 | ``` 20 | 21 | > May be used on the client. 22 | 23 | 24 |

class BatchLoader( batchLoadFunc [, options] )

25 | 26 | Create a new batch-loader given a batch loading function and options. 27 | 28 | - **Arguments:** 29 | - `{Function} batchLoadFunc` 30 | - `{Object} [ options ]` 31 | - `{Boolean} batch` 32 | - `{Boolean} cache` 33 | - `{Function} cacheKeyFn` 34 | - `{Object} cacheMap` 35 | - `{Object} context` 36 | - `{Number} maxBatchSize` 37 | 38 | | Argument | Type | Default | Description | 39 | | --------------- | :--------: | ------- | ------------------------------------------------ | 40 | | `batchLoadFunc` | `Function` | | See [Batch Function](./guide.md#batch-function). | 41 | | `options` | `Object` | | Options. | 42 | 43 | | `options` | Argument | Type | Default | Description | 44 | | -------------- | -------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------- | ----------- | 45 | | `batch` | Boolean | `true` | Set to false to disable batching, invoking `batchLoadFunc` with a single load key. | 46 | | `cache` | Boolean | `true` | Set to false to disable memoization caching, creating a new Promise and new key in the `batchLoadFunc` for every load of the same key. | 47 | | `cacheKeyFn` | Function | `key => key` | Produces cache key for a given load key. Useful when keys are objects and two objects should be considered equivalent. | 48 | | `cacheMap` | Object | `new Map()` | Instance of Map (or an object with a similar API) to be used as cache. See below. | 49 | | `context` | Object | `null` | A context object to pass into `batchLoadFunc` as its second argument. | 50 | | `maxBatchSize` | Number | `Infinity` | Limits the number of keys when calling `batchLoadFunc`. | 51 | 52 | - **Example** 53 | 54 | ```js 55 | const BatchLoader = require("@feathers-plus/batch-loader"); 56 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 57 | 58 | const usersLoader = new BatchLoader( 59 | async (keys, context) => { 60 | const data = await users.find({ 61 | query: { id: { $in: getUniqueKeys(keys) } }, 62 | paginate: false, 63 | }); 64 | return getResultsByKey(keys, data, (user) => user.id, ""); 65 | }, 66 | { context: {}, batch: true, cache: true } 67 | ); 68 | ``` 69 | 70 | - **Pagination** 71 | 72 | The number of results returned by a query using `$in` is controlled by the pagination `max` set for that Feathers service. You need to specify a `paginate: false` option to ensure that records for all the keys are returned. 73 | 74 | The maximum number of keys the `batchLoadFunc` is called with can be controlled by the BatchLoader itself with the `maxBatchSize` option. 75 | 76 | - **option.cacheMap** 77 | 78 | The default cache will grow without limit, which is reasonable for short lived batch-loaders which are rebuilt on every request. The number of records cached can be limited with a _least-recently-used_ cache: 79 | 80 | ```js 81 | const BatchLoader = require('@feathers-plus/batch-loader'); 82 | const cache = require('@feathers-plus/cache'); 83 | 84 | const usersLoader = new BatchLoader( 85 | keys => { ... }, 86 | { cacheMap: cache({ max: 100 }) 87 | ); 88 | ``` 89 | 90 | > You can consider wrapping npm's `lru` on the browser. 91 | 92 | - **See also:** [Guide](./guide.md) 93 | 94 | 95 |

static BatchLoader.getUniqueKeys( keys )

96 | 97 | Returns the unique elements in an array. 98 | 99 | - **Arguments:** 100 | - `{Array} keys` 101 | 102 | | Argument | Type | Default | Description | 103 | | -------- | ------------------------------ | ------- | --------------------------------- | 104 | | `keys` | `Array<` `String /` `Number >` | | The keys. May contain duplicates. | 105 | 106 | - **Example:** 107 | 108 | ```js 109 | const usersLoader = new BatchLoader(async keys => 110 | const data = users.find({ query: { id: { $in: getUniqueKeys(keys) } } }) 111 | ... 112 | ); 113 | ``` 114 | 115 | - **Details** 116 | 117 | The array of keys may contain duplicates when the batch-loader's memoization cache is disabled. 118 | 119 |

Function does not handle keys of type Object nor Array.

120 | 121 | 122 |

static BatchLoader.getResultsByKey( keys, records, getRecordKeyFunc, type [, options] )

123 | 124 | Reorganizes the records from the service call into the result expected from the batch function. 125 | 126 | - **Arguments:** 127 | - `{Array} keys` 128 | - `{Array} records` 129 | - `{Function} getRecordKeyFunc` 130 | - `{String} type` 131 | - `{Object} [ options ]` 132 | - `{null | []} defaultElem` 133 | - `{Function} onError` 134 | 135 | | Argument | Type | Default | Description | 136 | | ------------------ | :---------------------------: | ------- | ------------------------------------------------------------------------------------------------------------- | 137 | | `keys` | `Array<` `String /` `Number>` | | An array of `key` elements, which the value the batch loader function will use to find the records requested. | 138 | | `records` | `Array< ` `Object >` | | An array of records which, in total, resolve all the `keys`. | 139 | | `getRecordKeyFunc` | `Function` | | See below. | 140 | | `type` | `String` | | The type of value the batch loader must return for each key. | 141 | | `options` | `Object` | | Options. | 142 | 143 | | `type` | Value | Description | 144 | | -------- | :----------------------------------------------: | ----------- | 145 | | `''` | An optional single record. | 146 | | `'!'` | A required single record. | 147 | | `'[]'` | A required array including 0, 1 or more records. | 148 | | `'[]!'` | Alias of `'[]'` | 149 | | `'[!]'` | A required array including 1 or more records. | 150 | | `'[!]!'` | Alias of `'[!]'` | 151 | 152 | | `options` | Argument | Type | Default | Description | 153 | | ------------- | ------------- | :--------------: | ---------------------------------------------------------------------------------------------- | ----------- | 154 | | `defaultElem` | `{null / []}` | `null` | The value to return for a `key` having no record(s). | 155 | | `onError` | `Function` | `(i, msg) => {}` | Handler for detected errors, e.g. `(i, msg) =>` `{ throw new Error(msg,` `'on element', i); }` | 156 | 157 | - **Example** 158 | 159 | ```js 160 | const usersLoader = new BatchLoader(async (keys) => { 161 | const data = users.find({ query: { id: { $in: getUniqueKeys(keys) } } }); 162 | return getResultsByKey(keys, data, (user) => user.id, "", { 163 | defaultElem: [], 164 | }); 165 | }); 166 | ``` 167 | 168 | - **Details** 169 | 170 |

Function does not handle keys of type Object nor Array.

171 | 172 | - **getRecordKeyFunc** 173 | 174 | A function which, given a record, returns the key it satisfies, e.g. 175 | 176 | ```js 177 | (user) => user.id; 178 | ``` 179 | 180 | - **See also:** [Batch-Function](./guide.md#Batch-Function) 181 | 182 | 183 |

batchLoader.load( key )

184 | 185 | Loads a key, returning a Promise for the value represented by that key. 186 | 187 | - **Arguments:** 188 | - `{String | Number | Object | Array} key` 189 | 190 | | Argument | Type | Default | Description | 191 | | -------- | :--------------------------------: | ------- | ---------------------------------------------------- | 192 | | `key` | `String` `Number` `Object` `Array` | | The key the batch-loader uses to find the result(s). | 193 | 194 | - **Example:** 195 | 196 | ```js 197 | const batchLoader = new BatchLoader( ... ); 198 | const user = await batchLoader.load(key); 199 | ``` 200 | 201 | 202 |

batchLoader.loadMany( keys )

203 | 204 | Loads multiple keys, promising a arrays of values. 205 | 206 | - **Arguments** 207 | - `{Array} keys` 208 | 209 | | Argument | Type | Default | Description | 210 | | -------- | :---------------------------------------------------: | ------- | ---------------------------------------------------- | 211 | | `keys` | `Array<` `String /` ` Number /` ` Object /` ` Array>` | | The keys the batch-loader will return result(s) for. | 212 | 213 | - **Example** 214 | 215 | ```js 216 | const usersLoader = new BatchLoader( ... ); 217 | const users = await usersLoader.loadMany([ key1, key2 ]); 218 | ``` 219 | 220 | - **Details** 221 | 222 | This is a convenience method. `usersLoader.loadMany([ key1, key2 ])` is equivalent to the more verbose: 223 | 224 | ```js 225 | Promise.all([usersLoader.load(key1), usersLoader.load(key2)]); 226 | ``` 227 | 228 | 229 |

batchLoader.clear( key )

230 | 231 | Clears the value at key from the cache, if it exists. 232 | 233 | - **Arguments:** 234 | - `{String | Number | Object | Array} key` 235 | 236 | | Argument | Type | Default | Description | 237 | | -------- | :--------------------------------: | ------- | --------------------------------- | 238 | | `key` | `String` `Number` `Object` `Array` | | The key to remove from the cache. | 239 | 240 | - **Details** 241 | 242 | The key is matches using strict equality. This is particularly important for `Object` and `Array` keys. 243 | 244 | 245 |

batchLoader.clearAll()

246 | 247 | Clears the entire cache. 248 | 249 | - **Details** 250 | 251 | To be used when some event results in unknown invalidations across this particular batch-loader. 252 | 253 | 254 |

batchLoader.prime( key, value )

255 | 256 | Primes the cache with the provided key and value. 257 | 258 | - **Arguments:** 259 | - `{String | Number | Object | Array} key` 260 | - `{Object} record` 261 | 262 | | Argument | Type | Default | Description | 263 | | -------- | :--------------------------------: | ------- | ------------------------------------ | 264 | | `key` | `String` `Number` `Object` `Array` | | The key in the cache for the record. | 265 | | `record` | `Object` | | The value for the `key`. | 266 | 267 | - **Details** 268 | 269 | **If the key already exists, no change is made.** To forcefully prime the cache, clear the key first with `batchloader.clear(key)`. 270 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | 2 | const { nextTick } = require('process'); 3 | 4 | // TODO: There are duplicates here. These should be removed 5 | // and docs/types updated. Would require a major version bump 6 | const resultTypes = { 7 | '[!]!': { collection: true, elemReqd: true }, 8 | '[!]': { collection: true, elemReqd: true }, 9 | '[]!': { collection: true, elemReqd: false }, 10 | '[]': { collection: true, elemReqd: false }, 11 | '!': { collection: false, elemReqd: true }, 12 | '': { collection: false, elemReqd: false } 13 | }; 14 | 15 | let resolvedPromise; 16 | 17 | const BatchLoader1 = module.exports = class BatchLoader { 18 | constructor (batchLoadFn, options) { 19 | if (typeof batchLoadFn !== 'function') { 20 | throw new Error([ 21 | 'BatchLoader must be constructed with a function which accepts', 22 | `Array and returns Promise>, but got: ${batchLoadFn}. (BatchLoader)` 23 | ].join(' ')); 24 | } 25 | 26 | this._batchLoadFn = batchLoadFn; 27 | this._options = options; 28 | this._promiseCache = getValidCacheMap(options); 29 | this._queue = []; 30 | this._context = options && options.context; 31 | } 32 | 33 | load (key) { 34 | if (key === undefined) { 35 | throw new Error( 36 | `batchLoader.load() must be called with a value, but got: ${String(key)} (BatchLoader).` 37 | ); 38 | } 39 | 40 | if (Array.isArray(key)) { 41 | throw new Error( 42 | 'batchLoader.load() called with an array. batchLoader.loadMany() must be used. (BatchLoader).' 43 | ); 44 | } 45 | 46 | const options = this._options; 47 | const shouldBatch = !options || options.batch !== false; 48 | const shouldCache = !options || options.cache !== false; 49 | const cacheKeyFn = options && options.cacheKeyFn; 50 | const cacheKey = cacheKeyFn ? cacheKeyFn(key) : key; 51 | 52 | if (shouldCache) { 53 | const cachedPromise = this._promiseCache.get(cacheKey); 54 | if (cachedPromise) return cachedPromise; 55 | } 56 | 57 | const promise = new Promise((resolve, reject) => { 58 | this._queue.push({ key, resolve, reject }); 59 | 60 | if (this._queue.length === 1) { 61 | if (shouldBatch) { 62 | enqueuePostPromiseJob(() => dispatchQueue(this, this._context)); 63 | } else { 64 | dispatchQueue(this, this._context); 65 | } 66 | } 67 | }); 68 | 69 | if (shouldCache) { 70 | this._promiseCache.set(cacheKey, promise); 71 | } 72 | 73 | return promise; 74 | } 75 | 76 | loadMany (keys) { 77 | if (!Array.isArray(keys)) { 78 | throw new Error( 79 | `batchLoader.loadMany must be called with an array but got: ${keys}. (BatchLoader)` 80 | ); 81 | } 82 | 83 | return Promise.all(keys.map(key => this.load(key))); 84 | } 85 | 86 | clear (key) { 87 | const cacheKeyFn = this._options && this._options.cacheKeyFn; 88 | const cacheKey = cacheKeyFn ? cacheKeyFn(key) : key; 89 | this._promiseCache.delete(cacheKey); 90 | return this; 91 | } 92 | 93 | clearAll () { 94 | this._promiseCache.clear(); 95 | return this; 96 | } 97 | 98 | prime (key, value) { 99 | const cacheKeyFn = this._options && this._options.cacheKeyFn; 100 | const cacheKey = cacheKeyFn ? cacheKeyFn(key) : key; 101 | 102 | if (this._promiseCache.get(cacheKey) === undefined) { 103 | const promise = value instanceof Error // Match behavior of load(key). 104 | ? Promise.reject(value) 105 | : Promise.resolve(value); 106 | 107 | this._promiseCache.set(cacheKey, promise); 108 | } 109 | 110 | return this; 111 | } 112 | 113 | static getResultsByKey (keys, resultArray, serializeRecordKey, resultType, options = {}) { 114 | const { onError = () => { }, defaultElem = null } = options; 115 | 116 | const getRecKey = typeof serializeRecordKey === 'function' 117 | ? serializeRecordKey 118 | : record => record[serializeRecordKey].toString(); 119 | 120 | if (!resultTypes[resultType]) { 121 | onError(null, 'Invalid resultType option in dataLoaderAlignResults.'); 122 | } 123 | 124 | const { collection, elemReqd } = resultTypes[resultType]; 125 | 126 | if (resultArray === null || resultArray === undefined) resultArray = []; 127 | if (typeof resultArray === 'object' && !Array.isArray(resultArray)) resultArray = [resultArray]; 128 | 129 | // hash = { '1': {id: 1, bar: 10} } or { '1': [{id: 1, bar: 10}, {id: 1, bar: 11}], 2: [{id: 2, bar: 12}] } 130 | const hash = Object.create(null); 131 | 132 | resultArray.forEach((obj, i) => { 133 | if (!obj && elemReqd) { 134 | onError(i, 'This result requires a non-null result.'); 135 | } 136 | 137 | const recKey = getRecKey(obj); 138 | 139 | if (!hash[recKey]) hash[recKey] = []; 140 | hash[recKey].push(obj); 141 | }); 142 | 143 | // Convert hash to single records if required. 144 | // from = { '1': [{id: 1, bar: 10}], '2': [{id: 2, bar: 12}] } 145 | // to = { '1': {id: 1, bar: 10}, '2': {id: 2, bar: 12} } 146 | if (!collection) { 147 | Object.keys(hash).forEach((key, i) => { 148 | const value = hash[key]; 149 | 150 | if (value.length !== 1) { 151 | onError(i, `This result needs a single result object. A collection of ${value.length} elements was found.`); 152 | } 153 | 154 | hash[key] = value[0]; 155 | }); 156 | } 157 | 158 | return keys.map((key, i) => { 159 | const value = hash[key] || defaultElem; 160 | 161 | if (!value && elemReqd) { 162 | onError(i, 'This key requires a non-null result. Null or empty-array found.'); 163 | } 164 | 165 | return value; 166 | }); 167 | } 168 | 169 | static getUniqueKeys (keys) { 170 | // This is one of the fastest algorithms 171 | const found = {}; 172 | const unique = []; 173 | 174 | keys.forEach(key => { 175 | if (!found[key]) { 176 | found[key] = unique.push(key); 177 | } 178 | }); 179 | 180 | return unique; 181 | } 182 | 183 | static loaderFactory (service, id, multi, options = {}) { 184 | const { getKey = rec => rec[id], paramNames, injects } = options; 185 | 186 | return context => new BatchLoader(async (keys, context) => { 187 | const result = await service.find(makeCallingParams( 188 | context, { [id]: { $in: BatchLoader1.getUniqueKeys(keys) } }, paramNames, injects 189 | )); 190 | 191 | return BatchLoader1.getResultsByKey(keys, result, getKey, multi ? '[!]' : '!'); 192 | }, 193 | { context } 194 | ); 195 | } 196 | }; 197 | 198 | function dispatchQueue (loader, context) { 199 | const queue = loader._queue; 200 | loader._queue = []; 201 | 202 | const maxBatchSize = loader._options && loader._options.maxBatchSize; 203 | if (maxBatchSize && maxBatchSize > 0 && maxBatchSize < queue.length) { 204 | for (let i = 0; i < queue.length / maxBatchSize; i++) { 205 | dispatchQueueBatch( 206 | loader, 207 | queue.slice(i * maxBatchSize, (i + 1) * maxBatchSize), 208 | context 209 | ); 210 | } 211 | } else { 212 | dispatchQueueBatch(loader, queue, context); 213 | } 214 | } 215 | 216 | function dispatchQueueBatch (loader, queue, context) { 217 | const keys = queue.map(({ key }) => key); 218 | 219 | const batchLoadFn = loader._batchLoadFn; 220 | const batchPromise = batchLoadFn(keys, context); 221 | 222 | if (!batchPromise || typeof batchPromise.then !== 'function') { 223 | throw new Error([ 224 | 'BatchLoader must be constructed with a function which accepts', 225 | 'Array and returns Promise>, but the function did', 226 | `not return a Promise: ${String(batchPromise)}.` 227 | ].join(' ')); 228 | } 229 | 230 | batchPromise 231 | .then(values => { 232 | // Assert the expected resolution from batchLoadFn. 233 | if (!Array.isArray(values)) { 234 | throw new Error([ 235 | 'BatchLoader must be constructed with a function which accepts', 236 | 'Array and returns Promise>, but the function did', 237 | `not return a Promise of an Array: ${String(values)}.` 238 | ].join(' ')); 239 | } 240 | if (values.length !== keys.length) { 241 | throw new Error([ 242 | 'DataLoader must be constructed with a function which accepts', 243 | 'Array and returns Promise>, but the function did', 244 | 'not return a Promise of an Array of the same length as the Array', 245 | 'of keys.', 246 | `\n\nKeys:\n${String(keys)}`, 247 | `\n\nValues:\n${String(values)}` 248 | ].join(' ')); 249 | } 250 | 251 | queue.forEach(({ key, resolve, reject }, index) => { 252 | const value = values[index]; 253 | if (value instanceof Error) { 254 | reject(value); 255 | } else { 256 | resolve(value); 257 | } 258 | }); 259 | }); 260 | } 261 | 262 | function enqueuePostPromiseJob (fn) { 263 | if (!resolvedPromise) { 264 | resolvedPromise = Promise.resolve(); 265 | } 266 | resolvedPromise.then(() => nextTick(fn)); 267 | } 268 | 269 | function getValidCacheMap (options) { 270 | const cacheMap = options && options.cacheMap; 271 | if (!cacheMap) return new Map(); 272 | 273 | const missingFunctions = ['get', 'set', 'delete', 'clear'].filter( 274 | fnName => typeof cacheMap[fnName] !== 'function' 275 | ); 276 | 277 | if (missingFunctions.length) { 278 | throw new TypeError([ 279 | 'options.cacheMap missing methods:', 280 | missingFunctions.join(', '), 281 | '(BatchLoader)' 282 | ].join(' ')); 283 | } 284 | 285 | return cacheMap; 286 | } 287 | 288 | // copied from feathers-hooks-common v4 so Batch-Loader can be used with FeathersJS v3 (Buzzard) 289 | function makeCallingParams ( 290 | context, query, include = ['provider', 'authenticated', 'user'], inject = {} 291 | ) { 292 | const included = query ? { query } : {}; 293 | const defaults = { _populate: 'skip', paginate: false }; 294 | 295 | if (include) { 296 | (Array.isArray(include) ? include : [include]).forEach(name => { 297 | if (context.params && name in context.params) included[name] = context.params[name]; 298 | }); 299 | } 300 | 301 | return Object.assign(defaults, included, inject); 302 | } 303 | 304 | /* shout out to Facebook's GraphQL utilities */ 305 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@feathers-plus/batch-loader", 3 | "description": "Reduce requests to backend services by batching calls and caching records.", 4 | "version": "0.3.6", 5 | "homepage": "https://github.com/feathersjs-ecosystem/batch-loader", 6 | "main": "lib/", 7 | "types": "types/", 8 | "keywords": [ 9 | "feathers", 10 | "feathers-ecosystem", 11 | "batch", 12 | "cache" 13 | ], 14 | "license": "MIT", 15 | "repository": { 16 | "type": "git", 17 | "url": "git@github.com:feathersjs-ecosystem/batch-loader.git" 18 | }, 19 | "author": { 20 | "name": "John J. Szwaronek", 21 | "email": "johnsz9999@gmail.com" 22 | }, 23 | "contributors": [], 24 | "bugs": { 25 | "url": "https://github.com/feathersjs-ecosystem/batch-loader/issues" 26 | }, 27 | "engines": { 28 | "node": ">= 6.0.0" 29 | }, 30 | "scripts": { 31 | "publish": "git push origin --tags && npm run changelog && git push origin", 32 | "release:pre": "npm version prerelease && npm publish --tag pre", 33 | "release:patch": "npm version patch && npm publish", 34 | "release:minor": "npm version minor && npm publish", 35 | "release:major": "npm version major && npm publish", 36 | "changelog": "github_changelog_generator && git add CHANGELOG.md && git commit -am \"Updating changelog\"", 37 | "lint": "semistandard lib/*.js lib/**/*.js tests/*.js tests/**/*.js --fix", 38 | "lint:types": "dtslint types/", 39 | "mocha": "mocha --recursive tests/", 40 | "coverage": "istanbul cover node_modules/mocha/bin/_mocha -- --recursive tests/", 41 | "test": "npm run lint && npm run lint:types && npm run coverage" 42 | }, 43 | "semistandard": { 44 | "sourceType": "module", 45 | "env": [ 46 | "mocha" 47 | ] 48 | }, 49 | "directories": { 50 | "lib": "lib" 51 | }, 52 | "devDependencies": { 53 | "@types/feathersjs__feathers": "^3.1.7", 54 | "asyncro": "^3.0.0", 55 | "chai": "^4.3.0", 56 | "dtslint": "^4.0.6", 57 | "istanbul": "1.1.0-alpha.1", 58 | "mocha": "^8.2.1", 59 | "semistandard": "^16.0.0", 60 | "typescript": "^4.1.4" 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /tests/counts-js-to-batchloader.test.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const BatchLoader = require('../lib/index'); 4 | const { posts, comments } = require('../tests/helpers/make-services'); 5 | 6 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 7 | 8 | let countJS = 0; 9 | let countResolver = 0; 10 | let countNoBatch = 0; 11 | let countBatch = 0; 12 | 13 | function tester (fn) { 14 | return posts.find() 15 | .then(posts => { 16 | return Promise.all(posts.map(post => { 17 | return fn(post.id) 18 | .then(comments => { 19 | post.commentRecords = comments; 20 | return post; 21 | }); 22 | })); 23 | }) 24 | .catch(err => console.log(err)); 25 | } 26 | 27 | function commentsBatchLoaderResolver (keys) { 28 | // console.log('... comments batchLoader resolver', keys); 29 | countResolver += 1; 30 | 31 | return comments.find({ query: { postId: { $in: getUniqueKeys(keys) } } }) 32 | .then(result => getResultsByKey(keys, result, comment => comment.postId, '[!]')); 33 | } 34 | 35 | describe('counts-js-to-batchloader.test.js', () => { 36 | it('Compare JS to BatchLoader with and without batch and cache.', () => { 37 | return Promise.resolve() 38 | .then(() => { 39 | // console.log('\n=== Normal JavaScript'); 40 | return tester(key => { 41 | countJS += 1; 42 | return comments.find({ query: { postId: key } }); 43 | }); 44 | }) 45 | 46 | .then(() => { 47 | // console.log('\n=== Using BatchLoader with neither batching mor caching'); 48 | 49 | const commentsBatchLoader1 = new BatchLoader( 50 | commentsBatchLoaderResolver, { batch: false, cache: false } 51 | ); 52 | 53 | countResolver = 0; 54 | return tester(key => commentsBatchLoader1.load(key)); 55 | }) 56 | 57 | .then(() => { 58 | countNoBatch = countResolver; 59 | // console.log('\n=== Using BatchLoader with batching and caching'); 60 | 61 | const commentsBatchLoader2 = new BatchLoader( 62 | commentsBatchLoaderResolver 63 | ); 64 | 65 | countResolver = 0; 66 | return tester(key => commentsBatchLoader2.load(key)); 67 | }) 68 | 69 | .then(posts => { 70 | countBatch = countResolver; 71 | 72 | assert(countJS, 4, 'countJS'); 73 | assert(countNoBatch, 4, 'countNoBatch'); 74 | assert(countBatch, 1, 'countBatch'); 75 | }); 76 | }); 77 | }); 78 | -------------------------------------------------------------------------------- /tests/counts-no-batch-to-batch.test.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const BatchLoader = require('../lib/index'); 4 | const { posts, comments, users } = require('./helpers/make-services'); 5 | 6 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 7 | 8 | let countUsersResolver = 0; 9 | let countCommentsResolver = 0; 10 | let countUsersNoBatch; 11 | let countCommentsNoBatch; 12 | let countUsersBatch; 13 | let countCommentsBatch; 14 | 15 | function tester (options) { 16 | const commentsBatchLoader = new BatchLoader( 17 | keys => { 18 | countUsersResolver += 1; 19 | return comments.find({ query: { postId: { $in: getUniqueKeys(keys) } } }) 20 | .then(result => getResultsByKey(keys, result, comment => comment.postId, '[!]')); 21 | }, 22 | options 23 | ); 24 | 25 | const usersBatchLoader = new BatchLoader( 26 | keys => { 27 | countCommentsResolver += 1; 28 | return users.find({ query: { id: { $in: getUniqueKeys(keys) } } }) 29 | .then(result => getResultsByKey(keys, result, user => user.id, '!')); 30 | }, 31 | options 32 | ); 33 | 34 | countUsersResolver = 0; 35 | countCommentsResolver = 0; 36 | 37 | return posts.find() 38 | .then(posts => { 39 | // Process each post record 40 | return Promise.all(posts.map(post => { 41 | return Promise.all([ 42 | 43 | // Attach comments records 44 | commentsBatchLoader.load(post.id) 45 | .then(comments => { 46 | post.commentRecords = comments; 47 | 48 | // Process each comment record 49 | return Promise.all(comments.map(comment => { 50 | // Attach user record 51 | return usersBatchLoader.load(comment.userId) 52 | .then(user => { comment.userRecord = user; }); 53 | })); 54 | }), 55 | 56 | // Attach star user records 57 | Promise.resolve() 58 | .then(() => { 59 | if (!post.starIds) return null; 60 | 61 | return usersBatchLoader.loadMany(post.starIds) // Note that 'loadMany' is used. 62 | .then(users => { post.starUserRecords = users; }); 63 | }) 64 | ]) 65 | .then(() => post); 66 | })); 67 | }) 68 | .catch(err => console.log(err)); 69 | } 70 | 71 | describe('counts-no-batch-to-batch.test.js', () => { 72 | it('Compare BatchLoader with neither batch nor cache, to BatchLoader with both.', () => { 73 | return Promise.resolve() 74 | .then(() => { 75 | // console.log('\n=== Using BatchLoader with neither batching nor caching'); 76 | 77 | return tester({ batch: false, cache: false }); 78 | }) 79 | 80 | .then(() => { 81 | countUsersNoBatch = countUsersResolver; 82 | countCommentsNoBatch = countCommentsResolver; 83 | // console.log('\n=== Using BatchLoader with batching and caching'); 84 | 85 | return tester(); 86 | }) 87 | 88 | .then(posts => { 89 | countUsersBatch = countUsersResolver; 90 | countCommentsBatch = countCommentsResolver; 91 | 92 | assert.equal(countUsersNoBatch, 4, 'countUsersNoBatch'); 93 | assert.equal(countCommentsNoBatch, 13, 'countCommentsNoBatch'); 94 | assert.equal(countUsersBatch, 1, 'countUsersBatch'); 95 | assert.equal(countCommentsBatch, 1, 'countCommentsBatch'); 96 | }); 97 | }); 98 | }); 99 | -------------------------------------------------------------------------------- /tests/get-results-by-key.text.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const BatchLoader = require('../lib'); 4 | 5 | const resultsObj = [0, 1, 2].map(id => ({ id, value: id })); 6 | const collection1 = [0, 1, 2, 1].map(id => ({ id, value: id })); 7 | const result1 = [ 8 | [{ id: 0, value: 0 }], 9 | [{ id: 1, value: 1 }, { id: 1, value: 1 }], 10 | [{ id: 2, value: 2 }] 11 | ]; 12 | 13 | let getResultsByKey; 14 | let onErrorCalled; 15 | let args; 16 | 17 | const onError = (index, detail) => { 18 | onErrorCalled = true; 19 | // console.log(`#${index} ${detail}`); 20 | }; 21 | 22 | describe('get-results-by-key.test.js', () => { 23 | beforeEach(() => { 24 | getResultsByKey = BatchLoader.getResultsByKey; 25 | onErrorCalled = false; 26 | }); 27 | 28 | describe('basic tests', () => { 29 | it('returns a function', () => { 30 | assert.isFunction(getResultsByKey); 31 | }); 32 | }); 33 | 34 | describe("test ''", () => { 35 | beforeEach(() => { 36 | args = ['id', '', { onError, isStrict: true }]; 37 | }); 38 | 39 | it('results in key order', () => { 40 | const actual = getResultsByKey([0, 1, 2], resultsObj, ...args); 41 | const expected = resultsObj; 42 | 43 | assert.deepEqual(actual, expected); 44 | }); 45 | 46 | it('results in mixed order', () => { 47 | const actual = getResultsByKey([2, 0, 1], resultsObj, ...args); 48 | const expected = [resultsObj[2], resultsObj[0], resultsObj[1]]; 49 | 50 | assert.deepEqual(actual, expected); 51 | }); 52 | 53 | it('results missing', () => { 54 | const actual = getResultsByKey([2, 99, 0, 98, 1, 97], resultsObj, ...args); 55 | const expected = [resultsObj[2], null, resultsObj[0], null, resultsObj[1], null]; 56 | 57 | assert.deepEqual(actual, expected); 58 | }); 59 | }); 60 | 61 | describe("test '!'", () => { 62 | beforeEach(() => { 63 | args = ['id', '!', { onError, isStrict: true }]; 64 | }); 65 | 66 | it('results in key order', () => { 67 | const actual = getResultsByKey([0, 1, 2], resultsObj, ...args); 68 | const expected = resultsObj; 69 | 70 | assert.deepEqual(actual, expected); 71 | }); 72 | 73 | it('results in mixed order', () => { 74 | const actual = getResultsByKey([2, 0, 1], resultsObj, ...args); 75 | const expected = [resultsObj[2], resultsObj[0], resultsObj[1]]; 76 | 77 | assert.deepEqual(actual, expected); 78 | }); 79 | 80 | it('results missing', () => { 81 | getResultsByKey([2, 99, 0, 98, 1, 97], resultsObj, ...args); 82 | 83 | assert(onErrorCalled, 'should not have succeeded'); 84 | }); 85 | }); 86 | 87 | describe("test '[]'", () => { 88 | beforeEach(() => { 89 | args = ['id', '[]', { onError, isStrict: true }]; 90 | }); 91 | 92 | it('results in key order', () => { 93 | const actual = getResultsByKey([0, 1, 2], collection1, ...args); 94 | assert.deepEqual(actual, result1); 95 | }); 96 | 97 | it('results in mixed order', () => { 98 | const actual = getResultsByKey([2, 0, 1], collection1, ...args); 99 | const expected = [result1[2], result1[0], result1[1]]; 100 | 101 | assert.deepEqual(actual, expected); 102 | }); 103 | 104 | it('results missing', () => { 105 | const actual = getResultsByKey([2, 99, 0, 98, 1, 97], collection1, ...args); 106 | const expected = [result1[2], null, result1[0], null, result1[1], null]; 107 | 108 | assert.deepEqual(actual, expected); 109 | }); 110 | 111 | it('results missing', () => { 112 | const actual = getResultsByKey([2, null, 0, 98, 1, 97], collection1, ...args); 113 | const expected = [result1[2], null, result1[0], null, result1[1], null]; 114 | 115 | assert.deepEqual(actual, expected); 116 | }); 117 | }); 118 | 119 | describe("test '[!]'", () => { 120 | beforeEach(() => { 121 | args = ['id', '[!]', { onError, isStrict: true }]; 122 | }); 123 | 124 | it('results in key order', () => { 125 | const actual = getResultsByKey([0, 1, 2], collection1, ...args); 126 | const expected = result1; 127 | 128 | assert.deepEqual(actual, expected); 129 | }); 130 | 131 | it('results in mixed order', () => { 132 | const actual = getResultsByKey([2, 0, 1], collection1, ...args); 133 | const expected = [result1[2], result1[0], result1[1]]; 134 | 135 | assert.deepEqual(actual, expected); 136 | }); 137 | 138 | it('results missing', () => { 139 | getResultsByKey([2, 99, 0, 98, 1, 97], collection1, ...args); 140 | 141 | assert(onErrorCalled, 'should not have succeeded'); 142 | }); 143 | }); 144 | 145 | describe("test '[]!'", () => { 146 | beforeEach(() => { 147 | args = ['id', '[]', { onError, isStrict: true }]; 148 | }); 149 | 150 | it('results in key order', () => { 151 | const actual = getResultsByKey([0, 1, 2], collection1, ...args); 152 | assert.deepEqual(actual, result1); 153 | }); 154 | 155 | it('results in mixed order', () => { 156 | const actual = getResultsByKey([2, 0, 1], collection1, ...args); 157 | const expected = [result1[2], result1[0], result1[1]]; 158 | 159 | assert.deepEqual(actual, expected); 160 | }); 161 | 162 | it('results missing', () => { 163 | const actual = getResultsByKey([2, 99, 0, 98, 1, 97], collection1, ...args); 164 | const expected = [result1[2], null, result1[0], null, result1[1], null]; 165 | 166 | assert.deepEqual(actual, expected); 167 | }); 168 | 169 | it('no results', () => { 170 | const actual = getResultsByKey([99, 98, 97], collection1, ...args); 171 | const expected = [null, null, null]; 172 | 173 | assert.deepEqual(actual, expected); 174 | }); 175 | }); 176 | 177 | describe("test '[!]!'", () => { 178 | beforeEach(() => { 179 | args = ['id', '[!]!', { onError, isStrict: true }]; 180 | }); 181 | 182 | it('results in key order', () => { 183 | const actual = getResultsByKey([0, 1, 2], collection1, ...args); 184 | const expected = result1; 185 | 186 | assert.deepEqual(actual, expected); 187 | }); 188 | 189 | it('results in mixed order', () => { 190 | const actual = getResultsByKey([2, 0, 1], collection1, ...args); 191 | const expected = [result1[2], result1[0], result1[1]]; 192 | 193 | assert.deepEqual(actual, expected); 194 | }); 195 | 196 | it('results missing', () => { 197 | getResultsByKey([2, 99, 0, 98, 1, 97], collection1, ...args); 198 | 199 | assert(onErrorCalled, 'should not have succeeded'); 200 | }); 201 | }); 202 | 203 | describe('test isStrict: false', () => { 204 | beforeEach(() => { 205 | 206 | }); 207 | 208 | it("test '', no defaultElem", () => { 209 | const actual = getResultsByKey([2, 99, 0, 98, 1, 97], resultsObj, 'id', '', { onError }); 210 | const expected = [resultsObj[2], null, resultsObj[0], null, resultsObj[1], null]; 211 | 212 | assert.deepEqual(actual, expected); 213 | }); 214 | 215 | it("test '[]!, no defaultElem'", () => { 216 | const actual = getResultsByKey([99, 98, 97], collection1, 'id', '[]!', { onError }); 217 | const expected = [null, null, null]; 218 | 219 | assert.deepEqual(actual, expected); 220 | }); 221 | 222 | it("test '', defaultElem = []", () => { 223 | const actual = getResultsByKey([2, 99, 0, 98, 1, 97], resultsObj, 'id', '', { onError, defaultElem: [] }); 224 | const expected = [resultsObj[2], [], resultsObj[0], [], resultsObj[1], []]; 225 | 226 | assert.deepEqual(actual, expected); 227 | }); 228 | 229 | it("test '[]!, defaultElem = []'", () => { 230 | const actual = getResultsByKey([99, 98, 97], collection1, 'id', '[]!', { onError, defaultElem: [] }); 231 | const expected = [[], [], []]; 232 | 233 | assert.deepEqual(actual, expected); 234 | }); 235 | }); 236 | }); 237 | -------------------------------------------------------------------------------- /tests/get-unique-keys.test.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const { getUniqueKeys } = require('../lib'); 4 | 5 | describe('get-unique-keys.test.js', () => { 6 | it('handles 0 element array', () => { 7 | assert.deepEqual(getUniqueKeys([]), []); 8 | }); 9 | 10 | it('handles 1 element array', () => { 11 | assert.deepEqual(getUniqueKeys([1]), [1]); 12 | }); 13 | 14 | it('handles array with no duplicates', () => { 15 | assert.deepEqual(getUniqueKeys([1, 'a', 2, 'b']), [1, 'a', 2, 'b']); 16 | }); 17 | 18 | it('handles array with duplicates', () => { 19 | assert.deepEqual(getUniqueKeys([1, 'a', 2, 'b', 2, 1, 'a', 'b']), [1, 'a', 2, 'b']); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /tests/helpers/make-services.js: -------------------------------------------------------------------------------- 1 | 2 | const postsStore = [ 3 | { id: 1, body: 'John post', userId: 101, starIds: [102, 103, 104] }, 4 | { id: 2, body: 'Marshall post', userId: 102, starIds: [101, 103, 104] }, 5 | { id: 3, body: 'Barbara post', userId: 103 }, 6 | { id: 4, body: 'Aubree post', userId: 104 } 7 | ]; 8 | 9 | const commentsStore = [ 10 | { id: 11, text: 'John post Marshall comment 11', postId: 1, userId: 102 }, 11 | { id: 12, text: 'John post Marshall comment 12', postId: 1, userId: 102 }, 12 | { id: 13, text: 'John post Marshall comment 13', postId: 1, userId: 102 }, 13 | { id: 14, text: 'Marshall post John comment 14', postId: 2, userId: 101 }, 14 | { id: 15, text: 'Marshall post John comment 15', postId: 2, userId: 101 }, 15 | { id: 16, text: 'Barbara post John comment 16', postId: 3, userId: 101 }, 16 | { id: 17, text: 'Aubree post Marshall comment 17', postId: 4, userId: 102 } 17 | ]; 18 | 19 | const usersStore = [ 20 | { id: 101, name: 'John' }, 21 | { id: 102, name: 'Marshall' }, 22 | { id: 103, name: 'Barbara' }, 23 | { id: 104, name: 'Aubree' } 24 | ]; 25 | 26 | module.exports = { 27 | posts: makeService(postsStore, 'posts'), 28 | comments: makeService(commentsStore, 'comments'), 29 | users: makeService(usersStore, 'users') 30 | }; 31 | 32 | function makeService (store1, name) { 33 | return { 34 | get (id) { 35 | // console.log(`... ${name} get ${id}`); 36 | const store = clone(store1); 37 | 38 | for (let i = 0, leni = store.length; i < leni; i++) { 39 | if (store[i].id === id) return asyncReturn(store[i]); 40 | } 41 | 42 | throw Error(`post id ${id} not found`); 43 | }, 44 | 45 | find (params) { 46 | // console.log(`... ${name} find`, params ? params.query : ''); 47 | const store = clone(store1); 48 | 49 | if (!params || !params.query) return asyncReturn(store); 50 | 51 | const field = Object.keys(params.query)[0]; 52 | const value = params.query[field]; 53 | 54 | return asyncReturn(store.filter(post => { 55 | return typeof value !== 'object' 56 | ? post[field] === value 57 | : value.$in.indexOf(post[field]) !== -1; 58 | })); 59 | } 60 | }; 61 | } 62 | 63 | function asyncReturn (value) { 64 | return new Promise(resolve => { 65 | setTimeout(() => { resolve(value); }, 10); 66 | }); 67 | } 68 | 69 | function clone (obj) { 70 | return JSON.parse(JSON.stringify(obj)); 71 | } 72 | -------------------------------------------------------------------------------- /tests/loader-factory.test.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const { users } = require('./helpers/make-services'); 4 | const { loaderFactory } = require('../lib'); 5 | 6 | describe('loader-factory.test.js', () => { 7 | it('can load an entity', () => { 8 | const context = { _loaders: { user: {} } }; 9 | context._loaders.user.id = loaderFactory(users, 'id', false)(context); 10 | 11 | return context._loaders.user.id.load(101) 12 | .then((user) => { 13 | assert.deepEqual({ id: 101, name: 'John' }, user); 14 | }); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /tests/loader-large-populate.test.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const { map, parallel } = require('asyncro'); 4 | const BatchLoader = require('../lib'); 5 | const { posts, comments, users } = require('./helpers/make-services'); 6 | 7 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 8 | 9 | const result = [ 10 | { 11 | id: 1, 12 | body: 'John post', 13 | userId: 101, 14 | starIds: [102, 103, 104], 15 | userRecord: { id: 101, name: 'John' }, 16 | starUserRecords: 17 | [{ id: 102, name: 'Marshall' }, 18 | { id: 103, name: 'Barbara' }, 19 | { id: 104, name: 'Aubree' }], 20 | commentRecords: 21 | [{ 22 | id: 11, 23 | text: 'John post Marshall comment 11', 24 | postId: 1, 25 | userId: 102, 26 | userRecord: { id: 102, name: 'Marshall' } 27 | }, 28 | { 29 | id: 12, 30 | text: 'John post Marshall comment 12', 31 | postId: 1, 32 | userId: 102, 33 | userRecord: { id: 102, name: 'Marshall' } 34 | }, 35 | { 36 | id: 13, 37 | text: 'John post Marshall comment 13', 38 | postId: 1, 39 | userId: 102, 40 | userRecord: { id: 102, name: 'Marshall' } 41 | }] 42 | }, 43 | { 44 | id: 2, 45 | body: 'Marshall post', 46 | userId: 102, 47 | starIds: [101, 103, 104], 48 | userRecord: { id: 102, name: 'Marshall' }, 49 | starUserRecords: 50 | [{ id: 101, name: 'John' }, 51 | { id: 103, name: 'Barbara' }, 52 | { id: 104, name: 'Aubree' }], 53 | commentRecords: 54 | [{ 55 | id: 14, 56 | text: 'Marshall post John comment 14', 57 | postId: 2, 58 | userId: 101, 59 | userRecord: { id: 101, name: 'John' } 60 | }, 61 | { 62 | id: 15, 63 | text: 'Marshall post John comment 15', 64 | postId: 2, 65 | userId: 101, 66 | userRecord: { id: 101, name: 'John' } 67 | }] 68 | }, 69 | { 70 | id: 3, 71 | body: 'Barbara post', 72 | userId: 103, 73 | userRecord: { id: 103, name: 'Barbara' }, 74 | commentRecords: 75 | [{ 76 | id: 16, 77 | text: 'Barbara post John comment 16', 78 | postId: 3, 79 | userId: 101, 80 | userRecord: { id: 101, name: 'John' } 81 | }] 82 | }, 83 | { 84 | id: 4, 85 | body: 'Aubree post', 86 | userId: 104, 87 | userRecord: { id: 104, name: 'Aubree' }, 88 | commentRecords: 89 | [{ 90 | id: 17, 91 | text: 'Aubree post Marshall comment 17', 92 | postId: 4, 93 | userId: 102, 94 | userRecord: { id: 102, name: 'Marshall' } 95 | }] 96 | } 97 | ]; 98 | 99 | describe('loader-large-populate.js', () => { 100 | it('batch & cache', async () => { 101 | const data = await tester({}); 102 | assert.deepEqual(data, result); 103 | }); 104 | 105 | it('no batch & no cache', async () => { 106 | const data = await tester({ batch: false, cache: false }); 107 | assert.deepEqual(data, result); 108 | }); 109 | }); 110 | 111 | async function tester (options) { 112 | const commentsBatchLoader = new BatchLoader(async keys => { 113 | const result = await comments.find({ query: { postId: { $in: getUniqueKeys(keys) } } }); 114 | return getResultsByKey(keys, result, comment => comment.postId, '[]'); 115 | }, 116 | options 117 | ); 118 | 119 | const usersBatchLoader = new BatchLoader(async keys => { 120 | const result = await users.find({ query: { id: { $in: getUniqueKeys(keys) } } }); 121 | return getResultsByKey(keys, result, user => user.id, ''); 122 | }, 123 | options 124 | ); 125 | 126 | const postRecords = await posts.find(); 127 | 128 | await map(postRecords, async post => { 129 | await parallel([ 130 | // Join one users record to posts, for post.userId === users.id 131 | async () => { 132 | post.userRecord = await usersBatchLoader.load(post.userId); 133 | }, 134 | // Join 0, 1 or many comments records to posts, where comments.postId === posts.id 135 | async () => { 136 | const commentRecords = await commentsBatchLoader.load(post.id); 137 | post.commentRecords = commentRecords; 138 | 139 | // Join one users record to comments, for comments.userId === users.id 140 | await map(commentRecords, async comment => { 141 | comment.userRecord = await usersBatchLoader.load(comment.userId); 142 | }); 143 | }, 144 | // Join 0, 1 or many users record to posts, where posts.starIds === users.id 145 | async () => { 146 | if (!post.starIds) return null; 147 | 148 | post.starUserRecords = await usersBatchLoader.loadMany(post.starIds); 149 | } 150 | ]); 151 | }); 152 | 153 | return postRecords; 154 | } 155 | -------------------------------------------------------------------------------- /tests/loader-no-loader.test.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const { posts, comments } = require('./helpers/make-services'); 4 | 5 | const result = [{ 6 | id: 1, 7 | body: 'John post', 8 | userId: 101, 9 | starIds: [102, 103, 104], 10 | commentRecords: 11 | [{ 12 | id: 11, 13 | text: 'John post Marshall comment 11', 14 | postId: 1, 15 | userId: 102 16 | }, 17 | { 18 | id: 12, 19 | text: 'John post Marshall comment 12', 20 | postId: 1, 21 | userId: 102 22 | }, 23 | { 24 | id: 13, 25 | text: 'John post Marshall comment 13', 26 | postId: 1, 27 | userId: 102 28 | }] 29 | }, 30 | { 31 | id: 2, 32 | body: 'Marshall post', 33 | userId: 102, 34 | starIds: [101, 103, 104], 35 | commentRecords: 36 | [{ 37 | id: 14, 38 | text: 'Marshall post John comment 14', 39 | postId: 2, 40 | userId: 101 41 | }, 42 | { 43 | id: 15, 44 | text: 'Marshall post John comment 15', 45 | postId: 2, 46 | userId: 101 47 | }] 48 | }, 49 | { 50 | id: 3, 51 | body: 'Barbara post', 52 | userId: 103, 53 | commentRecords: 54 | [{ 55 | id: 16, 56 | text: 'Barbara post John comment 16', 57 | postId: 3, 58 | userId: 101 59 | }] 60 | }, 61 | { 62 | id: 4, 63 | body: 'Aubree post', 64 | userId: 104, 65 | commentRecords: 66 | [{ 67 | id: 17, 68 | text: 'Aubree post Marshall comment 17', 69 | postId: 4, 70 | userId: 102 71 | }] 72 | }]; 73 | 74 | describe('loader-no-loader.test.js', () => { 75 | it('using Promises', () => { 76 | return posts.find() 77 | .then(postsRecords => Promise.all( 78 | postsRecords.map(post => comments.find({ query: { postId: post.id } }) 79 | .then(comments => { 80 | post.commentRecords = comments; 81 | return post; 82 | }) 83 | ) 84 | )) 85 | 86 | .then(data => { 87 | assert.deepEqual(data, result); 88 | }); 89 | }); 90 | 91 | it('using async/await', async () => { 92 | const postsRecords = await posts.find(); 93 | 94 | const data = await Promise.all(postsRecords.map(async post => { 95 | post.commentRecords = await comments.find({ query: { postId: post.id } }); 96 | return post; 97 | })); 98 | 99 | assert.deepEqual(data, result); 100 | }); 101 | }); 102 | -------------------------------------------------------------------------------- /tests/loader-small-populate.test.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const BatchLoader = require('../lib'); 4 | const { posts, comments } = require('./helpers/make-services'); 5 | 6 | const { getResultsByKey, getUniqueKeys } = BatchLoader; 7 | 8 | const options = {}; // { batch: false, cache: false } 9 | 10 | const commentsLoaderPromises = new BatchLoader( 11 | keys => comments.find({ query: { postId: { $in: getUniqueKeys(keys) } } }) 12 | .then(result => getResultsByKey(keys, result, comment => comment.postId, '[]')), 13 | options 14 | ); 15 | 16 | const commentsLoaderAwait = new BatchLoader(async keys => { 17 | const postRecords = await comments.find({ query: { postId: { $in: getUniqueKeys(keys) } } }); 18 | return getResultsByKey(keys, postRecords, comment => comment.postId, '[]'); 19 | }, 20 | options 21 | ); 22 | 23 | const result = [ 24 | { 25 | id: 1, 26 | body: 'John post', 27 | userId: 101, 28 | starIds: [102, 103, 104], 29 | commentRecords: 30 | [{ 31 | id: 11, 32 | text: 'John post Marshall comment 11', 33 | postId: 1, 34 | userId: 102 35 | }, 36 | { 37 | id: 12, 38 | text: 'John post Marshall comment 12', 39 | postId: 1, 40 | userId: 102 41 | }, 42 | { 43 | id: 13, 44 | text: 'John post Marshall comment 13', 45 | postId: 1, 46 | userId: 102 47 | }] 48 | }, 49 | { 50 | id: 2, 51 | body: 'Marshall post', 52 | userId: 102, 53 | starIds: [101, 103, 104], 54 | commentRecords: 55 | [{ 56 | id: 14, 57 | text: 'Marshall post John comment 14', 58 | postId: 2, 59 | userId: 101 60 | }, 61 | { 62 | id: 15, 63 | text: 'Marshall post John comment 15', 64 | postId: 2, 65 | userId: 101 66 | }] 67 | }, 68 | { 69 | id: 3, 70 | body: 'Barbara post', 71 | userId: 103, 72 | commentRecords: 73 | [{ 74 | id: 16, 75 | text: 'Barbara post John comment 16', 76 | postId: 3, 77 | userId: 101 78 | }] 79 | }, 80 | { 81 | id: 4, 82 | body: 'Aubree post', 83 | userId: 104, 84 | commentRecords: 85 | [{ 86 | id: 17, 87 | text: 'Aubree post Marshall comment 17', 88 | postId: 4, 89 | userId: 102 90 | }] 91 | }]; 92 | 93 | describe('loader-small-populate.test.js', () => { 94 | it('using Promises', () => { 95 | return posts.find() 96 | .then(postRecords => Promise.all(postRecords.map(post => commentsLoaderPromises.load(post.id) 97 | .then(comments => { 98 | post.commentRecords = comments; 99 | return post; 100 | }) 101 | ))) 102 | .then(data => { 103 | assert.deepEqual(data, result); 104 | }); 105 | }); 106 | 107 | it('using async/await', async () => { 108 | const postRecords = await posts.find(); 109 | const data = await Promise.all(postRecords.map(async post => { 110 | post.commentRecords = await commentsLoaderAwait.load(post.id); 111 | return post; 112 | })); 113 | 114 | assert.deepEqual(data, result); 115 | }); 116 | 117 | it('null key', async () => { 118 | const returns = await commentsLoaderAwait.load(null); 119 | 120 | assert.strictEqual(returns, null); 121 | }); 122 | }); 123 | -------------------------------------------------------------------------------- /tests/make-services-await.test.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const { users } = require('./helpers/make-services'); 4 | 5 | describe('make-services-await.test.js', () => { 6 | it('run service calls', async () => { 7 | assert.deepEqual(await users.get(101), { id: 101, name: 'John' }, 'result1'); 8 | assert.deepEqual(await users.find({ query: { id: 101 } }), [{ id: 101, name: 'John' }], 'result1'); 9 | assert.deepEqual(await users.find({ query: { id: { $in: [101, 103, 104] } } }), [ 10 | { id: 101, name: 'John' }, 11 | { id: 103, name: 'Barbara' }, 12 | { id: 104, name: 'Aubree' } 13 | ], 'result1'); 14 | assert.deepEqual(await users.find(), [ 15 | { id: 101, name: 'John' }, 16 | { id: 102, name: 'Marshall' }, 17 | { id: 103, name: 'Barbara' }, 18 | { id: 104, name: 'Aubree' } 19 | ], 'result1'); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /tests/make-services.test.js: -------------------------------------------------------------------------------- 1 | 2 | const { assert } = require('chai'); 3 | const { users } = require('./helpers/make-services'); 4 | 5 | let result1, result2, result3; 6 | 7 | describe('make-services.test.js', () => { 8 | it('run service calls', () => { 9 | return users.get(101) 10 | .then(result => { 11 | result1 = result; 12 | 13 | return users.find({ query: { id: 101 } }); 14 | }) 15 | .then(result => { 16 | result2 = result; 17 | 18 | return users.find({ query: { id: { $in: [101, 103, 104] } } }); 19 | }) 20 | .then(result => { 21 | result3 = result; 22 | 23 | return users.find(); 24 | }) 25 | .then(result => { 26 | assert.deepEqual(result1, { id: 101, name: 'John' }, 'result1'); 27 | assert.deepEqual(result2, [{ id: 101, name: 'John' }], 'result1'); 28 | assert.deepEqual(result3, [ 29 | { id: 101, name: 'John' }, 30 | { id: 103, name: 'Barbara' }, 31 | { id: 104, name: 'Aubree' } 32 | ], 'result1'); 33 | assert.deepEqual(result, [ 34 | { id: 101, name: 'John' }, 35 | { id: 102, name: 'Marshall' }, 36 | { id: 103, name: 'Barbara' }, 37 | { id: 104, name: 'Aubree' } 38 | ], 'result1'); 39 | }) 40 | .catch(err => console.log(err)); 41 | }); 42 | }); 43 | -------------------------------------------------------------------------------- /types/index.d.ts: -------------------------------------------------------------------------------- 1 | // TypeScript Version: 2.3 2 | 3 | /* 4 | Forked from facebook/dataloader (src/index.d.ts) 5 | 6 | -- original license start -- 7 | 8 | BSD License 9 | 10 | For DataLoader software 11 | 12 | Copyright (c) 2015, Facebook, Inc. All rights reserved. 13 | 14 | Redistribution and use in source and binary forms, with or without modification, 15 | are permitted provided that the following conditions are met: 16 | 17 | * Redistributions of source code must retain the above copyright notice, this 18 | list of conditions and the following disclaimer. 19 | 20 | * Redistributions in binary form must reproduce the above copyright notice, 21 | this list of conditions and the following disclaimer in the documentation 22 | and/or other materials provided with the distribution. 23 | 24 | * Neither the name Facebook nor the names of its contributors may be used to 25 | endorse or promote products derived from this software without specific 26 | prior written permission. 27 | 28 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 29 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 30 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 31 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR 32 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 33 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 34 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 35 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 36 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 37 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 | 39 | -- original license end -- 40 | */ 41 | 42 | import { HookContext, Service } from "@feathersjs/feathers"; 43 | 44 | declare class BatchLoader { 45 | constructor( 46 | batchLoadFn: BatchLoader.BatchLoadFn, 47 | options?: BatchLoader.Options 48 | ); 49 | 50 | /** 51 | * Reorganizes the records from the service call into the result expected from the batch function. 52 | */ 53 | static getResultsByKey( 54 | keys: ReadonlyArray, 55 | resultArray: ReadonlyArray, 56 | serializeRecordKey: ((item: any) => string) | string, 57 | resultType: "" | "!" | "[]" | "[]!" | "[!]" | "[!]!", 58 | options?: BatchLoader.GetResultsByKeyOptions 59 | ): object[]; 60 | 61 | /** 62 | * Returns the unique elements in an array. 63 | */ 64 | static getUniqueKeys(keys: string[]): string[]; 65 | 66 | static loaderFactory( 67 | service: Service, 68 | id: string, 69 | multi?: boolean, 70 | options?: BatchLoader.LoaderFactoryOptions 71 | ): (context: C) => BatchLoader; 72 | 73 | /** 74 | * Loads a key, returning a `Promise` for the value represented by that key. 75 | */ 76 | load(key: K): Promise; 77 | 78 | /** 79 | * Loads multiple keys, promising an array of values: 80 | * 81 | * var [ a, b ] = await myLoader.loadMany([ 'a', 'b' ]); 82 | * 83 | * This is equivalent to the more verbose: 84 | * 85 | * var [ a, b ] = await Promise.all([ 86 | * myLoader.load('a'), 87 | * myLoader.load('b') 88 | * ]); 89 | * 90 | */ 91 | loadMany(keys: K[]): Promise; 92 | 93 | /** 94 | * Clears the value at `key` from the cache, if it exists. Returns itself for 95 | * method chaining. 96 | */ 97 | clear(key: K): BatchLoader; 98 | 99 | /** 100 | * Clears the entire cache. To be used when some event results in unknown 101 | * invalidations across this particular `BatchLoader`. Returns itself for 102 | * method chaining. 103 | */ 104 | clearAll(): BatchLoader; 105 | 106 | /** 107 | * Adds the provied key and value to the cache. If the key already exists, no 108 | * change is made. Returns itself for method chaining. 109 | */ 110 | prime(key: K, value: V): BatchLoader; 111 | } 112 | 113 | declare namespace BatchLoader { 114 | interface GetResultsByKeyOptions { 115 | onError: (i: number, message: string) => void; 116 | defaultElem: any; 117 | } 118 | 119 | interface LoaderFactoryOptions { 120 | getKey?: (item: any) => string | number; 121 | paramNames: string | string[]; 122 | injects: object; 123 | } 124 | 125 | // If a custom cache is provided, it must be of this type (a subset of ES6 Map). 126 | interface CacheMap { 127 | get(key: K): V | void; 128 | set(key: K, value: V): any; 129 | delete(key: K): any; 130 | clear(): any; 131 | } 132 | 133 | // A Function, which when given an Array of keys, returns a Promise of an Array 134 | // of values or Errors. 135 | type BatchLoadFn = ( 136 | keys: K[], 137 | context: C 138 | ) => Promise>; 139 | 140 | // Optionally turn off batching or caching or provide a cache key function or a 141 | // custom cache instance. 142 | interface Options { 143 | /** 144 | * Default `true`. Set to `false` to disable batching, 145 | * instead immediately invoking `batchLoadFn` with a 146 | * single load key. 147 | */ 148 | batch?: boolean; 149 | 150 | /** 151 | * Default `Infinity`. Limits the number of items that get 152 | * passed in to the `batchLoadFn`. 153 | */ 154 | maxBatchSize?: number; 155 | 156 | /** 157 | * Default `true`. Set to `false` to disable memoization caching, 158 | * instead creating a new Promise and new key in the `batchLoadFn` for every 159 | * load of the same key. 160 | */ 161 | cache?: boolean; 162 | 163 | /** 164 | * A function to produce a cache key for a given load key. 165 | * Defaults to `key => key`. Useful to provide when JavaScript 166 | * objects are keys and two similarly shaped objects should 167 | * be considered equivalent. 168 | */ 169 | cacheKeyFn?: (key: any) => any; 170 | 171 | /** 172 | * An instance of Map (or an object with a similar API) to 173 | * be used as the underlying cache for this loader. 174 | * Default `new Map()`. 175 | */ 176 | cacheMap?: CacheMap>; 177 | 178 | /** 179 | * A feathers hook context object 180 | */ 181 | context?: C; 182 | } 183 | } 184 | 185 | export = BatchLoader; 186 | -------------------------------------------------------------------------------- /types/tests.ts: -------------------------------------------------------------------------------- 1 | import BatchLoader = require('@feathers-plus/batch-loader'); 2 | import { Application, Service } from '@feathersjs/feathers'; 3 | 4 | const app: Application = null as any; 5 | 6 | const myContext = { hello: 'world' }; 7 | 8 | new BatchLoader((keys, context) => { 9 | // $ExpectType { hello: string; } 10 | context; 11 | return app.service('users').find({ query: { id: { $in: keys } } }) 12 | .then(records => { 13 | return []; 14 | }); 15 | }, 16 | { context: myContext } 17 | ); 18 | 19 | // $ExpectType BatchLoader 20 | BatchLoader.loaderFactory(app.service('users'), 'id')(myContext); 21 | -------------------------------------------------------------------------------- /types/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "lib": ["es6"], 5 | "noImplicitAny": true, 6 | "noImplicitThis": true, 7 | "strictNullChecks": true, 8 | "strictFunctionTypes": true, 9 | "noEmit": true, 10 | "baseUrl": "./", 11 | "paths": { 12 | "@feathers-plus/batch-loader": [ 13 | "." 14 | ] 15 | } 16 | } 17 | } --------------------------------------------------------------------------------