├── .ackrc ├── .gitignore ├── .npmignore ├── .travis.yml ├── .yarnrc ├── LICENSE ├── README.md ├── jest.config.js ├── package.json ├── prettier.config.js ├── src ├── batchloader.test.ts ├── batchloader.ts ├── cacheloader.test.ts ├── cacheloader.ts ├── cacheproxyloader.test.ts ├── cacheproxyloader.ts ├── index.ts ├── mappedbatchloader.test.ts ├── mappedbatchloader.ts └── types.ts ├── tsconfig.browser.json ├── tsconfig.build.json ├── tsconfig.json ├── tsconfig.module.json ├── tsconfig.node.json ├── tslint.json └── yarn.lock /.ackrc: -------------------------------------------------------------------------------- 1 | --ignore-dir=.git 2 | --ignore-dir=.gradle 3 | --ignore-dir=.idea 4 | --ignore-dir=.rpt2_cache 5 | --ignore-dir=__generated__ 6 | --ignore-dir=build 7 | --ignore-dir=cjs 8 | --ignore-dir=dist 9 | --ignore-dir=es5 10 | --ignore-dir=es6 11 | --ignore-dir=lib 12 | --ignore-dir=node_modules 13 | --ignore-file=match:.DS_Store 14 | --ignore-file=match:.ackrc 15 | --ignore-file=match:.babelrc 16 | --ignore-file=match:.buckconfig 17 | --ignore-file=match:.eslintignore 18 | --ignore-file=match:.eslintrc 19 | --ignore-file=match:.flowconfig 20 | --ignore-file=match:.gitattributes 21 | --ignore-file=match:.gitignore 22 | --ignore-file=match:.iml 23 | --ignore-file=match:.npmignore 24 | --ignore-file=match:.watchmanconfig 25 | --ignore-file=match:Podfile.lock 26 | --ignore-file=match:npm-debug.log 27 | --ignore-file=match:package-lock.json 28 | --ignore-file=match:yarn.lock 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/vue,vim,git,node,code,nuxt,xcode,vuejs,linux,macos,kotlin,windows,android,angular,flutter,webstorm,firebase,reactnative,objective-c,androidstudio 2 | !**/ios/**/default.mode1v3 3 | !**/ios/**/default.mode2v3 4 | !**/ios/**/default.pbxuser 5 | !**/ios/**/default.perspectivev3 6 | !*.xcodeproj/project.pbxproj 7 | !*.xcodeproj/xcshareddata/ 8 | !*.xcworkspace/contents.xcworkspacedata 9 | !.vscode/extensions.json 10 | !.vscode/launch.json 11 | !.vscode/settings.json 12 | !.vscode/tasks.json 13 | !/gradle/wrapper/gradle-wrapper.jar 14 | !/packages/flutter_tools/test/data/dart_dependencies_test/**/.packages 15 | !default.mode1v3 16 | !default.mode2v3 17 | !default.pbxuser 18 | !default.perspectivev3 19 | !gradle-wrapper.jar 20 | $RECYCLE.BIN/ 21 | **/android/**/GeneratedPluginRegistrant.java 22 | **/android/**/gradle-wrapper.jar 23 | **/android/.gradle 24 | **/android/captures/ 25 | **/android/gradlew 26 | **/android/gradlew.bat 27 | **/android/local.properties 28 | **/doc/api/ 29 | **/ios/**/*.mode1v3 30 | **/ios/**/*.mode2v3 31 | **/ios/**/*.moved-aside 32 | **/ios/**/*.pbxuser 33 | **/ios/**/*.perspectivev3 34 | **/ios/**/*sync/ 35 | **/ios/**/.sconsign.dblite 36 | **/ios/**/.symlinks/ 37 | **/ios/**/.tags* 38 | **/ios/**/.vagrant/ 39 | **/ios/**/DerivedData/ 40 | **/ios/**/Icon? 41 | **/ios/**/Pods/ 42 | **/ios/**/profile 43 | **/ios/**/xcuserdata 44 | **/ios/.generated/ 45 | **/ios/Flutter/App.framework 46 | **/ios/Flutter/Flutter.framework 47 | **/ios/Flutter/Generated.xcconfig 48 | **/ios/Flutter/app.flx 49 | **/ios/Flutter/app.zip 50 | **/ios/Flutter/flutter_assets/ 51 | **/ios/Runner/GeneratedPluginRegistrant.* 52 | **/ios/ServiceDefinitions.json 53 | **/xcshareddata/WorkspaceSettings.xcsettings 54 | *-debug.log* 55 | *-error.log* 56 | *.BACKUP.* 57 | *.BASE.* 58 | *.LOCAL.* 59 | *.REMOTE.* 60 | *.aab 61 | *.ap_ 62 | *.apk 63 | *.class 64 | *.ctxt 65 | *.dSYM 66 | *.dSYM.zip 67 | *.dex 68 | *.ear 69 | *.hmap 70 | *.iml 71 | *.ipa 72 | *.ipr 73 | *.iws 74 | *.jar 75 | *.jks 76 | *.keystore 77 | *.lcov 78 | *.lnk 79 | *.log 80 | *.mode1v3 81 | *.mode2v3 82 | *.moved-aside 83 | *.nar 84 | *.orig 85 | *.pbxuser 86 | *.perspectivev3 87 | *.pid 88 | *.pid.lock 89 | *.rar 90 | *.rs.bk 91 | *.seed 92 | *.stackdump 93 | *.swp 94 | *.tar.gz 95 | *.tgz 96 | *.tsbuildinfo 97 | *.war 98 | *.xccheckout 99 | *.xcodeproj/* 100 | *.xcscmblueprint 101 | *_BACKUP_*.txt 102 | *_BASE_*.txt 103 | *_LOCAL_*.txt 104 | *_REMOTE_*.txt 105 | *google-services.json 106 | *secrets* 107 | *~ 108 | .AppleDB 109 | .AppleDesktop 110 | .AppleDouble 111 | .DS_Store* 112 | .DocumentRevisions-V100 113 | .LSOverride 114 | .Spotlight-V100 115 | .TemporaryItems 116 | .Trash-* 117 | .Trashes 118 | .VolumeIcon.icns 119 | ._* 120 | .apdisk 121 | .buckconfig.local 122 | .buckd/ 123 | .buckversion 124 | .cache 125 | .cargo-ok 126 | .classpath 127 | .com.apple.timemachine.donotpresent 128 | .cproject 129 | .dart_tool/ 130 | .directory 131 | .dynamodb/ 132 | .env 133 | .env.* 134 | .eslintcache 135 | .expo 136 | .externalNativeBuild 137 | .fakebuckversion 138 | .firebase 139 | .firebaserc 140 | .flutter-plugins 141 | .flutter-plugins 142 | .fseventsd 143 | .fuse_hidden* 144 | .fusebox/ 145 | .gradle/ 146 | .gradletasknamecache 147 | .grunt 148 | .idea 149 | .idea_modules/ 150 | .lock-wscript 151 | .mtj.tmp/ 152 | .navigation/ 153 | .netrwhist 154 | .next 155 | .nfs* 156 | .node_repl_history 157 | .npm 158 | .npmrc 159 | .nuxt 160 | .nyc_output 161 | .packages 162 | .project 163 | .pub-cache/ 164 | .pub/ 165 | .rpt2_cache 166 | .runtimeconfig.json 167 | .sass-cache 168 | .serverless/ 169 | .settings/ 170 | .signing/ 171 | .tern-port 172 | .vscode/* 173 | .vuepress/dist 174 | .yarn-integrity 175 | /*.gcno 176 | Cargo.lock 177 | Carthage/Build 178 | DerivedData/ 179 | Icon 180 | Network Trash Folder 181 | Release/ 182 | Session.vim 183 | Sessionx.vim 184 | Temporary Items 185 | Thumbs.db 186 | Thumbs.db:encryptable 187 | [._]*.s[a-v][a-z] 188 | [._]*.sw[a-p] 189 | [._]*.un~ 190 | [._]s[a-rt-v][a-z] 191 | [._]ss[a-gi-z] 192 | [._]sw[a-p] 193 | [Dd]esktop.ini 194 | __generated__ 195 | atlassian-ide-plugin.xml 196 | bin/ 197 | bower_components 198 | buck-out/ 199 | build/ 200 | captures/ 201 | cmake-build-*/ 202 | com_crashlytics_export_strings.xml 203 | connect.lock 204 | coverage/ 205 | crashlytics-build.properties 206 | crashlytics.properties 207 | dist/ 208 | docs/_book 209 | ehthumbs.db 210 | ehthumbs_vista.db 211 | fabric.properties 212 | fastlane/Preview.html 213 | fastlane/readme.md 214 | fastlane/report.xml 215 | fastlane/screenshots 216 | fastlane/screenshots/**/*.png 217 | fastlane/test_output 218 | freeline.py 219 | freeline/ 220 | freeline_project_description.json 221 | gen-external-apklibs 222 | gen/ 223 | gradle-app.setting 224 | hs_err_pid* 225 | iOSInjectionProject/ 226 | jspm_packages/ 227 | lib-cov 228 | lint/generated/ 229 | lint/intermediates/ 230 | lint/outputs/ 231 | lint/tmp/ 232 | local.properties 233 | logs 234 | node_modules 235 | now-secrets.json 236 | out/ 237 | output.json 238 | pids 239 | proguard/ 240 | release/ 241 | report.*.json 242 | tmp/ 243 | vcs.xml 244 | xcuserdata/ 245 | # End 246 | 247 | lib 248 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io/api/vue,vim,git,node,code,nuxt,xcode,vuejs,linux,macos,kotlin,windows,android,angular,flutter,webstorm,firebase,reactnative,objective-c,androidstudio 2 | !**/ios/**/default.mode1v3 3 | !**/ios/**/default.mode2v3 4 | !**/ios/**/default.pbxuser 5 | !**/ios/**/default.perspectivev3 6 | !*.xcodeproj/project.pbxproj 7 | !*.xcodeproj/xcshareddata/ 8 | !*.xcworkspace/contents.xcworkspacedata 9 | !.vscode/extensions.json 10 | !.vscode/launch.json 11 | !.vscode/settings.json 12 | !.vscode/tasks.json 13 | !/gradle/wrapper/gradle-wrapper.jar 14 | !/packages/flutter_tools/test/data/dart_dependencies_test/**/.packages 15 | !default.mode1v3 16 | !default.mode2v3 17 | !default.pbxuser 18 | !default.perspectivev3 19 | !gradle-wrapper.jar 20 | $RECYCLE.BIN/ 21 | **/android/**/GeneratedPluginRegistrant.java 22 | **/android/**/gradle-wrapper.jar 23 | **/android/.gradle 24 | **/android/captures/ 25 | **/android/gradlew 26 | **/android/gradlew.bat 27 | **/android/local.properties 28 | **/doc/api/ 29 | **/ios/**/*.mode1v3 30 | **/ios/**/*.mode2v3 31 | **/ios/**/*.moved-aside 32 | **/ios/**/*.pbxuser 33 | **/ios/**/*.perspectivev3 34 | **/ios/**/*sync/ 35 | **/ios/**/.sconsign.dblite 36 | **/ios/**/.symlinks/ 37 | **/ios/**/.tags* 38 | **/ios/**/.vagrant/ 39 | **/ios/**/DerivedData/ 40 | **/ios/**/Icon? 41 | **/ios/**/Pods/ 42 | **/ios/**/profile 43 | **/ios/**/xcuserdata 44 | **/ios/.generated/ 45 | **/ios/Flutter/App.framework 46 | **/ios/Flutter/Flutter.framework 47 | **/ios/Flutter/Generated.xcconfig 48 | **/ios/Flutter/app.flx 49 | **/ios/Flutter/app.zip 50 | **/ios/Flutter/flutter_assets/ 51 | **/ios/Runner/GeneratedPluginRegistrant.* 52 | **/ios/ServiceDefinitions.json 53 | **/xcshareddata/WorkspaceSettings.xcsettings 54 | *-debug.log* 55 | *-error.log* 56 | *.BACKUP.* 57 | *.BASE.* 58 | *.LOCAL.* 59 | *.REMOTE.* 60 | *.aab 61 | *.ap_ 62 | *.apk 63 | *.class 64 | *.ctxt 65 | *.dSYM 66 | *.dSYM.zip 67 | *.dex 68 | *.ear 69 | *.hmap 70 | *.iml 71 | *.ipa 72 | *.ipr 73 | *.iws 74 | *.jar 75 | *.jks 76 | *.keystore 77 | *.lcov 78 | *.lnk 79 | *.log 80 | *.mode1v3 81 | *.mode2v3 82 | *.moved-aside 83 | *.nar 84 | *.orig 85 | *.pbxuser 86 | *.perspectivev3 87 | *.pid 88 | *.pid.lock 89 | *.rar 90 | *.rs.bk 91 | *.seed 92 | *.stackdump 93 | *.swp 94 | *.tar.gz 95 | *.tgz 96 | *.tsbuildinfo 97 | *.war 98 | *.xccheckout 99 | *.xcodeproj/* 100 | *.xcscmblueprint 101 | *_BACKUP_*.txt 102 | *_BASE_*.txt 103 | *_LOCAL_*.txt 104 | *_REMOTE_*.txt 105 | *google-services.json 106 | *secrets* 107 | *~ 108 | .AppleDB 109 | .AppleDesktop 110 | .AppleDouble 111 | .DS_Store* 112 | .DocumentRevisions-V100 113 | .LSOverride 114 | .Spotlight-V100 115 | .TemporaryItems 116 | .Trash-* 117 | .Trashes 118 | .VolumeIcon.icns 119 | ._* 120 | .apdisk 121 | .buckconfig.local 122 | .buckd/ 123 | .buckversion 124 | .cache 125 | .cargo-ok 126 | .classpath 127 | .com.apple.timemachine.donotpresent 128 | .cproject 129 | .dart_tool/ 130 | .directory 131 | .dynamodb/ 132 | .env 133 | .env.* 134 | .eslintcache 135 | .expo 136 | .externalNativeBuild 137 | .fakebuckversion 138 | .firebase 139 | .firebaserc 140 | .flutter-plugins 141 | .flutter-plugins 142 | .fseventsd 143 | .fuse_hidden* 144 | .fusebox/ 145 | .gradle/ 146 | .gradletasknamecache 147 | .grunt 148 | .idea 149 | .idea_modules/ 150 | .lock-wscript 151 | .mtj.tmp/ 152 | .navigation/ 153 | .netrwhist 154 | .next 155 | .nfs* 156 | .node_repl_history 157 | .npm 158 | .npmrc 159 | .nuxt 160 | .nyc_output 161 | .packages 162 | .project 163 | .pub-cache/ 164 | .pub/ 165 | .rpt2_cache 166 | .runtimeconfig.json 167 | .sass-cache 168 | .serverless/ 169 | .settings/ 170 | .signing/ 171 | .tern-port 172 | .vscode/* 173 | .vuepress/dist 174 | .yarn-integrity 175 | /*.gcno 176 | Cargo.lock 177 | Carthage/Build 178 | DerivedData/ 179 | Icon 180 | Network Trash Folder 181 | Release/ 182 | Session.vim 183 | Sessionx.vim 184 | Temporary Items 185 | Thumbs.db 186 | Thumbs.db:encryptable 187 | [._]*.s[a-v][a-z] 188 | [._]*.sw[a-p] 189 | [._]*.un~ 190 | [._]s[a-rt-v][a-z] 191 | [._]ss[a-gi-z] 192 | [._]sw[a-p] 193 | [Dd]esktop.ini 194 | __generated__ 195 | atlassian-ide-plugin.xml 196 | bin/ 197 | bower_components 198 | buck-out/ 199 | build/ 200 | captures/ 201 | cmake-build-*/ 202 | com_crashlytics_export_strings.xml 203 | connect.lock 204 | coverage/ 205 | crashlytics-build.properties 206 | crashlytics.properties 207 | dist/ 208 | docs/_book 209 | ehthumbs.db 210 | ehthumbs_vista.db 211 | fabric.properties 212 | fastlane/Preview.html 213 | fastlane/readme.md 214 | fastlane/report.xml 215 | fastlane/screenshots 216 | fastlane/screenshots/**/*.png 217 | fastlane/test_output 218 | freeline.py 219 | freeline/ 220 | freeline_project_description.json 221 | gen-external-apklibs 222 | gen/ 223 | gradle-app.setting 224 | hs_err_pid* 225 | iOSInjectionProject/ 226 | jspm_packages/ 227 | lib-cov 228 | lint/generated/ 229 | lint/intermediates/ 230 | lint/outputs/ 231 | lint/tmp/ 232 | local.properties 233 | logs 234 | node_modules 235 | now-secrets.json 236 | out/ 237 | output.json 238 | pids 239 | proguard/ 240 | release/ 241 | report.*.json 242 | tmp/ 243 | vcs.xml 244 | xcuserdata/ 245 | # End 246 | 247 | # Additional to .gitignore 248 | !dist/ 249 | !out/ 250 | *.config.js 251 | *.config.ts 252 | *.md 253 | .*ignore 254 | .*rc 255 | .gitignore 256 | .travis.yml 257 | now*.json 258 | src 259 | tsconfig*.json 260 | tslint.json 261 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | 3 | node_js: 4 | - "node" 5 | - "lts/*" 6 | - "12" 7 | - "11" 8 | - "10" 9 | - "9" 10 | - "8" 11 | 12 | after_success: yarn test:coverage:report 13 | -------------------------------------------------------------------------------- /.yarnrc: -------------------------------------------------------------------------------- 1 | --install.ignore-engines true 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Joon Ho Cho 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BatchLoader 2 | BatchLoader is a batching utility for data fetching layer to reduce requests round trips, inspired by [Facebook's DataLoader](https://github.com/facebook/dataloader), written in [TypeScript](https://www.typescriptlang.org/index.html). 3 | 4 | BatchLoader is a simplified version of Facebook's DataLoader and can be used with any database such as MongoDB or with GraphQL. 5 | 6 | [![Build Status](https://travis-ci.org/joonhocho/batchloader.svg?branch=master)](https://travis-ci.org/joonhocho/batchloader) 7 | [![Coverage Status](https://coveralls.io/repos/github/joonhocho/batchloader/badge.svg?branch=master)](https://coveralls.io/github/joonhocho/batchloader?branch=master) 8 | [![npm version](https://badge.fury.io/js/batchloader.svg)](https://badge.fury.io/js/batchloader) 9 | [![Dependency Status](https://david-dm.org/joonhocho/batchloader.svg)](https://david-dm.org/joonhocho/batchloader) 10 | [![License](http://img.shields.io/:license-mit-blue.svg)](http://doge.mit-license.org) 11 | 12 | ## Comparison to DataLoader 13 | \+ written in TypeScript 14 | 15 | \+ Further reduces data fetching requests by filtering out duplicate keys 16 | 17 | \+ Similar api as DataLoader 18 | 19 | \+ Smaller in size (0 dependencies) 20 | 21 | \+ MappedBatchLoader can be used to compose a new loader using existing loaders. 22 | 23 | \- Removed caching functionalities. Leave caching to better cache libraries. 24 | 25 | It is a very simple batcher that only does batching, and it does it very well. 26 | 27 | ## Getting Started 28 | 29 | First, install BatchLoader using npm. 30 | 31 | ```sh 32 | npm install --save batchloader 33 | ``` 34 | or with Yarn, 35 | ```sh 36 | yarn add batchloader 37 | ``` 38 | 39 | > Note: BatchLoader assumes a JavaScript environment with global ES6 `Promise`, available in all supported versions of Node.js. 40 | 41 | 42 | ## Batching 43 | 44 | Create loaders by providing a batch loading function and key transformation function (used for finding duplicate keys). 45 | 46 | ```typescript 47 | import { BatchLoader } from 'batchloader'; 48 | 49 | const userLoader = new BatchLoader( 50 | (_ids: ObjectId[]) => User.getByIds(_ids), // [required] batch function. 51 | (_id: ObjectId) => _id.toString(), // [optional] key to unique id function. must return string. used for finding duplicate keys. 52 | 100 // [optional = 0] batch delay in ms. default 0 ms. 53 | ); 54 | 55 | const user1 = await userLoader.load(id1); 56 | 57 | const [user1, user2] = await userLoader.loadMany([id1, id2]); 58 | 59 | const [user1, user1, user1] = await userLoader.loadMany([id1, id1, id1]); // batch function receives only one id1 since duplicate ids. Still returs three items just as requested. 60 | 61 | const [user1, user2, user3, user2, user1] = await Promise.all([ 62 | userLoader.load(id1), 63 | userLoader.load(id2), 64 | userLoader.load(id3), 65 | userLoader.load(id2), 66 | userLoader.load(id1), 67 | ]); // batch function receives [id1, id2, id3] only without duplicate ids. 68 | ``` 69 | 70 | #### Batch Function 71 | 72 | A batch loading function must be of the following type: 73 | ```typescript 74 | (keys: Key[]) => Value[] | Promise // keys.length === values.length 75 | ``` 76 | Constraints 77 | - keys.length === values.length 78 | - keys[i] => values[i] 79 | - keys.length > 0 80 | 81 | #### KeyToUniqueId Function 82 | 83 | A function must return string value given a key: 84 | ```typescript 85 | (key: Key) => string 86 | ``` 87 | 88 | If key is not uniquely identifiable, simply pass `null` instead. This will disable filtering out duplicate keys, and still work the same way. 89 | ```typescript 90 | const loader = new BatchLoader( 91 | (keys: Key[]) => loadValues(keys), 92 | null // keys cannot be transformed into string. no duplicates filtering. 93 | ); 94 | const v1 = await loader.load(k1); 95 | const [v1, v2, v1] = await loader.loadMany([k1, k2, k1]); // batch function receives [k1, k2, k1] as keys 96 | ``` 97 | 98 | ## MappedBatchLoader 99 | 100 | You can map a loader to create another loader. 101 | ```typescript 102 | import { MappedBatchLoader } from 'batchloader'; 103 | 104 | const getUsername = (user) => user && user.username; 105 | 106 | const usernameLoader = userLoader.mapLoader(getUsername); 107 | // or 108 | const usernameLoader = new MappedBatchLoader(userLoader, getUsername); 109 | 110 | // same APIs as BatchLoader 111 | const username = await usernameLoader.load(userId); 112 | const [username1, username2] = await usernameLoader.loadMany([userId1, userId2]); 113 | const [user1, username1] = await Promise.all([ 114 | userLoader.load(id1), 115 | usernameLoader.load(id1), 116 | ]) // one round-trip request with keys being [id1], since usernameLoader is using userLoader internally and id1 is duplicate. 117 | 118 | const anotherMappedLoader = usernameLoader.mapLoader(mapFn); 119 | // or 120 | const anotherMappedLoader = new MappedBatchLoader(usernameLoader, mapFn); 121 | ``` 122 | 123 | ## Caching 124 | Unlike DataLoader, BatchLoader does not do any caching. 125 | This is intentional, because you may want to use your favorite cache library that is best suited for your own use case. 126 | You can add caching ability easily like so: 127 | 128 | ```typescript 129 | let userCache = {}; 130 | 131 | const cachedUserLoader = new BatchLoader( 132 | async (ids) => { 133 | const uncachedIds = ids.filter(id => !userCache[id]); 134 | const users = await getUsersByIds(uncachedIds); 135 | uncachedIds.forEach((id, i) => { userCache[id] = users[i]; }); 136 | return ids.map(id => userCache[id]); 137 | }, 138 | ... 139 | ); 140 | 141 | delete userCache[id1]; // delete cache by key 142 | userCache[id2] = user2; // add cache by key 143 | userCache = {}; // empty cache 144 | ``` 145 | Choose whatever caching library you like and simply add it like above. 146 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | globals: { 3 | 'ts-jest': { 4 | tsConfig: 'tsconfig.json', 5 | }, 6 | }, 7 | moduleFileExtensions: ['js', 'jsx', 'ts', 'tsx', 'json'], 8 | moduleNameMapper: { 9 | '^_src/(.*)': '/src/$1', 10 | }, 11 | testEnvironment: 'node', 12 | testRegex: '(/__tests__/.*|(\\.|/)(test|spec))\\.(t|j)sx?$', 13 | transform: { 14 | '^.+\\.tsx?$': 'ts-jest', 15 | }, 16 | transformIgnorePatterns: [], 17 | preset: 'ts-jest', 18 | testMatch: null, 19 | }; 20 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "batchloader", 3 | "version": "0.0.22", 4 | "description": "BatchLoader is a utility for data fetching layer to reduce requests via batching written in TypeScript. Inspired by Facebook's DataLoader", 5 | "keywords": [ 6 | "TypeScript", 7 | "batch", 8 | "batchloader", 9 | "dataloader" 10 | ], 11 | "author": "Joon Ho Cho", 12 | "license": "MIT", 13 | "homepage": "https://github.com/joonhocho/batchloader#readme", 14 | "repository": { 15 | "type": "git", 16 | "url": "git+https://github.com/joonhocho/batchloader.git" 17 | }, 18 | "bugs": { 19 | "url": "https://github.com/joonhocho/batchloader/issues" 20 | }, 21 | "module": "lib/index.js", 22 | "main": "dist/node/index.js", 23 | "browser": "dist/browser/index.js", 24 | "types": "lib/index.d.ts", 25 | "sideEffects": false, 26 | "scripts": { 27 | "all": "npm run clean && npm run format && npm run lint:fix && npm run build:all && npm run test", 28 | "build:all": "npm run build:module && npm run build:node && npm run build:browser", 29 | "build:browser": "tsc -p ./tsconfig.browser.json && tscpaths -p ./tsconfig.browser.json -s ./src -o ./dist/browser", 30 | "build:module": "tsc -p ./tsconfig.module.json && tscpaths -p ./tsconfig.module.json -s ./src -o ./lib", 31 | "build:node": "tsc -p ./tsconfig.node.json && tscpaths -p ./tsconfig.node.json -s ./src -o ./dist/node", 32 | "clean": "rm -rf ./lib ./dist ./coverage", 33 | "format": "prettier --write \"./*.{js,jsx,ts,tsx}\" \"./src/**/*.{js,jsx,ts,tsx}\"", 34 | "lint": "tslint -c ./tslint.json \"src/**/*.ts\"", 35 | "lint:fix": "tslint --fix -c ./tslint.json \"src/**/*.ts\"", 36 | "precommit": "npm run all", 37 | "prepublishOnly": "npm run all", 38 | "reinstall": "rm -rf ./node_modules ./package-lock.json ./yarn.lock && yarn", 39 | "start": "npm run test", 40 | "test": "jest", 41 | "test:coverage": "jest --coverage", 42 | "test:coverage:report": "jest --coverage && cat ./coverage/lcov.info | coveralls", 43 | "test:watch": "jest --watch" 44 | }, 45 | "pre-commit": "precommit", 46 | "peerDependencies": { 47 | "tslib": "^1.10.0" 48 | }, 49 | "devDependencies": { 50 | "@types/jest": "^24.0.20", 51 | "@types/node": "^12.11.7", 52 | "coveralls": "^3.0.7", 53 | "jest": "^24.9.0", 54 | "pre-commit": "^1.2.2", 55 | "prettier": "^1.18.2", 56 | "ts-jest": "^24.1.0", 57 | "tscpaths": "^0.0.9", 58 | "tslint": "^5.20.0", 59 | "tslint-config-airbnb": "^5.11.2", 60 | "tslint-config-prettier": "^1.18.0", 61 | "typescript": "^3.6.4" 62 | }, 63 | "engines": { 64 | "node": ">=8.0.0" 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /prettier.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | arrowParens: 'always', 3 | bracketSpacing: true, 4 | jsxBracketSameLine: false, 5 | singleQuote: true, 6 | trailingComma: 'es5', 7 | }; 8 | -------------------------------------------------------------------------------- /src/batchloader.test.ts: -------------------------------------------------------------------------------- 1 | import { BatchLoader } from './batchloader'; 2 | 3 | describe('BatchLoader', () => { 4 | test('with keyToUniqueId', async () => { 5 | const idss = [] as number[][]; 6 | const loader = new BatchLoader( 7 | (ids: number[]): Promise => 8 | new Promise((resolve): void => { 9 | idss.push(ids); 10 | setTimeout(() => resolve(ids.map((i) => i * 2)), 10); 11 | }), 12 | String 13 | ); 14 | 15 | expect(await loader.load(3)).toBe(6); 16 | expect(await loader.load(4)).toBe(8); 17 | expect(await loader.load(5)).toBe(10); 18 | 19 | expect(await loader.loadMany([])).toEqual([]); 20 | expect(await loader.loadMany([])).toEqual([]); 21 | expect(await loader.loadMany([1, 2, 3])).toEqual([2, 4, 6]); 22 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual([ 23 | 2, 24 | 4, 25 | 6, 26 | 4, 27 | 6, 28 | 4, 29 | 2, 30 | ]); 31 | expect(await loader.loadMany([])).toEqual([]); 32 | 33 | expect( 34 | await Promise.all([ 35 | loader.load(1), 36 | loader.load(2), 37 | loader.load(3), 38 | loader.load(2), 39 | loader.load(1), 40 | loader.load(2), 41 | loader.load(3), 42 | ]) 43 | ).toEqual([2, 4, 6, 4, 2, 4, 6]); 44 | 45 | expect(idss).toEqual([[3], [4], [5], [1, 2, 3], [1, 2, 3], [1, 2, 3]]); 46 | }); 47 | 48 | test('without keyToUniqueId', async () => { 49 | const idss = [] as number[][]; 50 | const loader = new BatchLoader( 51 | (ids: number[]): Promise => 52 | new Promise((resolve): void => { 53 | idss.push(ids); 54 | setTimeout(() => resolve(ids.map((i) => i * 2)), 10); 55 | }), 56 | null 57 | ); 58 | 59 | expect(await loader.load(3)).toBe(6); 60 | expect(await loader.load(4)).toBe(8); 61 | expect(await loader.load(5)).toBe(10); 62 | 63 | expect(await loader.loadMany([])).toEqual([]); 64 | expect(await loader.loadMany([1, 2, 3])).toEqual([2, 4, 6]); 65 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual([ 66 | 2, 67 | 4, 68 | 6, 69 | 4, 70 | 6, 71 | 4, 72 | 2, 73 | ]); 74 | 75 | expect( 76 | await Promise.all([ 77 | loader.load(1), 78 | loader.load(2), 79 | loader.load(3), 80 | loader.load(2), 81 | loader.load(1), 82 | loader.load(2), 83 | loader.load(3), 84 | ]) 85 | ).toEqual([2, 4, 6, 4, 2, 4, 6]); 86 | 87 | expect(idss).toEqual([ 88 | [3], 89 | [4], 90 | [5], 91 | [1, 2, 3], 92 | [1, 2, 3, 2, 3, 2, 1], 93 | [1, 2, 3, 2, 1, 2, 3], 94 | ]); 95 | }); 96 | 97 | test('batchSize', async () => { 98 | const idss = [] as number[][]; 99 | const loader = new BatchLoader( 100 | (ids: number[]): Promise => 101 | new Promise((resolve): void => { 102 | idss.push(ids); 103 | setTimeout(() => resolve(ids.map((i) => i * 2)), 10); 104 | }), 105 | String, 106 | 10, 107 | 2 108 | ); 109 | 110 | expect( 111 | await Promise.all([ 112 | loader.load(1), 113 | loader.load(2), 114 | loader.loadMany([3, 4, 5]), 115 | loader.load(6), 116 | loader.loadMany([7, 8]), 117 | ]) 118 | ).toEqual([2, 4, [6, 8, 10], 12, [14, 16]]); 119 | }); 120 | 121 | test('sync mapLoader', async () => { 122 | const idss = [] as number[][]; 123 | const loader = new BatchLoader( 124 | (ids: number[]): Promise => 125 | new Promise((resolve): void => { 126 | idss.push(ids); 127 | setTimeout(() => resolve(ids.map((i) => i * 2)), 10); 128 | }), 129 | String 130 | ).mapLoader(String); 131 | 132 | expect(await loader.load(3)).toBe('6'); 133 | expect(await loader.load(4)).toBe('8'); 134 | expect(await loader.load(5)).toBe('10'); 135 | 136 | expect(await loader.loadMany([])).toEqual([]); 137 | expect(await loader.loadMany([1, 2, 3])).toEqual([2, 4, 6].map(String)); 138 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual( 139 | [2, 4, 6, 4, 6, 4, 2].map(String) 140 | ); 141 | 142 | expect( 143 | await Promise.all([ 144 | loader.load(1), 145 | loader.load(2), 146 | loader.load(3), 147 | loader.load(2), 148 | loader.load(1), 149 | loader.load(2), 150 | loader.load(3), 151 | ]) 152 | ).toEqual([2, 4, 6, 4, 2, 4, 6].map(String)); 153 | 154 | expect(idss).toEqual([[3], [4], [5], [1, 2, 3], [1, 2, 3], [1, 2, 3]]); 155 | }); 156 | 157 | test('async mapLoader', async () => { 158 | const idss = [] as number[][]; 159 | const loader = new BatchLoader( 160 | (ids: number[]): Promise => 161 | new Promise((resolve): void => { 162 | idss.push(ids); 163 | setTimeout(() => resolve(ids.map((i) => i * 2)), 10); 164 | }), 165 | String 166 | ).mapLoader((x): Promise => Promise.resolve(String(x))); 167 | 168 | expect(await loader.load(3)).toBe('6'); 169 | expect(await loader.load(4)).toBe('8'); 170 | expect(await loader.load(5)).toBe('10'); 171 | 172 | expect(await loader.loadMany([])).toEqual([]); 173 | expect(await loader.loadMany([1, 2, 3])).toEqual([2, 4, 6].map(String)); 174 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual( 175 | [2, 4, 6, 4, 6, 4, 2].map(String) 176 | ); 177 | 178 | expect( 179 | await Promise.all([ 180 | loader.load(1), 181 | loader.load(2), 182 | loader.load(3), 183 | loader.load(2), 184 | loader.load(1), 185 | loader.load(2), 186 | loader.load(3), 187 | ]) 188 | ).toEqual([2, 4, 6, 4, 2, 4, 6].map(String)); 189 | 190 | expect(idss).toEqual([[3], [4], [5], [1, 2, 3], [1, 2, 3], [1, 2, 3]]); 191 | }); 192 | }); 193 | -------------------------------------------------------------------------------- /src/batchloader.ts: -------------------------------------------------------------------------------- 1 | import { CacheLoader, ICache } from '_src/cacheloader'; 2 | import { MappedBatchLoader } from '_src/mappedbatchloader'; 3 | import { IBatchLoader, MaybePromise } from '_src/types'; 4 | 5 | export type BatchLoadFn = ( 6 | keys: Key[] 7 | ) => Value[] | Promise; 8 | 9 | export type KeyToUniqueId = (key: Key) => string; 10 | 11 | const sleep = (ms: number): Promise => 12 | new Promise((resolve): void => { 13 | setTimeout(resolve, ms); 14 | }); 15 | 16 | export class BatchLoader implements IBatchLoader { 17 | protected queuedKeys: Key[] = []; 18 | protected batchPromise: Promise | null = null; 19 | 20 | constructor( 21 | protected batchFn: BatchLoadFn, 22 | protected keyToUniqueId: KeyToUniqueId | null, 23 | protected batchDelay = 0, 24 | protected batchSize = Number.MAX_SAFE_INTEGER 25 | ) {} 26 | 27 | public load(key: Key): Promise { 28 | const { queuedKeys } = this; 29 | const index = queuedKeys.length; 30 | queuedKeys.push(key); 31 | 32 | return this.triggerBatch().then((values) => values[index]); 33 | } 34 | 35 | public loadMany(keys: Key[]): Promise { 36 | if (keys.length) { 37 | const { queuedKeys } = this; 38 | const index = queuedKeys.length; 39 | queuedKeys.push(...keys); 40 | const { length } = keys; 41 | 42 | return this.triggerBatch().then((values) => 43 | values.slice(index, index + length) 44 | ); 45 | } 46 | return Promise.resolve([]); 47 | } 48 | 49 | public mapLoader( 50 | mapFn: (value: Value, key: Key) => MappedValue 51 | ): MappedBatchLoader { 52 | return new MappedBatchLoader(this, mapFn); 53 | } 54 | 55 | public cacheLoader(cache?: ICache): CacheLoader { 56 | return new CacheLoader(this, cache); 57 | } 58 | 59 | protected triggerBatch(): Promise { 60 | return ( 61 | this.batchPromise || 62 | (this.batchPromise = new Promise((resolve, reject): void => { 63 | setTimeout(() => { 64 | this.batchPromise = null; 65 | this.runBatchNow().then(resolve, reject); 66 | }, this.batchDelay); 67 | })) 68 | ); 69 | } 70 | 71 | protected runBatchNow(): Promise { 72 | const { queuedKeys, keyToUniqueId } = this; 73 | this.queuedKeys = []; 74 | 75 | if (keyToUniqueId) { 76 | const idMap: { [key: string]: true } = {}; 77 | const indexToId: string[] = []; 78 | const idToNewIndex: { [key: string]: number } = {}; 79 | 80 | let newIndex = 0; 81 | 82 | const uniqueKeys = []; 83 | const len = queuedKeys.length; 84 | for (let i = 0; i < len; i += 1) { 85 | const key = queuedKeys[i]; 86 | const id = keyToUniqueId(key); 87 | indexToId[i] = id; 88 | if (idMap[id] !== true) { 89 | idMap[id] = true; 90 | idToNewIndex[id] = newIndex; 91 | newIndex += 1; 92 | uniqueKeys.push(key); 93 | } 94 | } 95 | 96 | return this.maybeBatchInChunks(uniqueKeys).then((values) => 97 | queuedKeys.map((_key, i) => values[idToNewIndex[indexToId[i]]]) 98 | ); 99 | } 100 | 101 | return this.maybeBatchInChunks(queuedKeys); 102 | } 103 | 104 | private maybeBatchInChunks(keys: Key[]): Promise { 105 | if (keys.length <= this.batchSize) { 106 | return Promise.resolve(this.batchFn(keys)); 107 | } 108 | return this.batchInChunks(keys); 109 | } 110 | 111 | private async batchInChunks(keys: Key[]): Promise { 112 | const { batchSize, batchDelay } = this; 113 | 114 | const promises: Array> = []; 115 | const kLen = keys.length; 116 | for (let i = 0; i < kLen; i += batchSize) { 117 | promises.push(this.batchFn(keys.slice(i, i + batchSize))); 118 | if (batchDelay) { 119 | await sleep(batchDelay); 120 | } 121 | } 122 | 123 | const results = await Promise.all(promises); 124 | const rLen = results.length; 125 | let values: Value[] = []; 126 | for (let i = 0; i < rLen; i += 1) { 127 | values = values.concat(results[i]); 128 | } 129 | 130 | return values; 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /src/cacheloader.test.ts: -------------------------------------------------------------------------------- 1 | import { BatchLoader } from './batchloader'; 2 | 3 | describe('CacheLoader', () => { 4 | test('with keyToUniqueId', async () => { 5 | const idss = [] as number[][]; 6 | const bloader = new BatchLoader( 7 | (ids: number[]): Promise => 8 | new Promise((resolve): void => { 9 | idss.push(ids); 10 | setTimeout(() => resolve(ids.map((i) => i * 2)), 10); 11 | }), 12 | String 13 | ); 14 | const loader = bloader.cacheLoader(); 15 | 16 | expect(await loader.load(3)).toBe(6); 17 | expect(await loader.load(4)).toBe(8); 18 | expect(await loader.load(5)).toBe(10); 19 | 20 | expect(await loader.loadMany([])).toEqual([]); 21 | expect(await loader.loadMany([1, 2, 3])).toEqual([2, 4, 6]); 22 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual([ 23 | 2, 24 | 4, 25 | 6, 26 | 4, 27 | 6, 28 | 4, 29 | 2, 30 | ]); 31 | 32 | expect( 33 | await Promise.all([ 34 | loader.load(1), 35 | loader.load(2), 36 | loader.load(3), 37 | loader.load(2), 38 | loader.load(1), 39 | loader.load(2), 40 | loader.load(3), 41 | ]) 42 | ).toEqual([2, 4, 6, 4, 2, 4, 6]); 43 | 44 | expect(idss).toEqual([[3], [4], [5], [1, 2]]); 45 | 46 | // rerun 47 | idss.length = 0; 48 | expect(await loader.load(3)).toBe(6); 49 | expect(await loader.load(4)).toBe(8); 50 | expect(await loader.load(5)).toBe(10); 51 | 52 | expect(await loader.loadMany([])).toEqual([]); 53 | expect(await loader.loadMany([1, 2, 3])).toEqual([2, 4, 6]); 54 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual([ 55 | 2, 56 | 4, 57 | 6, 58 | 4, 59 | 6, 60 | 4, 61 | 2, 62 | ]); 63 | 64 | expect( 65 | await Promise.all([ 66 | loader.load(1), 67 | loader.load(2), 68 | loader.load(3), 69 | loader.load(2), 70 | loader.load(1), 71 | loader.load(2), 72 | loader.load(3), 73 | ]) 74 | ).toEqual([2, 4, 6, 4, 2, 4, 6]); 75 | 76 | expect(idss).toEqual([]); 77 | 78 | // rerun 79 | idss.length = 0; 80 | loader.clear(); 81 | 82 | expect(await loader.load(3)).toBe(6); 83 | expect(await loader.load(4)).toBe(8); 84 | expect(await loader.load(5)).toBe(10); 85 | 86 | expect(await loader.loadMany([])).toEqual([]); 87 | expect(await loader.loadMany([1, 2, 3])).toEqual([2, 4, 6]); 88 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual([ 89 | 2, 90 | 4, 91 | 6, 92 | 4, 93 | 6, 94 | 4, 95 | 2, 96 | ]); 97 | 98 | expect( 99 | await Promise.all([ 100 | loader.load(1), 101 | loader.load(2), 102 | loader.load(3), 103 | loader.load(2), 104 | loader.load(1), 105 | loader.load(2), 106 | loader.load(3), 107 | ]) 108 | ).toEqual([2, 4, 6, 4, 2, 4, 6]); 109 | 110 | expect(idss).toEqual([[3], [4], [5], [1, 2]]); 111 | 112 | const mappedLoader = loader.mapLoader(String); 113 | 114 | expect(await mappedLoader.load(3)).toBe('6'); 115 | expect(await mappedLoader.load(4)).toBe('8'); 116 | expect(await mappedLoader.load(5)).toBe('10'); 117 | 118 | expect(loader.get(8)).toBe(undefined); 119 | expect(loader.get(8)).toBe(undefined); 120 | loader.set(8, 12); 121 | expect(await loader.get(8)).toBe(12); 122 | expect(await loader.get(8)).toBe(12); 123 | loader.delete(8); 124 | expect(loader.get(8)).toBe(undefined); 125 | expect(loader.get(8)).toBe(undefined); 126 | }); 127 | }); 128 | -------------------------------------------------------------------------------- /src/cacheloader.ts: -------------------------------------------------------------------------------- 1 | import { MappedBatchLoader } from '_src/mappedbatchloader'; 2 | import { IBatchLoader, MaybePromise } from '_src/types'; 3 | 4 | export interface ICache { 5 | clear(): void; 6 | delete(key: Key): boolean; 7 | get(key: Key): Value | undefined; 8 | set(key: Key, value: Value): void; 9 | } 10 | 11 | export class CacheLoader 12 | implements IBatchLoader, ICache> { 13 | public promiseCache: Map>; 14 | 15 | constructor( 16 | protected loader: IBatchLoader, 17 | public cache: ICache = new Map() 18 | ) { 19 | this.promiseCache = new Map>(); 20 | } 21 | 22 | public load(key: Key): Promise { 23 | const { promiseCache } = this; 24 | let pv = promiseCache.get(key); 25 | if (pv) { 26 | return pv; 27 | } 28 | const value = this.cache.get(key); 29 | pv = 30 | value === undefined 31 | ? this.loader.load(key).then((val) => { 32 | this.set(key, val); 33 | return val; 34 | }) 35 | : Promise.resolve(value); 36 | promiseCache.set(key, pv); 37 | return pv; 38 | } 39 | 40 | public loadMany(keys: Key[]): Promise { 41 | return Promise.all(keys.map((key) => this.load(key))); 42 | } 43 | 44 | public mapLoader( 45 | mapFn: (value: Value, key: Key) => MappedValue 46 | ): MappedBatchLoader { 47 | return new MappedBatchLoader(this, mapFn); 48 | } 49 | 50 | public get(key: Key): Promise | undefined { 51 | let pv = this.promiseCache.get(key); 52 | if (pv) { 53 | return pv; 54 | } 55 | const v = this.cache.get(key); 56 | if (v === undefined) { 57 | return undefined; 58 | } 59 | pv = Promise.resolve(v); 60 | this.promiseCache.set(key, pv); 61 | return pv; 62 | } 63 | 64 | public set(key: Key, value: Value): void { 65 | this.promiseCache.delete(key); 66 | this.cache.set(key, value); 67 | } 68 | 69 | public delete(key: Key): boolean { 70 | const pDeleted = this.promiseCache.delete(key); 71 | const deleted = this.cache.delete(key); 72 | return pDeleted || deleted; 73 | } 74 | 75 | public clear(): void { 76 | this.promiseCache.clear(); 77 | this.cache.clear(); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/cacheproxyloader.test.ts: -------------------------------------------------------------------------------- 1 | import { BatchLoader } from './batchloader'; 2 | import { proxyLoaderWithCache } from './cacheproxyloader'; 3 | 4 | test('proxyLoaderWithCache', async () => { 5 | let cached: { [key: string]: number | null } = { 6 | a: 1, 7 | c: 3, 8 | e: null, 9 | }; 10 | const data: { [key: string]: number } = { 11 | a: 1, 12 | b: 2, 13 | c: 3, 14 | d: 4, 15 | }; 16 | 17 | const mgetArgs: string[][] = []; 18 | const msetArgs: Array> = []; 19 | 20 | const cache = { 21 | mget: (ks: string[]): Array => { 22 | mgetArgs.push(ks); 23 | return ks.map((k) => cached[k]); 24 | }, 25 | mset: (keyValues: Array<[string, number | null]>): void => { 26 | msetArgs.push(keyValues); 27 | keyValues.forEach(([k, v]) => { 28 | cached[k] = v; 29 | }); 30 | }, 31 | }; 32 | 33 | const loadArgs: string[][] = []; 34 | 35 | const loader = new BatchLoader((ks: string[]): Array => { 36 | loadArgs.push(ks); 37 | return ks.map((k) => data[k] || null); 38 | }, String); 39 | 40 | const proxyloader = proxyLoaderWithCache( 41 | cache, 42 | loader, 43 | String 44 | ); 45 | 46 | expect(await proxyloader.load('a')).toBe(1); 47 | expect(await proxyloader.load('b')).toBe(2); 48 | 49 | expect(await proxyloader.loadMany(['c', 'd'])).toEqual([3, 4]); 50 | 51 | expect( 52 | await Promise.all([ 53 | proxyloader.load('a'), 54 | proxyloader.load('b'), 55 | proxyloader.load('c'), 56 | proxyloader.loadMany(['a', 'b', 'c', 'd', 'e', 'f']), 57 | proxyloader.load('d'), 58 | proxyloader.load('e'), 59 | proxyloader.load('f'), 60 | ]) 61 | ).toEqual([1, 2, 3, [1, 2, 3, 4, null, null], 4, null, null]); 62 | 63 | cached = {}; 64 | 65 | expect(await proxyloader.loadMany(['a', 'b', 'c', 'd', 'e', 'f'])).toEqual([ 66 | 1, 67 | 2, 68 | 3, 69 | 4, 70 | null, 71 | null, 72 | ]); 73 | 74 | expect(mgetArgs).toEqual([ 75 | ['a'], 76 | ['b'], 77 | ['c', 'd'], 78 | ['a', 'b', 'c', 'd', 'e', 'f'], 79 | ['a', 'b', 'c', 'd', 'e', 'f'], 80 | ]); 81 | 82 | expect(msetArgs).toEqual([ 83 | [['b', 2]], 84 | [['d', 4]], 85 | [['f', null]], 86 | [['a', 1], ['b', 2], ['c', 3], ['d', 4], ['e', null], ['f', null]], 87 | ]); 88 | 89 | expect(loadArgs).toEqual([ 90 | ['b'], 91 | ['d'], 92 | ['f'], 93 | ['a', 'b', 'c', 'd', 'e', 'f'], 94 | ]); 95 | }); 96 | -------------------------------------------------------------------------------- /src/cacheproxyloader.ts: -------------------------------------------------------------------------------- 1 | import { BatchLoader, KeyToUniqueId } from '_src/batchloader'; 2 | import { IBatchLoader, MaybePromise } from '_src/types'; 3 | 4 | export interface IBatchCache { 5 | mget: (keys: Key[]) => MaybePromise>; 6 | mset: (keyValues: Array<[Key, Value]>) => MaybePromise; 7 | } 8 | 9 | export const proxyLoaderWithCache = ( 10 | cache: IBatchCache, 11 | loader: IBatchLoader, 12 | keyToUniqueId: KeyToUniqueId | null, 13 | batchDelay?: number, 14 | batchSize?: number 15 | ): BatchLoader => 16 | new BatchLoader( 17 | async (keys): Promise => { 18 | const values = await cache.mget(keys); 19 | 20 | const len = values.length; 21 | const missingKeys: Key[] = []; 22 | const missingIndexes: number[] = []; 23 | for (let i = 0; i < len; i += 1) { 24 | if (values[i] === undefined) { 25 | missingKeys.push(keys[i]); 26 | missingIndexes.push(i); 27 | } 28 | } 29 | 30 | if (missingKeys.length) { 31 | const missingValues = await loader.loadMany(missingKeys); 32 | const mlen = missingValues.length; 33 | const missingKeyValues: Array<[Key, Value]> = []; 34 | for (let i = 0; i < mlen; i += 1) { 35 | const value = missingValues[i]; 36 | values[missingIndexes[i]] = value; 37 | missingKeyValues.push([missingKeys[i], value]); 38 | } 39 | 40 | // do not await 41 | cache.mset(missingKeyValues); 42 | } 43 | 44 | return values as Value[]; 45 | }, 46 | keyToUniqueId, 47 | batchDelay, 48 | batchSize 49 | ); 50 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | export { BatchLoadFn, BatchLoader, KeyToUniqueId } from './batchloader'; 2 | export { CacheLoader, ICache } from './cacheloader'; 3 | export { IBatchCache, proxyLoaderWithCache } from './cacheproxyloader'; 4 | export { MappedBatchLoader } from './mappedbatchloader'; 5 | export { IBatchLoader, MaybePromise } from './types'; 6 | -------------------------------------------------------------------------------- /src/mappedbatchloader.test.ts: -------------------------------------------------------------------------------- 1 | import { BatchLoader } from './batchloader'; 2 | import { MappedBatchLoader } from './mappedbatchloader'; 3 | 4 | describe('MappedBatchLoader', () => { 5 | test('sync mapper', async () => { 6 | const idss = [] as number[][]; 7 | const loader1 = new BatchLoader( 8 | (ids: number[]): Promise => 9 | new Promise((resolve): void => { 10 | idss.push(ids); 11 | setTimeout(() => resolve(ids.map((i) => i * 2)), 10); 12 | }), 13 | String 14 | ); 15 | const loader = new MappedBatchLoader(loader1, String); 16 | 17 | expect(await loader.load(3)).toBe('6'); 18 | expect(await loader.load(4)).toBe('8'); 19 | expect(await loader.load(5)).toBe('10'); 20 | 21 | expect(await loader.loadMany([])).toEqual([]); 22 | expect(await loader.loadMany([1, 2, 3])).toEqual([2, 4, 6].map(String)); 23 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual( 24 | [2, 4, 6, 4, 6, 4, 2].map(String) 25 | ); 26 | 27 | expect( 28 | await Promise.all([ 29 | loader.load(1), 30 | loader.load(2), 31 | loader.load(3), 32 | loader.load(2), 33 | loader.load(1), 34 | loader.load(2), 35 | loader.load(3), 36 | ]) 37 | ).toEqual([2, 4, 6, 4, 2, 4, 6].map(String)); 38 | 39 | // test one round trip 40 | expect(await Promise.all([loader1.load(1), loader.load(1)])).toEqual([ 41 | 2, 42 | '2', 43 | ]); 44 | 45 | expect(idss).toEqual([[3], [4], [5], [1, 2, 3], [1, 2, 3], [1, 2, 3], [1]]); 46 | }); 47 | 48 | test('async mapper', async () => { 49 | const idss = [] as number[][]; 50 | const loader = new MappedBatchLoader( 51 | new BatchLoader( 52 | (ids: number[]): Promise => 53 | new Promise((resolve): void => { 54 | idss.push(ids); 55 | setTimeout(() => resolve(ids.map((i) => i * 2)), 10); 56 | }), 57 | String 58 | ), 59 | (x): Promise => Promise.resolve(String(x)) 60 | ); 61 | 62 | expect(await loader.load(3)).toBe('6'); 63 | expect(await loader.load(4)).toBe('8'); 64 | expect(await loader.load(5)).toBe('10'); 65 | 66 | expect(await loader.loadMany([])).toEqual([]); 67 | expect(await loader.loadMany([1, 2, 3])).toEqual([2, 4, 6].map(String)); 68 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual( 69 | [2, 4, 6, 4, 6, 4, 2].map(String) 70 | ); 71 | 72 | expect( 73 | await Promise.all([ 74 | loader.load(1), 75 | loader.load(2), 76 | loader.load(3), 77 | loader.load(2), 78 | loader.load(1), 79 | loader.load(2), 80 | loader.load(3), 81 | ]) 82 | ).toEqual([2, 4, 6, 4, 2, 4, 6].map(String)); 83 | 84 | expect(idss).toEqual([[3], [4], [5], [1, 2, 3], [1, 2, 3], [1, 2, 3]]); 85 | }); 86 | 87 | test('mapLoader()', async () => { 88 | const idss = [] as number[][]; 89 | const loader = new BatchLoader( 90 | (ids: number[]): Promise => 91 | new Promise((resolve): void => { 92 | idss.push(ids); 93 | setTimeout(() => resolve(ids.map((i) => i * 2)), 10); 94 | }), 95 | String 96 | ) 97 | .mapLoader((x): Promise => Promise.resolve(String(x))) 98 | .mapLoader((x) => `${x}${x}`); 99 | 100 | expect(await loader.load(3)).toBe('66'); 101 | expect(await loader.load(4)).toBe('88'); 102 | expect(await loader.load(5)).toBe('1010'); 103 | 104 | expect(await loader.loadMany([])).toEqual([]); 105 | expect(await loader.loadMany([1, 2, 3])).toEqual(['22', '44', '66']); 106 | expect(await loader.loadMany([1, 2, 3, 2, 3, 2, 1])).toEqual( 107 | [2, 4, 6, 4, 6, 4, 2].map((x) => `${x}${x}`) 108 | ); 109 | 110 | expect( 111 | await Promise.all([ 112 | loader.load(1), 113 | loader.load(2), 114 | loader.load(3), 115 | loader.load(2), 116 | loader.load(1), 117 | loader.load(2), 118 | loader.load(3), 119 | ]) 120 | ).toEqual([2, 4, 6, 4, 2, 4, 6].map((x) => `${x}${x}`)); 121 | 122 | expect(idss).toEqual([[3], [4], [5], [1, 2, 3], [1, 2, 3], [1, 2, 3]]); 123 | }); 124 | }); 125 | -------------------------------------------------------------------------------- /src/mappedbatchloader.ts: -------------------------------------------------------------------------------- 1 | import { IBatchLoader } from '_src/types'; 2 | 3 | export class MappedBatchLoader 4 | implements IBatchLoader { 5 | constructor( 6 | protected loader: IBatchLoader, 7 | protected mapFn: ( 8 | value: Value, 9 | key: Key 10 | ) => MappedValue | Promise 11 | ) {} 12 | 13 | public load(key: Key): Promise { 14 | return this.loader.load(key).then((value) => this.mapFn(value, key)); 15 | } 16 | 17 | public loadMany(keys: Key[]): Promise { 18 | return this.loader.loadMany(keys).then((values) => { 19 | let hasPromise = false; 20 | const results: Array> = []; 21 | const len = values.length; 22 | for (let i = 0; i < len; i += 1) { 23 | const res = this.mapFn(values[i], keys[i]); 24 | results.push(res); 25 | hasPromise = 26 | hasPromise || 27 | (res != null && typeof (res as any).then === 'function'); 28 | } 29 | return hasPromise ? Promise.all(results) : (results as MappedValue[]); 30 | }); 31 | } 32 | 33 | public mapLoader( 34 | mapFn: (value: MappedValue, key: Key) => RemappedValue 35 | ): MappedBatchLoader { 36 | return new MappedBatchLoader(this, mapFn); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | export interface IBatchLoader { 2 | load(key: Key): Promise; 3 | loadMany(keys: Key[]): Promise; 4 | mapLoader( 5 | mapFn: (value: Value) => MappedValue 6 | ): IBatchLoader; 7 | } 8 | 9 | export type MaybePromise = T | Promise; 10 | -------------------------------------------------------------------------------- /tsconfig.browser.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.build.json", 3 | "compilerOptions": { 4 | "outDir": "dist/browser", 5 | "declarationDir": "dist/browser", 6 | "lib": ["dom", "esnext"], 7 | "target": "es5", 8 | "module": "commonjs" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "compilerOptions": { 4 | "composite": true, 5 | "declaration": true, 6 | "declarationDir": "lib", 7 | "declarationMap": true, 8 | "emitDecoratorMetadata": true, 9 | "importHelpers": true, 10 | "noEmit": false, 11 | "noEmitHelpers": true, 12 | "preserveConstEnums": true, 13 | "removeComments": true, 14 | "sourceMap": true 15 | }, 16 | "exclude": ["node_modules", "src/**/*.test.ts"] 17 | } 18 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "alwaysStrict": true, 4 | "baseUrl": ".", 5 | "diagnostics": true, 6 | "esModuleInterop": true, 7 | "experimentalDecorators": true, 8 | "forceConsistentCasingInFileNames": true, 9 | "lib": ["esnext"], 10 | "module": "commonjs", 11 | "moduleResolution": "node", 12 | "noEmit": true, 13 | "noErrorTruncation": true, 14 | "noFallthroughCasesInSwitch": true, 15 | "noImplicitAny": true, 16 | "noImplicitReturns": true, 17 | "noImplicitThis": true, 18 | "noStrictGenericChecks": false, 19 | "noUnusedLocals": true, 20 | "noUnusedParameters": true, 21 | "paths": { 22 | "_src/*": ["src/*"] 23 | }, 24 | "resolveJsonModule": true, 25 | "rootDir": "src", 26 | "skipLibCheck": true, 27 | "strict": true, 28 | "strictBindCallApply": true, 29 | "strictFunctionTypes": true, 30 | "strictNullChecks": true, 31 | "strictPropertyInitialization": true, 32 | "suppressExcessPropertyErrors": false, 33 | "suppressImplicitAnyIndexErrors": false, 34 | "target": "esnext" 35 | }, 36 | "include": ["src/**/*.ts"] 37 | } 38 | -------------------------------------------------------------------------------- /tsconfig.module.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.build.json", 3 | "compilerOptions": { 4 | "outDir": "lib", 5 | "declarationDir": "lib", 6 | "tsBuildInfoFile": "lib/tsconfig.module.tsbuildinfo", 7 | "lib": ["esnext"], 8 | "target": "es2017", 9 | "module": "es6" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.build.json", 3 | "compilerOptions": { 4 | "outDir": "dist/node", 5 | "declarationDir": "dist/node", 6 | "lib": ["es2017"], 7 | "target": "es2017", 8 | "module": "commonjs" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "tslint:recommended", 4 | "tslint-config-prettier", 5 | "tslint-config-airbnb" 6 | ], 7 | "rules": { 8 | "align": false, 9 | "array-type": [true, "array-simple"], 10 | "callable-types": false, 11 | "import-name": false, 12 | "max-line-length": false, 13 | "no-boolean-literal-compare": false, 14 | "no-empty-interface": false, 15 | "no-inferrable-types": true, 16 | "no-this-assignment": [true, { "allow-destructuring": true }], 17 | "object-literal-sort-keys": false, 18 | "object-shorthand-properties-first": false, 19 | "prefer-array-literal": false, 20 | "semicolon": [true, "always", "ignore-bound-class-methods"], 21 | "ter-arrow-parens": [true, "always"], 22 | "ter-func-call-spacing": false, 23 | "ter-indent": false, 24 | "trailing-comma": [ 25 | true, 26 | { 27 | "multiline": { 28 | "arrays": "always", 29 | "objects": "always", 30 | "functions": "never", 31 | "imports": "always", 32 | "exports": "always", 33 | "typeLiterals": "always" 34 | }, 35 | "singleline": "never", 36 | "esSpecCompliant": true 37 | } 38 | ], 39 | "typedef": [true, "call-signature", "arrow-call-signature"], 40 | "variable-name": [ 41 | true, 42 | "ban-keywords", 43 | "check-format", 44 | "allow-leading-underscore", 45 | "allow-pascal-case" 46 | ] 47 | }, 48 | "jsRules": {} 49 | } 50 | --------------------------------------------------------------------------------