├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── build.gradle ├── gradle.properties ├── gradle ├── publishing.gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── settings.gradle └── src ├── main └── java │ └── org │ └── dataloader │ ├── BatchLoader.java │ ├── CacheKey.java │ ├── CacheMap.java │ ├── DataLoader.java │ ├── DataLoaderOptions.java │ └── impl │ ├── Assertions.java │ ├── CompletableFutureKit.java │ ├── DefaultCacheMap.java │ ├── PromisedValues.java │ └── PromisedValuesImpl.java └── test └── java ├── ReadmeExamples.java └── org └── dataloader ├── DataLoaderTest.java ├── JsonObject.java ├── fixtures ├── User.java └── UserManager.java └── impl └── PromisedValuesImplTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source 2 | *.com 3 | *.class 4 | *.dll 5 | *.exe 6 | *.o 7 | *.so 8 | 9 | # Packages 10 | *.7z 11 | *.dmg 12 | *.gz 13 | *.iso 14 | *.jar 15 | *.rar 16 | *.tar 17 | *.zip 18 | 19 | # Logs and databases 20 | *.log 21 | 22 | # OS generated files 23 | .DS_Store* 24 | ehthumbs.db 25 | Icon? 26 | Thumbs.db 27 | 28 | # Editor Files 29 | *~ 30 | *.swp 31 | 32 | # Gradle Files 33 | .gradle 34 | 35 | # Build output directies 36 | /target 37 | **/target 38 | /build/** 39 | **/build 40 | /node_modules 41 | **/node_modules 42 | 43 | # IntelliJ specific files/directories 44 | out 45 | .idea 46 | *.ipr 47 | *.iws 48 | *.iml 49 | 50 | # Eclipse specific files/directories 51 | .classpath 52 | .project 53 | .settings 54 | .metadata 55 | 56 | # NetBeans specific files/directories 57 | .nbattrs 58 | 59 | # Vagrant files 60 | .vagrant 61 | 62 | # Vert.x 63 | .vertx -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | sudo: false 3 | 4 | jdk: 5 | - oraclejdk8 6 | 7 | addons: 8 | apt: 9 | packages: 10 | - oracle-java8-installer 11 | 12 | notifications: 13 | email: false 14 | 15 | branches: 16 | only: 17 | - master 18 | 19 | script: 20 | - ./gradlew clean test 21 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # java-dataloader 2 | 3 | [![Build Status](https://travis-ci.org/bbakerman/java-dataloader.svg?branch=master)](https://travis-ci.org/bbakerman/java-dataloader.svg?branch=master)   4 | [![Apache licensed](https://img.shields.io/hexpm/l/plug.svg?maxAge=2592000)](https://github.com/bbakerman/java-dataloader/blob/master/LICENSE)   5 | [![Download](https://api.bintray.com/packages/bbakerman/java-dataloader/java-dataloader/images/download.svg) ](https://bintray.com/bbakerman/java-dataloader/java-dataloader/_latestVersion) 6 | 7 | This small and simple utility library is a pure Java 8 port of [Facebook DataLoader](https://github.com/facebook/dataloader). 8 | 9 | It can serve as integral part of your application's data layer to provide a 10 | consistent API over various back-ends and reduce message communication overhead through batching and caching. 11 | 12 | An important use case for `java-dataloader` is improving the efficiency of GraphQL query execution. Graphql fields 13 | are resolved in a independent manner and with a true graph of objects, you may be fetching the same object many times. 14 | 15 | A naive implementation of graphql data fetchers can easily lead to the dreaded "n+1" fetch problem. 16 | 17 | Most of the code is ported directly from Facebook's reference implementation, with one IMPORTANT adaptation to make 18 | it work for Java 8. ([more on this below](manual-dispatching)). 19 | 20 | But before reading on, be sure to take a short dive into the 21 | [original documentation](https://github.com/facebook/dataloader/blob/master/README.md) provided by Lee Byron (@leebyron) 22 | and Nicholas Schrock (@schrockn) from [Facebook](https://www.facebook.com/), the creators of the original data loader. 23 | 24 | ## Table of contents 25 | 26 | - [Features](#features) 27 | - [Examples](#examples) 28 | - [Differences to reference implementation](#differences-to-reference-implementation) 29 | - [Manual dispatching](#manual-dispatching) 30 | - [Let's get started!](#lets-get-started) 31 | - [Installing](#installing) 32 | - [Building](#building) 33 | - [Other information sources](#other-information-sources) 34 | - [Contributing](#contributing) 35 | - [Acknowledgements](#acknowledgements) 36 | - [Licensing](#licensing) 37 | 38 | ## Features 39 | 40 | `java-dataloader` is a feature-complete port of the Facebook reference implementation with [one major difference](#manual-dispatching). These features are: 41 | 42 | - Simple, intuitive API, using generics and fluent coding 43 | - Define batch load function with lambda expression 44 | - Schedule a load request in queue for batching 45 | - Add load requests from anywhere in code 46 | - Request returns a [`CompleteableFuture`](https://docs.oracle.com/javase/8/docs/api/java/util/concurrent/CompletableFuture.html) of the requested value 47 | - Can create multiple requests at once 48 | - Caches load requests, so data is only fetched once 49 | - Can clear individual cache keys, so data is re-fetched on next batch queue dispatch 50 | - Can prime the cache with key/values, to avoid data being fetched needlessly 51 | - Can configure cache key function with lambda expression to extract cache key from complex data loader key types 52 | - Individual batch futures complete / resolve as batch is processed 53 | - Results are ordered according to insertion order of load requests 54 | - Deals with partial errors when a batch future fails 55 | - Can disable batching and/or caching in configuration 56 | - Can supply your own [`CacheMap`](https://github.com/bbakerman/java-dataloader/blob/master/src/main/java/io/engagingspaces/vertx/dataloader/CacheMap.java) implementations 57 | - Has very high test coverage (see [Acknowledgements](#acknowlegdements)) 58 | 59 | ## Examples 60 | 61 | A `DataLoader` object requires a `BatchLoader` function that is responsible for loading a promise of values given 62 | a list of keys 63 | 64 | ```java 65 | 66 | BatchLoader userBatchLoader = new BatchLoader() { 67 | @Override 68 | public CompletionStage> load(List userIds) { 69 | return CompletableFuture.supplyAsync(() -> { 70 | return userManager.loadUsersById(userIds); 71 | }); 72 | } 73 | }; 74 | 75 | DataLoader userLoader = new DataLoader<>(userBatchLoader); 76 | 77 | ``` 78 | 79 | You can then use it to load values which will be `CompleteableFuture` promises to values 80 | 81 | ```java 82 | CompletableFuture load1 = userLoader.load(1L); 83 | ``` 84 | 85 | or you can use it to compose future computations as follows. The key requirement is that you call 86 | `dataloader.dispatch()` or its variant `dataloader.dispatchAndJoin()` at some point in order to make the underlying calls happen to the batch loader. 87 | 88 | In this version of data loader, this does not happen automatically. More on this in [Manual dispatching](#manual-dispatching) . 89 | 90 | ```java 91 | userLoader.load(1L) 92 | .thenAccept(user -> { 93 | System.out.println("user = " + user); 94 | userLoader.load(user.getInvitedByID()) 95 | .thenAccept(invitedBy -> { 96 | System.out.println("invitedBy = " + invitedBy); 97 | }); 98 | }); 99 | 100 | userLoader.load(2L) 101 | .thenAccept(user -> { 102 | System.out.println("user = " + user); 103 | userLoader.load(user.getInvitedByID()) 104 | .thenAccept(invitedBy -> { 105 | System.out.println("invitedBy = " + invitedBy); 106 | }); 107 | }); 108 | 109 | userLoader.dispatchAndJoin(); 110 | 111 | ``` 112 | 113 | As stated on the original Facebook project : 114 | 115 | >A naive application may have issued four round-trips to a backend for the required information, 116 | but with DataLoader this application will make at most two. 117 | 118 | > DataLoader allows you to decouple unrelated parts of your application without sacrificing the 119 | performance of batch data-loading. While the loader presents an API that loads individual values, all 120 | concurrent requests will be coalesced and presented to your batch loading function. This allows your 121 | application to safely distribute data fetching requirements throughout your application and 122 | maintain minimal outgoing data requests. 123 | 124 | In the example above, the first call to dispatch will cause the batched user keys (1 and 2) to be fired at the BatchLoader function to load 2 users. 125 | 126 | Since each `thenAccept` callback made more calls to `userLoader` to get the "user they they invited", another 2 user keys are given at the `BatchLoader` 127 | function for them. 128 | 129 | In this case the `userLoader.dispatchAndJoin()` is used to make a dispatch call, wait for it (aka join it), see if the data loader has more batched entries, (which is does) 130 | and then it repeats this until the data loader internal queue of keys is empty. At this point we have made 2 batched calls instead of the naive 4 calls we might have made if 131 | we did not "batch" the calls to load data. 132 | 133 | ## Batching requires batched backing APIs 134 | 135 | You will notice in our BatchLoader example that the backing service had the ability to get a list of users given 136 | a list of user ids in one call. 137 | 138 | ```java 139 | public CompletionStage> load(List userIds) { 140 | return CompletableFuture.supplyAsync(() -> { 141 | return userManager.loadUsersById(userIds); 142 | }); 143 | } 144 | ``` 145 | 146 | This is important consideration. By using `dataloader` you have batched up the requests for N keys in a list of keys that can be 147 | retrieved at one time. 148 | 149 | If you don't have batched backing services, then you cant be as efficient as possible as you will have to make N calls for each key. 150 | 151 | ```java 152 | BatchLoader lessEfficientUserBatchLoader = new BatchLoader() { 153 | @Override 154 | public CompletionStage> load(List userIds) { 155 | return CompletableFuture.supplyAsync(() -> { 156 | // 157 | // notice how it makes N calls to load by single user id out of the batch of N keys 158 | // 159 | return userIds.stream() 160 | .map(id -> userManager.loadUserById(id)) 161 | .collect(Collectors.toList()); 162 | }); 163 | } 164 | }; 165 | 166 | ``` 167 | 168 | That said, with key caching turn on (the default), it may still be more efficient using `dataloader` than without it. 169 | 170 | ## Differences to reference implementation 171 | 172 | ### Manual dispatching 173 | 174 | The original data loader was written in Javascript for NodeJS. NodeJS is single-threaded in nature, but simulates 175 | asynchronous logic by invoking functions on separate threads in an event loop, as explained 176 | [in this post](http://stackoverflow.com/a/19823583/3455094) on StackOverflow. 177 | 178 | NodeJS generates so-call 'ticks' in which queued functions are dispatched for execution, and Facebook `DataLoader` uses 179 | the `nextTick()` function in NodeJS to _automatically_ dequeue load requests and send them to the batch execution function 180 | for processing. 181 | 182 | And here there is an **IMPORTANT DIFFERENCE** compared to how `java-dataloader` operates!! 183 | 184 | In NodeJS the batch preparation will not affect the asynchronous processing behaviour in any way. It will just prepare 185 | batches in 'spare time' as it were. 186 | 187 | This is different in Java as you will actually _delay_ the execution of your load requests, until the moment where you make a 188 | call to `dataLoader.dispatch()`. 189 | 190 | Does this make Java `DataLoader` any less useful than the reference implementation? We would argue this is not the case, 191 | and there are also gains to this different mode of operation: 192 | 193 | - In contrast to the NodeJS implementation _you_ as developer are in full control of when batches are dispatched 194 | - You can attach any logic that determines when a dispatch takes place 195 | - You still retain all other features, full caching support and batching (e.g. to optimize message bus traffic, GraphQL query execution time, etc.) 196 | 197 | However, with batch execution control comes responsibility! If you forget to make the call to `dispatch()` then the futures 198 | in the load request queue will never be batched, and thus _will never complete_! So be careful when crafting your loader designs. 199 | 200 | ### Error object is not a thing in a type safe Java world 201 | 202 | In the reference JS implementation if the batch loader returns an `Error` object back then the `loadKey()` promise is rejected 203 | with that error. This allows fine grain (per object in the list) sets of error. If I ask for keys A,B,C and B errors out the promise 204 | for B can contain a specific error. 205 | 206 | This is not quite as neat in a Java implementation 207 | 208 | A batch loader function is defined as `BatchLoader` meaning for a key of type `K` it returns a value of type `V`. 209 | 210 | It cant just return some `Exception` as an object of type `V` since Java is type safe. 211 | 212 | You in order for a batch loader function to return an `Exception` it must be declared as `BatchLoader` which 213 | allows both values and exceptions to be returned . Some type safety is lost in this case if you want 214 | to use the mix of exceptions and values pattern. 215 | 216 | ## Let's get started! 217 | 218 | ### Installing 219 | 220 | Gradle users configure the `java-dataloader` dependency in `build.gradle`: 221 | 222 | ``` 223 | repositories { 224 | maven { 225 | jcenter() 226 | } 227 | } 228 | 229 | dependencies { 230 | compile 'org.dataloader:java-dataloader:1.0.0' 231 | } 232 | ``` 233 | 234 | ### Building 235 | 236 | To build from source use the Gradle wrapper: 237 | 238 | ``` 239 | ./gradlew clean build 240 | ``` 241 | 242 | 243 | ## Other information sources 244 | 245 | - [Facebook DataLoader Github repo](https://github.com/facebook/dataloader) 246 | - [Facebook DataLoader code walkthrough on YouTube](https://youtu.be/OQTnXNCDywA) 247 | - [Using DataLoader and GraphQL to batch requests](http://gajus.com/blog/9/using-dataloader-to-batch-requests) 248 | 249 | ## Contributing 250 | 251 | All your feedback and help to improve this project is very welcome. Please create issues for your bugs, ideas and 252 | enhancement requests, or better yet, contribute directly by creating a PR. 253 | 254 | When reporting an issue, please add a detailed instruction, and if possible a code snippet or test that can be used 255 | as a reproducer of your problem. 256 | 257 | When creating a pull request, please adhere to the current coding style where possible, and create tests with your 258 | code so it keeps providing an excellent test coverage level. PR's without tests may not be accepted unless they only 259 | deal with minor changes. 260 | 261 | ## Acknowledgements 262 | 263 | This library was originally written for use within a [VertX world](http://vertx.io/) and it used the vertx-core `Future` classes to implement 264 | itself. All the heavy lifting has been done by this project : [vertx-dataloader](https://github.com/engagingspaces/vertx-dataloader) 265 | including the extensive testing. 266 | 267 | This particular port was done to reduce the dependency on Vertx and to write a pure Java 8 implementation with no dependencies and also 268 | to use the more normative Java CompletableFuture. 269 | 270 | [vertx-core](http://vertx.io/docs/vertx-core/java/) is not a lightweight library by any means 271 | so having a pure Java 8 implementation is very desirable. 272 | 273 | 274 | This library is entirely inspired by the great works of [Lee Byron](https://github.com/leebyron) and 275 | [Nicholas Schrock](https://github.com/schrockn) from [Facebook](https://www.facebook.com/) whom we would like to thank, and 276 | especially @leebyron for taking the time and effort to provide 100% coverage on the codebase. The original set of tests 277 | were also ported. 278 | 279 | 280 | 281 | ## Licensing 282 | 283 | This project is licensed under the 284 | [Apache Commons v2.0](https://www.apache.org/licenses/LICENSE-2.0) license. 285 | 286 | Copyright © 2016 Arnold Schrijver, 2017 Brad Baker and others 287 | [contributors](https://github.com/bbakerman/java-dataloader/graphs/contributors) 288 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | import java.text.SimpleDateFormat 2 | 3 | buildscript { 4 | repositories { 5 | jcenter() 6 | } 7 | dependencies { 8 | classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.7' 9 | } 10 | } 11 | 12 | apply plugin: 'maven-publish' 13 | 14 | publishing { 15 | repositories { 16 | maven { 17 | url 'https://dl.bintray.com/bbakerman/maven' 18 | } 19 | } 20 | } 21 | 22 | ext { 23 | junitVersion = '4.12' 24 | } 25 | 26 | apply plugin: 'java' 27 | apply plugin: 'maven' 28 | apply from: "$projectDir/gradle/publishing.gradle" 29 | 30 | repositories { 31 | mavenLocal() 32 | mavenCentral() 33 | jcenter() 34 | maven { 35 | url 'https://dl.bintray.com/engagingspaces/maven' 36 | } 37 | } 38 | 39 | def releaseVersion = System.properties.RELEASE_VERSION 40 | version = releaseVersion ? releaseVersion : new SimpleDateFormat('yyyy-MM-dd\'T\'HH-mm-ss').format(new Date()) 41 | group = 'org.dataloader' 42 | 43 | compileJava { 44 | sourceCompatibility = 1.8 45 | targetCompatibility = 1.8 46 | 47 | options.compilerArgs = ["-Xlint:unchecked", "-Xdiags:verbose"] 48 | } 49 | 50 | dependencies { 51 | testCompile "junit:junit:$junitVersion" 52 | testCompile 'org.awaitility:awaitility:2.0.0' 53 | } 54 | 55 | task wrapper(type: Wrapper) { 56 | gradleVersion = '4.0' 57 | distributionUrl = "http://services.gradle.org/distributions/gradle-4.0-all.zip" 58 | } -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | projectTitle = Java Dataloader 2 | projectDescription = Port of Facebook Dataloader for Java -------------------------------------------------------------------------------- /gradle/publishing.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'maven-publish' 2 | apply plugin: 'com.jfrog.bintray' 3 | 4 | publishing { 5 | repositories { 6 | maven { 7 | url 'http://dl.bintray.com/bbakerman/java-dataloader' 8 | } 9 | } 10 | } 11 | 12 | task sourcesJar(type: Jar, dependsOn: classes) { 13 | classifier = 'sources' 14 | from sourceSets.main.allSource 15 | } 16 | 17 | task javadocJar(type: Jar, dependsOn: javadoc) { 18 | classifier = 'javadoc' 19 | from javadoc.destinationDir 20 | } 21 | 22 | artifacts { 23 | archives sourcesJar 24 | archives javadocJar 25 | } 26 | 27 | publishing { 28 | publications { 29 | maven(MavenPublication) { 30 | artifactId "$project.name" 31 | from components.java 32 | 33 | artifact sourcesJar { 34 | classifier 'sources' 35 | } 36 | artifact javadocJar { 37 | classifier 'javadoc' 38 | } 39 | pom.withXml { 40 | asNode().children().last() + { 41 | def builder = delegate 42 | builder.name projectTitle 43 | builder.description projectDescription 44 | builder.url 'https://github.com/bbakerman/java-dataloader' 45 | builder.licenses { 46 | builder.license { 47 | builder.name 'The Apache Software License, Version 2.0' 48 | builder.url 'http://www.apache.org/licenses/LICENSE-2.0.txt' 49 | builder.distribution 'repo' 50 | } 51 | } 52 | builder.scm { 53 | builder.url 'scm:git@github.com:bbakerman/java-dataloader.git' 54 | builder.connection 'git@github.com:bbakerman/java-dataloader.git' 55 | builder.developerConnection 'git@github.com:bbakerman/java-dataloader.git' 56 | } 57 | } 58 | } 59 | } 60 | } 61 | } 62 | 63 | bintray { 64 | user = System.getenv('BINTRAY_USER') 65 | key = System.getenv('BINTRAY_KEY') 66 | publications = ['maven'] 67 | publish = true 68 | pkg { 69 | repo = 'java-dataloader' 70 | name = "java-dataloader" 71 | desc = projectDescription 72 | licenses = ['Apache-2.0'] 73 | vcsUrl = 'https://github.com/bbakerman/java-dataloader.git' 74 | version { 75 | released = new Date() 76 | vcsTag = project.version 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bbakerman/java-dataloader/52bd96c7e2d34d1a758a3c826c008aa78b4a2416/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Wed Jun 21 10:10:06 AEST 2017 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=http\://services.gradle.org/distributions/gradle-4.0-all.zip 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Attempt to set APP_HOME 10 | # Resolve links: $0 may be a link 11 | PRG="$0" 12 | # Need this for relative symlinks. 13 | while [ -h "$PRG" ] ; do 14 | ls=`ls -ld "$PRG"` 15 | link=`expr "$ls" : '.*-> \(.*\)$'` 16 | if expr "$link" : '/.*' > /dev/null; then 17 | PRG="$link" 18 | else 19 | PRG=`dirname "$PRG"`"/$link" 20 | fi 21 | done 22 | SAVED="`pwd`" 23 | cd "`dirname \"$PRG\"`/" >/dev/null 24 | APP_HOME="`pwd -P`" 25 | cd "$SAVED" >/dev/null 26 | 27 | APP_NAME="Gradle" 28 | APP_BASE_NAME=`basename "$0"` 29 | 30 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 31 | DEFAULT_JVM_OPTS="" 32 | 33 | # Use the maximum available, or set MAX_FD != -1 to use that value. 34 | MAX_FD="maximum" 35 | 36 | warn () { 37 | echo "$*" 38 | } 39 | 40 | die () { 41 | echo 42 | echo "$*" 43 | echo 44 | exit 1 45 | } 46 | 47 | # OS specific support (must be 'true' or 'false'). 48 | cygwin=false 49 | msys=false 50 | darwin=false 51 | nonstop=false 52 | case "`uname`" in 53 | CYGWIN* ) 54 | cygwin=true 55 | ;; 56 | Darwin* ) 57 | darwin=true 58 | ;; 59 | MINGW* ) 60 | msys=true 61 | ;; 62 | NONSTOP* ) 63 | nonstop=true 64 | ;; 65 | esac 66 | 67 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 68 | 69 | # Determine the Java command to use to start the JVM. 70 | if [ -n "$JAVA_HOME" ] ; then 71 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 72 | # IBM's JDK on AIX uses strange locations for the executables 73 | JAVACMD="$JAVA_HOME/jre/sh/java" 74 | else 75 | JAVACMD="$JAVA_HOME/bin/java" 76 | fi 77 | if [ ! -x "$JAVACMD" ] ; then 78 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 79 | 80 | Please set the JAVA_HOME variable in your environment to match the 81 | location of your Java installation." 82 | fi 83 | else 84 | JAVACMD="java" 85 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 86 | 87 | Please set the JAVA_HOME variable in your environment to match the 88 | location of your Java installation." 89 | fi 90 | 91 | # Increase the maximum file descriptors if we can. 92 | if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then 93 | MAX_FD_LIMIT=`ulimit -H -n` 94 | if [ $? -eq 0 ] ; then 95 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 96 | MAX_FD="$MAX_FD_LIMIT" 97 | fi 98 | ulimit -n $MAX_FD 99 | if [ $? -ne 0 ] ; then 100 | warn "Could not set maximum file descriptor limit: $MAX_FD" 101 | fi 102 | else 103 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 104 | fi 105 | fi 106 | 107 | # For Darwin, add options to specify how the application appears in the dock 108 | if $darwin; then 109 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 110 | fi 111 | 112 | # For Cygwin, switch paths to Windows format before running java 113 | if $cygwin ; then 114 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 115 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 116 | JAVACMD=`cygpath --unix "$JAVACMD"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Escape application args 158 | save () { 159 | for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done 160 | echo " " 161 | } 162 | APP_ARGS=$(save "$@") 163 | 164 | # Collect all arguments for the java command, following the shell quoting and substitution rules 165 | eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" 166 | 167 | # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong 168 | if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then 169 | cd "$(dirname "$0")" 170 | fi 171 | 172 | exec "$JAVACMD" "$@" 173 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | set DIRNAME=%~dp0 12 | if "%DIRNAME%" == "" set DIRNAME=. 13 | set APP_BASE_NAME=%~n0 14 | set APP_HOME=%DIRNAME% 15 | 16 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 17 | set DEFAULT_JVM_OPTS= 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windows variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | 53 | :win9xME_args 54 | @rem Slurp the command line arguments. 55 | set CMD_LINE_ARGS= 56 | set _SKIP=2 57 | 58 | :win9xME_args_slurp 59 | if "x%~1" == "x" goto execute 60 | 61 | set CMD_LINE_ARGS=%* 62 | 63 | :execute 64 | @rem Setup the command line 65 | 66 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 67 | 68 | @rem Execute Gradle 69 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 70 | 71 | :end 72 | @rem End local scope for the variables with windows NT shell 73 | if "%ERRORLEVEL%"=="0" goto mainEnd 74 | 75 | :fail 76 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 77 | rem the _cmd.exe /c_ return code! 78 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 79 | exit /b 1 80 | 81 | :mainEnd 82 | if "%OS%"=="Windows_NT" endlocal 83 | 84 | :omega 85 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bbakerman/java-dataloader/52bd96c7e2d34d1a758a3c826c008aa78b4a2416/settings.gradle -------------------------------------------------------------------------------- /src/main/java/org/dataloader/BatchLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 The original author or authors 3 | * 4 | * All rights reserved. This program and the accompanying materials 5 | * are made available under the terms of the Eclipse Public License v1.0 6 | * and Apache License v2.0 which accompanies this distribution. 7 | * 8 | * The Eclipse Public License is available at 9 | * http://www.eclipse.org/legal/epl-v10.html 10 | * 11 | * The Apache License v2.0 is available at 12 | * http://www.opensource.org/licenses/apache2.0.php 13 | * 14 | * You may elect to redistribute this code under either of these licenses. 15 | */ 16 | 17 | package org.dataloader; 18 | 19 | import java.util.List; 20 | import java.util.concurrent.CompletionStage; 21 | 22 | /** 23 | * A function that is invoked for batch loading a list of data values indicated by the provided list of keys. The 24 | * function returns a promise of a list of results of individual load requests. 25 | * 26 | * There are a few constraints that must be upheld: 27 | *
    28 | *
  • The list of values must be the same size as the list of keys.
    • 29 | *
  • Each index in the list of values must correspond to the same index in the list of keys.
    • 30 | *
31 | * 32 | * For example, if your batch function was provided the list of keys: 33 | * 34 | * 
35 |  *  [
36 |  *      2, 9, 6, 1
37 |  *  ]
38 |  *
39 | * 40 | * and loading from a back-end service returned this list of values: 41 | * 42 | * 
43 |  *  [
44 |  *      { id: 9, name: 'Chicago' },
45 |  *      { id: 1, name: 'New York' },
46 |  *      { id: 2, name: 'San Francisco' },
47 |  *  ]
48 |  *
49 | * 50 | * then the batch loader function contract has been broken. 51 | * 52 | * The back-end service returned results in a different order than we requested, likely because it was more efficient for it to 53 | * do so. Also, it omitted a result for key 6, which we may interpret as no value existing for that key. 54 | * 55 | * To uphold the constraints of the batch function, it must return an List of values the same length as 56 | * the List of keys, and re-order them to ensure each index aligns with the original keys [ 2, 9, 6, 1 ]: 57 | * 58 | * 
59 |  *  [
60 |  *      { id: 2, name: 'San Francisco' },
61 |  *      { id: 9, name: 'Chicago' },
62 |  *      null,
63 |  *      { id: 1, name: 'New York' }
64 |  * ]
65 |  *
66 | * 67 | * @param type parameter indicating the type of keys to use for data load requests. 68 | * @param type parameter indicating the type of values returned 69 | * 70 | * @author Arnold Schrijver 71 | * @author Brad Baker 72 | */ 73 | @FunctionalInterface 74 | public interface BatchLoader { 75 | 76 | /** 77 | * Called to batch load the provided keys and return a promise to a list of values 78 | * 79 | * @param keys the collection of keys to load 80 | * 81 | * @return a promise of the values for those keys 82 | */ 83 | CompletionStage> load(List keys); 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/org/dataloader/CacheKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 The original author or authors 3 | * 4 | * All rights reserved. This program and the accompanying materials 5 | * are made available under the terms of the Eclipse Public License v1.0 6 | * and Apache License v2.0 which accompanies this distribution. 7 | * 8 | * The Eclipse Public License is available at 9 | * http://www.eclipse.org/legal/epl-v10.html 10 | * 11 | * The Apache License v2.0 is available at 12 | * http://www.opensource.org/licenses/apache2.0.php 13 | * 14 | * You may elect to redistribute this code under either of these licenses. 15 | */ 16 | 17 | package org.dataloader; 18 | 19 | /** 20 | * Function that is invoked on input keys of type {@code K} to derive keys that are required by the {@link CacheMap} 21 | * implementation. 22 | * 23 | * @param type parameter indicating the type of the input key 24 | * 25 | * @author Arnold Schrijver 26 | */ 27 | @FunctionalInterface 28 | public interface CacheKey { 29 | 30 | /** 31 | * Returns the cache key that is created from the provided input key. 32 | * 33 | * @param input the input key 34 | * 35 | * @return the cache key 36 | */ 37 | Object getKey(K input); 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/dataloader/CacheMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 The original author or authors 3 | * 4 | * All rights reserved. This program and the accompanying materials 5 | * are made available under the terms of the Eclipse Public License v1.0 6 | * and Apache License v2.0 which accompanies this distribution. 7 | * 8 | * The Eclipse Public License is available at 9 | * http://www.eclipse.org/legal/epl-v10.html 10 | * 11 | * The Apache License v2.0 is available at 12 | * http://www.opensource.org/licenses/apache2.0.php 13 | * 14 | * You may elect to redistribute this code under either of these licenses. 15 | */ 16 | 17 | package org.dataloader; 18 | 19 | import org.dataloader.impl.DefaultCacheMap; 20 | 21 | import java.util.concurrent.CompletableFuture; 22 | 23 | /** 24 | * Cache map interface for data loaders that use caching. 25 | *

26 | * The default implementation used by the data loader is based on a {@link java.util.LinkedHashMap}. Note that the 27 | * implementation could also have used a regular {@link java.util.Map} instead of this {@link CacheMap}, but 28 | * this aligns better to the reference data loader implementation provided by Facebook 29 | *

30 | * Also it doesn't require you to implement the full set of map overloads, just the required methods. 31 | * 32 | * @param type parameter indicating the type of the cache keys 33 | * @param type parameter indicating the type of the data that is cached 34 | * 35 | * @author Arnold Schrijver 36 | * @author Brad Baker 37 | */ 38 | public interface CacheMap { 39 | 40 | /** 41 | * Creates a new cache map, using the default implementation that is based on a {@link java.util.LinkedHashMap}. 42 | * 43 | * @param type parameter indicating the type of the cache keys 44 | * @param type parameter indicating the type of the data that is cached 45 | * 46 | * @return the cache map 47 | */ 48 | static CacheMap> simpleMap() { 49 | return new DefaultCacheMap<>(); 50 | } 51 | 52 | /** 53 | * Checks whether the specified key is contained in the cach map. 54 | * 55 | * @param key the key to check 56 | * 57 | * @return {@code true} if the cache contains the key, {@code false} otherwise 58 | */ 59 | boolean containsKey(U key); 60 | 61 | /** 62 | * Gets the specified key from the cache map. 63 | *

64 | * May throw an exception if the key does not exists, depending on the cache map implementation that is used, 65 | * so be sure to check {@link CacheMap#containsKey(Object)} first. 66 | * 67 | * @param key the key to retrieve 68 | * 69 | * @return the cached value, or {@code null} if not found (depends on cache implementation) 70 | */ 71 | V get(U key); 72 | 73 | /** 74 | * Creates a new cache map entry with the specified key and value, or updates the value if the key already exists. 75 | * 76 | * @param key the key to cache 77 | * @param value the value to cache 78 | * 79 | * @return the cache map for fluent coding 80 | */ 81 | CacheMap set(U key, V value); 82 | 83 | /** 84 | * Deletes the entry with the specified key from the cache map, if it exists. 85 | * 86 | * @param key the key to delete 87 | * 88 | * @return the cache map for fluent coding 89 | */ 90 | CacheMap delete(U key); 91 | 92 | /** 93 | * Clears all entries of the cache map 94 | * 95 | * @return the cache map for fluent coding 96 | */ 97 | CacheMap clear(); 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/org/dataloader/DataLoader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 The original author or authors 3 | * 4 | * All rights reserved. This program and the accompanying materials 5 | * are made available under the terms of the Eclipse Public License v1.0 6 | * and Apache License v2.0 which accompanies this distribution. 7 | * 8 | * The Eclipse Public License is available at 9 | * http://www.eclipse.org/legal/epl-v10.html 10 | * 11 | * The Apache License v2.0 is available at 12 | * http://www.opensource.org/licenses/apache2.0.php 13 | * 14 | * You may elect to redistribute this code under either of these licenses. 15 | */ 16 | 17 | package org.dataloader; 18 | 19 | import org.dataloader.impl.CompletableFutureKit; 20 | 21 | import java.util.ArrayList; 22 | import java.util.Collection; 23 | import java.util.LinkedHashMap; 24 | import java.util.List; 25 | import java.util.Map; 26 | import java.util.concurrent.CompletableFuture; 27 | import java.util.stream.Collectors; 28 | 29 | import static java.util.Collections.emptyList; 30 | import static java.util.Collections.singletonList; 31 | import static org.dataloader.impl.Assertions.assertState; 32 | import static org.dataloader.impl.Assertions.nonNull; 33 | 34 | /** 35 | * Data loader is a utility class that allows batch loading of data that is identified by a set of unique keys. For 36 | * each key that is loaded a separate {@link CompletableFuture} is returned, that completes as the batch function completes. 37 | *

38 | * With batching enabled the execution will start after calling {@link DataLoader#dispatch()}, causing the queue of 39 | * loaded keys to be sent to the batch function, clears the queue, and returns a promise to the values. 40 | *

41 | * As batch functions are executed the resulting futures are cached using a cache implementation of choice, so they 42 | * will only execute once. Individual cache keys can be cleared, so they will be re-fetched when referred to again. 43 | * It is also possible to clear the cache entirely, and prime it with values before they are used. 44 | *

45 | * Both caching and batching can be disabled. Configuration of the data loader is done by providing a 46 | * {@link DataLoaderOptions} instance on creation. 47 | * 48 | * @param type parameter indicating the type of the data load keys 49 | * @param type parameter indicating the type of the data that is returned 50 | * 51 | * @author Arnold Schrijver 52 | * @author Brad Baker 53 | */ 54 | public class DataLoader { 55 | 56 | private final BatchLoader batchLoadFunction; 57 | private final DataLoaderOptions loaderOptions; 58 | private final CacheMap> futureCache; 59 | private final Map> loaderQueue; 60 | 61 | /** 62 | * Creates a new data loader with the provided batch load function, and default options. 63 | * 64 | * @param batchLoadFunction the batch load function to use 65 | */ 66 | public DataLoader(BatchLoader batchLoadFunction) { 67 | this(batchLoadFunction, null); 68 | } 69 | 70 | /** 71 | * Creates a new data loader with the provided batch load function and options. 72 | * 73 | * @param batchLoadFunction the batch load function to use 74 | * @param options the batch load options 75 | */ 76 | public DataLoader(BatchLoader batchLoadFunction, DataLoaderOptions options) { 77 | this.batchLoadFunction = nonNull(batchLoadFunction); 78 | this.loaderOptions = options == null ? new DataLoaderOptions() : options; 79 | this.futureCache = determineCacheMap(loaderOptions); 80 | // order of keys matter in data loader 81 | this.loaderQueue = new LinkedHashMap<>(); 82 | } 83 | 84 | @SuppressWarnings("unchecked") 85 | private CacheMap> determineCacheMap(DataLoaderOptions loaderOptions) { 86 | return loaderOptions.cacheMap().isPresent() ? (CacheMap>) loaderOptions.cacheMap().get() : CacheMap.simpleMap(); 87 | } 88 | 89 | /** 90 | * Requests to load the data with the specified key asynchronously, and returns a future of the resulting value. 91 | *

92 | * If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to 93 | * start batch execution. If you forget this call the future will never be completed (unless already completed, 94 | * and returned from cache). 95 | * 96 | * @param key the key to load 97 | * 98 | * @return the future of the value 99 | */ 100 | public CompletableFuture load(K key) { 101 | Object cacheKey = getCacheKey(nonNull(key)); 102 | synchronized (futureCache) { 103 | if (loaderOptions.cachingEnabled() && futureCache.containsKey(cacheKey)) { 104 | return futureCache.get(cacheKey); 105 | } 106 | } 107 | 108 | CompletableFuture future = new CompletableFuture<>(); 109 | if (loaderOptions.batchingEnabled()) { 110 | synchronized (loaderQueue) { 111 | loaderQueue.put(key, future); 112 | } 113 | } else { 114 | // immediate execution of batch function 115 | CompletableFuture> batchedLoad = batchLoadFunction 116 | .load(singletonList(key)) 117 | .toCompletableFuture(); 118 | future = batchedLoad 119 | .thenApply(list -> list.get(0)); 120 | } 121 | if (loaderOptions.cachingEnabled()) { 122 | synchronized (futureCache) { 123 | futureCache.set(cacheKey, future); 124 | } 125 | } 126 | return future; 127 | } 128 | 129 | /** 130 | * Requests to load the list of data provided by the specified keys asynchronously, and returns a composite future 131 | * of the resulting values. 132 | *

133 | * If batching is enabled (the default), you'll have to call {@link DataLoader#dispatch()} at a later stage to 134 | * start batch execution. If you forget this call the future will never be completed (unless already completed, 135 | * and returned from cache). 136 | * 137 | * @param keys the list of keys to load 138 | * 139 | * @return the composite future of the list of values 140 | */ 141 | public CompletableFuture> loadMany(List keys) { 142 | synchronized (loaderQueue) { 143 | 144 | List> collect = keys.stream() 145 | .map(this::load) 146 | .collect(Collectors.toList()); 147 | 148 | return CompletableFutureKit.allOf(collect); 149 | } 150 | } 151 | 152 | /** 153 | * Dispatches the queued load requests to the batch execution function and returns a promise of the result. 154 | *

155 | * If batching is disabled, or there are no queued requests, then a succeeded promise is returned. 156 | * 157 | * @return the promise of the queued load requests 158 | */ 159 | public CompletableFuture> dispatch() { 160 | // 161 | // we copy the pre-loaded set of futures ready for dispatch 162 | final List keys = new ArrayList<>(); 163 | final List> queuedFutures = new ArrayList<>(); 164 | synchronized (loaderQueue) { 165 | loaderQueue.forEach((key, future) -> { 166 | keys.add(key); 167 | queuedFutures.add(future); 168 | }); 169 | loaderQueue.clear(); 170 | } 171 | if (!loaderOptions.batchingEnabled() || keys.size() == 0) { 172 | return CompletableFuture.completedFuture(emptyList()); 173 | } 174 | // 175 | // order of keys -> values matter in data loader hence the use of linked hash map 176 | // 177 | // See https://github.com/facebook/dataloader/blob/master/README.md for more details 178 | // 179 | 180 | // 181 | // when the promised list of values completes, we transfer the values into 182 | // the previously cached future objects that the client already has been given 183 | // via calls to load("foo") and loadMany(["foo","bar"]) 184 | // 185 | int maxBatchSize = loaderOptions.maxBatchSize(); 186 | if (maxBatchSize > 0 && maxBatchSize < keys.size()) { 187 | return sliceIntoBatchesOfBatches(keys, queuedFutures, maxBatchSize); 188 | } else { 189 | return dispatchQueueBatch(keys, queuedFutures); 190 | } 191 | } 192 | 193 | private CompletableFuture> sliceIntoBatchesOfBatches(List keys, List> queuedFutures, int maxBatchSize) { 194 | // the number of keys is > than what the batch loader function can accept 195 | // so make multiple calls to the loader 196 | List>> allBatches = new ArrayList<>(); 197 | int len = keys.size(); 198 | int batchCount = (int) Math.ceil(len / (double) maxBatchSize); 199 | for (int i = 0; i < batchCount; i++) { 200 | 201 | int fromIndex = i * maxBatchSize; 202 | int toIndex = Math.min((i + 1) * maxBatchSize, len); 203 | 204 | List subKeys = keys.subList(fromIndex, toIndex); 205 | List> subFutures = queuedFutures.subList(fromIndex, toIndex); 206 | 207 | allBatches.add(dispatchQueueBatch(subKeys, subFutures)); 208 | } 209 | // 210 | // now reassemble all the futures into one that is the complete set of results 211 | return CompletableFuture.allOf(allBatches.toArray(new CompletableFuture[allBatches.size()])) 212 | .thenApply(v -> allBatches.stream() 213 | .map(CompletableFuture::join) 214 | .flatMap(Collection::stream) 215 | .collect(Collectors.toList())); 216 | } 217 | 218 | private CompletableFuture> dispatchQueueBatch(List keys, List> queuedFutures) { 219 | return batchLoadFunction.load(keys) 220 | .toCompletableFuture() 221 | .thenApply(values -> { 222 | assertState(keys.size() == values.size(), "The size of the promised values MUST be the same size as the key list"); 223 | 224 | for (int idx = 0; idx < queuedFutures.size(); idx++) { 225 | Object value = values.get(idx); 226 | CompletableFuture future = queuedFutures.get(idx); 227 | if (value instanceof Throwable) { 228 | future.completeExceptionally((Throwable) value); 229 | } else { 230 | @SuppressWarnings("unchecked") 231 | V val = (V) value; 232 | future.complete(val); 233 | } 234 | } 235 | return values; 236 | }).exceptionally(ex -> { 237 | for (int idx = 0; idx < queuedFutures.size(); idx++) { 238 | K key = keys.get(idx); 239 | CompletableFuture future = queuedFutures.get(idx); 240 | future.completeExceptionally(ex); 241 | // clear any cached view of this key 242 | clear(key); 243 | } 244 | return emptyList(); 245 | }); 246 | } 247 | 248 | /** 249 | * Normally {@link #dispatch()} is an asynchronous operation but this version will 'join' on the 250 | * results if dispatch and wait for them to complete. If the {@link CompletableFuture} callbacks make more 251 | * calls to this data loader then the {@link #dispatchDepth()} will be > 0 and this method will loop 252 | * around and wait for any other extra batch loads to occur. 253 | * 254 | * @return the list of all results when the {@link #dispatchDepth()} reached 0 255 | */ 256 | public List dispatchAndJoin() { 257 | List results = new ArrayList<>(); 258 | 259 | List joinedResults = dispatch().join(); 260 | results.addAll(joinedResults); 261 | while (this.dispatchDepth() > 0) { 262 | joinedResults = dispatch().join(); 263 | results.addAll(joinedResults); 264 | } 265 | return results; 266 | } 267 | 268 | 269 | /** 270 | * @return the depth of the batched key loads that need to be dispatched 271 | */ 272 | public int dispatchDepth() { 273 | synchronized (loaderQueue) { 274 | return loaderQueue.size(); 275 | } 276 | } 277 | 278 | 279 | /** 280 | * Clears the future with the specified key from the cache, if caching is enabled, so it will be re-fetched 281 | * on the next load request. 282 | * 283 | * @param key the key to remove 284 | * 285 | * @return the data loader for fluent coding 286 | */ 287 | public DataLoader clear(K key) { 288 | Object cacheKey = getCacheKey(key); 289 | synchronized (futureCache) { 290 | futureCache.delete(cacheKey); 291 | } 292 | return this; 293 | } 294 | 295 | /** 296 | * Clears the entire cache map of the loader. 297 | * 298 | * @return the data loader for fluent coding 299 | */ 300 | public DataLoader clearAll() { 301 | synchronized (futureCache) { 302 | futureCache.clear(); 303 | } 304 | return this; 305 | } 306 | 307 | /** 308 | * Primes the cache with the given key and value. 309 | * 310 | * @param key the key 311 | * @param value the value 312 | * 313 | * @return the data loader for fluent coding 314 | */ 315 | public DataLoader prime(K key, V value) { 316 | Object cacheKey = getCacheKey(key); 317 | synchronized (futureCache) { 318 | if (!futureCache.containsKey(cacheKey)) { 319 | futureCache.set(cacheKey, CompletableFuture.completedFuture(value)); 320 | } 321 | } 322 | return this; 323 | } 324 | 325 | /** 326 | * Primes the cache with the given key and error. 327 | * 328 | * @param key the key 329 | * @param error the exception to prime instead of a value 330 | * 331 | * @return the data loader for fluent coding 332 | */ 333 | public DataLoader prime(K key, Exception error) { 334 | Object cacheKey = getCacheKey(key); 335 | if (!futureCache.containsKey(cacheKey)) { 336 | futureCache.set(cacheKey, CompletableFutureKit.failedFuture(error)); 337 | } 338 | return this; 339 | } 340 | 341 | /** 342 | * Gets the object that is used in the internal cache map as key, by applying the cache key function to 343 | * the provided key. 344 | *

345 | * If no cache key function is present in {@link DataLoaderOptions}, then the returned value equals the input key. 346 | * 347 | * @param key the input key 348 | * 349 | * @return the cache key after the input is transformed with the cache key function 350 | */ 351 | @SuppressWarnings("unchecked") 352 | public Object getCacheKey(K key) { 353 | return loaderOptions.cacheKeyFunction().isPresent() ? 354 | loaderOptions.cacheKeyFunction().get().getKey(key) : key; 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /src/main/java/org/dataloader/DataLoaderOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 The original author or authors 3 | * 4 | * All rights reserved. This program and the accompanying materials 5 | * are made available under the terms of the Eclipse Public License v1.0 6 | * and Apache License v2.0 which accompanies this distribution. 7 | * 8 | * The Eclipse Public License is available at 9 | * http://www.eclipse.org/legal/epl-v10.html 10 | * 11 | * The Apache License v2.0 is available at 12 | * http://www.opensource.org/licenses/apache2.0.php 13 | * 14 | * You may elect to redistribute this code under either of these licenses. 15 | */ 16 | 17 | package org.dataloader; 18 | 19 | import org.dataloader.impl.Assertions; 20 | 21 | import java.util.Optional; 22 | 23 | /** 24 | * Configuration options for {@link DataLoader} instances. 25 | * 26 | * @author Arnold Schrijver 27 | */ 28 | public class DataLoaderOptions { 29 | 30 | private boolean batchingEnabled; 31 | private boolean cachingEnabled; 32 | private CacheKey cacheKeyFunction; 33 | private CacheMap cacheMap; 34 | private int maxBatchSize; 35 | 36 | /** 37 | * Creates a new data loader options with default settings. 38 | */ 39 | public DataLoaderOptions() { 40 | batchingEnabled = true; 41 | cachingEnabled = true; 42 | maxBatchSize = -1; 43 | } 44 | 45 | /** 46 | * Clones the provided data loader options. 47 | * 48 | * @param other the other options instance 49 | */ 50 | public DataLoaderOptions(DataLoaderOptions other) { 51 | Assertions.nonNull(other); 52 | this.batchingEnabled = other.batchingEnabled; 53 | this.cachingEnabled = other.cachingEnabled; 54 | this.cacheKeyFunction = other.cacheKeyFunction; 55 | this.cacheMap = other.cacheMap; 56 | this.maxBatchSize = other.maxBatchSize; 57 | } 58 | 59 | public static DataLoaderOptions create() { 60 | return new DataLoaderOptions(); 61 | } 62 | 63 | /** 64 | * Option that determines whether to use batching (the default), or not. 65 | * 66 | * @return {@code true} when batching is enabled, {@code false} otherwise 67 | */ 68 | public boolean batchingEnabled() { 69 | return batchingEnabled; 70 | } 71 | 72 | /** 73 | * Sets the option that determines whether batch loading is enabled. 74 | * 75 | * @param batchingEnabled {@code true} to enable batch loading, {@code false} otherwise 76 | * 77 | * @return the data loader options for fluent coding 78 | */ 79 | public DataLoaderOptions setBatchingEnabled(boolean batchingEnabled) { 80 | this.batchingEnabled = batchingEnabled; 81 | return this; 82 | } 83 | 84 | /** 85 | * Option that determines whether to use caching of futures (the default), or not. 86 | * 87 | * @return {@code true} when caching is enabled, {@code false} otherwise 88 | */ 89 | public boolean cachingEnabled() { 90 | return cachingEnabled; 91 | } 92 | 93 | /** 94 | * Sets the option that determines whether caching is enabled. 95 | * 96 | * @param cachingEnabled {@code true} to enable caching, {@code false} otherwise 97 | * 98 | * @return the data loader options for fluent coding 99 | */ 100 | public DataLoaderOptions setCachingEnabled(boolean cachingEnabled) { 101 | this.cachingEnabled = cachingEnabled; 102 | return this; 103 | } 104 | 105 | /** 106 | * Gets an (optional) function to invoke for creation of the cache key, if caching is enabled. 107 | *

108 | * If missing the cache key defaults to the {@code key} type parameter of the data loader of type {@code K}. 109 | * 110 | * @return an optional with the function, or empty optional 111 | */ 112 | public Optional cacheKeyFunction() { 113 | return Optional.ofNullable(cacheKeyFunction); 114 | } 115 | 116 | /** 117 | * Sets the function to use for creating the cache key, if caching is enabled. 118 | * 119 | * @param cacheKeyFunction the cache key function to use 120 | * 121 | * @return the data loader options for fluent coding 122 | */ 123 | public DataLoaderOptions setCacheKeyFunction(CacheKey cacheKeyFunction) { 124 | this.cacheKeyFunction = cacheKeyFunction; 125 | return this; 126 | } 127 | 128 | /** 129 | * Gets the (optional) cache map implementation that is used for caching, if caching is enabled. 130 | *

131 | * If missing a standard {@link java.util.LinkedHashMap} will be used as the cache implementation. 132 | * 133 | * @return an optional with the cache map instance, or empty 134 | */ 135 | public Optional cacheMap() { 136 | return Optional.ofNullable(cacheMap); 137 | } 138 | 139 | /** 140 | * Sets the cache map implementation to use for caching, if caching is enabled. 141 | * 142 | * @param cacheMap the cache map instance 143 | * 144 | * @return the data loader options for fluent coding 145 | */ 146 | public DataLoaderOptions setCacheMap(CacheMap cacheMap) { 147 | this.cacheMap = cacheMap; 148 | return this; 149 | } 150 | 151 | /** 152 | * Gets the maximum number of keys that will be presented to the {@link BatchLoader} function 153 | * before they are split into multiple class 154 | * 155 | * @return the maximum batch size or -1 if there is no limit 156 | */ 157 | public int maxBatchSize() { 158 | return maxBatchSize; 159 | } 160 | 161 | /** 162 | * Sets the maximum number of keys that will be presented to the {@link BatchLoader} function 163 | * before they are split into multiple class 164 | * 165 | * @param maxBatchSize the maximum batch size 166 | * 167 | * @return the data loader options for fluent coding 168 | */ 169 | public DataLoaderOptions setMaxBatchSize(int maxBatchSize) { 170 | this.maxBatchSize = maxBatchSize; 171 | return this; 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /src/main/java/org/dataloader/impl/Assertions.java: -------------------------------------------------------------------------------- 1 | package org.dataloader.impl; 2 | 3 | import java.util.Objects; 4 | 5 | public class Assertions { 6 | 7 | public static void assertState(boolean state, String message) { 8 | if (!state) { 9 | throw new AssertionException(message); 10 | } 11 | } 12 | 13 | public static T nonNull(T t) { 14 | return Objects.requireNonNull(t, "nonNull object required"); 15 | } 16 | 17 | private static class AssertionException extends IllegalStateException { 18 | public AssertionException(String message) { 19 | super(message); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/dataloader/impl/CompletableFutureKit.java: -------------------------------------------------------------------------------- 1 | package org.dataloader.impl; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.CompletableFuture; 5 | import java.util.concurrent.ExecutionException; 6 | 7 | import static java.util.stream.Collectors.toList; 8 | 9 | /** 10 | * Some really basic helpers when working with CompletableFutures 11 | */ 12 | public class CompletableFutureKit { 13 | 14 | public static CompletableFuture failedFuture(Exception e) { 15 | CompletableFuture future = new CompletableFuture<>(); 16 | future.completeExceptionally(e); 17 | return future; 18 | } 19 | 20 | public static Throwable cause(CompletableFuture completableFuture) { 21 | if (!completableFuture.isCompletedExceptionally()) { 22 | return null; 23 | } 24 | try { 25 | completableFuture.get(); 26 | return null; 27 | } catch (InterruptedException e) { 28 | return e; 29 | } catch (ExecutionException e) { 30 | Throwable cause = e.getCause(); 31 | if (cause != null) { 32 | return cause; 33 | } 34 | return e; 35 | } 36 | } 37 | 38 | public static boolean succeeded(CompletableFuture future) { 39 | return future.isDone() && !future.isCompletedExceptionally(); 40 | } 41 | 42 | public static boolean failed(CompletableFuture future) { 43 | return future.isDone() && future.isCompletedExceptionally(); 44 | } 45 | 46 | public static CompletableFuture> allOf(List> cfs) { 47 | return CompletableFuture.allOf(cfs.toArray(new CompletableFuture[cfs.size()])) 48 | .thenApply(v -> cfs.stream() 49 | .map(CompletableFuture::join) 50 | .collect(toList()) 51 | ); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/org/dataloader/impl/DefaultCacheMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 The original author or authors 3 | * 4 | * All rights reserved. This program and the accompanying materials 5 | * are made available under the terms of the Eclipse Public License v1.0 6 | * and Apache License v2.0 which accompanies this distribution. 7 | * 8 | * The Eclipse Public License is available at 9 | * http://www.eclipse.org/legal/epl-v10.html 10 | * 11 | * The Apache License v2.0 is available at 12 | * http://www.opensource.org/licenses/apache2.0.php 13 | * 14 | * You may elect to redistribute this code under either of these licenses. 15 | */ 16 | 17 | package org.dataloader.impl; 18 | 19 | import org.dataloader.CacheMap; 20 | 21 | import java.util.HashMap; 22 | import java.util.Map; 23 | 24 | /** 25 | * Default implementation of {@link CacheMap} that is based on a regular {@link java.util.LinkedHashMap}. 26 | * 27 | * @param type parameter indicating the type of the cache keys 28 | * @param type parameter indicating the type of the data that is cached 29 | * 30 | * @author Arnold Schrijver 31 | */ 32 | public class DefaultCacheMap implements CacheMap { 33 | 34 | private Map cache; 35 | 36 | /** 37 | * Default constructor 38 | */ 39 | public DefaultCacheMap() { 40 | cache = new HashMap<>(); 41 | } 42 | 43 | /** 44 | * {@inheritDoc} 45 | */ 46 | @Override 47 | public boolean containsKey(U key) { 48 | return cache.containsKey(key); 49 | } 50 | 51 | /** 52 | * {@inheritDoc} 53 | */ 54 | @Override 55 | public V get(U key) { 56 | return cache.get(key); 57 | } 58 | 59 | /** 60 | * {@inheritDoc} 61 | */ 62 | @Override 63 | public CacheMap set(U key, V value) { 64 | cache.put(key, value); 65 | return this; 66 | } 67 | 68 | /** 69 | * {@inheritDoc} 70 | */ 71 | @Override 72 | public CacheMap delete(U key) { 73 | cache.remove(key); 74 | return this; 75 | } 76 | 77 | /** 78 | * {@inheritDoc} 79 | */ 80 | @Override 81 | public CacheMap clear() { 82 | cache.clear(); 83 | return this; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/org/dataloader/impl/PromisedValues.java: -------------------------------------------------------------------------------- 1 | package org.dataloader.impl; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.CancellationException; 5 | import java.util.concurrent.CompletableFuture; 6 | import java.util.concurrent.CompletionException; 7 | import java.util.concurrent.CompletionStage; 8 | import java.util.function.Consumer; 9 | 10 | import static java.util.Arrays.asList; 11 | 12 | /** 13 | * This allows multiple {@link CompletionStage}s to be combined together and completed 14 | * as one and should something go wrong, instead of throwing {@link CompletionException}s it captures the cause and returns null for that 15 | * data value, other wise it allows you to access them as a list of values. 16 | * 17 | * This class really encapsulate a list of promised values. It is considered finished when all of the underlying futures 18 | * are finished. 19 | * 20 | * You can get that list of values via {@link #toList()}. You can also compose a {@link CompletableFuture} of that 21 | * list of values via {@link #toCompletableFuture()} ()} 22 | * 23 | * @author Brad Baker 24 | */ 25 | public interface PromisedValues { 26 | 27 | /** 28 | * Returns a new {@link PromisedValues} that is completed when all of 29 | * the given {@link CompletionStage}s complete. If any of the given 30 | * {@link CompletionStage}s complete exceptionally, then the returned 31 | * {@link PromisedValues} also does so. 32 | * 33 | * @param cfs the {@link CompletionStage}s to combine 34 | * @param the type of values 35 | * 36 | * @return a new PromisedValues 37 | */ 38 | static PromisedValues allOf(List> cfs) { 39 | return PromisedValuesImpl.combineAllOf(cfs); 40 | } 41 | 42 | /** 43 | * Returns a new {@link PromisedValues} that is completed when all of 44 | * the given {@link CompletionStage}s complete. If any of the given 45 | * {@link CompletionStage}s complete exceptionally, then the returned 46 | * {@link PromisedValues} also does so. 47 | * 48 | * @param f1 the 1st completable future 49 | * @param f2 the 2nd completable future 50 | * @param the type of values 51 | * 52 | * @return a new PromisedValues 53 | */ 54 | static PromisedValues allOf(CompletionStage f1, CompletionStage f2) { 55 | return PromisedValuesImpl.combineAllOf(asList(f1, f2)); 56 | } 57 | 58 | /** 59 | * Returns a new {@link PromisedValues} that is completed when all of 60 | * the given {@link CompletionStage}s complete. If any of the given 61 | * {@link CompletionStage}s complete exceptionally, then the returned 62 | * {@link PromisedValues} also does so. 63 | * 64 | * @param f1 the 1st completable future 65 | * @param f2 the 2nd completable future 66 | * @param f3 the 3rd completable future 67 | * @param the type of values 68 | * 69 | * @return a new PromisedValues 70 | */ 71 | static PromisedValues allOf(CompletionStage f1, CompletionStage f2, CompletionStage f3) { 72 | return PromisedValuesImpl.combineAllOf(asList(f1, f2, f3)); 73 | } 74 | 75 | 76 | /** 77 | * Returns a new {@link PromisedValues} that is completed when all of 78 | * the given {@link CompletionStage}s complete. If any of the given 79 | * {@link CompletionStage}s complete exceptionally, then the returned 80 | * {@link PromisedValues} also does so. 81 | * 82 | * @param f1 the 1st completable future 83 | * @param f2 the 2nd completable future 84 | * @param f3 the 3rd completable future 85 | * @param f4 the 4th completable future 86 | * @param the type of values 87 | * 88 | * @return a new PromisedValues 89 | */ 90 | static PromisedValues allOf(CompletionStage f1, CompletionStage f2, CompletionStage f3, CompletionStage f4) { 91 | return PromisedValuesImpl.combineAllOf(asList(f1, f2, f3, f4)); 92 | } 93 | 94 | 95 | /** 96 | * Returns a new {@link PromisedValues} that is completed when all of 97 | * the given {@link PromisedValues}s complete. If any of the given 98 | * {@link PromisedValues}s complete exceptionally, then the returned 99 | * {@link PromisedValues} also does so. 100 | * 101 | * @param cfs the list to combine 102 | * @param the type of values 103 | * 104 | * @return a new PromisedValues 105 | */ 106 | static PromisedValues allPromisedValues(List> cfs) { 107 | return PromisedValuesImpl.combinePromisedValues(cfs); 108 | } 109 | 110 | /** 111 | * Returns a new {@link PromisedValues} that is completed when all of 112 | * the given {@link PromisedValues}s complete. If any of the given 113 | * {@link PromisedValues}s complete exceptionally, then the returned 114 | * {@link PromisedValues} also does so. 115 | * 116 | * @param pv1 the 1st promised value 117 | * @param pv2 the 2nd promised value 118 | * @param the type of values 119 | * 120 | * @return a new PromisedValues 121 | */ 122 | static PromisedValues allPromisedValues(PromisedValues pv1, PromisedValues pv2) { 123 | return PromisedValuesImpl.combinePromisedValues(asList(pv1, pv2)); 124 | } 125 | 126 | /** 127 | * Returns a new {@link PromisedValues} that is completed when all of 128 | * the given {@link PromisedValues}s complete. If any of the given 129 | * {@link PromisedValues}s complete exceptionally, then the returned 130 | * {@link PromisedValues} also does so. 131 | * 132 | * @param pv1 the 1st promised value 133 | * @param pv2 the 2nd promised value 134 | * @param pv3 the 3rd promised value 135 | * @param the type of values 136 | * 137 | * @return a new PromisedValues 138 | */ 139 | static PromisedValues allPromisedValues(PromisedValues pv1, PromisedValues pv2, PromisedValues pv3) { 140 | return PromisedValuesImpl.combinePromisedValues(asList(pv1, pv2, pv3)); 141 | } 142 | 143 | /** 144 | * Returns a new {@link PromisedValues} that is completed when all of 145 | * the given {@link PromisedValues}s complete. If any of the given 146 | * {@link PromisedValues}s complete exceptionally, then the returned 147 | * {@link PromisedValues} also does so. 148 | * 149 | * @param pv1 the 1st promised value 150 | * @param pv2 the 2nd promised value 151 | * @param pv3 the 3rd promised value 152 | * @param pv4 the 4th promised value 153 | * @param the type of values 154 | * 155 | * @return a new PromisedValues 156 | */ 157 | static PromisedValues allPromisedValues(PromisedValues pv1, PromisedValues pv2, PromisedValues pv3, PromisedValues pv4) { 158 | return PromisedValuesImpl.combinePromisedValues(asList(pv1, pv2, pv3, pv4)); 159 | } 160 | 161 | 162 | /** 163 | * When the all the futures complete, this call back will be invoked with this {@link PromisedValues} as a parameter 164 | * 165 | * @param handler the call back which will be given this object 166 | * 167 | * @return a new {@link PromisedValues} which you can compose more computations with 168 | */ 169 | PromisedValues thenAccept(Consumer> handler); 170 | 171 | /** 172 | * @return true if all the futures completed successfully 173 | */ 174 | boolean succeeded(); 175 | 176 | /** 177 | * @return true if any of the the futures completed unsuccessfully 178 | */ 179 | boolean failed(); 180 | 181 | /** 182 | * The true if the all the futures have completed (and hence this {@link PromisedValues} has completed) 183 | * 184 | * @return true if all the futures have completed 185 | */ 186 | boolean isDone(); 187 | 188 | /** 189 | * The exception cause or null if it didn't fail 190 | * 191 | * @return an exception or null if the future did not fail 192 | */ 193 | Throwable cause(); 194 | 195 | /** 196 | * The true if the {@link CompletionStage} at the specified index succeeded 197 | * 198 | * @param index the index of the {@link CompletionStage} 199 | * 200 | * @return true if the future at the specified index succeeded 201 | */ 202 | boolean succeeded(int index); 203 | 204 | /** 205 | * The exception cause at the specified index or null if it didn't fail 206 | * 207 | * @param index the index of the {@link CompletionStage} 208 | * 209 | * @return an exception or null if the future did not fail 210 | */ 211 | Throwable cause(int index); 212 | 213 | /** 214 | * The value at index or null if it failed 215 | * 216 | * @param index the index of the future 217 | * 218 | * @return the value of the future 219 | */ 220 | @SuppressWarnings("unchecked") 221 | T get(int index); 222 | 223 | /** 224 | * Returns the underlying values as a list 225 | * 226 | * @return the list of underlying values 227 | */ 228 | List toList(); 229 | 230 | /** 231 | * @return the number of {@link CompletionStage}s under the covers 232 | */ 233 | int size(); 234 | 235 | /** 236 | * Waits for the underlying futures to complete. To better 237 | * conform with the use of common functional forms, if a 238 | * computation involved in the completion of this 239 | * CompletableFuture threw an exception, this method throws an 240 | * (unchecked) {@link CompletionException} with the underlying 241 | * exception as its cause. 242 | * 243 | * @return the list of completed values similar to {@link #toList()} 244 | * 245 | * @throws CancellationException if the computation was cancelled 246 | * @throws CompletionException if this future completed 247 | * exceptionally or a completion computation threw an exception 248 | */ 249 | List join(); 250 | 251 | /** 252 | * @return this as a {@link CompletableFuture} that returns the list of underlying values 253 | */ 254 | CompletableFuture> toCompletableFuture(); 255 | } 256 | -------------------------------------------------------------------------------- /src/main/java/org/dataloader/impl/PromisedValuesImpl.java: -------------------------------------------------------------------------------- 1 | package org.dataloader.impl; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.concurrent.CompletableFuture; 6 | import java.util.concurrent.CompletionException; 7 | import java.util.concurrent.CompletionStage; 8 | import java.util.concurrent.ExecutionException; 9 | import java.util.concurrent.atomic.AtomicReference; 10 | import java.util.function.Consumer; 11 | import java.util.stream.Collectors; 12 | 13 | import static org.dataloader.impl.Assertions.assertState; 14 | import static org.dataloader.impl.Assertions.nonNull; 15 | 16 | public class PromisedValuesImpl implements PromisedValues { 17 | 18 | private final List> futures; 19 | private final CompletionStage controller; 20 | private AtomicReference cause; 21 | 22 | private PromisedValuesImpl(List> cs) { 23 | this.futures = nonNull(cs); 24 | this.cause = new AtomicReference<>(); 25 | List cfs = cs.stream().map(CompletionStage::toCompletableFuture).collect(Collectors.toList()); 26 | CompletableFuture[] futuresArray = cfs.toArray(new CompletableFuture[cfs.size()]); 27 | this.controller = CompletableFuture.allOf(futuresArray).handle((result, throwable) -> { 28 | setCause(throwable); 29 | return null; 30 | }); 31 | } 32 | 33 | private PromisedValuesImpl(PromisedValuesImpl other, CompletionStage controller) { 34 | this.futures = other.futures; 35 | this.cause = other.cause; 36 | this.controller = controller; 37 | } 38 | 39 | public static PromisedValues combineAllOf(List> cfs) { 40 | return new PromisedValuesImpl<>(nonNull(cfs)); 41 | } 42 | 43 | public static PromisedValues combinePromisedValues(List> promisedValues) { 44 | List> cfs = promisedValues.stream() 45 | .map(pv -> (PromisedValuesImpl) pv) 46 | .flatMap(pv -> pv.futures.stream()) 47 | .collect(Collectors.toList()); 48 | return new PromisedValuesImpl<>(cfs); 49 | } 50 | 51 | private void setCause(Throwable throwable) { 52 | if (throwable != null) { 53 | if (throwable instanceof CompletionException && throwable.getCause() != null) { 54 | cause.set(throwable.getCause()); 55 | } else { 56 | cause.set(throwable); 57 | } 58 | } 59 | } 60 | 61 | @Override 62 | public PromisedValues thenAccept(Consumer> handler) { 63 | nonNull(handler); 64 | CompletionStage newController = controller.handle((result, throwable) -> { 65 | setCause(throwable); 66 | handler.accept(this); 67 | return result; 68 | }); 69 | return new PromisedValuesImpl<>(this, newController); 70 | } 71 | 72 | 73 | @Override 74 | public boolean succeeded() { 75 | return isDone() && cause.get() == null; 76 | } 77 | 78 | @Override 79 | public boolean failed() { 80 | return isDone() && cause.get() != null; 81 | } 82 | 83 | @Override 84 | public boolean isDone() { 85 | return controller.toCompletableFuture().isDone(); 86 | } 87 | 88 | @Override 89 | public Throwable cause() { 90 | return cause.get(); 91 | } 92 | 93 | @Override 94 | public boolean succeeded(int index) { 95 | return CompletableFutureKit.succeeded(futures.get(index).toCompletableFuture()); 96 | } 97 | 98 | @Override 99 | public Throwable cause(int index) { 100 | return CompletableFutureKit.cause(futures.get(index).toCompletableFuture()); 101 | } 102 | 103 | @Override 104 | @SuppressWarnings("unchecked") 105 | public T get(int index) { 106 | assertState(isDone(), "The PromisedValues MUST be complete before calling the get() method"); 107 | try { 108 | CompletionStage future = futures.get(index); 109 | return future.toCompletableFuture().get(); 110 | } catch (InterruptedException | ExecutionException e) { 111 | return null; 112 | } 113 | } 114 | 115 | @Override 116 | public List toList() { 117 | assertState(isDone(), "The PromisedValues MUST be complete before calling the toList() method"); 118 | int size = size(); 119 | List list = new ArrayList<>(size); 120 | for (int index = 0; index < size; index++) { 121 | list.add(get(index)); 122 | } 123 | return list; 124 | } 125 | 126 | @Override 127 | public int size() { 128 | return futures.size(); 129 | } 130 | 131 | @Override 132 | public List join() { 133 | controller.toCompletableFuture().join(); 134 | return toList(); 135 | } 136 | 137 | @Override 138 | public CompletableFuture> toCompletableFuture() { 139 | return controller.thenApply(v -> toList()).toCompletableFuture(); 140 | } 141 | 142 | } 143 | -------------------------------------------------------------------------------- /src/test/java/ReadmeExamples.java: -------------------------------------------------------------------------------- 1 | import org.dataloader.BatchLoader; 2 | import org.dataloader.DataLoader; 3 | import org.dataloader.fixtures.User; 4 | import org.dataloader.fixtures.UserManager; 5 | 6 | import java.util.List; 7 | import java.util.concurrent.CompletableFuture; 8 | import java.util.concurrent.CompletionStage; 9 | import java.util.stream.Collectors; 10 | 11 | public class ReadmeExamples { 12 | 13 | UserManager userManager = new UserManager(); 14 | 15 | public static void main(String[] args) { 16 | ReadmeExamples examples = new ReadmeExamples(); 17 | examples.basicExample(); 18 | } 19 | 20 | @SuppressWarnings({"Convert2Lambda", "Convert2MethodRef", "CodeBlock2Expr"}) 21 | void basicExample() { 22 | 23 | BatchLoader lessEfficientUserBatchLoader = new BatchLoader() { 24 | @Override 25 | public CompletionStage> load(List userIds) { 26 | return CompletableFuture.supplyAsync(() -> { 27 | // 28 | // notice how it makes N calls to load by single user id out of the batch of N keys 29 | // 30 | return userIds.stream() 31 | .map(id -> userManager.loadUserById(id)) 32 | .collect(Collectors.toList()); 33 | }); 34 | } 35 | }; 36 | 37 | BatchLoader userBatchLoader = new BatchLoader() { 38 | @Override 39 | public CompletionStage> load(List userIds) { 40 | return CompletableFuture.supplyAsync(() -> { 41 | return userManager.loadUsersById(userIds); 42 | }); 43 | } 44 | }; 45 | 46 | DataLoader userLoader = new DataLoader<>(userBatchLoader); 47 | 48 | CompletionStage load1 = userLoader.load(1L); 49 | 50 | userLoader.load(1L) 51 | .thenAccept(user -> { 52 | System.out.println("user = " + user); 53 | userLoader.load(user.getInvitedByID()) 54 | .thenAccept(invitedBy -> { 55 | System.out.println("invitedBy = " + invitedBy); 56 | }); 57 | }); 58 | 59 | userLoader.load(2L) 60 | .thenAccept(user -> { 61 | System.out.println("user = " + user); 62 | userLoader.load(user.getInvitedByID()) 63 | .thenAccept(invitedBy -> { 64 | System.out.println("invitedBy = " + invitedBy); 65 | }); 66 | }); 67 | 68 | userLoader.dispatchAndJoin(); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/test/java/org/dataloader/DataLoaderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2016 The original author or authors 3 | * 4 | * All rights reserved. This program and the accompanying materials 5 | * are made available under the terms of the Eclipse Public License v1.0 6 | * and Apache License v2.0 which accompanies this distribution. 7 | * 8 | * The Eclipse Public License is available at 9 | * http://www.eclipse.org/legal/epl-v10.html 10 | * 11 | * The Apache License v2.0 is available at 12 | * http://www.opensource.org/licenses/apache2.0.php 13 | * 14 | * You may elect to redistribute this code under either of these licenses. 15 | */ 16 | 17 | package org.dataloader; 18 | 19 | import org.dataloader.fixtures.User; 20 | import org.dataloader.fixtures.UserManager; 21 | import org.dataloader.impl.CompletableFutureKit; 22 | import org.junit.Test; 23 | 24 | import java.util.ArrayList; 25 | import java.util.Collection; 26 | import java.util.LinkedHashMap; 27 | import java.util.List; 28 | import java.util.Map; 29 | import java.util.concurrent.CompletableFuture; 30 | import java.util.concurrent.CompletionStage; 31 | import java.util.concurrent.ExecutionException; 32 | import java.util.concurrent.atomic.AtomicBoolean; 33 | import java.util.function.Supplier; 34 | import java.util.stream.Collectors; 35 | 36 | import static java.util.Arrays.asList; 37 | import static java.util.Collections.emptyList; 38 | import static java.util.Collections.singletonList; 39 | import static org.awaitility.Awaitility.await; 40 | import static org.dataloader.impl.CompletableFutureKit.cause; 41 | import static org.dataloader.impl.CompletableFutureKit.failedFuture; 42 | import static org.hamcrest.Matchers.empty; 43 | import static org.hamcrest.Matchers.equalTo; 44 | import static org.hamcrest.Matchers.instanceOf; 45 | import static org.hamcrest.Matchers.is; 46 | import static org.junit.Assert.assertArrayEquals; 47 | import static org.junit.Assert.assertThat; 48 | 49 | /** 50 | * Tests for {@link DataLoader}. 51 | *

52 | * The tests are a port of the existing tests in 53 | * the facebook/dataloader project. 54 | *

55 | * Acknowledgments go to Lee Byron for providing excellent coverage. 56 | * 57 | * @author Arnold Schrijver 58 | */ 59 | public class DataLoaderTest { 60 | 61 | @Test 62 | public void should_Build_a_really_really_simple_data_loader() { 63 | AtomicBoolean success = new AtomicBoolean(); 64 | DataLoader identityLoader = new DataLoader<>(keysAsValues()); 65 | 66 | CompletionStage future1 = identityLoader.load(1); 67 | 68 | future1.thenAccept(value -> { 69 | assertThat(value, equalTo(1)); 70 | success.set(true); 71 | }); 72 | identityLoader.dispatch(); 73 | await().untilAtomic(success, is(true)); 74 | } 75 | 76 | @Test 77 | public void should_Support_loading_multiple_keys_in_one_call() { 78 | AtomicBoolean success = new AtomicBoolean(); 79 | DataLoader identityLoader = new DataLoader<>(keysAsValues()); 80 | 81 | CompletionStage> futureAll = identityLoader.loadMany(asList(1, 2)); 82 | futureAll.thenAccept(promisedValues -> { 83 | assertThat(promisedValues.size(), is(2)); 84 | success.set(true); 85 | }); 86 | identityLoader.dispatch(); 87 | await().untilAtomic(success, is(true)); 88 | assertThat(futureAll.toCompletableFuture().join(), equalTo(asList(1, 2))); 89 | } 90 | 91 | @Test 92 | public void should_Resolve_to_empty_list_when_no_keys_supplied() { 93 | AtomicBoolean success = new AtomicBoolean(); 94 | DataLoader identityLoader = new DataLoader<>(keysAsValues()); 95 | CompletableFuture> futureEmpty = identityLoader.loadMany(emptyList()); 96 | futureEmpty.thenAccept(promisedValues -> { 97 | assertThat(promisedValues.size(), is(0)); 98 | success.set(true); 99 | }); 100 | identityLoader.dispatch(); 101 | await().untilAtomic(success, is(true)); 102 | assertThat(futureEmpty.join(), empty()); 103 | } 104 | 105 | @Test 106 | public void should_Batch_multiple_requests() throws ExecutionException, InterruptedException { 107 | List> loadCalls = new ArrayList<>(); 108 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 109 | 110 | CompletableFuture future1 = identityLoader.load(1); 111 | CompletableFuture future2 = identityLoader.load(2); 112 | identityLoader.dispatch(); 113 | 114 | await().until(() -> future1.isDone() && future2.isDone()); 115 | assertThat(future1.get(), equalTo(1)); 116 | assertThat(future2.get(), equalTo(2)); 117 | assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); 118 | } 119 | 120 | @Test 121 | public void should_Coalesce_identical_requests() throws ExecutionException, InterruptedException { 122 | List> loadCalls = new ArrayList<>(); 123 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 124 | 125 | CompletableFuture future1a = identityLoader.load(1); 126 | CompletableFuture future1b = identityLoader.load(1); 127 | assertThat(future1a, equalTo(future1b)); 128 | identityLoader.dispatch(); 129 | 130 | await().until(future1a::isDone); 131 | assertThat(future1a.get(), equalTo(1)); 132 | assertThat(future1b.get(), equalTo(1)); 133 | assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); 134 | } 135 | 136 | @Test 137 | public void should_Cache_repeated_requests() throws ExecutionException, InterruptedException { 138 | List> loadCalls = new ArrayList<>(); 139 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 140 | 141 | CompletableFuture future1 = identityLoader.load("A"); 142 | CompletableFuture future2 = identityLoader.load("B"); 143 | identityLoader.dispatch(); 144 | 145 | await().until(() -> future1.isDone() && future2.isDone()); 146 | assertThat(future1.get(), equalTo("A")); 147 | assertThat(future2.get(), equalTo("B")); 148 | assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); 149 | 150 | CompletableFuture future1a = identityLoader.load("A"); 151 | CompletableFuture future3 = identityLoader.load("C"); 152 | identityLoader.dispatch(); 153 | 154 | await().until(() -> future1a.isDone() && future3.isDone()); 155 | assertThat(future1a.get(), equalTo("A")); 156 | assertThat(future3.get(), equalTo("C")); 157 | assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("C")))); 158 | 159 | CompletableFuture future1b = identityLoader.load("A"); 160 | CompletableFuture future2a = identityLoader.load("B"); 161 | CompletableFuture future3a = identityLoader.load("C"); 162 | identityLoader.dispatch(); 163 | 164 | await().until(() -> future1b.isDone() && future2a.isDone() && future3a.isDone()); 165 | assertThat(future1b.get(), equalTo("A")); 166 | assertThat(future2a.get(), equalTo("B")); 167 | assertThat(future3a.get(), equalTo("C")); 168 | assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("C")))); 169 | } 170 | 171 | @Test 172 | public void should_Not_redispatch_previous_load() throws ExecutionException, InterruptedException { 173 | List> loadCalls = new ArrayList<>(); 174 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 175 | 176 | CompletableFuture future1 = identityLoader.load("A"); 177 | identityLoader.dispatch(); 178 | 179 | CompletableFuture future2 = identityLoader.load("B"); 180 | identityLoader.dispatch(); 181 | 182 | await().until(() -> future1.isDone() && future2.isDone()); 183 | assertThat(future1.get(), equalTo("A")); 184 | assertThat(future2.get(), equalTo("B")); 185 | assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); 186 | } 187 | 188 | @Test 189 | public void should_Cache_on_redispatch() throws ExecutionException, InterruptedException { 190 | List> loadCalls = new ArrayList<>(); 191 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 192 | 193 | CompletableFuture future1 = identityLoader.load("A"); 194 | identityLoader.dispatch(); 195 | 196 | CompletableFuture> future2 = identityLoader.loadMany(asList("A", "B")); 197 | identityLoader.dispatch(); 198 | 199 | await().until(() -> future1.isDone() && future2.isDone()); 200 | assertThat(future1.get(), equalTo("A")); 201 | assertThat(future2.get(), equalTo(asList("A", "B"))); 202 | assertThat(loadCalls, equalTo(asList(singletonList("A"), singletonList("B")))); 203 | } 204 | 205 | @Test 206 | public void should_Clear_single_value_in_loader() throws ExecutionException, InterruptedException { 207 | List> loadCalls = new ArrayList<>(); 208 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 209 | 210 | CompletableFuture future1 = identityLoader.load("A"); 211 | CompletableFuture future2 = identityLoader.load("B"); 212 | identityLoader.dispatch(); 213 | 214 | await().until(() -> future1.isDone() && future2.isDone()); 215 | assertThat(future1.get(), equalTo("A")); 216 | assertThat(future2.get(), equalTo("B")); 217 | assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); 218 | 219 | identityLoader.clear("A"); 220 | 221 | CompletableFuture future1a = identityLoader.load("A"); 222 | CompletableFuture future2a = identityLoader.load("B"); 223 | identityLoader.dispatch(); 224 | 225 | await().until(() -> future1a.isDone() && future2a.isDone()); 226 | assertThat(future1a.get(), equalTo("A")); 227 | assertThat(future2a.get(), equalTo("B")); 228 | assertThat(loadCalls, equalTo(asList(asList("A", "B"), singletonList("A")))); 229 | } 230 | 231 | @Test 232 | public void should_Clear_all_values_in_loader() throws ExecutionException, InterruptedException { 233 | List> loadCalls = new ArrayList<>(); 234 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 235 | 236 | CompletableFuture future1 = identityLoader.load("A"); 237 | CompletableFuture future2 = identityLoader.load("B"); 238 | identityLoader.dispatch(); 239 | 240 | await().until(() -> future1.isDone() && future2.isDone()); 241 | assertThat(future1.get(), equalTo("A")); 242 | assertThat(future2.get(), equalTo("B")); 243 | assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); 244 | 245 | identityLoader.clearAll(); 246 | 247 | CompletableFuture future1a = identityLoader.load("A"); 248 | CompletableFuture future2a = identityLoader.load("B"); 249 | identityLoader.dispatch(); 250 | 251 | await().until(() -> future1a.isDone() && future2a.isDone()); 252 | assertThat(future1a.get(), equalTo("A")); 253 | assertThat(future2a.get(), equalTo("B")); 254 | assertThat(loadCalls, equalTo(asList(asList("A", "B"), asList("A", "B")))); 255 | } 256 | 257 | @Test 258 | public void should_Allow_priming_the_cache() throws ExecutionException, InterruptedException { 259 | List> loadCalls = new ArrayList<>(); 260 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 261 | 262 | identityLoader.prime("A", "A"); 263 | 264 | CompletableFuture future1 = identityLoader.load("A"); 265 | CompletableFuture future2 = identityLoader.load("B"); 266 | identityLoader.dispatch(); 267 | 268 | await().until(() -> future1.isDone() && future2.isDone()); 269 | assertThat(future1.get(), equalTo("A")); 270 | assertThat(future2.get(), equalTo("B")); 271 | assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); 272 | } 273 | 274 | @Test 275 | public void should_Not_prime_keys_that_already_exist() throws ExecutionException, InterruptedException { 276 | List> loadCalls = new ArrayList<>(); 277 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 278 | 279 | identityLoader.prime("A", "X"); 280 | 281 | CompletableFuture future1 = identityLoader.load("A"); 282 | CompletableFuture future2 = identityLoader.load("B"); 283 | CompletableFuture> composite = identityLoader.dispatch(); 284 | 285 | await().until(composite::isDone); 286 | assertThat(future1.get(), equalTo("X")); 287 | assertThat(future2.get(), equalTo("B")); 288 | 289 | identityLoader.prime("A", "Y"); 290 | identityLoader.prime("B", "Y"); 291 | 292 | CompletableFuture future1a = identityLoader.load("A"); 293 | CompletableFuture future2a = identityLoader.load("B"); 294 | CompletableFuture> composite2 = identityLoader.dispatch(); 295 | 296 | await().until(composite2::isDone); 297 | assertThat(future1a.get(), equalTo("X")); 298 | assertThat(future2a.get(), equalTo("B")); 299 | assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); 300 | } 301 | 302 | @Test 303 | public void should_Allow_to_forcefully_prime_the_cache() throws ExecutionException, InterruptedException { 304 | List> loadCalls = new ArrayList<>(); 305 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 306 | 307 | identityLoader.prime("A", "X"); 308 | 309 | CompletableFuture future1 = identityLoader.load("A"); 310 | CompletableFuture future2 = identityLoader.load("B"); 311 | CompletableFuture> composite = identityLoader.dispatch(); 312 | 313 | await().until(composite::isDone); 314 | assertThat(future1.get(), equalTo("X")); 315 | assertThat(future2.get(), equalTo("B")); 316 | 317 | identityLoader.clear("A").prime("A", "Y"); 318 | identityLoader.clear("B").prime("B", "Y"); 319 | 320 | CompletableFuture future1a = identityLoader.load("A"); 321 | CompletableFuture future2a = identityLoader.load("B"); 322 | CompletableFuture> composite2 = identityLoader.dispatch(); 323 | 324 | await().until(composite2::isDone); 325 | assertThat(future1a.get(), equalTo("Y")); 326 | assertThat(future2a.get(), equalTo("Y")); 327 | assertThat(loadCalls, equalTo(singletonList(singletonList("B")))); 328 | } 329 | 330 | @Test 331 | public void should_not_Cache_failed_fetches_on_complete_failure() { 332 | List> loadCalls = new ArrayList<>(); 333 | DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); 334 | 335 | CompletableFuture future1 = errorLoader.load(1); 336 | errorLoader.dispatch(); 337 | 338 | await().until(future1::isDone); 339 | assertThat(future1.isCompletedExceptionally(), is(true)); 340 | assertThat(cause(future1), instanceOf(IllegalStateException.class)); 341 | 342 | CompletableFuture future2 = errorLoader.load(1); 343 | errorLoader.dispatch(); 344 | 345 | await().until(future2::isDone); 346 | assertThat(future2.isCompletedExceptionally(), is(true)); 347 | assertThat(cause(future2), instanceOf(IllegalStateException.class)); 348 | assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); 349 | } 350 | 351 | @Test 352 | public void should_Resolve_to_error_to_indicate_failure() throws ExecutionException, InterruptedException { 353 | List> loadCalls = new ArrayList<>(); 354 | DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); 355 | 356 | CompletableFuture future1 = evenLoader.load(1); 357 | evenLoader.dispatch(); 358 | 359 | await().until(future1::isDone); 360 | assertThat(future1.isCompletedExceptionally(), is(true)); 361 | assertThat(cause(future1), instanceOf(IllegalStateException.class)); 362 | 363 | CompletableFuture future2 = evenLoader.load(2); 364 | evenLoader.dispatch(); 365 | 366 | await().until(future2::isDone); 367 | assertThat(future2.get(), equalTo(2)); 368 | assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(2)))); 369 | } 370 | 371 | // Accept any kind of key. 372 | 373 | @Test 374 | public void should_Represent_failures_and_successes_simultaneously() throws ExecutionException, InterruptedException { 375 | AtomicBoolean success = new AtomicBoolean(); 376 | List> loadCalls = new ArrayList<>(); 377 | DataLoader evenLoader = idLoaderOddEvenExceptions(new DataLoaderOptions(), loadCalls); 378 | 379 | CompletableFuture future1 = evenLoader.load(1); 380 | CompletableFuture future2 = evenLoader.load(2); 381 | CompletableFuture future3 = evenLoader.load(3); 382 | CompletableFuture future4 = evenLoader.load(4); 383 | CompletableFuture> result = evenLoader.dispatch(); 384 | result.thenAccept(promisedValues -> success.set(true)); 385 | 386 | await().untilAtomic(success, is(true)); 387 | 388 | assertThat(future1.isCompletedExceptionally(), is(true)); 389 | assertThat(cause(future1), instanceOf(IllegalStateException.class)); 390 | assertThat(future2.get(), equalTo(2)); 391 | assertThat(future3.isCompletedExceptionally(), is(true)); 392 | assertThat(future4.get(), equalTo(4)); 393 | 394 | assertThat(loadCalls, equalTo(singletonList(asList(1, 2, 3, 4)))); 395 | } 396 | 397 | // Accepts options 398 | 399 | @Test 400 | public void should_Cache_failed_fetches() { 401 | List> loadCalls = new ArrayList<>(); 402 | DataLoader errorLoader = idLoaderAllExceptions(new DataLoaderOptions(), loadCalls); 403 | 404 | CompletableFuture future1 = errorLoader.load(1); 405 | errorLoader.dispatch(); 406 | 407 | await().until(future1::isDone); 408 | assertThat(future1.isCompletedExceptionally(), is(true)); 409 | assertThat(cause(future1), instanceOf(IllegalStateException.class)); 410 | 411 | CompletableFuture future2 = errorLoader.load(1); 412 | errorLoader.dispatch(); 413 | 414 | await().until(future2::isDone); 415 | assertThat(future2.isCompletedExceptionally(), is(true)); 416 | assertThat(cause(future2), instanceOf(IllegalStateException.class)); 417 | 418 | assertThat(loadCalls, equalTo(singletonList(singletonList(1)))); 419 | } 420 | 421 | 422 | // Accepts object key in custom cacheKey function 423 | 424 | @Test 425 | public void should_Handle_priming_the_cache_with_an_error() { 426 | List> loadCalls = new ArrayList<>(); 427 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 428 | 429 | identityLoader.prime(1, new IllegalStateException("Error")); 430 | 431 | CompletableFuture future1 = identityLoader.load(1); 432 | identityLoader.dispatch(); 433 | 434 | await().until(future1::isDone); 435 | assertThat(future1.isCompletedExceptionally(), is(true)); 436 | assertThat(cause(future1), instanceOf(IllegalStateException.class)); 437 | assertThat(loadCalls, equalTo(emptyList())); 438 | } 439 | 440 | @Test 441 | public void should_Clear_values_from_cache_after_errors() { 442 | List> loadCalls = new ArrayList<>(); 443 | DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); 444 | 445 | CompletableFuture future1 = errorLoader.load(1); 446 | future1.handle((value, t) -> { 447 | if (t != null) { 448 | // Presumably determine if this error is transient, and only clear the cache in that case. 449 | errorLoader.clear(1); 450 | } 451 | return null; 452 | }); 453 | errorLoader.dispatch(); 454 | 455 | await().until(future1::isDone); 456 | assertThat(future1.isCompletedExceptionally(), is(true)); 457 | assertThat(cause(future1), instanceOf(IllegalStateException.class)); 458 | 459 | CompletableFuture future2 = errorLoader.load(1); 460 | future2.handle((value, t) -> { 461 | if (t != null) { 462 | // Again, only do this if you can determine the error is transient. 463 | errorLoader.clear(1); 464 | } 465 | return null; 466 | }); 467 | errorLoader.dispatch(); 468 | 469 | await().until(future2::isDone); 470 | assertThat(future2.isCompletedExceptionally(), is(true)); 471 | assertThat(cause(future2), instanceOf(IllegalStateException.class)); 472 | assertThat(loadCalls, equalTo(asList(singletonList(1), singletonList(1)))); 473 | } 474 | 475 | @Test 476 | public void should_Propagate_error_to_all_loads() { 477 | List> loadCalls = new ArrayList<>(); 478 | DataLoader errorLoader = idLoaderBlowsUps(new DataLoaderOptions(), loadCalls); 479 | 480 | CompletableFuture future1 = errorLoader.load(1); 481 | CompletableFuture future2 = errorLoader.load(2); 482 | errorLoader.dispatch(); 483 | 484 | await().until(future1::isDone); 485 | assertThat(future1.isCompletedExceptionally(), is(true)); 486 | Throwable cause = cause(future1); 487 | assert cause != null; 488 | assertThat(cause, instanceOf(IllegalStateException.class)); 489 | assertThat(cause.getMessage(), equalTo("Error")); 490 | 491 | await().until(future2::isDone); 492 | cause = cause(future2); 493 | assert cause != null; 494 | assertThat(cause.getMessage(), equalTo(cause.getMessage())); 495 | assertThat(loadCalls, equalTo(singletonList(asList(1, 2)))); 496 | } 497 | 498 | @Test 499 | public void should_Accept_objects_as_keys() { 500 | List> loadCalls = new ArrayList<>(); 501 | DataLoader identityLoader = idLoader(new DataLoaderOptions(), loadCalls); 502 | 503 | Object keyA = new Object(); 504 | Object keyB = new Object(); 505 | 506 | // Fetches as expected 507 | 508 | identityLoader.load(keyA); 509 | identityLoader.load(keyB); 510 | 511 | identityLoader.dispatch().thenAccept(promisedValues -> { 512 | assertThat(promisedValues.get(0), equalTo(keyA)); 513 | assertThat(promisedValues.get(1), equalTo(keyB)); 514 | }); 515 | 516 | assertThat(loadCalls.size(), equalTo(1)); 517 | assertThat(loadCalls.get(0).size(), equalTo(2)); 518 | assertThat(loadCalls.get(0).toArray()[0], equalTo(keyA)); 519 | assertThat(loadCalls.get(0).toArray()[1], equalTo(keyB)); 520 | 521 | // Caching 522 | identityLoader.clear(keyA); 523 | //noinspection SuspiciousMethodCalls 524 | loadCalls.remove(keyA); 525 | 526 | identityLoader.load(keyA); 527 | identityLoader.load(keyB); 528 | 529 | identityLoader.dispatch().thenAccept(promisedValues -> { 530 | assertThat(promisedValues.get(0), equalTo(keyA)); 531 | assertThat(identityLoader.getCacheKey(keyB), equalTo(keyB)); 532 | }); 533 | 534 | assertThat(loadCalls.size(), equalTo(2)); 535 | assertThat(loadCalls.get(1).size(), equalTo(1)); 536 | assertThat(loadCalls.get(1).toArray()[0], equalTo(keyA)); 537 | } 538 | 539 | @Test 540 | public void should_Disable_caching() throws ExecutionException, InterruptedException { 541 | List> loadCalls = new ArrayList<>(); 542 | DataLoader identityLoader = 543 | idLoader(DataLoaderOptions.create().setCachingEnabled(false), loadCalls); 544 | 545 | CompletableFuture future1 = identityLoader.load("A"); 546 | CompletableFuture future2 = identityLoader.load("B"); 547 | identityLoader.dispatch(); 548 | 549 | await().until(() -> future1.isDone() && future2.isDone()); 550 | assertThat(future1.get(), equalTo("A")); 551 | assertThat(future2.get(), equalTo("B")); 552 | assertThat(loadCalls, equalTo(singletonList(asList("A", "B")))); 553 | 554 | CompletableFuture future1a = identityLoader.load("A"); 555 | CompletableFuture future3 = identityLoader.load("C"); 556 | identityLoader.dispatch(); 557 | 558 | await().until(() -> future1a.isDone() && future3.isDone()); 559 | assertThat(future1a.get(), equalTo("A")); 560 | assertThat(future3.get(), equalTo("C")); 561 | assertThat(loadCalls, equalTo(asList(asList("A", "B"), asList("A", "C")))); 562 | 563 | CompletableFuture future1b = identityLoader.load("A"); 564 | CompletableFuture future2a = identityLoader.load("B"); 565 | CompletableFuture future3a = identityLoader.load("C"); 566 | identityLoader.dispatch(); 567 | 568 | await().until(() -> future1b.isDone() && future2a.isDone() && future3a.isDone()); 569 | assertThat(future1b.get(), equalTo("A")); 570 | assertThat(future2a.get(), equalTo("B")); 571 | assertThat(future3a.get(), equalTo("C")); 572 | assertThat(loadCalls, equalTo(asList(asList("A", "B"), 573 | asList("A", "C"), asList("A", "B", "C")))); 574 | } 575 | 576 | // It is resilient to job queue ordering 577 | 578 | @Test 579 | public void should_Accept_objects_with_a_complex_key() throws ExecutionException, InterruptedException { 580 | List> loadCalls = new ArrayList<>(); 581 | DataLoaderOptions options = DataLoaderOptions.create().setCacheKeyFunction(getJsonObjectCacheMapFn()); 582 | DataLoader identityLoader = idLoader(options, loadCalls); 583 | 584 | JsonObject key1 = new JsonObject().put("id", 123); 585 | JsonObject key2 = new JsonObject().put("id", 123); 586 | 587 | CompletableFuture future1 = identityLoader.load(key1); 588 | CompletableFuture future2 = identityLoader.load(key2); 589 | identityLoader.dispatch(); 590 | 591 | await().until(() -> future1.isDone() && future2.isDone()); 592 | assertThat(loadCalls, equalTo(singletonList(singletonList(key1)))); 593 | assertThat(future1.get(), equalTo(key1)); 594 | assertThat(future2.get(), equalTo(key1)); 595 | } 596 | 597 | // Helper methods 598 | 599 | @Test 600 | public void should_Clear_objects_with_complex_key() throws ExecutionException, InterruptedException { 601 | List> loadCalls = new ArrayList<>(); 602 | DataLoaderOptions options = DataLoaderOptions.create().setCacheKeyFunction(getJsonObjectCacheMapFn()); 603 | DataLoader identityLoader = idLoader(options, loadCalls); 604 | 605 | JsonObject key1 = new JsonObject().put("id", 123); 606 | JsonObject key2 = new JsonObject().put("id", 123); 607 | 608 | CompletableFuture future1 = identityLoader.load(key1); 609 | identityLoader.dispatch(); 610 | 611 | await().until(future1::isDone); 612 | identityLoader.clear(key2); // clear equivalent object key 613 | 614 | CompletableFuture future2 = identityLoader.load(key1); 615 | identityLoader.dispatch(); 616 | 617 | await().until(future2::isDone); 618 | assertThat(loadCalls, equalTo(asList(singletonList(key1), singletonList(key1)))); 619 | assertThat(future1.get(), equalTo(key1)); 620 | assertThat(future2.get(), equalTo(key1)); 621 | } 622 | 623 | @Test 624 | public void should_Accept_objects_with_different_order_of_keys() throws ExecutionException, InterruptedException { 625 | List> loadCalls = new ArrayList<>(); 626 | DataLoaderOptions options = DataLoaderOptions.create().setCacheKeyFunction(getJsonObjectCacheMapFn()); 627 | DataLoader identityLoader = idLoader(options, loadCalls); 628 | 629 | JsonObject key1 = new JsonObject().put("a", 123).put("b", 321); 630 | JsonObject key2 = new JsonObject().put("b", 321).put("a", 123); 631 | 632 | // Fetches as expected 633 | 634 | CompletableFuture future1 = identityLoader.load(key1); 635 | CompletableFuture future2 = identityLoader.load(key2); 636 | identityLoader.dispatch(); 637 | 638 | await().until(() -> future1.isDone() && future2.isDone()); 639 | assertThat(loadCalls, equalTo(singletonList(singletonList(key1)))); 640 | assertThat(loadCalls.size(), equalTo(1)); 641 | assertThat(future1.get(), equalTo(key1)); 642 | assertThat(future2.get(), equalTo(key1)); 643 | } 644 | 645 | @Test 646 | public void should_Allow_priming_the_cache_with_an_object_key() throws ExecutionException, InterruptedException { 647 | List> loadCalls = new ArrayList<>(); 648 | DataLoaderOptions options = DataLoaderOptions.create().setCacheKeyFunction(getJsonObjectCacheMapFn()); 649 | DataLoader identityLoader = idLoader(options, loadCalls); 650 | 651 | JsonObject key1 = new JsonObject().put("id", 123); 652 | JsonObject key2 = new JsonObject().put("id", 123); 653 | 654 | identityLoader.prime(key1, key1); 655 | 656 | CompletableFuture future1 = identityLoader.load(key1); 657 | CompletableFuture future2 = identityLoader.load(key2); 658 | identityLoader.dispatch(); 659 | 660 | await().until(() -> future1.isDone() && future2.isDone()); 661 | assertThat(loadCalls, equalTo(emptyList())); 662 | assertThat(future1.get(), equalTo(key1)); 663 | assertThat(future2.get(), equalTo(key1)); 664 | } 665 | 666 | @Test 667 | public void should_Accept_a_custom_cache_map_implementation() throws ExecutionException, InterruptedException { 668 | CustomCacheMap customMap = new CustomCacheMap(); 669 | List> loadCalls = new ArrayList<>(); 670 | DataLoaderOptions options = DataLoaderOptions.create().setCacheMap(customMap); 671 | DataLoader identityLoader = idLoader(options, loadCalls); 672 | 673 | // Fetches as expected 674 | 675 | CompletableFuture future1 = identityLoader.load("a"); 676 | CompletableFuture future2 = identityLoader.load("b"); 677 | CompletableFuture> composite = identityLoader.dispatch(); 678 | 679 | await().until(composite::isDone); 680 | assertThat(future1.get(), equalTo("a")); 681 | assertThat(future2.get(), equalTo("b")); 682 | 683 | assertThat(loadCalls, equalTo(singletonList(asList("a", "b")))); 684 | assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "b").toArray()); 685 | 686 | CompletableFuture future3 = identityLoader.load("c"); 687 | CompletableFuture future2a = identityLoader.load("b"); 688 | composite = identityLoader.dispatch(); 689 | 690 | await().until(composite::isDone); 691 | assertThat(future3.get(), equalTo("c")); 692 | assertThat(future2a.get(), equalTo("b")); 693 | 694 | assertThat(loadCalls, equalTo(asList(asList("a", "b"), singletonList("c")))); 695 | assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "b", "c").toArray()); 696 | 697 | // Supports clear 698 | 699 | identityLoader.clear("b"); 700 | assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "c").toArray()); 701 | 702 | CompletableFuture future2b = identityLoader.load("b"); 703 | composite = identityLoader.dispatch(); 704 | 705 | await().until(composite::isDone); 706 | assertThat(future2b.get(), equalTo("b")); 707 | assertThat(loadCalls, equalTo(asList(asList("a", "b"), 708 | singletonList("c"), singletonList("b")))); 709 | assertArrayEquals(customMap.stash.keySet().toArray(), asList("a", "c", "b").toArray()); 710 | 711 | // Supports clear all 712 | 713 | identityLoader.clearAll(); 714 | assertArrayEquals(customMap.stash.keySet().toArray(), emptyList().toArray()); 715 | } 716 | 717 | @Test 718 | public void batching_disabled_should_dispatch_immediately() throws Exception { 719 | List> loadCalls = new ArrayList<>(); 720 | DataLoaderOptions options = DataLoaderOptions.create().setBatchingEnabled(false); 721 | DataLoader identityLoader = idLoader(options, loadCalls); 722 | 723 | CompletableFuture fa = identityLoader.load("A"); 724 | CompletableFuture fb = identityLoader.load("B"); 725 | 726 | // caching is on still 727 | CompletableFuture fa1 = identityLoader.load("A"); 728 | CompletableFuture fb1 = identityLoader.load("B"); 729 | 730 | List values = CompletableFutureKit.allOf(asList(fa, fb, fa1, fb1)).join(); 731 | 732 | assertThat(fa.join(), equalTo("A")); 733 | assertThat(fb.join(), equalTo("B")); 734 | assertThat(fa1.join(), equalTo("A")); 735 | assertThat(fb1.join(), equalTo("B")); 736 | 737 | assertThat(values, equalTo(asList("A", "B", "A", "B"))); 738 | 739 | assertThat(loadCalls, equalTo(asList( 740 | singletonList("A"), 741 | singletonList("B")))); 742 | 743 | } 744 | 745 | @Test 746 | public void batching_disabled_and_caching_disabled_should_dispatch_immediately_and_forget() throws Exception { 747 | List> loadCalls = new ArrayList<>(); 748 | DataLoaderOptions options = DataLoaderOptions.create().setBatchingEnabled(false).setCachingEnabled(false); 749 | DataLoader identityLoader = idLoader(options, loadCalls); 750 | 751 | CompletableFuture fa = identityLoader.load("A"); 752 | CompletableFuture fb = identityLoader.load("B"); 753 | 754 | // caching is off 755 | CompletableFuture fa1 = identityLoader.load("A"); 756 | CompletableFuture fb1 = identityLoader.load("B"); 757 | 758 | List values = CompletableFutureKit.allOf(asList(fa, fb, fa1, fb1)).join(); 759 | 760 | assertThat(fa.join(), equalTo("A")); 761 | assertThat(fb.join(), equalTo("B")); 762 | assertThat(fa1.join(), equalTo("A")); 763 | assertThat(fb1.join(), equalTo("B")); 764 | 765 | assertThat(values, equalTo(asList("A", "B", "A", "B"))); 766 | 767 | assertThat(loadCalls, equalTo(asList( 768 | singletonList("A"), 769 | singletonList("B"), 770 | singletonList("A"), 771 | singletonList("B") 772 | ))); 773 | 774 | } 775 | 776 | @Test 777 | public void batches_multiple_requests_with_max_batch_size() throws Exception { 778 | List> loadCalls = new ArrayList<>(); 779 | DataLoader identityLoader = idLoader(DataLoaderOptions.create().setMaxBatchSize(2), loadCalls); 780 | 781 | CompletableFuture f1 = identityLoader.load(1); 782 | CompletableFuture f2 = identityLoader.load(2); 783 | CompletableFuture f3 = identityLoader.load(3); 784 | 785 | identityLoader.dispatch(); 786 | 787 | CompletableFuture.allOf(f1, f2, f3).join(); 788 | 789 | assertThat(f1.join(), equalTo(1)); 790 | assertThat(f2.join(), equalTo(2)); 791 | assertThat(f3.join(), equalTo(3)); 792 | 793 | assertThat(loadCalls, equalTo(asList(asList(1, 2), singletonList(3)))); 794 | 795 | } 796 | 797 | @Test 798 | public void can_split_max_batch_sizes_correctly() throws Exception { 799 | List> loadCalls = new ArrayList<>(); 800 | DataLoader identityLoader = idLoader(DataLoaderOptions.create().setMaxBatchSize(5), loadCalls); 801 | 802 | for (int i = 0; i < 21; i++) { 803 | identityLoader.load(i); 804 | } 805 | List> expectedCalls = new ArrayList<>(); 806 | expectedCalls.add(listFrom(0, 5)); 807 | expectedCalls.add(listFrom(5, 10)); 808 | expectedCalls.add(listFrom(10, 15)); 809 | expectedCalls.add(listFrom(15, 20)); 810 | expectedCalls.add(listFrom(20, 21)); 811 | 812 | List result = identityLoader.dispatch().join(); 813 | 814 | assertThat(result, equalTo(listFrom(0, 21))); 815 | assertThat(loadCalls, equalTo(expectedCalls)); 816 | 817 | } 818 | 819 | private Collection listFrom(int i, int max) { 820 | List ints = new ArrayList<>(); 821 | for (int j = i; j < max; j++) { 822 | ints.add(j); 823 | } 824 | return ints; 825 | } 826 | 827 | @Test 828 | public void should_Batch_loads_occurring_within_futures() { 829 | List> loadCalls = new ArrayList<>(); 830 | DataLoader identityLoader = idLoader(DataLoaderOptions.create(), loadCalls); 831 | 832 | Supplier nullValue = () -> null; 833 | 834 | CompletableFuture.supplyAsync(nullValue).thenAccept(v1 -> { 835 | identityLoader.load("a"); 836 | CompletableFuture.supplyAsync(nullValue).thenAccept(v2 -> { 837 | identityLoader.load("b"); 838 | CompletableFuture.supplyAsync(nullValue).thenAccept(v3 -> { 839 | identityLoader.load("c"); 840 | CompletableFuture.supplyAsync(nullValue).thenAccept( 841 | v4 -> 842 | identityLoader.load("d")); 843 | }); 844 | }); 845 | }); 846 | 847 | identityLoader.dispatchAndJoin(); 848 | 849 | assertThat(loadCalls, equalTo( 850 | singletonList(asList("a", "b", "c", "d")))); 851 | } 852 | 853 | @Test 854 | public void can_call_a_loader_from_a_loader() throws Exception { 855 | List> deepLoadCalls = new ArrayList<>(); 856 | DataLoader deepLoader = new DataLoader<>(keys -> { 857 | deepLoadCalls.add(keys); 858 | return CompletableFuture.completedFuture(keys); 859 | }); 860 | 861 | List> aLoadCalls = new ArrayList<>(); 862 | DataLoader aLoader = new DataLoader<>(keys -> { 863 | aLoadCalls.add(keys); 864 | return deepLoader.loadMany(keys); 865 | }); 866 | 867 | List> bLoadCalls = new ArrayList<>(); 868 | DataLoader bLoader = new DataLoader<>(keys -> { 869 | bLoadCalls.add(keys); 870 | return deepLoader.loadMany(keys); 871 | }); 872 | 873 | CompletableFuture a1 = aLoader.load("A1"); 874 | CompletableFuture a2 = aLoader.load("A2"); 875 | CompletableFuture b1 = bLoader.load("B1"); 876 | CompletableFuture b2 = bLoader.load("B2"); 877 | 878 | CompletableFuture.allOf( 879 | aLoader.dispatch(), 880 | deepLoader.dispatch(), 881 | bLoader.dispatch(), 882 | deepLoader.dispatch() 883 | ).join(); 884 | 885 | assertThat(a1.get(), equalTo("A1")); 886 | assertThat(a2.get(), equalTo("A2")); 887 | assertThat(b1.get(), equalTo("B1")); 888 | assertThat(b2.get(), equalTo("B2")); 889 | 890 | assertThat(aLoadCalls, equalTo( 891 | singletonList(asList("A1", "A2")))); 892 | 893 | assertThat(bLoadCalls, equalTo( 894 | singletonList(asList("B1", "B2")))); 895 | 896 | assertThat(deepLoadCalls, equalTo( 897 | asList(asList("A1", "A2"), asList("B1", "B2")))); 898 | } 899 | 900 | @Test 901 | public void should_allow_composition_of_data_loader_calls() throws Exception { 902 | UserManager userManager = new UserManager(); 903 | 904 | BatchLoader userBatchLoader = userIds -> CompletableFuture 905 | .supplyAsync(() -> userIds 906 | .stream() 907 | .map(userManager::loadUserById) 908 | .collect(Collectors.toList())); 909 | DataLoader userLoader = new DataLoader<>(userBatchLoader); 910 | 911 | AtomicBoolean gandalfCalled = new AtomicBoolean(false); 912 | AtomicBoolean sarumanCalled = new AtomicBoolean(false); 913 | 914 | userLoader.load(1L) 915 | .thenAccept(user -> userLoader.load(user.getInvitedByID()) 916 | .thenAccept(invitedBy -> { 917 | gandalfCalled.set(true); 918 | assertThat(invitedBy.getName(), equalTo("Manwë")); 919 | })); 920 | 921 | userLoader.load(2L) 922 | .thenAccept(user -> userLoader.load(user.getInvitedByID()) 923 | .thenAccept(invitedBy -> { 924 | sarumanCalled.set(true); 925 | assertThat(invitedBy.getName(), equalTo("Aulë")); 926 | })); 927 | 928 | List allResults = userLoader.dispatchAndJoin(); 929 | 930 | await().untilTrue(gandalfCalled); 931 | await().untilTrue(sarumanCalled); 932 | 933 | assertThat(allResults.size(), equalTo(4)); 934 | } 935 | 936 | 937 | private static CacheKey getJsonObjectCacheMapFn() { 938 | return key -> key.stream() 939 | .map(entry -> entry.getKey() + ":" + entry.getValue()) 940 | .sorted() 941 | .collect(Collectors.joining()); 942 | } 943 | 944 | private static DataLoader idLoader(DataLoaderOptions options, List> loadCalls) { 945 | return new DataLoader<>(keys -> { 946 | loadCalls.add(new ArrayList<>(keys)); 947 | @SuppressWarnings("unchecked") 948 | List values = keys.stream() 949 | .map(k -> (V) k) 950 | .collect(Collectors.toList()); 951 | return CompletableFuture.completedFuture(values); 952 | }, options); 953 | } 954 | 955 | private static DataLoader idLoaderBlowsUps( 956 | DataLoaderOptions options, List> loadCalls) { 957 | return new DataLoader<>(keys -> { 958 | loadCalls.add(new ArrayList<>(keys)); 959 | return futureError(); 960 | }, options); 961 | } 962 | 963 | private static DataLoader idLoaderAllExceptions( 964 | DataLoaderOptions options, List> loadCalls) { 965 | return new DataLoader<>(keys -> { 966 | loadCalls.add(new ArrayList<>(keys)); 967 | 968 | List errors = keys.stream().map(k -> new IllegalStateException("Error")).collect(Collectors.toList()); 969 | return CompletableFuture.completedFuture(errors); 970 | }, options); 971 | } 972 | 973 | private static DataLoader idLoaderOddEvenExceptions( 974 | DataLoaderOptions options, List> loadCalls) { 975 | return new DataLoader<>(keys -> { 976 | loadCalls.add(new ArrayList<>(keys)); 977 | 978 | List errors = new ArrayList<>(); 979 | for (Integer key : keys) { 980 | if (key % 2 == 0) { 981 | errors.add(key); 982 | } else { 983 | errors.add(new IllegalStateException("Error")); 984 | } 985 | } 986 | return CompletableFuture.completedFuture(errors); 987 | }, options); 988 | } 989 | 990 | private static CompletableFuture futureError() { 991 | return failedFuture(new IllegalStateException("Error")); 992 | } 993 | 994 | 995 | private BatchLoader keysAsValues() { 996 | return CompletableFuture::completedFuture; 997 | } 998 | 999 | public class CustomCacheMap implements CacheMap { 1000 | 1001 | public Map stash; 1002 | 1003 | public CustomCacheMap() { 1004 | stash = new LinkedHashMap<>(); 1005 | } 1006 | 1007 | @Override 1008 | public boolean containsKey(String key) { 1009 | return stash.containsKey(key); 1010 | } 1011 | 1012 | @Override 1013 | public Object get(String key) { 1014 | return stash.get(key); 1015 | } 1016 | 1017 | @Override 1018 | public CacheMap set(String key, Object value) { 1019 | stash.put(key, value); 1020 | return this; 1021 | } 1022 | 1023 | @Override 1024 | public CacheMap delete(String key) { 1025 | stash.remove(key); 1026 | return this; 1027 | } 1028 | 1029 | @Override 1030 | public CacheMap clear() { 1031 | stash.clear(); 1032 | return this; 1033 | } 1034 | } 1035 | } 1036 | 1037 | -------------------------------------------------------------------------------- /src/test/java/org/dataloader/JsonObject.java: -------------------------------------------------------------------------------- 1 | package org.dataloader; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.Map; 5 | import java.util.stream.Stream; 6 | 7 | class JsonObject { 8 | 9 | private final Map values; 10 | 11 | JsonObject() { 12 | values = new LinkedHashMap<>(); 13 | } 14 | 15 | public JsonObject put(String key, Object value) { 16 | values.put(key, value); 17 | return this; 18 | } 19 | 20 | @Override 21 | public boolean equals(Object o) { 22 | if (this == o) return true; 23 | if (o == null || getClass() != o.getClass()) return false; 24 | 25 | JsonObject that = (JsonObject) o; 26 | 27 | return values.equals(that.values); 28 | } 29 | 30 | @Override 31 | public int hashCode() { 32 | return values.hashCode(); 33 | } 34 | 35 | public Stream> stream() { 36 | return values.entrySet().stream(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/org/dataloader/fixtures/User.java: -------------------------------------------------------------------------------- 1 | package org.dataloader.fixtures; 2 | 3 | public class User { 4 | final Long id; 5 | final String name; 6 | final Long invitedByID; 7 | 8 | public User(Long id, Long invitedByID, String name) { 9 | this.id = id; 10 | this.name = name; 11 | this.invitedByID = invitedByID; 12 | } 13 | 14 | public Long getId() { 15 | return id; 16 | } 17 | 18 | public String getName() { 19 | return name; 20 | } 21 | 22 | public Long getInvitedByID() { 23 | return invitedByID; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return "User{" + 29 | "name='" + name + '\'' + 30 | "invitedByID='" + invitedByID + '\'' + 31 | '}'; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/org/dataloader/fixtures/UserManager.java: -------------------------------------------------------------------------------- 1 | package org.dataloader.fixtures; 2 | 3 | import java.util.LinkedHashMap; 4 | import java.util.List; 5 | import java.util.Map; 6 | import java.util.stream.Collectors; 7 | 8 | public class UserManager { 9 | 10 | public static final User ILÚVATAR = new User(-1L, -1L, "Ilúvatar"); 11 | public static final User AULË = new User(10001L, -1L, "Aulë"); 12 | public static final User OROMË = new User(10002L, -1L, "Oromë"); 13 | public static final User YAVANNA = new User(10003L, -1L, "Yavanna"); 14 | public static final User MANWË = new User(10004L, -1L, "Manwë"); 15 | public static final User MORGOTH = new User(10005L, -1L, "Morgoth"); 16 | public static final User CURUNIR = new User(2L, 10001L, "Curunir"); 17 | public static final User ALATAR = new User(3L, 10002L, "Alatar"); 18 | public static final User AIWENDIL = new User(4L, 10003L, "Aiwendil"); 19 | public static final User OLÓRIN = new User(1L, 10004L, "Olórin"); 20 | public static final User SAURON = new User(5L, 10005L, "Sauron"); 21 | 22 | final Map users = new LinkedHashMap<>(); 23 | 24 | { 25 | add(ILÚVATAR); 26 | 27 | add(AULË); 28 | add(OROMË); 29 | add(YAVANNA); 30 | add(MANWË); 31 | add(MORGOTH); 32 | 33 | add(CURUNIR); 34 | add(ALATAR); 35 | add(AIWENDIL); 36 | add(OLÓRIN); 37 | add(SAURON); 38 | } 39 | 40 | private void add(User user) { 41 | users.put(user.getId(), user); 42 | } 43 | 44 | public User loadUserById(Long userId) { 45 | return users.get(userId); 46 | } 47 | 48 | public List loadUsersById(List userIds) { 49 | return userIds.stream().map(this::loadUserById).collect(Collectors.toList()); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/test/java/org/dataloader/impl/PromisedValuesImplTest.java: -------------------------------------------------------------------------------- 1 | package org.dataloader.impl; 2 | 3 | import org.junit.Test; 4 | 5 | import java.util.Collections; 6 | import java.util.List; 7 | import java.util.concurrent.CompletableFuture; 8 | import java.util.concurrent.atomic.AtomicBoolean; 9 | 10 | import static java.util.Arrays.asList; 11 | import static java.util.concurrent.CompletableFuture.supplyAsync; 12 | import static org.awaitility.Awaitility.await; 13 | import static org.hamcrest.Matchers.equalTo; 14 | import static org.hamcrest.Matchers.instanceOf; 15 | import static org.hamcrest.Matchers.is; 16 | import static org.hamcrest.Matchers.nullValue; 17 | import static org.junit.Assert.assertThat; 18 | 19 | public class PromisedValuesImplTest { 20 | 21 | @Test 22 | public void will_compose_multiple_futures() throws Exception { 23 | 24 | CompletableFuture f1 = supplyAsync(() -> 666); 25 | CompletableFuture f2 = supplyAsync(() -> 999); 26 | 27 | PromisedValues promisedValues = PromisedValues.allOf(f1, f2); 28 | 29 | assertThat(promisedValues.size(), equalTo(2)); 30 | 31 | await().until(promisedValues::isDone, is(true)); 32 | assertThat(promisedValues.toList(), equalTo(asList(666, 999))); 33 | } 34 | 35 | @Test 36 | public void will_allow_extra_composition_of_futures() throws Exception { 37 | 38 | CompletableFuture f1 = supplyAsync(() -> 666); 39 | CompletableFuture f2 = supplyAsync(() -> 999); 40 | 41 | PromisedValues promisedValues = PromisedValues.allOf(f1, f2); 42 | 43 | AtomicBoolean acceptCalled = new AtomicBoolean(false); 44 | promisedValues.toCompletableFuture().thenAccept(list -> { 45 | acceptCalled.set(true); 46 | assertThat(list, equalTo(asList(666, 999))); 47 | }); 48 | 49 | await().untilTrue(acceptCalled); 50 | 51 | assertThat(promisedValues.isDone(), equalTo(true)); 52 | assertThat(promisedValues.succeeded(), equalTo(true)); 53 | assertThat(promisedValues.failed(), equalTo(false)); 54 | assertThat(promisedValues.cause(), nullValue()); 55 | 56 | 57 | assertThat(promisedValues.size(), equalTo(2)); 58 | assertThat(promisedValues.toList(), equalTo(asList(666, 999))); 59 | assertThat(promisedValues.join(), equalTo(asList(666, 999))); 60 | 61 | } 62 | 63 | @Test 64 | public void empty_list_works() throws Exception { 65 | PromisedValues promisedValues = PromisedValues.allOf(Collections.emptyList()); 66 | 67 | assertThat(promisedValues.isDone(), equalTo(true)); 68 | assertThat(promisedValues.succeeded(), equalTo(true)); 69 | assertThat(promisedValues.failed(), equalTo(false)); 70 | assertThat(promisedValues.cause(), nullValue()); 71 | 72 | assertThat(promisedValues.size(), equalTo(0)); 73 | assertThat(promisedValues.toList(), equalTo(Collections.emptyList())); 74 | assertThat(promisedValues.join(), equalTo(Collections.emptyList())); 75 | } 76 | 77 | 78 | @Test 79 | public void can_compose_multiple_promised_values() throws Exception { 80 | 81 | CompletableFuture a1 = supplyAsync(() -> "A1"); 82 | CompletableFuture a2 = supplyAsync(() -> "A2"); 83 | 84 | PromisedValues promisedValues1 = PromisedValues.allOf(a1, a2); 85 | 86 | CompletableFuture b1 = supplyAsync(() -> "B1"); 87 | CompletableFuture b2 = supplyAsync(() -> "B2"); 88 | 89 | PromisedValues promisedValues2 = PromisedValues.allOf(b1, b2); 90 | 91 | PromisedValues combinePromisedValues = PromisedValues.allPromisedValues(promisedValues1, promisedValues2); 92 | 93 | List joinedResult = combinePromisedValues.toCompletableFuture().join(); 94 | 95 | assertThat(joinedResult.size(), equalTo(4)); 96 | assertThat(joinedResult, equalTo(asList("A1", "A2", "B1", "B2"))); 97 | 98 | assertThat(combinePromisedValues.size(), equalTo(4)); 99 | assertThat(combinePromisedValues.succeeded(), equalTo(true)); 100 | assertThat(combinePromisedValues.failed(), equalTo(false)); 101 | assertThat(combinePromisedValues.isDone(), equalTo(true)); 102 | 103 | assertThat(combinePromisedValues.get(0), equalTo("A1")); 104 | assertThat(combinePromisedValues.get(1), equalTo("A2")); 105 | assertThat(combinePromisedValues.get(2), equalTo("B1")); 106 | assertThat(combinePromisedValues.get(3), equalTo("B2")); 107 | 108 | assertThat(combinePromisedValues.toList(), equalTo(asList("A1", "A2", "B1", "B2"))); 109 | 110 | } 111 | 112 | @Test 113 | public void can_compose_multiple_promised_values_and_fail_as_one() throws Exception { 114 | 115 | CompletableFuture a1 = supplyAsync(() -> "A1"); 116 | CompletableFuture a2 = supplyAsync(() -> "A2"); 117 | 118 | PromisedValues promisedValues1 = PromisedValues.allOf(a1, a2); 119 | 120 | CompletableFuture b1 = supplyAsync(() -> { 121 | throw new IllegalStateException("Bang"); 122 | }); 123 | CompletableFuture b2 = supplyAsync(() -> "B2"); 124 | 125 | PromisedValues promisedValues2 = PromisedValues.allOf(b1, b2); 126 | 127 | PromisedValues combinePromisedValues = PromisedValues.allPromisedValues(promisedValues1, promisedValues2); 128 | 129 | AtomicBoolean acceptCalled = new AtomicBoolean(); 130 | combinePromisedValues.thenAccept(pv -> { 131 | 132 | acceptCalled.set(true); 133 | 134 | assertThat(pv.succeeded(), equalTo(false)); 135 | assertThat(pv.failed(), equalTo(true)); 136 | assertThat(pv.isDone(), equalTo(true)); 137 | 138 | assertThat(pv.size(), equalTo(4)); 139 | 140 | assertThat(pv.get(0), equalTo("A1")); 141 | assertThat(pv.succeeded(0), equalTo(true)); 142 | assertThat(pv.cause(0), nullValue()); 143 | 144 | assertThat(pv.get(1), equalTo("A2")); 145 | assertThat(pv.succeeded(1), equalTo(true)); 146 | assertThat(pv.cause(1), nullValue()); 147 | 148 | // the one that went bad 149 | assertThat(pv.get(2), nullValue()); 150 | assertThat(pv.succeeded(2), equalTo(false)); 151 | assertThat(pv.cause(2), instanceOf(IllegalStateException.class)); 152 | 153 | assertThat(pv.get(3), equalTo("B2")); 154 | assertThat(pv.succeeded(3), equalTo(true)); 155 | assertThat(pv.cause(3), nullValue()); 156 | 157 | assertThat(pv.toList(), equalTo(asList("A1", "A2", null, "B2"))); 158 | 159 | }).join(); 160 | 161 | await().untilTrue(acceptCalled); 162 | 163 | } 164 | 165 | @Test 166 | public void exceptions_are_captured_and_reported() throws Exception { 167 | CompletableFuture f1 = supplyAsync(() -> 1); 168 | CompletableFuture f2 = supplyAsync(() -> { 169 | throw new IllegalStateException("bang"); 170 | }); 171 | 172 | PromisedValues promisedValues = PromisedValues.allOf(f1, f2); 173 | List result = promisedValues.toCompletableFuture().join(); 174 | 175 | assertThat(promisedValues.isDone(), equalTo(true)); 176 | assertThat(promisedValues.succeeded(), equalTo(false)); 177 | assertThat(promisedValues.cause(), instanceOf(IllegalStateException.class)); 178 | 179 | assertThat(promisedValues.toList(), equalTo(asList(1, null))); 180 | assertThat(result, equalTo(asList(1, null))); 181 | } 182 | } --------------------------------------------------------------------------------