├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── pom.xml └── src ├── main └── java │ └── com │ └── github │ └── phantomthief │ └── util │ ├── CursorIterator.java │ ├── CursorIteratorEx.java │ ├── GetByCursorDAO.java │ └── PageScroller.java └── test ├── java └── com │ └── github │ └── phantomthieft │ └── test │ ├── CursorIteratorExTest.java │ ├── CursorIteratorTest.java │ ├── MutableDAO.java │ ├── User.java │ └── UserDAO.java └── resources └── logback.xml /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | .idea 3 | *.iml 4 | 5 | # Mobile Tools for Java (J2ME) 6 | .mtj.tmp/ 7 | 8 | # Package Files # 9 | *.jar 10 | *.war 11 | *.ear 12 | 13 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 14 | hs_err_pid* 15 | /target/ 16 | 17 | .project 18 | .settings 19 | .classpath 20 | 21 | .DS_Store 22 | 23 | .README.md.html 24 | 25 | pom.xml.releaseBackup 26 | release.properties 27 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | jdk: 3 | - openjdk8 4 | after_success: 5 | - mvn clean test jacoco:report coveralls:report 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The Artistic License 2.0 2 | 3 | Copyright (c) 2015 PhantomThief 4 | 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | Preamble 9 | 10 | This license establishes the terms under which a given free software 11 | Package may be copied, modified, distributed, and/or redistributed. 12 | The intent is that the Copyright Holder maintains some artistic 13 | control over the development of that Package while still keeping the 14 | Package available as open source and free software. 15 | 16 | You are always permitted to make arrangements wholly outside of this 17 | license directly with the Copyright Holder of a given Package. If the 18 | terms of this license do not permit the full use that you propose to 19 | make of the Package, you should contact the Copyright Holder and seek 20 | a different licensing arrangement. 21 | 22 | Definitions 23 | 24 | "Copyright Holder" means the individual(s) or organization(s) 25 | named in the copyright notice for the entire Package. 26 | 27 | "Contributor" means any party that has contributed code or other 28 | material to the Package, in accordance with the Copyright Holder's 29 | procedures. 30 | 31 | "You" and "your" means any person who would like to copy, 32 | distribute, or modify the Package. 33 | 34 | "Package" means the collection of files distributed by the 35 | Copyright Holder, and derivatives of that collection and/or of 36 | those files. A given Package may consist of either the Standard 37 | Version, or a Modified Version. 38 | 39 | "Distribute" means providing a copy of the Package or making it 40 | accessible to anyone else, or in the case of a company or 41 | organization, to others outside of your company or organization. 42 | 43 | "Distributor Fee" means any fee that you charge for Distributing 44 | this Package or providing support for this Package to another 45 | party. It does not mean licensing fees. 46 | 47 | "Standard Version" refers to the Package if it has not been 48 | modified, or has been modified only in ways explicitly requested 49 | by the Copyright Holder. 50 | 51 | "Modified Version" means the Package, if it has been changed, and 52 | such changes were not explicitly requested by the Copyright 53 | Holder. 54 | 55 | "Original License" means this Artistic License as Distributed with 56 | the Standard Version of the Package, in its current version or as 57 | it may be modified by The Perl Foundation in the future. 58 | 59 | "Source" form means the source code, documentation source, and 60 | configuration files for the Package. 61 | 62 | "Compiled" form means the compiled bytecode, object code, binary, 63 | or any other form resulting from mechanical transformation or 64 | translation of the Source form. 65 | 66 | 67 | Permission for Use and Modification Without Distribution 68 | 69 | (1) You are permitted to use the Standard Version and create and use 70 | Modified Versions for any purpose without restriction, provided that 71 | you do not Distribute the Modified Version. 72 | 73 | 74 | Permissions for Redistribution of the Standard Version 75 | 76 | (2) You may Distribute verbatim copies of the Source form of the 77 | Standard Version of this Package in any medium without restriction, 78 | either gratis or for a Distributor Fee, provided that you duplicate 79 | all of the original copyright notices and associated disclaimers. At 80 | your discretion, such verbatim copies may or may not include a 81 | Compiled form of the Package. 82 | 83 | (3) You may apply any bug fixes, portability changes, and other 84 | modifications made available from the Copyright Holder. The resulting 85 | Package will still be considered the Standard Version, and as such 86 | will be subject to the Original License. 87 | 88 | 89 | Distribution of Modified Versions of the Package as Source 90 | 91 | (4) You may Distribute your Modified Version as Source (either gratis 92 | or for a Distributor Fee, and with or without a Compiled form of the 93 | Modified Version) provided that you clearly document how it differs 94 | from the Standard Version, including, but not limited to, documenting 95 | any non-standard features, executables, or modules, and provided that 96 | you do at least ONE of the following: 97 | 98 | (a) make the Modified Version available to the Copyright Holder 99 | of the Standard Version, under the Original License, so that the 100 | Copyright Holder may include your modifications in the Standard 101 | Version. 102 | 103 | (b) ensure that installation of your Modified Version does not 104 | prevent the user installing or running the Standard Version. In 105 | addition, the Modified Version must bear a name that is different 106 | from the name of the Standard Version. 107 | 108 | (c) allow anyone who receives a copy of the Modified Version to 109 | make the Source form of the Modified Version available to others 110 | under 111 | 112 | (i) the Original License or 113 | 114 | (ii) a license that permits the licensee to freely copy, 115 | modify and redistribute the Modified Version using the same 116 | licensing terms that apply to the copy that the licensee 117 | received, and requires that the Source form of the Modified 118 | Version, and of any works derived from it, be made freely 119 | available in that license fees are prohibited but Distributor 120 | Fees are allowed. 121 | 122 | 123 | Distribution of Compiled Forms of the Standard Version 124 | or Modified Versions without the Source 125 | 126 | (5) You may Distribute Compiled forms of the Standard Version without 127 | the Source, provided that you include complete instructions on how to 128 | get the Source of the Standard Version. Such instructions must be 129 | valid at the time of your distribution. If these instructions, at any 130 | time while you are carrying out such distribution, become invalid, you 131 | must provide new instructions on demand or cease further distribution. 132 | If you provide valid instructions or cease distribution within thirty 133 | days after you become aware that the instructions are invalid, then 134 | you do not forfeit any of your rights under this license. 135 | 136 | (6) You may Distribute a Modified Version in Compiled form without 137 | the Source, provided that you comply with Section 4 with respect to 138 | the Source of the Modified Version. 139 | 140 | 141 | Aggregating or Linking the Package 142 | 143 | (7) You may aggregate the Package (either the Standard Version or 144 | Modified Version) with other packages and Distribute the resulting 145 | aggregation provided that you do not charge a licensing fee for the 146 | Package. Distributor Fees are permitted, and licensing fees for other 147 | components in the aggregation are permitted. The terms of this license 148 | apply to the use and Distribution of the Standard or Modified Versions 149 | as included in the aggregation. 150 | 151 | (8) You are permitted to link Modified and Standard Versions with 152 | other works, to embed the Package in a larger work of your own, or to 153 | build stand-alone binary or bytecode versions of applications that 154 | include the Package, and Distribute the result without restriction, 155 | provided the result does not expose a direct interface to the Package. 156 | 157 | 158 | Items That are Not Considered Part of a Modified Version 159 | 160 | (9) Works (including, but not limited to, modules and scripts) that 161 | merely extend or make use of the Package, do not, by themselves, cause 162 | the Package to be a Modified Version. In addition, such works are not 163 | considered parts of the Package itself, and are not subject to the 164 | terms of this license. 165 | 166 | 167 | General Provisions 168 | 169 | (10) Any use, modification, and distribution of the Standard or 170 | Modified Versions is governed by this Artistic License. By using, 171 | modifying or distributing the Package, you accept this license. Do not 172 | use, modify, or distribute the Package, if you do not accept this 173 | license. 174 | 175 | (11) If your Modified Version has been derived from a Modified 176 | Version made by someone other than you, you are nevertheless required 177 | to ensure that your Modified Version complies with the requirements of 178 | this license. 179 | 180 | (12) This license does not grant you the right to use any trademark, 181 | service mark, tradename, or logo of the Copyright Holder. 182 | 183 | (13) This license includes the non-exclusive, worldwide, 184 | free-of-charge patent license to make, have made, use, offer to sell, 185 | sell, import and otherwise transfer the Package with respect to any 186 | patent claims licensable by the Copyright Holder that are necessarily 187 | infringed by the Package. If you institute patent litigation 188 | (including a cross-claim or counterclaim) against any party alleging 189 | that the Package constitutes direct or contributory patent 190 | infringement, then this Artistic License to you shall terminate on the 191 | date that such litigation is filed. 192 | 193 | (14) Disclaimer of Warranty: 194 | THE PACKAGE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS "AS 195 | IS' AND WITHOUT ANY EXPRESS OR IMPLIED WARRANTIES. THE IMPLIED 196 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OR 197 | NON-INFRINGEMENT ARE DISCLAIMED TO THE EXTENT PERMITTED BY YOUR LOCAL 198 | LAW. UNLESS REQUIRED BY LAW, NO COPYRIGHT HOLDER OR CONTRIBUTOR WILL 199 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL 200 | DAMAGES ARISING IN ANY WAY OUT OF THE USE OF THE PACKAGE, EVEN IF 201 | ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Cursor Iterator 2 | ======================= 3 | [![Build Status](https://travis-ci.org/PhantomThief/cursor-iterator.svg)](https://travis-ci.org/PhantomThief/cursor-iterator) 4 | [![Coverage Status](https://coveralls.io/repos/PhantomThief/cursor-iterator/badge.svg?branch=master)](https://coveralls.io/r/PhantomThief/cursor-iterator?branch=master) 5 | [![Total alerts](https://img.shields.io/lgtm/alerts/g/PhantomThief/cursor-iterator.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/PhantomThief/cursor-iterator/alerts/) 6 | [![Language grade: Java](https://img.shields.io/lgtm/grade/java/g/PhantomThief/cursor-iterator.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/PhantomThief/cursor-iterator/context:java) 7 | [![Maven Central](https://img.shields.io/maven-central/v/com.github.phantomthief/cursor-iterator)](https://search.maven.org/artifact/com.github.phantomthief/cursor-iterator/) 8 | 9 | 一个简单的适合移动端无限下拉构建数据的后端支持组件 10 | 11 | ## 使用方法 12 | 13 | ```Java 14 | public class UserDAO { 15 | 16 | private static final int MAX_USER_ID = 938; 17 | 18 | // A fake DAO for test 19 | public static List getUsersAscById(Integer startId, int limit) { 20 | if (startId == null) { 21 | startId = 0; 22 | } 23 | List result = IntStream.range(startId, Math.min(startId + limit, MAX_USER_ID)) 24 | .mapToObj(User::new) 25 | .collect(Collectors.toList()); 26 | System.out.println("get users asc by id, startId:" + startId + ", limit:" + limit 27 | + ", result:" + result); 28 | return result; 29 | } 30 | } 31 | 32 | // 声明 33 | CursorIterator users = CursorIterator.newGenericBuilder() 34 | .start(startId) 35 | .cursorExtractor(User::getId) 36 | .bufferSize(countPerFetch) 37 | .buildEx(UserDAO::getUsersAscById); 38 | 39 | // jdk1.8 Stream方式 40 | List collect = users.stream() 41 | .filter(user -> user.getId() % 11 == 0) 42 | .limit(5) 43 | .collect(Collectors.toList()); 44 | 45 | // 传统迭代器模式 46 | List finalResult = new ArrayList<>(); 47 | for (User user : users) { 48 | if (user.getId() % 11 == 0) { // filter 49 | System.out.println("add to final result:" + user); 50 | finalResult.add(user); 51 | } else { 52 | System.out.println("ignore add user:" + user); 53 | } 54 | if (finalResult.size() == 50) { 55 | break; 56 | } 57 | } 58 | ``` 59 | 60 | ## 注意事项 61 | 62 | * GetByCursorDAO返回的元素不能有null，因为如果结尾的元素是null，CursorIterator将无法根据null计算下一次迭代滑动窗口时的起始位置 -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | com.github.phantomthief 4 | cursor-iterator 5 | 1.0.14-SNAPSHOT 6 | 7 | 8 | 1.3.9 9 | 28.1-jre 10 | 11 | 5.7.0 12 | 1.1.8 13 | 14 | 3.0.0-M5 15 | 1.6.8 16 | 0.8.6 17 | 4.3.0 18 | 3.2.0 19 | 3.8.1 20 | 3.2.1 21 | 2.2.6 22 | 3.2.0 23 | 24 | 25 | 26 | org.sonatype.oss 27 | oss-parent 28 | 9 29 | 30 | 31 | Cursor Iterator 32 | A cursor iterator implements for batch build cursor data 33 | 34 | https://github.com/PhantomThief/cursor-iterator 35 | 36 | 37 | 38 | w.vela 39 | 40 | 41 | 42 | 43 | 44 | The Artistic License 2.0 45 | http://www.perlfoundation.org/artistic_license_2_0 46 | repo 47 | 48 | 49 | 50 | 51 | scm:git:git@github.com:PhantomThief/cursor-iterator.git 52 | https://github.com/PhantomThief/cursor-iterator.git 53 | scm:git:git@github.com:PhantomThief/cursor-iterator.git 54 | 55 | 56 | 57 | 58 | 59 | 60 | org.junit 61 | junit-bom 62 | ${junit.version} 63 | pom 64 | import 65 | 66 | 67 | 68 | 69 | 70 | 71 | com.google.guava 72 | guava 73 | ${guava.version} 74 | 75 | 76 | 77 | com.google.code.findbugs 78 | jsr305 79 | ${jsr305.version} 80 | true 81 | 82 | 83 | 84 | org.junit.jupiter 85 | junit-jupiter-api 86 | test 87 | 88 | 89 | org.junit.jupiter 90 | junit-jupiter-engine 91 | test 92 | 93 | 94 | ch.qos.logback 95 | logback-classic 96 | ${logback-classic.version} 97 | test 98 | 99 | 100 | 101 | 102 | 103 | 104 | org.apache.maven.plugins 105 | maven-compiler-plugin 106 | ${maven-compiler-plugin.version} 107 | 108 | 1.8 109 | 1.8 110 | true 111 | true 112 | UTF-8 113 | true 114 | 115 | -parameters 116 | 117 | 118 | 119 | 120 | org.apache.maven.plugins 121 | maven-source-plugin 122 | ${maven-source-plugin.version} 123 | 124 | 125 | attach-sources 126 | 127 | jar 128 | 129 | 130 | 131 | 132 | 133 | org.apache.maven.plugins 134 | maven-javadoc-plugin 135 | ${maven-javadoc-plugin.version} 136 | 137 | none 138 | 8 139 | 140 | 141 | 142 | attach-javadocs 143 | 144 | jar 145 | 146 | 147 | 148 | 149 | 150 | org.sonatype.plugins 151 | nexus-staging-maven-plugin 152 | ${nexus-staging-maven-plugin.version} 153 | true 154 | 155 | sonatype-nexus-staging 156 | https://oss.sonatype.org/ 157 | true 158 | 159 | 160 | 161 | org.jacoco 162 | jacoco-maven-plugin 163 | ${jacoco-maven-plugin.version} 164 | 165 | 166 | prepare-agent 167 | 168 | prepare-agent 169 | 170 | 171 | 172 | 173 | 174 | org.eluder.coveralls 175 | coveralls-maven-plugin 176 | ${coveralls-maven-plugin.version} 177 | 178 | 179 | maven-surefire-plugin 180 | ${maven-surefire-plugin.version} 181 | 182 | 183 | pl.project13.maven 184 | git-commit-id-plugin 185 | ${git-commit-id-plugin.version} 186 | 187 | 188 | get-the-git-infos 189 | 190 | revision 191 | 192 | 193 | 194 | 195 | false 196 | 8 197 | yyyyMMddHHmmssSSS 198 | false 199 | false 200 | true 201 | 202 | true 203 | 204 | 205 | git.branch 206 | git.build 207 | git.commit.id 208 | git.commit.time 209 | git.commit.user 210 | git.remote.origin.url 211 | 212 | 213 | 214 | 215 | org.apache.maven.plugins 216 | maven-jar-plugin 217 | ${maven-jar-plugin.version} 218 | 219 | 220 | 221 | true 222 | 223 | 224 | ${git.commit.id} 225 | ${git.build.time} 226 | ${git.branch} 227 | ${java.version} 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | sonatype-nexus-snapshots 238 | https://oss.sonatype.org/content/repositories/snapshots 239 | 240 | 241 | sonatype-nexus-staging 242 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 243 | 244 | 245 | 246 | -------------------------------------------------------------------------------- /src/main/java/com/github/phantomthief/util/CursorIterator.java: -------------------------------------------------------------------------------- 1 | package com.github.phantomthief.util; 2 | 3 | import static com.github.phantomthief.util.PageScroller.MODE_TRIM_FIRST; 4 | import static com.github.phantomthief.util.PageScroller.MODE_TRIM_LAST; 5 | import static com.google.common.base.Preconditions.checkArgument; 6 | import static com.google.common.base.Preconditions.checkNotNull; 7 | import static java.util.Spliterator.IMMUTABLE; 8 | import static java.util.Spliterator.NONNULL; 9 | import static java.util.Spliterator.ORDERED; 10 | import static java.util.Spliterators.spliteratorUnknownSize; 11 | 12 | import java.util.Iterator; 13 | import java.util.List; 14 | import java.util.function.Function; 15 | import java.util.function.IntSupplier; 16 | import java.util.stream.Stream; 17 | import java.util.stream.StreamSupport; 18 | 19 | import javax.annotation.CheckReturnValue; 20 | import javax.annotation.Nonnull; 21 | 22 | import com.google.common.collect.AbstractIterator; 23 | 24 | /** 25 | * 游标迭代器 26 | *

提供一种迭代器机制，可以连续地遍历瀑布流式的数据接口

27 | *

例如某个接口允许从指定的ID起始读n条记录返回：

28 | * 
{@code
 29 |  * interface UserRepository {
 30 |  *     List findAllUsers(Integer startId, int limit);
 31 |  * }
 32 |  * }
33 | * 如果要遍历列表的全部，通常可以设置一个游标ID，并多次调用findAllUsers来完成，例如： 34 | * 
{@code
 35 |  * void foreachUser() {
 36 |  *     Integer batchSize = 100;
 37 |  *     Integer cursorId = 1;
 38 |  *     while (null != cursorId) {
 39 |  *         List users = findAllUsers(cursorId, batchSize + 1);
 40 |  *         (users.size() > batchSize ? users.subList(0, batchSize) : users).forEach(user -> {
 41 |  *             // 访问单个用户
 42 |  *         });
 43 |  *         if (users.size() >= batchSize + 1) {
 44 |  *            cursorId = users.get(batchSize).getId();
 45 |  *         } else {
 46 |  *            cursorId = null;
 47 |  *         }
 48 |  *     }
 49 |  * }
 50 |  * }
51 | * CursorIterator将这一过程抽象成一个组件，可直接返回一个承载全部列表的迭代器，并在访问时延迟加载后面的分段，使用CursorIterator实现例如： 52 | * 
{@code
 53 |  * CursorIterator users = CursorIterator.newGenericBuilder()
 54 |  *         .start(1)
 55 |  *         .bufferSize(100)
 56 |  *         .cursorExtractor(User::getId)
 57 |  *         .buildEx(UserRepository::findAllUsers);
 58 |  * // 直接作为Stream访问
 59 |  * users.stream().forEach(user -> {
 60 |  *     // 访问单个用户
 61 |  * });
 62 |  * }
63 | * 64 | * @param ID类型泛型 65 | * @param 实体对象泛型 66 | * @author w.vela 67 | */ 68 | public class CursorIterator implements Iterable { 69 | 70 | private static final int DEFAULT_BUFFER_SIZE = 30; 71 | private final PageScroller pageScroller; 72 | 73 | private CursorIterator(PageScroller pageScroller) { 74 | this.pageScroller = pageScroller; 75 | } 76 | 77 | /** 78 | * 创建通用游标迭代器的构造器 79 | * 80 | * @param ID泛型类型 81 | * @param 返回实体对象的泛型类型 82 | * @return 构造器对象 83 | */ 84 | @CheckReturnValue 85 | @Nonnull 86 | public static GenericBuilder newGenericBuilder() { 87 | return new GenericBuilder<>(newBuilder()); 88 | } 89 | 90 | /** 91 | * 已废弃：请使用泛型版本{@link #newGenericBuilder()} 92 | */ 93 | @Deprecated 94 | @CheckReturnValue 95 | @Nonnull 96 | public static Builder newBuilder() { 97 | return new Builder<>(); 98 | } 99 | 100 | /** 101 | * 获取迭代器 102 | * 103 | * @return 返回迭代器对象 104 | */ 105 | @Nonnull 106 | @Override 107 | public Iterator iterator() { 108 | return new AbstractIterator() { 109 | 110 | private final Iterator> pageIterator = pageScroller.iterator(); 111 | private Iterator entityIteratorInPage; 112 | 113 | @Override 114 | protected Entity computeNext() { 115 | if (entityIteratorInPage == null || !entityIteratorInPage.hasNext()) { 116 | if (pageIterator.hasNext()) { 117 | entityIteratorInPage = pageIterator.next().iterator(); 118 | } else { 119 | return endOfData(); 120 | } 121 | } 122 | return entityIteratorInPage.next(); 123 | } 124 | }; 125 | } 126 | 127 | /** 128 | * 获取Stream 129 | * 130 | * @return 返回一个Stream对象 131 | */ 132 | public Stream stream() { 133 | return StreamSupport 134 | .stream(spliteratorUnknownSize(iterator(), (NONNULL | IMMUTABLE | ORDERED)), false); 135 | } 136 | 137 | /** 138 | * 泛型游标迭代器构造器 139 | * 140 | * @param ID泛型类型 141 | * @param 实体对象泛型类型 142 | */ 143 | public static class GenericBuilder { 144 | 145 | private final Builder builder; 146 | 147 | private GenericBuilder(Builder builder) { 148 | this.builder = builder; 149 | } 150 | 151 | /** 152 | * 已废弃：请使用 {@link #buildEx} 代替 153 | *

警告：此方法在遍历列表删除时有缺陷，会引起跳过删除部分ID，{@link #buildEx}在迭代中删除记录是友好的

154 | * 155 | * @see #buildEx 156 | */ 157 | @Deprecated 158 | @Nonnull 159 | public CursorIterator build(GetByCursorDAO dao) { 160 | return builder.build(dao); 161 | } 162 | 163 | /** 164 | * 构造游标迭代器 165 | * 166 | * @param dao 游标数据访问对象，提供根据初始ID、读取条数读取一列列表的数据访问方法 167 | * @return 构造的游标迭代器对象 168 | */ 169 | @Nonnull 170 | public CursorIterator buildEx(GetByCursorDAO dao) { 171 | return builder.buildEx(dao); 172 | } 173 | 174 | /** 175 | * 设置游标提取函数 176 | * 177 | * @param function 提供一个函数，从传入的实体上提取游标ID对象 178 | * @return 当前构造器对象 179 | */ 180 | @CheckReturnValue 181 | @Nonnull 182 | public GenericBuilder 183 | cursorExtractor(Function function) { 184 | builder.cursorExtractor(function); 185 | return this; 186 | } 187 | 188 | /** 189 | * 设置起始ID，此ID对应的记录将作为迭代器返回的第一条数据对象 190 | * 191 | * @param init 起始ID 192 | * @return 当前构造器对象 193 | */ 194 | @CheckReturnValue 195 | @Nonnull 196 | public GenericBuilder start(Id init) { 197 | builder.start(init); 198 | return this; 199 | } 200 | 201 | /** 202 | * 设置一次取列表数据返回的记录数 203 | * 204 | * @param bufferSize 一次取列表数据返回的记录数 205 | * @return 当前构造器对象 206 | */ 207 | @CheckReturnValue 208 | @Nonnull 209 | public GenericBuilder bufferSize(int bufferSize) { 210 | builder.bufferSize(bufferSize); 211 | return this; 212 | } 213 | 214 | /** 215 | * 设置一次取列表数据返回的记录数的提供器 216 | * 217 | * @param bufferSize 一次取列表数据返回的记录数提供器函数 218 | * @return 当前构造器对象 219 | */ 220 | @CheckReturnValue 221 | @Nonnull 222 | public GenericBuilder bufferSize(IntSupplier bufferSize) { 223 | builder.bufferSize(bufferSize); 224 | return this; 225 | } 226 | 227 | /** 228 | * 设置最多取的页数，大于等于1 229 | * 230 | * @param maxNumberOfPages 最多取的页数 231 | * @return 当前构造器对象 232 | */ 233 | @CheckReturnValue 234 | @Nonnull 235 | public GenericBuilder maxNumberOfPages(int maxNumberOfPages) { 236 | builder.maxNumberOfPages(maxNumberOfPages); 237 | return this; 238 | } 239 | } 240 | 241 | /** 242 | * 使用 {@link #newGenericBuilder()} 代替 243 | */ 244 | @SuppressWarnings("unchecked") 245 | @Deprecated 246 | public static class Builder { 247 | 248 | private GetByCursorDAO dao; 249 | private IntSupplier bufferSize; 250 | private Function function; 251 | private Id init; 252 | private int maxNumberOfPages = 0; 253 | private boolean mode = MODE_TRIM_FIRST; 254 | 255 | /** 256 | * 使用 {@link #buildEx} 代替，后者在迭代中删除是友好的 257 | */ 258 | @Deprecated 259 | @Nonnull 260 | public CursorIterator build(GetByCursorDAO dao) { 261 | Builder thisBuilder = (Builder) this; 262 | thisBuilder.dao = (GetByCursorDAO) dao; 263 | return thisBuilder.build(); 264 | } 265 | 266 | @Nonnull 267 | public CursorIterator buildEx(GetByCursorDAO dao) { 268 | this.mode = MODE_TRIM_LAST; 269 | return build(dao); 270 | } 271 | 272 | @CheckReturnValue 273 | @Nonnull 274 | public Builder bufferSize(int bufferSize) { 275 | checkArgument(bufferSize > 0); 276 | return bufferSize(() -> bufferSize); 277 | } 278 | 279 | @CheckReturnValue 280 | @Nonnull 281 | public Builder bufferSize(@Nonnull IntSupplier bufferSize) { 282 | this.bufferSize = checkNotNull(bufferSize); 283 | return this; 284 | } 285 | 286 | @CheckReturnValue 287 | @Nonnull 288 | public Builder cursorExtractor(Function function) { 289 | Builder thisBuilder = (Builder) this; 290 | thisBuilder.function = (Function) function; 291 | return thisBuilder; 292 | } 293 | 294 | @CheckReturnValue 295 | @Nonnull 296 | public Builder start(I init) { 297 | Builder thisBuilder = (Builder) this; 298 | thisBuilder.init = init; 299 | return thisBuilder; 300 | } 301 | 302 | @CheckReturnValue 303 | @Nonnull 304 | public Builder maxNumberOfPages(int maxNumberOfPages) { 305 | Builder thisBuilder = (Builder) this; 306 | thisBuilder.maxNumberOfPages = maxNumberOfPages; 307 | return thisBuilder; 308 | } 309 | 310 | private CursorIterator build() { 311 | ensure(); 312 | PageScroller scroller = new PageScroller<>(dao, init, bufferSize, function, 313 | mode); 314 | if (maxNumberOfPages > 0) { 315 | scroller.setMaxNumberOfPages(maxNumberOfPages); 316 | } 317 | return new CursorIterator<>(scroller); 318 | } 319 | 320 | private void ensure() { 321 | checkNotNull(dao); 322 | checkNotNull(function); 323 | 324 | if (bufferSize == null) { 325 | bufferSize = () -> DEFAULT_BUFFER_SIZE; 326 | } 327 | } 328 | } 329 | } 330 | -------------------------------------------------------------------------------- /src/main/java/com/github/phantomthief/util/CursorIteratorEx.java: -------------------------------------------------------------------------------- 1 | package com.github.phantomthief.util; 2 | 3 | import java.util.Iterator; 4 | import java.util.Objects; 5 | import java.util.Spliterator; 6 | import java.util.Spliterators; 7 | import java.util.function.Function; 8 | import java.util.function.Predicate; 9 | import java.util.stream.Stream; 10 | import java.util.stream.StreamSupport; 11 | 12 | import javax.annotation.CheckReturnValue; 13 | import javax.annotation.Nonnull; 14 | 15 | /** 16 | * 游标迭代器增强版 17 | *

提供一种迭代器机制，可以连续地遍历瀑布流式的数据接口，相比{@link CursorIterator}，此类允许定制游标处理时的细节

18 | *

例如某个接口允许从指定的ID起始读n条记录，并返回下一页的起始ID：

19 | * 
{@code
 20 |  * interface UserRepository {
 21 |  *     ScanResult scan(Integer startId, int limit);
 22 |  * }
 23 |  * class ScanResult {
 24 |  *     private final List users;
 25 |  *     private final Integer nextCursor;
 26 |  *     // getters/setters...
 27 |  * }
 28 |  * }
29 | * 使用CursorIteratorEx可以很简洁地实现遍历过程： 30 | * 
{@code
 31 |  * Integer startId = 100;
 32 |  * int countPerFetch = 10;
 33 |  * CursorIteratorEx users = newBuilder()
 34 |  *     .withDataRetriever((Integer cursor) -> repository.scan(cursor, countPerFetch))
 35 |  *     .withCursorExtractor(ScanResult::getNextCursor)
 36 |  *     .withDataExtractor((ScanResult s) -> s.getUsers().iterator())
 37 |  *     .withInitCursor(startId)
 38 |  *     .build();
 39 |  * }
40 | * 41 | * @param 返回实体的类型泛型 42 | * @param ID类型泛型 43 | * @param 列表读取结果对象的泛型 44 | * @author w.vela 45 | */ 46 | public class CursorIteratorEx implements Iterable { 47 | 48 | private final C initCursor; 49 | private final boolean checkFirstCursor; 50 | private final Function dataRetriever; 51 | private final Function cursorExtractor; 52 | private final Function> dataExtractor; 53 | private final Predicate endChecker; 54 | 55 | private CursorIteratorEx(C initCursor, boolean checkFirstCursor, Function dataRetriever, 56 | Function cursorExtractor, Function> dataExtractor, 57 | Predicate endChecker) { 58 | this.initCursor = initCursor; 59 | this.checkFirstCursor = checkFirstCursor; 60 | this.dataRetriever = dataRetriever; 61 | this.cursorExtractor = cursorExtractor; 62 | this.dataExtractor = dataExtractor; 63 | this.endChecker = endChecker; 64 | } 65 | 66 | /** 67 | * 创建游标迭代器构造器对象 68 | * 69 | * @return 游标迭代器构造器对象 70 | */ 71 | @CheckReturnValue 72 | @Nonnull 73 | public static Builder newBuilder() { 74 | return new Builder<>(); 75 | } 76 | 77 | /** 78 | * 获取迭代器 79 | * 80 | * @return 返回迭代器对象 81 | */ 82 | @Nonnull 83 | @Override 84 | public Iterator iterator() { 85 | return new RollingIterator(); 86 | } 87 | 88 | /** 89 | * 获取Stream 90 | * 91 | * @return 返回一个Stream对象 92 | */ 93 | public Stream stream() { 94 | return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator(), 95 | (Spliterator.NONNULL | Spliterator.IMMUTABLE)), false); 96 | } 97 | 98 | /** 99 | * 游标迭代器构造器 100 | * 101 | * @param 返回实体的类型泛型 102 | * @param ID类型泛型 103 | * @param 列表读取结果对象的泛型 104 | */ 105 | @SuppressWarnings("unchecked") 106 | public static final class Builder { 107 | 108 | private C initCursor; 109 | private boolean checkFirstCursor; 110 | private Function dataRetriever; 111 | private Function cursorExtractor; 112 | private Function> dataExtractor; 113 | private Predicate endChecker; 114 | 115 | /** 116 | * 设置起始ID，此ID对应的记录将作为迭代器返回的第一条数据对象 117 | * 118 | * @param initCursor 起始ID 119 | * @param ID类型泛型 120 | * @return 当前构造器对象 121 | */ 122 | @CheckReturnValue 123 | @Nonnull 124 | public Builder withInitCursor(C1 initCursor) { 125 | Builder thisBuilder = (Builder) this; 126 | thisBuilder.initCursor = initCursor; 127 | return thisBuilder; 128 | } 129 | 130 | /** 131 | * 设置是否对首个传入的游标进行终末检查 132 | * 133 | * @param check 是否对首个传入的游标进行终末检查，默认不检查 134 | * @return 当前构造器对象 135 | */ 136 | @CheckReturnValue 137 | @Nonnull 138 | public Builder firstCursorCheckEnd(boolean check) { 139 | this.checkFirstCursor = check; 140 | return this; 141 | } 142 | 143 | /** 144 | * 数据读取函数 145 | * 146 | * @param dataRetriever 数据读取函数，传入当前起始的ID，返回查询结果对象 147 | * @param ID类型泛型 148 | * @param 查询结果泛型 149 | * @return 当前构造器对象 150 | */ 151 | @CheckReturnValue 152 | @Nonnull 153 | public Builder withDataRetriever(Function dataRetriever) { 154 | Builder thisBuilder = (Builder) this; 155 | thisBuilder.dataRetriever = dataRetriever; 156 | return thisBuilder; 157 | } 158 | 159 | /** 160 | * 获取下一条游标函数 161 | * 162 | * @param cursorExtractor 获取下一条游标函数，传入当前获得的查询结果对象，返回下一条游标ID对象 163 | * @param ID类型泛型 164 | * @param 查询结果泛型 165 | * @return 当前构造器对象 166 | */ 167 | @CheckReturnValue 168 | @Nonnull 169 | public Builder withCursorExtractor(Function cursorExtractor) { 170 | Builder thisBuilder = (Builder) this; 171 | thisBuilder.cursorExtractor = cursorExtractor; 172 | return thisBuilder; 173 | } 174 | 175 | /** 176 | * 数据提取器函数 177 | * 178 | * @param dataExtractor 数据提取器函数，传入当前获得的查询结果对象，返回结果实体集合的迭代器对象 179 | * @param 实体类型泛型 180 | * @param 查询结果泛型 181 | * @return 当前构造器对象 182 | */ 183 | @CheckReturnValue 184 | @Nonnull 185 | public Builder 186 | withDataExtractor(Function> dataExtractor) { 187 | Builder thisBuilder = (Builder) this; 188 | thisBuilder.dataExtractor = dataExtractor; 189 | return thisBuilder; 190 | } 191 | 192 | /** 193 | * 设置游标终末检查器 194 | * 195 | * @param endChecker 游标终末检查器，传入当前的游标值，判断是否还有下一条记录存在，有返回true，否则返回false。 196 | * 默认根据游标是否为null来进行判断，不为空证明有下一条记录 197 | * @param ID类型泛型 198 | * @return 当前构造器对象 199 | */ 200 | @CheckReturnValue 201 | @Nonnull 202 | public Builder withEndChecker(Predicate endChecker) { 203 | Builder thisBuilder = (Builder) this; 204 | thisBuilder.endChecker = endChecker; 205 | return thisBuilder; 206 | } 207 | 208 | /** 209 | * 构造游标迭代器 210 | * 211 | * @param 返回实体的类型泛型 212 | * @param ID类型泛型 213 | * @param 列表读取结果对象的泛型 214 | * @return 游标迭代器对象 215 | */ 216 | @SuppressWarnings("rawtypes") 217 | @Nonnull 218 | public CursorIteratorEx build() { 219 | ensure(); 220 | return new CursorIteratorEx(initCursor, checkFirstCursor, dataRetriever, 221 | cursorExtractor, dataExtractor, endChecker); 222 | } 223 | 224 | private void ensure() { 225 | if (dataExtractor == null) { 226 | throw new NullPointerException("data extractor is null."); 227 | } 228 | if (dataRetriever == null) { 229 | throw new NullPointerException("data retriever is null."); 230 | } 231 | if (cursorExtractor == null) { 232 | throw new NullPointerException("data retriever is null."); 233 | } 234 | if (endChecker == null) { 235 | endChecker = Objects::isNull; 236 | } 237 | } 238 | 239 | } 240 | 241 | private final class RollingIterator implements Iterator { 242 | 243 | private C currentCursor; 244 | private R currentData; 245 | private Iterator currentIterator; 246 | 247 | RollingIterator() { 248 | currentCursor = initCursor; 249 | if (checkFirstCursor && endChecker.test(currentCursor)) { 250 | return; 251 | } 252 | currentData = dataRetriever.apply(currentCursor); 253 | if (currentData != null) { 254 | currentIterator = dataExtractor.apply(currentData); 255 | currentCursor = cursorExtractor.apply(currentData); 256 | } 257 | } 258 | 259 | @Override 260 | public boolean hasNext() { 261 | if (currentIterator == null) { 262 | return false; 263 | } 264 | if (currentIterator.hasNext()) { 265 | return true; 266 | } 267 | roll(); 268 | return currentIterator != null && currentIterator.hasNext(); 269 | } 270 | 271 | private void roll() { 272 | if (endChecker.test(currentCursor)) { 273 | currentData = null; 274 | currentIterator = null; 275 | return; 276 | } 277 | currentData = dataRetriever.apply(currentCursor); 278 | if (currentData == null) { 279 | currentIterator = null; 280 | } else { 281 | currentCursor = cursorExtractor.apply(currentData); 282 | currentIterator = dataExtractor.apply(currentData); 283 | } 284 | } 285 | 286 | @Override 287 | public T next() { 288 | return currentIterator.next(); 289 | } 290 | } 291 | } 292 | -------------------------------------------------------------------------------- /src/main/java/com/github/phantomthief/util/GetByCursorDAO.java: -------------------------------------------------------------------------------- 1 | package com.github.phantomthief.util; 2 | 3 | import java.util.List; 4 | 5 | import javax.annotation.Nullable; 6 | 7 | /** 8 | * 通过ID作为游标的列表数据访问对象接口 9 | * 10 | * @author w.vela 11 | */ 12 | public interface GetByCursorDAO { 13 | 14 | /** 15 | * 从指定的游标开始（包括），读取limit条记录作为列表返回 16 | * 17 | * @param cursor 起始游标，包括 18 | * @param limit 返回记录数 19 | */ 20 | List getByCursor(@Nullable Id cursor, int limit); 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/github/phantomthief/util/PageScroller.java: -------------------------------------------------------------------------------- 1 | package com.github.phantomthief.util; 2 | 3 | import static java.util.Collections.emptyList; 4 | 5 | import java.util.Iterator; 6 | import java.util.List; 7 | import java.util.function.Function; 8 | import java.util.function.IntSupplier; 9 | 10 | import javax.annotation.Nonnull; 11 | 12 | import com.google.common.collect.AbstractIterator; 13 | 14 | /** 15 | * 按页从数据库里面取. 16 | * PageScroller 是无状态的, 所有状态全放在 iterator 中. 17 | * 18 | * @author lixian 19 | */ 20 | class PageScroller implements Iterable> { 21 | 22 | static final boolean MODE_TRIM_FIRST = true; 23 | static final boolean MODE_TRIM_LAST = false; 24 | 25 | private final GetByCursorDAO dao; 26 | private final Id initCursor; 27 | private final IntSupplier bufferSize; 28 | private final Function entityIdFunction; 29 | private int maxNumberOfPages = Integer.MAX_VALUE; 30 | private final boolean mode; 31 | 32 | PageScroller(GetByCursorDAO dao, Id initCursor, IntSupplier bufferSize, 33 | Function entityIdFunction, boolean mode) { 34 | this.dao = dao; 35 | this.initCursor = initCursor; 36 | this.bufferSize = bufferSize; 37 | this.entityIdFunction = entityIdFunction; 38 | this.mode = mode; 39 | } 40 | 41 | public void setMaxNumberOfPages(int maxNumberOfPages) { 42 | this.maxNumberOfPages = maxNumberOfPages; 43 | } 44 | 45 | @Nonnull 46 | @Override 47 | public Iterator> iterator() { 48 | if (mode == MODE_TRIM_FIRST) { 49 | return new TrimFirstIterator(); 50 | } else { 51 | return new TrimLastIterator(); 52 | } 53 | } 54 | 55 | private class TrimFirstIterator extends AbstractIterator> { 56 | 57 | private List previousPage; 58 | private boolean firstTime = true; 59 | private int pageIndex = 0; 60 | 61 | @Override 62 | protected List computeNext() { 63 | int thisBufferSize = bufferSize.getAsInt(); 64 | List page; 65 | if (firstTime) { 66 | firstTime = false; 67 | // 第一次, 正常取 68 | page = dao.getByCursor(initCursor, thisBufferSize); 69 | } else { 70 | if (pageIndex >= maxNumberOfPages) { 71 | // 已经取到限制的页数了 72 | page = emptyList(); 73 | } else if (previousPage.size() < thisBufferSize) { 74 | // 上页还不满, fail fast 75 | page = emptyList(); 76 | } else { 77 | Id start = entityIdFunction.apply(previousPage.get(previousPage.size() - 1)); 78 | page = fetchOnePageExcludeStart(dao, start, thisBufferSize); 79 | } 80 | } 81 | 82 | previousPage = page; 83 | pageIndex++; 84 | return page.isEmpty() ? endOfData() : page; 85 | } 86 | 87 | /** 88 | * 由于 dao 实现中, start 是被包含的, 使用上一次 cursor 取的时候希望去除 start, 所以还需要多取一个 89 | */ 90 | private List fetchOnePageExcludeStart(GetByCursorDAO dao, Id start, 91 | int limit) { 92 | List entities = dao.getByCursor(start, limit + 1); 93 | return entities.isEmpty() ? entities : entities.subList(1, entities.size()); 94 | } 95 | } 96 | 97 | private class TrimLastIterator extends AbstractIterator> { 98 | 99 | private int pageIndex = 0; 100 | private Id cursor = initCursor; 101 | private boolean noNext = false; 102 | 103 | @Override 104 | protected List computeNext() { 105 | if (noNext) { 106 | return endOfData(); 107 | } 108 | pageIndex++; 109 | if (pageIndex > maxNumberOfPages) { 110 | return endOfData(); 111 | } 112 | int thisBufferSize = bufferSize.getAsInt(); 113 | List list = dao.getByCursor(cursor, thisBufferSize + 1); 114 | if (list.isEmpty()) { 115 | return endOfData(); 116 | } 117 | if (list.size() >= thisBufferSize + 1) { 118 | cursor = entityIdFunction.apply(list.get(thisBufferSize)); 119 | return list.subList(0, thisBufferSize); 120 | } else { 121 | noNext = true; 122 | return list; 123 | } 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/test/java/com/github/phantomthieft/test/CursorIteratorExTest.java: -------------------------------------------------------------------------------- 1 | package com.github.phantomthieft.test; 2 | 3 | import static com.github.phantomthief.util.CursorIteratorEx.newBuilder; 4 | import static java.util.stream.Collectors.toList; 5 | import static org.slf4j.LoggerFactory.getLogger; 6 | 7 | import java.util.List; 8 | 9 | import org.junit.jupiter.api.Test; 10 | import org.slf4j.Logger; 11 | 12 | import com.github.phantomthief.util.CursorIteratorEx; 13 | import com.github.phantomthieft.test.UserDAO.ScanResult; 14 | 15 | /** 16 | * @author w.vela 17 | */ 18 | class CursorIteratorExTest { 19 | 20 | private final Logger logger = getLogger(getClass()); 21 | 22 | @Test 23 | void test() { 24 | UserDAO userDAO = new UserDAO(); 25 | Integer startId = 100; 26 | int countPerFetch = 10; 27 | CursorIteratorEx users = newBuilder() 28 | .withDataRetriever((Integer cursor) -> userDAO.scan(cursor, countPerFetch)) 29 | .withCursorExtractor(ScanResult::getNextCursor) 30 | .withDataExtractor((ScanResult s) -> s.getUsers().iterator()) 31 | .withInitCursor(startId) 32 | .build(); 33 | 34 | List collect = users.stream() 35 | .filter(user -> user.getId() % 11 == 0) 36 | .limit(5) 37 | .collect(toList()); 38 | collect.forEach(u -> logger.info("user:{}", u)); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/com/github/phantomthieft/test/CursorIteratorTest.java: -------------------------------------------------------------------------------- 1 | package com.github.phantomthieft.test; 2 | 3 | import static com.github.phantomthief.util.CursorIterator.newBuilder; 4 | import static java.util.stream.Collectors.toList; 5 | import static org.junit.jupiter.api.Assertions.assertEquals; 6 | import static org.junit.jupiter.api.Assertions.assertTrue; 7 | import static org.slf4j.LoggerFactory.getLogger; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.concurrent.ThreadLocalRandom; 12 | 13 | import org.junit.jupiter.api.Test; 14 | import org.slf4j.Logger; 15 | 16 | import com.github.phantomthief.util.CursorIterator; 17 | 18 | /** 19 | * @author w.vela 20 | */ 21 | class CursorIteratorTest { 22 | 23 | private final Logger logger = getLogger(getClass()); 24 | 25 | @Test 26 | void test() { 27 | UserDAO userDAO = new UserDAO(); 28 | Integer startId = 100; 29 | int countPerFetch = 10; 30 | CursorIterator users = newBuilder() 31 | .start(startId) 32 | .bufferSize(countPerFetch) 33 | .cursorExtractor(User::getId) 34 | .build(userDAO::getUsersAscById); 35 | 36 | List finalResult = new ArrayList<>(); 37 | for (User user : users) { 38 | if (user.getId() % 11 == 0) { // filter 39 | logger.info("add to final result:" + user); 40 | finalResult.add(user); 41 | } else { 42 | logger.info("ignore add user:" + user); 43 | } 44 | if (finalResult.size() == 50) { 45 | break; 46 | } 47 | } 48 | } 49 | 50 | @Test 51 | void testIterateTwice() { 52 | UserDAO userDAO = new UserDAO(); 53 | Integer startId = 100; 54 | int countPerFetch = 10; 55 | CursorIterator users = newBuilder() 56 | .start(startId) 57 | .bufferSize(countPerFetch) 58 | .cursorExtractor(User::getId) 59 | .build(userDAO::getUsersAscById); 60 | iterateOnce(users); 61 | iterateOnce(users); 62 | } 63 | 64 | @Test 65 | void testPageThreshold() { 66 | UserDAO userDAO = new UserDAO(); 67 | Integer startId = 100; 68 | int countPerFetch = 10; 69 | CursorIterator users = newBuilder() 70 | .start(startId) 71 | .bufferSize(countPerFetch) 72 | .cursorExtractor(User::getId) 73 | .maxNumberOfPages(3).build(userDAO::getUsersAscById); 74 | int i = 100; 75 | for (User user : users) { 76 | assertEquals(i++, user.getId()); 77 | } 78 | assertEquals(130, i); 79 | } 80 | 81 | private void iterateOnce(CursorIterator users) { 82 | int i = 100; 83 | for (User user : users) { 84 | assertEquals(i++, user.getId()); 85 | } 86 | assertEquals(938, i); 87 | } 88 | 89 | @Test 90 | void testBuilder() { 91 | UserDAO userDAO = new UserDAO(); 92 | Integer startId = 100; 93 | int countPerFetch = 10; 94 | CursorIterator users = newBuilder() 95 | .start(startId) 96 | .cursorExtractor(User::getId) 97 | .bufferSize(countPerFetch) 98 | .build(userDAO::getUsersAscById); 99 | 100 | List collect = users.stream() 101 | .filter(user -> user.getId() % 11 == 0) 102 | .limit(5) 103 | .collect(toList()); 104 | collect.forEach(u -> logger.info("user:{}", u)); 105 | } 106 | 107 | @Test 108 | void testGenericBuilder() { 109 | UserDAO userDAO = new UserDAO(); 110 | Integer startId = 100; 111 | int countPerFetch = 10; 112 | CursorIterator users = CursorIterator. newGenericBuilder() 113 | .start(startId) 114 | .cursorExtractor(User::getId) 115 | .bufferSize(countPerFetch) 116 | .build(userDAO::getUsersAscById); 117 | 118 | List collect = users.stream() 119 | .filter(user -> user.getId() % 11 == 0) 120 | .limit(5) 121 | .collect(toList()); 122 | collect.forEach(u -> logger.info("user:{}", u)); 123 | } 124 | 125 | @Test 126 | void testDeleteWhileIterator() { 127 | testDeleting(1005); 128 | testDeleting(1000); 129 | testNoDeleting(1005); 130 | testNoDeleting(1000); 131 | } 132 | 133 | @Test 134 | void testDynamicBufferSize() { 135 | UserDAO userDAO = new UserDAO(); 136 | int[] daoCount = {0}; 137 | CursorIterator users = newBuilder() 138 | .start(null) 139 | .cursorExtractor(User::getId) 140 | .bufferSize(() -> ThreadLocalRandom.current().nextInt(1, 10)) 141 | .buildEx((startId, limit) -> { 142 | daoCount[0]++; 143 | return userDAO.getUsersAscById(startId, limit); 144 | }); 145 | List result = users.stream() 146 | .limit(200) 147 | .collect(toList()); 148 | assertEquals(200, result.size()); 149 | for (int i = 0; i < 200; i++) { 150 | assertEquals(i, result.get(i).getId()); 151 | } 152 | assertTrue(daoCount[0] > 20); 153 | } 154 | 155 | private void testDeleting(int allSize) { 156 | MutableDAO dao = new MutableDAO(allSize); 157 | CursorIterator cursor = CursorIterator. newGenericBuilder() 158 | .start(0) 159 | .cursorExtractor(User::getId) 160 | .bufferSize(10) 161 | .buildEx(dao::getByCursor); 162 | List users = new ArrayList<>(); 163 | for (User user : cursor) { 164 | users.add(user); 165 | assertTrue(dao.deleteUser(user.getId())); 166 | } 167 | for (int i = 1; i <= allSize; i++) { 168 | assertEquals(new User(i), users.get(i - 1)); 169 | } 170 | assertEquals(allSize, users.size()); 171 | assertEquals(0, cursor.stream().count()); 172 | } 173 | 174 | private void testNoDeleting(int allSize) { 175 | MutableDAO dao = new MutableDAO(allSize); 176 | CursorIterator cursor = CursorIterator. newGenericBuilder() 177 | .start(0) 178 | .cursorExtractor(User::getId) 179 | .bufferSize(10) 180 | .buildEx(dao::getByCursor); 181 | List users = new ArrayList<>(); 182 | for (User user : cursor) { 183 | users.add(user); 184 | } 185 | for (int i = 1; i <= allSize; i++) { 186 | assertEquals(new User(i), users.get(i - 1)); 187 | } 188 | assertEquals(allSize, users.size()); 189 | } 190 | 191 | @Test 192 | void testPageSize() { 193 | int allSize = 1005; 194 | MutableDAO dao = new MutableDAO(allSize); 195 | int bufferSize = 10; 196 | int maxNumberOfPages = 3; 197 | CursorIterator cursor = CursorIterator. newGenericBuilder() 198 | .start(0) 199 | .cursorExtractor(User::getId) 200 | .bufferSize(bufferSize) 201 | .maxNumberOfPages(maxNumberOfPages) 202 | .buildEx(dao::getByCursor); 203 | List users = new ArrayList<>(); 204 | for (User user : cursor) { 205 | users.add(user); 206 | assertTrue(dao.deleteUser(user.getId())); 207 | } 208 | int read = bufferSize * maxNumberOfPages; 209 | assertEquals(read, users.size()); 210 | for (int i = 1; i <= read; i++) { 211 | assertEquals(new User(i), users.get(i - 1)); 212 | } 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/test/java/com/github/phantomthieft/test/MutableDAO.java: -------------------------------------------------------------------------------- 1 | package com.github.phantomthieft.test; 2 | 3 | import static java.util.stream.Collectors.toList; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | /** 9 | * @author w.vela 10 | * Created on 2018-12-04. 11 | */ 12 | class MutableDAO { 13 | 14 | private final List userList; 15 | 16 | MutableDAO(int allUserCount) { 17 | this.userList = new ArrayList<>(allUserCount); 18 | for (int i = 1; i <= allUserCount; i++) { 19 | userList.add(new User(i)); 20 | } 21 | } 22 | 23 | List getByCursor(int startId, int limit) { 24 | return userList.stream() 25 | .filter(user -> user.getId() >= startId) 26 | .limit(limit) 27 | .collect(toList()); 28 | } 29 | 30 | boolean deleteUser(int userId) { 31 | return userList.removeIf(user -> user.getId() == userId); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/test/java/com/github/phantomthieft/test/User.java: -------------------------------------------------------------------------------- 1 | package com.github.phantomthieft.test; 2 | 3 | import java.util.Objects; 4 | 5 | /** 6 | * @author w.vela 7 | */ 8 | public class User { 9 | 10 | private final int id; 11 | 12 | public User(int id) { 13 | this.id = id; 14 | } 15 | 16 | public int getId() { 17 | return id; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return "User [id=" + id + "]"; 23 | } 24 | 25 | @Override 26 | public boolean equals(Object o) { 27 | if (this == o) { 28 | return true; 29 | } 30 | if (!(o instanceof User)) { 31 | return false; 32 | } 33 | User user = (User) o; 34 | return id == user.id; 35 | } 36 | 37 | @Override 38 | public int hashCode() { 39 | return Objects.hash(id); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/com/github/phantomthieft/test/UserDAO.java: -------------------------------------------------------------------------------- 1 | package com.github.phantomthieft.test; 2 | 3 | import static java.lang.Math.min; 4 | import static java.util.stream.IntStream.range; 5 | import static org.slf4j.LoggerFactory.getLogger; 6 | 7 | import java.util.List; 8 | import java.util.stream.Collectors; 9 | 10 | import org.slf4j.Logger; 11 | 12 | /** 13 | * @author w.vela 14 | */ 15 | public class UserDAO { 16 | 17 | private static final int MAX_USER_ID = 938; 18 | 19 | private final Logger logger = getLogger(getClass()); 20 | 21 | // A fake DAO for test 22 | public List getUsersAscById(Integer startId, int limit) { 23 | if (startId == null) { 24 | startId = 0; 25 | } 26 | List result = range(startId, min(startId + limit, MAX_USER_ID)).mapToObj(User::new) 27 | .collect(Collectors.toList()); 28 | logger.trace("get users asc by id, startId:" + startId + ", limit:" + limit + ", result:" 29 | + result); 30 | return result; 31 | } 32 | 33 | // A fake DAO for test 34 | public ScanResult scan(Integer startId, int limit) { 35 | if (startId == null) { 36 | startId = 0; 37 | } 38 | List result = range(startId, min(startId + limit, MAX_USER_ID)).mapToObj(User::new) 39 | .collect(Collectors.toList()); 40 | logger.trace("get users asc by id, startId:" + startId + ", limit:" + limit + ", result:" 41 | + result); 42 | Integer nextCursor = startId + limit > MAX_USER_ID ? null : startId + limit; 43 | return new ScanResult(result, nextCursor); 44 | } 45 | 46 | public static final class ScanResult { 47 | 48 | private final List users; 49 | private final Integer nextCursor; 50 | 51 | private ScanResult(List users, Integer nextCursor) { 52 | this.users = users; 53 | this.nextCursor = nextCursor; 54 | } 55 | 56 | public List getUsers() { 57 | return users; 58 | } 59 | 60 | public Integer getNextCursor() { 61 | return nextCursor; 62 | } 63 | 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/test/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %level \(%F:%L\) - %msg %n 7 | 8 | true 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | --------------------------------------------------------------------------------