├── .gitignore ├── LICENSE ├── README.md ├── liteBatch-core ├── pom.xml └── src │ └── main │ └── java │ ├── META-INF │ └── MANIFEST.MF │ └── com │ └── litesalt │ └── batch │ ├── DBRowBatchListenerBuilder.java │ ├── FileRowBatchListenerBuilder.java │ ├── annotation │ ├── AliasField.java │ ├── AliasTable.java │ └── ExcludeField.java │ ├── context │ ├── HandlerContext.java │ └── QueueContext.java │ ├── entity │ └── DBColumnMetaData.java │ ├── enums │ ├── FileSavedCapacity.java │ └── TargetType.java │ ├── handler │ ├── DBRowBatchHandler.java │ ├── FileRowBatchHandler.java │ └── RowBatchHandler.java │ ├── listener │ └── RowBatchListener.java │ ├── monitor │ └── QueueStatusMonitor.java │ ├── queue │ ├── MemoryRowBatchQueue.java │ ├── RedisRowBatchQueue.java │ └── RowBatchQueue.java │ └── util │ ├── CamelCaseUtils.java │ └── Reflections.java └── liteBatch-test ├── pom.xml └── src └── main ├── java └── com │ └── litesalt │ └── batch │ └── test │ ├── Person.java │ ├── PersonVo.java │ └── TestMain.java └── resources ├── config.properties ├── log4j.properties └── spring-db.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | 10 | 11 | # Packages # 12 | ############ 13 | # it's better to unpack these files and commit the raw source 14 | # git has its own built in compression methods 15 | *.7z 16 | *.dmg 17 | *.gz 18 | *.iso 19 | *.jar 20 | *.rar 21 | *.tar 22 | *.zip 23 | *.war 24 | *.del 25 | *.pmd 26 | .tern-project 27 | 28 | 29 | # Logs and databases # 30 | ###################### 31 | *.log 32 | *.log.* 33 | # OS generated files # 34 | ###################### 35 | .DS_Store* 36 | ehthumbs.db 37 | Icon? 38 | Thumbs.db 39 | 40 | 41 | # Editor Files # 42 | ################ 43 | *~ 44 | *.swp 45 | 46 | 47 | # Gradle Files # 48 | ################ 49 | .gradle 50 | 51 | 52 | # Build output directies 53 | /target 54 | */target 55 | /build 56 | */build 57 | 58 | 59 | # IntelliJ specific files/directories 60 | out 61 | .idea 62 | *.ipr 63 | *.iws 64 | *.iml 65 | atlassian-ide-plugin.xml 66 | 67 | 68 | # Eclipse specific files/directories 69 | .classpath 70 | .project 71 | .settings 72 | .metadata 73 | .myeclipse 74 | 75 | 76 | # NetBeans specific files/directories 77 | .nbattrs 78 | 79 | *.mymetadata 80 | /logs 81 | */logs 82 | 83 | /payClear-timer/.tern-project 84 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, and 10 | distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by the copyright 13 | owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities 16 | that control, are controlled by, or are under common control with that entity. 17 | For the purposes of this definition, "control" means (i) the power, direct or 18 | indirect, to cause the direction or management of such entity, whether by 19 | contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the 20 | outstanding shares, or (iii) beneficial ownership of such entity. 21 | 22 | "You" (or "Your") shall mean an individual or Legal Entity exercising 23 | permissions granted by this License. 24 | 25 | "Source" form shall mean the preferred form for making modifications, including 26 | but not limited to software source code, documentation source, and configuration 27 | files. 28 | 29 | "Object" form shall mean any form resulting from mechanical transformation or 30 | translation of a Source form, including but not limited to compiled object code, 31 | generated documentation, and conversions to other media types. 32 | 33 | "Work" shall mean the work of authorship, whether in Source or Object form, made 34 | available under the License, as indicated by a copyright notice that is included 35 | in or attached to the work (an example is provided in the Appendix below). 36 | 37 | "Derivative Works" shall mean any work, whether in Source or Object form, that 38 | is based on (or derived from) the Work and for which the editorial revisions, 39 | annotations, elaborations, or other modifications represent, as a whole, an 40 | original work of authorship. For the purposes of this License, Derivative Works 41 | shall not include works that remain separable from, or merely link (or bind by 42 | name) to the interfaces of, the Work and Derivative Works thereof. 43 | 44 | "Contribution" shall mean any work of authorship, including the original version 45 | of the Work and any modifications or additions to that Work or Derivative Works 46 | thereof, that is intentionally submitted to Licensor for inclusion in the Work 47 | by the copyright owner or by an individual or Legal Entity authorized to submit 48 | on behalf of the copyright owner. For the purposes of this definition, 49 | "submitted" means any form of electronic, verbal, or written communication sent 50 | to the Licensor or its representatives, including but not limited to 51 | communication on electronic mailing lists, source code control systems, and 52 | issue tracking systems that are managed by, or on behalf of, the Licensor for 53 | the purpose of discussing and improving the Work, but excluding communication 54 | that is conspicuously marked or otherwise designated in writing by the copyright 55 | owner as "Not a Contribution." 56 | 57 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf 58 | of whom a Contribution has been received by Licensor and subsequently 59 | incorporated within the Work. 60 | 61 | 2. Grant of Copyright License. 62 | 63 | Subject to the terms and conditions of this License, each Contributor hereby 64 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 65 | irrevocable copyright license to reproduce, prepare Derivative Works of, 66 | publicly display, publicly perform, sublicense, and distribute the Work and such 67 | Derivative Works in Source or Object form. 68 | 69 | 3. Grant of Patent License. 70 | 71 | Subject to the terms and conditions of this License, each Contributor hereby 72 | grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, 73 | irrevocable (except as stated in this section) patent license to make, have 74 | made, use, offer to sell, sell, import, and otherwise transfer the Work, where 75 | such license applies only to those patent claims licensable by such Contributor 76 | that are necessarily infringed by their Contribution(s) alone or by combination 77 | of their Contribution(s) with the Work to which such Contribution(s) was 78 | submitted. If You institute patent litigation against any entity (including a 79 | cross-claim or counterclaim in a lawsuit) alleging that the Work or a 80 | Contribution incorporated within the Work constitutes direct or contributory 81 | patent infringement, then any patent licenses granted to You under this License 82 | for that Work shall terminate as of the date such litigation is filed. 83 | 84 | 4. Redistribution. 85 | 86 | You may reproduce and distribute copies of the Work or Derivative Works thereof 87 | in any medium, with or without modifications, and in Source or Object form, 88 | provided that You meet the following conditions: 89 | 90 | You must give any other recipients of the Work or Derivative Works a copy of 91 | this License; and 92 | You must cause any modified files to carry prominent notices stating that You 93 | changed the files; and 94 | You must retain, in the Source form of any Derivative Works that You distribute, 95 | all copyright, patent, trademark, and attribution notices from the Source form 96 | of the Work, excluding those notices that do not pertain to any part of the 97 | Derivative Works; and 98 | If the Work includes a "NOTICE" text file as part of its distribution, then any 99 | Derivative Works that You distribute must include a readable copy of the 100 | attribution notices contained within such NOTICE file, excluding those notices 101 | that do not pertain to any part of the Derivative Works, in at least one of the 102 | following places: within a NOTICE text file distributed as part of the 103 | Derivative Works; within the Source form or documentation, if provided along 104 | with the Derivative Works; or, within a display generated by the Derivative 105 | Works, if and wherever such third-party notices normally appear. The contents of 106 | the NOTICE file are for informational purposes only and do not modify the 107 | License. You may add Your own attribution notices within Derivative Works that 108 | You distribute, alongside or as an addendum to the NOTICE text from the Work, 109 | provided that such additional attribution notices cannot be construed as 110 | modifying the License. 111 | You may add Your own copyright statement to Your modifications and may provide 112 | additional or different license terms and conditions for use, reproduction, or 113 | distribution of Your modifications, or for any such Derivative Works as a whole, 114 | provided Your use, reproduction, and distribution of the Work otherwise complies 115 | with the conditions stated in this License. 116 | 117 | 5. Submission of Contributions. 118 | 119 | Unless You explicitly state otherwise, any Contribution intentionally submitted 120 | for inclusion in the Work by You to the Licensor shall be under the terms and 121 | conditions of this License, without any additional terms or conditions. 122 | Notwithstanding the above, nothing herein shall supersede or modify the terms of 123 | any separate license agreement you may have executed with Licensor regarding 124 | such Contributions. 125 | 126 | 6. Trademarks. 127 | 128 | This License does not grant permission to use the trade names, trademarks, 129 | service marks, or product names of the Licensor, except as required for 130 | reasonable and customary use in describing the origin of the Work and 131 | reproducing the content of the NOTICE file. 132 | 133 | 7. Disclaimer of Warranty. 134 | 135 | Unless required by applicable law or agreed to in writing, Licensor provides the 136 | Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, 137 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, 138 | including, without limitation, any warranties or conditions of TITLE, 139 | NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are 140 | solely responsible for determining the appropriateness of using or 141 | redistributing the Work and assume any risks associated with Your exercise of 142 | permissions under this License. 143 | 144 | 8. Limitation of Liability. 145 | 146 | In no event and under no legal theory, whether in tort (including negligence), 147 | contract, or otherwise, unless required by applicable law (such as deliberate 148 | and grossly negligent acts) or agreed to in writing, shall any Contributor be 149 | liable to You for damages, including any direct, indirect, special, incidental, 150 | or consequential damages of any character arising as a result of this License or 151 | out of the use or inability to use the Work (including but not limited to 152 | damages for loss of goodwill, work stoppage, computer failure or malfunction, or 153 | any and all other commercial damages or losses), even if such Contributor has 154 | been advised of the possibility of such damages. 155 | 156 | 9. Accepting Warranty or Additional Liability. 157 | 158 | While redistributing the Work or Derivative Works thereof, You may choose to 159 | offer, and charge a fee for, acceptance of support, warranty, indemnity, or 160 | other liability obligations and/or rights consistent with this License. However, 161 | in accepting such obligations, You may act only on Your own behalf and on Your 162 | sole responsibility, not on behalf of any other Contributor, and only if You 163 | agree to indemnify, defend, and hold each Contributor harmless for any liability 164 | incurred by, or claims asserted against, such Contributor by reason of your 165 | accepting any such warranty or additional liability. 166 | 167 | END OF TERMS AND CONDITIONS 168 | 169 | APPENDIX: How to apply the Apache License to your work 170 | 171 | To apply the Apache License to your work, attach the following boilerplate 172 | notice, with the fields enclosed by brackets "{}" replaced with your own 173 | identifying information. (Don't include the brackets!) The text should be 174 | enclosed in the appropriate comment syntax for the file format. We also 175 | recommend that a file or class name and description of purpose be included on 176 | the same "printed page" as the copyright notice for easier identification within 177 | third-party archives. 178 | 179 | Copyright 2016 铂赛东 180 | 181 | Licensed under the Apache License, Version 2.0 (the "License"); 182 | you may not use this file except in compliance with the License. 183 | You may obtain a copy of the License at 184 | 185 | http://www.apache.org/licenses/LICENSE-2.0 186 | 187 | Unless required by applicable law or agreed to in writing, software 188 | distributed under the License is distributed on an "AS IS" BASIS, 189 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 190 | See the License for the specific language governing permissions and 191 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bryan31/liteBatch/c6575aacc35b292105525d95fe6845bfe23024b2/README.md -------------------------------------------------------------------------------- /liteBatch-core/pom.xml: -------------------------------------------------------------------------------- 1 |  2 | 4 | com.lightSalt.liteBatch 5 | liteBatch-core 6 | jar 7 | 4.0.0 8 | 1.1.0 9 | 10 | 11 | UTF-8 12 | 1.7 13 | 3.4 14 | 4.1.7.RELEASE 15 | 1.7.21 16 | 1.2.17 17 | 1.7.5 18 | 1.7.13 19 | 2.9.0 20 | 1.2.7 21 | 22 | 23 | 24 | 25 | org.apache.commons 26 | commons-lang3 27 | ${commons.lang3.version} 28 | 29 | 30 | org.springframework 31 | spring-jdbc 32 | ${spring.version} 33 | 34 | 35 | log4j 36 | log4j 37 | ${log4j.version} 38 | 39 | 40 | org.slf4j 41 | slf4j-api 42 | ${org.slf4j.version} 43 | 44 | 45 | org.slf4j 46 | slf4j-log4j12 47 | ${log4j-slf4j.version} 48 | 49 | 50 | redis.clients 51 | jedis 52 | ${jedis.version} 53 | 54 | 55 | com.alibaba 56 | fastjson 57 | ${fastjson.version} 58 | 59 | 60 | 61 | 62 | 63 | 64 | org.apache.maven.plugins 65 | maven-compiler-plugin 66 | 67 | UTF-8 68 | ${java.version} 69 | ${java.version} 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Class-Path: 3 | 4 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/DBRowBatchListenerBuilder.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch; 2 | 3 | import org.springframework.jdbc.core.JdbcTemplate; 4 | 5 | import com.litesalt.batch.context.HandlerContext; 6 | import com.litesalt.batch.context.QueueContext; 7 | import com.litesalt.batch.enums.TargetType; 8 | import com.litesalt.batch.handler.DBRowBatchHandler; 9 | import com.litesalt.batch.listener.RowBatchListener; 10 | import com.litesalt.batch.queue.MemoryRowBatchQueue; 11 | import com.litesalt.batch.queue.RedisRowBatchQueue; 12 | 13 | /** 14 | * @author Paul-xiong 15 | * @date 2017年2月27日 16 | * @description 批插监听管理器构造器 17 | */ 18 | public class DBRowBatchListenerBuilder { 19 | 20 | /** 21 | * 构建内存批插监听管理器 22 | * 23 | * @param jdbcTemplate 24 | * @param submitCapacity 25 | * @param clazz 26 | * @return 27 | */ 28 | public static RowBatchListener buildMemoryRowBatchListener(JdbcTemplate jdbcTemplate, long submitCapacity, Class clazz) { 29 | return buildMemoryRowBatchListener(jdbcTemplate, submitCapacity, clazz, false); 30 | } 31 | 32 | /** 33 | * 构建内存批插监听管理器 34 | * 35 | * @param jdbcTemplate 36 | * @param submitCapacity 37 | * @param clazz 38 | * @param syn 39 | * @return 40 | */ 41 | public static RowBatchListener buildMemoryRowBatchListener(JdbcTemplate jdbcTemplate, long submitCapacity, Class clazz, boolean syn) { 42 | MemoryRowBatchQueue queue = new MemoryRowBatchQueue(); 43 | HandlerContext context = new HandlerContext(queue, submitCapacity, clazz, syn); 44 | DBRowBatchHandler rowBatchHandler = new DBRowBatchHandler(context, jdbcTemplate); 45 | RowBatchListener listener = new RowBatchListener(rowBatchHandler); 46 | return listener; 47 | } 48 | 49 | /** 50 | * 构建redis批插监听管理器 51 | * 52 | * @param jdbcTemplate 53 | * @param submitCapacity 54 | * @param clazz 55 | * @param host 56 | * @param port 57 | * @param auth 58 | * @return 59 | */ 60 | public static RowBatchListener buildRedisRowBatchListener(JdbcTemplate jdbcTemplate, long submitCapacity, Class clazz, String host, int port, String auth) { 61 | return buildRedisRowBatchListener(jdbcTemplate, submitCapacity, clazz, host, port, auth, false); 62 | } 63 | 64 | /** 65 | * 构建redis批插监听管理器 66 | * 67 | * @param jdbcTemplate 68 | * @param submitCapacity 69 | * @param clazz 70 | * @param host 71 | * @param port 72 | * @param syn 73 | * @return 74 | */ 75 | public static RowBatchListener buildRedisRowBatchListener(JdbcTemplate jdbcTemplate, long submitCapacity, Class clazz, String host, int port, String auth, boolean syn) { 76 | QueueContext qContext = new QueueContext(TargetType.DB, clazz); 77 | RedisRowBatchQueue queue = new RedisRowBatchQueue(qContext, host, port, auth); 78 | HandlerContext context = new HandlerContext<>(queue, submitCapacity, clazz, syn); 79 | DBRowBatchHandler rowBatchHandler = new DBRowBatchHandler(context, jdbcTemplate); 80 | RowBatchListener listener = new RowBatchListener(rowBatchHandler); 81 | return listener; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/FileRowBatchListenerBuilder.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch; 2 | 3 | import java.io.File; 4 | 5 | import com.litesalt.batch.context.HandlerContext; 6 | import com.litesalt.batch.context.QueueContext; 7 | import com.litesalt.batch.enums.FileSavedCapacity; 8 | import com.litesalt.batch.enums.TargetType; 9 | import com.litesalt.batch.handler.FileRowBatchHandler; 10 | import com.litesalt.batch.listener.RowBatchListener; 11 | import com.litesalt.batch.queue.MemoryRowBatchQueue; 12 | import com.litesalt.batch.queue.RedisRowBatchQueue; 13 | 14 | /** 15 | * @author Paul-xiong 16 | * @date 2017年2月27日 17 | * @description 批插监听管理器构造器 18 | */ 19 | public class FileRowBatchListenerBuilder { 20 | 21 | /** 22 | * 构建内存批插监听管理器 23 | * 24 | * @param file 25 | * @param submitCapacity 26 | * @param clazz 27 | * @param capacity 28 | * @return 29 | */ 30 | public static RowBatchListener buildMemoryRowBatchListener(File file, long submitCapacity, Class clazz, FileSavedCapacity capacity) { 31 | return buildMemoryRowBatchListener(file, submitCapacity, clazz, capacity, false); 32 | } 33 | 34 | /** 35 | * 构建内存批插监听管理器 36 | * 37 | * @param file 38 | * @param submitCapacity 39 | * @param clazz 40 | * @param capacity 41 | * @param syn 42 | * @return 43 | */ 44 | public static RowBatchListener buildMemoryRowBatchListener(File file, long submitCapacity, Class clazz, FileSavedCapacity capacity, boolean syn) { 45 | QueueContext qContext = new QueueContext(TargetType.FILE); 46 | MemoryRowBatchQueue queue = new MemoryRowBatchQueue(qContext); 47 | HandlerContext hContext = new HandlerContext<>(queue, submitCapacity, clazz, syn); 48 | FileRowBatchHandler rowBatchHandler = new FileRowBatchHandler<>(hContext, file, capacity); 49 | RowBatchListener listener = new RowBatchListener<>(rowBatchHandler); 50 | return listener; 51 | } 52 | 53 | /** 54 | * 构建redis批插监听管理器 55 | * 56 | * @param file 57 | * @param submitCapacity 58 | * @param clazz 59 | * @param capacity 60 | * @param host 61 | * @param port 62 | * @param auth 63 | * @return 64 | */ 65 | public static RowBatchListener buildRedisRowBatchListener(File file, long submitCapacity, Class clazz, FileSavedCapacity capacity, String host, int port, String auth) { 66 | return buildRedisRowBatchListener(file, submitCapacity, clazz, capacity, host, port, auth, false); 67 | } 68 | 69 | /** 70 | * 构建redis批插监听管理器 71 | * 72 | * @param file 73 | * @param submitCapacity 74 | * @param clazz 75 | * @param host 76 | * @param port 77 | * @param auth 78 | * @param syn 79 | * @return 80 | */ 81 | public static RowBatchListener buildRedisRowBatchListener(File file, long submitCapacity, Class clazz, FileSavedCapacity capacity, String host, int port, String auth, boolean syn) { 82 | QueueContext qContext = new QueueContext(TargetType.FILE, clazz, file.getName()); 83 | RedisRowBatchQueue queue = new RedisRowBatchQueue(qContext, host, port, auth); 84 | HandlerContext hContext = new HandlerContext<>(queue, submitCapacity, clazz, syn); 85 | FileRowBatchHandler rowBatchHandler = new FileRowBatchHandler(hContext, file, capacity); 86 | RowBatchListener listener = new RowBatchListener(rowBatchHandler); 87 | return listener; 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/annotation/AliasField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C), 上海布鲁爱电子商务有限公司 3 | */ 4 | package com.litesalt.batch.annotation; 5 | 6 | import java.lang.annotation.Documented; 7 | import java.lang.annotation.ElementType; 8 | import java.lang.annotation.Retention; 9 | import java.lang.annotation.RetentionPolicy; 10 | import java.lang.annotation.Target; 11 | 12 | /** 13 | * @author Paul-xiong 14 | * @date 2017年3月16日 15 | * @description 16 | */ 17 | @Target({ ElementType.FIELD }) 18 | @Retention(RetentionPolicy.RUNTIME) 19 | @Documented 20 | public @interface AliasField { 21 | public String value(); 22 | } 23 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/annotation/AliasTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C), 上海布鲁爱电子商务有限公司 3 | */ 4 | package com.litesalt.batch.annotation; 5 | 6 | import java.lang.annotation.Documented; 7 | import java.lang.annotation.ElementType; 8 | import java.lang.annotation.Retention; 9 | import java.lang.annotation.RetentionPolicy; 10 | import java.lang.annotation.Target; 11 | 12 | /** 13 | * @author Paul-xiong 14 | * @date 2017年3月16日 15 | * @description 16 | */ 17 | @Target({ ElementType.TYPE }) 18 | @Retention(RetentionPolicy.RUNTIME) 19 | @Documented 20 | public @interface AliasTable { 21 | public String value(); 22 | } 23 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/annotation/ExcludeField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C), 上海布鲁爱电子商务有限公司 3 | */ 4 | package com.litesalt.batch.annotation; 5 | 6 | import java.lang.annotation.Documented; 7 | import java.lang.annotation.ElementType; 8 | import java.lang.annotation.Retention; 9 | import java.lang.annotation.RetentionPolicy; 10 | import java.lang.annotation.Target; 11 | 12 | /** 13 | * @author Paul-xiong 14 | * @date 2017年3月16日 15 | * @description 16 | */ 17 | @Target({ ElementType.FIELD }) 18 | @Retention(RetentionPolicy.RUNTIME) 19 | @Documented 20 | public @interface ExcludeField { 21 | } 22 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/context/HandlerContext.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch.context; 2 | 3 | import com.litesalt.batch.queue.MemoryRowBatchQueue; 4 | import com.litesalt.batch.queue.RowBatchQueue; 5 | 6 | /** 7 | * @author Paul-xiong 8 | * @date 2017年3月17日 9 | * @description 批插操作器上下文 10 | */ 11 | public class HandlerContext { 12 | 13 | /** 14 | * 默认提交数量 15 | */ 16 | private final static long DEFAULT_SUBMIT_CAPACITY = 5000; 17 | /** 18 | * 缓存队列 19 | */ 20 | private RowBatchQueue queue; 21 | /** 22 | * 触发异步插入的提交数量 23 | */ 24 | private long submitCapacity; 25 | /** 26 | * 操作的泛型类 27 | */ 28 | private Class clazz; 29 | /** 30 | * 是否同步批插 31 | */ 32 | private boolean syn; 33 | 34 | public HandlerContext() { 35 | this(new MemoryRowBatchQueue(), null); 36 | } 37 | 38 | public HandlerContext(RowBatchQueue queue, Class clazz) { 39 | this(queue, DEFAULT_SUBMIT_CAPACITY, clazz); 40 | } 41 | 42 | public HandlerContext(RowBatchQueue queue, long submitCapacity, Class clazz) { 43 | this(queue, submitCapacity, clazz, false); 44 | } 45 | 46 | public HandlerContext(RowBatchQueue queue, long submitCapacity, Class clazz, boolean syn) { 47 | super(); 48 | this.queue = queue; 49 | this.submitCapacity = submitCapacity; 50 | this.clazz = clazz; 51 | this.syn = syn; 52 | } 53 | 54 | public RowBatchQueue getQueue() { 55 | return queue; 56 | } 57 | 58 | public void setQueue(RowBatchQueue queue) { 59 | this.queue = queue; 60 | } 61 | 62 | public long getSubmitCapacity() { 63 | return submitCapacity; 64 | } 65 | 66 | public void setSubmitCapacity(long submitCapacity) { 67 | this.submitCapacity = submitCapacity; 68 | } 69 | 70 | public Class getClazz() { 71 | return clazz; 72 | } 73 | 74 | public void setClazz(Class clazz) { 75 | this.clazz = clazz; 76 | } 77 | 78 | public boolean isSyn() { 79 | return syn; 80 | } 81 | 82 | public void setSyn(boolean syn) { 83 | this.syn = syn; 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/context/QueueContext.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch.context; 2 | 3 | import com.litesalt.batch.enums.TargetType; 4 | 5 | /** 6 | * @author Paul-xiong 7 | * @date 2017年3月17日 8 | * @description 队列上下文 9 | */ 10 | public class QueueContext { 11 | /** 12 | * 13 | */ 14 | private TargetType type; 15 | /** 16 | * 17 | */ 18 | private Class clazz; 19 | /** 20 | * redis key扩展 21 | */ 22 | private String redisKeyExt; 23 | 24 | public QueueContext() { 25 | this(TargetType.DB); 26 | } 27 | 28 | public QueueContext(TargetType type) { 29 | this(type, null); 30 | } 31 | 32 | public QueueContext(TargetType type, Class clazz) { 33 | this(type, clazz, null); 34 | } 35 | 36 | public QueueContext(TargetType type, Class clazz, String redisKeyExt) { 37 | super(); 38 | this.type = type; 39 | this.clazz = clazz; 40 | this.redisKeyExt = redisKeyExt; 41 | } 42 | 43 | public TargetType getType() { 44 | return type; 45 | } 46 | 47 | public void setType(TargetType type) { 48 | this.type = type; 49 | } 50 | 51 | public Class getClazz() { 52 | return clazz; 53 | } 54 | 55 | public void setClazz(Class clazz) { 56 | this.clazz = clazz; 57 | } 58 | 59 | public String getRedisKeyExt() { 60 | return redisKeyExt; 61 | } 62 | 63 | public void setRedisKeyExt(String redisKeyExt) { 64 | this.redisKeyExt = redisKeyExt; 65 | } 66 | 67 | } 68 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/entity/DBColumnMetaData.java: -------------------------------------------------------------------------------- 1 | /** 2 | *

Title: liteBatch

3 | *

Description: 一个轻量级，高性能的快速批插工具

4 | *

Copyright: Copyright (c) 2016

5 | * @author Bryan.Zhang 6 | * @email 47483522@qq.com 7 | * @Date 2016-11-10 8 | * @version 1.0 9 | */ 10 | package com.litesalt.batch.entity; 11 | 12 | /** 13 | * 数据列元信息 14 | */ 15 | public class DBColumnMetaData { 16 | 17 | private String columnName; 18 | 19 | private int dataType; 20 | 21 | private Object columnDef; 22 | 23 | public DBColumnMetaData(String columnName, int dataType, Object columnDef) { 24 | this.columnName = columnName; 25 | this.dataType = dataType; 26 | this.columnDef = columnDef; 27 | } 28 | 29 | public String getColumnName() { 30 | return columnName; 31 | } 32 | 33 | public void setColumnName(String columnName) { 34 | this.columnName = columnName; 35 | } 36 | 37 | public int getDataType() { 38 | return dataType; 39 | } 40 | 41 | public void setDataType(int dataType) { 42 | this.dataType = dataType; 43 | } 44 | 45 | public Object getColumnDef() { 46 | return columnDef; 47 | } 48 | 49 | public void setColumnDef(Object columnDef) { 50 | this.columnDef = columnDef; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/enums/FileSavedCapacity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C), 上海布鲁爱电子商务有限公司 3 | */ 4 | package com.litesalt.batch.enums; 5 | 6 | /** 7 | * @author Paul-xiong 8 | * @date 2017年3月16日 9 | * @description 文件保存策略 10 | */ 11 | public enum FileSavedCapacity { 12 | SINGLE, // 单个文件 13 | DAILY,// 每天一个文件 14 | ; 15 | } 16 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/enums/TargetType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C), 上海布鲁爱电子商务有限公司 3 | */ 4 | package com.litesalt.batch.enums; 5 | 6 | /** 7 | * @author Paul-xiong 8 | * @date 2017年3月16日 9 | * @description 队列数据导向目标类型 10 | */ 11 | public enum TargetType { 12 | DB, // 数据导向DB 13 | FILE,// 数据导向文件 14 | ; 15 | } 16 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/handler/DBRowBatchHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | *

Title: liteBatch

3 | *

Description: 一个轻量级，高性能的快速批插工具

4 | *

Copyright: Copyright (c) 2016

5 | * @author Bryan.Zhang 6 | * @email 47483522@qq.com 7 | * @Date 2016-11-10 8 | * @version 1.0 9 | */ 10 | package com.litesalt.batch.handler; 11 | 12 | import java.lang.reflect.Field; 13 | import java.math.BigDecimal; 14 | import java.sql.DatabaseMetaData; 15 | import java.sql.PreparedStatement; 16 | import java.sql.ResultSet; 17 | import java.sql.SQLException; 18 | import java.sql.Timestamp; 19 | import java.sql.Types; 20 | import java.util.Date; 21 | import java.util.HashMap; 22 | import java.util.List; 23 | import java.util.Map; 24 | 25 | import org.apache.commons.lang3.StringUtils; 26 | import org.springframework.jdbc.core.BatchPreparedStatementSetter; 27 | import org.springframework.jdbc.core.JdbcTemplate; 28 | 29 | import com.litesalt.batch.annotation.AliasField; 30 | import com.litesalt.batch.annotation.AliasTable; 31 | import com.litesalt.batch.annotation.ExcludeField; 32 | import com.litesalt.batch.context.HandlerContext; 33 | import com.litesalt.batch.entity.DBColumnMetaData; 34 | import com.litesalt.batch.util.CamelCaseUtils; 35 | import com.litesalt.batch.util.Reflections; 36 | 37 | /** 38 | * 批插处理器 39 | */ 40 | public class DBRowBatchHandler extends RowBatchHandler { 41 | 42 | private JdbcTemplate jdbcTemplate; 43 | 44 | private String insertSql; 45 | 46 | private Field[] fields; 47 | 48 | private Map metaMap = new HashMap(); 49 | 50 | // ================private================== 51 | private void prepareSql() { 52 | StringBuffer sql = new StringBuffer(); 53 | Class clazz = context.getClazz(); 54 | sql.append(" insert into ").append(getAliasTable(clazz)); 55 | sql.append("("); 56 | int m = 0; 57 | for (Field field : clazz.getDeclaredFields()) { 58 | ExcludeField excludeField = field.getAnnotation(ExcludeField.class); 59 | if (excludeField == null) { 60 | m++; 61 | sql.append(getAliasField(field)); 62 | sql.append(","); 63 | } 64 | } 65 | sql = sql.replace(sql.length() - 1, sql.length(), ""); 66 | sql.append(") values("); 67 | for (int n = 0; n < m; n++) { 68 | sql.append("?"); 69 | if (n < m - 1) { 70 | sql.append(","); 71 | } 72 | 73 | } 74 | sql.append(")"); 75 | insertSql = sql.toString(); 76 | } 77 | 78 | private boolean initDBMetaData() { 79 | boolean flag = false; 80 | try { 81 | Class clazz = context.getClazz(); 82 | DatabaseMetaData metaData = jdbcTemplate.getDataSource().getConnection().getMetaData(); 83 | ResultSet rs = metaData.getColumns(null, "%", getAliasTable(clazz), "%"); 84 | while (rs.next()) { 85 | metaMap.put(rs.getString("COLUMN_NAME"), new DBColumnMetaData(rs.getString("COLUMN_NAME"), rs.getInt("DATA_TYPE"), rs.getObject("COLUMN_DEF"))); 86 | } 87 | flag = true; 88 | } catch (Exception e) { 89 | logger.error("init db metadata wrong", e); 90 | flag = false; 91 | } 92 | return flag; 93 | } 94 | 95 | private String getAliasTable(Class clazz) { 96 | AliasTable aliasTable = clazz.getAnnotation(AliasTable.class); 97 | if (aliasTable != null && StringUtils.isNotBlank(aliasTable.value())) { 98 | return aliasTable.value(); 99 | } else { 100 | return CamelCaseUtils.toUnderScoreCase(clazz.getSimpleName()); 101 | } 102 | } 103 | 104 | private String getAliasField(Field field) { 105 | AliasField aliasField = field.getAnnotation(AliasField.class); 106 | if (aliasField != null && StringUtils.isNotBlank(aliasField.value())) { 107 | return aliasField.value(); 108 | } else { 109 | return CamelCaseUtils.toUnderScoreCase(field.getName()); 110 | } 111 | } 112 | 113 | // ======================================== 114 | 115 | public DBRowBatchHandler(HandlerContext context, JdbcTemplate jdbcTemplate) { 116 | super(context); 117 | this.jdbcTemplate = jdbcTemplate; 118 | 119 | if (initDBMetaData()) { 120 | Class clazz = context.getClazz(); 121 | fields = clazz.getDeclaredFields(); 122 | 123 | prepareSql(); 124 | } 125 | } 126 | 127 | @Override 128 | public void rowBatch(final List batchList) { 129 | long startTimeMillis = System.currentTimeMillis(); 130 | logger.info("开始批次插入数据库"); 131 | if (batchList != null && batchList.size() > 0) { 132 | jdbcTemplate.batchUpdate(insertSql, new BatchPreparedStatementSetter() { 133 | @Override 134 | public void setValues(PreparedStatement ps, int i) throws SQLException { 135 | T t = batchList.get(i); 136 | Object o = null; 137 | DBColumnMetaData metaData = null; 138 | int n = 0; 139 | for (Field field : fields) { 140 | ExcludeField excludeField = field.getAnnotation(ExcludeField.class); 141 | if (excludeField == null) { 142 | n++; 143 | o = Reflections.invokeGetter(t, field.getName()); 144 | metaData = metaMap.get(getAliasField(field)); 145 | 146 | // 如果值为null，还要看默认值，如果有默认值，取元数据中的默认值 147 | if (o == null) { 148 | if (metaData.getColumnDef() == null) { 149 | ps.setNull(n, Types.NULL); 150 | continue; 151 | } 152 | o = metaData.getColumnDef(); 153 | } 154 | 155 | switch (metaData.getDataType()) { 156 | case Types.CHAR: 157 | case Types.VARCHAR: 158 | case Types.NVARCHAR: 159 | ps.setString(n, (String) o); 160 | break; 161 | case Types.BLOB: 162 | case Types.LONGVARBINARY: 163 | ps.setBytes(n, (byte[]) o); 164 | break; 165 | case Types.TINYINT: 166 | case Types.SMALLINT: 167 | ps.setShort(n, Short.parseShort(o.toString())); 168 | break; 169 | case Types.INTEGER: 170 | ps.setInt(n, Integer.parseInt(o.toString())); 171 | break; 172 | case Types.BIGINT: 173 | ps.setLong(n, Long.parseLong(o.toString())); 174 | break; 175 | case Types.DATE: 176 | ps.setDate(n, new java.sql.Date(((Date) o).getTime())); 177 | break; 178 | case Types.TIMESTAMP: 179 | ps.setTimestamp(n, new Timestamp(((Date) o).getTime())); 180 | break; 181 | case Types.DECIMAL: 182 | ps.setBigDecimal(n, (BigDecimal) o); 183 | break; 184 | } 185 | } 186 | } 187 | } 188 | 189 | @Override 190 | public int getBatchSize() { 191 | return batchList != null ? batchList.size() : 0; 192 | } 193 | }); 194 | logger.info("this batch spend " + (System.currentTimeMillis() - startTimeMillis) + " millisecond"); 195 | } 196 | } 197 | 198 | } 199 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/handler/FileRowBatchHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C), 上海布鲁爱电子商务有限公司 3 | */ 4 | package com.litesalt.batch.handler; 5 | 6 | import java.io.File; 7 | import java.io.FileOutputStream; 8 | import java.io.OutputStream; 9 | import java.io.PrintWriter; 10 | import java.text.SimpleDateFormat; 11 | import java.util.Calendar; 12 | import java.util.List; 13 | 14 | import com.litesalt.batch.context.HandlerContext; 15 | import com.litesalt.batch.enums.FileSavedCapacity; 16 | 17 | /** 18 | * @author Paul-xiong 19 | * @date 2017年3月16日 20 | * @description 21 | */ 22 | public class FileRowBatchHandler extends RowBatchHandler { 23 | 24 | private File file; 25 | 26 | private String originFilePath; 27 | 28 | private Calendar fileDate = Calendar.getInstance(); 29 | 30 | private SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd"); 31 | 32 | private FileSavedCapacity capacity = FileSavedCapacity.SINGLE; 33 | 34 | private File getFile() { 35 | if (capacity.equals(FileSavedCapacity.DAILY)) { 36 | Calendar now = Calendar.getInstance(); 37 | if (now.get(Calendar.YEAR) > fileDate.get(Calendar.YEAR) || now.get(Calendar.MONTH) > fileDate.get(Calendar.MONTH) || now.get(Calendar.DAY_OF_MONTH) > fileDate.get(Calendar.DAY_OF_MONTH)) { 38 | fileDate = now; 39 | file = new File(originFilePath + "-" + simpleDateFormat.format(fileDate.getTime())); 40 | } 41 | } 42 | return file; 43 | } 44 | // ------------------------------------- 45 | 46 | public FileRowBatchHandler(HandlerContext context, File file, FileSavedCapacity capacity) { 47 | super(context); 48 | this.originFilePath = file.getAbsolutePath(); 49 | if (capacity.equals(FileSavedCapacity.DAILY)) { 50 | this.file = new File(originFilePath + "-" + simpleDateFormat.format(fileDate.getTime())); 51 | } else { 52 | this.file = file; 53 | } 54 | this.capacity = capacity; 55 | } 56 | 57 | @Override 58 | public void rowBatch(List batchList) { 59 | logger.info("开始批次插入文件"); 60 | if (batchList != null && batchList.size() > 0) { 61 | OutputStream os = null; 62 | PrintWriter pw = null; 63 | try { 64 | os = new FileOutputStream(getFile(), true); 65 | pw = new PrintWriter(os, true); 66 | for (T t : batchList) { 67 | if (t != null) { 68 | pw.println(t); 69 | } 70 | } 71 | pw.flush(); 72 | } catch (Exception e) { 73 | logger.error("批次插入文件异常: ", e); 74 | } finally { 75 | try { 76 | pw.close(); 77 | } catch (Exception e2) { 78 | logger.error("批次插入文件异常: {}", e2); 79 | } 80 | } 81 | } 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/handler/RowBatchHandler.java: -------------------------------------------------------------------------------- 1 | /** 2 | *

Title: liteBatch

3 | *

Description: 一个轻量级，高性能的快速批插工具

4 | *

Copyright: Copyright (c) 2016

5 | * @author Bryan.Zhang 6 | * @email 47483522@qq.com 7 | * @Date 2016-11-10 8 | * @version 1.0 9 | */ 10 | package com.litesalt.batch.handler; 11 | 12 | import java.util.List; 13 | import java.util.Observable; 14 | import java.util.concurrent.ExecutorService; 15 | import java.util.concurrent.Executors; 16 | import java.util.concurrent.atomic.AtomicLong; 17 | 18 | import org.apache.log4j.Logger; 19 | 20 | import com.litesalt.batch.context.HandlerContext; 21 | import com.litesalt.batch.monitor.QueueStatusMonitor; 22 | import com.litesalt.batch.queue.RowBatchQueue; 23 | 24 | /** 25 | * 批插处理器 26 | */ 27 | public abstract class RowBatchHandler extends Observable { 28 | 29 | protected final Logger logger = Logger.getLogger(RowBatchHandler.class); 30 | 31 | protected AtomicLong loopSize = new AtomicLong(0); 32 | 33 | protected HandlerContext context; 34 | 35 | private ExecutorService threadPool = Executors.newFixedThreadPool(10); 36 | 37 | // ======================================== 38 | 39 | public RowBatchHandler(HandlerContext context) { 40 | super(); 41 | // ======添加观察者===== 42 | this.addObserver(new QueueStatusMonitor(this)); 43 | // =================== 44 | this.context = context; 45 | } 46 | 47 | public abstract void rowBatch(final List batchList); 48 | 49 | public void insertWithBatch(List items) { 50 | try { 51 | RowBatchQueue queue = context.getQueue(); 52 | if (queue != null && items != null && items.size() > 0) { 53 | queue.put(items); 54 | if (!context.isSyn()) { 55 | loopSize.addAndGet(items.size()); 56 | final long submitCapacity = context.getSubmitCapacity(); 57 | if (loopSize.get() >= submitCapacity) { 58 | threadPool.submit(new Thread() { 59 | @Override 60 | public void run() { 61 | try { 62 | rowBatch(take(submitCapacity)); 63 | } catch (Exception e) { 64 | logger.error("批次插入发生异常", e); 65 | } 66 | } 67 | }); 68 | loopSize.set(0); 69 | // 向观察者发送通知 70 | this.setChanged(); 71 | this.notifyObservers(); 72 | } 73 | } 74 | } 75 | } catch (Exception e) { 76 | e.printStackTrace(); 77 | } 78 | } 79 | 80 | public void flush() { 81 | rowBatch(takeAll()); 82 | } 83 | 84 | public List take(long len) { 85 | try { 86 | RowBatchQueue queue = context.getQueue(); 87 | if (queue != null) { 88 | return queue.take(len); 89 | } else { 90 | return null; 91 | } 92 | } catch (Exception e) { 93 | logger.error("take is interrupted", e); 94 | return null; 95 | } 96 | } 97 | 98 | public List takeAll() { 99 | try { 100 | RowBatchQueue queue = context.getQueue(); 101 | if (queue != null) { 102 | return queue.takeAll(); 103 | } else { 104 | return null; 105 | } 106 | } catch (Exception e) { 107 | logger.error("take is interrupted", e); 108 | return null; 109 | } 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/listener/RowBatchListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | *

Title: liteBatch

3 | *

Description: 一个轻量级，高性能的快速批插工具

4 | *

Copyright: Copyright (c) 2016

5 | * @author Bryan.Zhang 6 | * @email 47483522@qq.com 7 | * @Date 2016-11-10 8 | * @version 1.0 9 | */ 10 | package com.litesalt.batch.listener; 11 | 12 | import java.util.Arrays; 13 | import java.util.List; 14 | 15 | import org.apache.log4j.Logger; 16 | 17 | import com.litesalt.batch.handler.RowBatchHandler; 18 | 19 | /** 20 | * 批插监听管理器 21 | */ 22 | public class RowBatchListener { 23 | 24 | private final static Logger log = Logger.getLogger(RowBatchListener.class); 25 | 26 | private RowBatchHandler rowBatchHandler; 27 | 28 | public RowBatchListener(RowBatchHandler rowBatchHandler) { 29 | log.info("开始监听批次插入"); 30 | this.rowBatchHandler = rowBatchHandler; 31 | } 32 | 33 | public void insertOneWithBatch(T t) { 34 | if (t == null) { 35 | log.warn("po must not be null!"); 36 | return; 37 | } 38 | this.rowBatchHandler.insertWithBatch(Arrays.asList(t)); 39 | } 40 | 41 | public void insertBatch(List coll) { 42 | this.rowBatchHandler.insertWithBatch(coll); 43 | } 44 | 45 | public void flush() { 46 | this.rowBatchHandler.flush(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/monitor/QueueStatusMonitor.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch.monitor; 2 | 3 | import java.util.Date; 4 | import java.util.Observable; 5 | import java.util.Observer; 6 | import java.util.Timer; 7 | import java.util.TimerTask; 8 | 9 | import org.apache.log4j.Logger; 10 | 11 | import com.litesalt.batch.handler.RowBatchHandler; 12 | 13 | /** 14 | * @author Paul-xiong 15 | * @date 2017年3月2日 16 | * @description 队列状态监控器 17 | */ 18 | public class QueueStatusMonitor implements Observer { 19 | 20 | private final Logger log = Logger.getLogger(QueueStatusMonitor.class); 21 | 22 | private final long time = 600 * 1000; // 定时时间 23 | 24 | private Date lastBatchTime = new Date(); 25 | 26 | private Timer timer = new Timer();; 27 | 28 | public QueueStatusMonitor() { 29 | this(null); 30 | } 31 | 32 | public QueueStatusMonitor(final RowBatchHandler handler) { 33 | super(); 34 | timer.schedule(new TimerTask() { 35 | @Override 36 | public void run() { 37 | try { 38 | if (handler != null) { 39 | Date now = new Date(); 40 | if (now.getTime() - lastBatchTime.getTime() > time) { 41 | log.info("队列健康状态监视器开始工作"); 42 | handler.flush(); 43 | } 44 | } 45 | } catch (Exception e) { 46 | log.error("队列健康状态监视器工作异常", e); 47 | } 48 | } 49 | }, 0, time); 50 | } 51 | 52 | @Override 53 | public void update(Observable o, Object arg) { 54 | lastBatchTime = new Date(); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/queue/MemoryRowBatchQueue.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch.queue; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Queue; 6 | import java.util.concurrent.ConcurrentLinkedQueue; 7 | 8 | import com.litesalt.batch.context.QueueContext; 9 | 10 | /** 11 | * @author Paul-xiong 12 | * @date 2017年2月26日 13 | * @description 内存插入队列 14 | */ 15 | public class MemoryRowBatchQueue extends RowBatchQueue { 16 | 17 | private Queue items; 18 | 19 | public MemoryRowBatchQueue() { 20 | this(new QueueContext()); 21 | } 22 | 23 | public MemoryRowBatchQueue(QueueContext context) { 24 | super(context); 25 | items = new ConcurrentLinkedQueue(); 26 | } 27 | 28 | @Override 29 | public void put(T t) { 30 | items.add(t); 31 | } 32 | 33 | @Override 34 | public void put(List ts) { 35 | if (ts != null && ts.size() > 0) { 36 | for (T t : ts) { 37 | put(t); 38 | } 39 | } 40 | } 41 | 42 | @Override 43 | public T take() { 44 | return items.poll(); 45 | } 46 | 47 | @Override 48 | public List take(long len) { 49 | List rt = new ArrayList(); 50 | while (len > 0) { 51 | T item = take(); 52 | if (item != null) { 53 | rt.add(item); 54 | } 55 | len--; 56 | } 57 | return rt; 58 | } 59 | 60 | @Override 61 | public List takeAll() { 62 | return take(items.size()); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/queue/RedisRowBatchQueue.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch.queue; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | 7 | import org.apache.commons.lang3.StringUtils; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import com.alibaba.fastjson.JSONObject; 12 | import com.litesalt.batch.context.QueueContext; 13 | 14 | import redis.clients.jedis.Jedis; 15 | import redis.clients.jedis.JedisPool; 16 | import redis.clients.jedis.JedisPoolConfig; 17 | import redis.clients.jedis.Pipeline; 18 | import redis.clients.jedis.Response; 19 | 20 | /** 21 | * @author Paul-xiong 22 | * @date 2017年2月26日 23 | * @description redis插入队列 24 | */ 25 | public class RedisRowBatchQueue extends RowBatchQueue { 26 | 27 | private final static String DEFAULT_HOST = "localhost"; 28 | 29 | private final static int DEFAULT_PORT = 6379; 30 | 31 | private final Logger logger = LoggerFactory.getLogger(RedisRowBatchQueue.class); 32 | 33 | private JedisPool jedisPool; 34 | 35 | private String key; 36 | 37 | /** 38 | * 生成redis队列key 39 | * 40 | * @return 41 | */ 42 | private void buildKey() { 43 | Class clazz = context.getClazz(); 44 | StringBuilder stringBuilder = new StringBuilder("ROW_BATCH_QUEUE_").append(clazz.getSimpleName().toUpperCase()); 45 | if (context != null) { 46 | if (context.getType() != null) { 47 | stringBuilder.append("_").append(context.getType().toString().toUpperCase()); 48 | } 49 | if (StringUtils.isNotBlank(context.getRedisKeyExt())) { 50 | stringBuilder.append("_").append(context.getRedisKeyExt().toUpperCase()); 51 | } 52 | } 53 | key = stringBuilder.toString(); 54 | } 55 | 56 | // ================================== 57 | public RedisRowBatchQueue() { 58 | this(DEFAULT_HOST, DEFAULT_PORT, null); 59 | } 60 | 61 | public RedisRowBatchQueue(String host, int port, String auth) { 62 | this(new QueueContext(), host, port, auth); 63 | } 64 | 65 | public RedisRowBatchQueue(QueueContext context, String host, int port, String auth) { 66 | super(context); 67 | JedisPoolConfig jedisPoolConfig = new JedisPoolConfig(); 68 | jedisPoolConfig.setMaxTotal(100); 69 | this.jedisPool = new JedisPool(jedisPoolConfig, host, port, 3000, auth); 70 | this.buildKey(); 71 | } 72 | 73 | @Override 74 | public void put(T t) { 75 | put(Arrays.asList(t)); 76 | } 77 | 78 | @Override 79 | public void put(List ts) { 80 | if (ts != null && ts.size() > 0) { 81 | Jedis jedis = this.jedisPool.getResource(); 82 | try { 83 | Pipeline pipe = jedis.pipelined(); 84 | for (T t : ts) { 85 | String value = JSONObject.toJSONString(t); 86 | pipe.rpush(key, value); 87 | } 88 | pipe.sync(); 89 | } catch (Exception e) { 90 | this.logger.error("Redis exception: {}", e.getMessage(), e); 91 | } finally { 92 | if (jedis != null) { 93 | jedis.close(); 94 | } 95 | } 96 | } 97 | } 98 | 99 | @Override 100 | public T take() { 101 | List take = take(1); 102 | if (take != null && take.size() > 0) { 103 | return take.get(0); 104 | } 105 | return null; 106 | } 107 | 108 | @Override 109 | public List take(long len) { 110 | List rt = new ArrayList(); 111 | Jedis jedis = null; 112 | try { 113 | if (len > 0) { 114 | jedis = this.jedisPool.getResource(); 115 | Pipeline pipe = jedis.pipelined(); 116 | List> responseList = new ArrayList>(); 117 | while (len > 0) { 118 | responseList.add(pipe.lpop(key)); 119 | len--; 120 | } 121 | pipe.sync(); 122 | String item = null; 123 | for (Response response : responseList) { 124 | item = response.get(); 125 | if (StringUtils.isNotBlank(item)) { 126 | rt.add(JSONObject.parseObject(item, context.getClazz())); 127 | } 128 | } 129 | } 130 | } catch (Exception e) { 131 | this.logger.error("Redis exception: {}", e.getMessage(), e); 132 | } finally { 133 | if (jedis != null) { 134 | jedis.close(); 135 | } 136 | } 137 | return rt; 138 | } 139 | 140 | @Override 141 | public List takeAll() { 142 | Jedis jedis = null; 143 | try { 144 | jedis = this.jedisPool.getResource(); 145 | Long len = jedis.llen(key); 146 | return take(len != null ? len : 0); 147 | } catch (Exception e) { 148 | this.logger.error("Redis exception: {}", e.getMessage(), e); 149 | return new ArrayList(); 150 | } finally { 151 | if (jedis != null) { 152 | jedis.close(); 153 | } 154 | } 155 | } 156 | 157 | } 158 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/queue/RowBatchQueue.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch.queue; 2 | 3 | import java.util.List; 4 | 5 | import com.litesalt.batch.context.QueueContext; 6 | 7 | /** 8 | * @author Paul-xiong 9 | * @date 2017年2月26日 10 | * @description 批量插入队列 11 | */ 12 | public abstract class RowBatchQueue { 13 | 14 | protected QueueContext context; 15 | 16 | public RowBatchQueue() { 17 | this(new QueueContext()); 18 | } 19 | 20 | public RowBatchQueue(QueueContext context) { 21 | super(); 22 | this.context = context; 23 | } 24 | 25 | public abstract void put(T t); 26 | 27 | public abstract void put(List ts); 28 | 29 | public abstract T take(); 30 | 31 | public abstract List take(long len); 32 | 33 | public abstract List takeAll(); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/util/CamelCaseUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | *

Title: liteBatch

3 | *

Description: 一个轻量级，高性能的快速批插工具

4 | *

Copyright: Copyright (c) 2016

5 | * @author Bryan.Zhang 6 | * @email 47483522@qq.com 7 | * @Date 2016-11-10 8 | * @version 1.0 9 | */ 10 | package com.litesalt.batch.util; 11 | 12 | /** 13 | * 驼峰-数据库命名转换器 14 | */ 15 | public class CamelCaseUtils { 16 | 17 | private static final char SEPARATOR = '_'; 18 | 19 | public static String toUnderScoreCase(String s) { 20 | if (s == null) { 21 | return null; 22 | } 23 | 24 | StringBuilder sb = new StringBuilder(); 25 | boolean upperCase = false; 26 | for (int i = 0; i < s.length(); i++) { 27 | char c = s.charAt(i); 28 | 29 | boolean nextUpperCase = true; 30 | 31 | if (i < (s.length() - 1)) { 32 | nextUpperCase = Character.isUpperCase(s.charAt(i + 1)); 33 | } 34 | 35 | if ((i > 0) && Character.isUpperCase(c)) { 36 | if (!upperCase || !nextUpperCase) { 37 | sb.append(SEPARATOR); 38 | } 39 | upperCase = true; 40 | } else { 41 | upperCase = false; 42 | } 43 | 44 | sb.append(Character.toLowerCase(c)); 45 | } 46 | 47 | return sb.toString(); 48 | } 49 | 50 | public static String toCamelCase(String s) { 51 | if (s == null) { 52 | return null; 53 | } 54 | s = s.toLowerCase(); 55 | StringBuilder sb = new StringBuilder(s.length()); 56 | boolean upperCase = false; 57 | for (int i = 0; i < s.length(); i++) { 58 | char c = s.charAt(i); 59 | if (c == SEPARATOR) { 60 | upperCase = true; 61 | } else if (upperCase) { 62 | sb.append(Character.toUpperCase(c)); 63 | upperCase = false; 64 | } else { 65 | sb.append(c); 66 | } 67 | } 68 | return sb.toString(); 69 | } 70 | 71 | public static String toCapitalizeCamelCase(String s) { 72 | if (s == null) { 73 | return null; 74 | } 75 | s = toCamelCase(s); 76 | return s.substring(0, 1).toUpperCase() + s.substring(1); 77 | } 78 | 79 | public static void main(String[] args) { 80 | System.out.println(CamelCaseUtils.toUnderScoreCase("userName")); 81 | } 82 | } -------------------------------------------------------------------------------- /liteBatch-core/src/main/java/com/litesalt/batch/util/Reflections.java: -------------------------------------------------------------------------------- 1 | /** 2 | *

Title: liteBatch

3 | *

Description: 一个轻量级，高性能的快速批插工具

4 | *

Copyright: Copyright (c) 2016

5 | * @author Bryan.Zhang 6 | * @email 47483522@qq.com 7 | * @Date 2016-11-10 8 | * @version 1.0 9 | */ 10 | package com.litesalt.batch.util; 11 | 12 | import java.lang.reflect.Field; 13 | import java.lang.reflect.InvocationTargetException; 14 | import java.lang.reflect.Method; 15 | import java.lang.reflect.Modifier; 16 | import java.lang.reflect.ParameterizedType; 17 | import java.lang.reflect.Type; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | import org.apache.commons.lang3.Validate; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | /** 25 | * 反射工具类. 26 | * 提供调用getter/setter方法, 访问私有变量, 调用私有方法, 获取泛型类型Class, 被AOP过的真实类等工具函数. 27 | */ 28 | @SuppressWarnings("rawtypes") 29 | public class Reflections { 30 | 31 | private static final String SETTER_PREFIX = "set"; 32 | 33 | private static final String GETTER_PREFIX = "get"; 34 | 35 | private static final String CGLIB_CLASS_SEPARATOR = "$$"; 36 | 37 | private static Logger logger = LoggerFactory.getLogger(Reflections.class); 38 | 39 | /** 40 | * 调用Getter方法. 41 | * 支持多级，如：对象名.对象名.方法 42 | */ 43 | public static Object invokeGetter(Object obj, String propertyName) { 44 | Object object = obj; 45 | for (String name : StringUtils.split(propertyName, ".")){ 46 | String getterMethodName = GETTER_PREFIX + StringUtils.capitalize(name); 47 | object = invokeMethod(object, getterMethodName, new Class[] {}, new Object[] {}); 48 | } 49 | return object; 50 | } 51 | 52 | /** 53 | * 调用Setter方法, 仅匹配方法名。 54 | * 支持多级，如：对象名.对象名.方法 55 | */ 56 | public static void invokeSetter(Object obj, String propertyName, Object value) { 57 | Object object = obj; 58 | String[] names = StringUtils.split(propertyName, "."); 59 | for (int i=0; i[] parameterTypes, 112 | final Object[] args) { 113 | Method method = getAccessibleMethod(obj, methodName, parameterTypes); 114 | if (method == null) { 115 | throw new IllegalArgumentException("Could not find method [" + methodName + "] on target [" + obj + "]"); 116 | } 117 | 118 | try { 119 | return method.invoke(obj, args); 120 | } catch (Exception e) { 121 | throw convertReflectionExceptionToUnchecked(e); 122 | } 123 | } 124 | 125 | /** 126 | * 直接调用对象方法, 无视private/protected修饰符， 127 | * 用于一次性调用的情况，否则应使用getAccessibleMethodByName()函数获得Method后反复调用. 128 | * 只匹配函数名，如果有多个同名函数调用第一个。 129 | */ 130 | public static Object invokeMethodByName(final Object obj, final String methodName, final Object[] args) { 131 | Method method = getAccessibleMethodByName(obj, methodName); 132 | if (method == null) { 133 | throw new IllegalArgumentException("Could not find method [" + methodName + "] on target [" + obj + "]"); 134 | } 135 | 136 | try { 137 | return method.invoke(obj, args); 138 | } catch (Exception e) { 139 | throw convertReflectionExceptionToUnchecked(e); 140 | } 141 | } 142 | 143 | /** 144 | * 循环向上转型, 获取对象的DeclaredField, 并强制设置为可访问. 145 | * 146 | * 如向上转型到Object仍无法找到, 返回null. 147 | */ 148 | public static Field getAccessibleField(final Object obj, final String fieldName) { 149 | Validate.notNull(obj, "object can't be null"); 150 | Validate.notBlank(fieldName, "fieldName can't be blank"); 151 | for (Class superClass = obj.getClass(); superClass != Object.class; superClass = superClass.getSuperclass()) { 152 | try { 153 | Field field = superClass.getDeclaredField(fieldName); 154 | makeAccessible(field); 155 | return field; 156 | } catch (NoSuchFieldException e) {//NOSONAR 157 | // Field不在当前类定义,继续向上转型 158 | continue;// new add 159 | } 160 | } 161 | return null; 162 | } 163 | 164 | /** 165 | * 循环向上转型, 获取对象的DeclaredMethod,并强制设置为可访问. 166 | * 如向上转型到Object仍无法找到, 返回null. 167 | * 匹配函数名+参数类型。 168 | * 169 | * 用于方法需要被多次调用的情况. 先使用本函数先取得Method,然后调用Method.invoke(Object obj, Object... args) 170 | */ 171 | public static Method getAccessibleMethod(final Object obj, final String methodName, 172 | final Class... parameterTypes) { 173 | Validate.notNull(obj, "object can't be null"); 174 | Validate.notBlank(methodName, "methodName can't be blank"); 175 | 176 | for (Class searchType = obj.getClass(); searchType != Object.class; searchType = searchType.getSuperclass()) { 177 | try { 178 | Method method = searchType.getDeclaredMethod(methodName, parameterTypes); 179 | makeAccessible(method); 180 | return method; 181 | } catch (NoSuchMethodException e) { 182 | // Method不在当前类定义,继续向上转型 183 | continue;// new add 184 | } 185 | } 186 | return null; 187 | } 188 | 189 | /** 190 | * 循环向上转型, 获取对象的DeclaredMethod,并强制设置为可访问. 191 | * 如向上转型到Object仍无法找到, 返回null. 192 | * 只匹配函数名。 193 | * 194 | * 用于方法需要被多次调用的情况. 先使用本函数先取得Method,然后调用Method.invoke(Object obj, Object... args) 195 | */ 196 | public static Method getAccessibleMethodByName(final Object obj, final String methodName) { 197 | Validate.notNull(obj, "object can't be null"); 198 | Validate.notBlank(methodName, "methodName can't be blank"); 199 | 200 | for (Class searchType = obj.getClass(); searchType != Object.class; searchType = searchType.getSuperclass()) { 201 | Method[] methods = searchType.getDeclaredMethods(); 202 | for (Method method : methods) { 203 | if (method.getName().equals(methodName)) { 204 | makeAccessible(method); 205 | return method; 206 | } 207 | } 208 | } 209 | return null; 210 | } 211 | 212 | /** 213 | * 改变private/protected的方法为public，尽量不调用实际改动的语句，避免JDK的SecurityManager抱怨。 214 | */ 215 | public static void makeAccessible(Method method) { 216 | if ((!Modifier.isPublic(method.getModifiers()) || !Modifier.isPublic(method.getDeclaringClass().getModifiers())) 217 | && !method.isAccessible()) { 218 | method.setAccessible(true); 219 | } 220 | } 221 | 222 | /** 223 | * 改变private/protected的成员变量为public，尽量不调用实际改动的语句，避免JDK的SecurityManager抱怨。 224 | */ 225 | public static void makeAccessible(Field field) { 226 | if ((!Modifier.isPublic(field.getModifiers()) || !Modifier.isPublic(field.getDeclaringClass().getModifiers()) || Modifier 227 | .isFinal(field.getModifiers())) && !field.isAccessible()) { 228 | field.setAccessible(true); 229 | } 230 | } 231 | 232 | /** 233 | * 通过反射, 获得Class定义中声明的泛型参数的类型, 注意泛型必须定义在父类处 234 | * 如无法找到, 返回Object.class. 235 | * eg. 236 | * public UserDao extends HibernateDao 237 | * 238 | * @param clazz The class to introspect 239 | * @return the first generic declaration, or Object.class if cannot be determined 240 | */ 241 | @SuppressWarnings("unchecked") 242 | public static Class getClassGenricType(final Class clazz) { 243 | return getClassGenricType(clazz, 0); 244 | } 245 | 246 | /** 247 | * 通过反射, 获得Class定义中声明的父类的泛型参数的类型. 248 | * 如无法找到, 返回Object.class. 249 | * 250 | * 如public UserDao extends HibernateDao 251 | * 252 | * @param clazz clazz The class to introspect 253 | * @param index the Index of the generic ddeclaration,start from 0. 254 | * @return the index generic declaration, or Object.class if cannot be determined 255 | */ 256 | public static Class getClassGenricType(final Class clazz, final int index) { 257 | 258 | Type genType = clazz.getGenericSuperclass(); 259 | 260 | if (!(genType instanceof ParameterizedType)) { 261 | logger.warn(clazz.getSimpleName() + "'s superclass not ParameterizedType"); 262 | return Object.class; 263 | } 264 | 265 | Type[] params = ((ParameterizedType) genType).getActualTypeArguments(); 266 | 267 | if (index >= params.length || index < 0) { 268 | logger.warn("Index: " + index + ", Size of " + clazz.getSimpleName() + "'s Parameterized Type: " 269 | + params.length); 270 | return Object.class; 271 | } 272 | if (!(params[index] instanceof Class)) { 273 | logger.warn(clazz.getSimpleName() + " not set the actual class on superclass generic parameter"); 274 | return Object.class; 275 | } 276 | 277 | return (Class) params[index]; 278 | } 279 | 280 | public static Class getUserClass(Object instance) { 281 | Class clazz = instance.getClass(); 282 | if (clazz != null && clazz.getName().contains(CGLIB_CLASS_SEPARATOR)) { 283 | Class superClass = clazz.getSuperclass(); 284 | if (superClass != null && !Object.class.equals(superClass)) { 285 | return superClass; 286 | } 287 | } 288 | return clazz; 289 | 290 | } 291 | 292 | /** 293 | * 将反射时的checked exception转换为unchecked exception. 294 | */ 295 | public static RuntimeException convertReflectionExceptionToUnchecked(Exception e) { 296 | if (e instanceof IllegalAccessException || e instanceof IllegalArgumentException 297 | || e instanceof NoSuchMethodException) { 298 | return new IllegalArgumentException(e); 299 | } else if (e instanceof InvocationTargetException) { 300 | return new RuntimeException(((InvocationTargetException) e).getTargetException()); 301 | } else if (e instanceof RuntimeException) { 302 | return (RuntimeException) e; 303 | } 304 | return new RuntimeException("Unexpected Checked Exception.", e); 305 | } 306 | } -------------------------------------------------------------------------------- /liteBatch-test/pom.xml: -------------------------------------------------------------------------------- 1 |  2 | 4 | com.lightSalt.liteBatch 5 | liteBatch-test 6 | jar 7 | 4.0.0 8 | 1.0.0 9 | 10 | 11 | UTF-8 12 | 1.7 13 | 3.4 14 | 4.2.5.RELEASE 15 | 1.7.21 16 | 1.2.17 17 | 1.7.5 18 | 1.7.13 19 | 5.1.13 20 | 1.0.14 21 | 1.1.0 22 | 4.12 23 | 24 | 25 | 26 | 27 | org.apache.commons 28 | commons-lang3 29 | ${commons.lang3.version} 30 | 31 | 32 | org.springframework 33 | spring-core 34 | ${spring.version} 35 | 36 | 37 | org.springframework 38 | spring-beans 39 | ${spring.version} 40 | 41 | 42 | org.springframework 43 | spring-context 44 | ${spring.version} 45 | 46 | 47 | org.springframework 48 | spring-jdbc 49 | ${spring.version} 50 | 51 | 52 | org.springframework 53 | spring-test 54 | ${spring.version} 55 | 56 | 57 | org.springframework 58 | spring-aop 59 | ${spring.version} 60 | 61 | 62 | org.springframework 63 | spring-orm 64 | ${spring.version} 65 | 66 | 67 | org.springframework 68 | spring-web 69 | ${spring.version} 70 | 71 | 72 | org.springframework 73 | spring-webmvc 74 | ${spring.version} 75 | 76 | 77 | org.springframework 78 | spring-context-support 79 | ${spring.version} 80 | 81 | 82 | log4j 83 | log4j 84 | ${log4j.version} 85 | 86 | 87 | org.slf4j 88 | slf4j-api 89 | ${org.slf4j.version} 90 | 91 | 92 | org.slf4j 93 | slf4j-log4j12 94 | ${log4j-slf4j.version} 95 | 96 | 97 | mysql 98 | mysql-connector-java 99 | ${mysql.version} 100 | 101 | 102 | com.alibaba 103 | druid 104 | ${druid.version} 105 | 106 | 107 | com.lightSalt.liteBatch 108 | liteBatch-core 109 | ${liteBatch.version} 110 | 111 | 112 | 113 | junit 114 | junit 115 | ${junit.version} 116 | 117 | 118 | 119 | 120 | 121 | 122 | org.apache.maven.plugins 123 | maven-compiler-plugin 124 | 125 | UTF-8 126 | ${java.version} 127 | ${java.version} 128 | 129 | 130 | 131 | 132 | -------------------------------------------------------------------------------- /liteBatch-test/src/main/java/com/litesalt/batch/test/Person.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch.test; 2 | 3 | import java.io.Serializable; 4 | import java.util.Date; 5 | 6 | public class Person implements Serializable{ 7 | 8 | private static final long serialVersionUID = 1L; 9 | private Long id; 10 | private String name; 11 | private Integer age; 12 | private String address; 13 | private String mobile; 14 | private String email; 15 | private String company; 16 | private String title; 17 | private Date createTime; 18 | 19 | public Long getId() { 20 | return id; 21 | } 22 | public void setId(Long id) { 23 | this.id = id; 24 | } 25 | public String getName() { 26 | return name; 27 | } 28 | public void setName(String name) { 29 | this.name = name; 30 | } 31 | public Integer getAge() { 32 | return age; 33 | } 34 | public void setAge(Integer age) { 35 | this.age = age; 36 | } 37 | public String getAddress() { 38 | return address; 39 | } 40 | public void setAddress(String address) { 41 | this.address = address; 42 | } 43 | public String getMobile() { 44 | return mobile; 45 | } 46 | public void setMobile(String mobile) { 47 | this.mobile = mobile; 48 | } 49 | public String getEmail() { 50 | return email; 51 | } 52 | public void setEmail(String email) { 53 | this.email = email; 54 | } 55 | public String getCompany() { 56 | return company; 57 | } 58 | public void setCompany(String company) { 59 | this.company = company; 60 | } 61 | public String getTitle() { 62 | return title; 63 | } 64 | public void setTitle(String title) { 65 | this.title = title; 66 | } 67 | public Date getCreateTime() { 68 | return createTime; 69 | } 70 | public void setCreateTime(Date createTime) { 71 | this.createTime = createTime; 72 | } 73 | } -------------------------------------------------------------------------------- /liteBatch-test/src/main/java/com/litesalt/batch/test/PersonVo.java: -------------------------------------------------------------------------------- 1 | /** 2 | *

Title: litePromise

3 | *

Description: litePromise,give you a promise

4 | *

Copyright: Copyright (c) 2016

5 | * @author Bryan.Zhang 6 | * @email 47483522@qq.com 7 | * @Date 2016-11-11 8 | * @version 1.0 9 | */ 10 | package com.litesalt.batch.test; 11 | 12 | import java.util.Date; 13 | 14 | import com.litesalt.batch.annotation.AliasField; 15 | import com.litesalt.batch.annotation.AliasTable; 16 | import com.litesalt.batch.annotation.ExcludeField; 17 | 18 | @AliasTable("person") 19 | public class PersonVo { 20 | @AliasField("name") 21 | private String personName; 22 | @AliasField("age") 23 | private Integer personAge; 24 | @ExcludeField 25 | private String coName; 26 | @AliasField("create_time") 27 | private Date insertDate; 28 | public String getPersonName() { 29 | return personName; 30 | } 31 | public void setPersonName(String personName) { 32 | this.personName = personName; 33 | } 34 | public Integer getPersonAge() { 35 | return personAge; 36 | } 37 | public void setPersonAge(Integer personAge) { 38 | this.personAge = personAge; 39 | } 40 | public String getCoName() { 41 | return coName; 42 | } 43 | public void setCoName(String coName) { 44 | this.coName = coName; 45 | } 46 | public Date getInsertDate() { 47 | return insertDate; 48 | } 49 | public void setInsertDate(Date insertDate) { 50 | this.insertDate = insertDate; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /liteBatch-test/src/main/java/com/litesalt/batch/test/TestMain.java: -------------------------------------------------------------------------------- 1 | package com.litesalt.batch.test; 2 | 3 | import java.util.Date; 4 | import java.util.Random; 5 | 6 | import javax.annotation.Resource; 7 | 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | import org.springframework.jdbc.core.JdbcTemplate; 11 | import org.springframework.test.context.ContextConfiguration; 12 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 13 | 14 | import com.litesalt.batch.DBRowBatchListenerBuilder; 15 | import com.litesalt.batch.listener.RowBatchListener; 16 | 17 | @RunWith(SpringJUnit4ClassRunner.class) 18 | @ContextConfiguration(locations = { "classpath:spring-db.xml" }) 19 | public class TestMain { 20 | @Resource 21 | private JdbcTemplate jdbcTemplate; 22 | 23 | @Test 24 | public void testBatch1() throws Exception { 25 | long start = System.currentTimeMillis(); 26 | RowBatchListener rowBatchListener = DBRowBatchListenerBuilder.buildRedisRowBatchListener(jdbcTemplate, 10000, Person.class, "192.168.20.48", 27 | 6379,null); 28 | 29 | // RowBatchListener rowBatchListener = RowBatchListenerBuilder.buildMemoryRowBatchListener(jdbcTemplate, 5000, Person.class); 30 | 31 | try { 32 | Random random = new Random(); 33 | Person person = null; 34 | for (int i = 0; i < 5000; i++) { 35 | person = new Person(); 36 | person.setAge(random.nextInt(100)); 37 | person.setAddress("XX马路1号"); 38 | person.setCompany("天天 向上科技有限公司"); 39 | person.setName("张三"); 40 | person.setCreateTime(new Date()); 41 | rowBatchListener.insertOneWithBatch(person); 42 | } 43 | } catch (Exception e) { 44 | e.printStackTrace(); 45 | } finally { 46 | } 47 | long end = System.currentTimeMillis(); 48 | System.out.println("耗时"+(end-start)); 49 | System.in.read(); 50 | } 51 | 52 | @Test 53 | public void testBatch2() throws Exception { 54 | RowBatchListener rowBatchListener = DBRowBatchListenerBuilder.buildMemoryRowBatchListener(jdbcTemplate, 5000, PersonVo.class); 55 | try { 56 | Random random = new Random(); 57 | PersonVo personVo = null; 58 | for (int i = 0; i < 286000; i++) { 59 | personVo = new PersonVo(); 60 | personVo.setPersonName("李四"); 61 | personVo.setPersonAge(random.nextInt(100)); 62 | personVo.setCoName("XX公司"); 63 | rowBatchListener.insertOneWithBatch(personVo); 64 | } 65 | } catch (Exception e) { 66 | e.printStackTrace(); 67 | } finally { 68 | } 69 | 70 | System.in.read(); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /liteBatch-test/src/main/resources/config.properties: -------------------------------------------------------------------------------- 1 | hh.jdbcUrl=jdbc\:mysql\://localhost\:3366/poseidon?characterEncoding\=UTF-8&rewriteBatchedStatements\=true 2 | hh.driverClass=com.mysql.jdbc.Driver 3 | hh.user=root 4 | hh.password=123456 5 | hh.initialPoolSize=1 6 | hh.minPoolSize=5 7 | hh.maxPoolSize=10 8 | hh.automaticTestTable=test_c3p0 9 | -------------------------------------------------------------------------------- /liteBatch-test/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=info,Console,DailyFile 2 | log4j.appender.Console=org.apache.log4j.ConsoleAppender 3 | log4j.appender.Console.layout=org.apache.log4j.PatternLayout 4 | log4j.appender.Console.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss,SSS} [%p] %m >> %c\:%L%n 5 | 6 | log4j.appender.DailyFile=org.apache.log4j.DailyRollingFileAppender 7 | log4j.appender.DailyFile.Threshold = info 8 | log4j.appender.DailyFile.File=./logs/test.log 9 | log4j.appender.DailyFile.encoding=UTF-8 10 | log4j.appender.DailyFile.Append=true 11 | log4j.appender.DailyFile.layout=org.apache.log4j.PatternLayout 12 | log4j.appender.DailyFile.layout.ConversionPattern=%d{yyyy-MM-dd HH\:mm\:ss,SSS} [%p] %m >> %c\:%L%n -------------------------------------------------------------------------------- /liteBatch-test/src/main/resources/spring-db.xml: -------------------------------------------------------------------------------- 1 | 2 | 15 | 16 | 17 | 18 | 19 | classpath:config.properties 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | --------------------------------------------------------------------------------