├── README.md ├── bak └── TransferDefinitions.xml ├── pom.xml └── src └── main ├── java └── com │ └── xtonic │ ├── config │ ├── Config.java │ ├── DataSourceDeifiniton.java │ ├── ErrorMsg.java │ ├── SrcData.java │ ├── TableChanges.java │ ├── TransferTableDefiniton.java │ └── TypeHandlerDefinition.java │ ├── container │ ├── DefaultErrorLogComsumer.java │ ├── EorrorDataLogConsumer.java │ ├── ErrorMsgProvider.java │ ├── Main.java │ └── Runcontainer.java │ ├── context │ ├── AbstarctTransferContext.java │ ├── AbstractRegistFactory.java │ ├── RegistConfig.java │ ├── RegistDataSourceFacotroy.java │ ├── RegistTableDefinitionFactory.java │ ├── RegistTypeHandlerFactory.java │ ├── RegisterFactory.java │ ├── TransferContext.java │ └── impl │ │ ├── BaseTrnasferContext.java │ │ └── TrnasferAppContext.java │ ├── dataSource │ ├── AbstractTransferDataSource.java │ ├── DefaultTransferDataSource.java │ └── TransferDataSource.java │ ├── entity │ ├── Test_Table_1.java │ ├── Test_Table_2.java │ ├── Test_Table_3.java │ ├── Test_Table_4.java │ ├── Test_Table_5.java │ ├── Test_Table_6.java │ └── User.java │ ├── exception │ └── GetTotalCountException.java │ ├── resources │ ├── ClassPathResource.java │ └── Resource.java │ ├── task │ ├── AbstractInsertDataTask.java │ ├── AbstractTransferTask.java │ ├── TransferTask.java │ ├── TransferTaskConfig.java │ ├── TransferTaskFactory.java │ └── impl │ │ ├── DefaultTransferTask.java │ │ ├── InsertTask.java │ │ └── MoreSrcsTransferTask.java │ ├── type │ ├── TypeChangeHander.java │ └── impl │ │ └── StringToIntegerHandler.java │ ├── utils │ ├── ClassUtils.java │ ├── PageQuerySQLTemplate.java │ └── Xml2JsonUtil.java │ ├── validator │ ├── DataSourceValidator.java │ ├── DefinitonValidator.java │ ├── TableValidator.java │ ├── TypeHandlerValidator.java │ └── impl │ │ └── DefaultDefinitionValidator.java │ └── xmlDefinition │ ├── DefaultLoadDefinition.java │ └── LoadDefinition.java └── resources ├── TransferDefinitions.xml ├── TransferDefinitions.xsd ├── log4j.properties └── logs ├── :user.txt ├── error.log └── log.log /README.md: -------------------------------------------------------------------------------- 1 | ###配置文件说明 2 | ---------- 3 | 执行容器的上下文是根据配置文件生成的，有数据源，类型装唤起，自定义的属性，还有最核心的，迁移任务配置； 4 | 5 | * **数据源配置** 6 | 7 | 8 | 9 | 10 | jdbc:mysql://192.168.31.108:3306/test 11 | xxx 12 | xxxx 13 | com.mysql.jdbc.Driver 14 | 15 | 16 | jdbc:mysql://192.168.31.108:3306/testA 17 | xxx 18 | xxxx 19 | com.mysql.jdbc.Driver 20 | 21 | 22 | 23 | 可以配多个数据源；每一个数据源都有一个自己的ID； 24 | dbType: 数据库的类型： mySql, sqlServer, oracle .... 25 | 26 | * **属性配置** 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 配置一些常用的属性，现在需要有个俩个配置： 36 | 37 | pageSize: 分页查询的时候，单页数据量的配置； 38 | 39 | fialRecordFilePath：记录迁移失败数据的文件的存放路径； 40 | 41 | **NOTE：** 如果有需要其他的配置，在进行自定义扩展的时候，可以按上面的格式进行配置，然后就可以从执行上下文中直接用getProperty(name) 来获取配置的值。 42 | 43 | * **迁移任务配置** 44 | 45 | 我们将需要迁移的目标数据库的表当做一个任务，而它数据可能来自多个表，甚至不同库的不同表，我们把他们分为下面种情况： 46 | 47 | 48 | **A：同库单表** 49 | 50 | 52 | 53 | 55 | 56 |
57 | 58 | 59 | **B：同库多表** 60 | 61 | 62 | 64 | 65 | 67 | 68 | 69 | 70 | 71 |
72 | 73 | 74 | **C：多库多表** 75 | 76 | 78 | 79 | 81 | 83 | 84 | 85 | 86 | 87 |
88 | 89 | 90 | 上面是三种情况（应该是可以满足大多数的数据迁移需求）的配置列子；下面是对各个配置项的说明，及其注意点： 91 | 92 | **table节点 表示的是一个迁移任务，其配置说明如下：** 93 | 94 | beanClass: 目标表对应JAVA BEAN； 95 | 96 | table: 目标表的表名 97 | 98 | targetDataSourceRef：目标表所在数据库对应的数据源的ID （DataSources 中配置的数据源） 99 | 100 | taskImplClass: 配置迁移任务的实现类；如果没有配置，就采用默认的是实现类是： 101 | 102 | com.xtonic.task.impl.DefaultTransferTask； 103 | 104 | //在不涉及到多库多表的迁移任务的时候，采用这个默认的； 它满足的是单库单表，或单库多表的情景； 105 | 106 | //如果是多库多表的情况，就采用下面这个这个实现类： 107 | 108 | com.xtonic.task.impl.MoreSrcsTransferTask 109 | 110 | *如果这俩个实现类都不能满足要求，就可以采用自己的扩展编写的实现类，在后续的**扩展说明**章节进行说明；* 111 | 112 | 113 | **srcs节点： 表示数据源，配置的是如何获取要插入到目标表中的数据。可以有多个；** 114 | 115 | srcDataSourceRef ： 数据源的ID； 在DataSource节点所配置的数据源的ID 116 | 117 | keyColumn：关键字段，一般是主键，如果有多个字段组成的主键采用‘,’进行分割， keyColumn="id，username" 118 | 119 | 在多库多表的情况下， 并且采用的是默认的com.xtonic.task.impl.MoreSrcsTransferTask的实现类的情况下： 120 | 几个数据源直接的keyColumn应该是一样额，是通过这个KeyColumn去关联不同数据库直接的数据的； 121 | 122 | 比如： 123 | 124 | 126 | 这样的话，会采用 id = id , userName = name 这样的规则来匹对数据； 127 | 128 | 129 | 130 | querySql: 从配置的数据库中获取源数据的SQL；这个SQL有几点要求： 131 | 132 | 1：不允许采用 select * from table； 这种写法， 要采用select columnA, columnB.... from table 这种写法； 133 | 2：字段名必须与 beanClass属性配置的 目标表的JAVA BEAN的属性对应， 如果源数据表的字段与目标表的字段的名字有出入，请在querySQL中采用别名的写法， 字段的别名应该与JAVA BEAN的属性名对应； 134 | 3： 字段的数量：多条SQL加起来的字段的总数，应该小于或等于JAVA BEAN的属性的个数； 135 | 4： 如果配置了 KeyColumn属性，那么keyColumn字段应该在 querySQL里面存在； 136 | 137 | 138 | **changes 节点： 是用来反映数据库结构变化的情况，进行再数据迁移的过程对对应的数据字段进行转换处理的处理器；** 139 | 140 | columnName ：有变动的字段，该值应该是JAVA BEAN 里面对应的属性名； 其实也是目标表里面字段名； 141 | targetType ：转换后的数据类型； 142 | srcType： 转换前的数据类型； 143 | handlerRef ： 转换处理的ID； （在转换器的配置） 144 | 145 | 146 | * **类型转换器的配置** 147 | 148 | 149 | 150 | 151 | 152 | 153 | handlerid ： 转换器的ID； 154 | 155 | handlerClass ： 转换器的是显现类； 该转换器，可能会涉及业务规则，或有一些特殊的情况，可以根据实际情况自己进行相关实现，并配置； **该实现类必须实现：com.xtonic.type.TypeChangeHander 接口；** 156 | 157 | 158 | ###扩展说明 159 | ----------- 160 | 在执行迁移任务的时候，主要三个核心的步骤，如下： 161 | 162 | 1. 获取数据，返回一个List集合，集合的类型为Mpa对象，KEY：为字段名，Value：属性值； 163 | 2. 将集合的一条条数据转换为对应的JAVE Bean； 中间包括了，类型转换处理； 164 | 3. 将一个个，JAVA BEAN 的数据，插入到对应的目标表中； 165 | 166 | 如果自带的任务实现类无法满足业务需求的话，可以自定义自己的任务实现类，并在XML中配置； 167 | 168 | ***该任务实现类要继承 com.xtonic.task.AbstractTransferTask;***，并实现getDataFromSrc（） 返回的数据格式严格按照上述的规范。 169 | 170 | 针对每一条数据进行特殊的处理，可以重写customProcessValues(List values, List fieldNames)方法； 171 | 172 | values: 一个JAVA BEAN存的数据的值。 173 | fieldNames: JAVA BENA对应的字段； 174 | values 与 fieldNames一一对应； 175 | 176 | ###运行 177 | ------ 178 | import java.io.IOException; 179 | 180 | import com.xtonic.context.TransferContext; 181 | import com.xtonic.context.impl.TrnasferAppContext; 182 | 183 | public class Main { 184 | public static void main(String[] args) throws IOException, InterruptedException { 185 | //初始化XML文件，生成上下文 186 | TransferContext context = new TrnasferAppContext("TransferDefinitions.xml"); 187 | //将上下文丢到容器里面进行运行； 188 | Runcontainer containor = new Runcontainer(context); 189 | containor.excute(); 190 | } 191 | } 192 | 193 | 194 | -------------------------------------------------------------------------------- /bak/TransferDefinitions.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | jdbc:mysql://192.168.31.108:3306/test 8 | root 9 | jzh9866 10 | com.mysql.jdbc.Driver 11 | 12 | 13 | jdbc:mysql://192.168.31.108:3306/testA 14 | root 15 | jzh9866 16 | com.mysql.jdbc.Driver 17 | 18 | 19 | 20 | 32 | 34 | 35 | 37 | 39 | 40 | 41 | 42 | 43 |
44 | 51 | 58 | 59 | 66 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.xtonic 5 | DateTransfer 6 | 0.0.1-SNAPSHOT 7 | 8 | 9 | net.sf.json-lib 10 | json-lib 11 | 2.4 12 | jdk15 13 | 14 | 15 | org.jdom 16 | jdom 17 | 1.1 18 | 19 | 20 | 21 | xom 22 | xom 23 | 1.2.5 24 | 25 | 26 | 27 | commons-dbutils 28 | commons-dbutils 29 | 1.6 30 | 31 | 32 | mysql 33 | mysql-connector-java 34 | 5.1.38 35 | 36 | 37 | 38 | org.mybatis 39 | mybatis 40 | 3.2.8 41 | 42 | 43 | log4j 44 | log4j 45 | 1.2.9 46 | 47 | 48 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/config/Config.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.config; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | public class Config { 7 | private static final Config CONFIG= new Config(); 8 | private Config(){}; 9 | public static Config initConfig(Map paramterMap){ 10 | if(commonParameters.isEmpty() && !paramterMap.isEmpty() ){ 11 | commonParameters.putAll(paramterMap); 12 | return CONFIG; 13 | } 14 | return CONFIG; 15 | } 16 | private final static Map commonParameters = new HashMap(); 17 | 18 | //TODO: 这里需要做同步的处理。需要优化，暂时先这样处理； 19 | public synchronized Object getParameter(String key){ 20 | return commonParameters.get(key); 21 | } 22 | public synchronized void setParameter(String key, Object value){ 23 | commonParameters.put(key, value); 24 | }; 25 | @Override 26 | public String toString() { 27 | return commonParameters.toString(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/config/DataSourceDeifiniton.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.config; 2 | 3 | public class DataSourceDeifiniton { 4 | private String url; 5 | private String password; 6 | private String username; 7 | private String driverClass; 8 | private String id; 9 | private String dbType; 10 | 11 | public String getDbType() { 12 | return dbType; 13 | } 14 | public void setDbType(String dbType) { 15 | this.dbType = dbType; 16 | } 17 | public String getUrl() { 18 | return url; 19 | } 20 | public void setUrl(String url) { 21 | this.url = url; 22 | } 23 | public String getPassword() { 24 | return password; 25 | } 26 | public void setPassword(String password) { 27 | this.password = password; 28 | } 29 | public String getUsername() { 30 | return username; 31 | } 32 | public void setUsername(String username) { 33 | this.username = username; 34 | } 35 | public String getDriverClass() { 36 | return driverClass; 37 | } 38 | public void setDriverClass(String driverClass) { 39 | this.driverClass = driverClass; 40 | } 41 | public String getId() { 42 | return id; 43 | } 44 | public void setId(String id) { 45 | this.id = id; 46 | } 47 | @Override 48 | public String toString() { 49 | return "DataSourceDeifiniton [url=" + url + ", password=" + password + ", username=" + username 50 | + ", driverClass=" + driverClass + ", id=" + id + "]"; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/config/ErrorMsg.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.config; 2 | 3 | public class ErrorMsg { 4 | private String tableName; 5 | private String errorMsg; 6 | 7 | public ErrorMsg(String tableName,String errorMsg) { 8 | this.tableName = tableName; 9 | this.errorMsg = errorMsg; 10 | } 11 | 12 | public String getTableName() { 13 | return tableName; 14 | } 15 | public void setTableName(String tableName) { 16 | this.tableName = tableName; 17 | } 18 | public String getErrorMsg() { 19 | return errorMsg; 20 | } 21 | public void setErrorMsg(String errorMsg) { 22 | this.errorMsg = errorMsg; 23 | } 24 | @Override 25 | public String toString() { 26 | return "ErrorMsg [tableName=" + tableName + ", errorMsg=" + errorMsg + "]"; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/config/SrcData.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.config; 2 | 3 | public class SrcData { 4 | private String querySql; 5 | private String srcDataSourceRef; 6 | private String keyColumn; 7 | private Boolean isMainSrc; 8 | 9 | public String getQuerySql() { 10 | return querySql; 11 | } 12 | public void setQuerySql(String querySql) { 13 | this.querySql = querySql; 14 | } 15 | public String getSrcDataSourceRef() { 16 | return srcDataSourceRef; 17 | } 18 | public void setSrcDataSourceRef(String srcDataSourceRef) { 19 | this.srcDataSourceRef = srcDataSourceRef; 20 | } 21 | public String getKeyColumn() { 22 | return keyColumn; 23 | } 24 | public void setKeyColumn(String keyColumn) { 25 | this.keyColumn = keyColumn; 26 | } 27 | public Boolean getIsMainSrc() { 28 | return isMainSrc; 29 | } 30 | public void setIsMainSrc(Boolean isMainSrc) { 31 | this.isMainSrc = isMainSrc; 32 | } 33 | @Override 34 | public String toString() { 35 | return "SrcData [querySql=" + querySql + ", srcDataSourceRef=" + srcDataSourceRef + ", keyColumn=" + keyColumn 36 | + ", isMainSrc=" + isMainSrc + "]"; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/config/TableChanges.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.config; 2 | 3 | public class TableChanges { 4 | private String columnName; 5 | private String targetType; 6 | private String srcType; 7 | private String handlerRef; 8 | 9 | public String getHandlerRef() { 10 | return handlerRef; 11 | } 12 | public void setHandlerRef(String handlerRef) { 13 | this.handlerRef = handlerRef; 14 | } 15 | public String getColumnName() { 16 | return columnName; 17 | } 18 | public void setColumnName(String columnName) { 19 | this.columnName = columnName; 20 | } 21 | public String getTargetType() { 22 | return targetType; 23 | } 24 | public void setTargetType(String targetType) { 25 | this.targetType = targetType; 26 | } 27 | 28 | public String getSrcType() { 29 | return srcType; 30 | } 31 | public void setSrcType(String srcType) { 32 | this.srcType = srcType; 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return "TableChanges [columnName=" + columnName + ", targetType=" + targetType + ", srcType=" + srcType 38 | + ", handlerRef=" + handlerRef + "]"; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/config/TransferTableDefiniton.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.config; 2 | 3 | import java.util.List; 4 | 5 | public class TransferTableDefiniton { 6 | private String beanClass; 7 | private String table; 8 | private String targetDataSourceRef; 9 | private String id; 10 | private String taskImplClass; 11 | private List srcs; 12 | private List changes; 13 | public String getBeanClass() { 14 | return beanClass; 15 | } 16 | public void setBeanClass(String beanClass) { 17 | this.beanClass = beanClass; 18 | } 19 | public String getTable() { 20 | return table; 21 | } 22 | public void setTable(String table) { 23 | this.table = table; 24 | } 25 | public String getTargetDataSourceRef() { 26 | return targetDataSourceRef; 27 | } 28 | public void setTargetDataSourceRef(String targetDataSourceRef) { 29 | this.targetDataSourceRef = targetDataSourceRef; 30 | } 31 | public String getId() { 32 | return id; 33 | } 34 | public void setId(String id) { 35 | this.id = id; 36 | } 37 | public List getSrcs() { 38 | return srcs; 39 | } 40 | public void setSrcs(List srcs) { 41 | this.srcs = srcs; 42 | } 43 | public List getChanges() { 44 | return changes; 45 | } 46 | public void setChanges(List changes) { 47 | this.changes = changes; 48 | } 49 | public String getTaskImplClass() { 50 | return taskImplClass; 51 | } 52 | public void setTaskImplClass(String taskImplClass) { 53 | this.taskImplClass = taskImplClass; 54 | } 55 | @Override 56 | public String toString() { 57 | return "TransferTableDefiniton [beanClass=" + beanClass + ", table=" + table + ", targetDataSourceRef=" 58 | + targetDataSourceRef + ", id=" + id + ", srcs=" + srcs + ", changes=" + changes + "]"; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/config/TypeHandlerDefinition.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.config; 2 | 3 | public class TypeHandlerDefinition { 4 | private String handlerid; 5 | private String handlerClass; 6 | public String getHandlerid() { 7 | return handlerid; 8 | } 9 | public void setHandlerid(String handlerid) { 10 | this.handlerid = handlerid; 11 | } 12 | public String getHandlerClass() { 13 | return handlerClass; 14 | } 15 | public void setHandlerClass(String handlerClass) { 16 | this.handlerClass = handlerClass; 17 | } 18 | @Override 19 | public String toString() { 20 | return "TypeHandlerDefinition [handlerid=" + handlerid + ", handlerClass=" + handlerClass + "]"; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/container/DefaultErrorLogComsumer.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.container; 2 | 3 | import java.io.BufferedWriter; 4 | import java.io.File; 5 | import java.io.FileNotFoundException; 6 | import java.io.FileOutputStream; 7 | import java.io.IOException; 8 | import java.io.OutputStreamWriter; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | import java.util.concurrent.Future; 12 | 13 | import org.apache.commons.logging.Log; 14 | import org.apache.commons.logging.LogFactory; 15 | 16 | import com.xtonic.config.ErrorMsg; 17 | import com.xtonic.context.TransferContext; 18 | 19 | public class DefaultErrorLogComsumer implements EorrorDataLogConsumer { 20 | private static final Log LOGER = LogFactory.getLog(DefaultErrorLogComsumer.class); 21 | private String failRecordFilePath; 22 | private Map files= new HashMap(); 23 | private Map fileStreams = new HashMap(); 24 | private boolean allTransferTasksDone = false; 25 | 26 | 27 | public DefaultErrorLogComsumer(TransferContext context) { 28 | String filePath = (String) context.getProperty("fialRecordFilePath"); 29 | if(filePath == null){ 30 | filePath = this.getClass().getResource(".").getFile().toString(); 31 | } 32 | this.failRecordFilePath = filePath; 33 | } 34 | 35 | public Object call() throws Exception { 36 | while(true){ 37 | messageConsumer(); 38 | if(allTransferTasksDone && messageQueue.isEmpty()){ 39 | break; 40 | } 41 | } 42 | LOGER.info("错误数据记录器停止工作"); 43 | return "错误数据记录器停止工作"; 44 | } 45 | 46 | public void messageConsumer() { 47 | ErrorMsg errorMsg = null; 48 | try { 49 | errorMsg = messageQueue.take(); //如果messageQueue 为空，在这里阻塞等待； 50 | } catch (InterruptedException e) { 51 | if(messageQueue.isEmpty() && allTransferTasksDone){ 52 | return; 53 | } 54 | LOGER.error("从队列中获取错误信息失败",e); 55 | return; 56 | } 57 | File file = getFile(errorMsg.getTableName()); 58 | System.out.println(" .............." + file.getAbsolutePath()); 59 | BufferedWriter writer = getFileString(errorMsg.getTableName(),file); 60 | 61 | try { 62 | writer.write(errorMsg.getErrorMsg()+ "\r\n"); 63 | writer.flush(); 64 | } catch (IOException e) { 65 | LOGER.error("将错误信息："+errorMsg.getErrorMsg() + "写入"+file.getName()+"失败",e); 66 | } 67 | } 68 | 69 | private BufferedWriter getFileString(String tableName, File file) { 70 | BufferedWriter writer = fileStreams.get(tableName); 71 | if(writer == null){ 72 | try { 73 | writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, true))); 74 | } catch (FileNotFoundException e) { 75 | LOGER.error("创建工作流失败："); 76 | } 77 | } 78 | fileStreams.put(tableName, writer); 79 | return writer; 80 | } 81 | 82 | private File getFile(String tableName) { 83 | File file = files.get(tableName); 84 | if(file != null){ 85 | return file; 86 | } 87 | file = new File(failRecordFilePath+File.pathSeparator+tableName + ".txt"); 88 | files.put(tableName, file); 89 | return file; 90 | } 91 | 92 | public void stopConsumer() { 93 | allTransferTasksDone = true; 94 | System.out.println(Thread.currentThread().getName()); 95 | Thread.currentThread().interrupted(); 96 | } 97 | 98 | public void stopConsumer(Future consumerTask) { 99 | allTransferTasksDone = true; 100 | System.out.println(Thread.currentThread().getName()); 101 | consumerTask.cancel(true); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/container/EorrorDataLogConsumer.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.container; 2 | 3 | import java.util.concurrent.BlockingQueue; 4 | import java.util.concurrent.Callable; 5 | import java.util.concurrent.Future; 6 | import java.util.concurrent.LinkedBlockingQueue; 7 | 8 | import com.xtonic.config.ErrorMsg; 9 | 10 | public interface EorrorDataLogConsumer extends Callable { 11 | public static final BlockingQueue messageQueue = new LinkedBlockingQueue(); 12 | public static boolean stopConsumerFlag = false; 13 | 14 | void stopConsumer(Future consumerTask); 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/container/ErrorMsgProvider.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.container; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | 6 | import com.xtonic.config.ErrorMsg; 7 | 8 | public class ErrorMsgProvider { 9 | private static final Log LOGGER = LogFactory.getLog(ErrorMsgProvider.class); 10 | public static void putErrorMsgData(ErrorMsg errorMsg){ 11 | try { 12 | EorrorDataLogConsumer.messageQueue.put(errorMsg); 13 | } catch (Exception e) { 14 | LOGGER.error("错误信息入队列失败； 错误信息为："+ errorMsg.toString(),e); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/container/Main.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.container; 2 | 3 | import java.io.IOException; 4 | 5 | import com.xtonic.context.TransferContext; 6 | import com.xtonic.context.impl.TrnasferAppContext; 7 | 8 | public class Main { 9 | public static void main(String[] args) throws IOException, InterruptedException { 10 | //初始化XML文件，生成上下文 11 | TransferContext context = new TrnasferAppContext("TransferDefinitions.xml"); 12 | //将上下文丢到容器里面进行运行； 13 | Runcontainer containor = new Runcontainer(context); 14 | containor.excute(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/container/Runcontainer.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.container; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Map; 6 | import java.util.concurrent.ExecutionException; 7 | import java.util.concurrent.ExecutorService; 8 | import java.util.concurrent.Executors; 9 | import java.util.concurrent.Future; 10 | import java.util.concurrent.TimeUnit; 11 | import java.util.concurrent.TimeoutException; 12 | 13 | import org.apache.commons.logging.Log; 14 | import org.apache.commons.logging.LogFactory; 15 | 16 | import com.xtonic.context.TransferContext; 17 | import com.xtonic.task.TransferTask; 18 | import com.xtonic.task.TransferTaskConfig; 19 | 20 | public class Runcontainer { 21 | private static final Log LOGGER = LogFactory.getLog(Runcontainer.class); 22 | 23 | private ExecutorService transferWorker = Executors.newFixedThreadPool(10); 24 | private List> taskResults = new ArrayList>(); 25 | @SuppressWarnings("rawtypes") 26 | private EorrorDataLogConsumer errorLogDealer; 27 | private Future consumerTask; 28 | private TransferContext context; 29 | private int taskCount = 0; 30 | 31 | @SuppressWarnings("unchecked") 32 | public Runcontainer(TransferContext context) { 33 | LOGGER.info("迁移任务容器启动................"); 34 | this.context = context; 35 | errorLogDealer = new DefaultErrorLogComsumer(context); 36 | this.consumerTask = transferWorker.submit(errorLogDealer); 37 | LOGGER.info("完成容器注册任务上下文及错误记录处理器；错误记录处理器为：" + EorrorDataLogConsumer.class.getName()); 38 | } 39 | 40 | public void excute() { 41 | long st = System.currentTimeMillis(); 42 | Map> tasks = context.getTransferTasks(); 43 | for (String key : tasks.keySet()) { 44 | LOGGER.info("提交执行任务,任务ID：" + ((TransferTaskConfig) context.getTransferTask(key)).getId()); 45 | taskCount++; 46 | try { 47 | Future result = transferWorker.submit(tasks.get(key)); 48 | taskResults.add(result); 49 | } catch (Exception e) { 50 | LOGGER.error("提交执行任务失败,任务ID：" + ((TransferTaskConfig) context.getTransferTask(key)).getId(), e); 51 | } 52 | } 53 | checkExcuteResult(); 54 | System.out.println("所有的任务的完成时间：" + (System.currentTimeMillis() - st) ); 55 | } 56 | 57 | private void checkExcuteResult() { 58 | int finishedCount = 0; 59 | while (true) { 60 | for (int i = 0; i < taskResults.size(); i ++) { 61 | try { 62 | String resultStr = taskResults.get(i).get(5, TimeUnit.SECONDS); 63 | taskResults.remove(i); 64 | i --; 65 | finishedCount ++; 66 | if (resultStr != null) { 67 | LOGGER.info(resultStr); 68 | } 69 | } catch (Exception e) { 70 | if(e instanceof TimeoutException){ 71 | //System.out.println("超时！！！！！！"); 72 | continue; 73 | } 74 | e.printStackTrace(); 75 | } 76 | } 77 | if(finishedCount == taskCount){ 78 | break; 79 | } 80 | } 81 | LOGGER.info("所有的任务均都执行完毕，关闭迁移任务容器...................."); 82 | errorLogDealer.stopConsumer(consumerTask); 83 | transferWorker.shutdown(); 84 | } 85 | 86 | public TransferContext getContext() { 87 | return context; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/AbstarctTransferContext.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import com.xtonic.config.Config; 7 | import com.xtonic.dataSource.TransferDataSource; 8 | import com.xtonic.task.TransferTask; 9 | import com.xtonic.type.TypeChangeHander; 10 | 11 | public abstract class AbstarctTransferContext implements TransferContext{ 12 | protected Config config; 13 | protected final Map> transferTasks = new HashMap>(); 14 | protected final Map dataSources = new HashMap(); 15 | protected final Map> typeHandlers = new HashMap>(); 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/AbstractRegistFactory.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | 6 | import com.xtonic.config.DataSourceDeifiniton; 7 | import com.xtonic.config.TransferTableDefiniton; 8 | import com.xtonic.config.TypeHandlerDefinition; 9 | import com.xtonic.dataSource.DefaultTransferDataSource; 10 | import com.xtonic.task.TransferTaskFactory; 11 | import com.xtonic.type.TypeChangeHander; 12 | import com.xtonic.xmlDefinition.DefaultLoadDefinition; 13 | import com.xtonic.xmlDefinition.LoadDefinition; 14 | 15 | public abstract class AbstractRegistFactory extends AbstarctTransferContext implements RegisterFactory { 16 | private static final Log LOGGER = LogFactory.getLog(AbstractRegistFactory.class); 17 | 18 | protected LoadDefinition loadDefinition = new DefaultLoadDefinition(this); 19 | 20 | public void registerTypeHandler(TypeHandlerDefinition definition) { 21 | Class clazz = null; 22 | try { 23 | clazz = Class.forName(definition.getHandlerClass()); 24 | } catch (ClassNotFoundException e) { 25 | LOGGER.error("没有找到ID：" + definition.getHandlerid() + "的类型转换器的实现类；", e); 26 | throw new RuntimeException("没有找到ID：" + definition.getHandlerid() + "的类型转换器的实现类；", e); 27 | } 28 | TypeChangeHander hander = null; 29 | try { 30 | hander = (TypeChangeHander) clazz.newInstance(); 31 | } catch (Exception e) { 32 | LOGGER.error("ID：" + definition.getHandlerid() + "的类型转换器的实现类实例化失败", e); 33 | throw new RuntimeException("ID：" + definition.getHandlerid() + "的类型转换器的实现类实例化失败", e); 34 | } 35 | typeHandlers.put(definition.getHandlerid(), hander); 36 | LOGGER.info("类型处理器 ID：" + definition.getHandlerid() + "完成在上下文中的注册"); 37 | } 38 | 39 | public void registTableDefinition(TransferTableDefiniton tableDefinition) { 40 | String task_key = tableDefinition.getBeanClass() + "_" + tableDefinition.getId(); 41 | transferTasks.put(task_key, TransferTaskFactory.creatTask(this, tableDefinition)); 42 | LOGGER.info("任务 ID：" + tableDefinition.getId() + "的任务初始化完成并在上下文中的注册"); 43 | } 44 | 45 | public void registDataSources(DataSourceDeifiniton datasourceDefinitions) { 46 | dataSources.put(datasourceDefinitions.getId(), new DefaultTransferDataSource(datasourceDefinitions)); 47 | LOGGER.info("完成DataSource ID：" + datasourceDefinitions.getId() + " 的数据源 ID在上下文的注册"); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/RegistConfig.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context; 2 | 3 | import com.xtonic.config.Config; 4 | 5 | public interface RegistConfig { 6 | void registConfig(Config config); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/RegistDataSourceFacotroy.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context; 2 | 3 | import com.xtonic.config.DataSourceDeifiniton; 4 | 5 | public interface RegistDataSourceFacotroy { 6 | void registDataSources(DataSourceDeifiniton datasourceDefinitions); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/RegistTableDefinitionFactory.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context; 2 | 3 | import com.xtonic.config.TransferTableDefiniton; 4 | 5 | public interface RegistTableDefinitionFactory { 6 | void registTableDefinition(TransferTableDefiniton tableDefinition); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/RegistTypeHandlerFactory.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context; 2 | 3 | import com.xtonic.config.TypeHandlerDefinition; 4 | 5 | public interface RegistTypeHandlerFactory { 6 | void registerTypeHandler(TypeHandlerDefinition definition); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/RegisterFactory.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context; 2 | 3 | public interface RegisterFactory extends RegistTypeHandlerFactory,RegistConfig, RegistDataSourceFacotroy, RegistTableDefinitionFactory{ 4 | } 5 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/TransferContext.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context; 2 | 3 | import java.util.Map; 4 | 5 | import com.xtonic.dataSource.TransferDataSource; 6 | import com.xtonic.task.TransferTask; 7 | import com.xtonic.type.TypeChangeHander; 8 | 9 | public interface TransferContext { 10 | 11 | 12 | 13 | 14 | /** 15 | * 获取系统的一些通用配置参数； 16 | * @return 17 | */ 18 | Object getProperty(String Key); 19 | /** 20 | * 获取迁移任务 21 | * @param id 22 | */ 23 | TransferTask getTransferTask(String id); 24 | 25 | TransferDataSource getDataSources(String id); 26 | 27 | TypeChangeHander getTypeHandler(String id); 28 | 29 | Map> getTransferTasks(); 30 | 31 | void setParameter(String key,Object value); 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/impl/BaseTrnasferContext.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context.impl; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | 9 | import com.xtonic.config.Config; 10 | import com.xtonic.context.AbstractRegistFactory; 11 | import com.xtonic.dataSource.TransferDataSource; 12 | import com.xtonic.resources.Resource; 13 | import com.xtonic.task.TransferTask; 14 | import com.xtonic.type.TypeChangeHander; 15 | 16 | public class BaseTrnasferContext extends AbstractRegistFactory { 17 | private static final Log LOGGER = LogFactory.getLog(BaseTrnasferContext.class); 18 | public BaseTrnasferContext(){}; 19 | 20 | public BaseTrnasferContext(Resource resource) throws IOException{ 21 | LOGGER.info("开始加载解析XML文件:" + resource.getFile().getName()); 22 | this.loadDefinition.loadXMLDefiniton(resource); 23 | } 24 | 25 | public Object getProperty(String Key) { 26 | return config.getParameter(Key); 27 | } 28 | 29 | public TransferTask getTransferTask(String id) { 30 | return transferTasks.get(id); 31 | } 32 | 33 | public TransferDataSource getDataSources(String id) { 34 | return dataSources.get(id); 35 | } 36 | 37 | public Map> getTransferTasks() { 38 | return transferTasks; 39 | } 40 | 41 | public void registConfig(Config config) { 42 | this.config = config; 43 | } 44 | 45 | public TypeChangeHander getTypeHandler(String id) { 46 | return typeHandlers.get(id); 47 | } 48 | 49 | public void setParameter(String key, Object value) { 50 | config.setParameter(key, value); 51 | } 52 | } -------------------------------------------------------------------------------- /src/main/java/com/xtonic/context/impl/TrnasferAppContext.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.context.impl; 2 | 3 | import java.io.IOException; 4 | 5 | import com.xtonic.resources.ClassPathResource; 6 | import com.xtonic.resources.Resource; 7 | 8 | public class TrnasferAppContext extends BaseTrnasferContext{ 9 | 10 | public TrnasferAppContext(String path) throws IOException { 11 | this(new ClassPathResource(path)); 12 | } 13 | 14 | public TrnasferAppContext(Resource resource) throws IOException{ 15 | super(resource); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/dataSource/AbstractTransferDataSource.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.dataSource; 2 | 3 | import org.apache.commons.dbutils.DbUtils; 4 | 5 | import com.xtonic.config.DataSourceDeifiniton; 6 | 7 | public abstract class AbstractTransferDataSource implements TransferDataSource{ 8 | protected String url; 9 | protected String password; 10 | protected String username; 11 | protected String driverClass; 12 | protected String dbType; 13 | 14 | public AbstractTransferDataSource(DataSourceDeifiniton dataSourceDefinition){ 15 | this.url = dataSourceDefinition.getUrl(); 16 | this.password = dataSourceDefinition.getPassword(); 17 | this.username = dataSourceDefinition.getUsername(); 18 | this.driverClass = dataSourceDefinition.getDriverClass(); 19 | this.dbType = dataSourceDefinition.getDbType(); 20 | boolean flag = DbUtils.loadDriver(this.driverClass); 21 | if(!flag){ 22 | throw new RuntimeException("加载数据库驱动出错：driverClass = "+this.driverClass); 23 | } 24 | } 25 | 26 | public String getDBType() { 27 | return this.dbType; 28 | } 29 | 30 | 31 | 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/dataSource/DefaultTransferDataSource.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.dataSource; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.SQLException; 6 | 7 | import com.xtonic.config.DataSourceDeifiniton; 8 | 9 | public class DefaultTransferDataSource extends AbstractTransferDataSource { 10 | 11 | public DefaultTransferDataSource(DataSourceDeifiniton dataSourceDefinition) { 12 | super(dataSourceDefinition); 13 | } 14 | 15 | public Connection getConnection() { 16 | try { 17 | return DriverManager.getConnection(url, username, password); 18 | } catch (SQLException e) { 19 | throw new RuntimeException("获取SQL链接是失败，" 20 | + "URL:" + url + "; username: " + username + ";password: " + password,e); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/dataSource/TransferDataSource.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.dataSource; 2 | 3 | import java.sql.Connection; 4 | 5 | public interface TransferDataSource { 6 | Connection getConnection(); 7 | 8 | String getDBType(); 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/entity/Test_Table_1.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.entity; 2 | 3 | public class Test_Table_1 { 4 | private Integer id; 5 | private String c1; 6 | private String c2; 7 | private String c3; 8 | private String c4; 9 | private String c5; 10 | private String c6; 11 | private String c7; 12 | private String c8; 13 | private String c9; 14 | private String c10; 15 | private String c11; 16 | private String c12; 17 | private String c13; 18 | private String c14; 19 | private String c15; 20 | private String c16; 21 | public Integer getId() { 22 | return id; 23 | } 24 | public void setId(Integer id) { 25 | this.id = id; 26 | } 27 | public String getC1() { 28 | return c1; 29 | } 30 | public void setC1(String c1) { 31 | this.c1 = c1; 32 | } 33 | public String getC2() { 34 | return c2; 35 | } 36 | public void setC2(String c2) { 37 | this.c2 = c2; 38 | } 39 | public String getC3() { 40 | return c3; 41 | } 42 | public void setC3(String c3) { 43 | this.c3 = c3; 44 | } 45 | public String getC4() { 46 | return c4; 47 | } 48 | public void setC4(String c4) { 49 | this.c4 = c4; 50 | } 51 | public String getC5() { 52 | return c5; 53 | } 54 | public void setC5(String c5) { 55 | this.c5 = c5; 56 | } 57 | public String getC6() { 58 | return c6; 59 | } 60 | public void setC6(String c6) { 61 | this.c6 = c6; 62 | } 63 | public String getC7() { 64 | return c7; 65 | } 66 | public void setC7(String c7) { 67 | this.c7 = c7; 68 | } 69 | public String getC8() { 70 | return c8; 71 | } 72 | public void setC8(String c8) { 73 | this.c8 = c8; 74 | } 75 | public String getC9() { 76 | return c9; 77 | } 78 | public void setC9(String c9) { 79 | this.c9 = c9; 80 | } 81 | public String getC10() { 82 | return c10; 83 | } 84 | public void setC10(String c10) { 85 | this.c10 = c10; 86 | } 87 | public String getC11() { 88 | return c11; 89 | } 90 | public void setC11(String c11) { 91 | this.c11 = c11; 92 | } 93 | public String getC12() { 94 | return c12; 95 | } 96 | public void setC12(String c12) { 97 | this.c12 = c12; 98 | } 99 | public String getC13() { 100 | return c13; 101 | } 102 | public void setC13(String c13) { 103 | this.c13 = c13; 104 | } 105 | public String getC14() { 106 | return c14; 107 | } 108 | public void setC14(String c14) { 109 | this.c14 = c14; 110 | } 111 | public String getC15() { 112 | return c15; 113 | } 114 | public void setC15(String c15) { 115 | this.c15 = c15; 116 | } 117 | public String getC16() { 118 | return c16; 119 | } 120 | public void setC16(String c16) { 121 | this.c16 = c16; 122 | } 123 | @Override 124 | public String toString() { 125 | return "Test_Table_1 [id=" + id + ", c1=" + c1 + ", c2=" + c2 + ", c3=" + c3 + ", c4=" + c4 + ", c5=" + c5 126 | + ", c6=" + c6 + ", c7=" + c7 + ", c8=" + c8 + ", c9=" + c9 + ", c10=" + c10 + ", c11=" + c11 + ", c12=" 127 | + c12 + ", c13=" + c13 + ", c14=" + c14 + ", c15=" + c15 + ", c16=" + c16 + "]"; 128 | } 129 | 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/entity/Test_Table_2.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.entity; 2 | 3 | public class Test_Table_2 { 4 | private Integer id; 5 | private String c1; 6 | private String c2; 7 | private String c3; 8 | private String c4; 9 | private String c5; 10 | private String c6; 11 | private String c7; 12 | private String c8; 13 | private String c9; 14 | private String c10; 15 | private String c11; 16 | private String c12; 17 | private String c13; 18 | private String c14; 19 | private String c15; 20 | private String c16; 21 | public Integer getId() { 22 | return id; 23 | } 24 | public void setId(Integer id) { 25 | this.id = id; 26 | } 27 | public String getC1() { 28 | return c1; 29 | } 30 | public void setC1(String c1) { 31 | this.c1 = c1; 32 | } 33 | public String getC2() { 34 | return c2; 35 | } 36 | public void setC2(String c2) { 37 | this.c2 = c2; 38 | } 39 | public String getC3() { 40 | return c3; 41 | } 42 | public void setC3(String c3) { 43 | this.c3 = c3; 44 | } 45 | public String getC4() { 46 | return c4; 47 | } 48 | public void setC4(String c4) { 49 | this.c4 = c4; 50 | } 51 | public String getC5() { 52 | return c5; 53 | } 54 | public void setC5(String c5) { 55 | this.c5 = c5; 56 | } 57 | public String getC6() { 58 | return c6; 59 | } 60 | public void setC6(String c6) { 61 | this.c6 = c6; 62 | } 63 | public String getC7() { 64 | return c7; 65 | } 66 | public void setC7(String c7) { 67 | this.c7 = c7; 68 | } 69 | public String getC8() { 70 | return c8; 71 | } 72 | public void setC8(String c8) { 73 | this.c8 = c8; 74 | } 75 | public String getC9() { 76 | return c9; 77 | } 78 | public void setC9(String c9) { 79 | this.c9 = c9; 80 | } 81 | public String getC10() { 82 | return c10; 83 | } 84 | public void setC10(String c10) { 85 | this.c10 = c10; 86 | } 87 | public String getC11() { 88 | return c11; 89 | } 90 | public void setC11(String c11) { 91 | this.c11 = c11; 92 | } 93 | public String getC12() { 94 | return c12; 95 | } 96 | public void setC12(String c12) { 97 | this.c12 = c12; 98 | } 99 | public String getC13() { 100 | return c13; 101 | } 102 | public void setC13(String c13) { 103 | this.c13 = c13; 104 | } 105 | public String getC14() { 106 | return c14; 107 | } 108 | public void setC14(String c14) { 109 | this.c14 = c14; 110 | } 111 | public String getC15() { 112 | return c15; 113 | } 114 | public void setC15(String c15) { 115 | this.c15 = c15; 116 | } 117 | public String getC16() { 118 | return c16; 119 | } 120 | public void setC16(String c16) { 121 | this.c16 = c16; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/entity/Test_Table_3.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.entity; 2 | 3 | public class Test_Table_3 { 4 | private Integer id; 5 | private String c1; 6 | private String c2; 7 | private String c3; 8 | private String c4; 9 | private String c5; 10 | private String c6; 11 | private String c7; 12 | private String c8; 13 | private String c9; 14 | private String c10; 15 | private String c11; 16 | private String c12; 17 | private String c13; 18 | private String c14; 19 | private String c15; 20 | private String c16; 21 | public Integer getId() { 22 | return id; 23 | } 24 | public void setId(Integer id) { 25 | this.id = id; 26 | } 27 | public String getC1() { 28 | return c1; 29 | } 30 | public void setC1(String c1) { 31 | this.c1 = c1; 32 | } 33 | public String getC2() { 34 | return c2; 35 | } 36 | public void setC2(String c2) { 37 | this.c2 = c2; 38 | } 39 | public String getC3() { 40 | return c3; 41 | } 42 | public void setC3(String c3) { 43 | this.c3 = c3; 44 | } 45 | public String getC4() { 46 | return c4; 47 | } 48 | public void setC4(String c4) { 49 | this.c4 = c4; 50 | } 51 | public String getC5() { 52 | return c5; 53 | } 54 | public void setC5(String c5) { 55 | this.c5 = c5; 56 | } 57 | public String getC6() { 58 | return c6; 59 | } 60 | public void setC6(String c6) { 61 | this.c6 = c6; 62 | } 63 | public String getC7() { 64 | return c7; 65 | } 66 | public void setC7(String c7) { 67 | this.c7 = c7; 68 | } 69 | public String getC8() { 70 | return c8; 71 | } 72 | public void setC8(String c8) { 73 | this.c8 = c8; 74 | } 75 | public String getC9() { 76 | return c9; 77 | } 78 | public void setC9(String c9) { 79 | this.c9 = c9; 80 | } 81 | public String getC10() { 82 | return c10; 83 | } 84 | public void setC10(String c10) { 85 | this.c10 = c10; 86 | } 87 | public String getC11() { 88 | return c11; 89 | } 90 | public void setC11(String c11) { 91 | this.c11 = c11; 92 | } 93 | public String getC12() { 94 | return c12; 95 | } 96 | public void setC12(String c12) { 97 | this.c12 = c12; 98 | } 99 | public String getC13() { 100 | return c13; 101 | } 102 | public void setC13(String c13) { 103 | this.c13 = c13; 104 | } 105 | public String getC14() { 106 | return c14; 107 | } 108 | public void setC14(String c14) { 109 | this.c14 = c14; 110 | } 111 | public String getC15() { 112 | return c15; 113 | } 114 | public void setC15(String c15) { 115 | this.c15 = c15; 116 | } 117 | public String getC16() { 118 | return c16; 119 | } 120 | public void setC16(String c16) { 121 | this.c16 = c16; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/entity/Test_Table_4.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.entity; 2 | 3 | public class Test_Table_4 { 4 | private Integer id; 5 | private String c1; 6 | private String c2; 7 | private String c3; 8 | private String c4; 9 | private String c5; 10 | private String c6; 11 | private String c7; 12 | private String c8; 13 | private String c9; 14 | private String c10; 15 | private String c11; 16 | private String c12; 17 | private String c13; 18 | private String c14; 19 | private String c15; 20 | private String c16; 21 | public Integer getId() { 22 | return id; 23 | } 24 | public void setId(Integer id) { 25 | this.id = id; 26 | } 27 | public String getC1() { 28 | return c1; 29 | } 30 | public void setC1(String c1) { 31 | this.c1 = c1; 32 | } 33 | public String getC2() { 34 | return c2; 35 | } 36 | public void setC2(String c2) { 37 | this.c2 = c2; 38 | } 39 | public String getC3() { 40 | return c3; 41 | } 42 | public void setC3(String c3) { 43 | this.c3 = c3; 44 | } 45 | public String getC4() { 46 | return c4; 47 | } 48 | public void setC4(String c4) { 49 | this.c4 = c4; 50 | } 51 | public String getC5() { 52 | return c5; 53 | } 54 | public void setC5(String c5) { 55 | this.c5 = c5; 56 | } 57 | public String getC6() { 58 | return c6; 59 | } 60 | public void setC6(String c6) { 61 | this.c6 = c6; 62 | } 63 | public String getC7() { 64 | return c7; 65 | } 66 | public void setC7(String c7) { 67 | this.c7 = c7; 68 | } 69 | public String getC8() { 70 | return c8; 71 | } 72 | public void setC8(String c8) { 73 | this.c8 = c8; 74 | } 75 | public String getC9() { 76 | return c9; 77 | } 78 | public void setC9(String c9) { 79 | this.c9 = c9; 80 | } 81 | public String getC10() { 82 | return c10; 83 | } 84 | public void setC10(String c10) { 85 | this.c10 = c10; 86 | } 87 | public String getC11() { 88 | return c11; 89 | } 90 | public void setC11(String c11) { 91 | this.c11 = c11; 92 | } 93 | public String getC12() { 94 | return c12; 95 | } 96 | public void setC12(String c12) { 97 | this.c12 = c12; 98 | } 99 | public String getC13() { 100 | return c13; 101 | } 102 | public void setC13(String c13) { 103 | this.c13 = c13; 104 | } 105 | public String getC14() { 106 | return c14; 107 | } 108 | public void setC14(String c14) { 109 | this.c14 = c14; 110 | } 111 | public String getC15() { 112 | return c15; 113 | } 114 | public void setC15(String c15) { 115 | this.c15 = c15; 116 | } 117 | public String getC16() { 118 | return c16; 119 | } 120 | public void setC16(String c16) { 121 | this.c16 = c16; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/entity/Test_Table_5.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.entity; 2 | 3 | public class Test_Table_5 { 4 | private Integer id; 5 | private String c1; 6 | private String c2; 7 | private String c3; 8 | private String c4; 9 | private String c5; 10 | private String c6; 11 | private String c7; 12 | private String c8; 13 | private String c9; 14 | private String c10; 15 | private String c11; 16 | private String c12; 17 | private String c13; 18 | private String c14; 19 | private String c15; 20 | private String c16; 21 | public Integer getId() { 22 | return id; 23 | } 24 | public void setId(Integer id) { 25 | this.id = id; 26 | } 27 | public String getC1() { 28 | return c1; 29 | } 30 | public void setC1(String c1) { 31 | this.c1 = c1; 32 | } 33 | public String getC2() { 34 | return c2; 35 | } 36 | public void setC2(String c2) { 37 | this.c2 = c2; 38 | } 39 | public String getC3() { 40 | return c3; 41 | } 42 | public void setC3(String c3) { 43 | this.c3 = c3; 44 | } 45 | public String getC4() { 46 | return c4; 47 | } 48 | public void setC4(String c4) { 49 | this.c4 = c4; 50 | } 51 | public String getC5() { 52 | return c5; 53 | } 54 | public void setC5(String c5) { 55 | this.c5 = c5; 56 | } 57 | public String getC6() { 58 | return c6; 59 | } 60 | public void setC6(String c6) { 61 | this.c6 = c6; 62 | } 63 | public String getC7() { 64 | return c7; 65 | } 66 | public void setC7(String c7) { 67 | this.c7 = c7; 68 | } 69 | public String getC8() { 70 | return c8; 71 | } 72 | public void setC8(String c8) { 73 | this.c8 = c8; 74 | } 75 | public String getC9() { 76 | return c9; 77 | } 78 | public void setC9(String c9) { 79 | this.c9 = c9; 80 | } 81 | public String getC10() { 82 | return c10; 83 | } 84 | public void setC10(String c10) { 85 | this.c10 = c10; 86 | } 87 | public String getC11() { 88 | return c11; 89 | } 90 | public void setC11(String c11) { 91 | this.c11 = c11; 92 | } 93 | public String getC12() { 94 | return c12; 95 | } 96 | public void setC12(String c12) { 97 | this.c12 = c12; 98 | } 99 | public String getC13() { 100 | return c13; 101 | } 102 | public void setC13(String c13) { 103 | this.c13 = c13; 104 | } 105 | public String getC14() { 106 | return c14; 107 | } 108 | public void setC14(String c14) { 109 | this.c14 = c14; 110 | } 111 | public String getC15() { 112 | return c15; 113 | } 114 | public void setC15(String c15) { 115 | this.c15 = c15; 116 | } 117 | public String getC16() { 118 | return c16; 119 | } 120 | public void setC16(String c16) { 121 | this.c16 = c16; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/entity/Test_Table_6.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.entity; 2 | 3 | public class Test_Table_6 { 4 | private Integer id; 5 | private String c1; 6 | private String c2; 7 | private String c3; 8 | private String c4; 9 | private String c5; 10 | private String c6; 11 | private String c7; 12 | private String c8; 13 | private String c9; 14 | private String c10; 15 | private String c11; 16 | private String c12; 17 | private String c13; 18 | private String c14; 19 | private String c15; 20 | private String c16; 21 | public Integer getId() { 22 | return id; 23 | } 24 | public void setId(Integer id) { 25 | this.id = id; 26 | } 27 | public String getC1() { 28 | return c1; 29 | } 30 | public void setC1(String c1) { 31 | this.c1 = c1; 32 | } 33 | public String getC2() { 34 | return c2; 35 | } 36 | public void setC2(String c2) { 37 | this.c2 = c2; 38 | } 39 | public String getC3() { 40 | return c3; 41 | } 42 | public void setC3(String c3) { 43 | this.c3 = c3; 44 | } 45 | public String getC4() { 46 | return c4; 47 | } 48 | public void setC4(String c4) { 49 | this.c4 = c4; 50 | } 51 | public String getC5() { 52 | return c5; 53 | } 54 | public void setC5(String c5) { 55 | this.c5 = c5; 56 | } 57 | public String getC6() { 58 | return c6; 59 | } 60 | public void setC6(String c6) { 61 | this.c6 = c6; 62 | } 63 | public String getC7() { 64 | return c7; 65 | } 66 | public void setC7(String c7) { 67 | this.c7 = c7; 68 | } 69 | public String getC8() { 70 | return c8; 71 | } 72 | public void setC8(String c8) { 73 | this.c8 = c8; 74 | } 75 | public String getC9() { 76 | return c9; 77 | } 78 | public void setC9(String c9) { 79 | this.c9 = c9; 80 | } 81 | public String getC10() { 82 | return c10; 83 | } 84 | public void setC10(String c10) { 85 | this.c10 = c10; 86 | } 87 | public String getC11() { 88 | return c11; 89 | } 90 | public void setC11(String c11) { 91 | this.c11 = c11; 92 | } 93 | public String getC12() { 94 | return c12; 95 | } 96 | public void setC12(String c12) { 97 | this.c12 = c12; 98 | } 99 | public String getC13() { 100 | return c13; 101 | } 102 | public void setC13(String c13) { 103 | this.c13 = c13; 104 | } 105 | public String getC14() { 106 | return c14; 107 | } 108 | public void setC14(String c14) { 109 | this.c14 = c14; 110 | } 111 | public String getC15() { 112 | return c15; 113 | } 114 | public void setC15(String c15) { 115 | this.c15 = c15; 116 | } 117 | public String getC16() { 118 | return c16; 119 | } 120 | public void setC16(String c16) { 121 | this.c16 = c16; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/entity/User.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.entity; 2 | 3 | public class User { 4 | private int id; 5 | private String username; 6 | private int age; 7 | private String intersted; 8 | private String memo; 9 | public int getId() { 10 | return id; 11 | } 12 | public void setId(int id) { 13 | this.id = id; 14 | } 15 | public String getUsername() { 16 | return username; 17 | } 18 | public void setUsername(String username) { 19 | this.username = username; 20 | } 21 | public int getAge() { 22 | return age; 23 | } 24 | public void setAge(int age) { 25 | this.age = age; 26 | } 27 | public String getIntersted() { 28 | return intersted; 29 | } 30 | public void setIntersted(String intersted) { 31 | this.intersted = intersted; 32 | } 33 | public String getMemo() { 34 | return memo; 35 | } 36 | public void setMemo(String memo) { 37 | this.memo = memo; 38 | } 39 | @Override 40 | public String toString() { 41 | return "User [id=" + id + ", username=" + username + ", age=" + age + ", intersted=" + intersted + ", memo=" 42 | + memo + "]"; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/exception/GetTotalCountException.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.exception; 2 | 3 | public class GetTotalCountException extends Exception { 4 | 5 | private static final long serialVersionUID = 2146723536917271795L; 6 | 7 | public GetTotalCountException(String msg ,Exception e ) { 8 | super(msg,e); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/resources/ClassPathResource.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.resources; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | 7 | public class ClassPathResource implements Resource { 8 | 9 | private final String path; 10 | 11 | private final File file; 12 | 13 | public ClassPathResource(String path){ 14 | this.path = path; 15 | this.file = new File(path); 16 | } 17 | 18 | public File getFile() { 19 | return this.file; 20 | } 21 | 22 | public boolean isExist() { 23 | return file.exists(); 24 | } 25 | 26 | public InputStream getInputStream() throws IOException { 27 | 28 | return this.getClass().getClassLoader().getResourceAsStream(path); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/resources/Resource.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.resources; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | 7 | public interface Resource { 8 | 9 | File getFile(); 10 | 11 | boolean isExist(); 12 | 13 | InputStream getInputStream() throws IOException; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/task/AbstractInsertDataTask.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.task; 2 | 3 | import java.lang.reflect.Method; 4 | import java.sql.Connection; 5 | import java.sql.PreparedStatement; 6 | import java.sql.SQLException; 7 | import java.text.MessageFormat; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | import org.apache.commons.dbutils.DbUtils; 12 | 13 | import com.xtonic.utils.ClassUtils; 14 | import com.xtonic.utils.PageQuerySQLTemplate; 15 | 16 | public abstract class AbstractInsertDataTask extends TransferTaskConfig { 17 | private int count =0; 18 | private PreparedStatement pst = null; 19 | public boolean insertData(Connection conn, Object obj, String insertSql, Class clazz) { 20 | 21 | 22 | List values = getValues(obj, beanClass); 23 | 24 | //如果有需要对即将插入的数据进行特殊处理的；在自己的实现类里面重写这个方法就可以； 25 | customProcessValues(values,fieldNames); 26 | 27 | try { 28 | conn.setAutoCommit(false); 29 | if(pst == null){ 30 | pst = conn.prepareStatement(insertSql); 31 | } 32 | for (int i = 0; i < values.size(); i++) { 33 | pst.setObject(i + 1, values.get(i)); 34 | } 35 | count++; 36 | pst.addBatch(); 37 | //pst.executeUpdate(); 38 | if(count!=0 && count % 2000 ==0 ){ 39 | System.out.println("id:"+getId()+"的任务已经迁移了" +count +"条"); 40 | pst.executeBatch(); 41 | pst.clearBatch(); 42 | conn.commit(); 43 | } 44 | 45 | if(count == totalDataCount){ 46 | System.out.println("id:"+getId()+"的任务已经迁移了" +count +"条"); 47 | pst.executeBatch(); 48 | pst.clearBatch(); 49 | conn.commit(); 50 | } 51 | 52 | //DbUtils.close(pst); 53 | return true; 54 | } catch (SQLException e) { 55 | e.printStackTrace(); 56 | return false; 57 | } 58 | } 59 | 60 | /** 61 | * 重写这个方法，进行一些特殊的处理； 62 | * @param values 63 | * @param fieldNames 64 | */ 65 | public void customProcessValues(List values, List fieldNames) { 66 | } 67 | 68 | @SuppressWarnings({ "rawtypes", "unchecked" }) 69 | private List getValues(Object obj, Class clazz) { 70 | List list = new ArrayList(); 71 | Method method = null; 72 | String methodName = ""; 73 | for (String name : fieldNames) { 74 | try { 75 | methodName = ClassUtils.getBeanMethodName("get", name, clazz); 76 | method = clazz.getMethod(methodName, null); 77 | list.add(method.invoke(obj, null)); 78 | } catch (Exception e) { 79 | // TODO: 不能抛运行时异常 80 | throw new RuntimeException("初始化Context出错：实体类" + clazz.getName() + " 的属性" + name + "没有对应的get方法", e); 81 | } 82 | } 83 | return list; 84 | } 85 | 86 | 87 | 88 | protected String getQuerySQL(String dbType,String querySql,int pageNo, int pageSize){ 89 | String tmp = PageQuerySQLTemplate.creatPageableQuerSql(dbType, querySql); 90 | 91 | return MessageFormat.format(tmp, String.valueOf(pageNo*pageSize),String.valueOf(pageSize)); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/task/AbstractTransferTask.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.task; 2 | 3 | import java.lang.reflect.Field; 4 | import java.lang.reflect.Method; 5 | import java.sql.Connection; 6 | import java.sql.PreparedStatement; 7 | import java.sql.ResultSet; 8 | import java.sql.SQLException; 9 | import java.sql.Statement; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | import org.apache.commons.dbutils.DbUtils; 15 | import org.apache.commons.dbutils.QueryRunner; 16 | import org.apache.commons.dbutils.handlers.MapListHandler; 17 | import org.apache.commons.logging.Log; 18 | import org.apache.commons.logging.LogFactory; 19 | 20 | import com.xtonic.config.ErrorMsg; 21 | import com.xtonic.config.SrcData; 22 | import com.xtonic.config.TableChanges; 23 | import com.xtonic.container.ErrorMsgProvider; 24 | import com.xtonic.dataSource.TransferDataSource; 25 | import com.xtonic.exception.GetTotalCountException; 26 | import com.xtonic.type.TypeChangeHander; 27 | import com.xtonic.utils.ClassUtils; 28 | 29 | public abstract class AbstractTransferTask extends AbstractInsertDataTask { 30 | private static final Log LOGGER = LogFactory.getLog(AbstractTransferTask.class); 31 | 32 | 33 | public String call() throws Exception { 34 | Long startTime = System.currentTimeMillis(); 35 | LOGGER.info("开始执行ID为"+getId()+"的迁移任务"); 36 | int pageNo = 0; 37 | int count = 0; 38 | TransferDataSource insertDataSource = config.getDataSources(targetTableDataSourceRef); 39 | Connection insertConn = insertDataSource.getConnection(); 40 | insertConn.setAutoCommit(true); 41 | while (true) { 42 | List> resultData = getDataFromSrc(srcs, pageNo, pageSize); 43 | 44 | List dataList = insertBeanGenerator(resultData, beanClass,pageNo); 45 | 46 | for (int i = 0; i < dataList.size(); i++) { 47 | //LOGGER.debug("ID：" + id+"任务，执行插入SQL，插入数据"+dataList.get(i).toString()); 48 | if (!insertData(insertConn, dataList.get(i), insertSQL, beanClass)) { 49 | String errorMsg = "任务ID：" + id + "的数据：" + dataList.get(i).toString() + "插入到表：" + targetTableName + "表中失败！"; 50 | LOGGER.error(errorMsg); 51 | ErrorMsgProvider.putErrorMsgData(new ErrorMsg(targetTableName, errorMsg)); 52 | } 53 | } 54 | // 退出条件 55 | pageNo = pageNo + 1; 56 | count = count + resultData.size(); 57 | if (pageNo == totalPage || count == totalDataCount) { 58 | break; 59 | } 60 | } 61 | DbUtils.close(insertConn); 62 | //LOGGER.info("完成" + targetTableName + "表的数据迁移！"); 63 | System.out.println(targetTableName +"任务完成时间："+(System.currentTimeMillis() - startTime) + "毫秒"); 64 | return "完成" + targetTableName + "表的数据迁移！"; 65 | } 66 | 67 | private List insertBeanGenerator(List> resultData, Class beanClass,int pageNo) { 68 | List dataList = new ArrayList(); 69 | for (int i = 0; i < resultData.size(); i++) { 70 | Map beanValue = resultData.get(i); 71 | try { 72 | T bean = beanClass.newInstance(); 73 | TableChanges change = null; 74 | String filedName = null; 75 | for (String key : beanValue.keySet()) { 76 | filedName = columnToFieldName.get(key.toUpperCase()); 77 | if (changes.containsKey(filedName)) { 78 | change = changes.get(filedName); 79 | } 80 | setBeanValue(beanClass, bean, beanValue.get(key), filedName, change); 81 | change = null; 82 | } 83 | dataList.add(bean); 84 | } catch (Exception e) { 85 | String errorMsg = "任务ID：" + id + "的数据：" + beanValue.toString() + "生成实体类失败：" + beanClass.getName(); 86 | LOGGER.error(errorMsg,e); 87 | } 88 | } 89 | if(dataList.size() != resultData.size()){ 90 | int tmpCount = resultData.size() - dataList.size(); 91 | LOGGER.info("Id:"+getId()+"的任务在获取第"+pageNo+"页数据的后，有"+tmpCount + "条记录转化为实体类失败，具体请查看错误信息"); 92 | } 93 | return dataList; 94 | } 95 | 96 | @SuppressWarnings({ "rawtypes", "unchecked" }) 97 | public void setBeanValue(Class clazz, Object obj, Object value, String propertyName, TableChanges change) 98 | throws Exception { 99 | if (change != null) { 100 | TypeChangeHander hander = config.getTypeHandler(change.getHandlerRef()); 101 | value = hander.changeType(value, Class.forName(change.getSrcType()), Class.forName(change.getTargetType())); 102 | } 103 | String methodName; 104 | try { 105 | Field field = clazz.getDeclaredField(propertyName); 106 | methodName = ClassUtils.getBeanMethodName("set", propertyName, clazz); 107 | Method method = clazz.getMethod(methodName, field.getType()); 108 | method.setAccessible(true); 109 | method.invoke(obj, value); 110 | } catch (Exception e) { 111 | throw new Exception("表实体类" + clazz.getName() + "注入属性值失败", e); 112 | } 113 | } 114 | 115 | protected List> doGetDataFromSrc(SrcData srcData, int pageNo, int pageSize) { 116 | List> resultMap = new ArrayList>(); 117 | QueryRunner runner = new QueryRunner(); 118 | String querySQL = srcData.getQuerySql(); 119 | String dbType = config.getDataSources(srcData.getSrcDataSourceRef()).getDBType(); 120 | querySQL = getQuerySQL(dbType, querySQL, pageNo, pageSize); 121 | LOGGER.info("ID："+id+"的任务执行查询SQL："+querySQL); 122 | Connection srcConn = config.getDataSources(srcData.getSrcDataSourceRef()).getConnection(); 123 | try { 124 | resultMap = runner.query(srcConn, querySQL, new MapListHandler()); 125 | } catch (SQLException e) { 126 | String errorMsg = "任务ID：" + id + "的数据：" + querySQL + "获取源数据失败：分页信息为：pageNo:"+ pageNo+"pageSize:"+pageSize; 127 | LOGGER.error(errorMsg,e); 128 | ErrorMsgProvider.putErrorMsgData(new ErrorMsg(targetTableName, errorMsg)); 129 | return resultMap; 130 | } 131 | return resultMap; 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/task/TransferTask.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.task; 2 | 3 | import java.sql.Connection; 4 | import java.util.List; 5 | import java.util.Map; 6 | import java.util.concurrent.Callable; 7 | 8 | import com.xtonic.config.SrcData; 9 | 10 | public interface TransferTask extends Callable { 11 | /** 12 | * 从源表中获取要被插入的对象； 13 | * @return 14 | */ 15 | List> getDataFromSrc(List srcDatas, int pageNo, int pageSize); 16 | /** 17 | * 将一条数据插入到数据库里面； 18 | * @param t 19 | * @return 20 | */ 21 | boolean insertData(Connection conn,Object obj,String insertSql,Class clazz); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/task/TransferTaskConfig.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.task; 2 | 3 | import java.sql.Connection; 4 | import java.sql.ResultSet; 5 | import java.sql.Statement; 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import org.apache.commons.dbutils.DbUtils; 11 | import org.apache.commons.logging.Log; 12 | import org.apache.commons.logging.LogFactory; 13 | 14 | import com.xtonic.config.SrcData; 15 | import com.xtonic.config.TableChanges; 16 | import com.xtonic.config.TransferTableDefiniton; 17 | import com.xtonic.context.TransferContext; 18 | import com.xtonic.dataSource.TransferDataSource; 19 | import com.xtonic.exception.GetTotalCountException; 20 | 21 | public abstract class TransferTaskConfig implements TransferTask { 22 | private static final Log LOGGER = LogFactory.getLog(TransferTaskConfig.class); 23 | 24 | protected String id; 25 | @SuppressWarnings("rawtypes") 26 | protected Class beanClass; 27 | protected TransferContext config; 28 | protected String targetTableName; 29 | protected String targetTableDataSourceRef; 30 | protected List fieldNames = new ArrayList(); 31 | protected Map changes; 32 | protected List srcs; 33 | protected Map columnToFieldName; 34 | protected int totalDataCount =0; 35 | protected int totalPage = 0; 36 | protected int pageSize = 0; 37 | protected String insertSQL = null; 38 | 39 | 40 | public Map getColumnToFieldName() { 41 | return columnToFieldName; 42 | } 43 | public void setColumnToFieldName(Map columnToFieldName) { 44 | this.columnToFieldName = columnToFieldName; 45 | } 46 | public String getId() { 47 | return id; 48 | } 49 | public void setId(String id) { 50 | this.id = id; 51 | } 52 | @SuppressWarnings("rawtypes") 53 | public Class getBeanClass() { 54 | return beanClass; 55 | } 56 | @SuppressWarnings("rawtypes") 57 | public void setBeanClass(Class beanClass) { 58 | this.beanClass = beanClass; 59 | } 60 | public TransferContext getConfig() { 61 | return config; 62 | } 63 | public void setConfig(TransferContext config) { 64 | this.config = config; 65 | } 66 | public String getTargetTableName() { 67 | return targetTableName; 68 | } 69 | public void setTargetTableName(String targetTableName) { 70 | this.targetTableName = targetTableName; 71 | } 72 | public String getTargetTableDataSourceRef() { 73 | return targetTableDataSourceRef; 74 | } 75 | public void setTargetTableDataSourceRef(String targetTableDataSourceRef) { 76 | this.targetTableDataSourceRef = targetTableDataSourceRef; 77 | } 78 | public Map getChanges() { 79 | return changes; 80 | } 81 | public void setChanges(Map changes) { 82 | this.changes = changes; 83 | } 84 | public List getFieldNames() { 85 | return fieldNames; 86 | } 87 | public void setFieldNames(List fieldNames) { 88 | this.fieldNames = fieldNames; 89 | } 90 | public List getSrcs() { 91 | return srcs; 92 | } 93 | public void setSrcs(List srcs) { 94 | this.srcs = srcs; 95 | } 96 | public int getTotalDataCount() { 97 | return totalDataCount; 98 | } 99 | public void setTotalDataCount(int totalDataCount) { 100 | this.totalDataCount = totalDataCount; 101 | } 102 | 103 | public int getTotalPage() { 104 | return totalPage; 105 | } 106 | public void setTotalPage(int totalPage) { 107 | this.totalPage = totalPage; 108 | } 109 | public int getPageSize() { 110 | return pageSize; 111 | } 112 | public void setPageSize(int pageSize) { 113 | this.pageSize = pageSize; 114 | } 115 | public String getInsertSQL() { 116 | return insertSQL; 117 | } 118 | public void setInsertSQL(String insertSQL) { 119 | this.insertSQL = insertSQL; 120 | } 121 | /** 122 | * 获取待复制数据的总量 123 | * 124 | * @param list 125 | * @return 126 | * @throws GetTotalCountException 127 | */ 128 | protected int getCount(List list) throws GetTotalCountException { 129 | String queryCountSql = null; 130 | int count = 0; 131 | for (SrcData src : list) { 132 | int tmp = count; 133 | queryCountSql = "SELECT COUNT(1) FROM ( " + src.getQuerySql() + " ) $1"; 134 | count = doQueryCount(queryCountSql, config.getDataSources(src.getSrcDataSourceRef())); 135 | //如果该数据源是主数据源，则总数量以主数据源为准；否则就取数据量最大的数据源的数据作为总数据量； 136 | if(src.getIsMainSrc()){ 137 | return count; 138 | } 139 | count = tmp > count ? tmp : count; 140 | } 141 | return count; 142 | } 143 | 144 | protected int doQueryCount(String queryCountSql, TransferDataSource transferDataSource) 145 | throws GetTotalCountException { 146 | Connection conn = transferDataSource.getConnection(); 147 | Statement stat = null; 148 | ResultSet result = null; 149 | try { 150 | stat = conn.createStatement(); 151 | result = stat.executeQuery(queryCountSql); 152 | result.next(); 153 | int tmpCount = result.getInt(1); 154 | LOGGER.info("执行SQL语句："+ queryCountSql +"获取记录条数："+ tmpCount+"条记录"); 155 | return tmpCount; 156 | } catch (Exception e) { 157 | throw new GetTotalCountException("取總數據量失敗,SQL:" + queryCountSql, e); 158 | } finally { 159 | DbUtils.closeQuietly(conn, stat, result); 160 | } 161 | } 162 | /** 163 | * 获取插入数据的SQL语句 164 | * @param fieldNames2 165 | * 166 | * @return 167 | */ 168 | protected String getInserSQL(String tableName, List fieldNames) { 169 | String columNamesStr = ""; 170 | String valuesStr = ""; 171 | for (String fieldName : fieldNames) { 172 | columNamesStr = columNamesStr + fieldName + ","; 173 | valuesStr = valuesStr + "? ,"; 174 | } 175 | columNamesStr = columNamesStr.substring(0, columNamesStr.length() - 1); 176 | valuesStr = valuesStr.substring(0, valuesStr.length() - 1); 177 | 178 | if (columNamesStr.length() == 0) { 179 | throw new RuntimeException("初始化Context出错：表名：" + tableName + " 对应的实体类没有定义任何属性"); 180 | } 181 | 182 | String insertSQL = "INSERT INTO " + tableName + " ( " + columNamesStr + " ) VALUES ( " + valuesStr + " )"; 183 | 184 | return insertSQL; 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/task/TransferTaskFactory.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.task; 2 | 3 | import java.lang.reflect.Field; 4 | import java.util.ArrayList; 5 | import java.util.HashMap; 6 | import java.util.HashSet; 7 | import java.util.Hashtable; 8 | import java.util.Iterator; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import org.apache.commons.logging.Log; 13 | import org.apache.commons.logging.LogFactory; 14 | 15 | import com.xtonic.config.TableChanges; 16 | import com.xtonic.config.TransferTableDefiniton; 17 | import com.xtonic.context.TransferContext; 18 | import com.xtonic.exception.GetTotalCountException; 19 | import com.xtonic.task.impl.DefaultTransferTask; 20 | 21 | public class TransferTaskFactory { 22 | private static final Log LOGGER = LogFactory.getLog(TransferTaskFactory.class); 23 | @SuppressWarnings({ "unchecked", "rawtypes" }) 24 | public static TransferTask creatTask(TransferContext config,TransferTableDefiniton tableDefinition){ 25 | LOGGER.info("开始初始化任务ID为：" +tableDefinition.getId()+"的任务" ); 26 | TransferTaskConfig task; 27 | String taskImplClassName = tableDefinition.getTaskImplClass(); 28 | if(taskImplClassName == null){ 29 | task = new DefaultTransferTask(); 30 | }else{ 31 | try { 32 | Class clazz = Class.forName(taskImplClassName); 33 | task = (TransferTaskConfig) clazz.newInstance(); 34 | } catch (Exception e) { 35 | LOGGER.error("任务ID："+tableDefinition.getId() + "指定的自定义数据迁移任务实现类不存在！",e); 36 | throw new RuntimeException("任务ID："+tableDefinition.getId() + "指定的自定义数据迁移任务实现类不存在！",e); 37 | } 38 | } 39 | Class tableBeanClazz = null; 40 | try { 41 | tableBeanClazz = Class.forName(tableDefinition.getBeanClass()); 42 | } catch (ClassNotFoundException e) { 43 | LOGGER.error("初始化Context出错：表名：" + tableDefinition.getTable() + " 对应的实体类没有定义", e); 44 | throw new RuntimeException("初始化Context出错：表名：" + tableDefinition.getTable() + " 对应的实体类没有定义", e); 45 | } 46 | 47 | List fieldNames = getFieldNameList(tableBeanClazz); 48 | Map changes = getChangesMap(tableDefinition); 49 | Map columnToFieldName = getColumnFieldMap(config, tableDefinition, fieldNames); 50 | 51 | task.setColumnToFieldName(columnToFieldName); 52 | task.setBeanClass(tableBeanClazz); 53 | task.setId(tableDefinition.getId()); 54 | task.setTargetTableName(tableDefinition.getTable()); 55 | task.setTargetTableDataSourceRef(tableDefinition.getTargetDataSourceRef()); 56 | task.setChanges(changes); 57 | task.setFieldNames(fieldNames); 58 | task.setConfig(config); 59 | task.setSrcs(tableDefinition.getSrcs()); 60 | try { 61 | task.setTotalDataCount(task.getCount(tableDefinition.getSrcs())); 62 | } catch (GetTotalCountException e) { 63 | LOGGER.error("ID:"+tableDefinition.getId()+"的任务初始化失败，获取数据总量失败",e); 64 | } 65 | if(task.getTotalDataCount() == 0){ 66 | LOGGER.info("ID:"+tableDefinition.getId()+"的任务的待迁移的数据量为0，请确认是否正确；因为数据源的数据为0，所以不生成迁移任务！"); 67 | return null; 68 | } 69 | 70 | Integer pageSize = Integer.valueOf(config.getProperty("pageSize")==null ? "0" : (String)config.getProperty("pageSize")); 71 | //如果没有配置单页容量，则默认为6000； 72 | if(pageSize == 0){ 73 | task.setPageSize(6000); 74 | } 75 | task.setPageSize(pageSize); 76 | task.setTotalPage(getTotalPage(task.getTotalDataCount(),task.getPageSize())); 77 | LOGGER.debug("ID："+tableDefinition.getId() + "的任务的数据按照pageSize："+task.getPageSize() +"分为"+task.getTotalPage()+"页"); 78 | 79 | task.setInsertSQL(task.getInserSQL(tableDefinition.getTable(),fieldNames)); 80 | LOGGER.info("ID：" + tableDefinition.getId()+"任务，执行插入SQL为："+task.getInsertSQL()); 81 | 82 | 83 | 84 | return task; 85 | } 86 | 87 | private static List getFieldNameList(Class tableBeanClazz) { 88 | List fieldNames = new ArrayList(); 89 | Field[] fields = tableBeanClazz.getDeclaredFields(); 90 | for(Field field : fields){ 91 | fieldNames.add(field.getName()); 92 | } 93 | return fieldNames; 94 | } 95 | /** 96 | * 返回一个类型转换器的MAP； 97 | * 98 | * KEY： 需要进行类型转换的属性名； VALUE： 类型转换配置信息 99 | * 100 | */ 101 | private static Map getChangesMap(TransferTableDefiniton tableDefinition) { 102 | Map changes = new Hashtable(); 103 | for(TableChanges change : tableDefinition.getChanges()){ 104 | changes.put(change.getColumnName(), change); 105 | } 106 | LOGGER.debug(tableDefinition.getId()+"任务的类型转化器："+ changes.toString()); 107 | return changes; 108 | } 109 | 110 | /** 111 | * 返回一个 SQL语句中的字段与实体类的属性的对应的关系的MAP 112 | * 113 | * KEY： sql语句中的字段名， VALUE： 实体类中的属性名 114 | */ 115 | private static Map getColumnFieldMap(TransferContext config, TransferTableDefiniton tableDefinition, 116 | List fieldNames) { 117 | Map columnToFieldName = new HashMap(); 118 | HashSet columnFromSql = (HashSet)config.getProperty(tableDefinition.getId()); 119 | Iterator it = columnFromSql.iterator(); 120 | String columnName = null; 121 | while(it.hasNext()){ 122 | columnName = it.next(); 123 | for(String fieldName : fieldNames){ 124 | if(fieldName.equalsIgnoreCase(columnName)){ 125 | columnToFieldName.put(columnName, fieldName); 126 | break; 127 | } 128 | } 129 | } 130 | LOGGER.debug(tableDefinition.getId()+"任务的 SQL语句中的字段与实体类的属性的对应的关系："+ columnToFieldName.toString()); 131 | return columnToFieldName; 132 | } 133 | 134 | private static int getTotalPage(int total,int pageSize) { 135 | //Integer pageSize = Integer.valueOf((String) config.getProperty("pageSize")); 136 | if (total % pageSize == 0) { 137 | return total / pageSize; 138 | } else { 139 | return total / pageSize + 1; 140 | } 141 | } 142 | 143 | 144 | 145 | } 146 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/task/impl/DefaultTransferTask.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.task.impl; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | import com.xtonic.config.SrcData; 7 | import com.xtonic.task.AbstractTransferTask; 8 | 9 | @SuppressWarnings("rawtypes") 10 | public class DefaultTransferTask extends AbstractTransferTask { 11 | 12 | 13 | @SuppressWarnings("unchecked") 14 | public List> getDataFromSrc(List srcDatas, int pageNo, int pageSize) { 15 | return doGetDataFromSrc((SrcData)srcDatas.get(0), pageNo, pageSize); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/task/impl/InsertTask.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.task.impl; 2 | 3 | import java.sql.Connection; 4 | import java.sql.PreparedStatement; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import com.xtonic.config.SrcData; 10 | import com.xtonic.task.AbstractTransferTask; 11 | 12 | public class InsertTask extends AbstractTransferTask { 13 | 14 | public List getDataFromSrc(List srcDatas, int pageNo, int pageSize) { 15 | SrcData srcData = (SrcData) srcDatas.get(0); 16 | Connection conn = config.getDataSources(srcData.getSrcDataSourceRef()).getConnection(); 17 | /*String sql = "INSERT INTO `testA`.`TEST_TABLE_6` (`id`,`c1`,`c2`,`c3`,`c4`,`c5`,`c6`," + "`c7`,`c8`,`c9`," 18 | + "`c10`,`11`,`c12`,`c13`,`c14`,`c15`,`c16`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";*/ 19 | 20 | //String sql ="INSERT INTO src5User (id,inttrested,memo) VALUES (?,?,?)"; 21 | 22 | String sql ="INSERT INTO src3User (id,name,age) VALUES (?,?,?)"; 23 | 24 | 25 | PreparedStatement pst = null; 26 | try { 27 | conn.setAutoCommit(false); 28 | pst = conn.prepareStatement(sql); 29 | for(int i = 1; i <500001; i ++){ 30 | pst.setInt(1, i); 31 | /*pst.setString(2, "TEST_TABLE_1_c1_"+(i+1)); 32 | pst.setString(3, "TEST_TABLE_1_c2_"+(i+1)); 33 | pst.setString(4, "TEST_TABLE_1_c3_"+(i+1)); 34 | pst.setString(5, "TEST_TABLE_1_c4_"+(i+1)); 35 | pst.setString(6, "TEST_TABLE_1_c5_"+(i+1)); 36 | pst.setString(7, "TEST_TABLE_1_c6_"+(i+1)); 37 | pst.setString(8, "TEST_TABLE_1_c7_"+(i+1)); 38 | pst.setString(9, "TEST_TABLE_1_c8_"+(i+1)); 39 | pst.setString(10, "TEST_TABLE_1_c9_"+(i+1)); 40 | pst.setString(11, "TEST_TABLE_1_c10_"+(i+1)); 41 | pst.setString(12, "TEST_TABLE_1_c11_"+(i+1)); 42 | pst.setString(13, "TEST_TABLE_1_c12_"+(i+1)); 43 | pst.setString(14, "TEST_TABLE_1_c13_"+(i+1)); 44 | pst.setString(15, "TEST_TABLE_1_c14_"+(i+1)); 45 | pst.setString(16, "TEST_TABLE_1_c15_"+(i+1)); 46 | pst.setString(17, "TEST_TABLE_1_c16_"+(i+1));*/ 47 | pst.setString(2, "text_mix_name"+i); 48 | pst.setString(3, i+""); 49 | pst.addBatch(); 50 | if(i != 0 &&i % 100000 == 0){ 51 | pst.executeBatch(); 52 | pst.clearBatch(); 53 | conn.commit(); 54 | } 55 | System.out.println("插入"+i+"条记录"); 56 | } 57 | pst.executeBatch(); 58 | pst.clearBatch(); 59 | conn.commit(); 60 | } catch (Exception e) { 61 | e.printStackTrace(); 62 | } 63 | return new ArrayList>(); 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/task/impl/MoreSrcsTransferTask.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.task.impl; 2 | 3 | import java.sql.Connection; 4 | import java.sql.SQLException; 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import org.apache.commons.dbutils.DbUtils; 11 | import org.apache.commons.dbutils.QueryRunner; 12 | import org.apache.commons.dbutils.handlers.MapListHandler; 13 | import org.apache.commons.logging.Log; 14 | import org.apache.commons.logging.LogFactory; 15 | 16 | import com.xtonic.config.ErrorMsg; 17 | import com.xtonic.config.SrcData; 18 | import com.xtonic.container.ErrorMsgProvider; 19 | import com.xtonic.dataSource.TransferDataSource; 20 | import com.xtonic.exception.GetTotalCountException; 21 | import com.xtonic.task.AbstractTransferTask; 22 | 23 | @SuppressWarnings("rawtypes") 24 | public class MoreSrcsTransferTask extends AbstractTransferTask { 25 | private static final Log LOGGER = LogFactory.getLog(MoreSrcsTransferTask.class); 26 | 27 | private SrcData mainSrcData; 28 | private int selectDataCount = 0; 29 | private Map conns = new HashMap(); 30 | 31 | @SuppressWarnings("unchecked") 32 | public List getDataFromSrc(List srcDatas, int pageNo, int pageSize) { 33 | long st = System.currentTimeMillis(); 34 | List> resultMap = new ArrayList>(); 35 | SrcData mainSrcData = selectMainSrc(srcDatas, totalDataCount); 36 | resultMap = doGetDataFromSrc(mainSrcData, pageNo, pageSize); 37 | srcDatas.remove(mainSrcData); 38 | for (int i = 0; i < resultMap.size(); i++) { 39 | Map record = resultMap.get(i); 40 | SrcData subSrcData = null; 41 | String subQuerySql = ""; 42 | List> subResultMap = new ArrayList>(); 43 | for (int j = 0; j < srcDatas.size(); j++) { 44 | subSrcData = (SrcData) srcDatas.get(j); 45 | subQuerySql = subSrcData.getQuerySql(); 46 | subQuerySql = createdSubQuerySql(record, subQuerySql, mainSrcData.getKeyColumn(), 47 | subSrcData.getKeyColumn()); 48 | QueryRunner runner = new QueryRunner(); 49 | Connection srcConn = getConns(subSrcData.getSrcDataSourceRef()); 50 | // 51 | try { 52 | subResultMap = runner.query(srcConn, subQuerySql, new MapListHandler()); 53 | } catch (SQLException e) { 54 | //resultMap.remove(i); 55 | String errorMsg = "任务ID：" + id + "的数据：" + subQuerySql + "获取源数据失败"; 56 | LOGGER.error(errorMsg, e); 57 | ErrorMsgProvider.putErrorMsgData(new ErrorMsg(targetTableName, errorMsg)); 58 | } 59 | if (subResultMap.size() > 1) { 60 | LOGGER.error("关键字配置错误，一条主记录，对应俩条记录: 查询SQL为：" + subQuerySql); 61 | //resultMap.remove(i); 62 | continue; 63 | } 64 | record.putAll(subResultMap.get(0)); 65 | 66 | } 67 | } 68 | System.out.println(getId() + "获取数据耗时：" +(System.currentTimeMillis() - st)); 69 | selectDataCount = selectDataCount + resultMap.size(); 70 | closeConnection(selectDataCount); 71 | return resultMap; 72 | } 73 | 74 | private void closeConnection(int selectDataCount2) { 75 | if(selectDataCount == totalDataCount){ 76 | for(String key : conns.keySet()){ 77 | DbUtils.closeQuietly(conns.get(key)); 78 | } 79 | } 80 | } 81 | 82 | private Connection getConns(String srcDataSourceRef) { 83 | Connection conn = conns.get(srcDataSourceRef); 84 | if(conn == null){ 85 | conn = config.getDataSources(srcDataSourceRef).getConnection(); 86 | } 87 | conns.put(srcDataSourceRef, conn); 88 | return conn; 89 | } 90 | 91 | private SrcData selectMainSrc(List srcDatas, int totalCount) { 92 | if (mainSrcData != null) { 93 | return mainSrcData; 94 | } 95 | try { 96 | for (int i = 0; i < srcDatas.size(); i++) { 97 | SrcData srcData = (SrcData) srcDatas.get(i); 98 | //配置文件中指定的主语句； 99 | if(srcData.getIsMainSrc()){ 100 | this.mainSrcData = srcData; 101 | return srcData; 102 | } 103 | //如果没有配置，则获取每一个数据源的总数据量，当与总数据量一直时候，就当选为主数据源； 104 | String queryCountSql = "SELECT COUNT(1) FROM ( " + srcData.getQuerySql() + " ) $1"; 105 | TransferDataSource dataSource = config.getDataSources(srcData.getSrcDataSourceRef()); 106 | int count = doQueryCount(queryCountSql, dataSource); 107 | if (count == totalCount) { 108 | this.mainSrcData = srcData; 109 | return srcData; 110 | } 111 | } 112 | } catch (GetTotalCountException e) { 113 | e.printStackTrace(); 114 | return null; 115 | } 116 | return null; 117 | } 118 | 119 | private String createdSubQuerySql(Map record, String subQuerySql, String mainKeyColumn, 120 | String subKeyColumn) { 121 | String[] mainKeyColumns = mainKeyColumn.split(","); 122 | String[] subKeyColumns = subKeyColumn.split(","); 123 | String where = " where 1 =1 "; 124 | for (int i = 0; i < mainKeyColumns.length; i++) { 125 | where = where + "And " + subKeyColumns[i] + " = " + record.get(mainKeyColumns[i].toUpperCase()); 126 | } 127 | return (subQuerySql + where).toUpperCase(); 128 | } 129 | 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/type/TypeChangeHander.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.type; 2 | 3 | public interface TypeChangeHander { 4 | T changeType(Object value, Class SrcTypeClazz, Class targetTypeClazz); 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/type/impl/StringToIntegerHandler.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.type.impl; 2 | 3 | import com.xtonic.type.TypeChangeHander; 4 | 5 | public class StringToIntegerHandler implements TypeChangeHander { 6 | 7 | public Integer changeType(Object value, Class SrcTypeClazz, Class targetTypeClazz) { 8 | if(SrcTypeClazz != String.class){ 9 | return null; 10 | } 11 | return Integer.valueOf((String)value); 12 | } 13 | 14 | 15 | 16 | 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/utils/ClassUtils.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.utils; 2 | 3 | import java.lang.reflect.Field; 4 | import java.lang.reflect.InvocationTargetException; 5 | import java.lang.reflect.Method; 6 | 7 | import com.xtonic.config.TableChanges; 8 | import com.xtonic.entity.User; 9 | 10 | public class ClassUtils { 11 | public static String getBeanMethodName(String prefix,String fieldName,Class clazz) throws NoSuchFieldException, SecurityException{ 12 | Field field = clazz.getDeclaredField(fieldName); 13 | if(field.getType() == boolean.class &&fieldName.toUpperCase().startsWith("IS")){ 14 | String tmpStr = fieldName.substring(2); 15 | return fieldName; 16 | } 17 | 18 | if(Character.isUpperCase(fieldName.charAt(0))){ 19 | return prefix + fieldName; 20 | } 21 | 22 | if(Character.isUpperCase(fieldName.charAt(1))){ 23 | return prefix + fieldName; 24 | } 25 | 26 | return prefix + fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1); 27 | 28 | } 29 | 30 | public static void setBeanValue(Class clazz, Object obj, Object value,String propertyName,TableChanges change) throws Exception{ 31 | if(change != null){ 32 | value = dealChange(value,Class.forName(change.getSrcType()) , Class.forName(change.getTargetType())); 33 | } 34 | String methodName; 35 | try { 36 | Field field = clazz.getDeclaredField(propertyName); 37 | methodName = getBeanMethodName("set", propertyName, clazz); 38 | Method method = clazz.getMethod(methodName, field.getType()); 39 | method.setAccessible(true); 40 | method.invoke(obj, value); 41 | } catch (Exception e) { 42 | e.printStackTrace(); 43 | throw new Exception("表实体类"+clazz.getName()+"注入属性值失败",e); 44 | } 45 | } 46 | 47 | private static V dealChange(Object value,Class srcTypeClazz , Class targetTypeClazz){ 48 | if(srcTypeClazz == String.class && targetTypeClazz == Integer.class){ 49 | return (V) Integer.valueOf((String)value); 50 | } 51 | return null; 52 | } 53 | 54 | public static void main(String[] args) throws Exception { 55 | User tmp = new User(); 56 | 57 | setBeanValue(User.class,tmp,"1","username",null); 58 | 59 | System.out.println(tmp.toString()); 60 | 61 | 62 | 63 | } 64 | 65 | 66 | 67 | } 68 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/utils/PageQuerySQLTemplate.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.utils; 2 | 3 | public class PageQuerySQLTemplate { 4 | 5 | public static String creatPageableQuerSql(String dataBaseType,String querySql){ 6 | if("mysql".equalsIgnoreCase(dataBaseType)){ 7 | return creatMySqlQuerySql(querySql); 8 | } 9 | if("sqlserver".equalsIgnoreCase(dataBaseType)){ 10 | return querySql; 11 | } 12 | if("oracle".equalsIgnoreCase(dataBaseType)){ 13 | return querySql; 14 | } 15 | return querySql; 16 | }; 17 | 18 | 19 | public static String creatMySqlQuerySql(String querySql){ 20 | String tmpQuerySql = querySql.toUpperCase(); 21 | StringBuilder sql = new StringBuilder(); 22 | // sql.append(tmpQuerySql.substring(0, tmpQuerySql.indexOf("FROM"))) 23 | sql.append("SELECT * ") 24 | .append(" FROM (") 25 | .append(tmpQuerySql) 26 | .append(")$1") 27 | .append(" limit {0},{1}"); 28 | return sql.toString(); 29 | } 30 | 31 | 32 | 33 | 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/utils/Xml2JsonUtil.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.utils; 2 | 3 | import java.io.InputStream; 4 | import java.util.Map; 5 | 6 | import net.sf.json.JSONObject; 7 | import net.sf.json.xml.XMLSerializer; 8 | 9 | public class Xml2JsonUtil { 10 | public static String xml2JSON(String xml){ 11 | return new XMLSerializer().read(xml).toString(); 12 | } 13 | 14 | @SuppressWarnings("unchecked") 15 | public static Map xml2Map(String xml){ 16 | JSONObject json = (JSONObject) new XMLSerializer().read(xml); 17 | return (Map) json; 18 | } 19 | 20 | @SuppressWarnings("unchecked") 21 | public static Map json2XML(InputStream inputStream) { 22 | JSONObject json = (JSONObject) new XMLSerializer().readFromStream(inputStream); 23 | return (Map) json; 24 | } 25 | public static String json2XML(String json){ 26 | JSONObject jobj = JSONObject.fromObject(json); 27 | String xml = new XMLSerializer().write(jobj); 28 | return xml; 29 | } 30 | 31 | // 测试 32 | public static void main(String[] args) { 33 | 34 | String xml = "" + "" + "" + "MapGuideddddddd" 35 | + "true" + "" + "ddd" 36 | + "" + "" + "" + "ccc" + "ggg" 37 | + "" + "aaa" + "" + "" + "" 38 | + "" + "" + "33333333" + "" + "" + "444" + "" + ""; 39 | 40 | System.out.println(xml); 41 | 42 | System.out.println(xml2JSON(xml)); 43 | Map tmpMap = xml2Map(xml); 44 | System.out.println(tmpMap.toString()); 45 | 46 | // System.out.println(Xml2JsonUtil 47 | // .xml2JSON("" + "" + "" + "MapGuideddddddd" 48 | // + "true" + "" + "ddd" 49 | // + "" + "" + "" + "ccc" + "ggg" 50 | // + "" + "aaa" + "" + "" + "" 51 | // + "" + "" + "33333333" + "" + "" + "444" + "" + "")); 52 | } 53 | 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/validator/DataSourceValidator.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.validator; 2 | 3 | import com.xtonic.config.DataSourceDeifiniton; 4 | 5 | public interface DataSourceValidator { 6 | String dataSourceDefinitionValidator(DataSourceDeifiniton definition); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/validator/DefinitonValidator.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.validator; 2 | 3 | public interface DefinitonValidator extends TableValidator,DataSourceValidator,TypeHandlerValidator { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/validator/TableValidator.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.validator; 2 | 3 | import com.xtonic.config.TransferTableDefiniton; 4 | 5 | public interface TableValidator { 6 | String tableDefinitionValidator(TransferTableDefiniton definition); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/validator/TypeHandlerValidator.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.validator; 2 | 3 | import com.xtonic.config.TypeHandlerDefinition; 4 | 5 | public interface TypeHandlerValidator { 6 | String typeHandlerDeifnitonValidator(TypeHandlerDefinition definition); 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/validator/impl/DefaultDefinitionValidator.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.validator.impl; 2 | 3 | import java.lang.reflect.Field; 4 | import java.util.HashSet; 5 | import java.util.List; 6 | 7 | import org.apache.commons.lang.StringUtils; 8 | import org.apache.commons.logging.Log; 9 | import org.apache.commons.logging.LogFactory; 10 | 11 | import com.xtonic.config.DataSourceDeifiniton; 12 | import com.xtonic.config.SrcData; 13 | import com.xtonic.config.TransferTableDefiniton; 14 | import com.xtonic.config.TypeHandlerDefinition; 15 | import com.xtonic.context.TransferContext; 16 | import com.xtonic.type.TypeChangeHander; 17 | import com.xtonic.validator.DefinitonValidator; 18 | 19 | public class DefaultDefinitionValidator implements DefinitonValidator { 20 | private static final Log LOGGER = LogFactory.getLog(DefaultDefinitionValidator.class); 21 | private static final String QUERY_SQL_FLAG = "SELECT"; 22 | 23 | private TransferContext context; 24 | 25 | public DefaultDefinitionValidator(TransferContext context ){ 26 | this.context = context; 27 | } 28 | 29 | public String tableDefinitionValidator(TransferTableDefiniton definition) { 30 | LOGGER.info("开始对ID为："+ definition.getId() +"的任务进行校验！"); 31 | String errorMsg = null; 32 | Class tableEntityClazz = null; 33 | try { 34 | tableEntityClazz = Class.forName(definition.getBeanClass()); 35 | } catch (ClassNotFoundException e) { 36 | errorMsg = "任务ID：" + definition.getId() + "的实体类未找到"+ definition.getBeanClass(); 37 | return errorMsg; 38 | } 39 | List srcList = definition.getSrcs(); 40 | HashSet tmpDataSrcList = new HashSet(); 41 | String querySql = null; 42 | HashSet columnsFromSrcSQL = new HashSet(); 43 | for (SrcData src : srcList) { 44 | if(context.getDataSources(src.getSrcDataSourceRef())== null){ 45 | errorMsg = "任务ID：" + definition.getId() + "的数据源："+ src.getSrcDataSourceRef() +"未被定义"; 46 | return errorMsg; 47 | } 48 | if (!tmpDataSrcList.add(src.getSrcDataSourceRef())) { 49 | errorMsg = "数据源：" + src.getSrcDataSourceRef() + " 重复，有多个数据源来自同一个库" 50 | + "请将他们归并未下面格式的SQL： SELECT A.COLUMN1,A.COLUMN2,B.COLUMN1 " 51 | + "FROM TABLE1 A LEFT JOIN TABLE2 B ON (A.KEY = B.KEY)"; 52 | return errorMsg; 53 | } 54 | // 1：判断主键ID，是否设置； 55 | if (src.getKeyColumn() == null || src.getKeyColumn().trim().length() == 0) { 56 | errorMsg = "数据源：" + src.getQuerySql() + " 的数据源未设置keyColumn的值"; 57 | return errorMsg; 58 | } 59 | querySql = src.getQuerySql(); 60 | querySql = querySql.trim().toUpperCase(); 61 | if (querySql == null || !querySql.startsWith(QUERY_SQL_FLAG)) { 62 | errorMsg = "table_id：" + definition.getId() + " 的数据源的sql不是查询SQL"; 63 | return errorMsg; 64 | } 65 | String columnStr = querySql.substring(querySql.indexOf(QUERY_SQL_FLAG) + QUERY_SQL_FLAG.length(), 66 | querySql.indexOf("FROM")); 67 | String[] columns = columnStr.split(","); 68 | String tmpStr = null; 69 | for (int i = 0; i < columns.length; i++) { 70 | tmpStr = columns[i].trim(); 71 | tmpStr = tmpStr.substring(tmpStr.indexOf(".") + 1); 72 | if ((tmpStr.indexOf("AS")) != -1) { 73 | tmpStr = tmpStr.substring(tmpStr.indexOf("AS") + "AS".length()).trim(); 74 | } 75 | if (columnsFromSrcSQL.contains(tmpStr) && (tmpStr.indexOf(src.getKeyColumn().toUpperCase()) == -1)) { 76 | errorMsg = "table_id：" + definition.getId() + " 的数据源的有重复的字段"; 77 | return errorMsg; 78 | } 79 | columnsFromSrcSQL.add(tmpStr); 80 | } 81 | 82 | // config.setParameter(definition.getId(), columnsFromSrcSQL.clone()); 83 | context.setParameter(definition.getId(), columnsFromSrcSQL.clone()); 84 | 85 | // 判断keyColumn字段是否被 包含在SELECT语句里面 86 | String[] keColumns = src.getKeyColumn().split(","); 87 | for (int j = 0; j < keColumns.length; j++) { 88 | if (!columnsFromSrcSQL.contains(keColumns[j].trim().toUpperCase())) { 89 | errorMsg = "数据源SQL语句：" + src.getQuerySql() + " 的数据源未包含keyColumn的字段"; 90 | return errorMsg; 91 | } 92 | } 93 | 94 | } 95 | Field[] fields = tableEntityClazz.getDeclaredFields(); 96 | for (int k = 0; k < fields.length; k++) { 97 | if (columnsFromSrcSQL.contains(fields[k].getName().toUpperCase())) { 98 | columnsFromSrcSQL.remove(fields[k].getName().toUpperCase()); 99 | } 100 | } 101 | if (!columnsFromSrcSQL.isEmpty()) { 102 | errorMsg = "任务ID：" + definition.getId() + "的sql语句中的字段："+columnsFromSrcSQL.toString() + "未在实体类中定义；请检查" ; 103 | return errorMsg; 104 | } 105 | LOGGER.info("开始对ID为："+ definition.getId() +"的任务的校验结束！"); 106 | return errorMsg; 107 | } 108 | 109 | 110 | public String dataSourceDefinitionValidator(DataSourceDeifiniton definition) { 111 | LOGGER.info("开始对ID为："+ definition.getId() +"的数据源进行校验！"); 112 | if(context.getDataSources(definition.getId()) != null){ 113 | return "DataSource ID：" + definition.getId() + " 的数据源 ID重复"; 114 | } 115 | LOGGER.info("开始对ID为："+ definition.getId() +"的数据源校验结束！"); 116 | return null; 117 | } 118 | 119 | 120 | public String typeHandlerDeifnitonValidator(TypeHandlerDefinition definition) { 121 | LOGGER.info("开始对ID为："+ definition.getHandlerid() +"的类型转换器进行校验！"); 122 | if (StringUtils.isEmpty(definition.getHandlerid())) { 123 | return "有类型转换器 的ID没有定义或为空字符串，请检查typehandlers节点的内容"; 124 | } 125 | if (StringUtils.isEmpty(definition.getHandlerClass())) { 126 | return "有类型转换器 的实现类没有定义或为空字符串，请检查typehandlers节点的内容"; 127 | } 128 | Class clazz = null; 129 | try { 130 | clazz = Class.forName(definition.getHandlerClass()); 131 | } catch (ClassNotFoundException e) { 132 | return "没有找到ID：" + definition.getHandlerid() + "的类型转换器的实现类；"; 133 | } 134 | 135 | if (!TypeChangeHander.class.isAssignableFrom(clazz)) { 136 | return "ID：" + definition.getHandlerid() + "的类型转换器的实现类没有显示TypeChangeHander接口"; 137 | } 138 | if((context.getTypeHandler(definition.getHandlerid())!= null)){ 139 | return "类型处理器 ID：" + definition.getHandlerid() + "重复"; 140 | } 141 | LOGGER.info("开始对ID为："+ definition.getHandlerid() +"的类型转换器校验结束！"); 142 | return null; 143 | } 144 | 145 | 146 | } 147 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/xmlDefinition/DefaultLoadDefinition.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.xmlDefinition; 2 | 3 | import java.io.IOException; 4 | import java.util.ArrayList; 5 | import java.util.Collection; 6 | import java.util.HashMap; 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | import org.apache.commons.logging.Log; 11 | import org.apache.commons.logging.LogFactory; 12 | 13 | import com.xtonic.config.Config; 14 | import com.xtonic.config.DataSourceDeifiniton; 15 | import com.xtonic.config.SrcData; 16 | import com.xtonic.config.TableChanges; 17 | import com.xtonic.config.TransferTableDefiniton; 18 | import com.xtonic.config.TypeHandlerDefinition; 19 | import com.xtonic.context.RegisterFactory; 20 | import com.xtonic.context.TransferContext; 21 | import com.xtonic.resources.Resource; 22 | import com.xtonic.utils.Xml2JsonUtil; 23 | import com.xtonic.validator.DefinitonValidator; 24 | import com.xtonic.validator.impl.DefaultDefinitionValidator; 25 | 26 | public class DefaultLoadDefinition implements LoadDefinition { 27 | private static final Log LOGGER = LogFactory.getLog(DefaultLoadDefinition.class); 28 | private static final String IS_LIST = "is_List"; 29 | private static final String IS_MAP = "is_MAP"; 30 | private static final String DATASOURCES_KEY = "dataSources"; 31 | private static final String TABLES_KEY = "tables"; 32 | private static final String PROPERTIES_KEY = "properties"; 33 | private static final String TYPEHANDLERS_KEY = "typehandlers"; 34 | 35 | 36 | private Map xmlMap; 37 | private List tableDefinitonList = new ArrayList(); 38 | private List dataSourceDefinitionList = new ArrayList(); 39 | private List typeHandlerDeinfitionList = new ArrayList(); 40 | 41 | private Config config; 42 | 43 | private DefinitonValidator validator; 44 | private RegisterFactory registerFactory; 45 | 46 | public DefaultLoadDefinition(RegisterFactory registerFactory) { 47 | LOGGER.info("初始化XML任务定义文件加载器完成；" + DefaultLoadDefinition.class.getName() ); 48 | this.registerFactory = registerFactory; 49 | this.validator = new DefaultDefinitionValidator((TransferContext) registerFactory); 50 | } 51 | public void loadXMLDefiniton(Resource resource) throws IOException{ 52 | xmlMap = Xml2JsonUtil.json2XML(resource.getInputStream()); 53 | LOGGER.debug("将任务定义XML文件转换为JSON："+ xmlMap.toString()); 54 | doLoadXMLDefiniton(); 55 | LOGGER.info("开始解析各类型的定义bean并注册到上下文中" ); 56 | doRegistConfig(); 57 | //先进行数据源的注册，再进行迁移任务的注册； 58 | doRegistDataSource(); 59 | doRegistTransferTable(); 60 | doRegistTypeHanlder(); 61 | LOGGER.info("完成解析各类型的定义bean并注册到上下文中" ); 62 | 63 | } 64 | 65 | private void doRegistTypeHanlder() { 66 | for(TypeHandlerDefinition definition : typeHandlerDeinfitionList){ 67 | String errorMsg = null; 68 | if((errorMsg = validator.typeHandlerDeifnitonValidator(definition)) == null){ 69 | this.registerFactory.registerTypeHandler(definition); 70 | }else{ 71 | LOGGER.error("ID：" + definition.getHandlerid() + "的类型转换器定义有误，注册失败;错误信息为："+ errorMsg); 72 | throw new RuntimeException("ID：" + definition.getHandlerid() + "的类型转换器定义有误，注册失败;错误信息为："+ errorMsg); 73 | } 74 | } 75 | } 76 | private void doRegistConfig() { 77 | registerFactory.registConfig(config); 78 | } 79 | 80 | private void doRegistTransferTable() { 81 | for(TransferTableDefiniton definition : tableDefinitonList){ 82 | String errorMsg = null; 83 | if((errorMsg = validator.tableDefinitionValidator(definition)) == null){ 84 | this.registerFactory.registTableDefinition(definition); 85 | }else{ 86 | LOGGER.error("注册任务失败,错误信息：" + errorMsg); 87 | throw new RuntimeException("注册任务失败,错误信息："+ errorMsg); 88 | } 89 | } 90 | } 91 | private void doRegistDataSource() { 92 | for(DataSourceDeifiniton dataSourceDefiniton : dataSourceDefinitionList){ 93 | String errorMsg = null; 94 | if((errorMsg =validator.dataSourceDefinitionValidator(dataSourceDefiniton)) == null){ 95 | this.registerFactory.registDataSources(dataSourceDefiniton); 96 | }else{ 97 | LOGGER.error( errorMsg); 98 | throw new RuntimeException("ID：" + dataSourceDefiniton.getId() + "的数据源定义有误，注册失败;错误信息为："+errorMsg); 99 | } 100 | } 101 | } 102 | private void doLoadXMLDefiniton() { 103 | LOGGER.info("开始将XML文件的定义内容转化对应的bean；"); 104 | buildCommonConfig(); 105 | LOGGER.info("开始将XML文件的定义内容转化对应的bean --- config解析完毕，内容为："+config.toString()); 106 | buildDataSourceDefinitionList(); 107 | LOGGER.info("开始将XML文件的定义内容转化对应的bean --- DataSources解析完毕，内容为："+dataSourceDefinitionList.toString()); 108 | buildTableDefinitionList(); 109 | LOGGER.info("开始将XML文件的定义内容转化对应的bean --- tables解析完毕，内容为："+tableDefinitonList.toString()); 110 | buildTypeChangeHandler(); 111 | LOGGER.info("开始将XML文件的定义内容转化对应的bean --- changeHandlers解析完毕，内容为："+typeHandlerDeinfitionList.toString()); 112 | } 113 | 114 | @SuppressWarnings("unchecked") 115 | private void buildTypeChangeHandler() { 116 | Map parameterMap = null; 117 | List tableDefinitions = (List) xmlMap.get(TYPEHANDLERS_KEY); 118 | for(int i = 0; i < tableDefinitions.size(); i++){ 119 | parameterMap = (Map) tableDefinitions.get(i); 120 | TypeHandlerDefinition typeHnadlerDefinition = new TypeHandlerDefinition(); 121 | typeHnadlerDefinition.setHandlerClass((String)parameterMap.get("@handlerClass")); 122 | typeHnadlerDefinition.setHandlerid((String)parameterMap.get("@handlerid")); 123 | typeHandlerDeinfitionList.add(typeHnadlerDefinition); 124 | } 125 | } 126 | 127 | @SuppressWarnings({ "unchecked", "rawtypes" }) 128 | private void buildCommonConfig() { 129 | Map propertiesMap = new HashMap(); 130 | List propertyList = (List) xmlMap.get(PROPERTIES_KEY); 131 | Map parameterMap = null; 132 | 133 | for(int i = 0 ; i < propertyList.size(); i ++){ 134 | parameterMap = (Map) propertyList.get(i); 135 | if(parameterMap == null){ 136 | continue; 137 | } 138 | String key = (String) parameterMap.get("@name"); 139 | String value = (String) parameterMap.get("@value"); 140 | propertiesMap.put(key, value); 141 | } 142 | this.config = Config.initConfig(propertiesMap); 143 | } 144 | @SuppressWarnings({ "unchecked", "rawtypes" }) 145 | private void buildTableDefinitionList() { 146 | Map parameterMap = null; 147 | List tableDefinitions = (List) xmlMap.get(TABLES_KEY); 148 | for(int i = 0; i < tableDefinitions.size(); i++){ 149 | parameterMap = (Map) tableDefinitions.get(i); 150 | addTableDefinitions(parameterMap); 151 | } 152 | } 153 | 154 | @SuppressWarnings({ "rawtypes", "unchecked" }) 155 | private void addTableDefinitions(Map parameterMap) { 156 | TransferTableDefiniton tableDefinition = new TransferTableDefiniton(); 157 | tableDefinition.setBeanClass((String)parameterMap.get("@beanClass")); 158 | tableDefinition.setId((String)parameterMap.get("@id")); 159 | tableDefinition.setTargetDataSourceRef((String)parameterMap.get("@targetDataSourceRef")); 160 | tableDefinition.setTable((String)parameterMap.get("@table")); 161 | tableDefinition.setTaskImplClass((String)parameterMap.get("@taskImplClass")); 162 | List srcList = new ArrayList(); 163 | Map tmpMap = null; 164 | List tmpList = (List) parameterMap.get("srcs"); 165 | for(int i = 0; i < tmpList.size(); i++){ 166 | tmpMap = (Map) tmpList.get(i); 167 | SrcData srcData = new SrcData(); 168 | srcData.setQuerySql((String)tmpMap.get("@querySql")); 169 | srcData.setSrcDataSourceRef((String)tmpMap.get("@srcDataSourceRef")); 170 | srcData.setKeyColumn((String)tmpMap.get("@keyColumn")); 171 | if("true".equalsIgnoreCase((String)tmpMap.get("@isMainSrc"))){ 172 | srcData.setIsMainSrc(true); 173 | }else{ 174 | srcData.setIsMainSrc(false); 175 | } 176 | srcList.add(srcData); 177 | } 178 | tableDefinition.setSrcs(srcList); 179 | 180 | List changesList = new ArrayList(); 181 | tmpList = (List) parameterMap.get("changes"); 182 | if(tmpList != null){ 183 | for(int i = 0; i < tmpList.size(); i++){ 184 | tmpMap = (Map) tmpList.get(i); 185 | TableChanges changes = new TableChanges(); 186 | changes.setColumnName((String)tmpMap.get("@columnName")); 187 | changes.setSrcType((String)tmpMap.get("@srcType")); 188 | changes.setTargetType((String)tmpMap.get("@targetType")); 189 | changes.setHandlerRef((String)tmpMap.get("@handlerRef")); 190 | changesList.add(changes); 191 | } 192 | } 193 | tableDefinition.setChanges(changesList); 194 | 195 | tableDefinitonList.add(tableDefinition); 196 | 197 | } 198 | @SuppressWarnings({ "unchecked", "rawtypes" }) 199 | private void buildDataSourceDefinitionList() { 200 | String collectionType = checkCollectionType(xmlMap.get(DATASOURCES_KEY)); 201 | Map parameterMap = null; 202 | if(collectionType.equals(IS_LIST)){ 203 | List dataSources = (List) xmlMap.get(DATASOURCES_KEY); 204 | for(int i = 0 ; i < dataSources.size();i++){ 205 | parameterMap = (Map) dataSources.get(i); 206 | addDataSource(parameterMap); 207 | } 208 | return; 209 | } 210 | parameterMap = (Map) xmlMap.get(DATASOURCES_KEY); 211 | addDataSource(parameterMap); 212 | 213 | 214 | } 215 | private void addDataSource(Map parameterMap) { 216 | DataSourceDeifiniton dataSource = new DataSourceDeifiniton(); 217 | dataSource.setDriverClass((String)parameterMap.get("driverClass")); 218 | dataSource.setPassword((String)parameterMap.get("password")); 219 | dataSource.setUrl((String)parameterMap.get("url")); 220 | dataSource.setUsername((String)parameterMap.get("username")); 221 | dataSource.setId((String)parameterMap.get("@id")); 222 | dataSource.setDbType((String)parameterMap.get("@dbType")); 223 | dataSourceDefinitionList.add(dataSource); 224 | } 225 | private String checkCollectionType(Object object) { 226 | if(Collection.class.isAssignableFrom(object.getClass())){ 227 | return IS_LIST; 228 | } 229 | return IS_MAP; 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /src/main/java/com/xtonic/xmlDefinition/LoadDefinition.java: -------------------------------------------------------------------------------- 1 | package com.xtonic.xmlDefinition; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import com.xtonic.config.Config; 7 | import com.xtonic.config.DataSourceDeifiniton; 8 | import com.xtonic.config.TransferTableDefiniton; 9 | import com.xtonic.resources.Resource; 10 | 11 | public interface LoadDefinition { 12 | 13 | void loadXMLDefiniton(Resource resource) throws IOException; 14 | } 15 | -------------------------------------------------------------------------------- /src/main/resources/TransferDefinitions.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | jdbc:mysql://192.168.31.108:3306/test 8 | root 9 | jzh9866 10 | com.mysql.jdbc.Driver 11 | 12 | 13 | jdbc:mysql://192.168.31.108:3306/testA 14 | root 15 | jzh9866 16 | com.mysql.jdbc.Driver 17 | 18 | 19 | 20 | 22 | 23 | 25 | 26 | 27 | 28 | 29 |
30 | 32 | 33 | 35 | 37 | 38 | 39 | 40 | 41 |
42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /src/main/resources/TransferDefinitions.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 9 | 10 | 12 | 13 | 15 | 16 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 29 | 30 | 32 | 33 | 35 | 36 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 60 | 61 | 63 | 64 | 65 | 67 | 68 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 86 | 87 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 97 | 98 | 100 | 101 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | ### set log levels ### 2 | log4j.rootLogger = debug , stdout 3 | 4 | 5 | ### \u8F93\u51FA\u5230\u63A7\u5236\u53F0 ### 6 | log4j.appender.stdout = org.apache.log4j.ConsoleAppender 7 | log4j.appender.stdout.Target = System.out 8 | log4j.appender.stdout.layout = org.apache.log4j.PatternLayout 9 | log4j.appender.stdout.layout.ConversionPattern = %d{ABSOLUTE} %5p %c{1}:%L - %m%n 10 | 11 | 12 | ### \u8F93\u51FA\u5230\u65E5\u5FD7\u6587\u4EF6 ### 13 | log4j.appender.D = org.apache.log4j.DailyRollingFileAppender 14 | log4j.appender.D.File = /home/neil/MyStudio/Workspace/DateTransfer/src/main/resources/logs/log.log 15 | log4j.appender.D.Append = true 16 | log4j.appender.D.Threshold = DEBUG 17 | log4j.appender.D.layout = org.apache.log4j.PatternLayout 18 | log4j.appender.D.layout.ConversionPattern = %d{ABSOLUTE} [ %t:%r ] - [ %p ] %m%n 19 | 20 | 21 | ### \u4FDD\u5B58\u5F02\u5E38\u4FE1\u606F\u5230\u5355\u72EC\u6587\u4EF6 ### 22 | log4j.appender.E = org.apache.log4j.DailyRollingFileAppender 23 | log4j.appender.E.File = /home/neil/MyStudio/Workspace/DateTransfer/src/main/resources/logs/error.log 24 | log4j.appender.E.Append = true 25 | log4j.appender.E.Threshold = ERROR 26 | log4j.appender.E.layout = org.apache.log4j.PatternLayout 27 | log4j.appender.E.layout.ConversionPattern = %d{ABSOLUTE} [ %t:%r ] - [ %p ] %m%n -------------------------------------------------------------------------------- /src/main/resources/logs/:user.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lifedance/DataTransfer/b77bfef70e23054011144bd1a31b701725a8d232/src/main/resources/logs/:user.txt -------------------------------------------------------------------------------- /src/main/resources/logs/error.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lifedance/DataTransfer/b77bfef70e23054011144bd1a31b701725a8d232/src/main/resources/logs/error.log -------------------------------------------------------------------------------- /src/main/resources/logs/log.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Lifedance/DataTransfer/b77bfef70e23054011144bd1a31b701725a8d232/src/main/resources/logs/log.log --------------------------------------------------------------------------------