├── .gitignore ├── LICENSE ├── README.md ├── application-global.yml ├── pom.xml ├── src └── main │ ├── java │ └── org │ │ └── poem │ │ ├── MetadataGatherApp.java │ │ ├── Swagger2Config.java │ │ ├── code │ │ ├── api │ │ │ ├── DsggatherStatisticsService.java │ │ │ ├── GatherService.java │ │ │ ├── MetadataService.java │ │ │ └── vo │ │ │ │ ├── DbVO.java │ │ │ │ ├── GatherDBTableFieldsVO.java │ │ │ │ ├── GatherDBVO.java │ │ │ │ ├── GatherTableVO.java │ │ │ │ ├── GatherVO.java │ │ │ │ ├── QueryGatherDBTableFieldsVO.java │ │ │ │ ├── TableFieldsVO.java │ │ │ │ ├── TableVO.java │ │ │ │ ├── column │ │ │ │ └── entity │ │ │ │ │ └── DataSetVO.java │ │ │ │ ├── databases │ │ │ │ └── entity │ │ │ │ │ └── DateBaseEntity.java │ │ │ │ ├── page │ │ │ │ └── Page.java │ │ │ │ ├── query │ │ │ │ ├── DbQueryVO.java │ │ │ │ ├── TableFieldsQueryVO.java │ │ │ │ └── TableQueryVO.java │ │ │ │ ├── statistics │ │ │ │ └── DsggatherStatisticsVO.java │ │ │ │ └── table │ │ │ │ ├── entity │ │ │ │ └── TableEntity.java │ │ │ │ └── vo │ │ │ │ └── CreateTableFiels.java │ │ ├── controller │ │ │ └── GatherController.java │ │ ├── dao │ │ │ ├── BaseDao.java │ │ │ ├── BaseDaoImpl.java │ │ │ ├── gather │ │ │ │ └── GatherInfoDao.java │ │ │ ├── info │ │ │ │ ├── DsgGatherDBDao.java │ │ │ │ ├── DsgGatherTableDao.java │ │ │ │ └── DsgGatherTableFieldsDao.java │ │ │ └── statistics │ │ │ │ └── DsggatherStatisticsDao.java │ │ ├── entities │ │ │ ├── DefaultCatalog.java │ │ │ ├── Indexes.java │ │ │ ├── Keys.java │ │ │ ├── Kylo.java │ │ │ ├── Tables.java │ │ │ └── tables │ │ │ │ ├── DsgGatherDb.java │ │ │ │ ├── DsgGatherInfo.java │ │ │ │ ├── DsgGatherStatistics.java │ │ │ │ ├── DsgGatherTable.java │ │ │ │ ├── DsgGatherTableFields.java │ │ │ │ └── records │ │ │ │ ├── DsgGatherDbRecord.java │ │ │ │ ├── DsgGatherInfoRecord.java │ │ │ │ ├── DsgGatherStatisticsRecord.java │ │ │ │ ├── DsgGatherTableFieldsRecord.java │ │ │ │ └── DsgGatherTableRecord.java │ │ └── service │ │ │ ├── DsggatherStatisticsServiceImpl.java │ │ │ ├── GatherServiceImpl.java │ │ │ ├── MetadataServiceImpl.java │ │ │ ├── connect │ │ │ ├── DataType.java │ │ │ ├── GatherBuilder.java │ │ │ └── GatherConnection.java │ │ │ └── databases │ │ │ ├── GatherDataBaseInter.java │ │ │ ├── MysqlGatherData.java │ │ │ ├── OracleGatherData.java │ │ │ └── abstracts │ │ │ └── AbstractGatherDataBaseInter.java │ │ ├── conf │ │ ├── JooqConfiguration.java │ │ └── MysqlConf.java │ │ ├── gen │ │ └── JooqPgGen.java │ │ ├── hadoop │ │ ├── common │ │ │ ├── DBDataImport.java │ │ │ ├── DateTimeUtil.java │ │ │ ├── FileWrite.java │ │ │ ├── NumberValidationUtils.java │ │ │ ├── constant │ │ │ │ └── Constant.java │ │ │ └── dbUtil │ │ │ │ ├── ConnUtil.java │ │ │ │ ├── DBUtil.java │ │ │ │ └── DataSourceType.java │ │ ├── hive │ │ │ ├── HiveDataType.java │ │ │ ├── HiveJdbcClient.java │ │ │ ├── HiveOperate.java │ │ │ └── RemoteOpera.java │ │ ├── model │ │ │ └── ColumnInfo.java │ │ └── rdbs │ │ │ ├── MysqlDataType.java │ │ │ ├── OracleDataType.java │ │ │ └── SqlServerDataType.java │ │ ├── loghelper │ │ ├── LoggerHelper.java │ │ └── file │ │ │ ├── FileService.java │ │ │ └── impl │ │ │ ├── HdfsFileServiceImpl.java │ │ │ └── LocalFileServiceImpl.java │ │ └── utils │ │ ├── GatherDataSourceUtils.java │ │ ├── HdfsUtils.java │ │ ├── SnowFlake.java │ │ ├── SpringUtils.java │ │ └── ThreadUtils.java │ ├── resources │ ├── application-gen.yml │ ├── application.yml │ └── db │ │ └── V0.01__init.sql │ └── test │ └── org │ └── poem │ └── service │ └── MetadataServiceImplTest.java ├── target ├── classes │ ├── META-INF │ │ └── spring-configuration-metadata.json │ ├── application-gen.yml │ ├── application.yml │ └── db │ │ └── V0.01__init.sql ├── maven-archiver │ └── pom.properties └── maven-status │ └── maven-compiler-plugin │ └── compile │ └── default-compile │ ├── createdFiles.lst │ └── inputFiles.lst └── tree /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | /.idea/ 25 | /53-metadata-gather.iml 26 | /metadata-gather.iml 27 | -------------------------------------------------------------------------------- /application-global.yml: -------------------------------------------------------------------------------- 1 | #该配置文件用于配置全局变量 2 | global: 3 | ip: 127.0.0.1 #本机以太网ip地址，需要手动配置 4 | #数据源配置 5 | datasource: 6 | name: test #数据库源名称 7 | #database-name: db_dcp_dev 8 | database-name: kylo 9 | url: 127.0.0.1 #数据库所在的url地址 10 | port: 3306 #数据库端口 11 | username: root #数据库用户名 12 | password: 123456 #数据库密码 13 | gather: 14 | server: 15 | port: 10015 16 | logger: 17 | storage: LOCAL # LOCAL or HDFS 18 | path: 19 | -------------------------------------------------------------------------------- /src/main/java/org/poem/MetadataGatherApp.java: -------------------------------------------------------------------------------- 1 | package org.poem; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import springfox.documentation.swagger2.annotations.EnableSwagger2; 6 | 7 | /** 8 | * @author Administrator 9 | */ 10 | @EnableSwagger2 11 | @SpringBootApplication 12 | public class MetadataGatherApp { 13 | 14 | public static void main(String[] args) { 15 | SpringApplication.run( MetadataGatherApp.class, args ); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/poem/Swagger2Config.java: -------------------------------------------------------------------------------- 1 | package org.poem; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import springfox.documentation.builders.ApiInfoBuilder; 6 | import springfox.documentation.builders.ParameterBuilder; 7 | import springfox.documentation.builders.PathSelectors; 8 | import springfox.documentation.builders.RequestHandlerSelectors; 9 | import springfox.documentation.schema.ModelRef; 10 | import springfox.documentation.service.ApiInfo; 11 | import springfox.documentation.service.Contact; 12 | import springfox.documentation.service.Parameter; 13 | import springfox.documentation.spi.DocumentationType; 14 | import springfox.documentation.spring.web.plugins.Docket; 15 | import springfox.documentation.swagger2.annotations.EnableSwagger2; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | 20 | 21 | /** 22 | * @author poem 23 | */ 24 | @Configuration 25 | @EnableSwagger2 26 | public class Swagger2Config { 27 | 28 | 29 | @Bean 30 | public Docket createAllRestApi() { 31 | List paraList = new ArrayList<>(); 32 | 33 | return new Docket(DocumentationType.SWAGGER_2) 34 | .groupName("API-All") 35 | .apiInfo(apiInfo()) 36 | .select() 37 | .apis(RequestHandlerSelectors.basePackage("org.poem.code.controller")) 38 | .paths(PathSelectors.any()) 39 | .build() 40 | .globalOperationParameters(paraList); 41 | } 42 | 43 | private ApiInfo apiInfo() { 44 | return new ApiInfoBuilder() 45 | .title("数据采集小工具") 46 | .description("数据采集 接口") 47 | .termsOfServiceUrl("未提供") 48 | .contact(new Contact("poem", "", "xue_2013@sina.com")) 49 | .version("1.0.0") 50 | .build(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/DsggatherStatisticsService.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api; 2 | 3 | import org.poem.code.api.vo.statistics.DsggatherStatisticsVO; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * 9 | * @author Administrator 10 | */ 11 | public interface DsggatherStatisticsService { 12 | 13 | /** 14 | * 保存 15 | * @param dsggatherStatisticsVO 16 | */ 17 | void save(DsggatherStatisticsVO dsggatherStatisticsVO); 18 | 19 | 20 | /** 21 | * 查询 22 | * @return 23 | */ 24 | List query(); 25 | 26 | 27 | /** 28 | * 29 | * @param dsggatherStatisticsVO 30 | */ 31 | public void saveCurrent(DsggatherStatisticsVO dsggatherStatisticsVO); 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/GatherService.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api; 2 | 3 | import org.poem.code.api.vo.*; 4 | import org.poem.code.api.vo.column.entity.DataSetVO; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * @author Administrator 10 | */ 11 | public interface GatherService { 12 | 13 | /** 14 | * 测试连接成功 15 | */ 16 | boolean testGather(String gatherId); 17 | 18 | /** 19 | * 获取数据的schema 20 | * 21 | * @param gatherId 22 | * @return 23 | */ 24 | List getSchema(String gatherId); 25 | 26 | 27 | /** 28 | * 获取数据表字端信息 29 | */ 30 | 31 | List getTable(GatherVO gatherVO); 32 | 33 | /** 34 | * @param table 35 | * @return 36 | */ 37 | TableVO getTableById(String table); 38 | 39 | /** 40 | * 获取表的字端信息 41 | * @param tableVO 42 | * @return 43 | */ 44 | List getTableFieles(TableVO tableVO); 45 | 46 | 47 | /** 48 | * 获取 某一个库 某一个表 的某一个列 49 | * @param gatherId 50 | * @param db 51 | * @return 52 | */ 53 | public GatherDBTableFieldsVO getAllGatherDBTableFieldsVO(String gatherId, String db, String table); 54 | 55 | 56 | /** 57 | * 获取查询 58 | * @param sql 执行的sql 59 | * @param sourceId source id 60 | * @return 61 | */ 62 | public DataSetVO executeSQL(String sql, String sourceId); 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/MetadataService.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api; 2 | 3 | import org.poem.code.api.vo.*; 4 | import org.poem.code.api.vo.query.DbQueryVO; 5 | import org.poem.code.api.vo.query.TableFieldsQueryVO; 6 | import org.poem.code.api.vo.query.TableQueryVO; 7 | import org.poem.code.entities.tables.records.DsgGatherDbRecord; 8 | import org.poem.code.entities.tables.records.DsgGatherTableFieldsRecord; 9 | import org.poem.code.entities.tables.records.DsgGatherTableRecord; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * @author Administrator 15 | */ 16 | public interface MetadataService { 17 | /** 18 | * 保存连接信息 19 | */ 20 | 21 | void savgeGather(GatherVO gatherVO); 22 | 23 | /** 24 | * 获取连接信息 25 | */ 26 | 27 | List getAllGather(); 28 | 29 | 30 | /** 31 | * @param gatherId 32 | * @return 33 | */ 34 | GatherVO getGatherVOById(String gatherId); 35 | 36 | /** 37 | * 保存 38 | * 39 | * @param dbVOS 40 | */ 41 | List saveDB(List dbVOS, String gatherInfoId); 42 | 43 | 44 | /** 45 | * @param dbQueryVO 46 | * @return 47 | */ 48 | List getAllDB(DbQueryVO dbQueryVO); 49 | 50 | /** 51 | * @param tableVOS 52 | */ 53 | List saveTable(List tableVOS, String dbId); 54 | 55 | /** 56 | * @param tableQueryVO 57 | * @return 58 | */ 59 | List getTable(TableQueryVO tableQueryVO); 60 | 61 | 62 | /** 63 | * 获取表的信息 64 | * @param tableFieldsQueryVO 65 | * @return 66 | */ 67 | public List getTableFileds(TableFieldsQueryVO tableFieldsQueryVO); 68 | /** 69 | * 保存数据 70 | * 71 | * @param fieldsVOS 72 | */ 73 | List saveTableFields(List fieldsVOS, String tableId); 74 | 75 | /** 76 | * 删除 77 | * @param gatherId 78 | */ 79 | void deleteAllDataGatherInfoId(String gatherId); 80 | 81 | /** 82 | * @param gatherDBTableFieldsVO 83 | */ 84 | public void saveGather(GatherDBTableFieldsVO gatherDBTableFieldsVO); 85 | 86 | /** 87 | * 查询保存的数据 88 | * @param queryGatherDBTableFieldsVO 89 | * @return 90 | */ 91 | GatherDBTableFieldsVO getDBTableAndFields(QueryGatherDBTableFieldsVO queryGatherDBTableFieldsVO); 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/DbVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | 7 | /** 8 | * @author Administrator 9 | */ 10 | @Data 11 | @ApiModel("数据库") 12 | public class DbVO { 13 | 14 | @ApiModelProperty("id") 15 | private String id; 16 | 17 | @ApiModelProperty("数据库名字") 18 | private String name; 19 | 20 | @ApiModelProperty("连接信息id") 21 | private String gatherId; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/GatherDBTableFieldsVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | 7 | import java.util.List; 8 | 9 | @Data 10 | @ApiModel("连接信息") 11 | public class GatherDBTableFieldsVO { 12 | 13 | @ApiModelProperty("连接信息Id") 14 | private String gratherid; 15 | 16 | @ApiModelProperty("数据库信息") 17 | private List gatherDBVOS; 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/GatherDBVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | 7 | import java.util.List; 8 | /** 9 | * @author Administrator 10 | */ 11 | @Data 12 | @ApiModel("数据库表信息") 13 | public class GatherDBVO { 14 | 15 | @ApiModelProperty("数据库") 16 | private DbVO dbVO; 17 | 18 | @ApiModelProperty("表") 19 | List gatherTableVOS; 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/GatherTableVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | 7 | import java.util.List; 8 | 9 | @Data 10 | @ApiModel("数据表和列") 11 | public class GatherTableVO { 12 | 13 | @ApiModelProperty("数据表") 14 | private TableVO tableVO; 15 | 16 | @ApiModelProperty("数据表列") 17 | private List tableFieldsVOS; 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/GatherVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | 7 | /** 8 | * @author Administrator 9 | */ 10 | @Data 11 | @ApiModel("数据源连接信息") 12 | public class GatherVO { 13 | 14 | /** 15 | * 16 | */ 17 | @ApiModelProperty("为空") 18 | private String gatherId; 19 | /** 20 | * id 21 | */ 22 | @ApiModelProperty("id 新添加任务为空") 23 | private String id; 24 | 25 | /** 26 | * 27 | */ 28 | @ApiModelProperty("为空") 29 | private String dbId; 30 | 31 | /** 32 | * 33 | */ 34 | @ApiModelProperty("连接名字") 35 | private String name; 36 | /** 37 | * 地址 38 | */ 39 | @ApiModelProperty("地址") 40 | private String ip; 41 | 42 | /** 43 | * 端口 44 | */ 45 | @ApiModelProperty("端口") 46 | private String port; 47 | 48 | /** 49 | * 用户名 50 | */ 51 | @ApiModelProperty("用户名") 52 | private String userName; 53 | 54 | /** 55 | * 密码 56 | */ 57 | @ApiModelProperty("密码") 58 | private String password; 59 | 60 | /** 61 | * 数据类型 62 | */ 63 | @ApiModelProperty("数据类型 MYSQL ORACLE") 64 | private String type; 65 | 66 | /** 67 | * oracle 服务名字 68 | */ 69 | @ApiModelProperty("oracle 服务名字") 70 | private String serverName; 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/QueryGatherDBTableFieldsVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | 7 | @Data 8 | @ApiModel("连接信息") 9 | public class QueryGatherDBTableFieldsVO { 10 | 11 | @ApiModelProperty("连接信息Id") 12 | private String gratherid; 13 | 14 | @ApiModelProperty("数据库") 15 | private String db; 16 | 17 | @ApiModelProperty("数据库id") 18 | private String dbId; 19 | 20 | @ApiModelProperty("表") 21 | private String table; 22 | 23 | @ApiModelProperty("表id") 24 | private String tableId; 25 | 26 | 27 | @ApiModelProperty("字端信息") 28 | private String fields; 29 | 30 | @ApiModelProperty("字端表述") 31 | private String fieldsDes; 32 | 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/TableFieldsVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | 7 | /** 8 | * @author Administrator 9 | */ 10 | @Data 11 | @ApiModel("数据表列") 12 | public class TableFieldsVO { 13 | 14 | @ApiModelProperty("表id") 15 | private String tableId; 16 | 17 | @ApiModelProperty("列id") 18 | private String field; 19 | 20 | @ApiModelProperty("列的数据类型") 21 | private String dataType; 22 | 23 | @ApiModelProperty("表的描述") 24 | private String description; 25 | 26 | @ApiModelProperty("默认值") 27 | private String defaultValue; 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/TableVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | 7 | /** 8 | * @author Administrator 9 | */ 10 | @Data 11 | @ApiModel("数据表") 12 | public class TableVO { 13 | 14 | @ApiModelProperty("id") 15 | private String id; 16 | 17 | @ApiModelProperty("库id") 18 | private String dbId; 19 | 20 | @ApiModelProperty("表名字") 21 | private String table; 22 | 23 | @ApiModelProperty("表的中文注释名字") 24 | private String name; 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/column/entity/DataSetVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo.column.entity; 2 | 3 | 4 | import org.poem.code.api.vo.table.entity.TableEntity; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * @author Administrator 10 | */ 11 | public class DataSetVO extends TableEntity { 12 | /** 13 | * 表的名字 14 | */ 15 | private String tableName; 16 | 17 | /** 18 | * 表头 19 | */ 20 | private List columns; 21 | 22 | /** 23 | * 数据 24 | */ 25 | private List> datas; 26 | 27 | /** 28 | * 数据量 29 | */ 30 | private Long total; 31 | 32 | public DataSetVO() { 33 | } 34 | 35 | public List getColumns() { 36 | return columns; 37 | } 38 | 39 | public void setColumns(List columns) { 40 | this.columns = columns; 41 | } 42 | 43 | public List> getDatas() { 44 | return datas; 45 | } 46 | 47 | public void setDatas(List> datas) { 48 | this.datas = datas; 49 | } 50 | 51 | @Override 52 | public String getTableName() { 53 | return tableName; 54 | } 55 | 56 | @Override 57 | public void setTableName(String tableName) { 58 | this.tableName = tableName; 59 | } 60 | 61 | public Long getTotal() { 62 | return total; 63 | } 64 | 65 | public void setTotal(Long total) { 66 | this.total = total; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/databases/entity/DateBaseEntity.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo.databases.entity; 2 | 3 | public class DateBaseEntity { 4 | 5 | /** 6 | * 数据库 7 | */ 8 | private String databaseName; 9 | 10 | public String getDatabaseName() { 11 | return databaseName; 12 | } 13 | 14 | public void setDatabaseName(String databaseName) { 15 | this.databaseName = databaseName; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/page/Page.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo.page; 2 | 3 | 4 | /** 5 | * @author Administrator 6 | */ 7 | public class Page { 8 | 9 | /** 10 | * 起始记录下标 11 | */ 12 | private int beginIndex; 13 | /** 14 | * 截止记录下标 15 | */ 16 | private int endIndex; 17 | 18 | public Page(int beginIndex, int endIndex) { 19 | this.beginIndex = beginIndex; 20 | this.endIndex = endIndex; 21 | } 22 | 23 | public int getBeginIndex() { 24 | return beginIndex; 25 | } 26 | 27 | public void setBeginIndex(int beginIndex) { 28 | this.beginIndex = beginIndex; 29 | } 30 | 31 | public int getEndIndex() { 32 | return endIndex; 33 | } 34 | 35 | public void setEndIndex(int endIndex) { 36 | this.endIndex = endIndex; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/query/DbQueryVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo.query; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | /** 9 | * @author Administrator 10 | */ 11 | @Data 12 | @ApiModel("数据库") 13 | @NoArgsConstructor 14 | public class DbQueryVO { 15 | 16 | @ApiModelProperty("数据库名字") 17 | private String name; 18 | 19 | @ApiModelProperty("连接信息id") 20 | private String gatherId; 21 | 22 | public DbQueryVO(String name, String gatherId) { 23 | this.name = name; 24 | this.gatherId = gatherId; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/query/TableFieldsQueryVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo.query; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | /** 9 | * @author Administrator 10 | */ 11 | @Data 12 | @ApiModel("数据表列") 13 | @NoArgsConstructor 14 | public class TableFieldsQueryVO { 15 | 16 | @ApiModelProperty("表id") 17 | private String tableId; 18 | 19 | @ApiModelProperty("列id") 20 | private String field; 21 | 22 | @ApiModelProperty("表的描述") 23 | private String description; 24 | 25 | public TableFieldsQueryVO(String tableId, String field, String description) { 26 | this.tableId = tableId; 27 | this.field = field; 28 | this.description = description; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/query/TableQueryVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo.query; 2 | 3 | import io.swagger.annotations.ApiModel; 4 | import io.swagger.annotations.ApiModelProperty; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | /** 9 | * @author Administrator 10 | */ 11 | @NoArgsConstructor 12 | @Data 13 | @ApiModel("数据表") 14 | public class TableQueryVO { 15 | 16 | @ApiModelProperty("库id") 17 | private String dbId; 18 | 19 | @ApiModelProperty("表名字") 20 | private String tableId; 21 | 22 | @ApiModelProperty("表的中文注释名字") 23 | private String name; 24 | 25 | public TableQueryVO(String dbId, String tableId, String name) { 26 | this.dbId = dbId; 27 | this.tableId = tableId; 28 | this.name = name; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/statistics/DsggatherStatisticsVO.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo.statistics; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @author Administrator 7 | */ 8 | @Data 9 | public class DsggatherStatisticsVO { 10 | 11 | String gatherId; 12 | 13 | private Integer dbCount; 14 | 15 | private Integer tableCount; 16 | 17 | private Integer fieldCount ; 18 | 19 | 20 | public DsggatherStatisticsVO(){ 21 | this.dbCount = 0; 22 | this.tableCount = 0; 23 | this.fieldCount = 0; 24 | } 25 | 26 | public DsggatherStatisticsVO(String gatherId, Integer dbCount, Integer tableCount, Integer fieldCount) { 27 | this.gatherId = gatherId; 28 | this.dbCount = dbCount; 29 | this.tableCount = tableCount; 30 | this.fieldCount = fieldCount; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/table/entity/TableEntity.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo.table.entity; 2 | 3 | 4 | import org.poem.code.api.vo.databases.entity.DateBaseEntity; 5 | 6 | /** 7 | * @author Administrator 8 | */ 9 | public class TableEntity extends DateBaseEntity { 10 | 11 | /** 12 | * 表名 13 | */ 14 | private String tableName; 15 | 16 | private String comment; 17 | 18 | public String getComment() { 19 | return comment; 20 | } 21 | 22 | public void setComment(String comment) { 23 | this.comment = comment; 24 | } 25 | 26 | public String getTableName() { 27 | return tableName; 28 | } 29 | 30 | public void setTableName(String tableName) { 31 | this.tableName = tableName; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/api/vo/table/vo/CreateTableFiels.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.api.vo.table.vo; 2 | 3 | import lombok.Getter; 4 | import lombok.NoArgsConstructor; 5 | import lombok.Setter; 6 | 7 | @Setter 8 | @Getter 9 | @NoArgsConstructor 10 | public class CreateTableFiels { 11 | 12 | private String field; 13 | 14 | private String type; 15 | 16 | private String content; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/controller/GatherController.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.controller; 2 | 3 | import io.swagger.annotations.*; 4 | import org.apache.commons.lang3.StringUtils; 5 | import org.poem.code.api.GatherService; 6 | import org.poem.code.api.MetadataService; 7 | import org.poem.code.api.vo.*; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.web.bind.annotation.*; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * @author Administrator 15 | */ 16 | @RestController 17 | @RequestMapping("/v1/gather") 18 | @Api(value = "/v1/gather", tags = {"01-元数据采集"}) 19 | public class GatherController { 20 | 21 | @Autowired 22 | private GatherService gatherService; 23 | 24 | @Autowired 25 | private MetadataService metadataService; 26 | 27 | 28 | /** 29 | * 30 | * @return 31 | */ 32 | @ApiOperation(value = "0101-获取所有的连接信息", notes = "获取所有的连接信息") 33 | @ApiResponses({ 34 | @ApiResponse(code = 400, message = "请求参数没有填好"), 35 | @ApiResponse(code = 404, message = "请求路径没有找到"), 36 | @ApiResponse(code = 500, message = "数据库发生错误") 37 | }) 38 | @GetMapping(value = "/getAllGather") 39 | public List getAllGather() { 40 | return this.metadataService.getAllGather(); 41 | } 42 | 43 | /** 44 | * 保存连接信息 45 | * @param gatherVO 46 | */ 47 | @ApiOperation(value = "0102-保存连接信息", notes = "保存连接信息") 48 | @ApiResponses({ 49 | @ApiResponse(code = 400, message = "请求参数没有填好"), 50 | @ApiResponse(code = 404, message = "请求路径没有找到"), 51 | @ApiResponse(code = 500, message = "数据库发生错误") 52 | }) 53 | @PostMapping(value = "/saveGatherInfo") 54 | public void savgeGather(@RequestBody GatherVO gatherVO) { 55 | this.metadataService.savgeGather( gatherVO ); 56 | } 57 | 58 | /** 59 | * 删除所有保存的连接信息 60 | * @param gatherId 61 | */ 62 | @ApiOperation(value = "0103-删除所有保存的连接信息", notes = "删除所有保存的连接信息") 63 | @ApiResponses({ 64 | @ApiResponse(code = 400, message = "请求参数没有填好"), 65 | @ApiResponse(code = 404, message = "请求路径没有找到"), 66 | @ApiResponse(code = 500, message = "数据库发生错误") 67 | }) 68 | @ApiImplicitParams({ 69 | @ApiImplicitParam(name = "gatherId", required = true, value = "连接信息id", dataType = "String", example = "123456"), 70 | }) 71 | @PostMapping(value = "/deleteAllDataGatherInfoId") 72 | public void deleteAllDataGatherInfoId(String gatherId) { 73 | this.metadataService.deleteAllDataGatherInfoId( gatherId ); 74 | } 75 | 76 | 77 | 78 | /** 79 | * 根据信息连接信息获取库 80 | * @param queryGatherDBTableFieldsVO 81 | * @return 82 | */ 83 | @ApiOperation(value = "0103-根据信息连接信息获取库、表、字端", notes = "根据信息连接信息获取库、表、字端") 84 | @ApiResponses({ 85 | @ApiResponse(code = 400, message = "请求参数没有填好"), 86 | @ApiResponse(code = 404, message = "请求路径没有找到"), 87 | @ApiResponse(code = 500, message = "数据库发生错误") 88 | }) 89 | @GetMapping(value = "/getDBAndTableAndFields") 90 | public GatherDBTableFieldsVO getDBAndTableAndFields(QueryGatherDBTableFieldsVO queryGatherDBTableFieldsVO) { 91 | if (StringUtils.isBlank( queryGatherDBTableFieldsVO.getGratherid() )){ 92 | return null; 93 | } 94 | return this.metadataService.getDBTableAndFields( queryGatherDBTableFieldsVO ); 95 | } 96 | 97 | /** 98 | * 根据信息连接信息获取库 99 | * @param gatherId 100 | * @return 101 | */ 102 | @ApiOperation(value = "0105-数据库-测试连接是否正常", notes = "数据库-测试连接是否正常") 103 | @ApiResponses({ 104 | @ApiResponse(code = 400, message = "请求参数没有填好"), 105 | @ApiResponse(code = 404, message = "请求路径没有找到"), 106 | @ApiResponse(code = 500, message = "数据库发生错误") 107 | }) 108 | @ApiImplicitParams({ 109 | @ApiImplicitParam(name = "gatherId", required = true, value = "连接信息id", dataType = "String", example = "123456"), 110 | }) 111 | @GetMapping(value = "/testGather/{gatherId}") 112 | public boolean testGather(@PathVariable("gatherId") String gatherId){ 113 | return gatherService.testGather( gatherId ); 114 | } 115 | 116 | /** 117 | * 获取数据的schema 118 | * 119 | * @param gatherId 120 | * @return 121 | */ 122 | 123 | @ApiOperation(value = "0106-数据库-获取数据的schema", notes = "数据库-获取数据的schema") 124 | @ApiResponses({ 125 | @ApiResponse(code = 400, message = "请求参数没有填好"), 126 | @ApiResponse(code = 404, message = "请求路径没有找到"), 127 | @ApiResponse(code = 500, message = "数据库发生错误") 128 | }) 129 | @ApiImplicitParams({ 130 | @ApiImplicitParam(name = "gatherId", required = true, value = "连接信息id", dataType = "String", example = "123456"), 131 | }) 132 | @GetMapping(value = "/getSchema/{gatherId}") 133 | public List getSchema(@PathVariable("gatherId") String gatherId){ 134 | return gatherService.getSchema( gatherId ); 135 | } 136 | 137 | 138 | /** 139 | * 获取数据表字端信息 140 | * @param gatherVO 141 | * @return 142 | */ 143 | @ApiOperation(value = "0107-数据库-获取数据表字端信息", notes = "获取数据表字端信息") 144 | @ApiResponses({ 145 | @ApiResponse(code = 400, message = "请求参数没有填好"), 146 | @ApiResponse(code = 404, message = "请求路径没有找到"), 147 | @ApiResponse(code = 500, message = "数据库发生错误") 148 | }) 149 | @GetMapping(value = "/getTable") 150 | public List getTable(GatherVO gatherVO){ 151 | return gatherService.getTable( gatherVO ); 152 | } 153 | 154 | /** 155 | * 获取表的字端信息 156 | * @param tableVO 157 | * @return 158 | */ 159 | @ApiOperation(value = "0108-数据库-获取表的字端信息", notes = "获取表的字端信息") 160 | @ApiResponses({ 161 | @ApiResponse(code = 400, message = "请求参数没有填好"), 162 | @ApiResponse(code = 404, message = "请求路径没有找到"), 163 | @ApiResponse(code = 500, message = "数据库发生错误") 164 | }) 165 | @GetMapping(value = "/getTableFields") 166 | public List getTableFields(TableVO tableVO){ 167 | return gatherService.getTableFieles( tableVO ); 168 | } 169 | 170 | 171 | /** 172 | * 根据信息连接信息获取库 173 | * @param gatherId 174 | */ 175 | @ApiOperation(value = "0109-数据库-根据信息连接信息获取库、表、字端,并且保存到数据库中", 176 | notes = "数据库-根据信息连接信息获取库、表、字端,并且保存到数据库中") 177 | @ApiResponses({ 178 | @ApiResponse(code = 400, message = "请求参数没有填好"), 179 | @ApiResponse(code = 404, message = "请求路径没有找到"), 180 | @ApiResponse(code = 500, message = "数据库发生错误") 181 | }) 182 | @PostMapping(value = "/getDbAndTableAndFieldAndSave") 183 | public void getDbAndTableAndFieldAndSave(String gatherId,String db, String table) { 184 | GatherDBTableFieldsVO dbTableFieldsVO = this.gatherService.getAllGatherDBTableFieldsVO( gatherId , db, table); 185 | this.metadataService.saveGather( dbTableFieldsVO ); 186 | } 187 | 188 | /** 189 | * 190 | * @param sql 191 | * @param sourceId 192 | */ 193 | @ApiOperation(value = "0110-执行sql 前端不做", 194 | notes = "执行sql 前端不做") 195 | @ApiResponses({ 196 | @ApiResponse(code = 400, message = "请求参数没有填好"), 197 | @ApiResponse(code = 404, message = "请求路径没有找到"), 198 | @ApiResponse(code = 500, message = "数据库发生错误") 199 | }) 200 | @ApiImplicitParams({ 201 | @ApiImplicitParam(name = "source", required = true, value = "连接信息id", dataType = "String", example = "123456"), 202 | @ApiImplicitParam(name = "sql", required = true, value = "sql", dataType = "String") 203 | }) 204 | @PostMapping(value = "/executeSql") 205 | public void executeSql(String sql, String sourceId) { 206 | this.gatherService.executeSQL( sql, sourceId ); 207 | } 208 | 209 | 210 | } 211 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/dao/BaseDao.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.dao; 2 | 3 | 4 | import org.jooq.Condition; 5 | import org.jooq.Field; 6 | import org.jooq.SortField; 7 | import org.jooq.TableRecord; 8 | import org.jooq.exception.DataAccessException; 9 | 10 | import java.util.Collection; 11 | import java.util.List; 12 | import java.util.Optional; 13 | 14 | public interface BaseDao, T> { 15 | void insert(R object) throws DataAccessException; 16 | 17 | void insert(Collection objects) throws DataAccessException; 18 | 19 | void update(R object) throws DataAccessException; 20 | 21 | void updateOneFieldById(T id, Field field, Z value) throws DataAccessException; 22 | 23 | void updateOneFieldByField(Field setField, Z setValue, Field whereField, S whereValue) throws DataAccessException; 24 | 25 | void delete(R object) throws DataAccessException; 26 | 27 | void delete(Collection objects) throws DataAccessException; 28 | 29 | void deleteById(Collection ids) throws DataAccessException; 30 | 31 | R findById(T id) throws DataAccessException; 32 | 33 | List fetch(Field field, Z... values) throws DataAccessException; 34 | 35 | List fetch(Field field, Collection values) throws DataAccessException; 36 | 37 | R fetchOne(Field field, Z value) throws DataAccessException; 38 | 39 | Optional fetchOptional(Field field, Z value) throws DataAccessException; 40 | 41 | 42 | List findByConditions(List conditions, Collection> sortFields); 43 | 44 | List findByCondition(Condition condition, Collection> sortFields); 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/dao/BaseDaoImpl.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.dao; 2 | 3 | 4 | import org.jooq.*; 5 | import org.jooq.exception.DataAccessException; 6 | import org.jooq.impl.DSL; 7 | 8 | import javax.annotation.PostConstruct; 9 | import java.util.ArrayList; 10 | import java.util.Collection; 11 | import java.util.List; 12 | import java.util.Optional; 13 | 14 | public abstract class BaseDaoImpl, T> implements BaseDao { 15 | 16 | private final Table table; 17 | 18 | public BaseDaoImpl(Table table) { 19 | this.table = table; 20 | } 21 | 22 | @PostConstruct 23 | public abstract Configuration getConfiguration(); 24 | 25 | @Override 26 | public void insert(R object) throws DataAccessException { 27 | DSL.using( getConfiguration() ).insertInto( object.getTable() ).set( object ).execute(); 28 | } 29 | 30 | @Override 31 | public void insert(Collection objects) throws DataAccessException { 32 | DSL.using( getConfiguration() ).batchInsert( objects ).execute(); 33 | 34 | } 35 | 36 | @Override 37 | public void update(R object) throws DataAccessException { 38 | DSL.using( getConfiguration() ).executeUpdate( object ); 39 | } 40 | 41 | @Override 42 | public void updateOneFieldById(T id, Field field, Z value) throws DataAccessException { 43 | Field primaryField = getPrimaryKey(); 44 | if (primaryField != null) { 45 | DSL.using( getConfiguration() ).update( table ).set( field, value ).where( primaryField.eq( id ) ).execute(); 46 | } 47 | } 48 | 49 | @Override 50 | public void updateOneFieldByField(Field setField, Z setValue, Field whereField, S whereValue) throws DataAccessException { 51 | DSL.using( getConfiguration() ).update( table ).set( setField, setValue ).where( whereField.eq( whereValue ) ).execute(); 52 | } 53 | 54 | @Override 55 | public void delete(R object) throws DataAccessException { 56 | DSL.using( getConfiguration() ).executeDelete( object ); 57 | } 58 | 59 | @Override 60 | public void delete(Collection objects) throws DataAccessException { 61 | DSL.using( getConfiguration() ).batchDelete( objects ).execute(); 62 | } 63 | 64 | @Override 65 | public void deleteById(Collection ids) throws DataAccessException { 66 | Field primaryField = getPrimaryKey(); 67 | if (primaryField != null && ids.size() > 0) { 68 | DSL.using( getConfiguration() ).delete( this.table ).where( primaryField.in( ids ) ).execute(); 69 | } 70 | } 71 | 72 | @Override 73 | public R findById(T id) throws DataAccessException { 74 | Field primaryField = getPrimaryKey(); 75 | if (primaryField != null) { 76 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( primaryField.eq( id ) ).fetchAny(); 77 | } 78 | return null; 79 | } 80 | 81 | @Override 82 | public List fetch(Field field, Z... values) throws DataAccessException { 83 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( new Condition[]{field.in( values )} ).fetch(); 84 | } 85 | 86 | public List findByIds(T... ids) throws DataAccessException { 87 | Field primaryField = getPrimaryKey(); 88 | if (primaryField != null) { 89 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( primaryField.in( ids ) ).fetch(); 90 | } 91 | return new ArrayList<>(); 92 | } 93 | 94 | public List findByIdList(Collection ids) throws DataAccessException { 95 | Field primaryField = getPrimaryKey(); 96 | if (primaryField != null) { 97 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( primaryField.in( ids ) ).fetch(); 98 | } 99 | return new ArrayList<>(); 100 | } 101 | 102 | @Override 103 | public List fetch(Field field, Collection values) throws DataAccessException { 104 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( new Condition[]{field.in( values )} ).fetch(); 105 | } 106 | 107 | @Override 108 | public R fetchOne(Field field, Z value) throws DataAccessException { 109 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( new Condition[]{field.eq( value )} ).fetchOne(); 110 | } 111 | 112 | @Override 113 | public Optional fetchOptional(Field field, Z value) throws DataAccessException { 114 | return Optional.ofNullable( this.fetchOne( field, value ) ); 115 | } 116 | 117 | 118 | private Field getPrimaryKey() { 119 | UniqueKey key = this.table.getPrimaryKey(); 120 | if (key == null || key.getFields() == null || key.getFields().isEmpty()) { 121 | return null; 122 | } 123 | List> fields = key.getFields(); 124 | return fields.get( 0 ); 125 | } 126 | 127 | @Override 128 | public List findByConditions(List conditions, Collection> sortFields) { 129 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( conditions ).orderBy( sortFields ).fetch(); 130 | } 131 | 132 | public List findByConditions(List conditions) { 133 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( conditions ).fetch(); 134 | } 135 | 136 | public List findByCondition(Condition condition) { 137 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( condition ).fetch(); 138 | } 139 | 140 | public List findAll() { 141 | return DSL.using( getConfiguration() ).selectFrom( this.table ).fetch(); 142 | } 143 | 144 | @Override 145 | public List findByCondition(Condition condition, Collection> sortFields) { 146 | return DSL.using( getConfiguration() ).selectFrom( this.table ).where( condition ).orderBy( sortFields ).fetch(); 147 | } 148 | 149 | public void deleteByConditions(Condition... conditions) { 150 | DSL.using( getConfiguration() ).deleteFrom( this.table ).where( conditions ).execute(); 151 | } 152 | 153 | public void deleteByConditions(Collection conditions) { 154 | DSL.using( getConfiguration() ).deleteFrom( this.table ).where( conditions ).execute(); 155 | } 156 | 157 | public Optional fetchAny() throws DataAccessException { 158 | return Optional.ofNullable( DSL.using( getConfiguration() ).selectFrom( this.table ).limit( 1 ).fetchAny() ); 159 | } 160 | 161 | public List findByCondition(Collection> sortFields) { 162 | return DSL.using( getConfiguration() ).selectFrom( this.table ).orderBy( sortFields ).fetch(); 163 | } 164 | 165 | } 166 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/dao/gather/GatherInfoDao.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.dao.gather; 2 | 3 | import org.jooq.Configuration; 4 | import org.jooq.DSLContext; 5 | import org.poem.code.dao.BaseDaoImpl; 6 | import org.poem.code.entities.tables.DsgGatherInfo; 7 | import org.poem.code.entities.tables.records.DsgGatherInfoRecord; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.stereotype.Repository; 10 | 11 | /** 12 | * @author Administrator 13 | */ 14 | @Repository 15 | public class GatherInfoDao extends BaseDaoImpl { 16 | 17 | @Autowired 18 | private DSLContext dslContext; 19 | 20 | public GatherInfoDao() { 21 | super( DsgGatherInfo.DSG_GATHER_INFO ); 22 | } 23 | 24 | @Override 25 | public Configuration getConfiguration() { 26 | return dslContext.configuration(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/dao/info/DsgGatherDBDao.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.dao.info; 2 | 3 | import org.jooq.Configuration; 4 | import org.jooq.DSLContext; 5 | import org.poem.code.dao.BaseDaoImpl; 6 | import org.poem.code.entities.tables.DsgGatherDb; 7 | import org.poem.code.entities.tables.records.DsgGatherDbRecord; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.stereotype.Repository; 10 | 11 | @Repository 12 | public class DsgGatherDBDao extends BaseDaoImpl { 13 | 14 | 15 | @Autowired 16 | private DSLContext dslContext; 17 | 18 | public DsgGatherDBDao() { 19 | super( DsgGatherDb.DSG_GATHER_DB ); 20 | } 21 | 22 | @Override 23 | public Configuration getConfiguration() { 24 | return dslContext.configuration(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/dao/info/DsgGatherTableDao.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.dao.info; 2 | 3 | import org.jooq.Configuration; 4 | import org.jooq.DSLContext; 5 | import org.poem.code.dao.BaseDaoImpl; 6 | import org.poem.code.entities.tables.DsgGatherTable; 7 | import org.poem.code.entities.tables.records.DsgGatherTableRecord; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.stereotype.Repository; 10 | 11 | /** 12 | * @author Administrator 13 | */ 14 | @Repository 15 | public class DsgGatherTableDao extends BaseDaoImpl { 16 | 17 | 18 | @Autowired 19 | private DSLContext dslContext; 20 | 21 | public DsgGatherTableDao() { 22 | super( DsgGatherTable.DSG_GATHER_TABLE); 23 | } 24 | 25 | @Override 26 | public Configuration getConfiguration() { 27 | return dslContext.configuration(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/dao/info/DsgGatherTableFieldsDao.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.dao.info; 2 | 3 | import org.jooq.Configuration; 4 | import org.jooq.DSLContext; 5 | import org.poem.code.dao.BaseDaoImpl; 6 | import org.poem.code.entities.tables.DsgGatherTableFields; 7 | import org.poem.code.entities.tables.records.DsgGatherTableFieldsRecord; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.stereotype.Repository; 10 | 11 | @Repository 12 | public class DsgGatherTableFieldsDao extends BaseDaoImpl { 13 | 14 | 15 | @Autowired 16 | private DSLContext dslContext; 17 | 18 | public DsgGatherTableFieldsDao() { 19 | super( DsgGatherTableFields.DSG_GATHER_TABLE_FIELDS); 20 | } 21 | 22 | @Override 23 | public Configuration getConfiguration() { 24 | return dslContext.configuration(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/dao/statistics/DsggatherStatisticsDao.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.dao.statistics; 2 | 3 | 4 | import org.jooq.Configuration; 5 | import org.jooq.DSLContext; 6 | import org.poem.code.dao.BaseDaoImpl; 7 | import org.poem.code.entities.tables.DsgGatherStatistics; 8 | import org.poem.code.entities.tables.records.DsgGatherStatisticsRecord; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.stereotype.Repository; 11 | 12 | /** 13 | * @author Administrator 14 | */ 15 | @Repository 16 | public class DsggatherStatisticsDao extends BaseDaoImpl { 17 | 18 | 19 | @Autowired 20 | private DSLContext dslContext; 21 | 22 | public DsggatherStatisticsDao() { 23 | super( DsgGatherStatistics.DSG_GATHER_STATISTICS ); 24 | } 25 | 26 | @Override 27 | public Configuration getConfiguration() { 28 | return dslContext.configuration(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/DefaultCatalog.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities; 5 | 6 | 7 | import java.util.ArrayList; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import javax.annotation.Generated; 12 | 13 | import org.jooq.Schema; 14 | import org.jooq.impl.CatalogImpl; 15 | 16 | 17 | /** 18 | * This class is generated by jOOQ. 19 | */ 20 | @Generated( 21 | value = { 22 | "http://www.jooq.org", 23 | "jOOQ version:3.11.9" 24 | }, 25 | comments = "This class is generated by jOOQ" 26 | ) 27 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 28 | public class DefaultCatalog extends CatalogImpl { 29 | 30 | private static final long serialVersionUID = -1644450025; 31 | 32 | /** 33 | * The reference instance of 34 | */ 35 | public static final DefaultCatalog DEFAULT_CATALOG = new DefaultCatalog(); 36 | 37 | /** 38 | * The schema kylo. 39 | */ 40 | public final Kylo KYLO = org.poem.code.entities.Kylo.KYLO; 41 | 42 | /** 43 | * No further instances allowed 44 | */ 45 | private DefaultCatalog() { 46 | super(""); 47 | } 48 | 49 | @Override 50 | public final List getSchemas() { 51 | List result = new ArrayList(); 52 | result.addAll(getSchemas0()); 53 | return result; 54 | } 55 | 56 | private final List getSchemas0() { 57 | return Arrays.asList( 58 | Kylo.KYLO); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/Indexes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities; 5 | 6 | 7 | import javax.annotation.Generated; 8 | 9 | import org.jooq.Index; 10 | import org.jooq.OrderField; 11 | import org.jooq.impl.Internal; 12 | import org.poem.code.entities.tables.DsgGatherDb; 13 | import org.poem.code.entities.tables.DsgGatherInfo; 14 | import org.poem.code.entities.tables.DsgGatherStatistics; 15 | import org.poem.code.entities.tables.DsgGatherTable; 16 | import org.poem.code.entities.tables.DsgGatherTableFields; 17 | 18 | 19 | /** 20 | * A class modelling indexes of tables of the kylo schema. 21 | */ 22 | @Generated( 23 | value = { 24 | "http://www.jooq.org", 25 | "jOOQ version:3.11.9" 26 | }, 27 | comments = "This class is generated by jOOQ" 28 | ) 29 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 30 | public class Indexes { 31 | 32 | // ------------------------------------------------------------------------- 33 | // INDEX definitions 34 | // ------------------------------------------------------------------------- 35 | 36 | public static final Index DSG_GATHER_DB_PRIMARY = Indexes0.DSG_GATHER_DB_PRIMARY; 37 | public static final Index DSG_GATHER_INFO_PRIMARY = Indexes0.DSG_GATHER_INFO_PRIMARY; 38 | public static final Index DSG_GATHER_STATISTICS_PRIMARY = Indexes0.DSG_GATHER_STATISTICS_PRIMARY; 39 | public static final Index DSG_GATHER_TABLE_PRIMARY = Indexes0.DSG_GATHER_TABLE_PRIMARY; 40 | public static final Index DSG_GATHER_TABLE_FIELDS_PRIMARY = Indexes0.DSG_GATHER_TABLE_FIELDS_PRIMARY; 41 | 42 | // ------------------------------------------------------------------------- 43 | // [#1459] distribute members to avoid static initialisers > 64kb 44 | // ------------------------------------------------------------------------- 45 | 46 | private static class Indexes0 { 47 | public static Index DSG_GATHER_DB_PRIMARY = Internal.createIndex("PRIMARY", DsgGatherDb.DSG_GATHER_DB, new OrderField[] { DsgGatherDb.DSG_GATHER_DB.ID }, true); 48 | public static Index DSG_GATHER_INFO_PRIMARY = Internal.createIndex("PRIMARY", DsgGatherInfo.DSG_GATHER_INFO, new OrderField[] { DsgGatherInfo.DSG_GATHER_INFO.ID }, true); 49 | public static Index DSG_GATHER_STATISTICS_PRIMARY = Internal.createIndex("PRIMARY", DsgGatherStatistics.DSG_GATHER_STATISTICS, new OrderField[] { DsgGatherStatistics.DSG_GATHER_STATISTICS.ID }, true); 50 | public static Index DSG_GATHER_TABLE_PRIMARY = Internal.createIndex("PRIMARY", DsgGatherTable.DSG_GATHER_TABLE, new OrderField[] { DsgGatherTable.DSG_GATHER_TABLE.ID }, true); 51 | public static Index DSG_GATHER_TABLE_FIELDS_PRIMARY = Internal.createIndex("PRIMARY", DsgGatherTableFields.DSG_GATHER_TABLE_FIELDS, new OrderField[] { DsgGatherTableFields.DSG_GATHER_TABLE_FIELDS.ID }, true); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/Keys.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities; 5 | 6 | 7 | import javax.annotation.Generated; 8 | 9 | import org.jooq.UniqueKey; 10 | import org.jooq.impl.Internal; 11 | import org.poem.code.entities.tables.DsgGatherDb; 12 | import org.poem.code.entities.tables.DsgGatherInfo; 13 | import org.poem.code.entities.tables.DsgGatherStatistics; 14 | import org.poem.code.entities.tables.DsgGatherTable; 15 | import org.poem.code.entities.tables.DsgGatherTableFields; 16 | import org.poem.code.entities.tables.records.DsgGatherDbRecord; 17 | import org.poem.code.entities.tables.records.DsgGatherInfoRecord; 18 | import org.poem.code.entities.tables.records.DsgGatherStatisticsRecord; 19 | import org.poem.code.entities.tables.records.DsgGatherTableFieldsRecord; 20 | import org.poem.code.entities.tables.records.DsgGatherTableRecord; 21 | 22 | 23 | /** 24 | * A class modelling foreign key relationships and constraints of tables of 25 | * the kylo schema. 26 | */ 27 | @Generated( 28 | value = { 29 | "http://www.jooq.org", 30 | "jOOQ version:3.11.9" 31 | }, 32 | comments = "This class is generated by jOOQ" 33 | ) 34 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 35 | public class Keys { 36 | 37 | // ------------------------------------------------------------------------- 38 | // IDENTITY definitions 39 | // ------------------------------------------------------------------------- 40 | 41 | 42 | // ------------------------------------------------------------------------- 43 | // UNIQUE and PRIMARY KEY definitions 44 | // ------------------------------------------------------------------------- 45 | 46 | public static final UniqueKey KEY_DSG_GATHER_DB_PRIMARY = UniqueKeys0.KEY_DSG_GATHER_DB_PRIMARY; 47 | public static final UniqueKey KEY_DSG_GATHER_INFO_PRIMARY = UniqueKeys0.KEY_DSG_GATHER_INFO_PRIMARY; 48 | public static final UniqueKey KEY_DSG_GATHER_STATISTICS_PRIMARY = UniqueKeys0.KEY_DSG_GATHER_STATISTICS_PRIMARY; 49 | public static final UniqueKey KEY_DSG_GATHER_TABLE_PRIMARY = UniqueKeys0.KEY_DSG_GATHER_TABLE_PRIMARY; 50 | public static final UniqueKey KEY_DSG_GATHER_TABLE_FIELDS_PRIMARY = UniqueKeys0.KEY_DSG_GATHER_TABLE_FIELDS_PRIMARY; 51 | 52 | // ------------------------------------------------------------------------- 53 | // FOREIGN KEY definitions 54 | // ------------------------------------------------------------------------- 55 | 56 | 57 | // ------------------------------------------------------------------------- 58 | // [#1459] distribute members to avoid static initialisers > 64kb 59 | // ------------------------------------------------------------------------- 60 | 61 | private static class UniqueKeys0 { 62 | public static final UniqueKey KEY_DSG_GATHER_DB_PRIMARY = Internal.createUniqueKey(DsgGatherDb.DSG_GATHER_DB, "KEY_dsg_gather_db_PRIMARY", DsgGatherDb.DSG_GATHER_DB.ID); 63 | public static final UniqueKey KEY_DSG_GATHER_INFO_PRIMARY = Internal.createUniqueKey(DsgGatherInfo.DSG_GATHER_INFO, "KEY_dsg_gather_info_PRIMARY", DsgGatherInfo.DSG_GATHER_INFO.ID); 64 | public static final UniqueKey KEY_DSG_GATHER_STATISTICS_PRIMARY = Internal.createUniqueKey(DsgGatherStatistics.DSG_GATHER_STATISTICS, "KEY_dsg_gather_statistics_PRIMARY", DsgGatherStatistics.DSG_GATHER_STATISTICS.ID); 65 | public static final UniqueKey KEY_DSG_GATHER_TABLE_PRIMARY = Internal.createUniqueKey(DsgGatherTable.DSG_GATHER_TABLE, "KEY_dsg_gather_table_PRIMARY", DsgGatherTable.DSG_GATHER_TABLE.ID); 66 | public static final UniqueKey KEY_DSG_GATHER_TABLE_FIELDS_PRIMARY = Internal.createUniqueKey(DsgGatherTableFields.DSG_GATHER_TABLE_FIELDS, "KEY_dsg_gather_table_fields_PRIMARY", DsgGatherTableFields.DSG_GATHER_TABLE_FIELDS.ID); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/Kylo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities; 5 | 6 | 7 | import java.util.ArrayList; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import javax.annotation.Generated; 12 | 13 | import org.jooq.Catalog; 14 | import org.jooq.Table; 15 | import org.jooq.impl.SchemaImpl; 16 | import org.poem.code.entities.tables.DsgGatherDb; 17 | import org.poem.code.entities.tables.DsgGatherInfo; 18 | import org.poem.code.entities.tables.DsgGatherStatistics; 19 | import org.poem.code.entities.tables.DsgGatherTable; 20 | import org.poem.code.entities.tables.DsgGatherTableFields; 21 | 22 | 23 | /** 24 | * This class is generated by jOOQ. 25 | */ 26 | @Generated( 27 | value = { 28 | "http://www.jooq.org", 29 | "jOOQ version:3.11.9" 30 | }, 31 | comments = "This class is generated by jOOQ" 32 | ) 33 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 34 | public class Kylo extends SchemaImpl { 35 | 36 | private static final long serialVersionUID = 703595012; 37 | 38 | /** 39 | * The reference instance of kylo 40 | */ 41 | public static final Kylo KYLO = new Kylo(); 42 | 43 | /** 44 | * 采集到库信息 45 | */ 46 | public final DsgGatherDb DSG_GATHER_DB = org.poem.code.entities.tables.DsgGatherDb.DSG_GATHER_DB; 47 | 48 | /** 49 | * 元数据采集信息 50 | */ 51 | public final DsgGatherInfo DSG_GATHER_INFO = org.poem.code.entities.tables.DsgGatherInfo.DSG_GATHER_INFO; 52 | 53 | /** 54 | * 采集表信息 55 | */ 56 | public final DsgGatherStatistics DSG_GATHER_STATISTICS = org.poem.code.entities.tables.DsgGatherStatistics.DSG_GATHER_STATISTICS; 57 | 58 | /** 59 | * 采集表信息 60 | */ 61 | public final DsgGatherTable DSG_GATHER_TABLE = org.poem.code.entities.tables.DsgGatherTable.DSG_GATHER_TABLE; 62 | 63 | /** 64 | * 采集表字端信息 65 | */ 66 | public final DsgGatherTableFields DSG_GATHER_TABLE_FIELDS = org.poem.code.entities.tables.DsgGatherTableFields.DSG_GATHER_TABLE_FIELDS; 67 | 68 | /** 69 | * No further instances allowed 70 | */ 71 | private Kylo() { 72 | super("kylo", null); 73 | } 74 | 75 | 76 | /** 77 | * {@inheritDoc} 78 | */ 79 | @Override 80 | public Catalog getCatalog() { 81 | return DefaultCatalog.DEFAULT_CATALOG; 82 | } 83 | 84 | @Override 85 | public final List> getTables() { 86 | List result = new ArrayList(); 87 | result.addAll(getTables0()); 88 | return result; 89 | } 90 | 91 | private final List> getTables0() { 92 | return Arrays.>asList( 93 | DsgGatherDb.DSG_GATHER_DB, 94 | DsgGatherInfo.DSG_GATHER_INFO, 95 | DsgGatherStatistics.DSG_GATHER_STATISTICS, 96 | DsgGatherTable.DSG_GATHER_TABLE, 97 | DsgGatherTableFields.DSG_GATHER_TABLE_FIELDS); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/Tables.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities; 5 | 6 | 7 | import javax.annotation.Generated; 8 | 9 | import org.poem.code.entities.tables.DsgGatherDb; 10 | import org.poem.code.entities.tables.DsgGatherInfo; 11 | import org.poem.code.entities.tables.DsgGatherStatistics; 12 | import org.poem.code.entities.tables.DsgGatherTable; 13 | import org.poem.code.entities.tables.DsgGatherTableFields; 14 | 15 | 16 | /** 17 | * Convenience access to all tables in kylo 18 | */ 19 | @Generated( 20 | value = { 21 | "http://www.jooq.org", 22 | "jOOQ version:3.11.9" 23 | }, 24 | comments = "This class is generated by jOOQ" 25 | ) 26 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 27 | public class Tables { 28 | 29 | /** 30 | * 采集到库信息 31 | */ 32 | public static final DsgGatherDb DSG_GATHER_DB = org.poem.code.entities.tables.DsgGatherDb.DSG_GATHER_DB; 33 | 34 | /** 35 | * 元数据采集信息 36 | */ 37 | public static final DsgGatherInfo DSG_GATHER_INFO = org.poem.code.entities.tables.DsgGatherInfo.DSG_GATHER_INFO; 38 | 39 | /** 40 | * 采集表信息 41 | */ 42 | public static final DsgGatherStatistics DSG_GATHER_STATISTICS = org.poem.code.entities.tables.DsgGatherStatistics.DSG_GATHER_STATISTICS; 43 | 44 | /** 45 | * 采集表信息 46 | */ 47 | public static final DsgGatherTable DSG_GATHER_TABLE = org.poem.code.entities.tables.DsgGatherTable.DSG_GATHER_TABLE; 48 | 49 | /** 50 | * 采集表字端信息 51 | */ 52 | public static final DsgGatherTableFields DSG_GATHER_TABLE_FIELDS = org.poem.code.entities.tables.DsgGatherTableFields.DSG_GATHER_TABLE_FIELDS; 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/tables/DsgGatherDb.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities.tables; 5 | 6 | 7 | import java.sql.Timestamp; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import javax.annotation.Generated; 12 | 13 | import org.jooq.Field; 14 | import org.jooq.ForeignKey; 15 | import org.jooq.Index; 16 | import org.jooq.Name; 17 | import org.jooq.Record; 18 | import org.jooq.Schema; 19 | import org.jooq.Table; 20 | import org.jooq.TableField; 21 | import org.jooq.UniqueKey; 22 | import org.jooq.impl.DSL; 23 | import org.jooq.impl.TableImpl; 24 | import org.poem.code.entities.Indexes; 25 | import org.poem.code.entities.Keys; 26 | import org.poem.code.entities.Kylo; 27 | import org.poem.code.entities.tables.records.DsgGatherDbRecord; 28 | 29 | 30 | /** 31 | * 采集到库信息 32 | */ 33 | @Generated( 34 | value = { 35 | "http://www.jooq.org", 36 | "jOOQ version:3.11.9" 37 | }, 38 | comments = "This class is generated by jOOQ" 39 | ) 40 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 41 | public class DsgGatherDb extends TableImpl { 42 | 43 | private static final long serialVersionUID = 742227479; 44 | 45 | /** 46 | * The reference instance of kylo.dsg_gather_db 47 | */ 48 | public static final DsgGatherDb DSG_GATHER_DB = new DsgGatherDb(); 49 | 50 | /** 51 | * The class holding records for this type 52 | */ 53 | @Override 54 | public Class getRecordType() { 55 | return DsgGatherDbRecord.class; 56 | } 57 | 58 | /** 59 | * The column kylo.dsg_gather_db.id. id 60 | */ 61 | public final TableField ID = createField("id", org.jooq.impl.SQLDataType.VARCHAR(50).nullable(false), this, "id"); 62 | 63 | /** 64 | * The column kylo.dsg_gather_db.gather_id. 元数据采集信息 65 | */ 66 | public final TableField GATHER_ID = createField("gather_id", org.jooq.impl.SQLDataType.VARCHAR(500).nullable(false), this, "元数据采集信息"); 67 | 68 | /** 69 | * The column kylo.dsg_gather_db.schema. schema 70 | */ 71 | public final TableField SCHEMA = createField("schema", org.jooq.impl.SQLDataType.VARCHAR(500), this, "schema"); 72 | 73 | /** 74 | * The column kylo.dsg_gather_db.description. 描述 75 | */ 76 | public final TableField DESCRIPTION = createField("description", org.jooq.impl.SQLDataType.VARCHAR(225), this, "描述"); 77 | 78 | /** 79 | * The column kylo.dsg_gather_db.create_time. 80 | */ 81 | public final TableField CREATE_TIME = createField("create_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 82 | 83 | /** 84 | * The column kylo.dsg_gather_db.update_time. 85 | */ 86 | public final TableField UPDATE_TIME = createField("update_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 87 | 88 | /** 89 | * Create a kylo.dsg_gather_db table reference 90 | */ 91 | public DsgGatherDb() { 92 | this(DSL.name("dsg_gather_db"), null); 93 | } 94 | 95 | /** 96 | * Create an aliased kylo.dsg_gather_db table reference 97 | */ 98 | public DsgGatherDb(String alias) { 99 | this(DSL.name(alias), DSG_GATHER_DB); 100 | } 101 | 102 | /** 103 | * Create an aliased kylo.dsg_gather_db table reference 104 | */ 105 | public DsgGatherDb(Name alias) { 106 | this(alias, DSG_GATHER_DB); 107 | } 108 | 109 | private DsgGatherDb(Name alias, Table aliased) { 110 | this(alias, aliased, null); 111 | } 112 | 113 | private DsgGatherDb(Name alias, Table aliased, Field[] parameters) { 114 | super(alias, null, aliased, parameters, DSL.comment("采集到库信息")); 115 | } 116 | 117 | public DsgGatherDb(Table child, ForeignKey key) { 118 | super(child, key, DSG_GATHER_DB); 119 | } 120 | 121 | /** 122 | * {@inheritDoc} 123 | */ 124 | @Override 125 | public Schema getSchema() { 126 | return Kylo.KYLO; 127 | } 128 | 129 | /** 130 | * {@inheritDoc} 131 | */ 132 | @Override 133 | public List getIndexes() { 134 | return Arrays.asList(Indexes.DSG_GATHER_DB_PRIMARY); 135 | } 136 | 137 | /** 138 | * {@inheritDoc} 139 | */ 140 | @Override 141 | public UniqueKey getPrimaryKey() { 142 | return Keys.KEY_DSG_GATHER_DB_PRIMARY; 143 | } 144 | 145 | /** 146 | * {@inheritDoc} 147 | */ 148 | @Override 149 | public List> getKeys() { 150 | return Arrays.>asList(Keys.KEY_DSG_GATHER_DB_PRIMARY); 151 | } 152 | 153 | /** 154 | * {@inheritDoc} 155 | */ 156 | @Override 157 | public DsgGatherDb as(String alias) { 158 | return new DsgGatherDb(DSL.name(alias), this); 159 | } 160 | 161 | /** 162 | * {@inheritDoc} 163 | */ 164 | @Override 165 | public DsgGatherDb as(Name alias) { 166 | return new DsgGatherDb(alias, this); 167 | } 168 | 169 | /** 170 | * Rename this table 171 | */ 172 | @Override 173 | public DsgGatherDb rename(String name) { 174 | return new DsgGatherDb(DSL.name(name), null); 175 | } 176 | 177 | /** 178 | * Rename this table 179 | */ 180 | @Override 181 | public DsgGatherDb rename(Name name) { 182 | return new DsgGatherDb(name, null); 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/tables/DsgGatherInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities.tables; 5 | 6 | 7 | import java.sql.Timestamp; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import javax.annotation.Generated; 12 | 13 | import org.jooq.Field; 14 | import org.jooq.ForeignKey; 15 | import org.jooq.Index; 16 | import org.jooq.Name; 17 | import org.jooq.Record; 18 | import org.jooq.Schema; 19 | import org.jooq.Table; 20 | import org.jooq.TableField; 21 | import org.jooq.UniqueKey; 22 | import org.jooq.impl.DSL; 23 | import org.jooq.impl.TableImpl; 24 | import org.poem.code.entities.Indexes; 25 | import org.poem.code.entities.Keys; 26 | import org.poem.code.entities.Kylo; 27 | import org.poem.code.entities.tables.records.DsgGatherInfoRecord; 28 | 29 | 30 | /** 31 | * 元数据采集信息 32 | */ 33 | @Generated( 34 | value = { 35 | "http://www.jooq.org", 36 | "jOOQ version:3.11.9" 37 | }, 38 | comments = "This class is generated by jOOQ" 39 | ) 40 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 41 | public class DsgGatherInfo extends TableImpl { 42 | 43 | private static final long serialVersionUID = -1641434727; 44 | 45 | /** 46 | * The reference instance of kylo.dsg_gather_info 47 | */ 48 | public static final DsgGatherInfo DSG_GATHER_INFO = new DsgGatherInfo(); 49 | 50 | /** 51 | * The class holding records for this type 52 | */ 53 | @Override 54 | public Class getRecordType() { 55 | return DsgGatherInfoRecord.class; 56 | } 57 | 58 | /** 59 | * The column kylo.dsg_gather_info.id. id 60 | */ 61 | public final TableField ID = createField("id", org.jooq.impl.SQLDataType.VARCHAR(50).nullable(false), this, "id"); 62 | 63 | /** 64 | * The column kylo.dsg_gather_info.name. 连接的名字 65 | */ 66 | public final TableField NAME = createField("name", org.jooq.impl.SQLDataType.VARCHAR(500).nullable(false), this, "连接的名字"); 67 | 68 | /** 69 | * The column kylo.dsg_gather_info.ip. 连接ip 70 | */ 71 | public final TableField IP = createField("ip", org.jooq.impl.SQLDataType.VARCHAR(500), this, "连接ip"); 72 | 73 | /** 74 | * The column kylo.dsg_gather_info.port. 连接的端口 75 | */ 76 | public final TableField PORT = createField("port", org.jooq.impl.SQLDataType.VARCHAR(500), this, "连接的端口"); 77 | 78 | /** 79 | * The column kylo.dsg_gather_info.user. 数据库连接的用户名字 80 | */ 81 | public final TableField USER = createField("user", org.jooq.impl.SQLDataType.VARCHAR(500), this, "数据库连接的用户名字"); 82 | 83 | /** 84 | * The column kylo.dsg_gather_info.password. 数据库连接的密码 85 | */ 86 | public final TableField PASSWORD = createField("password", org.jooq.impl.SQLDataType.VARCHAR(500), this, "数据库连接的密码"); 87 | 88 | /** 89 | * The column kylo.dsg_gather_info.type. 数据库类型 mysql或者是 orcale 90 | */ 91 | public final TableField TYPE = createField("type", org.jooq.impl.SQLDataType.VARCHAR(500), this, "数据库类型 mysql或者是 orcale"); 92 | 93 | /** 94 | * The column kylo.dsg_gather_info.service_name. oracle 服务名字 95 | */ 96 | public final TableField SERVICE_NAME = createField("service_name", org.jooq.impl.SQLDataType.VARCHAR(255), this, "oracle 服务名字"); 97 | 98 | /** 99 | * The column kylo.dsg_gather_info.remark. 描述 100 | */ 101 | public final TableField REMARK = createField("remark", org.jooq.impl.SQLDataType.VARCHAR(500), this, "描述"); 102 | 103 | /** 104 | * The column kylo.dsg_gather_info.delete. 是否删除 105 | */ 106 | public final TableField DELETE = createField("delete", org.jooq.impl.SQLDataType.TINYINT, this, "是否删除"); 107 | 108 | /** 109 | * The column kylo.dsg_gather_info.status. 状态 0 没有启动 1 启动 110 | */ 111 | public final TableField STATUS = createField("status", org.jooq.impl.SQLDataType.INTEGER, this, "状态 0 没有启动 1 启动"); 112 | 113 | /** 114 | * The column kylo.dsg_gather_info.create_time. 115 | */ 116 | public final TableField CREATE_TIME = createField("create_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 117 | 118 | /** 119 | * The column kylo.dsg_gather_info.update_time. 120 | */ 121 | public final TableField UPDATE_TIME = createField("update_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 122 | 123 | /** 124 | * Create a kylo.dsg_gather_info table reference 125 | */ 126 | public DsgGatherInfo() { 127 | this(DSL.name("dsg_gather_info"), null); 128 | } 129 | 130 | /** 131 | * Create an aliased kylo.dsg_gather_info table reference 132 | */ 133 | public DsgGatherInfo(String alias) { 134 | this(DSL.name(alias), DSG_GATHER_INFO); 135 | } 136 | 137 | /** 138 | * Create an aliased kylo.dsg_gather_info table reference 139 | */ 140 | public DsgGatherInfo(Name alias) { 141 | this(alias, DSG_GATHER_INFO); 142 | } 143 | 144 | private DsgGatherInfo(Name alias, Table aliased) { 145 | this(alias, aliased, null); 146 | } 147 | 148 | private DsgGatherInfo(Name alias, Table aliased, Field[] parameters) { 149 | super(alias, null, aliased, parameters, DSL.comment("元数据采集信息")); 150 | } 151 | 152 | public DsgGatherInfo(Table child, ForeignKey key) { 153 | super(child, key, DSG_GATHER_INFO); 154 | } 155 | 156 | /** 157 | * {@inheritDoc} 158 | */ 159 | @Override 160 | public Schema getSchema() { 161 | return Kylo.KYLO; 162 | } 163 | 164 | /** 165 | * {@inheritDoc} 166 | */ 167 | @Override 168 | public List getIndexes() { 169 | return Arrays.asList(Indexes.DSG_GATHER_INFO_PRIMARY); 170 | } 171 | 172 | /** 173 | * {@inheritDoc} 174 | */ 175 | @Override 176 | public UniqueKey getPrimaryKey() { 177 | return Keys.KEY_DSG_GATHER_INFO_PRIMARY; 178 | } 179 | 180 | /** 181 | * {@inheritDoc} 182 | */ 183 | @Override 184 | public List> getKeys() { 185 | return Arrays.>asList(Keys.KEY_DSG_GATHER_INFO_PRIMARY); 186 | } 187 | 188 | /** 189 | * {@inheritDoc} 190 | */ 191 | @Override 192 | public DsgGatherInfo as(String alias) { 193 | return new DsgGatherInfo(DSL.name(alias), this); 194 | } 195 | 196 | /** 197 | * {@inheritDoc} 198 | */ 199 | @Override 200 | public DsgGatherInfo as(Name alias) { 201 | return new DsgGatherInfo(alias, this); 202 | } 203 | 204 | /** 205 | * Rename this table 206 | */ 207 | @Override 208 | public DsgGatherInfo rename(String name) { 209 | return new DsgGatherInfo(DSL.name(name), null); 210 | } 211 | 212 | /** 213 | * Rename this table 214 | */ 215 | @Override 216 | public DsgGatherInfo rename(Name name) { 217 | return new DsgGatherInfo(name, null); 218 | } 219 | } 220 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/tables/DsgGatherStatistics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities.tables; 5 | 6 | 7 | import java.sql.Timestamp; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import javax.annotation.Generated; 12 | 13 | import org.jooq.Field; 14 | import org.jooq.ForeignKey; 15 | import org.jooq.Index; 16 | import org.jooq.Name; 17 | import org.jooq.Record; 18 | import org.jooq.Schema; 19 | import org.jooq.Table; 20 | import org.jooq.TableField; 21 | import org.jooq.UniqueKey; 22 | import org.jooq.impl.DSL; 23 | import org.jooq.impl.TableImpl; 24 | import org.poem.code.entities.Indexes; 25 | import org.poem.code.entities.Keys; 26 | import org.poem.code.entities.Kylo; 27 | import org.poem.code.entities.tables.records.DsgGatherStatisticsRecord; 28 | 29 | 30 | /** 31 | * 采集表信息 32 | */ 33 | @Generated( 34 | value = { 35 | "http://www.jooq.org", 36 | "jOOQ version:3.11.9" 37 | }, 38 | comments = "This class is generated by jOOQ" 39 | ) 40 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 41 | public class DsgGatherStatistics extends TableImpl { 42 | 43 | private static final long serialVersionUID = -415158464; 44 | 45 | /** 46 | * The reference instance of kylo.dsg_gather_statistics 47 | */ 48 | public static final DsgGatherStatistics DSG_GATHER_STATISTICS = new DsgGatherStatistics(); 49 | 50 | /** 51 | * The class holding records for this type 52 | */ 53 | @Override 54 | public Class getRecordType() { 55 | return DsgGatherStatisticsRecord.class; 56 | } 57 | 58 | /** 59 | * The column kylo.dsg_gather_statistics.id. id 60 | */ 61 | public final TableField ID = createField("id", org.jooq.impl.SQLDataType.VARCHAR(50).nullable(false), this, "id"); 62 | 63 | /** 64 | * The column kylo.dsg_gather_statistics.gather_info_id. 元数据信息, 和任务在一起 65 | */ 66 | public final TableField GATHER_INFO_ID = createField("gather_info_id", org.jooq.impl.SQLDataType.VARCHAR(500).nullable(false), this, "元数据信息, 和任务在一起"); 67 | 68 | /** 69 | * The column kylo.dsg_gather_statistics.gather_db_count. 抓取的数据库数量 70 | */ 71 | public final TableField GATHER_DB_COUNT = createField("gather_db_count", org.jooq.impl.SQLDataType.INTEGER.nullable(false), this, "抓取的数据库数量"); 72 | 73 | /** 74 | * The column kylo.dsg_gather_statistics.gather_table_count. 抓取的表的个数 75 | */ 76 | public final TableField GATHER_TABLE_COUNT = createField("gather_table_count", org.jooq.impl.SQLDataType.INTEGER.nullable(false), this, "抓取的表的个数"); 77 | 78 | /** 79 | * The column kylo.dsg_gather_statistics.gather_field_count. 抓取的表列的个数 80 | */ 81 | public final TableField GATHER_FIELD_COUNT = createField("gather_field_count", org.jooq.impl.SQLDataType.INTEGER.nullable(false), this, "抓取的表列的个数"); 82 | 83 | /** 84 | * The column kylo.dsg_gather_statistics.create_time. 85 | */ 86 | public final TableField CREATE_TIME = createField("create_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 87 | 88 | /** 89 | * The column kylo.dsg_gather_statistics.update_time. 90 | */ 91 | public final TableField UPDATE_TIME = createField("update_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 92 | 93 | /** 94 | * Create a kylo.dsg_gather_statistics table reference 95 | */ 96 | public DsgGatherStatistics() { 97 | this(DSL.name("dsg_gather_statistics"), null); 98 | } 99 | 100 | /** 101 | * Create an aliased kylo.dsg_gather_statistics table reference 102 | */ 103 | public DsgGatherStatistics(String alias) { 104 | this(DSL.name(alias), DSG_GATHER_STATISTICS); 105 | } 106 | 107 | /** 108 | * Create an aliased kylo.dsg_gather_statistics table reference 109 | */ 110 | public DsgGatherStatistics(Name alias) { 111 | this(alias, DSG_GATHER_STATISTICS); 112 | } 113 | 114 | private DsgGatherStatistics(Name alias, Table aliased) { 115 | this(alias, aliased, null); 116 | } 117 | 118 | private DsgGatherStatistics(Name alias, Table aliased, Field[] parameters) { 119 | super(alias, null, aliased, parameters, DSL.comment("采集表信息")); 120 | } 121 | 122 | public DsgGatherStatistics(Table child, ForeignKey key) { 123 | super(child, key, DSG_GATHER_STATISTICS); 124 | } 125 | 126 | /** 127 | * {@inheritDoc} 128 | */ 129 | @Override 130 | public Schema getSchema() { 131 | return Kylo.KYLO; 132 | } 133 | 134 | /** 135 | * {@inheritDoc} 136 | */ 137 | @Override 138 | public List getIndexes() { 139 | return Arrays.asList(Indexes.DSG_GATHER_STATISTICS_PRIMARY); 140 | } 141 | 142 | /** 143 | * {@inheritDoc} 144 | */ 145 | @Override 146 | public UniqueKey getPrimaryKey() { 147 | return Keys.KEY_DSG_GATHER_STATISTICS_PRIMARY; 148 | } 149 | 150 | /** 151 | * {@inheritDoc} 152 | */ 153 | @Override 154 | public List> getKeys() { 155 | return Arrays.>asList(Keys.KEY_DSG_GATHER_STATISTICS_PRIMARY); 156 | } 157 | 158 | /** 159 | * {@inheritDoc} 160 | */ 161 | @Override 162 | public DsgGatherStatistics as(String alias) { 163 | return new DsgGatherStatistics(DSL.name(alias), this); 164 | } 165 | 166 | /** 167 | * {@inheritDoc} 168 | */ 169 | @Override 170 | public DsgGatherStatistics as(Name alias) { 171 | return new DsgGatherStatistics(alias, this); 172 | } 173 | 174 | /** 175 | * Rename this table 176 | */ 177 | @Override 178 | public DsgGatherStatistics rename(String name) { 179 | return new DsgGatherStatistics(DSL.name(name), null); 180 | } 181 | 182 | /** 183 | * Rename this table 184 | */ 185 | @Override 186 | public DsgGatherStatistics rename(Name name) { 187 | return new DsgGatherStatistics(name, null); 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/tables/DsgGatherTable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities.tables; 5 | 6 | 7 | import java.sql.Timestamp; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import javax.annotation.Generated; 12 | 13 | import org.jooq.Field; 14 | import org.jooq.ForeignKey; 15 | import org.jooq.Index; 16 | import org.jooq.Name; 17 | import org.jooq.Record; 18 | import org.jooq.Schema; 19 | import org.jooq.Table; 20 | import org.jooq.TableField; 21 | import org.jooq.UniqueKey; 22 | import org.jooq.impl.DSL; 23 | import org.jooq.impl.TableImpl; 24 | import org.poem.code.entities.Indexes; 25 | import org.poem.code.entities.Keys; 26 | import org.poem.code.entities.Kylo; 27 | import org.poem.code.entities.tables.records.DsgGatherTableRecord; 28 | 29 | 30 | /** 31 | * 采集表信息 32 | */ 33 | @Generated( 34 | value = { 35 | "http://www.jooq.org", 36 | "jOOQ version:3.11.9" 37 | }, 38 | comments = "This class is generated by jOOQ" 39 | ) 40 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 41 | public class DsgGatherTable extends TableImpl { 42 | 43 | private static final long serialVersionUID = 1689055397; 44 | 45 | /** 46 | * The reference instance of kylo.dsg_gather_table 47 | */ 48 | public static final DsgGatherTable DSG_GATHER_TABLE = new DsgGatherTable(); 49 | 50 | /** 51 | * The class holding records for this type 52 | */ 53 | @Override 54 | public Class getRecordType() { 55 | return DsgGatherTableRecord.class; 56 | } 57 | 58 | /** 59 | * The column kylo.dsg_gather_table.id. id 60 | */ 61 | public final TableField ID = createField("id", org.jooq.impl.SQLDataType.VARCHAR(50).nullable(false), this, "id"); 62 | 63 | /** 64 | * The column kylo.dsg_gather_table.gather_db_id. 元数据数据库信息 65 | */ 66 | public final TableField GATHER_DB_ID = createField("gather_db_id", org.jooq.impl.SQLDataType.VARCHAR(500).nullable(false), this, "元数据数据库信息"); 67 | 68 | /** 69 | * The column kylo.dsg_gather_table.table. 数据库表 70 | */ 71 | public final TableField TABLE = createField("table", org.jooq.impl.SQLDataType.VARCHAR(500), this, "数据库表"); 72 | 73 | /** 74 | * The column kylo.dsg_gather_table.table_name. 数据库表名字 75 | */ 76 | public final TableField TABLE_NAME = createField("table_name", org.jooq.impl.SQLDataType.VARCHAR(1000), this, "数据库表名字"); 77 | 78 | /** 79 | * The column kylo.dsg_gather_table.comment. 描述 80 | */ 81 | public final TableField COMMENT = createField("comment", org.jooq.impl.SQLDataType.VARCHAR(1000), this, "描述"); 82 | 83 | /** 84 | * The column kylo.dsg_gather_table.create_time. 85 | */ 86 | public final TableField CREATE_TIME = createField("create_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 87 | 88 | /** 89 | * The column kylo.dsg_gather_table.update_time. 90 | */ 91 | public final TableField UPDATE_TIME = createField("update_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 92 | 93 | /** 94 | * Create a kylo.dsg_gather_table table reference 95 | */ 96 | public DsgGatherTable() { 97 | this(DSL.name("dsg_gather_table"), null); 98 | } 99 | 100 | /** 101 | * Create an aliased kylo.dsg_gather_table table reference 102 | */ 103 | public DsgGatherTable(String alias) { 104 | this(DSL.name(alias), DSG_GATHER_TABLE); 105 | } 106 | 107 | /** 108 | * Create an aliased kylo.dsg_gather_table table reference 109 | */ 110 | public DsgGatherTable(Name alias) { 111 | this(alias, DSG_GATHER_TABLE); 112 | } 113 | 114 | private DsgGatherTable(Name alias, Table aliased) { 115 | this(alias, aliased, null); 116 | } 117 | 118 | private DsgGatherTable(Name alias, Table aliased, Field[] parameters) { 119 | super(alias, null, aliased, parameters, DSL.comment("采集表信息")); 120 | } 121 | 122 | public DsgGatherTable(Table child, ForeignKey key) { 123 | super(child, key, DSG_GATHER_TABLE); 124 | } 125 | 126 | /** 127 | * {@inheritDoc} 128 | */ 129 | @Override 130 | public Schema getSchema() { 131 | return Kylo.KYLO; 132 | } 133 | 134 | /** 135 | * {@inheritDoc} 136 | */ 137 | @Override 138 | public List getIndexes() { 139 | return Arrays.asList(Indexes.DSG_GATHER_TABLE_PRIMARY); 140 | } 141 | 142 | /** 143 | * {@inheritDoc} 144 | */ 145 | @Override 146 | public UniqueKey getPrimaryKey() { 147 | return Keys.KEY_DSG_GATHER_TABLE_PRIMARY; 148 | } 149 | 150 | /** 151 | * {@inheritDoc} 152 | */ 153 | @Override 154 | public List> getKeys() { 155 | return Arrays.>asList(Keys.KEY_DSG_GATHER_TABLE_PRIMARY); 156 | } 157 | 158 | /** 159 | * {@inheritDoc} 160 | */ 161 | @Override 162 | public DsgGatherTable as(String alias) { 163 | return new DsgGatherTable(DSL.name(alias), this); 164 | } 165 | 166 | /** 167 | * {@inheritDoc} 168 | */ 169 | @Override 170 | public DsgGatherTable as(Name alias) { 171 | return new DsgGatherTable(alias, this); 172 | } 173 | 174 | /** 175 | * Rename this table 176 | */ 177 | @Override 178 | public DsgGatherTable rename(String name) { 179 | return new DsgGatherTable(DSL.name(name), null); 180 | } 181 | 182 | /** 183 | * Rename this table 184 | */ 185 | @Override 186 | public DsgGatherTable rename(Name name) { 187 | return new DsgGatherTable(name, null); 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/tables/DsgGatherTableFields.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities.tables; 5 | 6 | 7 | import java.sql.Timestamp; 8 | import java.util.Arrays; 9 | import java.util.List; 10 | 11 | import javax.annotation.Generated; 12 | 13 | import org.jooq.Field; 14 | import org.jooq.ForeignKey; 15 | import org.jooq.Index; 16 | import org.jooq.Name; 17 | import org.jooq.Record; 18 | import org.jooq.Schema; 19 | import org.jooq.Table; 20 | import org.jooq.TableField; 21 | import org.jooq.UniqueKey; 22 | import org.jooq.impl.DSL; 23 | import org.jooq.impl.TableImpl; 24 | import org.poem.code.entities.Indexes; 25 | import org.poem.code.entities.Keys; 26 | import org.poem.code.entities.Kylo; 27 | import org.poem.code.entities.tables.records.DsgGatherTableFieldsRecord; 28 | 29 | 30 | /** 31 | * 采集表字端信息 32 | */ 33 | @Generated( 34 | value = { 35 | "http://www.jooq.org", 36 | "jOOQ version:3.11.9" 37 | }, 38 | comments = "This class is generated by jOOQ" 39 | ) 40 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 41 | public class DsgGatherTableFields extends TableImpl { 42 | 43 | private static final long serialVersionUID = -1597933465; 44 | 45 | /** 46 | * The reference instance of kylo.dsg_gather_table_fields 47 | */ 48 | public static final DsgGatherTableFields DSG_GATHER_TABLE_FIELDS = new DsgGatherTableFields(); 49 | 50 | /** 51 | * The class holding records for this type 52 | */ 53 | @Override 54 | public Class getRecordType() { 55 | return DsgGatherTableFieldsRecord.class; 56 | } 57 | 58 | /** 59 | * The column kylo.dsg_gather_table_fields.id. id 60 | */ 61 | public final TableField ID = createField("id", org.jooq.impl.SQLDataType.VARCHAR(50).nullable(false), this, "id"); 62 | 63 | /** 64 | * The column kylo.dsg_gather_table_fields.gather_table_id. 表 65 | */ 66 | public final TableField GATHER_TABLE_ID = createField("gather_table_id", org.jooq.impl.SQLDataType.VARCHAR(500), this, "表"); 67 | 68 | /** 69 | * The column kylo.dsg_gather_table_fields.field_name. 字段 70 | */ 71 | public final TableField FIELD_NAME = createField("field_name", org.jooq.impl.SQLDataType.VARCHAR(225), this, "字段"); 72 | 73 | /** 74 | * The column kylo.dsg_gather_table_fields.data_type. 数据类型 75 | */ 76 | public final TableField DATA_TYPE = createField("data_type", org.jooq.impl.SQLDataType.VARCHAR(225), this, "数据类型"); 77 | 78 | /** 79 | * The column kylo.dsg_gather_table_fields.column_name. 描述 80 | */ 81 | public final TableField COLUMN_NAME = createField("column_name", org.jooq.impl.SQLDataType.VARCHAR(225), this, "描述"); 82 | 83 | /** 84 | * The column kylo.dsg_gather_table_fields.default_value. 默认值 85 | */ 86 | public final TableField DEFAULT_VALUE = createField("default_value", org.jooq.impl.SQLDataType.VARCHAR(225), this, "默认值"); 87 | 88 | /** 89 | * The column kylo.dsg_gather_table_fields.data_type_length. 数据长度 90 | */ 91 | public final TableField DATA_TYPE_LENGTH = createField("data_type_length", org.jooq.impl.SQLDataType.VARCHAR(225), this, "数据长度"); 92 | 93 | /** 94 | * The column kylo.dsg_gather_table_fields.create_time. 95 | */ 96 | public final TableField CREATE_TIME = createField("create_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 97 | 98 | /** 99 | * The column kylo.dsg_gather_table_fields.update_time. 100 | */ 101 | public final TableField UPDATE_TIME = createField("update_time", org.jooq.impl.SQLDataType.TIMESTAMP.nullable(false), this, ""); 102 | 103 | /** 104 | * Create a kylo.dsg_gather_table_fields table reference 105 | */ 106 | public DsgGatherTableFields() { 107 | this(DSL.name("dsg_gather_table_fields"), null); 108 | } 109 | 110 | /** 111 | * Create an aliased kylo.dsg_gather_table_fields table reference 112 | */ 113 | public DsgGatherTableFields(String alias) { 114 | this(DSL.name(alias), DSG_GATHER_TABLE_FIELDS); 115 | } 116 | 117 | /** 118 | * Create an aliased kylo.dsg_gather_table_fields table reference 119 | */ 120 | public DsgGatherTableFields(Name alias) { 121 | this(alias, DSG_GATHER_TABLE_FIELDS); 122 | } 123 | 124 | private DsgGatherTableFields(Name alias, Table aliased) { 125 | this(alias, aliased, null); 126 | } 127 | 128 | private DsgGatherTableFields(Name alias, Table aliased, Field[] parameters) { 129 | super(alias, null, aliased, parameters, DSL.comment("采集表字端信息")); 130 | } 131 | 132 | public DsgGatherTableFields(Table child, ForeignKey key) { 133 | super(child, key, DSG_GATHER_TABLE_FIELDS); 134 | } 135 | 136 | /** 137 | * {@inheritDoc} 138 | */ 139 | @Override 140 | public Schema getSchema() { 141 | return Kylo.KYLO; 142 | } 143 | 144 | /** 145 | * {@inheritDoc} 146 | */ 147 | @Override 148 | public List getIndexes() { 149 | return Arrays.asList(Indexes.DSG_GATHER_TABLE_FIELDS_PRIMARY); 150 | } 151 | 152 | /** 153 | * {@inheritDoc} 154 | */ 155 | @Override 156 | public UniqueKey getPrimaryKey() { 157 | return Keys.KEY_DSG_GATHER_TABLE_FIELDS_PRIMARY; 158 | } 159 | 160 | /** 161 | * {@inheritDoc} 162 | */ 163 | @Override 164 | public List> getKeys() { 165 | return Arrays.>asList(Keys.KEY_DSG_GATHER_TABLE_FIELDS_PRIMARY); 166 | } 167 | 168 | /** 169 | * {@inheritDoc} 170 | */ 171 | @Override 172 | public DsgGatherTableFields as(String alias) { 173 | return new DsgGatherTableFields(DSL.name(alias), this); 174 | } 175 | 176 | /** 177 | * {@inheritDoc} 178 | */ 179 | @Override 180 | public DsgGatherTableFields as(Name alias) { 181 | return new DsgGatherTableFields(alias, this); 182 | } 183 | 184 | /** 185 | * Rename this table 186 | */ 187 | @Override 188 | public DsgGatherTableFields rename(String name) { 189 | return new DsgGatherTableFields(DSL.name(name), null); 190 | } 191 | 192 | /** 193 | * Rename this table 194 | */ 195 | @Override 196 | public DsgGatherTableFields rename(Name name) { 197 | return new DsgGatherTableFields(name, null); 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/entities/tables/records/DsgGatherDbRecord.java: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is generated by jOOQ. 3 | */ 4 | package org.poem.code.entities.tables.records; 5 | 6 | 7 | import java.sql.Timestamp; 8 | 9 | import javax.annotation.Generated; 10 | 11 | import org.jooq.Field; 12 | import org.jooq.Record1; 13 | import org.jooq.Record6; 14 | import org.jooq.Row6; 15 | import org.jooq.impl.UpdatableRecordImpl; 16 | import org.poem.code.entities.tables.DsgGatherDb; 17 | 18 | 19 | /** 20 | * 采集到库信息 21 | */ 22 | @Generated( 23 | value = { 24 | "http://www.jooq.org", 25 | "jOOQ version:3.11.9" 26 | }, 27 | comments = "This class is generated by jOOQ" 28 | ) 29 | @SuppressWarnings({ "all", "unchecked", "rawtypes" }) 30 | public class DsgGatherDbRecord extends UpdatableRecordImpl implements Record6 { 31 | 32 | private static final long serialVersionUID = 1189173100; 33 | 34 | /** 35 | * Setter for kylo.dsg_gather_db.id. id 36 | */ 37 | public void setId(String value) { 38 | set(0, value); 39 | } 40 | 41 | /** 42 | * Getter for kylo.dsg_gather_db.id. id 43 | */ 44 | public String getId() { 45 | return (String) get(0); 46 | } 47 | 48 | /** 49 | * Setter for kylo.dsg_gather_db.gather_id. 元数据采集信息 50 | */ 51 | public void setGatherId(String value) { 52 | set(1, value); 53 | } 54 | 55 | /** 56 | * Getter for kylo.dsg_gather_db.gather_id. 元数据采集信息 57 | */ 58 | public String getGatherId() { 59 | return (String) get(1); 60 | } 61 | 62 | /** 63 | * Setter for kylo.dsg_gather_db.schema. schema 64 | */ 65 | public void setSchema(String value) { 66 | set(2, value); 67 | } 68 | 69 | /** 70 | * Getter for kylo.dsg_gather_db.schema. schema 71 | */ 72 | public String getSchema() { 73 | return (String) get(2); 74 | } 75 | 76 | /** 77 | * Setter for kylo.dsg_gather_db.description. 描述 78 | */ 79 | public void setDescription(String value) { 80 | set(3, value); 81 | } 82 | 83 | /** 84 | * Getter for kylo.dsg_gather_db.description. 描述 85 | */ 86 | public String getDescription() { 87 | return (String) get(3); 88 | } 89 | 90 | /** 91 | * Setter for kylo.dsg_gather_db.create_time. 92 | */ 93 | public void setCreateTime(Timestamp value) { 94 | set(4, value); 95 | } 96 | 97 | /** 98 | * Getter for kylo.dsg_gather_db.create_time. 99 | */ 100 | public Timestamp getCreateTime() { 101 | return (Timestamp) get(4); 102 | } 103 | 104 | /** 105 | * Setter for kylo.dsg_gather_db.update_time. 106 | */ 107 | public void setUpdateTime(Timestamp value) { 108 | set(5, value); 109 | } 110 | 111 | /** 112 | * Getter for kylo.dsg_gather_db.update_time. 113 | */ 114 | public Timestamp getUpdateTime() { 115 | return (Timestamp) get(5); 116 | } 117 | 118 | // ------------------------------------------------------------------------- 119 | // Primary key information 120 | // ------------------------------------------------------------------------- 121 | 122 | /** 123 | * {@inheritDoc} 124 | */ 125 | @Override 126 | public Record1 key() { 127 | return (Record1) super.key(); 128 | } 129 | 130 | // ------------------------------------------------------------------------- 131 | // Record6 type implementation 132 | // ------------------------------------------------------------------------- 133 | 134 | /** 135 | * {@inheritDoc} 136 | */ 137 | @Override 138 | public Row6 fieldsRow() { 139 | return (Row6) super.fieldsRow(); 140 | } 141 | 142 | /** 143 | * {@inheritDoc} 144 | */ 145 | @Override 146 | public Row6 valuesRow() { 147 | return (Row6) super.valuesRow(); 148 | } 149 | 150 | /** 151 | * {@inheritDoc} 152 | */ 153 | @Override 154 | public Field field1() { 155 | return DsgGatherDb.DSG_GATHER_DB.ID; 156 | } 157 | 158 | /** 159 | * {@inheritDoc} 160 | */ 161 | @Override 162 | public Field field2() { 163 | return DsgGatherDb.DSG_GATHER_DB.GATHER_ID; 164 | } 165 | 166 | /** 167 | * {@inheritDoc} 168 | */ 169 | @Override 170 | public Field field3() { 171 | return DsgGatherDb.DSG_GATHER_DB.SCHEMA; 172 | } 173 | 174 | /** 175 | * {@inheritDoc} 176 | */ 177 | @Override 178 | public Field field4() { 179 | return DsgGatherDb.DSG_GATHER_DB.DESCRIPTION; 180 | } 181 | 182 | /** 183 | * {@inheritDoc} 184 | */ 185 | @Override 186 | public Field field5() { 187 | return DsgGatherDb.DSG_GATHER_DB.CREATE_TIME; 188 | } 189 | 190 | /** 191 | * {@inheritDoc} 192 | */ 193 | @Override 194 | public Field field6() { 195 | return DsgGatherDb.DSG_GATHER_DB.UPDATE_TIME; 196 | } 197 | 198 | /** 199 | * {@inheritDoc} 200 | */ 201 | @Override 202 | public String component1() { 203 | return getId(); 204 | } 205 | 206 | /** 207 | * {@inheritDoc} 208 | */ 209 | @Override 210 | public String component2() { 211 | return getGatherId(); 212 | } 213 | 214 | /** 215 | * {@inheritDoc} 216 | */ 217 | @Override 218 | public String component3() { 219 | return getSchema(); 220 | } 221 | 222 | /** 223 | * {@inheritDoc} 224 | */ 225 | @Override 226 | public String component4() { 227 | return getDescription(); 228 | } 229 | 230 | /** 231 | * {@inheritDoc} 232 | */ 233 | @Override 234 | public Timestamp component5() { 235 | return getCreateTime(); 236 | } 237 | 238 | /** 239 | * {@inheritDoc} 240 | */ 241 | @Override 242 | public Timestamp component6() { 243 | return getUpdateTime(); 244 | } 245 | 246 | /** 247 | * {@inheritDoc} 248 | */ 249 | @Override 250 | public String value1() { 251 | return getId(); 252 | } 253 | 254 | /** 255 | * {@inheritDoc} 256 | */ 257 | @Override 258 | public String value2() { 259 | return getGatherId(); 260 | } 261 | 262 | /** 263 | * {@inheritDoc} 264 | */ 265 | @Override 266 | public String value3() { 267 | return getSchema(); 268 | } 269 | 270 | /** 271 | * {@inheritDoc} 272 | */ 273 | @Override 274 | public String value4() { 275 | return getDescription(); 276 | } 277 | 278 | /** 279 | * {@inheritDoc} 280 | */ 281 | @Override 282 | public Timestamp value5() { 283 | return getCreateTime(); 284 | } 285 | 286 | /** 287 | * {@inheritDoc} 288 | */ 289 | @Override 290 | public Timestamp value6() { 291 | return getUpdateTime(); 292 | } 293 | 294 | /** 295 | * {@inheritDoc} 296 | */ 297 | @Override 298 | public DsgGatherDbRecord value1(String value) { 299 | setId(value); 300 | return this; 301 | } 302 | 303 | /** 304 | * {@inheritDoc} 305 | */ 306 | @Override 307 | public DsgGatherDbRecord value2(String value) { 308 | setGatherId(value); 309 | return this; 310 | } 311 | 312 | /** 313 | * {@inheritDoc} 314 | */ 315 | @Override 316 | public DsgGatherDbRecord value3(String value) { 317 | setSchema(value); 318 | return this; 319 | } 320 | 321 | /** 322 | * {@inheritDoc} 323 | */ 324 | @Override 325 | public DsgGatherDbRecord value4(String value) { 326 | setDescription(value); 327 | return this; 328 | } 329 | 330 | /** 331 | * {@inheritDoc} 332 | */ 333 | @Override 334 | public DsgGatherDbRecord value5(Timestamp value) { 335 | setCreateTime(value); 336 | return this; 337 | } 338 | 339 | /** 340 | * {@inheritDoc} 341 | */ 342 | @Override 343 | public DsgGatherDbRecord value6(Timestamp value) { 344 | setUpdateTime(value); 345 | return this; 346 | } 347 | 348 | /** 349 | * {@inheritDoc} 350 | */ 351 | @Override 352 | public DsgGatherDbRecord values(String value1, String value2, String value3, String value4, Timestamp value5, Timestamp value6) { 353 | value1(value1); 354 | value2(value2); 355 | value3(value3); 356 | value4(value4); 357 | value5(value5); 358 | value6(value6); 359 | return this; 360 | } 361 | 362 | // ------------------------------------------------------------------------- 363 | // Constructors 364 | // ------------------------------------------------------------------------- 365 | 366 | /** 367 | * Create a detached DsgGatherDbRecord 368 | */ 369 | public DsgGatherDbRecord() { 370 | super(DsgGatherDb.DSG_GATHER_DB); 371 | } 372 | 373 | /** 374 | * Create a detached, initialised DsgGatherDbRecord 375 | */ 376 | public DsgGatherDbRecord(String id, String gatherId, String schema, String description, Timestamp createTime, Timestamp updateTime) { 377 | super(DsgGatherDb.DSG_GATHER_DB); 378 | 379 | set(0, id); 380 | set(1, gatherId); 381 | set(2, schema); 382 | set(3, description); 383 | set(4, createTime); 384 | set(5, updateTime); 385 | } 386 | } 387 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/service/DsggatherStatisticsServiceImpl.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.service; 2 | 3 | import org.apache.commons.collections.CollectionUtils; 4 | import org.poem.code.api.DsggatherStatisticsService; 5 | import org.poem.code.api.vo.statistics.DsggatherStatisticsVO; 6 | import org.poem.code.dao.statistics.DsggatherStatisticsDao; 7 | import org.poem.code.entities.tables.DsgGatherStatistics; 8 | import org.poem.code.entities.tables.records.DsgGatherStatisticsRecord; 9 | import org.poem.utils.SnowFlake; 10 | import org.poem.utils.ThreadUtils; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | import org.springframework.beans.factory.annotation.Autowired; 14 | import org.springframework.stereotype.Service; 15 | 16 | import java.sql.Timestamp; 17 | import java.util.List; 18 | import java.util.concurrent.*; 19 | import java.util.concurrent.atomic.AtomicInteger; 20 | import java.util.stream.Collectors; 21 | 22 | /** 23 | * 统计信息 24 | * @author Administrator 25 | */ 26 | @Service 27 | public class DsggatherStatisticsServiceImpl implements DsggatherStatisticsService { 28 | 29 | private static final Logger logger = LoggerFactory.getLogger( DsggatherStatisticsServiceImpl.class ); 30 | 31 | 32 | @Autowired 33 | private DsggatherStatisticsDao dsggatherStatisticsDao; 34 | 35 | 36 | /** 37 | * 38 | */ 39 | private static final ExecutorService EXECUTE_SERVICE; 40 | 41 | /** 42 | * 43 | */ 44 | static { 45 | logger.info( "Begin Init JDBC Thread Pool. " ); 46 | int procsssors = Runtime.getRuntime().availableProcessors(); 47 | final AtomicInteger threadNumber = new AtomicInteger( 0 ); 48 | EXECUTE_SERVICE = new ThreadPoolExecutor( procsssors * 2, Integer.MAX_VALUE, 7, TimeUnit.DAYS, 49 | new LinkedBlockingDeque<>(), new ThreadFactory() { 50 | @Override 51 | public Thread newThread(Runnable r) { 52 | Thread t = new Thread( r ); 53 | t.setName( "gather-" + threadNumber.getAndIncrement() ); 54 | t.setUncaughtExceptionHandler( (th, thr) -> { 55 | logger.error( th.getName() + " uncaught: " + thr.getClass().getName() ); 56 | thr.printStackTrace(); 57 | } ); 58 | return t; 59 | } 60 | } ); 61 | } 62 | 63 | /** 64 | * 保存数据 65 | * 保存 66 | */ 67 | @Override 68 | public void saveCurrent(DsggatherStatisticsVO dsggatherStatisticsVO){ 69 | EXECUTE_SERVICE.submit( new Runnable() { 70 | @Override 71 | public void run() { 72 | ThreadUtils.setTaskId( dsggatherStatisticsVO.getGatherId() ); 73 | save( dsggatherStatisticsVO ); 74 | } 75 | } ); 76 | } 77 | 78 | /** 79 | * @param dsggatherStatisticsVO 80 | */ 81 | @Override 82 | public synchronized void save(DsggatherStatisticsVO dsggatherStatisticsVO) { 83 | List records = 84 | dsggatherStatisticsDao.findByCondition( DsgGatherStatistics.DSG_GATHER_STATISTICS.GATHER_INFO_ID.eq( dsggatherStatisticsVO.getGatherId() ) ); 85 | DsgGatherStatisticsRecord dsgGatherStatisticsRecord; 86 | if (!CollectionUtils.isEmpty( records )) { 87 | dsgGatherStatisticsRecord = records.get( 0 ); 88 | } else { 89 | dsgGatherStatisticsRecord = new DsgGatherStatisticsRecord(); 90 | dsgGatherStatisticsRecord.setId( SnowFlake.genId() ); 91 | dsgGatherStatisticsRecord.setCreateTime( new Timestamp( System.currentTimeMillis() ) ); 92 | dsgGatherStatisticsRecord.setGatherDbCount( 0 ); 93 | dsgGatherStatisticsRecord.setGatherFieldCount( 0 ); 94 | dsgGatherStatisticsRecord.setGatherTableCount( 0 ); 95 | dsgGatherStatisticsRecord.setGatherInfoId( dsggatherStatisticsVO.getGatherId() ); 96 | } 97 | dsgGatherStatisticsRecord.setUpdateTime( new Timestamp( System.currentTimeMillis() ) ); 98 | dsgGatherStatisticsRecord.setGatherTableCount( dsgGatherStatisticsRecord.getGatherTableCount() + dsggatherStatisticsVO.getTableCount() ); 99 | dsgGatherStatisticsRecord.setGatherDbCount( dsgGatherStatisticsRecord.getGatherDbCount() + dsggatherStatisticsVO.getDbCount() ); 100 | dsgGatherStatisticsRecord.setGatherFieldCount( dsgGatherStatisticsRecord.getGatherFieldCount() + dsggatherStatisticsVO.getFieldCount() ); 101 | dsgGatherStatisticsRecord.setUpdateTime( new Timestamp( System.currentTimeMillis() ) ); 102 | if (CollectionUtils.isEmpty( records )) { 103 | this.dsggatherStatisticsDao.insert( dsgGatherStatisticsRecord ); 104 | } else { 105 | this.dsggatherStatisticsDao.update( dsgGatherStatisticsRecord ); 106 | } 107 | } 108 | 109 | /** 110 | * @return 111 | */ 112 | @Override 113 | public List query() { 114 | List records = 115 | dsggatherStatisticsDao.findAll(); 116 | return records.stream().map( 117 | o ->{ 118 | DsggatherStatisticsVO dsggatherStatisticsVO = new DsggatherStatisticsVO(); 119 | dsggatherStatisticsVO.setDbCount( o.getGatherDbCount()); 120 | dsggatherStatisticsVO.setTableCount( o.getGatherTableCount() ); 121 | dsggatherStatisticsVO.setFieldCount( o.getGatherFieldCount() ); 122 | return dsggatherStatisticsVO; 123 | } 124 | ).collect( Collectors.toList()); 125 | 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/service/connect/DataType.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.service.connect; 2 | 3 | 4 | /** 5 | * @author Administrator 6 | */ 7 | 8 | public enum DataType { 9 | /** 10 | * mysql 11 | */ 12 | MYSQL( "MYSQL" ), 13 | /** 14 | * oracle 15 | */ 16 | ORACLE( "ORACLE" ); 17 | 18 | private String name; 19 | 20 | private DataType(String name) { 21 | this.name = name; 22 | } 23 | 24 | public String getName() { 25 | return name; 26 | } 27 | 28 | public void setName(String name) { 29 | this.name = name; 30 | } 31 | 32 | public boolean eq(DataType dataType) { 33 | if (dataType == null) { 34 | return false; 35 | } 36 | String name = dataType.getName(); 37 | return name.equals( this.name ); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/service/connect/GatherBuilder.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.service.connect; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | public class GatherBuilder { 8 | 9 | private static final Logger logger = LoggerFactory.getLogger( GatherBuilder.class ); 10 | /** 11 | * 地址 12 | */ 13 | private String ip; 14 | 15 | /** 16 | * 端口号 17 | */ 18 | private String port; 19 | 20 | /** 21 | * 用户名 22 | */ 23 | private String userName; 24 | 25 | /** 26 | * 用户密码 27 | */ 28 | private String password; 29 | /** 30 | * s 31 | * 数据类型 32 | */ 33 | private DataType dataType; 34 | 35 | /** 36 | * oracle 服务名字 37 | */ 38 | private String serviceName; 39 | 40 | /** 41 | * @param ip 42 | * @return 43 | */ 44 | public GatherBuilder ip(String ip) { 45 | this.ip = ip; 46 | return this; 47 | } 48 | 49 | public GatherBuilder port(String port) { 50 | this.port = port; 51 | return this; 52 | } 53 | 54 | public GatherBuilder user(String user) { 55 | this.userName = user; 56 | return this; 57 | } 58 | 59 | public GatherBuilder password(String password) { 60 | this.password = password; 61 | return this; 62 | } 63 | 64 | public GatherBuilder dataType(DataType dataType) { 65 | this.dataType = dataType; 66 | return this; 67 | } 68 | 69 | public GatherBuilder serverName(String serviceName) { 70 | this.serviceName = serviceName; 71 | return this; 72 | } 73 | 74 | 75 | /** 76 | * @return 77 | * @throws ClassNotFoundException 78 | */ 79 | public GatherConnection budder() { 80 | if (StringUtils.isBlank( ip )) { 81 | throw new IllegalArgumentException( "ip not null !!!!!!!" ); 82 | } 83 | if (StringUtils.isBlank( password )) { 84 | throw new IllegalArgumentException( "password not null !!!!!!! " ); 85 | } 86 | if (StringUtils.isBlank( userName )) { 87 | throw new IllegalArgumentException( "user not null !!!!!!! " ); 88 | } 89 | if (StringUtils.isBlank( port )) { 90 | throw new IllegalArgumentException( "port not null !!!!!!! " ); 91 | } 92 | if (dataType == null) { 93 | throw new IllegalArgumentException( "dataType not null !!!!!!!" ); 94 | } 95 | if (StringUtils.isBlank( serviceName ) && dataType.eq( DataType.ORACLE )) { 96 | throw new IllegalArgumentException( "oracle server name can\'t be null !!!!!!!" ); 97 | } 98 | String url = null; 99 | try { 100 | if (dataType.equals( DataType.MYSQL )) { 101 | Class.forName( "com.mysql.cj.jdbc.Driver" ); 102 | url = "jdbc:mysql://" + ip + ":" + port + "?useSSL=false&serverTimezone=UTC"; 103 | } else if (dataType.eq( DataType.ORACLE )) { 104 | Class.forName( "oracle.jdbc.driver.OracleDriver" ); 105 | //jdbc:oracle:thin:@//localhost:1521:orcl 106 | //url = "jdbc:oracle:thin:@" + ip + ":" + port + ":orcl"; 107 | url = "jdbc:oracle:thin:@(description=(address=(protocol=tcp)(port=" + port + ")(host=" + ip + "))(connect_data=(service_name=" + serviceName + ")))"; 108 | } else { 109 | throw new IllegalArgumentException( "data type[" + dataType + "] is not unrecognizable !!!!!!" ); 110 | } 111 | } catch (ClassNotFoundException e) { 112 | e.printStackTrace(); 113 | logger.error( e.getLocalizedMessage(), e ); 114 | } 115 | if (url == null) { 116 | throw new NullPointerException( "url is null!!!!!!" ); 117 | } 118 | return new GatherConnection( url, password, userName ); 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/service/connect/GatherConnection.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.service.connect; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.Closeable; 7 | import java.io.IOException; 8 | import java.sql.Connection; 9 | import java.sql.DriverManager; 10 | import java.sql.SQLException; 11 | 12 | /** 13 | * @author Administrator 14 | */ 15 | public class GatherConnection implements Closeable { 16 | 17 | /** 18 | * 日志 19 | */ 20 | private static final Logger logger = LoggerFactory.getLogger( GatherConnection.class ); 21 | /** 22 | * 地址 23 | */ 24 | private String url; 25 | 26 | /** 27 | * 用户密码 28 | */ 29 | private String password; 30 | 31 | /** 32 | * 数据类型 33 | */ 34 | private String userName; 35 | 36 | /** 37 | * 数据连接 38 | */ 39 | 40 | private Connection connection; 41 | 42 | /** 43 | * 初始化 44 | * 45 | * @param url 46 | * @param password 47 | * @param userName 48 | */ 49 | public GatherConnection(String url, String password, String userName) { 50 | this.url = url; 51 | this.password = password; 52 | this.userName = userName; 53 | try { 54 | connection = DriverManager.getConnection( url, userName, password ); 55 | } catch (SQLException e) { 56 | logger.error( e.getMessage(), e ); 57 | e.printStackTrace(); 58 | } 59 | } 60 | 61 | /** 62 | * @return 63 | */ 64 | public Connection getConnection() { 65 | return connection; 66 | } 67 | 68 | 69 | /** 70 | * 必须关闭 71 | * 72 | * @throws IOException 73 | */ 74 | @Override 75 | public void close() { 76 | try { 77 | this.connection.close(); 78 | } catch (SQLException e) { 79 | logger.error( e.getMessage(), e ); 80 | e.printStackTrace(); 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/service/databases/GatherDataBaseInter.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.service.databases; 2 | 3 | import org.poem.code.api.vo.column.entity.DataSetVO; 4 | import org.poem.code.api.vo.databases.entity.DateBaseEntity; 5 | import org.poem.code.api.vo.page.Page; 6 | import org.poem.code.api.vo.table.entity.TableEntity; 7 | 8 | import java.sql.Connection; 9 | import java.util.List; 10 | 11 | /** 12 | * 数据结构 13 | * 获取数据库 14 | * 并且获取数据库表 15 | * @author Administrator 16 | */ 17 | public interface GatherDataBaseInter { 18 | 19 | 20 | /** 21 | * 22 | * 获取所有数据库 23 | * @param connection 24 | * @return 25 | */ 26 | public List getDataBases(Connection connection); 27 | 28 | 29 | /** 30 | * 获取所有的表列表 31 | * @return 32 | */ 33 | List getAllTableName(String db, Connection connection ); 34 | 35 | /** 36 | * 获取表的列信息 37 | * @param databases 38 | * @param table 39 | * @return 40 | */ 41 | DataSetVO getTaleFields(String databases, String table, Connection connection); 42 | 43 | 44 | 45 | /** 46 | * 获取数据表的数据 47 | * 48 | * @param tableEntity 　数据表 49 | * @return 50 | */ 51 | DataSetVO getTableColumnsDatas(TableEntity tableEntity, Connection connection); 52 | 53 | /** 54 | * 所有数据的个数 55 | * 56 | * @param tableEntity 57 | * @return 58 | */ 59 | long countTableDatas(TableEntity tableEntity, Connection connection); 60 | 61 | 62 | /** 63 | * 分页 64 | * 65 | * @param tableEntity 66 | * @param page 67 | * @return 68 | */ 69 | DataSetVO getTableColumnsDatas(TableEntity tableEntity, Page page, Connection connection); 70 | 71 | 72 | /** 73 | * 查询信息 74 | * @param sql 需要执行的sql 75 | * @param connection 连接信息 76 | * @return 77 | */ 78 | DataSetVO executeSql(String sql, Connection connection); 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/service/databases/MysqlGatherData.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.service.databases; 2 | 3 | import org.poem.code.service.databases.abstracts.AbstractGatherDataBaseInter; 4 | import org.springframework.stereotype.Service; 5 | 6 | /** 7 | * @author Administrator 8 | */ 9 | @Service(MysqlGatherData.BEAN) 10 | public class MysqlGatherData extends AbstractGatherDataBaseInter { 11 | public static final String BEAN = "mysql_gather_data"; 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/org/poem/code/service/databases/OracleGatherData.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.service.databases; 2 | 3 | import com.google.common.collect.Lists; 4 | import org.poem.code.api.vo.column.entity.DataSetVO; 5 | import org.poem.code.api.vo.databases.entity.DateBaseEntity; 6 | import org.poem.code.api.vo.table.entity.TableEntity; 7 | import org.poem.code.service.databases.abstracts.AbstractGatherDataBaseInter; 8 | import org.poem.utils.GatherDataSourceUtils; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.stereotype.Service; 12 | 13 | import java.sql.*; 14 | import java.util.ArrayList; 15 | import java.util.Arrays; 16 | import java.util.List; 17 | 18 | /** 19 | * @author Administrator 20 | */ 21 | @Service(OracleGatherData.BEAN) 22 | public class OracleGatherData extends AbstractGatherDataBaseInter { 23 | 24 | public static final String BEAN = "oracle_gather_data"; 25 | 26 | private static final Logger logger = LoggerFactory.getLogger( OracleGatherData.class ); 27 | 28 | 29 | /** 30 | * @param connection 31 | * @return 32 | */ 33 | @Override 34 | public List getDataBases(Connection connection) { 35 | List databases = Lists.newArrayList(); 36 | if (connection != null) { 37 | Statement stmt = null; 38 | ResultSet res = null; 39 | try { 40 | stmt = connection.createStatement(); 41 | res = stmt.executeQuery( "select DISTINCT OWNER from all_tab_comments where TABLE_TYPE='TABLE' " ); 42 | while (res.next()) { 43 | DateBaseEntity entity = new DateBaseEntity(); 44 | entity.setDatabaseName( res.getString( 1 ) ); 45 | databases.add( entity ); 46 | } 47 | } catch (SQLException e) { 48 | logger.error( e.getMessage(), e ); 49 | e.printStackTrace(); 50 | } finally { 51 | GatherDataSourceUtils.release( stmt, res ); 52 | } 53 | } 54 | return databases; 55 | } 56 | 57 | /** 58 | * 59 | * @param schema 60 | * @param conn 61 | * @return 62 | */ 63 | @Override 64 | public List getAllTableName(String schema, Connection conn) { 65 | Statement stmt = null; 66 | ResultSet res = null; 67 | if (conn == null) { 68 | return Lists.newArrayList(); 69 | } 70 | List tableEntities = Lists.newArrayList(); 71 | try { 72 | boolean cl = conn.isClosed(); 73 | if (!cl) { 74 | stmt = conn.createStatement(); 75 | res = stmt.executeQuery( "select TABLE_NAME, COMMENTS from all_tab_comments where TABLE_TYPE='TABLE' and OWNER = '" + schema + "'" ); 76 | while (res.next()) { 77 | TableEntity tableEntity = new TableEntity(); 78 | tableEntity.setDatabaseName( schema ); 79 | tableEntity.setComment( res.getString( 2 ) ); 80 | tableEntity.setTableName( res.getString( 1 ) ); 81 | tableEntities.add( tableEntity ); 82 | } 83 | } 84 | } catch (SQLException e) { 85 | logger.error( e.getMessage(), e ); 86 | e.printStackTrace(); 87 | } finally { 88 | GatherDataSourceUtils.release( stmt, res ); 89 | } 90 | return tableEntities; 91 | } 92 | 93 | /** 94 | * @param databases 95 | * @param table 96 | * @param connection 97 | * @return 98 | */ 99 | @Override 100 | public DataSetVO getTaleFields(String databases, String table, Connection connection) { 101 | List metas = Arrays.asList( "column_name", "type_name", "content" ); 102 | List> datas = new ArrayList<>(); 103 | DatabaseMetaData databaseMetaData = null; 104 | ResultSet colRet = null; 105 | try { 106 | databaseMetaData = connection.getMetaData(); 107 | String columnName; 108 | String columnType; 109 | colRet = databaseMetaData.getColumns( null, databases, table.toUpperCase(), "%" ); 110 | while (colRet.next()) { 111 | columnName = colRet.getString( "COLUMN_NAME" ); 112 | columnType = colRet.getString( "TYPE_NAME" ); 113 | List datass = Arrays.asList( columnName, columnType.toLowerCase(), colRet.getString( "REMARKS" ) ); 114 | datas.add( datass ); 115 | } 116 | } catch (SQLException e) { 117 | logger.error( e.getMessage(), e ); 118 | e.printStackTrace(); 119 | } finally { 120 | GatherDataSourceUtils.release( null, colRet ); 121 | } 122 | DataSetVO dataSetVO = new DataSetVO(); 123 | dataSetVO.setDatas( datas ); 124 | dataSetVO.setColumns( metas ); 125 | dataSetVO.setDatabaseName( databases ); 126 | dataSetVO.setTableName( table ); 127 | return dataSetVO; 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/main/java/org/poem/conf/JooqConfiguration.java: -------------------------------------------------------------------------------- 1 | package org.poem.conf; 2 | 3 | 4 | import com.mysql.cj.jdbc.MysqlDataSource; 5 | import org.jooq.SQLDialect; 6 | import org.jooq.conf.Settings; 7 | import org.jooq.impl.DataSourceConnectionProvider; 8 | import org.jooq.impl.DefaultConfiguration; 9 | import org.jooq.impl.DefaultDSLContext; 10 | import org.springframework.beans.factory.annotation.Autowired; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | import org.springframework.context.annotation.Import; 14 | import org.springframework.data.jdbc.repository.config.EnableJdbcRepositories; 15 | import org.springframework.data.jdbc.repository.config.JdbcConfiguration; 16 | import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; 17 | 18 | /** 19 | * @author Administrator 20 | */ 21 | @Configuration 22 | @EnableJdbcRepositories 23 | @Import(JdbcConfiguration.class) 24 | public class JooqConfiguration { 25 | 26 | @Autowired 27 | private MysqlDataSource dataSource; 28 | 29 | @Bean 30 | public DataSourceConnectionProvider connectionProvider() { 31 | return new DataSourceConnectionProvider( new TransactionAwareDataSourceProxy( dataSource ) ); 32 | } 33 | 34 | @Bean 35 | DefaultDSLContext dsl() { 36 | 37 | Settings settings = new Settings(); 38 | settings.withRenderSchema( false ); 39 | org.jooq.Configuration conf = new DefaultConfiguration(); 40 | TransactionAwareDataSourceProxy proxy = new TransactionAwareDataSourceProxy( dataSource ); 41 | conf.set( new DataSourceConnectionProvider( proxy ) ) 42 | .set( settings ) 43 | .set( SQLDialect.MYSQL ); 44 | DefaultDSLContext defaultDSLContext = new DefaultDSLContext( conf ); 45 | return defaultDSLContext; 46 | } 47 | 48 | public DefaultConfiguration configuration() { 49 | DefaultConfiguration jooqConfiguration = new DefaultConfiguration(); 50 | jooqConfiguration.set( connectionProvider() ); 51 | return jooqConfiguration; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/org/poem/conf/MysqlConf.java: -------------------------------------------------------------------------------- 1 | package org.poem.conf; 2 | 3 | import com.mysql.cj.jdbc.MysqlDataSource; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | 10 | /** 11 | * @author Administrator 12 | */ 13 | @Configuration 14 | public class MysqlConf { 15 | 16 | private static final Logger logger = LoggerFactory.getLogger( MysqlConf.class ); 17 | @Value("${spring.datasource.url}") 18 | private String connectionUrl; 19 | @Value("${spring.datasource.username}") 20 | private String username; 21 | @Value("${spring.datasource.password}") 22 | private String password; 23 | 24 | /** 25 | * 配置数据源 26 | * 27 | * @return 28 | */ 29 | @Bean(name = "mysqlDatasource") 30 | public MysqlDataSource initDataSource() { 31 | logger.info( "Init Mysql Datasource" ); 32 | MysqlDataSource basicDataSource = new MysqlDataSource(); 33 | basicDataSource.setUrl( connectionUrl ); 34 | basicDataSource.setUser( username ); 35 | basicDataSource.setPassword( password ); 36 | return basicDataSource; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/org/poem/gen/JooqPgGen.java: -------------------------------------------------------------------------------- 1 | package org.poem.gen; 2 | 3 | import org.jooq.codegen.GenerationTool; 4 | import org.jooq.meta.jaxb.*; 5 | import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; 6 | import org.springframework.core.io.ClassPathResource; 7 | 8 | import java.util.Properties; 9 | 10 | 11 | public class JooqPgGen { 12 | 13 | public static void main(String args[]) { 14 | YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); 15 | yaml.setResources( new ClassPathResource( "application-gen.yml" ) ); 16 | Properties properties = yaml.getObject(); 17 | JooqPgGen jg = new JooqPgGen(); 18 | jg.gen( properties ); 19 | System.out.println( "JooqPgGenerator finished." ); 20 | } 21 | 22 | /** 23 | * Jooq 自动化生成代码 24 | */ 25 | public void gen(Properties props) { 26 | 27 | String driverClassName = props.getProperty( "spring.datasource.driver-class-name" ); 28 | String username = props.getProperty( "spring.datasource.username" ); 29 | String url = props.getProperty( "spring.datasource.url" ); 30 | String password = props.getProperty( "spring.datasource.password" ); 31 | String schema = "kylo"; 32 | 33 | Jdbc jdbc = new Jdbc().withDriver( driverClassName ) 34 | .withUrl( url ) 35 | .withUser( username ) 36 | .withPassword( password ); 37 | Database database = new Database() 38 | .withName( "org.jooq.meta.mysql.MySQLDatabase" ) 39 | .withExcludes( "schema_version" ) 40 | .withSchemata( new Schema().withInputSchema( schema ) ); 41 | Generator generator = new Generator() 42 | .withName( "org.jooq.codegen.DefaultGenerator" ) 43 | .withDatabase( database ) 44 | .withGenerate( new Generate() 45 | ) 46 | .withTarget( new Target() 47 | .withPackageName( "org.poem.code.entities" ).withDirectory( "src/main/java" ) ); 48 | Configuration conf = new Configuration() 49 | .withJdbc( jdbc ) 50 | .withGenerator( generator ); 51 | 52 | try { 53 | GenerationTool.generate( conf ); 54 | } catch (Exception e) { 55 | e.printStackTrace(); 56 | } 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/common/DBDataImport.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.common; 2 | 3 | import org.poem.hadoop.common.dbUtil.ConnUtil; 4 | import org.poem.hadoop.common.dbUtil.DBUtil; 5 | import org.poem.hadoop.common.dbUtil.DataSourceType; 6 | import org.poem.hadoop.hive.HiveOperate; 7 | import org.poem.hadoop.model.ColumnInfo; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.sql.Connection; 12 | import java.util.List; 13 | 14 | /** 15 | * Created by sai.luo on 2017-5-19. 16 | */ 17 | public class DBDataImport extends Thread { 18 | 19 | private static final Logger log = LoggerFactory.getLogger(DBDataImport.class) ; 20 | private FileWrite fileWrite = new FileWrite(); 21 | private static final String split = "\t"; 22 | 23 | private String username ; 24 | private String password; 25 | private String url ; 26 | private int dataSourceType ; 27 | private String sourceTableName; 28 | private String localTempPath ; 29 | /** 30 | * 31 | * @param username 用户名 32 | * @param password 密码 33 | * @param url 数据库路径 34 | * @param dataSourceType 数据库类型 35 | * @param tableName 数据库表名 36 | */ 37 | public DBDataImport(String username, String password, String url, int dataSourceType, String tableName,String localTempPath) { 38 | this.username = username; 39 | this.password = password; 40 | this.url = url; 41 | this.dataSourceType = dataSourceType; 42 | this.sourceTableName = tableName; 43 | this.localTempPath = localTempPath ; 44 | } 45 | 46 | @Override 47 | public void run() { 48 | 49 | try { 50 | //1、获取数据库连接 51 | Connection conn = ConnUtil.getConn(DataSourceType.getDriverName(dataSourceType),url,username,password); 52 | //2、获取表的信息 53 | List columnMapList = DBUtil.getTableColumnInfo(sourceTableName, conn); 54 | //3、转换为hive表的对应类型 55 | DBUtil.tableColumnInfoToHiveColnmnInfo(columnMapList,dataSourceType); 56 | columnMapList.forEach(columnInfo -> log.info(columnInfo.toString())); 57 | //4、获取表的所有数据 58 | List> resultList = DBUtil.getTableDatas(sourceTableName, conn); 59 | int dataCount = resultList.size(); 60 | //5、写入文件 并导入到hive 61 | if (dataCount > 0) { 62 | String tmpFile = sourceTableName; 63 | fileWrite.writeMapList2File(resultList, localTempPath+ sourceTableName,split); 64 | HiveOperate.executeHiveOperate(sourceTableName,tmpFile,columnMapList,split,localTempPath); 65 | } 66 | } catch (Exception e) { 67 | e.printStackTrace(); 68 | log.error(e.getMessage()); 69 | } 70 | 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/common/FileWrite.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.common; 2 | 3 | import org.springframework.stereotype.Repository; 4 | 5 | import java.io.BufferedWriter; 6 | import java.io.FileWriter; 7 | import java.sql.Date; 8 | import java.sql.Timestamp; 9 | import java.util.List; 10 | 11 | /** 12 | * Created by suntree.xu on 2016-8-11. 13 | */ 14 | @Repository 15 | public class FileWrite { 16 | 17 | public static void main(String[] args){ 18 | try { 19 | FileWriter fw = new FileWriter("e:/tmp1.txt"); 20 | BufferedWriter bw=new BufferedWriter(fw); 21 | fw.write("sss"); 22 | fw.flush(); 23 | fw.close(); 24 | }catch (Exception e){ 25 | e.printStackTrace(); 26 | } 27 | 28 | } 29 | 30 | /** 31 | * 以字符写入 32 | * @param mapList 33 | * @param fileName 34 | * @throws Exception 35 | */ 36 | public void writeMapList2File(List> mapList, String fileName, String split) throws Exception{ 37 | FileWriter fw = new FileWriter(fileName); 38 | String sb; 39 | split = split==null?"|":split; 40 | for(int i = 0; i < mapList.size(); i++){ 41 | sb = ""; 42 | boolean hasBlank = false; 43 | List row = mapList.get(i); 44 | for(Object value :row){ 45 | if(value instanceof Date){ 46 | sb += DateTimeUtil.format((Date)value,"yyyy-MM-dd")+split; 47 | }else if(value instanceof Timestamp){ 48 | sb += DateTimeUtil.format((Timestamp)value,"yyyy-MM-dd HH:mm:ss")+split; 49 | }else if(value ==null){ 50 | sb +=split; 51 | hasBlank = true ; 52 | }else { 53 | sb += value+split; 54 | } 55 | } 56 | sb += "\n"; 57 | try{ 58 | fw.write(sb); 59 | fw.flush(); 60 | }catch (Exception e) { 61 | e.printStackTrace(); 62 | } 63 | } 64 | fw.close(); 65 | } 66 | 67 | public void writeList2File(List mapList,String fileName) throws Exception{ 68 | FileWriter fw = new FileWriter(fileName); 69 | String sb; 70 | for(int i = 0; i < mapList.size(); i++){ 71 | sb = ""; 72 | sb += mapList.get(i); 73 | sb += "\n"; 74 | fw.write(sb); 75 | fw.flush(); 76 | } 77 | fw.close(); 78 | } 79 | 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/common/NumberValidationUtils.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.common; 2 | 3 | import java.util.regex.Matcher; 4 | import java.util.regex.Pattern; 5 | 6 | /** 7 | * Created by sai.luo on 2017-5-16. 8 | */ 9 | public class NumberValidationUtils { 10 | 11 | public static void main(String[] args) { 12 | 13 | String a = "12."; 14 | boolean decimal = NumberValidationUtils.isDecimal(a); 15 | boolean wholeNumber = NumberValidationUtils.isWholeNumber(a); 16 | System.out.println(decimal); 17 | } 18 | private static boolean isMatch(String regex, String orginal){ 19 | if (orginal == null || orginal.trim().equals("")) { 20 | return false; 21 | } 22 | Pattern pattern = Pattern.compile(regex); 23 | Matcher isNum = pattern.matcher(orginal); 24 | return isNum.matches(); 25 | } 26 | 27 | public static boolean isPositiveInteger(String orginal) { 28 | return isMatch("^\\+{0,1}[1-9]\\d*", orginal); 29 | } 30 | 31 | public static boolean isNegativeInteger(String orginal) { 32 | return isMatch("^-[1-9]\\d*", orginal); 33 | } 34 | 35 | public static boolean isWholeNumber(String orginal) { 36 | return isMatch("[+-]{0,1}0", orginal) || isPositiveInteger(orginal) || isNegativeInteger(orginal); 37 | } 38 | 39 | public static boolean isPositiveDecimal(String orginal){ 40 | return isMatch("\\+{0,1}[0]\\.[1-9]*|\\+{0,1}[1-9]\\d*\\.\\d*", orginal); 41 | } 42 | 43 | public static boolean isNegativeDecimal(String orginal){ 44 | return isMatch("^-[0]\\.[1-9]*|^-[1-9]\\d*\\.\\d*", orginal); 45 | } 46 | 47 | public static boolean isDecimal(String orginal){ 48 | return isMatch("[-+]{0,1}\\d+\\.\\d*|[-+]{0,1}\\d*\\.\\d+", orginal); 49 | } 50 | 51 | public static boolean isRealNumber(String orginal){ 52 | return isWholeNumber(orginal) || isDecimal(orginal); 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/common/constant/Constant.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.common.constant; 2 | 3 | /** 4 | * 5 | */ 6 | public interface Constant { 7 | final String ROW_ID = "row_id"; 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/common/dbUtil/ConnUtil.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.common.dbUtil; 2 | 3 | import java.sql.*; 4 | 5 | /** 6 | * 7 | * @author sai.luo 8 | * @date 2017-5-18 9 | */ 10 | public class ConnUtil { 11 | 12 | 13 | /** 14 | * 得到连接 15 | * @return 16 | * @throws SQLException 17 | * @throws ClassNotFoundException 18 | */ 19 | public static Connection getConn(String driverName, String url, String user, String password) throws SQLException,ClassNotFoundException{ 20 | Class.forName(driverName); 21 | return DriverManager.getConnection(url, user, password); 22 | } 23 | 24 | /** 25 | * 关闭连接 26 | * @param conn 27 | * @throws SQLException 28 | */ 29 | static void close(Connection conn) throws SQLException{ 30 | if(conn != null){ 31 | conn.close(); 32 | } 33 | } 34 | 35 | /** 36 | * 关闭Statement 37 | * @param stmt 38 | * @throws SQLException 39 | */ 40 | static void close(Statement stmt) throws SQLException{ 41 | if(stmt != null){ 42 | stmt.close(); 43 | } 44 | } 45 | /** 46 | * 关闭结果集 47 | * @param rs 48 | * @throws 49 | */ 50 | static void close(ResultSet rs) throws SQLException{ 51 | if(rs != null){ 52 | rs.close(); 53 | } 54 | } 55 | 56 | /** 57 | * 58 | * @param rs 59 | * @param statement 60 | * @param connection 61 | */ 62 | public static void close(ResultSet rs,Statement statement,Connection connection){ 63 | if(rs != null){ 64 | try { 65 | rs.close(); 66 | } catch (SQLException e) { 67 | e.printStackTrace(); 68 | } 69 | } 70 | if(statement != null){ 71 | try { 72 | statement.close(); 73 | } catch (SQLException e) { 74 | e.printStackTrace(); 75 | } 76 | } 77 | if(connection != null){ 78 | try { 79 | connection.close(); 80 | } catch (SQLException e) { 81 | e.printStackTrace(); 82 | } 83 | } 84 | 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/common/dbUtil/DBUtil.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.common.dbUtil; 2 | 3 | 4 | import org.poem.hadoop.hive.HiveDataType; 5 | import org.poem.hadoop.model.ColumnInfo; 6 | import org.poem.hadoop.rdbs.MysqlDataType; 7 | import org.poem.hadoop.rdbs.OracleDataType; 8 | import org.poem.hadoop.rdbs.SqlServerDataType; 9 | import org.springframework.jdbc.support.JdbcUtils; 10 | 11 | import java.sql.*; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | /** 16 | * 17 | * @author sai.luo 18 | * @date 2017-5-18 19 | */ 20 | public class DBUtil { 21 | 22 | /** 23 | * 获取数据库表字段 类型 24 | * @param tableName 25 | * @param con 26 | * @return 27 | */ 28 | public static List getTableColumnInfo(String tableName, Connection con){ 29 | List resultList = new ArrayList<>(); 30 | try { 31 | Statement stmt = con.createStatement(); 32 | String sql = "select * from "+tableName; 33 | ResultSet res = stmt.executeQuery(sql); 34 | ResultSetMetaData rsmd = res.getMetaData(); 35 | 36 | for(int i=1;i<=rsmd.getColumnCount();i++) { 37 | resultList.add(new ColumnInfo(rsmd.getColumnName(i), rsmd.getColumnTypeName(i))); 38 | } 39 | 40 | 41 | }catch (Exception e){ 42 | e.printStackTrace(); 43 | } 44 | return resultList ; 45 | } 46 | 47 | /** 48 | * 获取全表表数据 49 | * @param tableName 50 | * @param con 51 | * @return 52 | */ 53 | public static List> getTableDatas(String tableName,Connection con){ 54 | List> resultList = new ArrayList<>(); 55 | ResultSet res = null ; 56 | Statement stmt = null ; 57 | try { 58 | stmt = con.createStatement(); 59 | String sql = "select * from "+tableName; 60 | res = stmt.executeQuery(sql); 61 | ResultSetMetaData metaData = res.getMetaData(); 62 | int var =0 ; 63 | while (res.next()){ 64 | List row = new ArrayList<>(); 65 | for (int i=1;i<=metaData.getColumnCount();i++){ 66 | row.add(JdbcUtils.getResultSetValue(res,i)); 67 | } 68 | resultList.add(row); 69 | } 70 | 71 | }catch (Exception e){ 72 | e.printStackTrace(); 73 | }finally { 74 | try { 75 | ConnUtil.close(res); 76 | ConnUtil.close(stmt); 77 | ConnUtil.close(con); 78 | } catch (SQLException e) { 79 | e.printStackTrace(); 80 | } 81 | } 82 | return resultList ; 83 | } 84 | 85 | /** 86 | * 87 | * @param list 88 | * @param dataSourceType 89 | * @return 90 | */ 91 | public static List tableColumnInfoToHiveColnmnInfo(List list,int dataSourceType){ 92 | list.forEach(columnInfo -> columnInfo.setColumnType(dataBaseDataTypeToHiveDataType(columnInfo.getColumnType(),dataSourceType))); 93 | return list ; 94 | 95 | } 96 | 97 | /** 98 | * 数据库数据类型转hive数据类型 99 | * @param dataBaseDataType 100 | * @param dataSourceType 101 | * @return 102 | */ 103 | public static String dataBaseDataTypeToHiveDataType(String dataBaseDataType,int dataSourceType){ 104 | switch (dataSourceType){ 105 | case DataSourceType.MYSQL_TYPE: 106 | return MysqlDataType.mysqlDataTypetoHiveDataType(dataBaseDataType); 107 | case DataSourceType.SQLSERVER_TYPE: 108 | return SqlServerDataType.sqlServerDataTypetoHiveDataType(dataBaseDataType); 109 | case DataSourceType.ORACLE_TYPE: 110 | return OracleDataType.OracleDataTypetoHiveDataType(dataBaseDataType); 111 | default:return HiveDataType.STRING; 112 | } 113 | 114 | } 115 | 116 | /** 117 | * 获取所有的表 118 | * @param conn 链接 119 | * @param database 数据库 120 | * @return 121 | * @throws SQLException 122 | */ 123 | public static List getAlltables(Connection conn, String database) throws SQLException { 124 | List tables = new ArrayList(); 125 | DatabaseMetaData dbMetaData = conn.getMetaData(); 126 | //可为:"TABLE", "VIEW", "SYSTEM TABLE", 127 | //"GLOBAL TEMPORARY", "LOCAL TEMPORARY", "ALIAS", "SYNONYM" 128 | String[] types = {"TABLE"}; 129 | ResultSet tabs = dbMetaData.getTables(database, null, "%",types/*只要表就好了*/); 130 | /*记录集的结构如下: 131 | TABLE_CAT String => table catalog (may be null) 132 | TABLE_SCHEM String => table schema (may be null) 133 | TABLE_NAME String => table name 134 | TABLE_TYPE String => table type. 135 | REMARKS String => explanatory comment on the table 136 | TYPE_CAT String => the types catalog (may be null) 137 | TYPE_SCHEM String => the types schema (may be null) 138 | TYPE_NAME String => type name (may be null) 139 | SELF_REFERENCING_COL_NAME String => name of the designated "identifier" column of a typed table (may be null) 140 | REF_GENERATION String => specifies how values in SELF_REFERENCING_COL_NAME are created. Values are "SYSTEM", "USER", "DERIVED". (may be null) 141 | */ 142 | while(tabs.next()){ 143 | //只要表名这一列 144 | tables.add(tabs.getObject("TABLE_NAME")); 145 | 146 | } 147 | return tables; 148 | } 149 | 150 | /** 151 | * 数据库导入 152 | * @param host 153 | * @param port 154 | * @param databaseName 155 | * @param dataSourceType 156 | * @return 157 | */ 158 | public static String genDatabaseUrl(String host,int port,String databaseName,int dataSourceType){ 159 | switch (dataSourceType){ 160 | case DataSourceType.MYSQL_TYPE: 161 | return "jdbc:mysql://"+host+":"+port+"/"+databaseName+"?serverTimezone=GMT%2b8&useUnicode=true&characterEncoding=UTF-8&useSSL=false"; 162 | case DataSourceType.SQLSERVER_TYPE: 163 | return "jdbc:sqlserver://"+host+":"+port+";DatabaseName="+databaseName; 164 | case DataSourceType.ORACLE_TYPE: 165 | return "jdbc:oracle:thin:@//"+host+":"+port+":"+databaseName; 166 | default: 167 | return null; 168 | 169 | } 170 | } 171 | 172 | } 173 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/common/dbUtil/DataSourceType.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.common.dbUtil; 2 | 3 | /** 4 | * 5 | * @author sai.luo 6 | * @date 2017-5-19 7 | */ 8 | public class DataSourceType { 9 | public static final int MYSQL_TYPE = 1; 10 | public static final int SQLSERVER_TYPE = 2; 11 | public static final int ORACLE_TYPE = 3; 12 | 13 | public static final String MYSQL_DRIVER_NAME = "com.mysql.cj.jdbc.Driver"; 14 | public static final String SQLSERVER_DRIVER_NAME = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; 15 | public static final String ORACLE_TYPE_DRIVER_NAME = "oracle.jdbc.driver.OracleDriver"; 16 | 17 | public static String getDriverName(int dataSourceType) { 18 | switch (dataSourceType) { 19 | case MYSQL_TYPE: 20 | return MYSQL_DRIVER_NAME; 21 | case SQLSERVER_TYPE: 22 | return SQLSERVER_DRIVER_NAME; 23 | case ORACLE_TYPE: 24 | return ORACLE_TYPE_DRIVER_NAME; 25 | default:return null; 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/hive/HiveDataType.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.hive; 2 | 3 | /** 4 | * 5 | * @author sai.luo 6 | * @date 2017-5-19 7 | */ 8 | public class HiveDataType { 9 | 10 | public static final String TINYINT ="TINYINT"; 11 | public static final String SMALLINT ="SMALLINT"; 12 | public static final String INT ="INT"; 13 | public static final String INTEGER ="INTEGER"; 14 | public static final String BIGINT ="BIGINT"; 15 | public static final String DOUBLE ="DOUBLE"; 16 | public static final String FLOAT ="FLOAT"; 17 | public static final String DECIMAL ="DECIMAL"; 18 | public static final String STRING ="STRING"; 19 | public static final String VARCHAR ="VARCHAR"; 20 | public static final String DATE ="DATE"; 21 | public static final String TIMESTAMP ="TIMESTAMP"; 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/hive/HiveJdbcClient.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.hive; 2 | import org.springframework.stereotype.Repository; 3 | 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.sql.*; 7 | import java.util.*; 8 | 9 | /** 10 | * hivejdbc连接类 11 | * @author suntree.xu 12 | * 13 | */ 14 | @Repository 15 | public class HiveJdbcClient { 16 | 17 | 18 | public static void main(String[] args) throws Exception { 19 | HiveJdbcClient hiveJdbcClient = new HiveJdbcClient(); 20 | 21 | 22 | List goods2 = hiveJdbcClient.getTableInfo("goods2"); 23 | for (String s : goods2) { 24 | System.out.println(s); 25 | } 26 | List> dataFromTable = hiveJdbcClient.getDataFromTable("goods2", Arrays.asList("id", "name"), null, null); 27 | for (Map map : dataFromTable) { 28 | map.forEach((key,value)->{ 29 | System.out.println(key+":"+value); 30 | }); 31 | } 32 | } 33 | 34 | /** 35 | * 获取jdbc连接 36 | * @return Connection 37 | */ 38 | public static Connection getConn(){ 39 | Connection con = null; 40 | try { 41 | Class.forName( getProperties("hive.driverName")); 42 | con = DriverManager.getConnection(getProperties("hive.connection.url"),getProperties("remote.username"),getProperties("remote.password")); 43 | } catch (Exception e) { 44 | // TODO Auto-generated catch block 45 | e.printStackTrace(); 46 | } 47 | return con; 48 | } 49 | 50 | /** 51 | * 获取指定表名的表字段信息 52 | * @param tableName 53 | * @return List 54 | * @throws Exception 55 | */ 56 | private List getTableInfo(String tableName) throws Exception { 57 | List resultList = new ArrayList(); 58 | List> resultMapList = describeTable(tableName); 59 | for (Map stringStringMap : resultMapList) { 60 | resultList.add(stringStringMap.get("col_name")); 61 | } 62 | return resultList; 63 | } 64 | 65 | 66 | static List> describeTable(String tableName){ 67 | List> resultList = new ArrayList>(); 68 | Connection con = getConn(); 69 | ResultSet res = null; 70 | String hiveql = "describe "+tableName; 71 | try { 72 | Statement stmt = con.createStatement(); 73 | res = stmt.executeQuery(hiveql); 74 | ResultSetMetaData rsmd = res.getMetaData(); 75 | while(res.next()) { 76 | Map map = new HashMap<>(); 77 | for (int i = 1; i <= rsmd.getColumnCount(); i++) { 78 | map.put(rsmd.getColumnName(i), res.getString(rsmd.getColumnName(i))); 79 | } 80 | resultList.add(map); 81 | } 82 | }catch (Exception e){ 83 | e.printStackTrace(); 84 | } 85 | return resultList; 86 | } 87 | 88 | /** 89 | * 根据表名称、查询字段、条件、限制条数返回数据,若参数为空,请填入""; 90 | * @param tableName 91 | * @param columnList 92 | * @param condition 93 | * @param limitInfo 94 | * @return 95 | */ 96 | private List> getDataFromTable(String tableName, List columnList, String condition, String limitInfo){ 97 | List> resultList = new ArrayList>(); 98 | Connection con = getConn(); 99 | ResultSet res = null; 100 | StringBuilder hiveql = new StringBuilder("select "); 101 | if(columnList !=null&&columnList.size() > 0){ 102 | for (int i = 0;i < columnList.size();i++){ 103 | if(i != columnList.size()-1) { 104 | hiveql.append(columnList.get(i)).append(","); 105 | } else { 106 | hiveql.append(columnList.get(i)); 107 | } 108 | } 109 | }else{ 110 | hiveql.append("* "); 111 | } 112 | hiveql.append(" from ").append(tableName); 113 | if(condition!=null&&!"".equals(condition)){ 114 | hiveql.append(" where ").append(condition); 115 | } 116 | if(limitInfo!=null&&!"".equals(limitInfo)){ 117 | hiveql.append(" ").append(limitInfo); 118 | } 119 | try { 120 | Statement stmt = con.createStatement(); 121 | res = stmt.executeQuery(hiveql.toString()); 122 | ResultSetMetaData rsmd = res.getMetaData(); 123 | while(res.next()) { 124 | Map map = new HashMap<>(); 125 | for (int i = 1; i <= rsmd.getColumnCount(); i++) { 126 | map.put(rsmd.getColumnName(i), res.getString(rsmd.getColumnName(i))); 127 | } 128 | resultList.add(map); 129 | } 130 | }catch (Exception e){ 131 | e.printStackTrace(); 132 | } 133 | return resultList; 134 | } 135 | 136 | public String excuteLoadData(String loadStr){ 137 | String result = ""; 138 | Connection con = getConn(); 139 | //ResultSet res = null; 140 | try { 141 | Statement stmt = con.createStatement(); 142 | ResultSet res = stmt.executeQuery(loadStr); 143 | } catch (Exception e){ 144 | result = "执行失败："+loadStr; 145 | e.printStackTrace(); 146 | } 147 | return result; 148 | } 149 | 150 | /** 151 | * 仅执行hiveql，不返回数据，只返回成功失败，比如执行创建表，加载数据等 152 | * @param hiveql 153 | * @return 154 | * @throws Exception 155 | */ 156 | public String excuteHiveql(String hiveql) throws Exception{ 157 | String result = ""; 158 | Connection con = getConn(); 159 | //ResultSet res = null; 160 | try { 161 | Statement stmt = con.createStatement(); 162 | int bool = stmt.executeUpdate(hiveql); 163 | result = "执行成功："+hiveql; 164 | }catch (Exception e){ 165 | result = "执行失败："+hiveql; 166 | e.printStackTrace(); 167 | } 168 | return result; 169 | } 170 | 171 | public int queryCount(String tableName){ 172 | int sum = 0; 173 | Connection con = getConn(); 174 | try { 175 | Statement stmt = con.createStatement(); 176 | ResultSet res = stmt.executeQuery("select count(*) from "+tableName); 177 | while(res.next()){ 178 | sum = res.getInt(1); 179 | } 180 | }catch (Exception e){ 181 | e.printStackTrace(); 182 | } 183 | return sum; 184 | } 185 | 186 | /** 187 | * 获取生成hive表的hiveql 188 | * hive表默认全部都是string类型，若有其他类型的需要另外封装方法 189 | * @param tableName 190 | * @param columnList 191 | * @return 192 | */ 193 | public String genCreateTablesql(String tableName,List columnList){ 194 | String sql = "create table "+tableName+"("; 195 | for(int i = 0;i> columnMapList){ 216 | String sql = "create table "+tableName+"("; 217 | 218 | for (Map map:columnMapList){ 219 | Set keys = map.keySet(); 220 | for (String key :keys){ 221 | sql += "`"+key +"`"+" " +map.get(key) + ","; 222 | } 223 | } 224 | sql = sql.substring(0,sql.length()-1); 225 | sql += ") row format delimited fields terminated by ','"; 226 | return sql; 227 | } 228 | 229 | public String genQueryDatasql(String tableName,List columnList){ 230 | String sql = "select "; 231 | if(columnList.size() == 0) { 232 | sql += "*"; 233 | }else { 234 | for (int i = 0; i < columnList.size(); i++) { 235 | if (i != columnList.size() - 1) { 236 | sql += columnList.get(i) + ","; 237 | } else { 238 | sql += columnList.get(i); 239 | } 240 | } 241 | } 242 | sql += " from "+tableName; 243 | return sql; 244 | } 245 | 246 | public boolean isTableDataExist(String tableName) { 247 | Connection con = getConn(); 248 | ResultSet res =null; 249 | String str = ""; 250 | try { 251 | Statement stmt = con.createStatement(); 252 | res = stmt.executeQuery("select * from "+tableName+" limit 1"); 253 | while(res.next()){ 254 | str = res.getString(1); 255 | } 256 | }catch (Exception e){ 257 | e.printStackTrace(); 258 | return false; 259 | } 260 | return !str.equals(""); 261 | //return false; 262 | } 263 | 264 | /** 265 | * 获取相关配置 266 | * @param key 267 | * @return 268 | */ 269 | public static String getProperties(String key){ 270 | String value = ""; 271 | Properties prop = new Properties(); 272 | InputStream in = HiveJdbcClient.class.getResourceAsStream("/hiveConfiguration.properties"); 273 | try { 274 | prop.load(in); 275 | value = prop.getProperty(key).trim(); 276 | } catch (IOException e) { 277 | e.printStackTrace(); 278 | } 279 | return value; 280 | } 281 | 282 | /** 283 | * 获取相关配置(公用) 284 | * @param key 285 | * @return 286 | */ 287 | public String getProperty(String key){ 288 | String value = ""; 289 | Properties prop = new Properties(); 290 | InputStream in = HiveJdbcClient.class.getResourceAsStream("/hiveConfiguration.properties"); 291 | try { 292 | prop.load(in); 293 | value = prop.getProperty(key).trim(); 294 | } catch (IOException e) { 295 | e.printStackTrace(); 296 | }finally { 297 | 298 | } 299 | return value; 300 | } 301 | class testThread extends Thread{ 302 | public void run() 303 | { 304 | System.out.println("testThread run"); 305 | try{ 306 | sleep(10000); 307 | }catch (Exception e){ 308 | e.printStackTrace(); 309 | } 310 | } 311 | } 312 | 313 | } 314 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/hive/HiveOperate.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.hive; 2 | 3 | 4 | import org.poem.hadoop.common.constant.Constant; 5 | import org.poem.hadoop.model.ColumnInfo; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | /** 14 | * Created by sai.luo on 2017-5-18. 15 | */ 16 | public class HiveOperate { 17 | private static final HiveJdbcClient hiveJdbcClient = new HiveJdbcClient(); 18 | private static RemoteOpera remoteOpera = new RemoteOpera(); 19 | private static final Logger log = LoggerFactory.getLogger(HiveOperate.class); 20 | 21 | 22 | public static List getTableColumnInfo(String tableName) throws Exception { 23 | List resultList = new ArrayList(); 24 | List> resultMapList = hiveJdbcClient.describeTable(tableName); 25 | for(int i=0;i columnInfoList,String split){ 41 | String sql = "create table "+tableName+"("; 42 | 43 | for (ColumnInfo columnInfo:columnInfoList){ 44 | sql += "`"+columnInfo.getColumnName() +"`"+" " +columnInfo.getColumnType() + ","; 45 | } 46 | sql = sql.substring(0,sql.length()-1); 47 | sql += ") row format delimited fields terminated by '"+split+"'"; 48 | return sql; 49 | } 50 | 51 | 52 | 53 | /** 54 | * 文件上传并且导入到hive中 55 | * @param tableName 56 | * @param tmpFile 57 | * @param columnMapList 58 | */ 59 | public static String executeHiveOperate(String tableName,String tmpFile,List columnMapList,String split,String localFilePath)throws Exception{ 60 | String result = tableName ; 61 | //上传服务器 62 | remoteOpera.transferFile(hiveJdbcClient.getProperty("remote.ip"),hiveJdbcClient.getProperty("remote.username"),hiveJdbcClient.getProperty("remote.password"),localFilePath+tmpFile,hiveJdbcClient.getProperty("remote.filepath")); 63 | 64 | //根据表名 表头 拼接sql 65 | String createTableSql = HiveOperate.genCreateTablesqlByColumnInfo(tableName,columnMapList,split); 66 | log.info(createTableSql); 67 | try { 68 | //建表 hive 69 | String info = hiveJdbcClient.excuteHiveql(createTableSql); 70 | log.info(info); 71 | String loadStr = "load data local inpath '/root/" + tmpFile + "' into table " + tableName; 72 | hiveJdbcClient.excuteHiveql(loadStr); 73 | //shell 删除临时文件 74 | String cmd1 = "rm -rf "+hiveJdbcClient.getProperty("remote.filepath") + "/" + tmpFile; 75 | remoteOpera.excuteCmd(hiveJdbcClient.getProperty("remote.ip"), hiveJdbcClient.getProperty("remote.username"), hiveJdbcClient.getProperty("remote.password"), cmd1); 76 | 77 | } catch (Exception e) { 78 | log.error("error: ",e); 79 | throw new RuntimeException(e.getCause().getLocalizedMessage()); 80 | } 81 | return result ; 82 | } 83 | 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/hive/RemoteOpera.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.hive; 2 | 3 | 4 | import ch.ethz.ssh2.Connection; 5 | import ch.ethz.ssh2.SCPClient; 6 | import ch.ethz.ssh2.Session; 7 | import ch.ethz.ssh2.StreamGobbler; 8 | import org.springframework.stereotype.Repository; 9 | 10 | import java.io.BufferedReader; 11 | import java.io.IOException; 12 | import java.io.InputStream; 13 | import java.io.InputStreamReader; 14 | 15 | 16 | /** 17 | * @author sangfor 18 | */ 19 | @Repository 20 | public class RemoteOpera { 21 | 22 | public static void main(String[] args) throws IOException{ 23 | Connection conn = new Connection("10.0.0.100", 22); 24 | conn.connect(); 25 | boolean isAuthenticated = conn.authenticateWithPassword("root", "hadoop"); 26 | if (isAuthenticated == false) 27 | throw new IOException("Authentication failed.1"); 28 | SCPClient client = new SCPClient(conn); 29 | client.put("D://xsx//Competence+//work.txt", "/home/compentence/"); 30 | /* Create a session */ 31 | 32 | /*Session sess = conn.openSession(); 33 | 34 | //sess.execCommand("uname -a && date && uptime && who"); 35 | sess.execCommand("date"); 36 | System.out.println("Here is some information about the remote host:"); 37 | 38 | 39 | * This basic example does not handle stderr, which is sometimes dangerous 40 | * (please read the FAQ). 41 | 42 | 43 | InputStream stdout = new StreamGobbler(sess.getStdout()); 44 | 45 | BufferedReader br = new BufferedReader(new InputStreamReader(stdout)); 46 | 47 | while (true) 48 | { 49 | String line = br.readLine(); 50 | if (line == null) 51 | break; 52 | System.out.println(line); 53 | } 54 | 55 | Show exit status, if available (otherwise "null") 56 | 57 | System.out.println("ExitCode: " + sess.getExitStatus()); 58 | 59 | Close this session 60 | 61 | sess.close();*/ 62 | 63 | /* Close the connection */ 64 | 65 | conn.close(); 66 | 67 | } 68 | 69 | /** 70 | * 71 | * @param host 72 | * @param username 73 | * @param password 74 | * @param localFilepath 75 | * @param remotePath 76 | */ 77 | public void transferFile(String host, String username, String password, String localFilepath, String remotePath){ 78 | try{ 79 | Connection conn = new Connection(host, 22); 80 | conn.connect(); 81 | boolean isAuthenticated = conn.authenticateWithPassword(username, password); 82 | if (!isAuthenticated) { 83 | throw new IOException("Authentication failed."); 84 | } 85 | SCPClient client = new SCPClient(conn); 86 | client.put(localFilepath, remotePath); 87 | conn.close(); 88 | }catch (Exception e){ 89 | } 90 | } 91 | 92 | /** 93 | * 远程执行shell命令 94 | * @param host 95 | * @param username 96 | * @param password 97 | * @param cmd 98 | */ 99 | void excuteCmd(String host, String username, String password, String cmd){ 100 | StringBuilder result = new StringBuilder(); 101 | try{ 102 | Connection conn = new Connection(host, 22); 103 | conn.connect(); 104 | boolean isAuthenticated = conn.authenticateWithPassword(username, password); 105 | if (!isAuthenticated) { 106 | throw new IOException("Authentication failed."); 107 | } 108 | Session sess = conn.openSession(); 109 | //sess.execCommand("uname -a && date && uptime && who"); 110 | sess.execCommand(cmd); 111 | //System.out.println("Here is some information about the remote host:"); 112 | InputStream stdout = new StreamGobbler(sess.getStdout()); 113 | BufferedReader br = new BufferedReader(new InputStreamReader(stdout)); 114 | while (true) 115 | { 116 | String line = br.readLine(); 117 | if (line == null) { 118 | break; 119 | } 120 | //System.out.println(line); 121 | result.append(line); 122 | } 123 | result.append("ExitCode: ").append(sess.getExitStatus()); 124 | //System.out.println("ExitCode: " + sess.getExitStatus()); 125 | sess.close(); 126 | /* Close the connection */ 127 | conn.close(); 128 | }catch (Exception e){ 129 | result = new StringBuilder(); 130 | result.append("命令执行错误，错误信息为："); 131 | result.append(e.toString()); 132 | } 133 | } 134 | 135 | 136 | } 137 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/model/ColumnInfo.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.model; 2 | 3 | /** 4 | * 5 | * @author sai.luo 6 | * @date 2017-5-17 7 | */ 8 | public class ColumnInfo { 9 | private String columnName ; 10 | private String columnType ; 11 | 12 | public ColumnInfo(String columnName, String columnType) { 13 | this.columnName = columnName; 14 | this.columnType = columnType; 15 | } 16 | 17 | public String getColumnName() { 18 | return columnName; 19 | } 20 | 21 | public void setColumnName(String columnName) { 22 | this.columnName = columnName; 23 | } 24 | 25 | public String getColumnType() { 26 | return columnType; 27 | } 28 | 29 | public void setColumnType(String columnType) { 30 | this.columnType = columnType; 31 | } 32 | 33 | @Override 34 | public boolean equals(Object obj) { 35 | if (this==obj) return true; 36 | if (obj == null || getClass() != obj.getClass()) return false; 37 | ColumnInfo columnInfo = (ColumnInfo)obj; 38 | return this.columnName.equals(columnInfo.columnName); 39 | } 40 | 41 | @Override 42 | public int hashCode() { 43 | return columnName != null ? columnName.hashCode() : 0; 44 | } 45 | 46 | @Override 47 | public String toString() { 48 | return "ColumnInfo{" + 49 | "columnName='" + columnName + '\'' + 50 | ", columnType='" + columnType + '\'' + 51 | '}'; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/rdbs/MysqlDataType.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.rdbs; 2 | 3 | 4 | import org.poem.hadoop.hive.HiveDataType; 5 | 6 | /** 7 | * Created by sai.luo on 2017-5-19. 8 | */ 9 | public class MysqlDataType { 10 | public static final String TINYINT ="TINYINT"; 11 | public static final String SMALLINT ="SMALLINT"; 12 | public static final String MEDIUMINT ="MEDIUMINT"; 13 | public static final String INT ="INT"; 14 | public static final String INTEGER ="INTEGER"; 15 | public static final String BIGINT ="BIGINT"; 16 | public static final String DOUBLE ="DOUBLE"; 17 | public static final String FLOAT ="FLOAT"; 18 | public static final String DECIMAL ="DECIMAL"; 19 | public static final String NUMERIC ="NUMERIC"; 20 | public static final String CHAR ="CHAR"; 21 | public static final String VARCHAR ="VARCHAR"; 22 | public static final String DATE ="DATE"; 23 | public static final String DATETIME ="DATETIME"; 24 | public static final String TIMESTAMP ="TIMESTAMP"; 25 | 26 | public static String mysqlDataTypetoHiveDataType(String databaseDataType){ 27 | switch (databaseDataType){ 28 | case MysqlDataType.TINYINT: 29 | return HiveDataType.TINYINT; 30 | case MysqlDataType.SMALLINT: 31 | return HiveDataType.SMALLINT; 32 | case MysqlDataType.MEDIUMINT: 33 | case MysqlDataType.INT: 34 | case MysqlDataType.INTEGER: 35 | return HiveDataType.INT; 36 | case MysqlDataType.BIGINT: 37 | return HiveDataType.BIGINT; 38 | case MysqlDataType.DOUBLE: 39 | return HiveDataType.DOUBLE; 40 | case MysqlDataType.FLOAT: 41 | return HiveDataType.FLOAT; 42 | case MysqlDataType.DECIMAL: 43 | return HiveDataType.DECIMAL; 44 | case MysqlDataType.NUMERIC: 45 | return HiveDataType.DOUBLE; 46 | case MysqlDataType.VARCHAR: 47 | return HiveDataType.STRING; 48 | case MysqlDataType.CHAR: 49 | return HiveDataType.STRING; 50 | case MysqlDataType.DATE: 51 | return HiveDataType.DATE; 52 | case MysqlDataType.DATETIME: 53 | return HiveDataType.TIMESTAMP; 54 | case MysqlDataType.TIMESTAMP: 55 | return HiveDataType.TIMESTAMP; 56 | default: 57 | return HiveDataType.STRING; 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/rdbs/OracleDataType.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.rdbs; 2 | 3 | 4 | import org.poem.hadoop.hive.HiveDataType; 5 | 6 | /** 7 | * Created by sai.luo on 2017-5-19. 8 | */ 9 | public class OracleDataType { 10 | 11 | public static final String TINYINT ="TINYINT"; 12 | public static final String SMALLINT ="SMALLINT"; 13 | public static final String MEDIUMINT ="MEDIUMINT"; 14 | public static final String INT ="INT"; 15 | public static final String INTEGER ="INTEGER"; 16 | public static final String BINARY_FLOAT ="BINARY_FLOAT"; 17 | public static final String BINARY_DOUBLE ="BINARY_DOUBLE"; 18 | public static final String DECIMAL ="DECIMAL"; 19 | public static final String NUMERIC ="NUMERIC"; 20 | public static final String CHAR ="CHAR"; 21 | public static final String NCHAR ="NCHAR"; 22 | public static final String VARCHAR ="VARCHAR"; 23 | public static final String VARCHAR2 ="VARCHAR2"; 24 | public static final String NVARCHAR ="NVARCHAR"; 25 | public static final String NVARCHAR2 ="NVARCHAR2"; 26 | public static final String DATE ="DATE"; 27 | public static final String DATETIME ="DATETIME"; 28 | public static final String TIMESTAMP ="TIMESTAMP"; 29 | 30 | public static String OracleDataTypetoHiveDataType(String databaseDataType){ 31 | switch (databaseDataType){ 32 | case OracleDataType.TINYINT: 33 | return HiveDataType.TINYINT; 34 | case OracleDataType.SMALLINT: 35 | return HiveDataType.SMALLINT; 36 | case OracleDataType.MEDIUMINT: 37 | case OracleDataType.INTEGER: 38 | case OracleDataType.INT: 39 | return HiveDataType.INT; 40 | case OracleDataType.BINARY_DOUBLE: 41 | return HiveDataType.DOUBLE; 42 | case OracleDataType.DECIMAL: 43 | return HiveDataType.DECIMAL; 44 | case OracleDataType.BINARY_FLOAT: 45 | return HiveDataType.FLOAT; 46 | case OracleDataType.NUMERIC: 47 | return HiveDataType.DOUBLE; 48 | case OracleDataType.CHAR: 49 | case OracleDataType.NCHAR: 50 | case OracleDataType.VARCHAR: 51 | case OracleDataType.VARCHAR2: 52 | case OracleDataType.NVARCHAR: 53 | case OracleDataType.NVARCHAR2: 54 | return HiveDataType.STRING; 55 | case OracleDataType.DATE: 56 | return HiveDataType.DATE; 57 | case OracleDataType.DATETIME: 58 | return HiveDataType.TIMESTAMP; 59 | case OracleDataType.TIMESTAMP: 60 | return HiveDataType.TIMESTAMP; 61 | default: 62 | return HiveDataType.STRING; 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/org/poem/hadoop/rdbs/SqlServerDataType.java: -------------------------------------------------------------------------------- 1 | package org.poem.hadoop.rdbs; 2 | 3 | 4 | import org.poem.hadoop.hive.HiveDataType; 5 | 6 | /** 7 | * Created by sai.luo on 2017-5-19. 8 | */ 9 | public class SqlServerDataType { 10 | public static final String TINYINT = "TINYINT"; 11 | public static final String SMALLINT = "SMALLINT"; 12 | public static final String MEDIUMINT = "MEDIUMINT"; 13 | public static final String INT = "INT"; 14 | public static final String INTEGER = "INTEGER"; 15 | public static final String BIGINT = "BIGINT"; 16 | public static final String REAL = "REAL"; 17 | public static final String DOUBLE = "DOUBLE"; 18 | public static final String FLOAT = "FLOAT"; 19 | public static final String DECIMAL = "DECIMAL"; 20 | public static final String NUMERIC = "NUMERIC"; 21 | public static final String CHAR = "CHAR"; 22 | public static final String VARCHAR = "VARCHAR"; 23 | public static final String NCHAR = "NCHAR"; 24 | public static final String NVARCHAR = "NVARCHAR"; 25 | public static final String DATE = "DATE"; 26 | public static final String DATETIME = "DATETIME"; 27 | public static final String DATETIME2 = "DATETIME2"; 28 | public static final String SMALLDATETIME = "SMALLDATETIME"; 29 | public static final String DATETIMEOFFSET = "DATETIMEOFFSET"; 30 | public static final String TIMESTAMP = "TIMESTAMP"; 31 | public static final String MONEY = "MONEY"; 32 | public static final String SMALLMONEY = "SMALLMONEY"; 33 | 34 | public static String sqlServerDataTypetoHiveDataType(String databaseDataType) { 35 | switch (databaseDataType.toUpperCase()) { 36 | case SqlServerDataType.TINYINT: 37 | return HiveDataType.TINYINT; 38 | case SqlServerDataType.SMALLINT: 39 | return HiveDataType.SMALLINT; 40 | case SqlServerDataType.MEDIUMINT: 41 | case SqlServerDataType.BIGINT: 42 | case SqlServerDataType.INTEGER: 43 | return HiveDataType.INT; 44 | case SqlServerDataType.FLOAT: 45 | case SqlServerDataType.SMALLMONEY: 46 | return HiveDataType.FLOAT; 47 | case SqlServerDataType.DOUBLE: 48 | case SqlServerDataType.NUMERIC: 49 | case SqlServerDataType.REAL: 50 | case SqlServerDataType.MONEY: 51 | return HiveDataType.DOUBLE; 52 | case SqlServerDataType.DECIMAL: 53 | return HiveDataType.DECIMAL; 54 | case SqlServerDataType.VARCHAR: 55 | case SqlServerDataType.NVARCHAR: 56 | case SqlServerDataType.NCHAR: 57 | case SqlServerDataType.CHAR: 58 | return HiveDataType.STRING; 59 | case SqlServerDataType.DATE: 60 | return HiveDataType.DATE; 61 | case SqlServerDataType.DATETIME: 62 | case SqlServerDataType.DATETIME2: 63 | case SqlServerDataType.SMALLDATETIME: 64 | case SqlServerDataType.DATETIMEOFFSET: 65 | 66 | case SqlServerDataType.TIMESTAMP: 67 | return HiveDataType.TIMESTAMP; 68 | default: 69 | return HiveDataType.STRING; 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/main/java/org/poem/loghelper/LoggerHelper.java: -------------------------------------------------------------------------------- 1 | package org.poem.loghelper; 2 | 3 | 4 | import lombok.Data; 5 | import org.apache.commons.lang3.StringUtils; 6 | import org.poem.loghelper.file.FileService; 7 | import org.poem.utils.ThreadUtils; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.boot.context.properties.ConfigurationProperties; 10 | import org.springframework.stereotype.Service; 11 | 12 | /** 13 | * 日志管理 14 | * 15 | * @author Administrator 16 | */ 17 | @Data 18 | @Service 19 | @ConfigurationProperties(prefix = "logging.gather") 20 | public class LoggerHelper { 21 | 22 | @Autowired 23 | private FileService fileService; 24 | 25 | private String storage; 26 | 27 | private String path; 28 | 29 | /** 30 | * 保存数据 31 | * @param content 32 | */ 33 | public void info(String content){ 34 | String gatherId = ThreadUtils.getTaskId().get(); 35 | if (StringUtils.isEmpty(content)){ 36 | content += ""; 37 | } 38 | content += "\n"; 39 | fileService.write( path, gatherId + ".log", content ); 40 | } 41 | 42 | } -------------------------------------------------------------------------------- /src/main/java/org/poem/loghelper/file/FileService.java: -------------------------------------------------------------------------------- 1 | package org.poem.loghelper.file; 2 | 3 | /** 4 | * 文件 5 | * @author Administrator 6 | */ 7 | public interface FileService { 8 | 9 | 10 | /** 11 | * 写入文件的信息 12 | * @param path 文件路径 13 | * @param fileName 文件名字 14 | * @param content 文件内容 15 | */ 16 | void write(String path, String fileName, String content); 17 | 18 | 19 | /** 20 | * 获取日志文件的内容 21 | * @param path 文件路径 22 | * @param fileName 文件名字 23 | * @return 文件内容 24 | */ 25 | String read(String path, String fileName); 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/poem/loghelper/file/impl/HdfsFileServiceImpl.java: -------------------------------------------------------------------------------- 1 | package org.poem.loghelper.file.impl; 2 | 3 | import org.poem.loghelper.file.FileService; 4 | import org.poem.utils.HdfsUtils; 5 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 6 | import org.springframework.stereotype.Service; 7 | 8 | /** 9 | * hdfs file service 10 | * @author Administrator 11 | */ 12 | @Service 13 | @ConditionalOnProperty(name = "logging.gather.storage",havingValue = "HDFS") 14 | public class HdfsFileServiceImpl implements FileService { 15 | 16 | 17 | /** 18 | * 19 | * @param path 文件路径 20 | * @param fileName 文件名字 21 | * @param content 文件内容 22 | */ 23 | @Override 24 | public void write(String path, String fileName, String content) { 25 | HdfsUtils.uploadFile( path, fileName , content ); 26 | } 27 | 28 | /** 29 | * 30 | * @param path 文件路径 31 | * @param fileName 文件名字 32 | * @return 33 | */ 34 | @Override 35 | public String read(String path, String fileName) { 36 | return HdfsUtils.getFileContent( path, fileName); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/poem/loghelper/file/impl/LocalFileServiceImpl.java: -------------------------------------------------------------------------------- 1 | package org.poem.loghelper.file.impl; 2 | 3 | import org.apache.commons.io.IOUtils; 4 | import org.apache.commons.lang3.StringUtils; 5 | import org.poem.loghelper.file.FileService; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 9 | import org.springframework.stereotype.Service; 10 | 11 | import java.io.*; 12 | import java.nio.ByteBuffer; 13 | import java.nio.channels.FileChannel; 14 | import java.nio.file.Files; 15 | import java.nio.file.attribute.FileAttribute; 16 | 17 | /** 18 | * @author Administrator 19 | */ 20 | @Service 21 | @ConditionalOnProperty(name = "logging.gather.storage",havingValue = "LOCAL") 22 | public class LocalFileServiceImpl implements FileService { 23 | 24 | 25 | private static final FileAttribute[] NO_ATTRIBUTES = new FileAttribute[0]; 26 | 27 | private static final Logger logger = LoggerFactory.getLogger( LocalFileServiceImpl.class ); 28 | 29 | /** 30 | * @param path 文件路径 31 | * @param fileName 文件名字 32 | * @param content 文件内容 33 | */ 34 | @Override 35 | public void write(String path, String fileName, String content) { 36 | if (StringUtils.isBlank( content )) { 37 | logger.warn( "content is empty" ); 38 | return; 39 | } 40 | if (StringUtils.isBlank( path )){ 41 | path = System.getProperty("user.dir"); 42 | } 43 | ByteArrayInputStream fin = null; 44 | FileOutputStream fout = null; 45 | FileChannel fic = null; 46 | FileChannel foc = null; 47 | try { 48 | if (!new File( path ).exists()) { 49 | Files.createDirectory( new File( path ).toPath(), NO_ATTRIBUTES ); 50 | } 51 | File dst = new File( path + File.separator + fileName ); 52 | if (!dst.exists()) { 53 | Files.createFile( dst.toPath(), NO_ATTRIBUTES ); 54 | } 55 | fin = new ByteArrayInputStream( content.getBytes() ); 56 | fout = new FileOutputStream( dst , true); 57 | foc = fout.getChannel(); 58 | // 16KB缓冲区 59 | byte[] bb = new byte[1024 << 4]; 60 | // 根据 read返回实际读出的字节数 中止循环 61 | int len; 62 | do { 63 | len = fin.read( bb ); 64 | foc.write( ByteBuffer.wrap( bb, 0, len ) ); 65 | } while (fin.read( bb ) > 0); 66 | } catch (IOException e) { 67 | e.printStackTrace(); 68 | logger.error( e.getMessage(), e ); 69 | } finally { 70 | IOUtils.closeQuietly( fin ); 71 | IOUtils.closeQuietly( fout ); 72 | IOUtils.closeQuietly( fic ); 73 | IOUtils.closeQuietly( foc ); 74 | } 75 | } 76 | 77 | /** 78 | * @param path 文件路径 79 | * @param fileName 文件名字 80 | * @return 81 | */ 82 | @Override 83 | public String read(String path, String fileName) { 84 | if (StringUtils.isBlank( path )){ 85 | path = System.getProperty("user.dir"); 86 | } 87 | if (!new File( path ).exists()) { 88 | logger.warn( "[" + path + "] not exist!!!!!!" ); 89 | return ""; 90 | } 91 | File file = new File( path + File.separator + fileName ); 92 | if (!file.exists()) { 93 | logger.warn( "file[" + file.getAbsolutePath() + "] not exists!!!!" ); 94 | return ""; 95 | } 96 | StringBuilder content = new StringBuilder( ); 97 | FileInputStream fin = null; 98 | try { 99 | fin = new FileInputStream( file); 100 | FileChannel channel = fin.getChannel(); 101 | // 字节 102 | int capacity = 1024 << 4; 103 | ByteBuffer bf = ByteBuffer.allocate( capacity ); 104 | int length = -1; 105 | while ((length = channel.read( bf )) != -1) { 106 | bf.clear(); 107 | byte[] bytes = bf.array(); 108 | content.append( new String( bytes, 0, length ) ); 109 | } 110 | channel.close(); 111 | } catch (FileNotFoundException e) { 112 | e.printStackTrace(); 113 | logger.error( e.getMessage(),e ); 114 | } catch (IOException e) { 115 | e.printStackTrace(); 116 | logger.error( e.getMessage(),e ); 117 | } finally { 118 | if (fin != null) { 119 | try { 120 | fin.close(); 121 | } catch (IOException e) { 122 | e.printStackTrace(); 123 | } 124 | } 125 | } 126 | return content.toString(); 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /src/main/java/org/poem/utils/GatherDataSourceUtils.java: -------------------------------------------------------------------------------- 1 | package org.poem.utils; 2 | 3 | import org.poem.code.service.connect.DataType; 4 | import org.poem.code.service.databases.GatherDataBaseInter; 5 | import org.poem.code.service.databases.MysqlGatherData; 6 | import org.poem.code.service.databases.OracleGatherData; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.sql.ResultSet; 11 | import java.sql.Statement; 12 | 13 | /** 14 | * @author Administrator 15 | */ 16 | public class GatherDataSourceUtils { 17 | 18 | private static final Logger logger = LoggerFactory.getLogger( GatherDataSourceUtils.class ); 19 | 20 | /** 21 | * 关闭 22 | * 23 | * @param pstmt 24 | * @param rs 25 | */ 26 | public static void release(Statement pstmt, ResultSet rs) { 27 | if (rs != null) { 28 | try { 29 | logger.debug( "关闭存储查询结果的ResultSet对象" ); 30 | rs.close(); 31 | } catch (Exception e) { 32 | e.printStackTrace(); 33 | } 34 | } 35 | if (pstmt != null) { 36 | try { 37 | logger.debug( "关闭负责执行SQL命令的Statement对象" ); 38 | pstmt.close(); 39 | } catch (Exception e) { 40 | e.printStackTrace(); 41 | } 42 | } 43 | } 44 | 45 | 46 | /** 47 | * 获取数据的bean类型 48 | * 49 | * @return 50 | */ 51 | public static GatherDataBaseInter getBean() { 52 | DataType dataType = ThreadUtils.getDataTypeThreadLocal().get(); 53 | if (DataType.MYSQL.eq( dataType )) { 54 | return SpringUtils.getBean( MysqlGatherData.BEAN, MysqlGatherData.class ); 55 | } else if (DataType.ORACLE.eq( dataType )) { 56 | return SpringUtils.getBean( OracleGatherData.BEAN, OracleGatherData.class ); 57 | } else { 58 | throw new IllegalArgumentException( "not unrecognizable type[" + dataType + "] !!!!!!!!!" ); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/org/poem/utils/HdfsUtils.java: -------------------------------------------------------------------------------- 1 | package org.poem.utils; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.fs.FSDataInputStream; 5 | import org.apache.hadoop.fs.FSDataOutputStream; 6 | import org.apache.hadoop.fs.FileSystem; 7 | import org.apache.hadoop.fs.Path; 8 | import org.apache.hadoop.io.IOUtils; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.*; 13 | import java.nio.ByteBuffer; 14 | 15 | public class HdfsUtils { 16 | 17 | private static final Logger logger = LoggerFactory.getLogger( HdfsUtils.class ); 18 | 19 | private static String ip; 20 | 21 | private static String port; 22 | 23 | /** 24 | * 获取文件的内容 25 | * @param path 26 | * @param file 27 | * @return 28 | */ 29 | public static String getFileContent(String path, String file) { 30 | Configuration configuration = new Configuration(); 31 | configuration.set( "fs.defaultFS", "hdfs://" + ip + ":" + port ); 32 | configuration.set( "dfs.client.use.datanode.hostname", "true" ); 33 | configuration.setBoolean( "dfs.support.append", true ); 34 | OutputStream ot = null; 35 | StringBuilder stringBuffer = new StringBuilder(); 36 | FSDataInputStream out = null; 37 | String pat = "hdfs://" + ip + ":" + port + "/" + path + "/" + file; 38 | Path hdfsPath = new Path( pat ); 39 | try { 40 | FileSystem fs = FileSystem.get( configuration ); 41 | if (!fs.exists( hdfsPath )) { 42 | logger.warn( "file [" + pat + "] not exist !!!!!" ); 43 | return null; 44 | } 45 | out = fs.open( hdfsPath ); 46 | int capacity = 1024 << 4; 47 | ByteBuffer bf = ByteBuffer.allocate( capacity ); 48 | int le = 0; 49 | while ((le = out.read( bf )) > 0) { 50 | stringBuffer.append( new String( bf.array(), 0, le ) ); 51 | } 52 | } catch (IOException e) { 53 | logger.error( e.getMessage(), e ); 54 | e.printStackTrace(); 55 | } finally { 56 | org.apache.commons.io.IOUtils.closeQuietly( ot ); 57 | org.apache.commons.io.IOUtils.closeQuietly( out ); 58 | } 59 | return stringBuffer.toString(); 60 | 61 | } 62 | 63 | /** 64 | * 上传本地文件到 HDFS 65 | * 66 | * @param source 67 | */ 68 | public static void uploadFile(String path, String file, String source) { 69 | Configuration configuration = new Configuration(); 70 | configuration.set( "fs.defaultFS", "hdfs://" + ip + ":" + port ); 71 | configuration.set( "dfs.client.use.datanode.hostname", "true" ); 72 | configuration.setBoolean( "dfs.support.append", true ); 73 | OutputStream ot = null; 74 | InputStream is1 = null; 75 | FSDataOutputStream out = null; 76 | String pat = "hdfs://" + ip + ":" + port + "/" + path + "/" + file; 77 | Path hdfsPath = new Path( pat ); 78 | try { 79 | FileSystem fs = FileSystem.get( configuration ); 80 | if (!fs.exists( hdfsPath )) { 81 | logger.info( "create hdfs :" + pat ); 82 | ot = fs.create( hdfsPath ); 83 | } 84 | 85 | is1 = new BufferedInputStream( new ByteArrayInputStream( source.getBytes() ) ); 86 | out = fs.append( hdfsPath ); 87 | IOUtils.copyBytes( is1, out, configuration ); 88 | } catch (IOException e) { 89 | logger.error( e.getMessage(), e ); 90 | e.printStackTrace(); 91 | } finally { 92 | org.apache.commons.io.IOUtils.closeQuietly( ot ); 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/main/java/org/poem/utils/SnowFlake.java: -------------------------------------------------------------------------------- 1 | package org.poem.utils; 2 | 3 | /** 4 | * twitter的snowflake算法 -- java实现 5 | * 6 | * @author beyond 7 | * @date 2016/11/26 8 | */ 9 | public class SnowFlake { 10 | 11 | /** 12 | * 起始的时间戳 13 | */ 14 | private final static long START_STMP = 1480166465631L; 15 | 16 | /** 17 | * 每一部分占用的位数 18 | */ 19 | private final static long SEQUENCE_BIT = 12; //序列号占用的位数 20 | private final static long MACHINE_BIT = 5; //机器标识占用的位数 21 | private final static long DATACENTER_BIT = 5;//数据中心占用的位数 22 | 23 | /** 24 | * 每一部分的最大值 25 | */ 26 | private final static long MAX_DATACENTER_NUM = -1L ^ (-1L << DATACENTER_BIT); 27 | private final static long MAX_MACHINE_NUM = -1L ^ (-1L << MACHINE_BIT); 28 | private final static long MAX_SEQUENCE = -1L ^ (-1L << SEQUENCE_BIT); 29 | 30 | /** 31 | * 每一部分向左的位移 32 | */ 33 | private final static long MACHINE_LEFT = SEQUENCE_BIT; 34 | private final static long DATACENTER_LEFT = SEQUENCE_BIT + MACHINE_BIT; 35 | private final static long TIMESTMP_LEFT = DATACENTER_LEFT + DATACENTER_BIT; 36 | private static SnowFlake snowFlake; 37 | 38 | static { 39 | snowFlake = new SnowFlake( 1, 1 ); 40 | } 41 | 42 | private long datacenterId; //数据中心 43 | private long machineId; //机器标识 44 | private long sequence = 0L; //序列号 45 | private long lastStmp = -1L;//上一次时间戳 46 | 47 | public SnowFlake(long datacenterId, long machineId) { 48 | if (datacenterId > MAX_DATACENTER_NUM || datacenterId < 0) { 49 | throw new IllegalArgumentException( "datacenterId can't be greater than MAX_DATACENTER_NUM or less than 0" ); 50 | } 51 | if (machineId > MAX_MACHINE_NUM || machineId < 0) { 52 | throw new IllegalArgumentException( "machineId can't be greater than MAX_MACHINE_NUM or less than 0" ); 53 | } 54 | this.datacenterId = datacenterId; 55 | this.machineId = machineId; 56 | } 57 | 58 | public static String genId() { 59 | return Long.toString( snowFlake.nextId() ); 60 | } 61 | 62 | /** 63 | * 产生下一个ID 64 | * 65 | * @return 66 | */ 67 | public synchronized long nextId() { 68 | long currStmp = getNewstmp(); 69 | if (currStmp < lastStmp) { 70 | throw new RuntimeException( "Clock moved backwards. Refusing to generate id" ); 71 | } 72 | 73 | if (currStmp == lastStmp) { 74 | //相同毫秒内，序列号自增 75 | sequence = (sequence + 1) & MAX_SEQUENCE; 76 | //同一毫秒的序列数已经达到最大 77 | if (sequence == 0L) { 78 | currStmp = getNextMill(); 79 | } 80 | } else { 81 | //不同毫秒内，序列号置为0 82 | sequence = 0L; 83 | } 84 | 85 | lastStmp = currStmp; 86 | 87 | return (currStmp - START_STMP) << TIMESTMP_LEFT //时间戳部分 88 | | datacenterId << DATACENTER_LEFT //数据中心部分 89 | | machineId << MACHINE_LEFT //机器标识部分 90 | | sequence; //序列号部分 91 | } 92 | 93 | private long getNextMill() { 94 | long mill = getNewstmp(); 95 | while (mill <= lastStmp) { 96 | mill = getNewstmp(); 97 | } 98 | return mill; 99 | } 100 | 101 | private long getNewstmp() { 102 | return System.currentTimeMillis(); 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/org/poem/utils/SpringUtils.java: -------------------------------------------------------------------------------- 1 | package org.poem.utils; 2 | 3 | import org.springframework.beans.BeansException; 4 | import org.springframework.context.ApplicationContext; 5 | import org.springframework.context.ApplicationContextAware; 6 | import org.springframework.stereotype.Component; 7 | 8 | /** 9 | * 10 | * @author poem 11 | */ 12 | @Component 13 | public class SpringUtils implements ApplicationContextAware { 14 | 15 | private static ApplicationContext applicationContext; 16 | 17 | @Override 18 | public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { 19 | if(SpringUtils.applicationContext == null) { 20 | SpringUtils.applicationContext = applicationContext; 21 | } 22 | } 23 | 24 | /** 25 | * 获取applicationContext 26 | * @return 27 | */ 28 | public static ApplicationContext getApplicationContext() { 29 | return applicationContext; 30 | } 31 | 32 | /** 33 | * 通过name获取 Bean. 34 | * @param name 35 | * @return 36 | */ 37 | public static Object getBean(String name){ 38 | return getApplicationContext().getBean(name); 39 | } 40 | 41 | /** 42 | * 通过class获取Bean. 43 | * @param clazz 44 | * @param 45 | * @return 46 | */ 47 | public static T getBean(Class clazz){ 48 | return getApplicationContext().getBean(clazz); 49 | } 50 | 51 | /** 52 | * 通过name,以及Clazz返回指定的Bean 53 | * @param name 54 | * @param clazz 55 | * @param 56 | * @return 57 | */ 58 | public static T getBean(String name,Class clazz){ 59 | return getApplicationContext().getBean(name, clazz); 60 | } 61 | 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/org/poem/utils/ThreadUtils.java: -------------------------------------------------------------------------------- 1 | package org.poem.utils; 2 | 3 | import org.poem.code.service.connect.DataType; 4 | 5 | /** 6 | * @author Administrator 7 | */ 8 | public class ThreadUtils { 9 | 10 | /** 11 | * 数据类型 12 | */ 13 | private static ThreadLocal dataTypeThreadLocal = new ThreadLocal<>(); 14 | 15 | private static ThreadLocal taskId = new ThreadLocal<>(); 16 | 17 | /** 18 | * 任务id 19 | * @return 20 | */ 21 | public static ThreadLocal getTaskId() { 22 | return taskId; 23 | } 24 | 25 | /** 26 | * 任务id 27 | * @param taskId 28 | */ 29 | public static void setTaskId(String taskId) { 30 | ThreadUtils.taskId.set( taskId ); 31 | } 32 | 33 | /** 34 | * 数据类型 35 | * @return 36 | */ 37 | public static ThreadLocal getDataTypeThreadLocal() { 38 | return ThreadUtils.dataTypeThreadLocal; 39 | } 40 | 41 | /** 42 | * 数据类型 43 | * @param dataType 44 | */ 45 | public static void setDataTypeThreadLocal(DataType dataType) { 46 | ThreadUtils.dataTypeThreadLocal.set( dataType ); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/resources/application-gen.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: ${global.target.server.port} 3 | spring: 4 | resources: 5 | static-locations: classpath:/webapp/ 6 | datasource: 7 | logAbandoned: true 8 | minIdle: 5 9 | removeAbandoned: true 10 | maxWait: 1000 11 | name: kylo 12 | removeAbandonedTimeout: 60 13 | url: jdbc:mysql://127.0.0.1:3306/kylo?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC 14 | username: root 15 | password: 123456 16 | driver-class-name: com.mysql.cj.jdbc.Driver 17 | flyway: 18 | locations: classpath:migration 19 | enabled: false 20 | jooq: 21 | sql-dialect: MYSQL 22 | profiles: 23 | include: global 24 | 25 | logging: 26 | level: 27 | root: info 28 | 29 | azkaban: 30 | url: http://192.168.23.226:8081 31 | username: azkaban 32 | password: azkaban 33 | execute: 34 | path: /opt/app/azkaban/urule-azkaban-executor 35 | -------------------------------------------------------------------------------- /src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: ${global.gather.server.port} 3 | spring: 4 | profiles: 5 | include: global 6 | application: 7 | name: metadata-gather 8 | datasource: 9 | name: ${global.datasource.name} 10 | url: jdbc:mysql://${global.datasource.url}:${global.datasource.port}/${global.datasource.database-name}?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC 11 | username: ${global.datasource.username} 12 | password: ${global.datasource.password} 13 | driver-class-name: com.mysql.cj.jdbc.Driver 14 | dbcp2: 15 | log-abandoned: true 16 | flyway: 17 | locations: classpath:migration 18 | enabled: false 19 | jooq: 20 | sql-dialect: MYSQL 21 | main: 22 | allow-bean-definition-overriding: true 23 | logging: 24 | gather: 25 | storage: ${global.gather.logger.storage} 26 | path: ${global.gather.logger.path} 27 | level: 28 | root: info 29 | 30 | -------------------------------------------------------------------------------- /src/main/resources/db/V0.01__init.sql: -------------------------------------------------------------------------------- 1 | -- 元数据采集信息 2 | drop table if exists `dsg_gather_info`; 3 | create table `dsg_gather_info` 4 | ( 5 | `id` varchar(50) not null comment 'id', 6 | `name` varchar(500) not null comment '连接的名字', 7 | `ip` varchar(500) default null comment '连接ip', 8 | `port` varchar(500) default null comment '连接的端口', 9 | `user` varchar(500) default null comment '数据库连接的用户名字', 10 | `password` varchar(500) default null comment '数据库连接的密码', 11 | `type` varchar(500) default null comment '数据库类型 mysql或者是 orcale', 12 | `remark` varchar(500) default null comment '描述', 13 | `delete` boolean default null comment '是否删除', 14 | `create_time` timestamp NOT NULL , 15 | `update_time` timestamp NOT NULL , 16 | primary key (`id`) 17 | ) engine = innodb 18 | default charset = utf8 comment ='元数据采集信息'; 19 | 20 | 21 | 22 | -- 采集到库信息 23 | drop table if exists `dsg_gather_db`; 24 | create table `dsg_gather_db` 25 | ( 26 | `id` varchar(50) not null comment 'id', 27 | `gather_id` varchar(500) not null comment '元数据采集信息', 28 | `schema` varchar(500) default null comment 'schema', 29 | `description` varchar(225) DEFAULT NULL comment '描述', 30 | `create_time` timestamp NOT NULL , 31 | `update_time` timestamp NOT NULL , 32 | primary key (`id`) 33 | ) engine = innodb 34 | default charset = utf8 comment ='采集到库信息'; 35 | 36 | 37 | 38 | 39 | -- 采集表信息 40 | drop table if exists `dsg_gather_table`; 41 | create table `dsg_gather_table` 42 | ( 43 | `id` varchar(50) not null comment 'id', 44 | `gather_db_id` varchar(500) not null comment '元数据数据库信息', 45 | `table` varchar(500) default null comment '数据库表', 46 | `table_name` varchar(1000) default null comment '数据库表名字', 47 | `comment` varchar(1000) DEFAULT NULL comment '描述', 48 | `create_time` timestamp NOT NULL , 49 | `update_time` timestamp NOT NULL , 50 | primary key (`id`) 51 | ) engine = innodb 52 | default charset = utf8 comment ='采集表信息'; 53 | 54 | 55 | -- 采集表字端信息 56 | drop table if exists `dsg_gather_table_fields`; 57 | create table `dsg_gather_table_fields` 58 | ( 59 | `id` varchar(50) not null comment 'id', 60 | `gather_table_id` varchar(500) default null comment '表', 61 | `field_name` varchar(225) DEFAULT NULL comment '字段', 62 | `data_type` varchar(225) DEFAULT NULL comment '数据类型', 63 | `column_name` varchar(225) DEFAULT NULL comment '描述', 64 | `default_value` varchar(225) DEFAULT NULL comment '默认值', 65 | `data_type_length` varchar(225) DEFAULT NULL comment '数据长度', 66 | `create_time` timestamp NOT NULL , 67 | `update_time` timestamp NOT NULL , 68 | primary key (`id`) 69 | ) engine = innodb 70 | default charset = utf8 comment ='采集表字端信息'; 71 | 72 | 73 | ALTER TABLE `dsg_gather_info` 74 | ADD COLUMN `service_name` varchar(255) NULL COMMENT 'oracle 服务名字' AFTER `type`; 75 | 76 | 77 | 78 | ALTER TABLE `dsg_gather_info` 79 | ADD COLUMN `status` int(3) NULL COMMENT '状态 0 没有启动 1 启动' AFTER `delete`; 80 | 81 | 82 | -- 采集的统计信息 83 | drop table if exists `dsg_gather_statistics`; 84 | create table `dsg_gather_statistics` 85 | ( 86 | `id` varchar(50) not null comment 'id', 87 | `gather_info_id` varchar(500) not null comment '元数据信息, 和任务在一起', 88 | `gather_db_count` int(5) not null comment '抓取的数据库数量', 89 | `gather_table_count` int(5) not null comment '抓取的表的个数', 90 | `gather_field_count` int(5) not null comment '抓取的表列的个数', 91 | `create_time` timestamp NOT NULL , 92 | `update_time` timestamp NOT NULL , 93 | primary key (`id`) 94 | ) engine = innodb 95 | default charset = utf8 comment ='采集表信息'; -------------------------------------------------------------------------------- /src/main/test/org/poem/service/MetadataServiceImplTest.java: -------------------------------------------------------------------------------- 1 | package org.poem.code.service; 2 | 3 | 4 | import com.alibaba.fastjson.JSONObject; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.poem.MetadataGatherApp; 8 | import org.poem.code.api.GatherService; 9 | import org.poem.code.api.MetadataService; 10 | import org.poem.code.api.vo.*; 11 | import org.poem.code.api.vo.query.DbQueryVO; 12 | import org.poem.code.api.vo.query.TableQueryVO; 13 | import org.springframework.beans.factory.annotation.Autowired; 14 | import org.springframework.boot.test.context.SpringBootTest; 15 | import org.springframework.test.context.junit.jupiter.web.SpringJUnitWebConfig; 16 | import org.springframework.test.context.junit4.SpringRunner; 17 | 18 | import java.util.List; 19 | 20 | @SpringBootTest 21 | @SpringJUnitWebConfig(classes = {MetadataGatherApp.class}) 22 | @RunWith(SpringRunner.class) 23 | public class MetadataServiceImplTest { 24 | 25 | 26 | @Autowired 27 | private MetadataService metadataService; 28 | 29 | @Autowired 30 | private GatherService gatherService; 31 | 32 | @org.junit.Test 33 | public void savgeGather() { 34 | GatherVO gatherVO = new GatherVO(); 35 | gatherVO.setIp( "127.0.0.1" ); 36 | gatherVO.setUserName( "root" ); 37 | gatherVO.setPassword( "123456" ); 38 | gatherVO.setType( "MYSQL" ); 39 | gatherVO.setPort( "3306" ); 40 | gatherVO.setName( "测试Mysql 数据库" ); 41 | metadataService.savgeGather( gatherVO ); 42 | } 43 | 44 | @org.junit.Test 45 | public void savgeOrcaleGather() { 46 | GatherVO gatherVO = new GatherVO(); 47 | gatherVO.setIp( "192.168.23.210" ); 48 | gatherVO.setUserName( "dsg" ); 49 | gatherVO.setPassword( "dsg" ); 50 | gatherVO.setType( "ORACLE" ); 51 | gatherVO.setServerName( "DB112A" ); 52 | gatherVO.setPort( "1521" ); 53 | gatherVO.setName( "测试 ORACLE 数据库" ); 54 | metadataService.savgeGather( gatherVO ); 55 | } 56 | 57 | @org.junit.Test 58 | public void getAllGather() { 59 | List gatherVOList = this.metadataService.getAllGather(); 60 | gatherVOList.forEach( 61 | o -> { 62 | System.out.println( JSONObject.toJSONString( o ) ); 63 | } 64 | ); 65 | } 66 | 67 | @org.junit.Test 68 | public void saveDB() { 69 | List dbVOS = gatherService.getSchema( "346086619914309632" ); 70 | for (DbVO dbVO : dbVOS) { 71 | System.out.println( JSONObject.toJSONString( dbVO ) ); 72 | } 73 | this.metadataService.deleteAllDataGatherInfoId( "346086619914309632" ); 74 | this.metadataService.saveDB( dbVOS, "346086619914309632" ); 75 | } 76 | 77 | 78 | @org.junit.Test 79 | public void getAllDB() { 80 | List dbVOS = metadataService.getAllDB( new DbQueryVO( null, "346086619914309632" ) ); 81 | for (DbVO dbVO : dbVOS) { 82 | System.out.println( JSONObject.toJSONString( dbVO ) ); 83 | } 84 | } 85 | 86 | @org.junit.Test 87 | public void saveTable() { 88 | GatherVO gatherVO = this.metadataService.getGatherVOById( "346086619914309632" ); 89 | gatherVO.setDbId( "346000843771875330" ); 90 | List tableVOS = this.gatherService.getTable( gatherVO ); 91 | for (TableVO tableVO : tableVOS) { 92 | System.out.println( JSONObject.toJSONString( tableVO ) ); 93 | } 94 | this.metadataService.saveTable( tableVOS, "346000843771875330" ); 95 | } 96 | 97 | @org.junit.Test 98 | public void getAllTable() { 99 | List tableVOS = metadataService.getTable( new TableQueryVO( "346000843771875330", null, null ) ); 100 | for (TableVO tableVO : tableVOS) { 101 | System.out.println( JSONObject.toJSONString( tableVO ) ); 102 | } 103 | } 104 | 105 | @org.junit.Test 106 | public void saveTableFields() { 107 | TableVO tableVO = this.gatherService.getTableById( "346006017080954881" ); 108 | List tableFieldsVOS = gatherService.getTableFieles( tableVO ); 109 | for (TableFieldsVO tableFieldsVO : tableFieldsVOS) { 110 | System.out.println( JSONObject.toJSONString( tableFieldsVO ) ); 111 | } 112 | this.metadataService.saveTableFields( tableFieldsVOS, "346006017080954881" ); 113 | } 114 | 115 | 116 | @Test 117 | public void getAllDataTableFiels() { 118 | List gatherVOList = this.metadataService.getAllGather(); 119 | gatherVOList.forEach( 120 | o -> { 121 | String gatherId = o.getGatherId(); 122 | GatherDBTableFieldsVO gatherDBTableFieldsVO = this.gatherService.getAllGatherDBTableFieldsVO( gatherId , 123 | null,null); 124 | this.metadataService.saveGather( gatherDBTableFieldsVO ); 125 | } 126 | ); 127 | } 128 | 129 | 130 | 131 | @Test 132 | public void getAllDataTableFiels1() { 133 | GatherDBTableFieldsVO gatherDBTableFieldsVO = 134 | this.gatherService.getAllGatherDBTableFieldsVO( "346086619914309632", null ,null); 135 | this.metadataService.saveGather( gatherDBTableFieldsVO ); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /target/classes/META-INF/spring-configuration-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "groups": [ 3 | { 4 | "name": "logging.gather", 5 | "type": "org.poem.loghelper.LoggerHelper", 6 | "sourceType": "org.poem.loghelper.LoggerHelper" 7 | } 8 | ], 9 | "properties": [ 10 | { 11 | "name": "logging.gather.file-service", 12 | "type": "org.poem.loghelper.file.FileService", 13 | "sourceType": "org.poem.loghelper.LoggerHelper" 14 | }, 15 | { 16 | "name": "logging.gather.path", 17 | "type": "java.lang.String", 18 | "sourceType": "org.poem.loghelper.LoggerHelper" 19 | }, 20 | { 21 | "name": "logging.gather.storage", 22 | "type": "java.lang.String", 23 | "sourceType": "org.poem.loghelper.LoggerHelper" 24 | } 25 | ], 26 | "hints": [] 27 | } -------------------------------------------------------------------------------- /target/classes/application-gen.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: ${global.target.server.port} 3 | spring: 4 | resources: 5 | static-locations: classpath:/webapp/ 6 | datasource: 7 | logAbandoned: true 8 | minIdle: 5 9 | removeAbandoned: true 10 | maxWait: 1000 11 | name: kylo 12 | removeAbandonedTimeout: 60 13 | url: jdbc:mysql://127.0.0.1:3306/kylo?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC 14 | username: root 15 | password: 123456 16 | driver-class-name: com.mysql.cj.jdbc.Driver 17 | flyway: 18 | locations: classpath:migration 19 | enabled: false 20 | jooq: 21 | sql-dialect: MYSQL 22 | profiles: 23 | include: global 24 | 25 | logging: 26 | level: 27 | root: info 28 | 29 | azkaban: 30 | url: http://192.168.23.226:8081 31 | username: azkaban 32 | password: azkaban 33 | execute: 34 | path: /opt/app/azkaban/urule-azkaban-executor 35 | -------------------------------------------------------------------------------- /target/classes/application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: ${global.gather.server.port} 3 | spring: 4 | profiles: 5 | include: global 6 | application: 7 | name: metadata-gather 8 | datasource: 9 | name: ${global.datasource.name} 10 | url: jdbc:mysql://${global.datasource.url}:${global.datasource.port}/${global.datasource.database-name}?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC 11 | username: ${global.datasource.username} 12 | password: ${global.datasource.password} 13 | driver-class-name: com.mysql.cj.jdbc.Driver 14 | dbcp2: 15 | log-abandoned: true 16 | flyway: 17 | locations: classpath:migration 18 | enabled: false 19 | jooq: 20 | sql-dialect: MYSQL 21 | main: 22 | allow-bean-definition-overriding: true 23 | logging: 24 | gather: 25 | storage: ${global.gather.logger.storage} 26 | path: ${global.gather.logger.path} 27 | level: 28 | root: info 29 | 30 | -------------------------------------------------------------------------------- /target/classes/db/V0.01__init.sql: -------------------------------------------------------------------------------- 1 | -- 元数据采集信息 2 | drop table if exists `dsg_gather_info`; 3 | create table `dsg_gather_info` 4 | ( 5 | `id` varchar(50) not null comment 'id', 6 | `name` varchar(500) not null comment '连接的名字', 7 | `ip` varchar(500) default null comment '连接ip', 8 | `port` varchar(500) default null comment '连接的端口', 9 | `user` varchar(500) default null comment '数据库连接的用户名字', 10 | `password` varchar(500) default null comment '数据库连接的密码', 11 | `type` varchar(500) default null comment '数据库类型 mysql或者是 orcale', 12 | `remark` varchar(500) default null comment '描述', 13 | `delete` boolean default null comment '是否删除', 14 | `create_time` timestamp NOT NULL , 15 | `update_time` timestamp NOT NULL , 16 | primary key (`id`) 17 | ) engine = innodb 18 | default charset = utf8 comment ='元数据采集信息'; 19 | 20 | 21 | 22 | -- 采集到库信息 23 | drop table if exists `dsg_gather_db`; 24 | create table `dsg_gather_db` 25 | ( 26 | `id` varchar(50) not null comment 'id', 27 | `gather_id` varchar(500) not null comment '元数据采集信息', 28 | `schema` varchar(500) default null comment 'schema', 29 | `description` varchar(225) DEFAULT NULL comment '描述', 30 | `create_time` timestamp NOT NULL , 31 | `update_time` timestamp NOT NULL , 32 | primary key (`id`) 33 | ) engine = innodb 34 | default charset = utf8 comment ='采集到库信息'; 35 | 36 | 37 | 38 | 39 | -- 采集表信息 40 | drop table if exists `dsg_gather_table`; 41 | create table `dsg_gather_table` 42 | ( 43 | `id` varchar(50) not null comment 'id', 44 | `gather_db_id` varchar(500) not null comment '元数据数据库信息', 45 | `table` varchar(500) default null comment '数据库表', 46 | `table_name` varchar(1000) default null comment '数据库表名字', 47 | `comment` varchar(1000) DEFAULT NULL comment '描述', 48 | `create_time` timestamp NOT NULL , 49 | `update_time` timestamp NOT NULL , 50 | primary key (`id`) 51 | ) engine = innodb 52 | default charset = utf8 comment ='采集表信息'; 53 | 54 | 55 | -- 采集表字端信息 56 | drop table if exists `dsg_gather_table_fields`; 57 | create table `dsg_gather_table_fields` 58 | ( 59 | `id` varchar(50) not null comment 'id', 60 | `gather_table_id` varchar(500) default null comment '表', 61 | `field_name` varchar(225) DEFAULT NULL comment '字段', 62 | `data_type` varchar(225) DEFAULT NULL comment '数据类型', 63 | `column_name` varchar(225) DEFAULT NULL comment '描述', 64 | `default_value` varchar(225) DEFAULT NULL comment '默认值', 65 | `data_type_length` varchar(225) DEFAULT NULL comment '数据长度', 66 | `create_time` timestamp NOT NULL , 67 | `update_time` timestamp NOT NULL , 68 | primary key (`id`) 69 | ) engine = innodb 70 | default charset = utf8 comment ='采集表字端信息'; 71 | 72 | 73 | ALTER TABLE `dsg_gather_info` 74 | ADD COLUMN `service_name` varchar(255) NULL COMMENT 'oracle 服务名字' AFTER `type`; 75 | 76 | 77 | 78 | ALTER TABLE `dsg_gather_info` 79 | ADD COLUMN `status` int(3) NULL COMMENT '状态 0 没有启动 1 启动' AFTER `delete`; 80 | 81 | 82 | -- 采集的统计信息 83 | drop table if exists `dsg_gather_statistics`; 84 | create table `dsg_gather_statistics` 85 | ( 86 | `id` varchar(50) not null comment 'id', 87 | `gather_info_id` varchar(500) not null comment '元数据信息, 和任务在一起', 88 | `gather_db_count` int(5) not null comment '抓取的数据库数量', 89 | `gather_table_count` int(5) not null comment '抓取的表的个数', 90 | `gather_field_count` int(5) not null comment '抓取的表列的个数', 91 | `create_time` timestamp NOT NULL , 92 | `update_time` timestamp NOT NULL , 93 | primary key (`id`) 94 | ) engine = innodb 95 | default charset = utf8 comment ='采集表信息'; -------------------------------------------------------------------------------- /target/maven-archiver/pom.properties: -------------------------------------------------------------------------------- 1 | #Created by Apache Maven 3.5.3 2 | version=1.0-SNAPSHOT 3 | groupId=org.poem 4 | artifactId=metadata-gather 5 | -------------------------------------------------------------------------------- /target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst: -------------------------------------------------------------------------------- 1 | org\poem\api\vo\query\TableFieldsQueryVO.class 2 | org\poem\entities\tables\records\DsgGatherTableFieldsRecord.class 3 | org\poem\service\connect\GatherConnection.class 4 | org\poem\api\vo\query\DbQueryVO.class 5 | org\poem\api\vo\GatherTableVO.class 6 | org\poem\entities\Kylo.class 7 | org\poem\api\GatherService.class 8 | org\poem\entities\Keys$UniqueKeys0.class 9 | org\poem\api\vo\table\vo\CreateTableFiels.class 10 | org\poem\utils\SpringUtils.class 11 | org\poem\entities\Keys.class 12 | org\poem\gen\JooqPgGen.class 13 | org\poem\api\vo\query\TableQueryVO.class 14 | org\poem\conf\JooqConfiguration.class 15 | org\poem\loghelper\file\FileService.class 16 | org\poem\utils\ThreadUtils.class 17 | org\poem\api\vo\TableVO.class 18 | org\poem\entities\Tables.class 19 | org\poem\entities\tables\DsgGatherTable.class 20 | org\poem\service\databases\OracleGatherData.class 21 | org\poem\api\MetadataService.class 22 | org\poem\api\vo\page\Page.class 23 | org\poem\entities\Indexes.class 24 | org\poem\loghelper\file\impl\HdfsFileServiceImpl.class 25 | org\poem\dao\BaseDao.class 26 | org\poem\dao\info\DsgGatherTableDao.class 27 | org\poem\api\vo\TableFieldsVO.class 28 | org\poem\controller\GatherController.class 29 | org\poem\api\vo\column\entity\DataSetVO.class 30 | org\poem\dao\BaseDaoImpl.class 31 | org\poem\utils\GatherDataSourceUtils.class 32 | org\poem\entities\tables\records\DsgGatherInfoRecord.class 33 | org\poem\entities\tables\records\DsgGatherDbRecord.class 34 | org\poem\service\connect\DataType.class 35 | org\poem\api\vo\databases\entity\DateBaseEntity.class 36 | org\poem\dao\gather\GatherInfoDao.class 37 | org\poem\service\connect\GatherBuilder.class 38 | org\poem\MetadataGatherApp.class 39 | org\poem\service\databases\MysqlGatherData.class 40 | org\poem\service\GatherServiceImpl.class 41 | org\poem\api\vo\GatherDBVO.class 42 | org\poem\conf\MysqlConf.class 43 | org\poem\entities\DefaultCatalog.class 44 | org\poem\dao\info\DsgGatherDBDao.class 45 | org\poem\entities\tables\DsgGatherDb.class 46 | org\poem\dao\info\DsgGatherTableFieldsDao.class 47 | org\poem\loghelper\LoggerHelper.class 48 | org\poem\service\databases\abstracts\AbstractGatherDataBaseInter.class 49 | org\poem\Swagger2Config.class 50 | org\poem\api\vo\GatherVO.class 51 | org\poem\entities\tables\records\DsgGatherTableRecord.class 52 | org\poem\loghelper\file\impl\LocalFileServiceImpl.class 53 | org\poem\entities\tables\DsgGatherInfo.class 54 | org\poem\api\vo\QueryGatherDBTableFieldsVO.class 55 | org\poem\api\vo\GatherDBTableFieldsVO.class 56 | org\poem\service\MetadataServiceImpl.class 57 | org\poem\utils\SnowFlake.class 58 | org\poem\api\vo\DbVO.class 59 | org\poem\utils\HdfsUtils.class 60 | org\poem\entities\Indexes$Indexes0.class 61 | org\poem\entities\tables\DsgGatherTableFields.class 62 | org\poem\service\databases\GatherDataBaseInter.class 63 | org\poem\api\vo\table\entity\TableEntity.class 64 | -------------------------------------------------------------------------------- /target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst: -------------------------------------------------------------------------------- 1 | C:\code\36-metadata-gather\src\main\java\org\poem\loghelper\file\impl\HdfsFileServiceImpl.java 2 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\table\vo\CreateTableFiels.java 3 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\Indexes.java 4 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\tables\DsgGatherTableFields.java 5 | C:\code\36-metadata-gather\src\main\java\org\poem\controller\GatherController.java 6 | C:\code\36-metadata-gather\src\main\java\org\poem\dao\info\DsgGatherTableDao.java 7 | C:\code\36-metadata-gather\src\main\java\org\poem\service\databases\abstracts\AbstractGatherDataBaseInter.java 8 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\tables\records\DsgGatherDbRecord.java 9 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\databases\entity\DateBaseEntity.java 10 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\table\entity\TableEntity.java 11 | C:\code\36-metadata-gather\src\main\java\org\poem\dao\BaseDao.java 12 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\DefaultCatalog.java 13 | C:\code\36-metadata-gather\src\main\java\org\poem\service\MetadataServiceImpl.java 14 | C:\code\36-metadata-gather\src\main\java\org\poem\utils\HdfsUtils.java 15 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\tables\DsgGatherDb.java 16 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\TableVO.java 17 | C:\code\36-metadata-gather\src\main\java\org\poem\conf\MysqlConf.java 18 | C:\code\36-metadata-gather\src\main\java\org\poem\loghelper\file\FileService.java 19 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\tables\records\DsgGatherTableRecord.java 20 | C:\code\36-metadata-gather\src\main\java\org\poem\gen\JooqPgGen.java 21 | C:\code\36-metadata-gather\src\main\java\org\poem\Swagger2Config.java 22 | C:\code\36-metadata-gather\src\main\java\org\poem\loghelper\file\impl\LocalFileServiceImpl.java 23 | C:\code\36-metadata-gather\src\main\java\org\poem\conf\JooqConfiguration.java 24 | C:\code\36-metadata-gather\src\main\java\org\poem\dao\info\DsgGatherTableFieldsDao.java 25 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\GatherDBTableFieldsVO.java 26 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\tables\DsgGatherTable.java 27 | C:\code\36-metadata-gather\src\main\java\org\poem\api\GatherService.java 28 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\GatherTableVO.java 29 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\Tables.java 30 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\Kylo.java 31 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\query\TableFieldsQueryVO.java 32 | C:\code\36-metadata-gather\src\main\java\org\poem\service\connect\DataType.java 33 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\GatherVO.java 34 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\tables\records\DsgGatherInfoRecord.java 35 | C:\code\36-metadata-gather\src\main\java\org\poem\service\connect\GatherBuilder.java 36 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\page\Page.java 37 | C:\code\36-metadata-gather\src\main\java\org\poem\MetadataGatherApp.java 38 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\column\entity\DataSetVO.java 39 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\query\DbQueryVO.java 40 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\Keys.java 41 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\query\TableQueryVO.java 42 | C:\code\36-metadata-gather\src\main\java\org\poem\dao\info\DsgGatherDBDao.java 43 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\GatherDBVO.java 44 | C:\code\36-metadata-gather\src\main\java\org\poem\utils\GatherDataSourceUtils.java 45 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\tables\records\DsgGatherTableFieldsRecord.java 46 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\TableFieldsVO.java 47 | C:\code\36-metadata-gather\src\main\java\org\poem\utils\ThreadUtils.java 48 | C:\code\36-metadata-gather\src\main\java\org\poem\dao\BaseDaoImpl.java 49 | C:\code\36-metadata-gather\src\main\java\org\poem\loghelper\LoggerHelper.java 50 | C:\code\36-metadata-gather\src\main\java\org\poem\utils\SpringUtils.java 51 | C:\code\36-metadata-gather\src\main\java\org\poem\service\GatherServiceImpl.java 52 | C:\code\36-metadata-gather\src\main\java\org\poem\utils\SnowFlake.java 53 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\DbVO.java 54 | C:\code\36-metadata-gather\src\main\java\org\poem\dao\gather\GatherInfoDao.java 55 | C:\code\36-metadata-gather\src\main\java\org\poem\service\databases\OracleGatherData.java 56 | C:\code\36-metadata-gather\src\main\java\org\poem\service\databases\GatherDataBaseInter.java 57 | C:\code\36-metadata-gather\src\main\java\org\poem\service\databases\MysqlGatherData.java 58 | C:\code\36-metadata-gather\src\main\java\org\poem\entities\tables\DsgGatherInfo.java 59 | C:\code\36-metadata-gather\src\main\java\org\poem\api\vo\QueryGatherDBTableFieldsVO.java 60 | C:\code\36-metadata-gather\src\main\java\org\poem\service\connect\GatherConnection.java 61 | C:\code\36-metadata-gather\src\main\java\org\poem\api\MetadataService.java 62 | --------------------------------------------------------------------------------