├── LICENSE ├── README.md ├── client ├── kd-config │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ ├── com │ │ │ │ └── knowlegene │ │ │ │ │ └── parent │ │ │ │ │ └── config │ │ │ │ │ ├── common │ │ │ │ │ ├── constantenum │ │ │ │ │ │ ├── DBOperationEnum.java │ │ │ │ │ │ ├── DatabaseTypeEnum.java │ │ │ │ │ │ ├── HiveTypeEnum.java │ │ │ │ │ │ ├── TimeTypeEnum.java │ │ │ │ │ │ └── TokenEnum.java │ │ │ │ │ └── event │ │ │ │ │ │ ├── ESExportType.java │ │ │ │ │ │ ├── ESImportType.java │ │ │ │ │ │ ├── FileExportType.java │ │ │ │ │ │ ├── FileImportType.java │ │ │ │ │ │ ├── GbaseExportType.java │ │ │ │ │ │ ├── GbaseImportType.java │ │ │ │ │ │ ├── HiveExportType.java │ │ │ │ │ │ ├── HiveImportType.java │ │ │ │ │ │ ├── MySQLExportType.java │ │ │ │ │ │ ├── MySQLImportType.java │ │ │ │ │ │ ├── Neo4jExportType.java │ │ │ │ │ │ ├── Neo4jImportType.java │ │ │ │ │ │ ├── OracleExportType.java │ │ │ │ │ │ ├── OracleImportType.java │ │ │ │ │ │ └── SwapJobEventType.java │ │ │ │ │ ├── dao │ │ │ │ │ └── AbstractSqlBaseDaoImpl.java │ │ │ │ │ ├── model │ │ │ │ │ ├── CustomizeHadoopOptions.java │ │ │ │ │ └── sql │ │ │ │ │ │ └── Lexer.java │ │ │ │ │ ├── pojo │ │ │ │ │ ├── BaseDO.java │ │ │ │ │ ├── BaseDTO.java │ │ │ │ │ └── sql │ │ │ │ │ │ ├── SQLOperator.java │ │ │ │ │ │ ├── SQLOperators.java │ │ │ │ │ │ ├── SQLResult.java │ │ │ │ │ │ └── SQLStatement.java │ │ │ │ │ └── util │ │ │ │ │ ├── BaseSqlParserFactoryUtil.java │ │ │ │ │ ├── BaseUtil.java │ │ │ │ │ ├── HSqlThreadLocalUtil.java │ │ │ │ │ ├── HdfsFileUtil.java │ │ │ │ │ ├── HdfsPipelineUtil.java │ │ │ │ │ ├── JdbcUtil.java │ │ │ │ │ ├── PipelineSingletonUtil.java │ │ │ │ │ ├── PropertiesUtil.java │ │ │ │ │ └── ProxyFactoryUtil.java │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── beam │ │ │ │ └── sdk │ │ │ │ └── io │ │ │ │ ├── elasticsearch │ │ │ │ └── ElasticsearchIO.java │ │ │ │ └── jdbc │ │ │ │ └── JdbcIO.java │ │ └── resources │ │ │ ├── log4j2.xml │ │ │ └── logback.xml │ │ └── test │ │ └── java │ │ └── com │ │ └── knowlegene │ │ └── parent │ │ └── config │ │ └── HdfsFileTest.java └── kd-process │ ├── pom.xml │ └── src │ ├── main │ └── java │ │ └── com │ │ └── knowlegene │ │ └── parent │ │ └── process │ │ ├── SwapDirectApplication.java │ │ ├── SwapFlinkApplication.java │ │ ├── SwapSparkApplication.java │ │ ├── common │ │ ├── annotation │ │ │ └── StoredAsProperty.java │ │ └── constantenum │ │ │ ├── Neo4jEnum.java │ │ │ └── OptionsEnum.java │ │ ├── io │ │ ├── es │ │ │ ├── ESSwap.java │ │ │ └── impl │ │ │ │ └── ESSwapImpl.java │ │ ├── file │ │ │ ├── HadoopFile.java │ │ │ └── HadoopFileImpl.java │ │ ├── jdbc │ │ │ ├── AbstractSwapBase.java │ │ │ ├── GbaseSwap.java │ │ │ ├── HiveSwap.java │ │ │ ├── MySQLSwap.java │ │ │ ├── OracleSwap.java │ │ │ └── impl │ │ │ │ ├── GbaseSwapExport.java │ │ │ │ ├── GbaseSwapImpl.java │ │ │ │ ├── GbaseSwapImport.java │ │ │ │ ├── HiveSwapExport.java │ │ │ │ ├── HiveSwapImpl.java │ │ │ │ ├── HiveSwapImport.java │ │ │ │ ├── MySQLSwapExport.java │ │ │ │ ├── MySQLSwapImpl.java │ │ │ │ ├── MySQLSwapImport.java │ │ │ │ ├── OracleSwapExport.java │ │ │ │ ├── OracleSwapImpl.java │ │ │ │ └── OracleSwapImport.java │ │ ├── logs │ │ │ └── LogPatternConverter.java │ │ └── neo4j │ │ │ ├── Neo4jIO.java │ │ │ ├── Neo4jSwap.java │ │ │ └── impl │ │ │ ├── Neo4jSwapExport.java │ │ │ ├── Neo4jSwapImpl.java │ │ │ └── Neo4jSwapImport.java │ │ ├── pojo │ │ ├── DataSources.java │ │ ├── DefaultHCatRecord.java │ │ ├── JobInfo.java │ │ ├── NestingFields.java │ │ ├── ObjectCoder.java │ │ ├── SQLOptions.java │ │ ├── SwapOptions.java │ │ ├── db │ │ │ └── DBOptions.java │ │ ├── es │ │ │ └── ESOptions.java │ │ ├── file │ │ │ └── FileOptions.java │ │ ├── hive │ │ │ └── HiveOptions.java │ │ └── neo4j │ │ │ ├── Neo4jCode.java │ │ │ ├── Neo4jObject.java │ │ │ └── Neo4jOptions.java │ │ ├── runners │ │ ├── SwapRunners.java │ │ ├── common │ │ │ ├── BaseJobRunners.java │ │ │ └── BaseRunners.java │ │ └── options │ │ │ ├── IndexerPipelineOptions.java │ │ │ ├── SwapFlinkPipelineOptions.java │ │ │ ├── SwapJavaPipelineOptions.java │ │ │ └── SwapSparkPipelineOptions.java │ │ ├── swap │ │ ├── ESExportJob.java │ │ ├── ESImportJob.java │ │ ├── ExportJobBase.java │ │ ├── FileExportJob.java │ │ ├── FileImportJob.java │ │ ├── GbaseExportJob.java │ │ ├── GbaseImportJob.java │ │ ├── HiveExportJob.java │ │ ├── HiveImportJob.java │ │ ├── ImportJobBase.java │ │ ├── JobBase.java │ │ ├── MySQLExportJob.java │ │ ├── MySQLImportJob.java │ │ ├── Neo4jExportJob.java │ │ ├── Neo4jImportJob.java │ │ ├── OracleExportJob.java │ │ ├── OracleImportJob.java │ │ ├── common │ │ │ └── BaseSwap.java │ │ ├── dispatcher │ │ │ └── SwapMaster.java │ │ └── event │ │ │ ├── ESExportTaskEvent.java │ │ │ ├── ESImportTaskEvent.java │ │ │ ├── FileExportTaskEvent.java │ │ │ ├── FileImportTaskEvent.java │ │ │ ├── GbaseExportTaskEvent.java │ │ │ ├── GbaseImportTaskEvent.java │ │ │ ├── HiveExportTaskEvent.java │ │ │ ├── HiveImportTaskEvent.java │ │ │ ├── MySQLExportTaskEvent.java │ │ │ ├── MySQLImportTaskEvent.java │ │ │ ├── Neo4jExportTaskEvent.java │ │ │ ├── Neo4jImportTaskEvent.java │ │ │ ├── OracleExportTaskEvent.java │ │ │ ├── OracleImportTaskEvent.java │ │ │ └── SwapJobEvent.java │ │ ├── tool │ │ └── BaseSwapTool.java │ │ ├── transform │ │ ├── CombineTransform.java │ │ ├── DistinctTransform.java │ │ ├── ESTransform.java │ │ ├── FilterTransform.java │ │ ├── JdbcTransform.java │ │ ├── PrintTransform.java │ │ ├── TypeConversion.java │ │ └── common │ │ │ └── BaseTransform.java │ │ └── util │ │ ├── ComboPooledDataSourceUtil.java │ │ ├── CommonUtil.java │ │ ├── DataSourceUtil.java │ │ ├── JobThreadLocalUtil.java │ │ ├── SqlUtil.java │ │ ├── SwapMasterUtil.java │ │ └── UUIDFactoryUtil.java │ └── test │ └── java │ └── com │ └── knowlegene │ └── parent │ └── process │ ├── kafka │ ├── KafkaConsumerTest.java │ └── KafkaProducerTest.java │ └── route │ └── swap │ ├── ESSwapTest.java │ ├── FileSwapTest.java │ ├── GbaseSwapTest.java │ ├── HiveSwapTest.java │ ├── MySQLSwapTest.java │ ├── Neo4jSwapTest.java │ └── OracleSwapTest.java ├── core └── scheduler │ ├── pom.xml │ └── src │ ├── main │ └── java │ │ └── com │ │ └── knowlegene │ │ └── parent │ │ └── scheduler │ │ ├── event │ │ ├── AbstractEvent.java │ │ ├── AsyncDispatcher.java │ │ ├── Dispatcher.java │ │ ├── Event.java │ │ └── EventHandler.java │ │ ├── pojo │ │ └── CacheEntry.java │ │ ├── service │ │ ├── AbstractService.java │ │ ├── CompositeService.java │ │ ├── Service.java │ │ ├── ServiceOperations.java │ │ ├── ServiceStateException.java │ │ └── ServiceStateModel.java │ │ └── utils │ │ └── CacheManager.java │ └── test │ └── java │ └── com │ └── knowlegene │ └── parent │ └── scheduler │ └── service │ ├── TestCacheManager.java │ ├── TestJobEvent.java │ ├── TestJobEventType.java │ ├── TestSimpleMRAppMaster.java │ ├── TestSimpleMRAppMasterRun.java │ ├── TestTaskEvent.java │ └── TestTaskEventType.java ├── docs └── img │ ├── clentsource.png │ └── source.png └── pom.xml /client/kd-config/src/main/java/com/knowlegene/parent/config/common/constantenum/DBOperationEnum.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.constantenum; 2 | 3 | /** 4 | * @Classname DBOperationEnum 5 | * @Description TODO 6 | * @Date 2020/6/10 14:07 7 | * @Created by limeng 8 | */ 9 | public enum DBOperationEnum { 10 | /** 11 | * 各个源操作 12 | */ 13 | HIVE_IMPORT_DB(1,"HIVE_IMPORT_DB"),HIVE_EXPORT_DB(2,"HIVE_EXPORT_DB"),HIVE_IMPORT(41,"HIVE_IMPORT"),HIVE_EXPORT(42,"HIVE_EXPORT"), 14 | MYSQL_IMPORT_DB(3,"MYSQL_IMPORT_DB"),MYSQL_EXPORT_DB(4,"MYSQL_EXPORT_DB"),MYSQL_IMPORT(43,"MYSQL_IMPORT"),MYSQL_EXPORT(44,"MYSQL_EXPORT"), 15 | ORACLE_IMPORT_DB(5,"ORACLE_IMPORT_DB"),ORACLE_EXPORT_DB(6,"ORACLE_EXPORT_DB"),ORACLE_IMPORT(45,"ORACLE_IMPORT"),ORACLE_EXPORT(46,"ORACLE_EXPORT"), 16 | GBASE_IMPORT_DB(7,"GBASE_IMPORT_DB"),GBASE_EXPORT_DB(8,"GBASE_EXPORT_DB"),GBASE_IMPORT(47,"GBASE_IMPORT"),GBASE_EXPORT(48,"GBASE_EXPORT"), 17 | ES_IMPORT(49,"ES_IMPORT"),ES_EXPORT(50,"ES_EXPORT"), 18 | FILE_IMPORT(51,"FILE_IMPORT"),FILE_EXPORT(52,"FILE_EXPORT"), 19 | NEO4J_IMPORT(53,"NEO4J_IMPORT"),NEO4J_EXPORT(54,"NEO4J_EXPORT"), 20 | 21 | 22 | PCOLLECTION_QUERYS(300,"PCOLLECTION_QUERYS"),IMPORT(400,"IMPORT"),EXPORT(401,"EXPORT"); 23 | 24 | 25 | /** 26 | * 类型标记 27 | */ 28 | private int value; 29 | 30 | /** 31 | * 类型名称 32 | */ 33 | private String name; 34 | 35 | public int getValue() { 36 | return value; 37 | } 38 | 39 | public void setValue(int value) { 40 | this.value = value; 41 | } 42 | 43 | public String getName() { 44 | return name; 45 | } 46 | 47 | public void setName(String name) { 48 | this.name = name; 49 | } 50 | 51 | DBOperationEnum() { 52 | } 53 | 54 | DBOperationEnum(int value, String name) { 55 | this.value = value; 56 | this.name = name; 57 | } 58 | 59 | 60 | public static DBOperationEnum getEnum(String keys){ 61 | DBOperationEnum[] values = DBOperationEnum.values(); 62 | for(DBOperationEnum d:values){ 63 | if(keys.equalsIgnoreCase(d.getName())) return d; 64 | } 65 | return null; 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/constantenum/DatabaseTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.constantenum; 2 | 3 | import com.knowlegene.parent.config.util.BaseUtil; 4 | 5 | /** 6 | * 数据类型 7 | * 8 | * @Author: limeng 9 | * @Date: 2019/7/19 16:42 10 | */ 11 | public enum DatabaseTypeEnum { 12 | /** 13 | * 各个源 14 | */ 15 | HIVE(1,"hive"),ORACLE(2,"oracle"),MYSQL(3,"mysql") 16 | ,ES(4,"es"),GBASE(5,"gbase"), 17 | FILE(6,"file"),NEO4J(7,"neo4j"); 18 | 19 | 20 | /** 21 | * 类型标记 22 | */ 23 | private int value; 24 | 25 | /** 26 | * 类型名称 27 | */ 28 | private String name; 29 | 30 | public int getValue() { 31 | return value; 32 | } 33 | 34 | public void setValue(int value) { 35 | this.value = value; 36 | } 37 | 38 | public String getName() { 39 | return name; 40 | } 41 | 42 | public void setName(String name) { 43 | this.name = name; 44 | } 45 | 46 | DatabaseTypeEnum() { 47 | } 48 | 49 | DatabaseTypeEnum(int value, String name) { 50 | this.value = value; 51 | this.name = name; 52 | } 53 | 54 | public static DatabaseTypeEnum queryValue(String name){ 55 | DatabaseTypeEnum result = null; 56 | if(BaseUtil.isBlank(name)) return result; 57 | 58 | for(DatabaseTypeEnum d:DatabaseTypeEnum.values()){ 59 | if(d.getName().equalsIgnoreCase(name)){ 60 | return d; 61 | } 62 | } 63 | return result; 64 | } 65 | 66 | 67 | 68 | public static boolean isDB(String name){ 69 | DatabaseTypeEnum databaseTypeEnum = queryValue(name); 70 | boolean result=false; 71 | switch (databaseTypeEnum){ 72 | case MYSQL: 73 | result = true; 74 | break; 75 | case GBASE: 76 | result = true; 77 | break; 78 | case ORACLE: 79 | result = true; 80 | break; 81 | } 82 | return result; 83 | } 84 | 85 | 86 | } 87 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/constantenum/HiveTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.constantenum; 2 | 3 | /** 4 | * @Author: limeng 5 | * @Date: 2019/7/23 23:51 6 | */ 7 | public enum HiveTypeEnum { 8 | /** 9 | * sql解析 10 | */ 11 | SELECT1 (1,"^(select)(.+)(from)(.+)"),SAVE1(2,"^(insert)(.+)"),DELETE1(3,"(delete from)(.+)"), 12 | CREATE1(4,"(create)(.+)"),TRUNCATE1(5,"(truncate)(.+)"),LOAD1(6,"(load)(.+)"),SET1(7,"^(set)(.+)") 13 | ,WITH1(8,"(with)(.+)"),SELECTNAME(9,"select"),INSERTNAME(10,"insert"),DELETENAME(11,"delete"),DDL(12,"ddl") 14 | ,DROP1(13,"(drop)(.+)") 15 | ,HCATALOGMETASTOREURIS(14,"hive.metastore.uris"),HIVEDATABASE(15,"hivedatabase"),HIVETABLE(16,"hivetable"), 16 | TEZ_ENGINE(17,"tez"),SPARK_ENGINE(18,"spark"),ENGINE(19,"hive_engine"); 17 | 18 | /** 19 | * hive参数 20 | */ 21 | /** 22 | * 类型标记 23 | */ 24 | private int value; 25 | 26 | /** 27 | * 类型名称 28 | */ 29 | private String name; 30 | 31 | public int getValue() { 32 | return value; 33 | } 34 | 35 | public void setValue(int value) { 36 | this.value = value; 37 | } 38 | 39 | public String getName() { 40 | return name; 41 | } 42 | 43 | public void setName(String name) { 44 | this.name = name; 45 | } 46 | 47 | HiveTypeEnum() { 48 | } 49 | 50 | HiveTypeEnum(int value, String name) { 51 | this.value = value; 52 | this.name = name; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/constantenum/TimeTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.constantenum; 2 | 3 | /** 4 | * 日期正则 5 | * @Author: limeng 6 | * @Date: 2019/7/23 23:51 7 | */ 8 | public enum TimeTypeEnum { 9 | /** 10 | * 11 | */ 12 | YMDHMSS(1,"yyyy-MM-dd HH:mm:ss.SSS"),YMDHMS(2,"yyyy-MM-dd HH:mm:ss"); 13 | 14 | /** 15 | * hive参数 16 | */ 17 | /** 18 | * 类型标记 19 | */ 20 | private int value; 21 | 22 | /** 23 | * 类型名称 24 | */ 25 | private String name; 26 | 27 | public int getValue() { 28 | return value; 29 | } 30 | 31 | public void setValue(int value) { 32 | this.value = value; 33 | } 34 | 35 | public String getName() { 36 | return name; 37 | } 38 | 39 | public void setName(String name) { 40 | this.name = name; 41 | } 42 | 43 | TimeTypeEnum() { 44 | } 45 | 46 | TimeTypeEnum(int value, String name) { 47 | this.value = value; 48 | this.name = name; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/ESExportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLExportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:22 7 | * @Created by limeng 8 | */ 9 | public enum ESExportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/ESImportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLImportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:21 7 | * @Created by limeng 8 | */ 9 | public enum ESImportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/FileExportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLExportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:22 7 | * @Created by limeng 8 | */ 9 | public enum FileExportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/FileImportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLImportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:21 7 | * @Created by limeng 8 | */ 9 | public enum FileImportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/GbaseExportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLExportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:22 7 | * @Created by limeng 8 | */ 9 | public enum GbaseExportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/GbaseImportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLImportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:21 7 | * @Created by limeng 8 | */ 9 | public enum GbaseImportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/HiveExportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname HiveExportTaskEvent 5 | * @Description TODO 6 | * @Date 2020/6/11 15:49 7 | * @Created by limeng 8 | */ 9 | public enum HiveExportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/HiveImportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname HiveImportTaskEvent 5 | * @Description TODO 6 | * @Date 2020/6/11 15:46 7 | * @Created by limeng 8 | */ 9 | public enum HiveImportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/MySQLExportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLExportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:22 7 | * @Created by limeng 8 | */ 9 | public enum MySQLExportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/MySQLImportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLImportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:21 7 | * @Created by limeng 8 | */ 9 | public enum MySQLImportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/Neo4jExportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname Neo4jExportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:22 7 | * @Created by limeng 8 | */ 9 | public enum Neo4jExportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/Neo4jImportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname Neo4jImportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:21 7 | * @Created by limeng 8 | */ 9 | public enum Neo4jImportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/OracleExportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLExportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:22 7 | * @Created by limeng 8 | */ 9 | public enum OracleExportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/OracleImportType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname MySQLImportType 5 | * @Description TODO 6 | * @Date 2020/6/22 15:21 7 | * @Created by limeng 8 | */ 9 | public enum OracleImportType { 10 | T_KILL, 11 | T_SCHEDULE, 12 | T_IMPORT, 13 | T_EXPORT 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/common/event/SwapJobEventType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.common.event; 2 | 3 | /** 4 | * @Classname JobEventType 5 | * @Description TODO 6 | * @Date 2020/6/10 15:44 7 | * @Created by limeng 8 | */ 9 | public enum SwapJobEventType { 10 | JOB_KILL, 11 | JOB_INIT, 12 | JOB_START 13 | } 14 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/model/CustomizeHadoopOptions.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.model; 2 | 3 | import org.apache.beam.sdk.io.hdfs.HadoopFileSystemOptions; 4 | import org.apache.beam.sdk.options.Description; 5 | 6 | /** 7 | * 自定义hadoop file 8 | * @Author: limeng 9 | * @Date: 2019/7/23 22:01 10 | */ 11 | public interface CustomizeHadoopOptions extends HadoopFileSystemOptions { 12 | @Description("input file") 13 | String getInputFile(); 14 | void setInputFile(String in); 15 | 16 | @Description("output") 17 | String getOutput(); 18 | void setOutput(String out); 19 | } 20 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/model/sql/Lexer.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.model.sql; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.TokenEnum; 4 | 5 | /** 6 | * @Author: limeng 7 | * @Date: 2019/7/31 14:44 8 | */ 9 | public class Lexer { 10 | private TokenEnum token; 11 | private Lexer next; 12 | private Lexer pre; 13 | 14 | public TokenEnum getToken() { 15 | return token; 16 | } 17 | 18 | public void setToken(TokenEnum token) { 19 | this.token = token; 20 | } 21 | 22 | public Lexer getNext() { 23 | return next; 24 | } 25 | 26 | public void setNext(Lexer next) { 27 | this.next = next; 28 | } 29 | 30 | public Lexer getPre() { 31 | return pre; 32 | } 33 | 34 | public void setPre(Lexer pre) { 35 | this.pre = pre; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/pojo/BaseDO.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.pojo; 2 | 3 | import lombok.Data; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * @Author: limeng 9 | * @Date: 2019/7/22 17:31 10 | */ 11 | @Data 12 | public class BaseDO implements Serializable { 13 | 14 | private Integer id; 15 | } 16 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/pojo/BaseDTO.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.pojo; 2 | 3 | import lombok.Data; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * @Author: limeng 9 | * @Date: 2019/8/1 15:12 10 | */ 11 | @Data 12 | public class BaseDTO implements Serializable { 13 | 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/pojo/sql/SQLOperator.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.pojo.sql; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * @Author: limeng 7 | * @Date: 2019/8/8 18:59 8 | */ 9 | public interface SQLOperator { 10 | public enum Operator { 11 | DDL, CREATE, INSERT 12 | } 13 | public List listBuilders(); 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/pojo/sql/SQLOperators.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.pojo.sql; 2 | 3 | 4 | 5 | import com.knowlegene.parent.config.pojo.BaseDO; 6 | import com.knowlegene.parent.config.util.BaseUtil; 7 | import lombok.Data; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | /** 13 | * @Author: limeng 14 | * @Date: 2019/8/8 19:26 15 | */ 16 | @Data 17 | public class SQLOperators extends BaseDO implements SQLOperator { 18 | 19 | private List queryBuilderqList = new ArrayList<>(); 20 | private Operator operator; 21 | protected String dbType; 22 | private Boolean isOrder; 23 | public SQLOperators() { 24 | } 25 | 26 | public SQLOperators(Operator operator) { 27 | this.operator = operator; 28 | } 29 | 30 | @Override 31 | public List listBuilders() { 32 | return queryBuilderqList; 33 | } 34 | 35 | 36 | public SQLOperators addInsert(String sql){ 37 | if(BaseUtil.isNotBlank(sql)){ 38 | queryBuilderqList.add(sql); 39 | } 40 | return this; 41 | } 42 | 43 | public SQLOperators addWhere(String sql){ 44 | if(BaseUtil.isNotBlank(sql)){ 45 | queryBuilderqList.add(sql); 46 | } 47 | return this; 48 | } 49 | 50 | public SQLOperators addJoin(String sql){ 51 | if(BaseUtil.isNotBlank(sql)){ 52 | queryBuilderqList.add(sql); 53 | } 54 | return this; 55 | } 56 | 57 | public SQLOperators addParent(String sql){ 58 | if(BaseUtil.isNotBlank(sql)){ 59 | queryBuilderqList.add(sql); 60 | } 61 | return this; 62 | } 63 | 64 | public SQLOperators addFrom(String sql){ 65 | if(BaseUtil.isNotBlank(sql)){ 66 | queryBuilderqList.add(sql); 67 | } 68 | return this; 69 | } 70 | 71 | public SQLOperators addSelect(String sql){ 72 | if(BaseUtil.isNotBlank(sql)){ 73 | queryBuilderqList.add(sql); 74 | } 75 | return this; 76 | } 77 | 78 | public SQLOperators addDDL(String sql){ 79 | if(BaseUtil.isNotBlank(sql)){ 80 | queryBuilderqList.add(sql); 81 | } 82 | return this; 83 | } 84 | 85 | @Override 86 | public String toString() { 87 | if(!BaseUtil.isBlankSet(queryBuilderqList)){ 88 | StringBuffer sb=new StringBuffer(); 89 | for (String sql:queryBuilderqList) { 90 | sb.append(sql+" "); 91 | } 92 | return sb.toString(); 93 | } 94 | return null; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/pojo/sql/SQLResult.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.pojo.sql; 2 | 3 | import com.knowlegene.parent.config.pojo.BaseDO; 4 | import lombok.Data; 5 | import org.springframework.util.CollectionUtils; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | 10 | /** 11 | * @Author: limeng 12 | * @Date: 2019/8/8 19:06 13 | */ 14 | @Data 15 | public class SQLResult extends BaseDO{ 16 | /** 17 | * 总况是否有序 18 | */ 19 | private Boolean isOrder; 20 | /** 21 | * 定义语句 22 | * 插入语句 23 | * 容器 24 | */ 25 | private List insertOperator = new ArrayList<>(); 26 | private List ddlOperator = new ArrayList<>(); 27 | 28 | 29 | public List getList(){ 30 | int count = insertOperator.size() + ddlOperator.size(); 31 | List sqls = null; 32 | if(count >= 1){ 33 | List insertOperators = this.getSqlList(insertOperator); 34 | List ddlOperators = this.getSqlList(ddlOperator); 35 | sqls = new ArrayList<>(); 36 | sqls.addAll(insertOperators); 37 | sqls.addAll(ddlOperators); 38 | } 39 | return sqls; 40 | } 41 | 42 | /** 43 | * 获取插入sql 44 | * @return 45 | */ 46 | public List getInserts(){ 47 | List sqls = null; 48 | if (!CollectionUtils.isEmpty(insertOperator)) { 49 | sqls = this.getSqlList(insertOperator); 50 | } 51 | return sqls; 52 | } 53 | 54 | /** 55 | * 获取ddl 56 | * @return 57 | */ 58 | public List getDdls(){ 59 | List sqls = null; 60 | if (!CollectionUtils.isEmpty(ddlOperator)) { 61 | sqls = this.getSqlList(ddlOperator); 62 | } 63 | return sqls; 64 | } 65 | private List getSqlList(List list){ 66 | List sqls = null; 67 | if (!CollectionUtils.isEmpty(list)) { 68 | sqls = new ArrayList<>(); 69 | for(SQLOperators operatorDO : list){ 70 | String sql = operatorDO.toString(); 71 | sqls.add(sql); 72 | } 73 | } 74 | return sqls; 75 | } 76 | 77 | /** 78 | * 插入 79 | * 80 | */ 81 | public SQLResult insert(SQLOperators operator){ 82 | if(operator!=null){ 83 | insertOperator.add(operator); 84 | } 85 | return this; 86 | } 87 | 88 | /** 89 | * with 90 | * set 91 | * create 92 | * truncate 93 | * 94 | */ 95 | public SQLResult ddl(SQLOperators operator){ 96 | if(operator!=null){ 97 | ddlOperator.add(operator); 98 | } 99 | return this; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/pojo/sql/SQLStatement.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.pojo.sql; 2 | 3 | 4 | import com.knowlegene.parent.config.pojo.BaseDO; 5 | import lombok.Data; 6 | 7 | /** 8 | * @Author: limeng 9 | * @Date: 2019/8/8 19:24 10 | */ 11 | @Data 12 | public class SQLStatement extends BaseDO { 13 | private String sql; 14 | 15 | public SQLStatement() { 16 | 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/util/BaseSqlParserFactoryUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.util; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DatabaseTypeEnum; 4 | import com.knowlegene.parent.config.common.constantenum.HiveTypeEnum; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.regex.Matcher; 9 | import java.util.regex.Pattern; 10 | 11 | /** 12 | * 解析sql 13 | * @Author: limeng 14 | * @Date: 2019/7/22 21:37 15 | */ 16 | public class BaseSqlParserFactoryUtil { 17 | private static Logger logger = LoggerFactory.getLogger(BaseSqlParserFactoryUtil.class); 18 | /** 19 | * 查询 20 | */ 21 | private static final String SELECT1 ="^(select)(.+)(from)(.+)"; 22 | 23 | private static final String SAVE1 = "^(insert)(.+)"; 24 | private static final String DELETE1="(delete from)(.+)"; 25 | private static final String CREATE1="(create)(.+)"; 26 | private static final String TRUNCATE1="(truncate)(.+)"; 27 | private static final String LOAD1="^(load)(.+)"; 28 | 29 | public static int generateParser(Integer source,String originalSql) { 30 | if(source == null){ 31 | return -1; 32 | } 33 | HiveTypeEnum select1 = HiveTypeEnum.SELECT1; 34 | HiveTypeEnum save1 = HiveTypeEnum.SAVE1; 35 | HiveTypeEnum delete1 = HiveTypeEnum.DELETE1; 36 | 37 | HiveTypeEnum create1 = HiveTypeEnum.CREATE1; 38 | HiveTypeEnum truncate1 = HiveTypeEnum.TRUNCATE1; 39 | HiveTypeEnum load1 = HiveTypeEnum.LOAD1; 40 | HiveTypeEnum drop1 = HiveTypeEnum.DROP1; 41 | HiveTypeEnum set1 = HiveTypeEnum.SET1; 42 | 43 | 44 | boolean sourceBoolean = source.equals(DatabaseTypeEnum.HIVE.getValue()); 45 | if(sourceBoolean && contains(originalSql,select1.getName())) { 46 | return select1.getValue(); 47 | }else if(sourceBoolean && contains(originalSql,save1.getName())){ 48 | return save1.getValue(); 49 | }else if(sourceBoolean && contains(originalSql,delete1.getName())){ 50 | return delete1.getValue(); 51 | }else if(sourceBoolean && contains(originalSql,create1.getName())){ 52 | return create1.getValue(); 53 | }else if(sourceBoolean && contains(originalSql,truncate1.getName())){ 54 | return truncate1.getValue(); 55 | }else if(sourceBoolean && contains(originalSql,load1.getName())){ 56 | return load1.getValue(); 57 | }else if(sourceBoolean && contains(originalSql,drop1.getName())){ 58 | return drop1.getValue(); 59 | }else if(sourceBoolean && contains(originalSql,set1.getName())){ 60 | return set1.getValue(); 61 | }else { 62 | logger.info("BaseSqlParserFactory generateParser error=>sql:{}",originalSql); 63 | return -1; 64 | } 65 | } 66 | 67 | 68 | /** 69 | * 包含 70 | * @param sql sql 71 | * @param regExp 正则 72 | * @return 73 | */ 74 | private static boolean contains(String sql,String regExp){ 75 | Pattern pattern=Pattern.compile(regExp,Pattern.CASE_INSENSITIVE); 76 | Matcher matcher=pattern.matcher(sql.toLowerCase()); 77 | return matcher.find(); 78 | } 79 | 80 | 81 | } 82 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/util/HSqlThreadLocalUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.util; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * @Author: limeng 7 | * @Date: 2019/8/15 23:15 8 | */ 9 | public class HSqlThreadLocalUtil { 10 | private static final ThreadLocal> CONTEXT_HOLDER = new ThreadLocal<>(); 11 | 12 | /** 13 | * @Description: id 14 | * @return void 15 | */ 16 | public static void setJob(List sqls) { 17 | CONTEXT_HOLDER.set(sqls); 18 | } 19 | 20 | /** 21 | * @Description: id 22 | */ 23 | public static List getJob() { 24 | return CONTEXT_HOLDER.get(); 25 | } 26 | 27 | /** 28 | * @Description: job 29 | */ 30 | public static void clearJob() { 31 | CONTEXT_HOLDER.remove(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/util/HdfsFileUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.util; 2 | 3 | import org.apache.beam.sdk.values.KV; 4 | import org.apache.hadoop.conf.Configuration; 5 | import org.apache.hadoop.fs.FSDataInputStream; 6 | import org.apache.hadoop.fs.FileSystem; 7 | import org.apache.hadoop.fs.Path; 8 | 9 | import java.io.BufferedReader; 10 | import java.io.IOException; 11 | import java.io.InputStream; 12 | import java.io.InputStreamReader; 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | import java.util.Properties; 16 | import java.util.regex.Matcher; 17 | import java.util.regex.Pattern; 18 | 19 | /** 20 | * hdfs file 21 | * @Author: limeng 22 | * @Date: 2019/7/25 21:58 23 | */ 24 | public class HdfsFileUtil { 25 | 26 | private Configuration conf; 27 | private static String REX=";"; 28 | //注释标记 29 | private String comment = "^[#\\-\\-]"; 30 | public HdfsFileUtil() { 31 | 32 | } 33 | 34 | /** 35 | * 读取文件内容 36 | */ 37 | public List getContent(String remoteFilePath) throws IOException { 38 | List result = null; 39 | this.getConf(remoteFilePath); 40 | FileSystem fs = FileSystem.get(conf); 41 | Path remotePath = new Path(remoteFilePath); 42 | FSDataInputStream in = fs.open(remotePath); 43 | result = this.getContentStream(in); 44 | in.close(); 45 | fs.close(); 46 | return result; 47 | } 48 | 49 | /** 50 | * 读取文件内容 51 | */ 52 | public List getContentStream(InputStream remoteFilePath) throws IOException { 53 | List result = new ArrayList<>(); 54 | BufferedReader d = new BufferedReader(new InputStreamReader(remoteFilePath)); 55 | String line = null; 56 | Pattern p = Pattern.compile(comment); 57 | Matcher m=null; 58 | while ((line = d.readLine()) != null) { 59 | m = p.matcher(line); 60 | if(!m.find()){ 61 | result.add(line); 62 | } 63 | } 64 | d.close(); 65 | return result; 66 | } 67 | 68 | public void getConf(String remoteFilePath){ 69 | conf = new Configuration(); 70 | Boolean hadoopPath = HdfsPipelineUtil.isHadoopPath(remoteFilePath); 71 | if(hadoopPath){ 72 | Properties properties = PropertiesUtil.getProperties(); 73 | String defaultName = properties.getProperty("kd.hdfs.defaultname"); 74 | conf = new Configuration(); 75 | conf.set("fs.default.name", defaultName); 76 | }else{ 77 | conf.set("fs.default.name", "file:///"); 78 | conf.set("mapred.job.tracker", "local"); 79 | } 80 | } 81 | 82 | /** 83 | * 清楚格式问题 84 | * 转换集合 85 | * @param sqlList 86 | * @return 87 | */ 88 | public List conversionSet(List sqlList) { 89 | List result = null; 90 | if(BaseUtil.isBlankSet(sqlList)){ 91 | return result; 92 | } 93 | result = new ArrayList<>(); 94 | StringBuffer sb= new StringBuffer(); 95 | for (String sql:sqlList){ 96 | sb.append(sql); 97 | if(sql.endsWith(REX)){ 98 | String newSql=sb.toString().replaceAll(";",""); 99 | result.add(newSql); 100 | sb= new StringBuffer(); 101 | } 102 | } 103 | return result; 104 | } 105 | 106 | 107 | public static KV splitMark(String str,String rex){ 108 | String[] split = str.split(rex); 109 | if(split != null && split.length>1){ 110 | return KV.of(split[0],"."+split[1]); 111 | } 112 | return KV.of(str,".cvs"); 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/util/HdfsPipelineUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.util; 2 | 3 | import com.knowlegene.parent.config.model.CustomizeHadoopOptions; 4 | import org.apache.beam.sdk.Pipeline; 5 | import org.apache.beam.sdk.io.TextIO; 6 | import org.apache.beam.sdk.options.PipelineOptions; 7 | import org.apache.beam.sdk.options.PipelineOptionsFactory; 8 | import org.apache.beam.sdk.values.PCollection; 9 | import org.apache.hadoop.conf.Configuration; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.io.Serializable; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | import java.util.Properties; 17 | 18 | 19 | /** 20 | * 获取文件 21 | * @Author: limeng 22 | * @Date: 2019/7/23 20:43 23 | */ 24 | public class HdfsPipelineUtil implements Serializable { 25 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 26 | private static final String START_HADOOP_PATH ="hdfs"; 27 | /** 28 | * 判断 29 | * @param path 路径 30 | * @return 31 | */ 32 | public static Boolean isHadoopPath(String path){ 33 | if(path.trim().startsWith(START_HADOOP_PATH)){ 34 | return true; 35 | } 36 | return false; 37 | } 38 | 39 | 40 | /** 41 | * 文件集合 42 | */ 43 | public static PCollection startDocumentImport(String path){ 44 | PCollection result=null; 45 | Pipeline pipeline = PipelineSingletonUtil.instance; 46 | PipelineOptions pipelineOptions = pipeline.getOptions(); 47 | if(BaseUtil.isNotBlank(path)){ 48 | if(isHadoopPath(path)){ 49 | result = hdfsDocumentImport(path); 50 | }else{ 51 | result = fileDocumentImport(path); 52 | } 53 | } 54 | return result; 55 | } 56 | 57 | /** 58 | * 获取hadoop文件 59 | * @return 60 | */ 61 | public static PCollection hdfsDocumentImport(String filePath){ 62 | Pipeline pipeline = initPipeline(); 63 | return pipeline.apply("hdfsDocumentImport", TextIO.read().from(filePath)); 64 | } 65 | 66 | /** 67 | * 或去本地文件 68 | * @return 69 | */ 70 | public static PCollection fileDocumentImport(String filePath){ 71 | Pipeline pipeline = PipelineSingletonUtil.instance; 72 | return pipeline.apply("fileDocumentImport", TextIO.read().from(filePath)); 73 | 74 | } 75 | 76 | /** 77 | * 初始化pipline 78 | * @return 79 | */ 80 | public static Pipeline initPipeline(){ 81 | CustomizeHadoopOptions options = PipelineOptionsFactory.create().as(CustomizeHadoopOptions.class); 82 | Configuration conf = new Configuration(); 83 | Properties properties = PropertiesUtil.getProperties(); 84 | String defaultName = properties.getProperty("kd.hdfs.defaultname"); 85 | String hdfsImpl = properties.getProperty("kd.hdfs.impl"); 86 | conf.set("fs.default.name", defaultName); 87 | conf.set("fs.hdfs.impl", hdfsImpl); 88 | List list = new ArrayList(); 89 | list.add(conf); 90 | options.setHdfsConfiguration(list); 91 | return Pipeline.create(options); 92 | } 93 | 94 | 95 | } 96 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/util/PipelineSingletonUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.util; 2 | 3 | import org.apache.beam.sdk.Pipeline; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import java.io.Serializable; 8 | 9 | /** 10 | * @Author: limeng 11 | * @Date: 2019/7/18 16:18 12 | */ 13 | public class PipelineSingletonUtil implements Serializable { 14 | 15 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 16 | private PipelineSingletonUtil() { 17 | } 18 | 19 | public static volatile Pipeline instance = null; 20 | 21 | public static Pipeline getInstance(Pipeline newPipeline){ 22 | if(instance == null){ 23 | synchronized (PipelineSingletonUtil.class){ 24 | if(instance == null && newPipeline != null){ 25 | instance = newPipeline; 26 | } 27 | } 28 | } 29 | return instance; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/util/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.util; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.IOException; 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | import java.util.Properties; 10 | 11 | /** 12 | * 获取参数 13 | * @Author: limeng 14 | * @Date: 2019/7/23 22:32 15 | */ 16 | public class PropertiesUtil { 17 | private static final String defaultPath="/database.properties"; 18 | private static Logger logger = LoggerFactory.getLogger(PropertiesUtil.class); 19 | 20 | private static Map propertiesMap=new HashMap<>(); 21 | 22 | public static Properties getProperties(){ 23 | return apply(defaultPath); 24 | } 25 | 26 | public static Properties getProperties(String path){ 27 | return apply(path); 28 | } 29 | 30 | private static Properties apply(String path){ 31 | Properties properties = propertiesMap.get(path); 32 | if(properties == null){ 33 | properties = new Properties(); 34 | try { 35 | properties.load(PropertiesUtil.class.getResourceAsStream(path)); 36 | propertiesMap.put(path,properties); 37 | } catch (IOException e) { 38 | logger.error("PropertiesUtil apply error=>msg:{}",e.getMessage()); 39 | } 40 | } 41 | return properties; 42 | } 43 | 44 | 45 | } 46 | -------------------------------------------------------------------------------- /client/kd-config/src/main/java/com/knowlegene/parent/config/util/ProxyFactoryUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config.util; 2 | 3 | import net.sf.cglib.proxy.Enhancer; 4 | import net.sf.cglib.proxy.MethodInterceptor; 5 | import net.sf.cglib.proxy.MethodProxy; 6 | import java.lang.reflect.Method; 7 | 8 | /** 9 | * 代理 10 | * @Author: limeng 11 | * @Date: 2019/7/31 18:35 12 | */ 13 | public class ProxyFactoryUtil implements MethodInterceptor { 14 | //维护目标对象 15 | private Object target; 16 | 17 | public ProxyFactoryUtil(Object target) { 18 | this.target = target; 19 | } 20 | 21 | /** 22 | * 给目标对象创建一个代理对象 23 | * @return 24 | */ 25 | public Object getProxyInstance(){ 26 | //1.工具类 27 | Enhancer en = new Enhancer(); 28 | //2.设置父类 29 | en.setSuperclass(target.getClass()); 30 | //3.设置回调函数 31 | en.setCallback(this); 32 | //4.创建子类(代理对象) 33 | return en.create(); 34 | 35 | } 36 | 37 | @Override 38 | public Object intercept(Object o, Method method, Object[] objects, MethodProxy methodProxy) throws Throwable { 39 | //执行目标对象的方法 40 | Object returnValue = method.invoke(target, objects); 41 | 42 | return returnValue; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /client/kd-config/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | logs/kd 8 | kd 9 | 10 | 11 | 12 | 13 | 14 | 17 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /client/kd-config/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | UTF-8 8 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /client/kd-config/src/test/java/com/knowlegene/parent/config/HdfsFileTest.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.config; 2 | 3 | import com.knowlegene.parent.config.util.HdfsFileUtil; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | import java.io.IOException; 8 | import java.util.List; 9 | 10 | /** 11 | * 测试hdfs 12 | * @Author: limeng 13 | * @Date: 2019/7/26 17:12 14 | */ 15 | public class HdfsFileTest { 16 | @Test 17 | public void getFile(){ 18 | String filePath="D:\\sqltext.txt"; 19 | String hdfsFilePath="hdfs://192.168.20.117:8020/testsql.txt"; 20 | 21 | HdfsFileUtil hdfsFileUtil = new HdfsFileUtil(); 22 | try { 23 | //本地文件 24 | List content = hdfsFileUtil.getContent(filePath); 25 | //hdfs测试 26 | List content1 = hdfsFileUtil.getContent(hdfsFilePath); 27 | Assert.assertNotNull(content); 28 | Assert.assertNotNull(content1); 29 | } catch (IOException e) { 30 | e.printStackTrace(); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/SwapDirectApplication.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process; 2 | 3 | import com.knowlegene.parent.process.runners.SwapRunners; 4 | import com.knowlegene.parent.process.runners.options.SwapJavaPipelineOptions; 5 | 6 | 7 | /** 8 | * java 运行 9 | * @Classname SwapDirectApplication 10 | * @Description TODO 11 | * @Date 2020/7/10 14:52 12 | * @Created by limeng 13 | */ 14 | public class SwapDirectApplication extends SwapRunners { 15 | public static void main(String[] args) { 16 | SwapDirectApplication application=new SwapDirectApplication(); 17 | application.start(SwapJavaPipelineOptions.class,args); 18 | } 19 | 20 | @Override 21 | public void setJobStream() { 22 | 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/SwapFlinkApplication.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process; 2 | 3 | import com.knowlegene.parent.process.runners.SwapRunners; 4 | import com.knowlegene.parent.process.runners.options.SwapFlinkPipelineOptions; 5 | 6 | /** 7 | * flink 8 | * @Classname SwapFlinkApplication 9 | * @Description TODO 10 | * @Date 2020/7/9 17:24 11 | * @Created by limeng 12 | */ 13 | public class SwapFlinkApplication extends SwapRunners { 14 | public static void main(String[] args) { 15 | SwapFlinkApplication application=new SwapFlinkApplication(); 16 | application.start(SwapFlinkPipelineOptions.class,args); 17 | } 18 | 19 | @Override 20 | public void setJobStream() { 21 | 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/SwapSparkApplication.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process; 2 | 3 | import com.knowlegene.parent.process.runners.SwapRunners; 4 | import com.knowlegene.parent.process.runners.options.SwapSparkPipelineOptions; 5 | 6 | 7 | /** 8 | * 9 | * spark 10 | * @Author: limeng 11 | * @Date: 2019/10/15 19:46 12 | */ 13 | public class SwapSparkApplication extends SwapRunners { 14 | public static void main(String[] args) { 15 | SwapSparkApplication application=new SwapSparkApplication(); 16 | application.start(SwapSparkPipelineOptions.class,args); 17 | 18 | } 19 | 20 | @Override 21 | public void setJobStream() { 22 | 23 | 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/common/annotation/StoredAsProperty.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.common.annotation; 2 | 3 | import java.lang.annotation.*; 4 | 5 | /** 6 | * @Author: limeng 7 | * @Date: 2019/8/20 14:03 8 | */ 9 | @Documented 10 | @Retention(RetentionPolicy.RUNTIME) 11 | @Target(ElementType.FIELD) 12 | public @interface StoredAsProperty { 13 | String value(); 14 | } 15 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/common/constantenum/Neo4jEnum.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.common.constantenum; 2 | 3 | /** 4 | * @Author: limeng 5 | * @Date: 2019/9/29 15:56 6 | */ 7 | public enum Neo4jEnum { 8 | 9 | RELATE(1,"relate"),SAVE(2,"save"),UPDATE(3,"update"),DELETE(4,"delete"), 10 | START_ID(5,"start_id"),END_ID(6,"end_id"),NODE_ID(7,"node_id"),NODE(8,"node"); 11 | /** 12 | * 类型标记 13 | */ 14 | private int value; 15 | 16 | /** 17 | * 类型名称 18 | */ 19 | private String name; 20 | 21 | public int getValue() { 22 | return value; 23 | } 24 | 25 | public void setValue(int value) { 26 | this.value = value; 27 | } 28 | 29 | public String getName() { 30 | return name; 31 | } 32 | 33 | public void setName(String name) { 34 | this.name = name; 35 | } 36 | 37 | Neo4jEnum() { 38 | } 39 | 40 | Neo4jEnum(int value, String name) { 41 | this.value = value; 42 | this.name = name; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/common/constantenum/OptionsEnum.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.common.constantenum; 2 | 3 | /** 4 | * @Author: limeng 5 | * @Date: 2019/8/12 14:46 6 | */ 7 | public interface OptionsEnum { 8 | public enum Indexer{ 9 | PATTERN("pattern"),SOURCENAME("sourceName"), 10 | ISSQLFILE("sqlFile"),FILEPATH("filePath"),ISORDERLINK("orderLink") 11 | ,JDBCSQL("jdbcSql"),ISTREESTATUS("treeStatus"); 12 | 13 | public final String name; 14 | 15 | Indexer(){ 16 | this(null); 17 | } 18 | 19 | Indexer(String name){ 20 | this.name = name; 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/es/ESSwap.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.es; 2 | 3 | import org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO; 4 | 5 | /** 6 | * es数据交换 7 | * @Author: limeng 8 | * @Date: 2019/9/9 18:40 9 | */ 10 | public interface ESSwap { 11 | 12 | ElasticsearchIO.Read getRead(String[] addresses, String index, String type); 13 | 14 | ElasticsearchIO.Write getWrite(String[] addresses, String index, String type); 15 | 16 | ElasticsearchIO.Write getWrite(String[] addresses, String index, String type, String idName); 17 | } 18 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/es/impl/ESSwapImpl.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.es.impl; 2 | 3 | import com.knowlegene.parent.config.util.BaseUtil; 4 | import com.knowlegene.parent.process.io.es.ESSwap; 5 | import com.knowlegene.parent.process.transform.ESTransform; 6 | import org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * @Author: limeng 12 | * @Date: 2019/9/9 20:00 13 | */ 14 | public class ESSwapImpl implements ESSwap { 15 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 16 | 17 | private boolean isEmpty(String[] addresses, String index, String type){ 18 | boolean result=false; 19 | if(addresses == null || addresses.length == 0){ 20 | logger.error("es addresses is null"); 21 | result = true; 22 | } 23 | if(BaseUtil.isBlank(index)){ 24 | logger.error("es index is null"); 25 | result = true; 26 | } 27 | if(BaseUtil.isBlank(type)){ 28 | logger.error("es type is null"); 29 | result = true; 30 | } 31 | return result; 32 | } 33 | 34 | /** 35 | * 读 36 | * get 37 | * @param addresses 地址 38 | * @param index 39 | * @param type 40 | * @return 41 | */ 42 | @Override 43 | public ElasticsearchIO.Read getRead(String[] addresses, String index, String type) { 44 | boolean empty = isEmpty(addresses, index, type); 45 | if(empty) return null; 46 | ElasticsearchIO.ConnectionConfiguration connectionConfiguration = ElasticsearchIO.ConnectionConfiguration.create(addresses, index, type); 47 | return ElasticsearchIO.read().withConnectionConfiguration(connectionConfiguration).withBatchSize(10000L); 48 | } 49 | 50 | /** 51 | * 写 52 | * post 53 | * _bulk 54 | * @param addresses 55 | * @param index 56 | * @param type 57 | * @return 58 | */ 59 | @Override 60 | public ElasticsearchIO.Write getWrite(String[] addresses, String index, String type) { 61 | boolean empty = isEmpty(addresses, index, type); 62 | if(empty) return null; 63 | ElasticsearchIO.ConnectionConfiguration connectionConfiguration = ElasticsearchIO.ConnectionConfiguration.create(addresses, index, type); 64 | return ElasticsearchIO.write().withConnectionConfiguration(connectionConfiguration); 65 | } 66 | 67 | @Override 68 | public ElasticsearchIO.Write getWrite(String[] addresses, String index, String type, String idName) { 69 | if(BaseUtil.isBlank(idName)){ 70 | return null; 71 | } 72 | return getWrite(addresses,index,type).withIdFn(new ESTransform.ExtractValueFn(idName)); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/file/HadoopFile.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.file; 2 | 3 | import org.apache.beam.sdk.values.PCollection; 4 | 5 | import java.io.InputStream; 6 | import java.util.List; 7 | 8 | /** 9 | * @Author: limeng 10 | * @Date: 2019/8/12 10:07 11 | */ 12 | public interface HadoopFile { 13 | /** 14 | * 读取文件 15 | * @param path 16 | * @return 17 | */ 18 | List readFile(String path); 19 | 20 | /** 21 | * 读取文件 22 | * @param path 23 | * @return 24 | */ 25 | List readFileStream(InputStream path); 26 | 27 | /** 28 | * textio 29 | */ 30 | PCollection readFileByIO(String path); 31 | 32 | 33 | } 34 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/file/HadoopFileImpl.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.file; 2 | 3 | import com.knowlegene.parent.config.util.HdfsFileUtil; 4 | import com.knowlegene.parent.config.util.HdfsPipelineUtil; 5 | import org.apache.beam.sdk.values.PCollection; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.io.IOException; 10 | import java.io.InputStream; 11 | import java.util.List; 12 | 13 | /** 14 | * @Author: limeng 15 | * @Date: 2019/8/12 10:07 16 | */ 17 | public class HadoopFileImpl implements HadoopFile { 18 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 19 | @Override 20 | public List readFile(String path) { 21 | HdfsFileUtil hdfsFileUtil = new HdfsFileUtil(); 22 | try { 23 | List content = hdfsFileUtil.getContent(path); 24 | List conversions = hdfsFileUtil.conversionSet(content); 25 | return conversions; 26 | } catch (IOException e) { 27 | logger.error("readFiles=>msg:{}",e.getMessage()); 28 | } 29 | return null; 30 | } 31 | 32 | @Override 33 | public List readFileStream(InputStream path) { 34 | HdfsFileUtil hdfsFileUtil = new HdfsFileUtil(); 35 | try { 36 | List content = hdfsFileUtil.getContentStream(path); 37 | List conversions = hdfsFileUtil.conversionSet(content); 38 | return conversions; 39 | } catch (IOException e) { 40 | logger.error("readFileStream=>msg:{}",e.getMessage()); 41 | } 42 | return null; 43 | } 44 | 45 | @Override 46 | public PCollection readFileByIO(String path) { 47 | return HdfsPipelineUtil.startDocumentImport(path); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/GbaseSwap.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc; 2 | 3 | import com.knowlegene.parent.process.pojo.ObjectCoder; 4 | import org.apache.beam.sdk.io.jdbc.JdbcIO; 5 | import org.apache.beam.sdk.schemas.Schema; 6 | import org.apache.beam.sdk.values.Row; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * @Author: limeng 12 | * @Date: 2019/10/12 14:06 13 | */ 14 | public interface GbaseSwap { 15 | JdbcIO.Read> queryByTable(String tableName); 16 | 17 | Schema desc(String tableName); 18 | 19 | JdbcIO.Write> saveByIO(String sql); 20 | 21 | JdbcIO.Read> query(String sql); 22 | } 23 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/HiveSwap.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc; 2 | 3 | import com.knowlegene.parent.process.pojo.ObjectCoder; 4 | import org.apache.beam.sdk.io.hcatalog.HCatalogIO; 5 | import org.apache.beam.sdk.io.jdbc.JdbcIO; 6 | import org.apache.beam.sdk.schemas.Schema; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * @Author: limeng 12 | * @Date: 2019/8/22 14:43 13 | */ 14 | public interface HiveSwap { 15 | /** 16 | * 详情 17 | * @param tableName 表名 18 | * @return 结果 19 | */ 20 | Schema descByTableName(String tableName, boolean isTimeStr); 21 | 22 | JdbcIO.Write> saveByIO(String sql); 23 | 24 | int saveCommon(String sql); 25 | 26 | 27 | HCatalogIO.Write saveByHCatalogIO(Map ops, Map partition); 28 | 29 | HCatalogIO.Read queryByHCatalogIO(Map ops); 30 | 31 | HCatalogIO.Read queryByHCatalogIO(Map ops, String filter); 32 | 33 | 34 | JdbcIO.Read> queryByTable(String tableName); 35 | 36 | JdbcIO.Read> query(String sql); 37 | 38 | } 39 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/MySQLSwap.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc; 2 | 3 | import com.knowlegene.parent.process.pojo.ObjectCoder; 4 | import org.apache.beam.sdk.io.jdbc.JdbcIO; 5 | import org.apache.beam.sdk.schemas.Schema; 6 | import org.apache.beam.sdk.values.Row; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * @Author: limeng 12 | * @Date: 2019/8/22 11:53 13 | */ 14 | public interface MySQLSwap { 15 | 16 | JdbcIO.Read> queryByTable(String tableName); 17 | 18 | JdbcIO.Write> saveByIO(String sql); 19 | 20 | 21 | Schema desc(String tableName); 22 | JdbcIO.Read> query(String sql); 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/OracleSwap.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc; 2 | 3 | import com.knowlegene.parent.process.pojo.ObjectCoder; 4 | import org.apache.beam.sdk.io.jdbc.JdbcIO; 5 | import org.apache.beam.sdk.schemas.Schema; 6 | import org.apache.beam.sdk.values.Row; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * @Author: limeng 12 | * @Date: 2019/8/19 20:16 13 | */ 14 | public interface OracleSwap { 15 | 16 | 17 | JdbcIO.Read> queryByTable(String tableName); 18 | JdbcIO.Write> saveByIO(String sql); 19 | Schema desc(String tableName); 20 | JdbcIO.Read> query(String sql); 21 | } 22 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/GbaseSwapExport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.scheduler.utils.CacheManager; 5 | 6 | import javax.sql.DataSource; 7 | 8 | /** 9 | * @Classname GbaseSwapExport 10 | * @Description TODO 11 | * @Date 2020/6/23 10:40 12 | * @Created by limeng 13 | */ 14 | public class GbaseSwapExport extends GbaseSwapImpl { 15 | @Override 16 | public DataSource getDataSource() { 17 | String name = DBOperationEnum.GBASE_EXPORT_DB.getName(); 18 | if(CacheManager.isExist(name)){ 19 | return (DataSource)CacheManager.getCache(name); 20 | } 21 | return null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/GbaseSwapImpl.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.util.BaseUtil; 4 | import com.knowlegene.parent.process.io.jdbc.AbstractSwapBase; 5 | import com.knowlegene.parent.process.io.jdbc.GbaseSwap; 6 | import com.knowlegene.parent.process.pojo.ObjectCoder; 7 | import org.apache.beam.sdk.io.jdbc.JdbcIO; 8 | import org.apache.beam.sdk.schemas.Schema; 9 | import org.apache.beam.sdk.values.Row; 10 | 11 | 12 | import javax.sql.DataSource; 13 | import java.util.Map; 14 | 15 | /** 16 | * @Author: limeng 17 | * @Date: 2019/10/12 14:16 18 | */ 19 | public abstract class GbaseSwapImpl extends AbstractSwapBase implements GbaseSwap { 20 | 21 | @Override 22 | public JdbcIO.Read> queryByTable(String tableName) { 23 | String sql = "select * from "+tableName; 24 | return this.query(sql); 25 | } 26 | 27 | @Override 28 | public Schema desc(String tableName) { 29 | try { 30 | return this.getSchema(tableName); 31 | }catch (Exception e){ 32 | getLogger().error("desc=>tableName:{},msg:{}",tableName,e.getMessage()); 33 | } 34 | return null; 35 | } 36 | 37 | @Override 38 | public JdbcIO.Read> query(String sql) { 39 | try { 40 | return this.select(sql); 41 | } catch (Exception e) { 42 | getLogger().error("query=>sql:{},msg:{}",sql,e.getMessage()); 43 | } 44 | return null; 45 | } 46 | 47 | 48 | @Override 49 | public JdbcIO.Write> saveByIO(String sql) { 50 | if(BaseUtil.isBlank(sql)){ 51 | getLogger().error("sql is null"); 52 | return null; 53 | } 54 | return this.batchSaveCommon(sql); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/GbaseSwapImport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.scheduler.utils.CacheManager; 5 | 6 | import javax.sql.DataSource; 7 | 8 | /** 9 | * @Classname GbaseSwapImport 10 | * @Description TODO 11 | * @Date 2020/6/23 10:39 12 | * @Created by limeng 13 | */ 14 | public class GbaseSwapImport extends GbaseSwapImpl { 15 | @Override 16 | public DataSource getDataSource() { 17 | String name = DBOperationEnum.GBASE_IMPORT_DB.getName(); 18 | if(CacheManager.isExist(name)){ 19 | return (DataSource)CacheManager.getCache(name); 20 | } 21 | return null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/HiveSwapExport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.scheduler.utils.CacheManager; 5 | 6 | import javax.sql.DataSource; 7 | 8 | /** 9 | * @Classname HiveSwapExport 10 | * @Description TODO 11 | * @Date 2020/6/11 15:23 12 | * @Created by limeng 13 | */ 14 | public class HiveSwapExport extends HiveSwapImpl { 15 | @Override 16 | public DataSource getDataSource() { 17 | String name = DBOperationEnum.HIVE_EXPORT_DB.getName(); 18 | if(CacheManager.isExist(name)){ 19 | return (DataSource)CacheManager.getCache(name); 20 | } 21 | return null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/HiveSwapImport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.scheduler.utils.CacheManager; 5 | 6 | import javax.sql.DataSource; 7 | 8 | /** 9 | * @Classname HiveSwapImport 10 | * @Description TODO 11 | * @Date 2020/6/11 15:21 12 | * @Created by limeng 13 | */ 14 | public class HiveSwapImport extends HiveSwapImpl{ 15 | 16 | @Override 17 | public DataSource getDataSource() { 18 | String name = DBOperationEnum.HIVE_IMPORT_DB.getName(); 19 | if(CacheManager.isExist(name)){ 20 | return (DataSource)CacheManager.getCache(name); 21 | } 22 | return null; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/MySQLSwapExport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.scheduler.utils.CacheManager; 5 | 6 | import javax.sql.DataSource; 7 | 8 | /** 9 | * @Classname MySQLSwapExport 10 | * @Description TODO 11 | * @Date 2020/6/19 20:37 12 | * @Created by limeng 13 | */ 14 | public class MySQLSwapExport extends MySQLSwapImpl { 15 | @Override 16 | public DataSource getDataSource() { 17 | String name = DBOperationEnum.MYSQL_EXPORT_DB.getName(); 18 | if(CacheManager.isExist(name)){ 19 | return (DataSource)CacheManager.getCache(name); 20 | } 21 | return null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/MySQLSwapImpl.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.util.BaseUtil; 4 | import com.knowlegene.parent.process.io.jdbc.AbstractSwapBase; 5 | import com.knowlegene.parent.process.io.jdbc.MySQLSwap; 6 | import com.knowlegene.parent.process.pojo.ObjectCoder; 7 | import org.apache.beam.sdk.io.jdbc.JdbcIO; 8 | import org.apache.beam.sdk.schemas.Schema; 9 | import org.apache.beam.sdk.values.Row; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import javax.sql.DataSource; 14 | import java.util.Map; 15 | 16 | /** 17 | * @Author: limeng 18 | * @Date: 2019/8/21 15:18 19 | */ 20 | public abstract class MySQLSwapImpl extends AbstractSwapBase implements MySQLSwap { 21 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 22 | 23 | /** 24 | * 查询 Row 25 | * @param tableName 26 | * @param 27 | * @return 28 | */ 29 | @Override 30 | public JdbcIO.Read> queryByTable(String tableName) { 31 | String sql = "select * from "+tableName; 32 | return this.query(sql); 33 | } 34 | 35 | /** 36 | * row sql 带批量的保存 37 | * @param sql 38 | * @return 39 | */ 40 | @Override 41 | public JdbcIO.Write> saveByIO(String sql) { 42 | if(BaseUtil.isBlank(sql)){ 43 | logger.error("sql is null"); 44 | return null; 45 | } 46 | return this.batchSaveCommon(sql); 47 | } 48 | 49 | 50 | /** 51 | * 查询 row 52 | * @param sql 53 | * @param 54 | * @return 55 | */ 56 | @Override 57 | public JdbcIO.Read> query(String sql) { 58 | try { 59 | return this.select(sql); 60 | } catch (Exception e) { 61 | logger.error("query=>sql:{},msg:{}",sql,e.getMessage()); 62 | } 63 | return null; 64 | } 65 | 66 | /** 67 | * 详情 68 | * @param tableName 表名 69 | * @return 70 | */ 71 | @Override 72 | public Schema desc(String tableName) { 73 | try { 74 | return this.getSchema(tableName,false); 75 | }catch (Exception e){ 76 | logger.error("desc=>tableName:{},msg:{}",tableName,e.getMessage()); 77 | } 78 | return null; 79 | } 80 | 81 | 82 | 83 | } 84 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/MySQLSwapImport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.scheduler.utils.CacheManager; 5 | 6 | import javax.sql.DataSource; 7 | 8 | /** 9 | * @Classname MySQLSwapImport 10 | * @Description TODO 11 | * @Date 2020/6/19 20:37 12 | * @Created by limeng 13 | */ 14 | public class MySQLSwapImport extends MySQLSwapImpl{ 15 | @Override 16 | public DataSource getDataSource() { 17 | String name = DBOperationEnum.MYSQL_IMPORT_DB.getName(); 18 | if(CacheManager.isExist(name)){ 19 | return (DataSource)CacheManager.getCache(name); 20 | } 21 | return null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/OracleSwapExport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.scheduler.utils.CacheManager; 5 | 6 | import javax.sql.DataSource; 7 | 8 | /** 9 | * @Classname OracleSwapExport 10 | * @Description TODO 11 | * @Date 2020/6/23 10:07 12 | * @Created by limeng 13 | */ 14 | public class OracleSwapExport extends OracleSwapImpl { 15 | @Override 16 | public DataSource getDataSource() { 17 | String name = DBOperationEnum.ORACLE_EXPORT_DB.getName(); 18 | if(CacheManager.isExist(name)){ 19 | return (DataSource)CacheManager.getCache(name); 20 | } 21 | return null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/OracleSwapImpl.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.util.BaseUtil; 4 | import com.knowlegene.parent.process.io.jdbc.AbstractSwapBase; 5 | import com.knowlegene.parent.process.io.jdbc.OracleSwap; 6 | import com.knowlegene.parent.process.pojo.ObjectCoder; 7 | import org.apache.beam.sdk.io.jdbc.JdbcIO; 8 | import org.apache.beam.sdk.schemas.Schema; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.util.Map; 13 | 14 | /** 15 | * @Author: limeng 16 | * @Date: 2019/8/26 19:09 17 | */ 18 | public abstract class OracleSwapImpl extends AbstractSwapBase implements OracleSwap { 19 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 20 | 21 | /** 22 | * 查询表 23 | * @param tableName 表名称 24 | * @param 25 | * @return 26 | */ 27 | @Override 28 | public JdbcIO.Read> queryByTable(String tableName) { 29 | String sql = "select * from "+tableName; 30 | return this.query(sql); 31 | } 32 | 33 | @Override 34 | public JdbcIO.Write> saveByIO(String sql) { 35 | if(BaseUtil.isBlank(sql)){ 36 | logger.error("sql is null"); 37 | return null; 38 | } 39 | return this.batchSaveCommon(sql); 40 | } 41 | 42 | /** 43 | * 查询 44 | * @param sql 45 | * @param 46 | * @return 47 | */ 48 | @Override 49 | public JdbcIO.Read> query(String sql) { 50 | try { 51 | return this.select(sql); 52 | } catch (Exception e) { 53 | logger.error("query=>sql:{},msg:{}",sql,e.getMessage()); 54 | } 55 | return null; 56 | } 57 | 58 | /** 59 | * 详情 60 | * @param tableName 表名 61 | * @return 62 | */ 63 | @Override 64 | public Schema desc(String tableName) { 65 | try { 66 | return this.getOracleSchema(tableName); 67 | }catch (Exception e){ 68 | logger.error("desc=>tableName:{},msg:{}",tableName,e.getMessage()); 69 | } 70 | return null; 71 | } 72 | 73 | 74 | 75 | 76 | 77 | } 78 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/jdbc/impl/OracleSwapImport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.jdbc.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.scheduler.utils.CacheManager; 5 | 6 | import javax.sql.DataSource; 7 | 8 | /** 9 | * @Classname OracleSwapImport 10 | * @Description TODO 11 | * @Date 2020/6/23 10:07 12 | * @Created by limeng 13 | */ 14 | public class OracleSwapImport extends OracleSwapImpl { 15 | @Override 16 | public DataSource getDataSource() { 17 | String name = DBOperationEnum.ORACLE_IMPORT_DB.getName(); 18 | if(CacheManager.isExist(name)){ 19 | return (DataSource)CacheManager.getCache(name); 20 | } 21 | return null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/logs/LogPatternConverter.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.logs; 2 | 3 | import com.knowlegene.parent.config.util.BaseUtil; 4 | import com.knowlegene.parent.process.util.JobThreadLocalUtil; 5 | import org.apache.logging.log4j.core.LogEvent; 6 | import org.apache.logging.log4j.core.config.plugins.Plugin; 7 | import org.apache.logging.log4j.core.pattern.ConverterKeys; 8 | import org.apache.logging.log4j.core.pattern.LogEventPatternConverter; 9 | import org.apache.logging.log4j.core.pattern.PatternConverter; 10 | import org.apache.logging.log4j.message.Message; 11 | 12 | /** 13 | * 日志标记 14 | * @Author: limeng 15 | * @Date: 2019/8/13 17:55 16 | */ 17 | @Plugin(name = "LogPatternConverter", category = PatternConverter.CATEGORY) 18 | @ConverterKeys({ "H" }) 19 | public class LogPatternConverter extends LogEventPatternConverter { 20 | 21 | private static final LogPatternConverter INSTANCE = new LogPatternConverter(); 22 | 23 | public static LogPatternConverter newInstance(final String[] options) { 24 | return INSTANCE; 25 | } 26 | 27 | private LogPatternConverter(){ 28 | super("LogId","logId"); 29 | } 30 | 31 | @Override 32 | public void format(LogEvent logEvent, StringBuilder stringBuilder) { 33 | final Message msg = logEvent.getMessage(); 34 | String result = null; 35 | if (msg != null) { 36 | //String result = msg.getFormattedMessage(); 37 | result = getPayplusLogUUID() ; 38 | stringBuilder.append(result); 39 | } 40 | } 41 | 42 | /** 43 | * 业务日志全局jobid 44 | * @return 45 | */ 46 | protected String getPayplusLogUUID() { 47 | StringBuilder builder = new StringBuilder(); 48 | String job = JobThreadLocalUtil.getJob(); 49 | if(BaseUtil.isNotBlank(job)){ 50 | builder.append("JobId("); 51 | builder.append(job); 52 | builder.append(")"); 53 | } 54 | return builder.toString(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/neo4j/Neo4jSwap.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.neo4j; 2 | 3 | import com.knowlegene.parent.process.pojo.ObjectCoder; 4 | import com.knowlegene.parent.process.pojo.neo4j.Neo4jObject; 5 | 6 | import java.util.Map; 7 | import java.util.Set; 8 | 9 | /** 10 | * @Author: limeng 11 | * @Date: 2019/9/23 17:48 12 | */ 13 | public interface Neo4jSwap { 14 | /** 15 | * 16 | * @param statement 17 | * @return 18 | */ 19 | Neo4jIO.Write write(String statement); 20 | 21 | Neo4jIO.Write relate(String statement, String label); 22 | 23 | Neo4jIO.Read query(String sql); 24 | 25 | 26 | Neo4jIO.Read>> query(String sql, String type); 27 | } 28 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/neo4j/impl/Neo4jSwapExport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.neo4j.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.process.pojo.neo4j.Neo4jOptions; 5 | 6 | /** 7 | * @Classname Neo4jSwapExport 8 | * @Description TODO 9 | * @Date 2020/7/1 18:23 10 | * @Created by limeng 11 | */ 12 | public class Neo4jSwapExport extends Neo4jSwapImpl { 13 | @Override 14 | public Neo4jOptions getDriver() { 15 | String name = DBOperationEnum.NEO4J_EXPORT.getName(); 16 | Object options = getOptions(name); 17 | if(options != null){ 18 | return (Neo4jOptions)options; 19 | } 20 | return null; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/io/neo4j/impl/Neo4jSwapImport.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.io.neo4j.impl; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.process.pojo.neo4j.Neo4jOptions; 5 | 6 | /** 7 | * @Classname Neo4jSwapImport 8 | * @Description TODO 9 | * @Date 2020/7/1 18:23 10 | * @Created by limeng 11 | */ 12 | public class Neo4jSwapImport extends Neo4jSwapImpl { 13 | @Override 14 | public Neo4jOptions getDriver() { 15 | String name = DBOperationEnum.NEO4J_IMPORT.getName(); 16 | Object options = getOptions(name); 17 | if(options != null){ 18 | return (Neo4jOptions)options; 19 | } 20 | return null; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/DataSources.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo; 2 | 3 | import lombok.Builder; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | import java.io.Serializable; 9 | 10 | /** 11 | * @Classname DataSources 12 | * @Description TODO 13 | * @Date 2020/6/10 14:45 14 | * @Created by limeng 15 | */ 16 | @Data 17 | @AllArgsConstructor 18 | @NoArgsConstructor 19 | @Builder 20 | public class DataSources implements Serializable { 21 | 22 | String className; 23 | String user; 24 | String password; 25 | String url; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/DefaultHCatRecord.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo; 2 | 3 | import org.apache.hive.hcatalog.common.HCatException; 4 | import org.apache.hive.hcatalog.data.HCatRecord; 5 | import org.apache.hive.hcatalog.data.ReaderWriter; 6 | import org.apache.hive.hcatalog.data.schema.HCatSchema; 7 | 8 | import java.io.DataInput; 9 | import java.io.DataOutput; 10 | import java.io.IOException; 11 | import java.io.Serializable; 12 | import java.util.ArrayList; 13 | import java.util.Iterator; 14 | import java.util.List; 15 | 16 | /** 17 | * @Author: limeng 18 | * @Date: 2019/8/30 17:56 19 | */ 20 | public class DefaultHCatRecord extends HCatRecord implements Serializable { 21 | private List contents; 22 | 23 | public DefaultHCatRecord() { 24 | this.contents = new ArrayList(); 25 | } 26 | 27 | public DefaultHCatRecord(int size) { 28 | this.contents = new ArrayList(size); 29 | 30 | for(int i = 0; i < size; ++i) { 31 | this.contents.add((Object)null); 32 | } 33 | 34 | } 35 | 36 | @Override 37 | public void remove(int idx) throws HCatException { 38 | this.contents.remove(idx); 39 | } 40 | 41 | public DefaultHCatRecord(List list) { 42 | this.contents = list; 43 | } 44 | 45 | @Override 46 | public Object get(int fieldNum) { 47 | return this.contents.get(fieldNum); 48 | } 49 | 50 | @Override 51 | public List getAll() { 52 | return this.contents; 53 | } 54 | 55 | @Override 56 | public void set(int fieldNum, Object val) { 57 | this.contents.set(fieldNum, val); 58 | } 59 | 60 | @Override 61 | public int size() { 62 | return this.contents.size(); 63 | } 64 | 65 | @Override 66 | public void readFields(DataInput in) throws IOException { 67 | this.contents.clear(); 68 | int len = in.readInt(); 69 | 70 | for(int i = 0; i < len; ++i) { 71 | this.contents.add(ReaderWriter.readDatum(in)); 72 | } 73 | 74 | } 75 | 76 | @Override 77 | public void write(DataOutput out) throws IOException { 78 | int sz = this.size(); 79 | out.writeInt(sz); 80 | 81 | for(int i = 0; i < sz; ++i) { 82 | ReaderWriter.writeDatum(out, this.contents.get(i)); 83 | } 84 | 85 | } 86 | 87 | @Override 88 | public int hashCode() { 89 | int hash = 1; 90 | Iterator i$ = this.contents.iterator(); 91 | 92 | while(i$.hasNext()) { 93 | Object o = i$.next(); 94 | if (o != null) { 95 | hash = 31 * hash + o.hashCode(); 96 | } 97 | } 98 | 99 | return hash; 100 | } 101 | 102 | @Override 103 | public String toString() { 104 | StringBuilder sb = new StringBuilder(); 105 | Iterator i$ = this.contents.iterator(); 106 | 107 | while(i$.hasNext()) { 108 | Object o = i$.next(); 109 | sb.append(o + "\t"); 110 | } 111 | 112 | return sb.toString(); 113 | } 114 | 115 | @Override 116 | public Object get(String fieldName, HCatSchema recordSchema) throws HCatException { 117 | return this.get(recordSchema.getPosition(fieldName)); 118 | } 119 | 120 | @Override 121 | public void set(String fieldName, HCatSchema recordSchema, Object value) throws HCatException { 122 | this.set(recordSchema.getPosition(fieldName), value); 123 | } 124 | 125 | @Override 126 | public void copy(HCatRecord r) throws HCatException { 127 | this.contents = r.getAll(); 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/JobInfo.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo; 2 | 3 | /** 4 | * @Author: limeng 5 | * @Date: 2019/8/13 16:38 6 | */ 7 | public class JobInfo { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/NestingFields.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo; 2 | 3 | import com.knowlegene.parent.config.util.BaseUtil; 4 | import lombok.Data; 5 | import org.apache.beam.sdk.schemas.Schema; 6 | import org.apache.beam.sdk.values.KV; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.*; 11 | 12 | /** 13 | * 嵌套对象 14 | * @Author: limeng 15 | * @Date: 2019/9/16 20:01 16 | */ 17 | @Data 18 | public class NestingFields { 19 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 20 | //唯一key名称 21 | private String[] keys; 22 | //合并成数组的列名称 23 | private Map nestings; 24 | 25 | private String nestingKeysName; 26 | 27 | 28 | 29 | 30 | 31 | /** 32 | * 合并字段转换kv 33 | * @return 34 | */ 35 | public KV> mapToKV2(){ 36 | Iterator iterator = nestings.keySet().iterator(); 37 | while (iterator.hasNext()){ 38 | String next = iterator.next(); 39 | if(BaseUtil.isNotBlank(next)){ 40 | String[] values = nestings.get(next); 41 | return KV.of(next.toLowerCase(),Arrays.asList(values)); 42 | } 43 | 44 | } 45 | return null; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/ObjectCoder.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo; 2 | 3 | import lombok.Data; 4 | import org.apache.beam.sdk.schemas.Schema; 5 | 6 | import java.io.Serializable; 7 | import java.util.Objects; 8 | 9 | /** 10 | * @Author: limeng 11 | * @Date: 2019/9/16 17:23 12 | */ 13 | @Data 14 | public class ObjectCoder implements Serializable { 15 | private Object value; 16 | private Schema.FieldType fieldType; 17 | private Integer index; 18 | 19 | public ObjectCoder(Object value, Schema.FieldType fieldType, Integer index) { 20 | this.value = value; 21 | this.fieldType = fieldType; 22 | this.index = index; 23 | } 24 | 25 | public ObjectCoder(Object value, Schema.FieldType fieldType) { 26 | this.value = value; 27 | this.fieldType = fieldType; 28 | } 29 | 30 | public ObjectCoder(Object value, Integer index) { 31 | this.value = value; 32 | this.index = index; 33 | } 34 | 35 | public ObjectCoder(Object value) { 36 | this.value = value; 37 | } 38 | 39 | public ObjectCoder() { 40 | } 41 | 42 | @Override 43 | public boolean equals(Object o) { 44 | if (this == o) return true; 45 | if (o == null || getClass() != o.getClass()) return false; 46 | ObjectCoder that = (ObjectCoder) o; 47 | return Objects.equals(value, that.value) && 48 | Objects.equals(fieldType, that.fieldType) && 49 | Objects.equals(index, that.index); 50 | } 51 | 52 | @Override 53 | public int hashCode() { 54 | return Objects.hash(value, fieldType, index); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/SQLOptions.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo; 2 | 3 | import com.knowlegene.parent.config.pojo.BaseDTO; 4 | import lombok.Data; 5 | 6 | /** 7 | * @Author: limeng 8 | * @Date: 2019/8/1 14:28 9 | */ 10 | @Data 11 | public class SQLOptions extends BaseDTO { 12 | /** 13 | * 操作类型 14 | */ 15 | private String optionType; 16 | private String select; 17 | private String from; 18 | private String where; 19 | private String join; 20 | private String insert; 21 | /** 22 | * set 23 | * with 24 | * create 25 | * truncate 26 | */ 27 | private String ddl; 28 | } 29 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/db/DBOptions.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo.db; 2 | 3 | import com.knowlegene.parent.process.common.annotation.StoredAsProperty; 4 | import lombok.Data; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @Classname DBOptions 10 | * @Description TODO 11 | * @Date 2020/6/22 10:41 12 | * @Created by limeng 13 | */ 14 | @Data 15 | public class DBOptions implements Serializable { 16 | @StoredAsProperty("db.url") 17 | private String url; 18 | @StoredAsProperty("db.table") 19 | private String tableName; 20 | @StoredAsProperty("db.username") 21 | private String username; 22 | @StoredAsProperty("db.password") 23 | private String password; 24 | @StoredAsProperty("db.driver.class") 25 | private String driverClass; 26 | @StoredAsProperty("db.sql") 27 | private String dbSQL; 28 | @StoredAsProperty("db.column") 29 | private String[] dbColumn; 30 | } 31 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/es/ESOptions.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo.es; 2 | 3 | import com.knowlegene.parent.process.common.annotation.StoredAsProperty; 4 | import com.knowlegene.parent.process.pojo.NestingFields; 5 | import lombok.Data; 6 | 7 | import java.io.Serializable; 8 | 9 | /** 10 | * @Classname ESOptions 11 | * @Description TODO 12 | * @Date 2020/6/23 11:07 13 | * @Created by limeng 14 | */ 15 | @Data 16 | public class ESOptions implements Serializable { 17 | @StoredAsProperty("es.addrs") 18 | private String[] esAddrs; 19 | @StoredAsProperty("es.index") 20 | private String esIndex; 21 | @StoredAsProperty("es.type") 22 | private String esType; 23 | @StoredAsProperty("es.query") 24 | private String esQuery; 25 | @StoredAsProperty("es.idFn") 26 | private String esIdFn; 27 | 28 | private NestingFields nestingFields; 29 | } 30 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/file/FileOptions.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo.file; 2 | 3 | import com.knowlegene.parent.process.common.annotation.StoredAsProperty; 4 | import lombok.Data; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @Classname FileOptions 10 | * @Description TODO 11 | * @Date 2020/6/23 14:10 12 | * @Created by limeng 13 | */ 14 | @Data 15 | public class FileOptions implements Serializable { 16 | @StoredAsProperty("file.url") 17 | private String filePath; 18 | @StoredAsProperty("field.delim") 19 | private String fieldDelim; 20 | @StoredAsProperty("field.fieldTitle") 21 | private String[] fieldTitle; 22 | } 23 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/hive/HiveOptions.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo.hive; 2 | 3 | import com.knowlegene.parent.process.common.annotation.StoredAsProperty; 4 | import lombok.Data; 5 | 6 | import java.io.Serializable; 7 | import java.util.Arrays; 8 | import java.util.Objects; 9 | 10 | /** 11 | * @Classname HiveOptions 12 | * @Description TODO 13 | * @Date 2020/6/11 16:48 14 | * @Created by limeng 15 | */ 16 | @Data 17 | public class HiveOptions implements Serializable { 18 | @StoredAsProperty("hive.driver.class") 19 | private String hiveClass; 20 | 21 | private String hiveUrl; 22 | @StoredAsProperty("hive.username") 23 | private String hiveUsername; 24 | @StoredAsProperty("hive.password") 25 | private String hivePassword; 26 | @StoredAsProperty("hive.table") 27 | private String hiveTableName; 28 | @StoredAsProperty("hive.database") 29 | private String hiveDatabase; 30 | @StoredAsProperty("hive.table.empty") 31 | private Boolean hiveTableEmpty; 32 | 33 | @StoredAsProperty("hive.sql") 34 | private String hiveSQL; 35 | @StoredAsProperty("hive.column") 36 | private String[] hiveColumn; 37 | @StoredAsProperty("hcatalog.metastore") 38 | private String hMetastoreHost; 39 | @StoredAsProperty("hcatalog.metastore.port") 40 | private String hMetastorePort; 41 | @StoredAsProperty("hcatalog.metastore.filter") 42 | private String hiveFilter; 43 | @StoredAsProperty("hive.metastore.partition") 44 | private String hivePartition; 45 | 46 | 47 | 48 | @Override 49 | public boolean equals(Object o) { 50 | if (this == o) return true; 51 | if (o == null || getClass() != o.getClass()) return false; 52 | HiveOptions that = (HiveOptions) o; 53 | return Objects.equals(hiveClass, that.hiveClass) && 54 | Objects.equals(hiveUrl, that.hiveUrl) && 55 | Objects.equals(hiveUsername, that.hiveUsername) && 56 | Objects.equals(hivePassword, that.hivePassword) && 57 | Objects.equals(hiveTableName, that.hiveTableName) && 58 | Objects.equals(hiveDatabase, that.hiveDatabase) && 59 | Objects.equals(hiveTableEmpty, that.hiveTableEmpty) && 60 | Objects.equals(hiveSQL, that.hiveSQL) && 61 | Arrays.equals(hiveColumn, that.hiveColumn) && 62 | Objects.equals(hMetastoreHost, that.hMetastoreHost) && 63 | Objects.equals(hMetastorePort, that.hMetastorePort) && 64 | Objects.equals(hiveFilter, that.hiveFilter) && 65 | Objects.equals(hivePartition, that.hivePartition) ; 66 | } 67 | 68 | @Override 69 | public int hashCode() { 70 | int result = Objects.hash(hiveClass, hiveUrl, hiveUsername, hivePassword, hiveTableName, hiveDatabase, hiveTableEmpty, hiveSQL, hMetastoreHost, hMetastorePort, hiveFilter, hivePartition); 71 | result = 31 * result + Arrays.hashCode(hiveColumn); 72 | return result; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/neo4j/Neo4jCode.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo.neo4j; 2 | 3 | import com.knowlegene.parent.config.util.BaseUtil; 4 | import com.knowlegene.parent.process.common.constantenum.Neo4jEnum; 5 | import lombok.Data; 6 | 7 | import java.util.*; 8 | 9 | /** 10 | * neo4j操作 11 | * @Author: limeng 12 | * @Date: 2019/9/29 15:07 13 | */ 14 | @Data 15 | public class Neo4jCode { 16 | private String Rex="\\s|\t|\n"; 17 | private String dsl; 18 | private List keys; 19 | private Integer type; 20 | private String startId=":START_ID"; 21 | private String endId=":END_ID"; 22 | private String id="ID"; 23 | 24 | /** 25 | * 节点属性 26 | * @param format 27 | */ 28 | private void setAttribute(String format){ 29 | if(BaseUtil.isNotBlank(format)){ 30 | String[] split = format.split(Rex); 31 | if(split.length>0){ 32 | keys = Arrays.asList(split); 33 | } 34 | } 35 | } 36 | 37 | /** 38 | * 操作 39 | */ 40 | private void options(){ 41 | if(!BaseUtil.isBlankSet(keys)){ 42 | boolean present1 = keys.stream().anyMatch(a -> a.contains(startId) || a.contains(endId)); 43 | boolean present2 = keys.stream().anyMatch(a -> a.contains(id) ); 44 | if(present1){ 45 | type = Neo4jEnum.RELATE.getValue(); 46 | }else if(present2){ 47 | type = Neo4jEnum.SAVE.getValue(); 48 | } 49 | } 50 | } 51 | 52 | /** 53 | * 生成操作语句 54 | */ 55 | private void createDSL(){ 56 | if(!BaseUtil.isBlankSet(keys) && type!=null){ 57 | boolean first=false; 58 | StringBuffer sb=new StringBuffer(); 59 | String label=null; 60 | if(type == Neo4jEnum.RELATE.getValue()){ 61 | for(String k:keys){ 62 | if(first && !k.contains(endId) && !k.contains(startId)){ 63 | sb.append(","); 64 | } 65 | 66 | if(k.contains(startId)){ 67 | label = BaseUtil.getBrackets(k); 68 | }else if(!k.contains(endId)){ 69 | sb.append(k+":{"+k+"}"); 70 | first =true; 71 | } 72 | } 73 | if(BaseUtil.isNotBlank(label)){ 74 | String result="MATCH (a:"+label+"),(b:"+label+") WHERE a.id={start_id} AND b.id={end_id} " + 75 | " CREATE (a)-[r:%s {"+sb.toString()+"}] ->(b)"; 76 | dsl = result; 77 | } 78 | }else if(type == Neo4jEnum.SAVE.getValue()){ 79 | for(String k:keys){ 80 | if(first){ 81 | sb.append(","); 82 | } 83 | first =true; 84 | if(k.contains(id)){ 85 | label = BaseUtil.getBrackets(k); 86 | sb.append("id:{id}"); 87 | }else{ 88 | sb.append(k+":{"+k+"}"); 89 | } 90 | } 91 | if(BaseUtil.isNotBlank(label)){ 92 | String format="CREATE (a:%s {%s})"; 93 | dsl = String.format(format,label,sb.toString()); 94 | } 95 | 96 | } 97 | } 98 | } 99 | 100 | public void toDSL(String format){ 101 | if(BaseUtil.isNotBlank(format)){ 102 | setAttribute(format); 103 | options(); 104 | createDSL(); 105 | } 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/neo4j/Neo4jObject.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo.neo4j; 2 | 3 | import com.knowlegene.parent.config.util.BaseUtil; 4 | import lombok.Data; 5 | 6 | import java.io.Serializable; 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | /** 12 | * @Author: limeng 13 | * @Date: 2019/9/23 17:12 14 | */ 15 | @Data 16 | public class Neo4jObject implements Serializable { 17 | private Map parMap; 18 | private Object[] Objects; 19 | 20 | public Object[] getObjectValue(){ 21 | Object[] result=null; 22 | if(!BaseUtil.isBlankMap(parMap)){ 23 | List list=new ArrayList<>(); 24 | parMap.forEach((k,v)->{ 25 | list.add(k); 26 | list.add(v); 27 | }); 28 | result = list.toArray(); 29 | }else if(Objects != null || Objects.length>0){ 30 | result = Objects; 31 | } 32 | return result; 33 | } 34 | 35 | public Neo4jObject() { 36 | } 37 | 38 | public Neo4jObject(Object[] objects) { 39 | Objects = objects; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/pojo/neo4j/Neo4jOptions.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.pojo.neo4j; 2 | 3 | import com.knowlegene.parent.process.common.annotation.StoredAsProperty; 4 | import lombok.Data; 5 | 6 | /** 7 | * @Author: limeng 8 | * @Date: 2019/9/23 19:49 9 | */ 10 | @Data 11 | public class Neo4jOptions { 12 | @StoredAsProperty("neo4j.cypher") 13 | private String cypher; 14 | @StoredAsProperty("neo4j.neourl") 15 | private String neoUrl; 16 | @StoredAsProperty("neo4j.neousername") 17 | private String neoUsername; 18 | @StoredAsProperty("neo4j.neopassword") 19 | private String neoPassword; 20 | @StoredAsProperty("neo4j.neoformat") 21 | private String neoFormat; 22 | @StoredAsProperty("neo4j.label") 23 | private String neoLabel; 24 | @StoredAsProperty("neo4j.type") 25 | private String neoType; 26 | 27 | 28 | 29 | } 30 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/runners/SwapRunners.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.runners; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DatabaseTypeEnum; 4 | import com.knowlegene.parent.config.util.BaseUtil; 5 | import com.knowlegene.parent.process.pojo.SwapOptions; 6 | import com.knowlegene.parent.process.runners.options.SwapSparkPipelineOptions; 7 | import com.knowlegene.parent.process.runners.common.BaseJobRunners; 8 | import com.knowlegene.parent.process.tool.BaseSwapTool; 9 | import lombok.Data; 10 | import org.apache.beam.sdk.options.PipelineOptions; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | /** 15 | * 数据交换处理流程 16 | * @Author: limeng 17 | * @Date: 2019/8/21 13:55 18 | */ 19 | @Data 20 | public abstract class SwapRunners extends BaseJobRunners { 21 | private Logger logger = LoggerFactory.getLogger(getClass()); 22 | private SwapOptions swapOptions; 23 | 24 | private Object swapObject =new Object(); 25 | 26 | public void start(){ 27 | this.startRunners(); 28 | this.init(); 29 | this.setJobStream(); 30 | this.join(); 31 | 32 | } 33 | 34 | public void start(Class clazz,String[] args){ 35 | this.startRunnersByArgs(clazz,args); 36 | argsAndOptions(this.getPipelineOptions().as(clazz)); 37 | this.init(); 38 | this.setJobStream(); 39 | this.join(); 40 | } 41 | 42 | public void start(String[] args){ 43 | this.startRunnersByArgs(SwapSparkPipelineOptions.class,args); 44 | argsAndOptions(this.getPipelineOptions().as(SwapSparkPipelineOptions.class)); 45 | this.init(); 46 | this.setJobStream(); 47 | this.join(); 48 | } 49 | 50 | /** 51 | * 参数 52 | */ 53 | private void argsAndOptions(Object pipelineOptions){ 54 | if(swapOptions == null && pipelineOptions!=null){ 55 | SwapOptions newSwap = new SwapOptions(); 56 | BaseUtil.copyNonNullProperties(newSwap,pipelineOptions); 57 | this.swapOptions = newSwap; 58 | } 59 | } 60 | 61 | @Override 62 | protected void init(){ 63 | if(swapOptions != null){ 64 | String fromName = swapOptions.getFromName(); 65 | String toName = swapOptions.getToName(); 66 | Integer fromVal = DatabaseTypeEnum.queryValue(fromName).getValue(); 67 | Integer toVal = DatabaseTypeEnum.queryValue(toName).getValue(); 68 | if(fromVal==null || toVal==null){ 69 | logger.error("fromName or toName is null"); 70 | } 71 | try { 72 | new BaseSwapTool(swapOptions).run(); 73 | } catch (Exception e) { 74 | e.printStackTrace(); 75 | } 76 | } 77 | } 78 | 79 | private void join(){ 80 | while (getSwapMaster().isAlive()){} 81 | } 82 | /** 83 | * 执行计划 84 | */ 85 | public abstract void setJobStream(); 86 | 87 | public SwapRunners(){ 88 | 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/runners/common/BaseJobRunners.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.runners.common; 2 | 3 | 4 | import com.knowlegene.parent.process.swap.dispatcher.SwapMaster; 5 | import com.knowlegene.parent.process.util.JobThreadLocalUtil; 6 | import com.knowlegene.parent.process.util.SwapMasterUtil; 7 | import com.knowlegene.parent.process.util.UUIDFactoryUtil; 8 | import org.apache.beam.sdk.options.PipelineOptions; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.Serializable; 13 | import java.util.Map; 14 | 15 | /** 16 | * @Author: limeng 17 | * @Date: 2019/8/23 15:35 18 | */ 19 | public abstract class BaseJobRunners implements Serializable { 20 | private final Logger logger; 21 | private BaseRunners baseRunners; 22 | public Logger getLogger() { 23 | return logger; 24 | } 25 | 26 | public BaseJobRunners() { 27 | this.logger = LoggerFactory.getLogger(getClass()); 28 | String uuidStr = UUIDFactoryUtil.getUUIDStr(); 29 | JobThreadLocalUtil.setJob(uuidStr); 30 | } 31 | 32 | /** 33 | * pipline开始 34 | */ 35 | public void startRunners(){ 36 | baseRunners = new BaseRunners(); 37 | baseRunners.start(); 38 | } 39 | 40 | /** 41 | * pipline开始 42 | */ 43 | public void startRunners(Class clazz, Map map){ 44 | baseRunners = new BaseRunners(clazz); 45 | baseRunners.start(map); 46 | } 47 | 48 | /** 49 | * pipline开始 50 | */ 51 | public void startRunnersByArgs(Class clazz, String[] args){ 52 | baseRunners = new BaseRunners(clazz); 53 | baseRunners.startByArgs(args); 54 | } 55 | 56 | public PipelineOptions getPipelineOptions(){ 57 | if(baseRunners != null){ 58 | return baseRunners.getPipelineOptions(); 59 | } 60 | return null; 61 | } 62 | 63 | /** 64 | * 执行dag 65 | */ 66 | public void run(){ 67 | baseRunners.run(); 68 | } 69 | 70 | 71 | protected static SwapMaster getSwapMaster(){ 72 | return SwapMasterUtil.instance; 73 | } 74 | 75 | protected abstract void init(); 76 | } 77 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/runners/common/BaseRunners.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.runners.common; 2 | 3 | import com.knowlegene.parent.process.runners.options.IndexerPipelineOptions; 4 | import com.knowlegene.parent.config.util.BaseUtil; 5 | import com.knowlegene.parent.config.util.PipelineSingletonUtil; 6 | import com.knowlegene.parent.process.common.constantenum.OptionsEnum; 7 | import lombok.Data; 8 | import org.apache.beam.sdk.Pipeline; 9 | import org.apache.beam.sdk.options.PipelineOptions; 10 | import org.apache.beam.sdk.options.PipelineOptionsFactory; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | import java.io.Serializable; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | 17 | /** 18 | * 执行类 19 | * @Author: limeng 20 | * @Date: 2019/8/12 10:28 21 | */ 22 | @Data 23 | public class BaseRunners implements Serializable, OptionsEnum { 24 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 25 | private Class clazz = null; 26 | private Pipeline pipeline; 27 | private PipelineOptions pipelineOptions; 28 | 29 | private static final String MARK_SPLIT = "\\|%a%b%c%\\|"; 30 | private static final String UNDERLINE_T = "_"; 31 | 32 | 33 | public BaseRunners() { 34 | } 35 | 36 | public BaseRunners(Class clazz) { 37 | this.clazz = clazz; 38 | } 39 | 40 | 41 | 42 | public void start(Map map){ 43 | String[] args = mapToStr(map); 44 | pipelineOptions = PipelineOptionsFactory.fromArgs(args).withValidation().as(clazz); 45 | pipeline = Pipeline.create(pipelineOptions); 46 | PipelineSingletonUtil.getInstance(pipeline); 47 | } 48 | 49 | public void startByArgs(String[] args){ 50 | pipelineOptions = PipelineOptionsFactory.fromArgs(args).withValidation().as(clazz); 51 | pipeline = Pipeline.create(pipelineOptions); 52 | PipelineSingletonUtil.getInstance(pipeline); 53 | } 54 | 55 | public void start(){ 56 | if(clazz == null){ 57 | pipelineOptions = PipelineOptionsFactory.create(); 58 | }else { 59 | pipelineOptions = PipelineOptionsFactory.fromArgs("").withValidation().as(clazz); 60 | } 61 | 62 | pipeline = Pipeline.create(pipelineOptions); 63 | PipelineSingletonUtil.getInstance(pipeline); 64 | } 65 | 66 | public void run(){ 67 | 68 | pipeline.run().waitUntilFinish(); 69 | } 70 | 71 | /** 72 | * map转str 73 | * @param map 74 | * @return 75 | */ 76 | private String[] mapToStr(Map map){ 77 | String[] result=null; 78 | if(!BaseUtil.isBlankMap(map)){ 79 | result = new String[map.size()]; 80 | int index=0; 81 | for(Map.Entry entry : map.entrySet()){ 82 | result[index] = "--"+entry.getKey()+"="+entry.getValue().toString(); 83 | index++; 84 | } 85 | }else{ 86 | logger.info("fromArgs is null"); 87 | } 88 | return result; 89 | } 90 | 91 | 92 | 93 | public static void main(String[] args) { 94 | 95 | BaseRunners runners = new BaseRunners(IndexerPipelineOptions.class); 96 | 97 | HashMap objectObjectHashMap = new HashMap<>(); 98 | objectObjectHashMap.put("pattern","cccc"); 99 | runners.start(objectObjectHashMap); 100 | Pipeline instance = PipelineSingletonUtil.instance; 101 | IndexerPipelineOptions options = (IndexerPipelineOptions) instance.getOptions(); 102 | System.out.println(options); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/runners/options/IndexerPipelineOptions.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.runners.options; 2 | 3 | import org.apache.beam.runners.spark.SparkPipelineOptions; 4 | import org.apache.beam.sdk.options.Default; 5 | import org.apache.beam.sdk.options.Description; 6 | /** 7 | * 参数 8 | * @Author: limeng 9 | * @Date: 2019/7/18 14:51 10 | */ 11 | public interface IndexerPipelineOptions extends SparkPipelineOptions { 12 | 13 | @Description("方式import/export") 14 | String getPattern(); 15 | void setPattern(String pattern); 16 | 17 | @Description("源名称") 18 | String getSourceName(); 19 | void setSourceName(String sourceName); 20 | 21 | @Description("是否批量处理sql任务") 22 | @Default.Boolean(false) 23 | Boolean isSqlFile(); 24 | void setSqlFile(Boolean sqlFile); 25 | 26 | @Description("获取文件地址，本地/hdfs") 27 | String getFilePath(); 28 | void setFilePath(String filePath); 29 | 30 | 31 | @Description("链路是否有序") 32 | @Default.Boolean(false) 33 | Boolean isOrderLink(); 34 | void setOrderLink(Boolean value); 35 | 36 | 37 | @Description("jdbc sql") 38 | String getJdbcSql(); 39 | void setJdbcSql(String jdbcSql); 40 | 41 | @Description("jdbc 树 状态") 42 | @Default.Boolean(false) 43 | Boolean isTreeStatus(); 44 | void setTreeStatus(Boolean value); 45 | 46 | 47 | } 48 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/ESExportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.config.common.event.ESExportType; 5 | import com.knowlegene.parent.config.util.BaseUtil; 6 | import com.knowlegene.parent.process.pojo.ObjectCoder; 7 | import com.knowlegene.parent.process.pojo.SwapOptions; 8 | import com.knowlegene.parent.process.pojo.es.ESOptions; 9 | import com.knowlegene.parent.process.swap.event.ESExportTaskEvent; 10 | import com.knowlegene.parent.process.transform.TypeConversion; 11 | import com.knowlegene.parent.scheduler.event.EventHandler; 12 | import com.knowlegene.parent.scheduler.utils.CacheManager; 13 | import org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO; 14 | import org.apache.beam.sdk.transforms.ParDo; 15 | import org.apache.beam.sdk.values.PCollection; 16 | 17 | import java.util.Map; 18 | 19 | 20 | /** 21 | * @Author: limeng 22 | * @Date: 2019/9/9 18:30 23 | */ 24 | public class ESExportJob extends ExportJobBase { 25 | private static volatile ESOptions esOptions= null; 26 | 27 | public ESExportJob() { 28 | 29 | } 30 | 31 | public ESExportJob(SwapOptions options) { 32 | super(options); 33 | } 34 | 35 | public static ESOptions getDbOptions(){ 36 | if(esOptions == null){ 37 | String name = DBOperationEnum.ES_EXPORT.getName(); 38 | Object options = getOptions(name); 39 | if(options != null){ 40 | esOptions = (ESOptions)options; 41 | } 42 | } 43 | return esOptions; 44 | } 45 | 46 | 47 | 48 | 49 | /** 50 | * get matchall 51 | * @return 52 | */ 53 | private static PCollection queryAll(){ 54 | String[] esAddrs = getDbOptions().getEsAddrs(); 55 | String esIndex = getDbOptions().getEsIndex(); 56 | String esType = getDbOptions().getEsType(); 57 | 58 | ElasticsearchIO.Read read = getESSwap().getRead(esAddrs, esIndex, esType); 59 | if(read == null){ 60 | getLogger().error("es read is null"); 61 | return null; 62 | }else{ 63 | String esQuery = getDbOptions().getEsQuery(); 64 | if(BaseUtil.isNotBlank(esQuery)){ 65 | read.withQuery(esQuery); 66 | } 67 | getLogger().info("queryall start=>index:{},type:{}",esIndex,esType); 68 | return getPipeline().apply(read); 69 | } 70 | 71 | } 72 | 73 | /** 74 | * 75 | * 转换 根据导入库的类型 76 | * @return 77 | */ 78 | private static PCollection> transformPCollection(){ 79 | PCollection> result = null; 80 | PCollection strings = queryAll(); 81 | if(strings != null){ 82 | return strings.apply(ParDo.of(new TypeConversion.JsonAndMap())); 83 | } 84 | return result; 85 | 86 | } 87 | 88 | 89 | 90 | public static PCollection> query() { 91 | return transformPCollection(); 92 | } 93 | 94 | 95 | public static class ESExportDispatcher implements EventHandler { 96 | @Override 97 | public void handle(ESExportTaskEvent event) { 98 | if (event.getType() == ESExportType.T_EXPORT) { 99 | getLogger().info("ESExportDispatcher is start"); 100 | PCollection> result = query(); 101 | if(result != null){ 102 | CacheManager.setCache(DBOperationEnum.PCOLLECTION_QUERYS.getName(), result); 103 | } 104 | } 105 | } 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/ExportJobBase.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | 4 | import com.knowlegene.parent.process.pojo.ObjectCoder; 5 | import com.knowlegene.parent.process.pojo.SwapOptions; 6 | import org.apache.beam.sdk.values.PCollection; 7 | 8 | 9 | import java.util.Map; 10 | 11 | /** 12 | * @Author: limeng 13 | * @Date: 2019/8/20 15:52 14 | */ 15 | public class ExportJobBase extends JobBase { 16 | 17 | 18 | public ExportJobBase() { 19 | } 20 | 21 | public ExportJobBase(SwapOptions options) { 22 | super(options); 23 | } 24 | 25 | 26 | 27 | 28 | public static PCollection> query() { 29 | return null; 30 | } 31 | 32 | 33 | public static void save(PCollection> rows) { 34 | 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/FileExportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.config.common.event.FileExportType; 5 | import com.knowlegene.parent.config.util.BaseUtil; 6 | import com.knowlegene.parent.process.pojo.ObjectCoder; 7 | import com.knowlegene.parent.process.pojo.SwapOptions; 8 | import com.knowlegene.parent.process.pojo.file.FileOptions; 9 | import com.knowlegene.parent.process.swap.event.FileExportTaskEvent; 10 | import com.knowlegene.parent.process.transform.TypeConversion; 11 | import com.knowlegene.parent.process.util.SqlUtil; 12 | import com.knowlegene.parent.scheduler.event.EventHandler; 13 | import com.knowlegene.parent.scheduler.utils.CacheManager; 14 | import org.apache.beam.sdk.io.TextIO; 15 | import org.apache.beam.sdk.schemas.Schema; 16 | import org.apache.beam.sdk.transforms.ParDo; 17 | import org.apache.beam.sdk.values.PCollection; 18 | 19 | import java.util.Map; 20 | 21 | 22 | /** 23 | * @Author: limeng 24 | * @Date: 2019/8/26 19:36 25 | */ 26 | public class FileExportJob extends ExportJobBase { 27 | 28 | private static volatile FileOptions fileOptions= null; 29 | 30 | public FileExportJob() { 31 | } 32 | 33 | public FileExportJob(SwapOptions options) { 34 | super(options); 35 | } 36 | 37 | public static FileOptions getDbOptions(){ 38 | if(fileOptions == null){ 39 | String name = DBOperationEnum.FILE_EXPORT.getName(); 40 | Object options = getOptions(name); 41 | if(options != null){ 42 | fileOptions = (FileOptions)options; 43 | } 44 | } 45 | return fileOptions; 46 | } 47 | 48 | 49 | 50 | 51 | private static PCollection queryByFile(){ 52 | String fieldDelim = getDbOptions().getFieldDelim(); 53 | String filePath = getDbOptions().getFilePath(); 54 | if(fieldDelim == null || fieldDelim == "" || BaseUtil.isBlank(filePath)){ 55 | getLogger().error("fieldDelim/filePath is null"); 56 | return null; 57 | } 58 | 59 | return getPipeline().apply(TextIO.read().from(filePath).withHintMatchesManyFiles()); 60 | } 61 | 62 | /** 63 | * 64 | * 转换 根据导入库的类型 65 | * @return 66 | */ 67 | private static PCollection> transformPCollection(){ 68 | PCollection> result = null; 69 | 70 | String[] fieldTitle = getDbOptions().getFieldTitle(); 71 | Schema type = SqlUtil.getSchemaByTitle(fieldTitle); 72 | if(type == null){ 73 | getLogger().error("schema is null"); 74 | return result; 75 | } 76 | String fieldDelim = getDbOptions().getFieldDelim(); 77 | PCollection files = queryByFile(); 78 | if(files!=null){ 79 | return files.apply(ParDo.of(new TypeConversion.StringAndMap(type,fieldDelim))); 80 | }else{ 81 | getLogger().info("files is null"); 82 | } 83 | return null; 84 | } 85 | 86 | 87 | 88 | 89 | 90 | public static PCollection> query(){ 91 | return transformPCollection(); 92 | } 93 | 94 | 95 | 96 | public static class FileExportDispatcher implements EventHandler { 97 | @Override 98 | public void handle(FileExportTaskEvent event) { 99 | if (event.getType() == FileExportType.T_EXPORT) { 100 | getLogger().info("FileExportDispatcher is start"); 101 | PCollection> result = query(); 102 | if(result != null){ 103 | CacheManager.setCache(DBOperationEnum.PCOLLECTION_QUERYS.getName(), result); 104 | } 105 | 106 | } 107 | } 108 | } 109 | 110 | } 111 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/FileImportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.config.common.event.FileImportType; 5 | import com.knowlegene.parent.config.util.BaseUtil; 6 | import com.knowlegene.parent.config.util.HdfsFileUtil; 7 | import com.knowlegene.parent.process.pojo.ObjectCoder; 8 | import com.knowlegene.parent.process.pojo.SwapOptions; 9 | import com.knowlegene.parent.process.pojo.file.FileOptions; 10 | import com.knowlegene.parent.process.swap.event.FileImportTaskEvent; 11 | import com.knowlegene.parent.process.transform.TypeConversion; 12 | import com.knowlegene.parent.process.util.SqlUtil; 13 | import com.knowlegene.parent.scheduler.event.EventHandler; 14 | import com.knowlegene.parent.scheduler.utils.CacheManager; 15 | import org.apache.beam.sdk.io.TextIO; 16 | import org.apache.beam.sdk.schemas.Schema; 17 | import org.apache.beam.sdk.transforms.ParDo; 18 | import org.apache.beam.sdk.values.KV; 19 | import org.apache.beam.sdk.values.PCollection; 20 | 21 | import java.util.Map; 22 | 23 | 24 | /** 25 | * 文件导入 26 | * @Author: limeng 27 | * @Date: 2019/8/20 18:58 28 | */ 29 | public class FileImportJob extends ImportJobBase{ 30 | 31 | private static volatile FileOptions fileOptions= null; 32 | 33 | public FileImportJob() { 34 | } 35 | 36 | public FileImportJob(SwapOptions opts) { 37 | super(opts); 38 | } 39 | 40 | 41 | public static FileOptions getDbOptions(){ 42 | if(fileOptions == null){ 43 | String name = DBOperationEnum.FILE_IMPORT.getName(); 44 | Object options = getOptions(name); 45 | if(options != null){ 46 | fileOptions = (FileOptions)options; 47 | } 48 | } 49 | return fileOptions; 50 | } 51 | 52 | /** 53 | * 保存文件 54 | * @param rows 55 | */ 56 | private static void saveByFile(PCollection> rows){ 57 | if(rows != null){ 58 | String filePath = getDbOptions().getFilePath(); 59 | String fieldDelim = getDbOptions().getFieldDelim(); 60 | if(fieldDelim == null || fieldDelim == ""){ 61 | fieldDelim = "\t"; 62 | } 63 | if(BaseUtil.isBlank(filePath)){ 64 | getLogger().info("filePath is null"); 65 | return; 66 | } 67 | KV splitKv = HdfsFileUtil.splitMark(filePath, "\\."); 68 | 69 | String[] fieldTitle = getDbOptions().getFieldTitle(); 70 | Schema schema = SqlUtil.getSchemaByTitle(fieldTitle); 71 | if(schema!=null){ 72 | rows.apply(ParDo.of(new TypeConversion.SortAndMapType(schema))) 73 | .apply(ParDo.of(new TypeConversion.MapAndString(fieldDelim))) 74 | .apply(TextIO.write().to(splitKv.getKey()).withSuffix(splitKv.getValue())); 75 | }else{ 76 | rows.apply(ParDo.of(new TypeConversion.MapAndString(fieldDelim))) 77 | .apply(TextIO.write().to(splitKv.getKey()).withSuffix(splitKv.getValue())); 78 | } 79 | 80 | 81 | } 82 | } 83 | 84 | 85 | 86 | public static void save(PCollection> rows) { 87 | if(rows!=null){ 88 | 89 | saveByFile(rows); 90 | } 91 | } 92 | 93 | 94 | 95 | public static class FileImportDispatcher implements EventHandler { 96 | @Override 97 | public void handle(FileImportTaskEvent event) { 98 | if(event.getType() == FileImportType.T_IMPORT){ 99 | getLogger().info("FileImportDispatcher is start"); 100 | 101 | if(CacheManager.isExist(DBOperationEnum.PCOLLECTION_QUERYS.getName())){ 102 | save((PCollection>)CacheManager.getCache(DBOperationEnum.PCOLLECTION_QUERYS.getName())); 103 | } 104 | 105 | } 106 | } 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/GbaseExportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.config.common.event.GbaseExportType; 5 | import com.knowlegene.parent.config.util.BaseUtil; 6 | import com.knowlegene.parent.config.util.JdbcUtil; 7 | import com.knowlegene.parent.process.pojo.ObjectCoder; 8 | import com.knowlegene.parent.process.pojo.db.DBOptions; 9 | import com.knowlegene.parent.process.pojo.SwapOptions; 10 | import com.knowlegene.parent.process.swap.event.GbaseExportTaskEvent; 11 | import com.knowlegene.parent.process.transform.TypeConversion; 12 | import com.knowlegene.parent.scheduler.event.EventHandler; 13 | import com.knowlegene.parent.scheduler.utils.CacheManager; 14 | import org.apache.beam.sdk.schemas.Schema; 15 | import org.apache.beam.sdk.transforms.ParDo; 16 | import org.apache.beam.sdk.values.PCollection; 17 | 18 | import java.util.Map; 19 | 20 | /** 21 | * @Author: limeng 22 | * @Date: 2019/10/11 18:19 23 | */ 24 | public class GbaseExportJob extends ExportJobBase { 25 | private volatile static DBOptions dbOptions = null; 26 | 27 | public GbaseExportJob() { 28 | } 29 | 30 | public GbaseExportJob(SwapOptions options) { 31 | super(options); 32 | } 33 | 34 | public static DBOptions getDbOptions(){ 35 | if(dbOptions == null){ 36 | String name = DBOperationEnum.GBASE_EXPORT.getName(); 37 | Object options = getOptions(name); 38 | if(options != null){ 39 | dbOptions = (DBOptions)options; 40 | } 41 | } 42 | return dbOptions; 43 | } 44 | 45 | 46 | private static Schema getGbaseSchemas(){ 47 | String[] dbColumn = getDbOptions().getDbColumn(); 48 | String tableName = getDbOptions().getTableName(); 49 | //表所有列 50 | Schema allSchema = getGbaseSwapExport().desc(tableName); 51 | 52 | getLogger().info("gbase=>tableName:{}",tableName); 53 | return JdbcUtil.columnConversion(dbColumn, allSchema); 54 | 55 | } 56 | 57 | 58 | /** 59 | * 查询sql 60 | * @return 61 | */ 62 | private static PCollection> queryBySQL(){ 63 | String sql = getDbOptions().getDbSQL(); 64 | String tableName = getDbOptions().getTableName(); 65 | Schema schema = getGbaseSchemas(); 66 | getLogger().info("gbase=>sql:{},tableName:{}",sql,tableName); 67 | 68 | return getPipeline().apply(getGbaseSwapExport().query(sql)) 69 | .apply(ParDo.of(new TypeConversion.MapObjectAndType(schema))); 70 | } 71 | 72 | 73 | /** 74 | * 查询表 75 | * @return 76 | */ 77 | private static PCollection> queryByTable(){ 78 | Schema schema = getGbaseSchemas(); 79 | String tableName = getDbOptions().getTableName(); 80 | getLogger().info("gbase=>tableName:{}",tableName); 81 | 82 | return getPipeline().apply(getGbaseSwapExport().queryByTable(tableName)) 83 | .apply(ParDo.of(new TypeConversion.MapObjectAndType(schema))); 84 | 85 | } 86 | 87 | 88 | public static PCollection> query() { 89 | String sql = getDbOptions().getDbSQL(); 90 | PCollection> rows=null; 91 | if(BaseUtil.isNotBlank(sql)){ 92 | rows = queryBySQL(); 93 | }else { 94 | rows = queryByTable(); 95 | } 96 | 97 | 98 | return rows; 99 | 100 | } 101 | 102 | 103 | public static class GbaseExportDispatcher implements EventHandler { 104 | @Override 105 | public void handle(GbaseExportTaskEvent event) { 106 | if (event.getType() == GbaseExportType.T_EXPORT) { 107 | getLogger().info("GbaseExportDispatcher is start"); 108 | PCollection> result = query(); 109 | if(result != null){ 110 | CacheManager.setCache(DBOperationEnum.PCOLLECTION_QUERYS.getName(), result); 111 | } 112 | 113 | } 114 | } 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/GbaseImportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | 4 | 5 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 6 | import com.knowlegene.parent.config.common.event.GbaseImportType; 7 | import com.knowlegene.parent.config.util.BaseUtil; 8 | import com.knowlegene.parent.config.util.JdbcUtil; 9 | import com.knowlegene.parent.process.pojo.ObjectCoder; 10 | import com.knowlegene.parent.process.pojo.db.DBOptions; 11 | import com.knowlegene.parent.process.pojo.SwapOptions; 12 | import com.knowlegene.parent.process.swap.event.GbaseImportTaskEvent; 13 | import com.knowlegene.parent.process.transform.TypeConversion; 14 | import com.knowlegene.parent.scheduler.event.EventHandler; 15 | import com.knowlegene.parent.scheduler.utils.CacheManager; 16 | import org.apache.beam.sdk.schemas.Schema; 17 | import org.apache.beam.sdk.transforms.ParDo; 18 | import org.apache.beam.sdk.values.PCollection; 19 | 20 | import java.util.Map; 21 | 22 | 23 | /** 24 | * @Author: limeng 25 | * @Date: 2019/8/20 16:40 26 | */ 27 | public class GbaseImportJob extends ImportJobBase{ 28 | private volatile static DBOptions dbOptions = null; 29 | 30 | public GbaseImportJob() { 31 | } 32 | 33 | public GbaseImportJob(SwapOptions opts) { 34 | super(opts); 35 | } 36 | 37 | public static DBOptions getDbOptions(){ 38 | if(dbOptions == null){ 39 | String name = DBOperationEnum.GBASE_IMPORT.getName(); 40 | Object options = getOptions(name); 41 | if(options != null){ 42 | dbOptions = (DBOptions)options; 43 | } 44 | } 45 | return dbOptions; 46 | } 47 | 48 | 49 | /** 50 | * 批量保存sql 51 | * @param schema 52 | * @param tableName 53 | * @return 54 | */ 55 | private static String getInsertSQL(Schema schema,String tableName){ 56 | String result=""; 57 | if(schema!=null && BaseUtil.isNotBlank(tableName)){ 58 | result = JdbcUtil.getInsertSQL(schema,tableName); 59 | } 60 | return result; 61 | } 62 | 63 | 64 | 65 | 66 | private static Schema getSchemas(){ 67 | String[] dbColumn = getDbOptions().getDbColumn(); 68 | String tableName = getDbOptions().getTableName(); 69 | //表所有列 70 | Schema allSchema = getGbaseSwapImport().desc(tableName); 71 | 72 | getLogger().info("gbase=>tableName:{}",tableName); 73 | return JdbcUtil.columnConversion(dbColumn, allSchema); 74 | 75 | } 76 | 77 | 78 | 79 | /** 80 | * 保存 81 | * @param rows 82 | * @param schema 83 | * @param tableName 84 | * @return 85 | */ 86 | private static void saveBySQL(PCollection> rows, Schema schema, String tableName){ 87 | if(rows !=null && schema!=null){ 88 | String insertSQL = getInsertSQL(schema, tableName); 89 | getLogger().info("insertSQL:{}",insertSQL); 90 | if(BaseUtil.isNotBlank(insertSQL)){ 91 | rows.apply(ParDo.of(new TypeConversion.SortAndMapType(schema))) 92 | .apply(getGbaseSwapImport().saveByIO(insertSQL)); 93 | } 94 | } 95 | } 96 | 97 | public static void save(PCollection> rows) { 98 | if(rows != null){ 99 | String tableName = getDbOptions().getTableName(); 100 | Schema schema = getSchemas(); 101 | if(schema == null){ 102 | getLogger().info("schema is null"); 103 | } 104 | 105 | saveBySQL(rows,schema,tableName); 106 | } 107 | } 108 | 109 | 110 | public static class GbaseImportDispatcher implements EventHandler { 111 | @Override 112 | public void handle(GbaseImportTaskEvent event) { 113 | if(event.getType() == GbaseImportType.T_IMPORT){ 114 | getLogger().info("GbaseImportDispatcher is start"); 115 | 116 | if(CacheManager.isExist(DBOperationEnum.PCOLLECTION_QUERYS.getName())){ 117 | save((PCollection>)CacheManager.getCache(DBOperationEnum.PCOLLECTION_QUERYS.getName())); 118 | } 119 | 120 | } 121 | } 122 | } 123 | 124 | 125 | } 126 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/ImportJobBase.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | 4 | import com.knowlegene.parent.process.pojo.ObjectCoder; 5 | import com.knowlegene.parent.process.pojo.SwapOptions; 6 | import org.apache.beam.sdk.values.PCollection; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * import 12 | * @Author: limeng 13 | * @Date: 2019/8/20 15:50 14 | */ 15 | public class ImportJobBase extends JobBase { 16 | 17 | 18 | 19 | public ImportJobBase() { 20 | } 21 | public ImportJobBase(SwapOptions opts) { 22 | super(opts); 23 | } 24 | 25 | 26 | 27 | 28 | 29 | public static PCollection> query() { 30 | return null; 31 | } 32 | 33 | 34 | public static void save(PCollection> rows) { } 35 | } 36 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/JobBase.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | 4 | import com.knowlegene.parent.config.common.event.SwapJobEventType; 5 | import com.knowlegene.parent.config.util.PipelineSingletonUtil; 6 | import com.knowlegene.parent.process.pojo.SwapOptions; 7 | import com.knowlegene.parent.process.swap.dispatcher.SwapMaster; 8 | import com.knowlegene.parent.process.swap.common.BaseSwap; 9 | import com.knowlegene.parent.process.swap.event.SwapJobEvent; 10 | import com.knowlegene.parent.process.tool.BaseSwapTool; 11 | import com.knowlegene.parent.process.util.SwapMasterUtil; 12 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 13 | import com.knowlegene.parent.scheduler.event.EventHandler; 14 | import com.knowlegene.parent.scheduler.utils.CacheManager; 15 | import lombok.Data; 16 | import org.apache.beam.sdk.Pipeline; 17 | import org.slf4j.Logger; 18 | import org.slf4j.LoggerFactory; 19 | 20 | /** 21 | * @Author: limeng 22 | * @Date: 2019/8/20 15:54 23 | */ 24 | @Data 25 | public class JobBase extends BaseSwap { 26 | protected static SwapOptions options; 27 | 28 | private static Logger logger = null; 29 | 30 | private static String fromPre = "from_"; 31 | private static String toSuffix = "to_"; 32 | 33 | public JobBase() { 34 | JobBase.logger = LoggerFactory.getLogger(this.getClass()); 35 | } 36 | 37 | public JobBase(SwapOptions options) { 38 | JobBase.logger = LoggerFactory.getLogger(this.getClass()); 39 | JobBase.options = options; 40 | } 41 | 42 | 43 | public static Logger getLogger(){ 44 | return logger; 45 | } 46 | 47 | 48 | protected static Object getOptions(String keys){ 49 | boolean exist = CacheManager.isExist(keys); 50 | Object result = null; 51 | if(!exist) return result; 52 | result = CacheManager.getCache(keys); 53 | return result; 54 | } 55 | 56 | 57 | public static class SwapJobEventDispatcher implements EventHandler { 58 | @Override 59 | public void handle(SwapJobEvent event) { 60 | 61 | if(event.getType() == SwapJobEventType.JOB_KILL){ 62 | try { 63 | getSwapMaster().serviceStop(); 64 | getSwapMaster().setAlive(false); 65 | } catch (Exception e) { 66 | logger.error("SwapMaster stop msg:{}",e.getMessage()); 67 | } 68 | }else if(event.getType() == SwapJobEventType.JOB_START){ 69 | getPipeline().run().waitUntilFinish(); 70 | } 71 | } 72 | } 73 | 74 | 75 | public void run(){ 76 | getSwapMaster().getDispatcher().getEventHandler().handle((AbstractEvent)BaseSwapTool.getTask(fromPre + options.getFromName())); 77 | getSwapMaster().getDispatcher().getEventHandler().handle((AbstractEvent)BaseSwapTool.getTask(toSuffix + options.getToName())); 78 | getSwapMaster().getDispatcher().getEventHandler().handle(new SwapJobEvent(SwapJobEventType.JOB_START)); 79 | getSwapMaster().getDispatcher().getEventHandler().handle(new SwapJobEvent(SwapJobEventType.JOB_KILL)); 80 | } 81 | 82 | protected static Pipeline getPipeline(){ 83 | return PipelineSingletonUtil.instance; 84 | } 85 | 86 | protected static SwapMaster getSwapMaster(){ 87 | return SwapMasterUtil.instance; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/MySQLExportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.config.common.event.MySQLExportType; 5 | import com.knowlegene.parent.config.util.BaseUtil; 6 | import com.knowlegene.parent.config.util.JdbcUtil; 7 | import com.knowlegene.parent.process.pojo.ObjectCoder; 8 | import com.knowlegene.parent.process.pojo.db.DBOptions; 9 | import com.knowlegene.parent.process.pojo.SwapOptions; 10 | import com.knowlegene.parent.process.swap.event.MySQLExportTaskEvent; 11 | import com.knowlegene.parent.process.transform.TypeConversion; 12 | import com.knowlegene.parent.scheduler.event.EventHandler; 13 | import com.knowlegene.parent.scheduler.utils.CacheManager; 14 | import org.apache.beam.sdk.schemas.Schema; 15 | import org.apache.beam.sdk.transforms.ParDo; 16 | import org.apache.beam.sdk.values.PCollection; 17 | 18 | import java.util.Map; 19 | 20 | /** 21 | * @Author: limeng 22 | * @Date: 2019/9/9 14:57 23 | */ 24 | public class MySQLExportJob extends ExportJobBase { 25 | 26 | private volatile static DBOptions dbOptions = null; 27 | 28 | public MySQLExportJob() { 29 | } 30 | 31 | public MySQLExportJob(SwapOptions options) { 32 | super(options); 33 | } 34 | 35 | public static DBOptions getDbOptions(){ 36 | if(dbOptions == null){ 37 | String name = DBOperationEnum.MYSQL_EXPORT.getName(); 38 | Object options = getOptions(name); 39 | if(options != null){ 40 | dbOptions = (DBOptions)options; 41 | } 42 | } 43 | return dbOptions; 44 | } 45 | 46 | 47 | 48 | private static Schema getMysqlSchemas(){ 49 | String tableName = getDbOptions().getTableName(); 50 | //表所有列 51 | Schema allSchema = getMySQLSwapExport().desc(tableName); 52 | getLogger().info("mysql=>tableName:{}",tableName); 53 | return allSchema; 54 | 55 | } 56 | 57 | /** 58 | * 查询sql 59 | * @return 60 | */ 61 | private static PCollection> queryBySQL(){ 62 | String sql = getDbOptions().getDbSQL(); 63 | String tableName = getDbOptions().getTableName(); 64 | String[] dbColumn = JdbcUtil.getColumnBySqlRex(sql); 65 | 66 | Schema allSchema = getMySQLSwapExport().desc(tableName); 67 | 68 | getLogger().info("mysql=>sql:{},tableName:{}",sql,tableName); 69 | 70 | Schema schema = JdbcUtil.columnConversion(dbColumn, allSchema); 71 | 72 | 73 | 74 | return getPipeline().apply(getMySQLSwapExport().query(sql)) 75 | .apply(ParDo.of(new TypeConversion.MapObjectAndType(schema))); 76 | } 77 | 78 | /** 79 | * 查询表 80 | * @return 81 | */ 82 | private static PCollection> queryByTable(){ 83 | 84 | Schema schema = getMysqlSchemas(); 85 | String tableName = getDbOptions().getTableName(); 86 | 87 | return getPipeline().apply(getMySQLSwapExport().queryByTable(tableName)) 88 | .apply(ParDo.of(new TypeConversion.MapObjectAndType(schema))); 89 | 90 | } 91 | 92 | public static PCollection> query(){ 93 | String sql = getDbOptions().getDbSQL(); 94 | PCollection> rows=null; 95 | if(BaseUtil.isNotBlank(sql)){ 96 | rows = queryBySQL(); 97 | 98 | }else{ 99 | rows = queryByTable(); 100 | } 101 | 102 | return rows; 103 | } 104 | 105 | 106 | 107 | public static class MySQLExportDispatcher implements EventHandler { 108 | @Override 109 | public void handle(MySQLExportTaskEvent event) { 110 | if (event.getType() == MySQLExportType.T_EXPORT) { 111 | getLogger().info("MySQLExportDispatcher is start"); 112 | PCollection> result = query(); 113 | if(result != null){ 114 | CacheManager.setCache(DBOperationEnum.PCOLLECTION_QUERYS.getName(), result); 115 | } 116 | } 117 | } 118 | } 119 | 120 | 121 | 122 | } 123 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/MySQLImportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | 4 | 5 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 6 | import com.knowlegene.parent.config.common.event.MySQLImportType; 7 | import com.knowlegene.parent.config.util.BaseUtil; 8 | import com.knowlegene.parent.config.util.JdbcUtil; 9 | import com.knowlegene.parent.process.pojo.ObjectCoder; 10 | import com.knowlegene.parent.process.pojo.db.DBOptions; 11 | import com.knowlegene.parent.process.pojo.SwapOptions; 12 | import com.knowlegene.parent.process.swap.event.MySQLImportTaskEvent; 13 | import com.knowlegene.parent.process.transform.TypeConversion; 14 | import com.knowlegene.parent.scheduler.event.EventHandler; 15 | import com.knowlegene.parent.scheduler.utils.CacheManager; 16 | import org.apache.beam.sdk.schemas.Schema; 17 | import org.apache.beam.sdk.transforms.ParDo; 18 | import org.apache.beam.sdk.values.PCollection; 19 | import java.util.Map; 20 | 21 | 22 | 23 | /** 24 | * @Author: limeng 25 | * @Date: 2019/8/20 16:40 26 | */ 27 | public class MySQLImportJob extends ImportJobBase{ 28 | private volatile static DBOptions dbOptions = null; 29 | 30 | public MySQLImportJob() { 31 | } 32 | 33 | public MySQLImportJob(SwapOptions opts) { 34 | super(opts); 35 | } 36 | 37 | public static DBOptions getDbOptions(){ 38 | if(dbOptions == null){ 39 | String name = DBOperationEnum.MYSQL_IMPORT.getName(); 40 | Object options = getOptions(name); 41 | if(options != null){ 42 | dbOptions = (DBOptions)options; 43 | } 44 | } 45 | return dbOptions; 46 | } 47 | 48 | 49 | /** 50 | * 批量保存sql 51 | * @param schema 52 | * @param tableName 53 | * @return 54 | */ 55 | private static String getInsertSQL(Schema schema,String tableName){ 56 | String result=""; 57 | if(schema!=null && BaseUtil.isNotBlank(tableName)){ 58 | result = JdbcUtil.getInsertSQL(schema,tableName); 59 | } 60 | return result; 61 | } 62 | 63 | 64 | 65 | 66 | private static Schema getMysqlSchemas(){ 67 | String[] dbColumn = getDbOptions().getDbColumn(); 68 | String tableName = getDbOptions().getTableName(); 69 | //表所有列 70 | Schema allSchema = getMySQLSwapImport().desc(tableName); 71 | 72 | getLogger().info("mysql=>tableName:{}",tableName); 73 | return JdbcUtil.columnConversion(dbColumn, allSchema); 74 | 75 | } 76 | 77 | 78 | 79 | /** 80 | * 保存 81 | * @param rows 82 | * @param schema 83 | * @param tableName 84 | * @return 85 | */ 86 | private static void saveBySQL(PCollection> rows, Schema schema, String tableName){ 87 | if(rows !=null && schema!=null){ 88 | String insertSQL = getInsertSQL(schema, tableName); 89 | getLogger().info("insertSQL:{}",insertSQL); 90 | if(BaseUtil.isNotBlank(insertSQL)){ 91 | rows.apply(ParDo.of(new TypeConversion.SortAndMapType(schema))) 92 | .apply(getMySQLSwapImport().saveByIO(insertSQL)); 93 | } 94 | } 95 | } 96 | 97 | public static void save(PCollection> rows) { 98 | if(rows != null){ 99 | String tableName = getDbOptions().getTableName(); 100 | Schema schema = getMysqlSchemas(); 101 | if(schema == null){ 102 | getLogger().info("schema is null"); 103 | } 104 | saveBySQL(rows,schema,tableName); 105 | } 106 | } 107 | 108 | 109 | public static class MySQLImportDispatcher implements EventHandler { 110 | @Override 111 | public void handle(MySQLImportTaskEvent event) { 112 | if(event.getType() == MySQLImportType.T_IMPORT){ 113 | getLogger().info("MySQLImportDispatcher is start"); 114 | 115 | if(CacheManager.isExist(DBOperationEnum.PCOLLECTION_QUERYS.getName())){ 116 | save((PCollection>)CacheManager.getCache(DBOperationEnum.PCOLLECTION_QUERYS.getName())); 117 | } 118 | 119 | } 120 | } 121 | } 122 | 123 | 124 | } 125 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/Neo4jExportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.config.common.event.Neo4jExportType; 5 | import com.knowlegene.parent.config.util.BaseUtil; 6 | import com.knowlegene.parent.process.pojo.ObjectCoder; 7 | import com.knowlegene.parent.process.pojo.SwapOptions; 8 | import com.knowlegene.parent.process.pojo.neo4j.Neo4jOptions; 9 | import com.knowlegene.parent.process.swap.event.Neo4jExportTaskEvent; 10 | import com.knowlegene.parent.process.transform.TypeConversion; 11 | import com.knowlegene.parent.scheduler.event.EventHandler; 12 | import com.knowlegene.parent.scheduler.utils.CacheManager; 13 | import org.apache.beam.sdk.transforms.ParDo; 14 | import org.apache.beam.sdk.values.PCollection; 15 | 16 | import java.util.Map; 17 | 18 | 19 | /** 20 | * @Author: limeng 21 | * @Date: 2019/9/23 17:46 22 | */ 23 | public class Neo4jExportJob extends ExportJobBase { 24 | private static Neo4jOptions neo4jOptions; 25 | 26 | public Neo4jExportJob() { 27 | } 28 | 29 | public Neo4jExportJob(SwapOptions options) { 30 | super(options); 31 | } 32 | 33 | private static Neo4jOptions getDbOptions(){ 34 | if(neo4jOptions == null){ 35 | String name = DBOperationEnum.NEO4J_EXPORT.getName(); 36 | Object options = getOptions(name); 37 | if(options != null){ 38 | neo4jOptions = (Neo4jOptions)options; 39 | } 40 | } 41 | return neo4jOptions; 42 | } 43 | 44 | 45 | 46 | public static PCollection> query() { 47 | String cypher = getDbOptions().getCypher(); 48 | String neoType = getDbOptions().getNeoType(); 49 | if(BaseUtil.isNotBlank(cypher) && BaseUtil.isNotBlank(neoType)){ 50 | return getPipeline().apply(getNeo4jSwapExport().query(cypher,neoType)) 51 | .apply(ParDo.of(new TypeConversion.ObjectCodersAndMap())); 52 | } 53 | return null; 54 | } 55 | 56 | 57 | public static class Neo4jExportDispatcher implements EventHandler { 58 | @Override 59 | public void handle(Neo4jExportTaskEvent event) { 60 | if (event.getType() == Neo4jExportType.T_EXPORT) { 61 | getLogger().info("Neo4jExportDispatcher is start"); 62 | PCollection> result = query(); 63 | if(result != null){ 64 | CacheManager.setCache(DBOperationEnum.PCOLLECTION_QUERYS.getName(), result); 65 | } 66 | 67 | } 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/OracleExportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.config.common.event.OracleExportType; 5 | import com.knowlegene.parent.config.util.BaseUtil; 6 | import com.knowlegene.parent.config.util.JdbcUtil; 7 | import com.knowlegene.parent.process.pojo.ObjectCoder; 8 | import com.knowlegene.parent.process.pojo.db.DBOptions; 9 | import com.knowlegene.parent.process.pojo.SwapOptions; 10 | import com.knowlegene.parent.process.swap.event.OracleExportTaskEvent; 11 | import com.knowlegene.parent.process.transform.TypeConversion; 12 | import com.knowlegene.parent.scheduler.event.EventHandler; 13 | import com.knowlegene.parent.scheduler.utils.CacheManager; 14 | import org.apache.beam.sdk.io.jdbc.JdbcIO; 15 | import org.apache.beam.sdk.schemas.Schema; 16 | import org.apache.beam.sdk.schemas.SchemaCoder; 17 | import org.apache.beam.sdk.transforms.ParDo; 18 | import org.apache.beam.sdk.values.PCollection; 19 | import org.apache.beam.sdk.values.Row; 20 | 21 | import java.util.Map; 22 | 23 | /** 24 | * @Author: limeng 25 | * @Date: 2019/9/12 16:22 26 | */ 27 | public class OracleExportJob extends ExportJobBase { 28 | private static volatile DBOptions dbOptions = null; 29 | 30 | public OracleExportJob() { 31 | } 32 | 33 | public OracleExportJob(SwapOptions options) { 34 | super(options); 35 | } 36 | 37 | 38 | private static DBOptions getDbOptions(){ 39 | if(dbOptions == null){ 40 | String name = DBOperationEnum.ORACLE_EXPORT.getName(); 41 | Object options = getOptions(name); 42 | if(options != null){ 43 | dbOptions = (DBOptions)options; 44 | } 45 | } 46 | return dbOptions; 47 | } 48 | 49 | /** 50 | * 查询sql 51 | * @return 52 | */ 53 | private static PCollection> queryBySQL(){ 54 | String sql = getDbOptions().getDbSQL(); 55 | String tableName = getDbOptions().getTableName(); 56 | String[] dbColumn = JdbcUtil.getColumnBySqlRex(sql); 57 | Schema allSchema = getOracleSwapExport().desc(tableName); 58 | 59 | Schema schema = JdbcUtil.columnConversion(dbColumn, allSchema); 60 | getLogger().info("query start=>tableName:{},sql:{}",tableName,sql); 61 | 62 | return getPipeline().apply(getOracleSwapExport().query(sql)) 63 | .apply(ParDo.of(new TypeConversion.MapObjectAndType(schema))); 64 | } 65 | 66 | 67 | /** 68 | * 查询表 69 | * @return 70 | */ 71 | private static PCollection> queryByTable(){ 72 | String[] dbColumn =getDbOptions().getDbColumn(); 73 | String tableName = getDbOptions().getTableName(); 74 | //表所有列 75 | Schema allSchema = getOracleSwapExport().desc(tableName); 76 | Schema schema = JdbcUtil.columnConversion(dbColumn, allSchema); 77 | 78 | getLogger().info("query start=>tableName:{}",tableName); 79 | 80 | return getPipeline().apply(getOracleSwapExport().queryByTable(tableName)) 81 | .apply(ParDo.of(new TypeConversion.MapObjectAndType(schema))); 82 | } 83 | 84 | 85 | public static PCollection> query(){ 86 | String sql = getDbOptions().getDbSQL(); 87 | if(BaseUtil.isNotBlank(sql)){ 88 | return queryBySQL(); 89 | }else{ 90 | return queryByTable(); 91 | } 92 | } 93 | 94 | 95 | public static class OracleExportDispatcher implements EventHandler { 96 | @Override 97 | public void handle(OracleExportTaskEvent event) { 98 | if (event.getType() == OracleExportType.T_EXPORT) { 99 | getLogger().info("OracleExportDispatcher is start"); 100 | PCollection> result = query(); 101 | if(result != null){ 102 | CacheManager.setCache(DBOperationEnum.PCOLLECTION_QUERYS.getName(), result); 103 | } 104 | } 105 | } 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/OracleImportJob.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap; 2 | 3 | import com.knowlegene.parent.config.common.constantenum.DBOperationEnum; 4 | import com.knowlegene.parent.config.common.event.OracleImportType; 5 | import com.knowlegene.parent.config.util.BaseUtil; 6 | import com.knowlegene.parent.config.util.JdbcUtil; 7 | import com.knowlegene.parent.process.pojo.ObjectCoder; 8 | import com.knowlegene.parent.process.pojo.db.DBOptions; 9 | import com.knowlegene.parent.process.pojo.SwapOptions; 10 | import com.knowlegene.parent.process.swap.event.OracleImportTaskEvent; 11 | import com.knowlegene.parent.process.transform.TypeConversion; 12 | import com.knowlegene.parent.scheduler.event.EventHandler; 13 | import com.knowlegene.parent.scheduler.utils.CacheManager; 14 | import org.apache.beam.sdk.schemas.Schema; 15 | import org.apache.beam.sdk.transforms.ParDo; 16 | import org.apache.beam.sdk.values.PCollection; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | 21 | 22 | /** 23 | * @Author: limeng 24 | * @Date: 2019/8/20 16:44 25 | */ 26 | public class OracleImportJob extends ImportJobBase{ 27 | private volatile static DBOptions dbOptions = null; 28 | 29 | public OracleImportJob() { 30 | } 31 | 32 | public OracleImportJob(SwapOptions opts) { 33 | super(opts); 34 | } 35 | 36 | 37 | 38 | private static DBOptions getDbOptions(){ 39 | if(dbOptions == null){ 40 | String name = DBOperationEnum.ORACLE_IMPORT.getName(); 41 | Object options = getOptions(name); 42 | if(options != null){ 43 | dbOptions = (DBOptions)options; 44 | } 45 | } 46 | return dbOptions; 47 | } 48 | 49 | /** 50 | * 保存 51 | * @param rows 52 | * @param schema 53 | * @param tableName 54 | * @return 55 | */ 56 | private static void saveBySQL(PCollection> rows, Schema schema, String tableName){ 57 | if(rows !=null && schema!=null){ 58 | String insertSQL = getInsertSQL(schema, tableName); 59 | getLogger().info("insertSQL:{}",insertSQL); 60 | if(BaseUtil.isNotBlank(insertSQL)){ 61 | rows.apply(ParDo.of(new TypeConversion.SortAndMapType(schema))) 62 | .apply(getOracleSwapImport().saveByIO(insertSQL)); 63 | } 64 | } 65 | } 66 | 67 | /** 68 | * 批量保存sql 69 | * @param schema 70 | * @param tableName 71 | * @return 72 | */ 73 | private static String getInsertSQL(Schema schema,String tableName){ 74 | String result=""; 75 | if(schema!=null && BaseUtil.isNotBlank(tableName)){ 76 | result = JdbcUtil.getInsertSQL(schema,tableName); 77 | } 78 | return result; 79 | } 80 | 81 | private static Schema getSchema(){ 82 | String[] dbColumn = getDbOptions().getDbColumn(); 83 | String tableName = getDbOptions().getTableName(); 84 | //表所有列 85 | Schema allSchema = getOracleSwapImport().desc(tableName); 86 | 87 | getLogger().info("mysql=>tableName:{}",tableName); 88 | return JdbcUtil.columnConversion(dbColumn, allSchema); 89 | } 90 | 91 | 92 | 93 | public static void save(PCollection> rows) { 94 | if(rows != null){ 95 | String tableName = getDbOptions().getTableName(); 96 | Schema schema = getSchema(); 97 | 98 | if(schema == null){ 99 | getLogger().info("schema is null"); 100 | } 101 | 102 | 103 | saveBySQL(rows,schema,tableName); 104 | } 105 | } 106 | 107 | 108 | public static class OracleImportDispatcher implements EventHandler { 109 | @Override 110 | public void handle(OracleImportTaskEvent event) { 111 | if(event.getType() == OracleImportType.T_IMPORT){ 112 | getLogger().info("OracleImportDispatcher is start"); 113 | 114 | if(CacheManager.isExist(DBOperationEnum.PCOLLECTION_QUERYS.getName())){ 115 | save((PCollection>)CacheManager.getCache(DBOperationEnum.PCOLLECTION_QUERYS.getName())); 116 | } 117 | 118 | } 119 | } 120 | } 121 | 122 | 123 | 124 | } 125 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/common/BaseSwap.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.common; 2 | 3 | 4 | 5 | import com.knowlegene.parent.process.io.es.ESSwap; 6 | import com.knowlegene.parent.process.io.es.impl.ESSwapImpl; 7 | import com.knowlegene.parent.process.io.jdbc.GbaseSwap; 8 | import com.knowlegene.parent.process.io.jdbc.HiveSwap; 9 | import com.knowlegene.parent.process.io.jdbc.MySQLSwap; 10 | import com.knowlegene.parent.process.io.jdbc.OracleSwap; 11 | import com.knowlegene.parent.process.io.jdbc.impl.*; 12 | import com.knowlegene.parent.process.io.neo4j.Neo4jSwap; 13 | import com.knowlegene.parent.process.io.neo4j.impl.Neo4jSwapExport; 14 | import com.knowlegene.parent.process.io.neo4j.impl.Neo4jSwapImport; 15 | import lombok.Data; 16 | 17 | 18 | import javax.annotation.Resource; 19 | import java.io.Serializable; 20 | 21 | /** 22 | * 公共交换方法 23 | * @Author: limeng 24 | * @Date: 2019/8/26 15:17 25 | */ 26 | @Data 27 | public class BaseSwap implements Serializable { 28 | @Resource 29 | private static HiveSwap hiveSwapImport; 30 | @Resource 31 | private static HiveSwap hiveSwapExport; 32 | 33 | 34 | @Resource 35 | private static MySQLSwap mySQLSwapImport; 36 | @Resource 37 | private static MySQLSwap mySQLSwapExport; 38 | 39 | 40 | @Resource 41 | private static OracleSwap oracleSwapImport; 42 | 43 | private static OracleSwap oracleSwapExport; 44 | 45 | 46 | @Resource 47 | private static GbaseSwap gbaseSwapImport; 48 | @Resource 49 | private static GbaseSwap gbaseSwapExport; 50 | 51 | 52 | @Resource 53 | private static ESSwap eSSwap; 54 | 55 | @Resource 56 | private static Neo4jSwap neo4jSwapImport; 57 | 58 | @Resource 59 | private static Neo4jSwap neo4jSwapExport; 60 | 61 | 62 | public BaseSwap() { 63 | hiveSwapImport = new HiveSwapImport(); 64 | hiveSwapExport = new HiveSwapExport(); 65 | 66 | mySQLSwapImport = new MySQLSwapImport(); 67 | mySQLSwapExport = new MySQLSwapExport(); 68 | 69 | oracleSwapImport = new OracleSwapImport(); 70 | oracleSwapExport = new OracleSwapExport(); 71 | 72 | gbaseSwapImport = new GbaseSwapImport(); 73 | gbaseSwapExport = new GbaseSwapExport(); 74 | 75 | eSSwap = new ESSwapImpl(); 76 | 77 | neo4jSwapImport = new Neo4jSwapImport(); 78 | neo4jSwapExport = new Neo4jSwapExport(); 79 | } 80 | 81 | protected static HiveSwap getHiveSwapImport(){ 82 | return hiveSwapImport; 83 | } 84 | 85 | protected static HiveSwap getHiveSwapExport(){ 86 | return hiveSwapExport; 87 | } 88 | 89 | protected static MySQLSwap getMySQLSwapImport(){ 90 | return mySQLSwapImport; 91 | } 92 | 93 | protected static MySQLSwap getMySQLSwapExport(){ 94 | return mySQLSwapExport; 95 | } 96 | 97 | 98 | protected static OracleSwap getOracleSwapImport() { 99 | return oracleSwapImport; 100 | } 101 | 102 | protected static OracleSwap getOracleSwapExport() { 103 | return oracleSwapExport; 104 | } 105 | 106 | protected static GbaseSwap getGbaseSwapImport() { 107 | return gbaseSwapImport; 108 | } 109 | 110 | protected static GbaseSwap getGbaseSwapExport() { 111 | return gbaseSwapExport; 112 | } 113 | 114 | 115 | protected static ESSwap getESSwap(){ 116 | return eSSwap; 117 | } 118 | 119 | 120 | 121 | protected static Neo4jSwap getNeo4jSwapImport() { 122 | return neo4jSwapImport; 123 | } 124 | 125 | protected static Neo4jSwap getNeo4jSwapExport() { 126 | return neo4jSwapExport; 127 | } 128 | } 129 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/dispatcher/SwapMaster.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.dispatcher; 2 | 3 | import com.knowlegene.parent.config.common.event.*; 4 | import com.knowlegene.parent.process.swap.*; 5 | import com.knowlegene.parent.process.util.SwapMasterUtil; 6 | import com.knowlegene.parent.scheduler.event.AsyncDispatcher; 7 | import com.knowlegene.parent.scheduler.event.Dispatcher; 8 | import com.knowlegene.parent.scheduler.service.CompositeService; 9 | import com.knowlegene.parent.scheduler.service.Service; 10 | 11 | /** 12 | * @Classname SwapMaster 13 | * @Description TODO 14 | * @Date 2020/6/11 15:58 15 | * @Created by limeng 16 | */ 17 | public class SwapMaster extends CompositeService { 18 | private Dispatcher dispatcher;//中央异步调度器 19 | 20 | private volatile boolean isAlive=true; 21 | 22 | public void setAlive(boolean alive) { 23 | isAlive = alive; 24 | } 25 | 26 | public boolean isAlive() { 27 | return isAlive; 28 | } 29 | 30 | public SwapMaster(String name) { 31 | super(name); 32 | SwapMasterUtil.getInstance(this); 33 | } 34 | 35 | @Override 36 | public void serviceInit() throws Exception { 37 | dispatcher = new AsyncDispatcher(); 38 | 39 | //注册Job和Task事件调度器 40 | dispatcher.register(SwapJobEventType.class,new JobBase.SwapJobEventDispatcher()); 41 | dispatcher.register(HiveExportType.class,new HiveExportJob.HiveExportDispatcher()); 42 | dispatcher.register(HiveImportType.class,new HiveImportJob.HiveImportDispatcher()); 43 | 44 | dispatcher.register(ESExportType.class,new ESExportJob.ESExportDispatcher()); 45 | dispatcher.register(ESImportType.class,new ESImportJob.ESImportDispatcher()); 46 | 47 | 48 | dispatcher.register(MySQLExportType.class,new MySQLExportJob.MySQLExportDispatcher()); 49 | dispatcher.register(MySQLImportType.class,new MySQLImportJob.MySQLImportDispatcher()); 50 | 51 | dispatcher.register(OracleExportType.class,new OracleExportJob.OracleExportDispatcher()); 52 | dispatcher.register(OracleImportType.class,new OracleImportJob.OracleImportDispatcher()); 53 | 54 | dispatcher.register(GbaseExportType.class,new GbaseExportJob.GbaseExportDispatcher()); 55 | dispatcher.register(GbaseImportType.class,new GbaseImportJob.GbaseImportDispatcher()); 56 | 57 | dispatcher.register(FileExportType.class,new FileExportJob.FileExportDispatcher()); 58 | dispatcher.register(FileImportType.class,new FileImportJob.FileImportDispatcher()); 59 | 60 | dispatcher.register(Neo4jExportType.class,new Neo4jExportJob.Neo4jExportDispatcher()); 61 | dispatcher.register(Neo4jImportType.class,new Neo4jImportJob.Neo4jImportDispatcher()); 62 | 63 | addService((Service)dispatcher); 64 | super.serviceInit(); 65 | } 66 | 67 | @Override 68 | public void serviceStart() throws Exception { 69 | super.serviceStart(); 70 | } 71 | 72 | @Override 73 | public void serviceStop() throws Exception { 74 | super.serviceStop(); 75 | } 76 | 77 | public Dispatcher getDispatcher() { 78 | return dispatcher; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/ESExportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | import com.knowlegene.parent.config.common.event.ESExportType; 4 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 5 | 6 | /** 7 | * @Classname 8 | * @Description TODO 9 | * @Date 2020/6/11 15:53 10 | * @Created by limeng 11 | */ 12 | public class ESExportTaskEvent extends AbstractEvent { 13 | 14 | 15 | public ESExportTaskEvent(ESExportType exportEnum) { 16 | super(exportEnum); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/ESImportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | 4 | import com.knowlegene.parent.config.common.event.ESImportType; 5 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 6 | 7 | /** 8 | * @Classname 9 | * @Description TODO 10 | * @Date 2020/6/11 15:29 11 | * @Created by limeng 12 | */ 13 | public class ESImportTaskEvent extends AbstractEvent { 14 | 15 | public ESImportTaskEvent(ESImportType importEnum) { 16 | super(importEnum); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/FileExportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | import com.knowlegene.parent.config.common.event.ESExportType; 4 | import com.knowlegene.parent.config.common.event.FileExportType; 5 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 6 | 7 | /** 8 | * @Classname 9 | * @Description TODO 10 | * @Date 2020/6/11 15:53 11 | * @Created by limeng 12 | */ 13 | public class FileExportTaskEvent extends AbstractEvent { 14 | 15 | 16 | public FileExportTaskEvent(FileExportType exportEnum) { 17 | super(exportEnum); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/FileImportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | 4 | 5 | import com.knowlegene.parent.config.common.event.FileImportType; 6 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 7 | 8 | /** 9 | * @Classname 10 | * @Description TODO 11 | * @Date 2020/6/11 15:29 12 | * @Created by limeng 13 | */ 14 | public class FileImportTaskEvent extends AbstractEvent { 15 | 16 | public FileImportTaskEvent(FileImportType importEnum) { 17 | super(importEnum); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/GbaseExportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | import com.knowlegene.parent.config.common.event.GbaseExportType; 4 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 5 | 6 | /** 7 | * @Classname 8 | * @Description TODO 9 | * @Date 2020/6/11 15:53 10 | * @Created by limeng 11 | */ 12 | public class GbaseExportTaskEvent extends AbstractEvent { 13 | 14 | 15 | public GbaseExportTaskEvent(GbaseExportType exportEnum) { 16 | super(exportEnum); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/GbaseImportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | 4 | import com.knowlegene.parent.config.common.event.GbaseImportType; 5 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 6 | 7 | /** 8 | * @Classname 9 | * @Description TODO 10 | * @Date 2020/6/11 15:29 11 | * @Created by limeng 12 | */ 13 | public class GbaseImportTaskEvent extends AbstractEvent { 14 | 15 | public GbaseImportTaskEvent(GbaseImportType importEnum) { 16 | super(importEnum); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/HiveExportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | import com.knowlegene.parent.config.common.event.HiveExportType; 4 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 5 | /** 6 | * @Classname HiveExportTaskEvent 7 | * @Description TODO 8 | * @Date 2020/6/11 15:53 9 | * @Created by limeng 10 | */ 11 | public class HiveExportTaskEvent extends AbstractEvent { 12 | 13 | 14 | public HiveExportTaskEvent(HiveExportType hiveExportEnum) { 15 | super(hiveExportEnum); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/HiveImportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | 4 | import com.knowlegene.parent.config.common.event.HiveImportType; 5 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 6 | 7 | /** 8 | * @Classname HiveTaskEvent 9 | * @Description TODO 10 | * @Date 2020/6/11 15:29 11 | * @Created by limeng 12 | */ 13 | public class HiveImportTaskEvent extends AbstractEvent { 14 | 15 | public HiveImportTaskEvent(HiveImportType hiveImportEnum) { 16 | super(hiveImportEnum); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/MySQLExportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | import com.knowlegene.parent.config.common.event.MySQLExportType; 4 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 5 | 6 | /** 7 | * @Classname MySQLExportTaskEvent 8 | * @Description TODO 9 | * @Date 2020/6/11 15:53 10 | * @Created by limeng 11 | */ 12 | public class MySQLExportTaskEvent extends AbstractEvent { 13 | 14 | 15 | public MySQLExportTaskEvent(MySQLExportType exportEnum) { 16 | super(exportEnum); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/MySQLImportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | 4 | import com.knowlegene.parent.config.common.event.MySQLImportType; 5 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 6 | 7 | /** 8 | * @Classname MySQLImportTaskEvent 9 | * @Description TODO 10 | * @Date 2020/6/11 15:29 11 | * @Created by limeng 12 | */ 13 | public class MySQLImportTaskEvent extends AbstractEvent { 14 | 15 | public MySQLImportTaskEvent(MySQLImportType importEnum) { 16 | super(importEnum); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/Neo4jExportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | import com.knowlegene.parent.config.common.event.Neo4jExportType; 4 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 5 | 6 | /** 7 | * @Classname 8 | * @Description TODO 9 | * @Date 2020/6/11 15:53 10 | * @Created by limeng 11 | */ 12 | public class Neo4jExportTaskEvent extends AbstractEvent { 13 | 14 | 15 | public Neo4jExportTaskEvent(Neo4jExportType exportEnum) { 16 | super(exportEnum); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/Neo4jImportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | 4 | import com.knowlegene.parent.config.common.event.Neo4jImportType; 5 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 6 | 7 | /** 8 | * @Classname 9 | * @Description TODO 10 | * @Date 2020/6/11 15:29 11 | * @Created by limeng 12 | */ 13 | public class Neo4jImportTaskEvent extends AbstractEvent { 14 | 15 | public Neo4jImportTaskEvent(Neo4jImportType importEnum) { 16 | super(importEnum); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/OracleExportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | import com.knowlegene.parent.config.common.event.OracleExportType; 4 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 5 | 6 | /** 7 | * @Classname 8 | * @Description TODO 9 | * @Date 2020/6/11 15:53 10 | * @Created by limeng 11 | */ 12 | public class OracleExportTaskEvent extends AbstractEvent { 13 | 14 | 15 | public OracleExportTaskEvent(OracleExportType exportEnum) { 16 | super(exportEnum); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/OracleImportTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | 4 | import com.knowlegene.parent.config.common.event.OracleImportType; 5 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 6 | 7 | /** 8 | * @Classname 9 | * @Description TODO 10 | * @Date 2020/6/11 15:29 11 | * @Created by limeng 12 | */ 13 | public class OracleImportTaskEvent extends AbstractEvent { 14 | 15 | public OracleImportTaskEvent(OracleImportType importEnum) { 16 | super(importEnum); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/swap/event/SwapJobEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.swap.event; 2 | 3 | import com.knowlegene.parent.config.common.event.SwapJobEventType; 4 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 5 | 6 | /** 7 | * @Classname SwapJobEvent 8 | * @Description TODO 9 | * @Date 2020/6/15 16:23 10 | * @Created by limeng 11 | */ 12 | public class SwapJobEvent extends AbstractEvent{ 13 | public SwapJobEvent(SwapJobEventType swapJobEventType) { 14 | super(swapJobEventType); 15 | } 16 | 17 | public SwapJobEvent(SwapJobEventType swapJobEventType, long timestamp) { 18 | super(swapJobEventType, timestamp); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/transform/CombineTransform.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.transform; 2 | 3 | import com.knowlegene.parent.process.pojo.ObjectCoder; 4 | import org.apache.beam.sdk.transforms.Combine; 5 | import org.apache.beam.sdk.values.Row; 6 | 7 | import java.util.HashSet; 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | /** 12 | * 合并 13 | * @Author: limeng 14 | * @Date: 2019/9/17 16:17 15 | */ 16 | public class CombineTransform { 17 | public static class UniqueMapSets extends Combine.CombineFn, Set>, Set>>{ 18 | 19 | @Override 20 | public Set> createAccumulator() { 21 | return new HashSet<>(); 22 | } 23 | 24 | @Override 25 | public Set> addInput(Set> accumulator, Map input) { 26 | if(input != null){ 27 | accumulator.add(input); 28 | } 29 | return accumulator; 30 | } 31 | 32 | @Override 33 | public Set> mergeAccumulators(Iterable>> accumulators) { 34 | Set> all = new HashSet<>(); 35 | for (Set> part : accumulators) { 36 | all.addAll(part); 37 | } 38 | return all; 39 | } 40 | 41 | @Override 42 | public Set> extractOutput(Set> accumulator) { 43 | return accumulator; 44 | } 45 | } 46 | 47 | public static class UniqueSets extends Combine.CombineFn, Set>{ 48 | @Override 49 | public Set createAccumulator() { 50 | return new HashSet<>(); 51 | } 52 | 53 | @Override 54 | public Set addInput(Set accumulator, Row input) { 55 | if(input != null){ 56 | accumulator.add(input); 57 | } 58 | return accumulator; 59 | } 60 | 61 | @Override 62 | public Set mergeAccumulators(Iterable> accumulators) { 63 | Set all = new HashSet<>(); 64 | for (Set part : accumulators) { 65 | all.addAll(part); 66 | } 67 | return all; 68 | } 69 | 70 | @Override 71 | public Set extractOutput(Set accumulator) { 72 | return accumulator; 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/transform/DistinctTransform.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.transform; 2 | 3 | 4 | import org.apache.beam.sdk.transforms.Distinct; 5 | import org.apache.beam.sdk.transforms.PTransform; 6 | import org.apache.beam.sdk.transforms.SerializableFunction; 7 | import org.apache.beam.sdk.values.PCollection; 8 | import org.apache.beam.sdk.values.TypeDescriptor; 9 | import org.apache.hive.hcatalog.common.HCatException; 10 | import org.apache.hive.hcatalog.data.HCatRecord; 11 | import org.apache.hive.hcatalog.data.schema.HCatSchema; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import javax.annotation.Nullable; 16 | 17 | /** 18 | * 去重自定义 19 | * @Author: limeng 20 | * @Date: 2019/9/2 20:13 21 | */ 22 | public class DistinctTransform { 23 | static class DistinctHCatRecord extends PTransform,PCollection>{ 24 | private final HCatSchema hCatSchema; 25 | private final String version; 26 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 27 | public DistinctHCatRecord( HCatSchema hCatSchema, String version) { 28 | this.hCatSchema = hCatSchema; 29 | this.version = version; 30 | } 31 | 32 | @Override 33 | public PCollection expand(PCollection input) { 34 | //去重 35 | PCollection result = input.apply(Distinct.withRepresentativeValueFn(new SerializableFunction() { 36 | @Override 37 | public String apply(HCatRecord input) { 38 | try { 39 | return input.get(version,hCatSchema).toString(); 40 | } catch (HCatException e) { 41 | logger.error("distinct is error =>msg:{}",e.getMessage()); 42 | return null; 43 | } 44 | } 45 | }).withRepresentativeType(TypeDescriptor.of(String.class))); 46 | 47 | return result; 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/transform/ESTransform.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.transform; 2 | 3 | import com.fasterxml.jackson.databind.JsonNode; 4 | import com.knowlegene.parent.process.pojo.ObjectCoder; 5 | import org.apache.beam.sdk.io.elasticsearch.ElasticsearchIO; 6 | import org.apache.beam.sdk.schemas.Schema; 7 | import org.apache.beam.sdk.schemas.SchemaCoder; 8 | import org.apache.beam.sdk.transforms.Combine; 9 | import org.apache.beam.sdk.transforms.PTransform; 10 | import org.apache.beam.sdk.transforms.ParDo; 11 | import org.apache.beam.sdk.values.KV; 12 | import org.apache.beam.sdk.values.PCollection; 13 | import org.apache.beam.sdk.values.PCollectionList; 14 | import org.apache.beam.sdk.values.Row; 15 | import org.slf4j.Logger; 16 | import org.slf4j.LoggerFactory; 17 | 18 | import javax.annotation.Nullable; 19 | import java.util.Arrays; 20 | import java.util.List; 21 | import java.util.Map; 22 | 23 | /** 24 | * @Author: limeng 25 | * @Date: 2019/9/12 15:53 26 | */ 27 | public class ESTransform { 28 | 29 | /** 30 | * es提前JsonNode 31 | */ 32 | public static class ExtractValueFn implements ElasticsearchIO.Write.FieldValueExtractFn{ 33 | private final String fieldName; 34 | 35 | public ExtractValueFn(String fieldName) { 36 | this.fieldName = fieldName; 37 | } 38 | @Override 39 | public String apply(JsonNode input) { 40 | return input.path(fieldName).asText(); 41 | } 42 | } 43 | 44 | public static class NestingFieldTransformMap extends PTransform>, PCollection>>{ 45 | Logger logger = LoggerFactory.getLogger(this.getClass()); 46 | private final List keys; 47 | //嵌套字段名称 48 | private final KV> nestings; 49 | //嵌套字段名称 50 | private final String keysName; 51 | 52 | public NestingFieldTransformMap(List keys, KV> nestings,String keysName) { 53 | this.keys = keys; 54 | this.nestings = nestings; 55 | this.keysName = keysName; 56 | } 57 | 58 | 59 | @Override 60 | public PCollection> expand(PCollection> input) { 61 | if(input == null){ 62 | return null; 63 | } 64 | 65 | return input.apply(ParDo.of(new FilterTransform.FilterByKeysMap(keys))) 66 | .apply(Combine.perKey(new CombineTransform.UniqueMapSets())) 67 | .apply(ParDo.of(new FilterTransform.FilterKeysAndMapJson(nestings,keysName))); 68 | 69 | } 70 | } 71 | 72 | public static class NestingFieldTransform extends PTransform, PCollection>{ 73 | Logger logger = LoggerFactory.getLogger(this.getClass()); 74 | private final List keys; 75 | private final Schema type; 76 | //嵌套字段名称 77 | private final KV> nestings; 78 | 79 | public NestingFieldTransform(List keys, Schema type, KV> nestings) { 80 | this.keys = keys; 81 | this.type = type; 82 | this.nestings = nestings; 83 | } 84 | 85 | public NestingFieldTransform(@Nullable String name, List keys, Schema type, KV> nestings) { 86 | super(name); 87 | this.keys = keys; 88 | this.type = type; 89 | this.nestings = nestings; 90 | } 91 | 92 | @Override 93 | public PCollection expand(PCollection input) { 94 | PCollection merages = null; 95 | if(input == null){ 96 | return merages; 97 | } 98 | 99 | merages = input.apply(ParDo.of(new FilterTransform.FilterByKeys(keys))) 100 | .apply(Combine.perKey(new CombineTransform.UniqueSets())) 101 | .apply(ParDo.of(new FilterTransform.FilterKeysAndJson(type, nestings))) 102 | .setCoder(SchemaCoder.of(type)); 103 | 104 | return merages; 105 | } 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/transform/PrintTransform.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.transform; 2 | 3 | import org.apache.beam.sdk.transforms.DoFn; 4 | import org.apache.beam.sdk.values.Row; 5 | import org.apache.hive.hcatalog.data.HCatRecord; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.List; 10 | 11 | /** 12 | * @Author: limeng 13 | * @Date: 2019/9/2 18:45 14 | */ 15 | public class PrintTransform { 16 | public static class Print extends DoFn { 17 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 18 | @ProcessElement 19 | public void processElement( ProcessContext c) { 20 | Row element = c.element(); 21 | List values = element.getValues(); 22 | StringBuilder stringBuilder = new StringBuilder(); 23 | for (Object o:values){ 24 | stringBuilder.append(o.toString()+" "); 25 | } 26 | logger.info(stringBuilder.toString()); 27 | c.output(element); 28 | } 29 | } 30 | 31 | public static class PrintByHCatIO extends DoFn { 32 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 33 | @ProcessElement 34 | public void processElement( ProcessContext c) { 35 | HCatRecord element = c.element(); 36 | List all = element.getAll(); 37 | StringBuilder stringBuilder = new StringBuilder(); 38 | for (Object o:all){ 39 | stringBuilder.append(o.toString()+" "); 40 | } 41 | logger.info(stringBuilder.toString()); 42 | c.output(element); 43 | } 44 | } 45 | 46 | public static class PrintByString extends DoFn { 47 | private Logger logger = LoggerFactory.getLogger(this.getClass()); 48 | @ProcessElement 49 | public void processElement( ProcessContext c) { 50 | String element = c.element(); 51 | logger.info(element); 52 | c.output(element); 53 | } 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/transform/common/BaseTransform.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.transform.common; 2 | 3 | 4 | 5 | 6 | 7 | /** 8 | * @Author: limeng 9 | * @Date: 2019/8/28 17:24 10 | */ 11 | 12 | public class BaseTransform { 13 | 14 | 15 | } 16 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/util/ComboPooledDataSourceUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.util; 2 | 3 | import com.mchange.v2.c3p0.ComboPooledDataSource; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import javax.sql.DataSource; 8 | import java.beans.PropertyVetoException; 9 | import java.io.Serializable; 10 | 11 | /** 12 | * @Author: limeng 13 | * @Date: 2019/8/21 17:55 14 | */ 15 | public class ComboPooledDataSourceUtil implements Serializable { 16 | private static Logger logger = LoggerFactory.getLogger(ComboPooledDataSourceUtil.class); 17 | 18 | public static DataSource setDataSource(String className,String user,String password,String url){ 19 | ComboPooledDataSource cpds = new ComboPooledDataSource(); 20 | cpds.setUser(user); 21 | try { 22 | cpds.setDriverClass(className); 23 | cpds.setPassword(password); 24 | cpds.setJdbcUrl(url); 25 | cpds.setMaxPoolSize(100); 26 | cpds.setMinPoolSize(5); 27 | cpds.setAcquireIncrement(5); 28 | cpds.setMaxIdleTime(600); 29 | //连接丢失 每10秒自动检测 30 | cpds.setTestConnectionOnCheckin(true); 31 | cpds.setPreferredTestQuery("SELECT 1"); 32 | cpds.setIdleConnectionTestPeriod(10); 33 | return cpds; 34 | } catch (PropertyVetoException e) { 35 | logger.error("dataSource error=>msg:{}", e.getMessage()); 36 | return null; 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/util/JobThreadLocalUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.util; 2 | 3 | /** 4 | * 设置jobid 5 | * @Author: limeng 6 | * @Date: 2019/8/13 17:36 7 | */ 8 | public class JobThreadLocalUtil { 9 | private static final ThreadLocal CONTEXT_HOLDER = new ThreadLocal(); 10 | 11 | /** 12 | * @Description: id 13 | * @return void 14 | */ 15 | public static void setJob(String type) { 16 | CONTEXT_HOLDER.set(type); 17 | } 18 | 19 | /** 20 | * @Description: id 21 | */ 22 | public static String getJob() { 23 | return CONTEXT_HOLDER.get(); 24 | } 25 | 26 | /** 27 | * @Description: job 28 | */ 29 | public static void clearJob() { 30 | CONTEXT_HOLDER.remove(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/util/SwapMasterUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.util; 2 | 3 | import com.knowlegene.parent.process.swap.dispatcher.SwapMaster; 4 | import com.knowlegene.parent.scheduler.event.Dispatcher; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @Classname DispatcherUtil 10 | * @Description TODO 11 | * @Date 2020/6/11 16:28 12 | * @Created by limeng 13 | */ 14 | public class SwapMasterUtil implements Serializable { 15 | 16 | private SwapMasterUtil() { 17 | } 18 | 19 | public static volatile SwapMaster instance = null; 20 | 21 | public static SwapMaster getInstance(SwapMaster newPipeline){ 22 | if(instance == null){ 23 | synchronized (Dispatcher.class){ 24 | if(instance == null && newPipeline != null){ 25 | instance = newPipeline; 26 | } 27 | } 28 | } 29 | return instance; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /client/kd-process/src/main/java/com/knowlegene/parent/process/util/UUIDFactoryUtil.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.util; 2 | 3 | import java.text.ParseException; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Calendar; 6 | import java.util.UUID; 7 | 8 | /** 9 | * uuid 10 | * @Author: limeng 11 | * @Date: 2019/8/13 17:37 12 | */ 13 | public class UUIDFactoryUtil { 14 | public UUIDFactoryUtil() { 15 | } 16 | 17 | public static String getUUIDStr() { 18 | UUID uuid = UUID.randomUUID(); 19 | String s = uuid.toString(); 20 | s = s.replaceAll("[-]", ""); 21 | return s.toUpperCase(); 22 | } 23 | 24 | public static String getNumberUUIDStr() { 25 | Calendar c = Calendar.getInstance(); 26 | Calendar date = Calendar.getInstance(); 27 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); 28 | try { 29 | date.setTime(sdf.parse("2013-01-01")); 30 | } catch (ParseException e) { 31 | e.printStackTrace(); 32 | } 33 | Long seconds = c.getTimeInMillis() - date.getTimeInMillis(); 34 | return seconds + ""; 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /client/kd-process/src/test/java/com/knowlegene/parent/process/kafka/KafkaConsumerTest.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.kafka; 2 | 3 | import com.google.common.collect.ImmutableMap; 4 | import org.apache.beam.sdk.Pipeline; 5 | import org.apache.beam.sdk.io.TextIO; 6 | import org.apache.beam.sdk.io.kafka.KafkaIO; 7 | import org.apache.beam.sdk.options.PipelineOptions; 8 | import org.apache.beam.sdk.options.PipelineOptionsFactory; 9 | import org.apache.beam.sdk.transforms.*; 10 | import org.apache.beam.sdk.values.KV; 11 | import org.apache.kafka.common.serialization.LongDeserializer; 12 | import org.apache.kafka.common.serialization.StringDeserializer; 13 | 14 | /** 15 | * @Author: limeng 16 | * @Date: 2019/7/15 11:56 17 | */ 18 | public class KafkaConsumerTest { 19 | static final String TOKENIZER_PATTERN = "[^\\p{L}]+"; 20 | 21 | public static void main(String[] args) { 22 | PipelineOptions options = PipelineOptionsFactory.create(); 23 | 24 | 25 | Pipeline p = Pipeline.create(options); 26 | 27 | p.apply(KafkaIO.read() 28 | .withBootstrapServers("kafka:29092") 29 | .withTopic("words") 30 | .withKeyDeserializer(LongDeserializer.class) 31 | .withValueDeserializer(StringDeserializer.class) 32 | 33 | .updateConsumerProperties(ImmutableMap.of("auto.offset.reset", (Object)"earliest")) 34 | 35 | .withMaxNumRecords(5) 36 | 37 | .withoutMetadata() // PCollection> 38 | ) 39 | .apply(Values.create()) 40 | .apply("ExtractWords", ParDo.of(new DoFn() { 41 | @ProcessElement 42 | public void processElement(ProcessContext c) { 43 | for (String word : c.element().split(TOKENIZER_PATTERN)) { 44 | if (!word.isEmpty()) { 45 | c.output(word); 46 | } 47 | } 48 | } 49 | })) 50 | .apply(Count.perElement()) 51 | .apply("FormatResults", MapElements.via(new SimpleFunction, String>() { 52 | @Override 53 | public String apply(KV input) { 54 | return input.getKey() + ": " + input.getValue(); 55 | } 56 | })) 57 | .apply(TextIO.write().to("wordcounts")); 58 | 59 | p.run().waitUntilFinish(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /client/kd-process/src/test/java/com/knowlegene/parent/process/kafka/KafkaProducerTest.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.process.kafka; 2 | 3 | import org.apache.beam.sdk.Pipeline; 4 | import org.apache.beam.sdk.io.kafka.KafkaIO; 5 | import org.apache.beam.sdk.options.PipelineOptions; 6 | import org.apache.beam.sdk.options.PipelineOptionsFactory; 7 | import org.apache.beam.sdk.transforms.Create; 8 | import org.apache.beam.sdk.values.KV; 9 | import org.apache.beam.sdk.values.PCollection; 10 | import org.apache.kafka.common.serialization.LongSerializer; 11 | import org.apache.kafka.common.serialization.StringSerializer; 12 | 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | 16 | /** 17 | * @Author: limeng 18 | * @Date: 2019/7/15 11:57 19 | */ 20 | public class KafkaProducerTest { 21 | public static void main(String[] args) { 22 | PipelineOptions options = PipelineOptionsFactory.create(); 23 | Pipeline p = Pipeline.create(options); 24 | 25 | // sample data 26 | List> kvs = new ArrayList<>(); 27 | kvs.add(KV.of(1L, "hi there")); 28 | kvs.add(KV.of(2L, "hi")); 29 | kvs.add(KV.of(3L, "hi sue bob")); 30 | kvs.add(KV.of(4L, "hi sue")); 31 | kvs.add(KV.of(5L, "hi bob")); 32 | 33 | PCollection> input = p 34 | .apply(Create.of(kvs)); 35 | 36 | input.apply(KafkaIO.write() 37 | .withBootstrapServers("kafka:29092") 38 | .withTopic("words") 39 | 40 | .withKeySerializer(LongSerializer.class) 41 | .withValueSerializer(StringSerializer.class) 42 | ); 43 | 44 | p.run().waitUntilFinish(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /core/scheduler/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | exchange-parent 7 | com.knowlegene.parent 8 | 0.1.0 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | 13 | com.knowlegene.parent.scheduler 14 | scheduler 15 | ${kd.project.version} 16 | 17 | 18 | UTF-8 19 | UTF-8 20 | 1.8 21 | 22 | 23 | 24 | 25 | 26 | 27 | org.apache.maven.plugins 28 | maven-compiler-plugin 29 | 30 | 1.8 31 | 1.8 32 | UTF-8 33 | true 34 | 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/event/AbstractEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.event; 2 | 3 | 4 | 5 | /** 6 | * @Classname AbstractEvent 7 | * @Description TODO 8 | * @Date 2020/6/5 15:46 9 | * @Created by limeng 10 | */ 11 | public abstract class AbstractEvent> implements Event { 12 | 13 | 14 | private final TYPE type; 15 | 16 | private final long timestamp; 17 | 18 | public AbstractEvent(TYPE type) { 19 | this.type = type; 20 | this.timestamp = 1L; 21 | } 22 | 23 | public AbstractEvent(TYPE type,long timestamp){ 24 | this.type = type; 25 | this.timestamp = timestamp; 26 | } 27 | 28 | 29 | @Override 30 | public long getTimestamp() { 31 | return timestamp; 32 | } 33 | 34 | @Override 35 | public TYPE getType() { 36 | return type; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return "EventType: " + getType(); 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/event/Dispatcher.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.event; 2 | 3 | /** 4 | * @Classname Dispatcher 5 | * @Description TODO 6 | * @Date 2020/6/5 15:45 7 | * @Created by limeng 8 | */ 9 | public interface Dispatcher { 10 | 11 | public static final boolean DEFAULT_DISPATCHER_EXIT_ON_ERROR = false; 12 | 13 | EventHandler getEventHandler(); 14 | 15 | void register(Class eventType, EventHandler handler); 16 | 17 | } 18 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/event/Event.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.event; 2 | 3 | /** 4 | * @Classname Event 5 | * @Description TODO 6 | * @Date 2020/6/5 15:43 7 | * @Created by limeng 8 | */ 9 | public interface Event> { 10 | 11 | TYPE getType(); 12 | long getTimestamp(); 13 | String toString(); 14 | } 15 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/event/EventHandler.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.event; 2 | 3 | /** 4 | * @Classname EventHandler 5 | * @Description TODO 6 | * @Date 2020/6/5 15:44 7 | * @Created by limeng 8 | */ 9 | public interface EventHandler { 10 | void handle(T event); 11 | } 12 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/pojo/CacheEntry.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.pojo; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @Classname CacheEntry 7 | * @Description TODO 8 | * @Date 2020/6/9 15:40 9 | * @Created by limeng 10 | */ 11 | @Data 12 | public class CacheEntry { 13 | private Object cacheValue; 14 | private Long ttlTime; 15 | 16 | private long defaultTime = 1000L; 17 | 18 | public CacheEntry(Object cacheValue) { 19 | this.cacheValue = cacheValue; 20 | this.ttlTime = defaultTime; 21 | } 22 | 23 | public CacheEntry(Object cacheValue, Long ttlTime) { 24 | this.cacheValue = cacheValue; 25 | this.ttlTime = ttlTime; 26 | } 27 | 28 | public CacheEntry() { 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/service/CompositeService.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | /** 10 | * @Classname CompositeService 11 | * @Description TODO 12 | * @Date 2020/6/4 19:38 13 | * @Created by limeng 14 | */ 15 | public class CompositeService extends AbstractService { 16 | 17 | private static final Log LOG = LogFactory.getLog(CompositeService.class); 18 | 19 | public CompositeService(String name) { 20 | super(name); 21 | } 22 | 23 | protected static final boolean STOP_ONLY_STARTED_SERVICES = false; 24 | 25 | private final List serviceList = new ArrayList<>(); 26 | 27 | 28 | public List getServices(){ 29 | synchronized (serviceList){ 30 | return new ArrayList<>(serviceList); 31 | } 32 | } 33 | 34 | protected void addService(Service service){ 35 | if(LOG.isDebugEnabled()){ 36 | LOG.debug("Adding service " + service.getName()); 37 | } 38 | 39 | synchronized (serviceList){ 40 | serviceList.add(service); 41 | } 42 | } 43 | 44 | protected boolean addIfService(Object object){ 45 | if(object instanceof Service){ 46 | addService((Service) object); 47 | return true; 48 | }else{ 49 | return false; 50 | } 51 | } 52 | 53 | protected synchronized boolean removeService(Service service){ 54 | synchronized (serviceList){ 55 | return serviceList.remove(service); 56 | } 57 | } 58 | 59 | @Override 60 | protected void serviceStart() throws Exception { 61 | List services = getServices(); 62 | if(LOG.isDebugEnabled()){ 63 | LOG.debug(getName() + ": starting services, size=" + services.size()); 64 | } 65 | for (Service service : services){ 66 | service.start(); 67 | } 68 | super.serviceStart(); 69 | } 70 | 71 | @Override 72 | protected void serviceStop() throws Exception { 73 | int numOfServicesToStop = serviceList.size(); 74 | if(LOG.isDebugEnabled()){ 75 | LOG.debug(getName() + ": stopping services, size=" + numOfServicesToStop); 76 | } 77 | stop(numOfServicesToStop,STOP_ONLY_STARTED_SERVICES); 78 | super.serviceStop(); 79 | } 80 | 81 | 82 | 83 | private void stop(int numOfServicesStarted, boolean stopOnlyStartedServices) { 84 | Exception firstException = null; 85 | List services = getServices(); 86 | for (int i = numOfServicesStarted -1 ; i >= 0 ; i--) { 87 | Service service = services.get(i); 88 | if(LOG.isDebugEnabled()){ 89 | LOG.debug("Stopping service #" + i + ": " + service); 90 | } 91 | 92 | STATE state = service.getServiceState(); 93 | 94 | if(state == STATE.STARTED || (!stopOnlyStartedServices && state == STATE.INITED)){ 95 | Exception ex = ServiceOperations.stopQuietly(LOG, service); 96 | if(ex != null && firstException == null){ 97 | firstException = ex; 98 | } 99 | } 100 | } 101 | 102 | if(firstException != null){ 103 | throw ServiceStateException.convert(firstException); 104 | } 105 | } 106 | 107 | @Override 108 | protected void serviceInit() throws Exception { 109 | List services = getServices(); 110 | if (LOG.isDebugEnabled()) { 111 | LOG.debug(getName() + ": initing services, size=" + services.size()); 112 | } 113 | for (Service service : services) { 114 | service.init(); 115 | } 116 | super.serviceInit(); 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/service/Service.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | 6 | /** 7 | * @Classname Service 8 | * @Description TODO 9 | * @Date 2020/6/3 13:38 10 | * @Created by limeng 11 | */ 12 | public interface Service extends Closeable { 13 | 14 | /** 15 | * Service states 16 | */ 17 | public enum STATE { 18 | 19 | NOTINITED(0, "NOTINITED"), 20 | 21 | INITED(1, "INITED"), 22 | 23 | 24 | STARTED(2, "STARTED"), 25 | 26 | 27 | STOPPED(3, "STOPPED"); 28 | 29 | 30 | private final int value; 31 | 32 | private final String statename; 33 | 34 | private STATE(int value, String name) { 35 | this.value = value; 36 | this.statename = name; 37 | } 38 | 39 | public int getValue() { 40 | return value; 41 | } 42 | 43 | 44 | @Override 45 | public String toString() { 46 | return statename; 47 | } 48 | } 49 | 50 | 51 | void start(); 52 | 53 | void stop(); 54 | 55 | @Override 56 | void close() throws IOException; 57 | 58 | String getName(); 59 | 60 | STATE getServiceState(); 61 | 62 | long getStartTime(); 63 | 64 | boolean isInState(STATE state); 65 | 66 | Throwable getFailureCause(); 67 | 68 | STATE getFailureState(); 69 | 70 | boolean waitForServiceToStop(long timeout); 71 | 72 | void init(); 73 | } 74 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/service/ServiceOperations.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | 6 | /** 7 | * @Classname ServiceOperations 8 | * @Description TODO 9 | * @Date 2020/6/5 15:27 10 | * @Created by limeng 11 | */ 12 | public final class ServiceOperations { 13 | private static final Log LOG = LogFactory.getLog(ServiceOperations.class); 14 | 15 | private ServiceOperations() { 16 | } 17 | 18 | 19 | public static void stop(Service service) { 20 | if (service != null) { 21 | service.stop(); 22 | } 23 | } 24 | 25 | public static Exception stopQuietly(Log log, Service service) { 26 | try { 27 | stop(service); 28 | } catch (Exception e) { 29 | log.warn("When stopping the service " + service.getName() 30 | + " : " + e, 31 | e); 32 | return e; 33 | } 34 | return null; 35 | } 36 | 37 | 38 | public static Exception stopQuietly(Service service) { 39 | return stopQuietly(LOG, service); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/service/ServiceStateException.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | /** 4 | * @Classname ServiceStateException 5 | * @Description TODO 6 | * @Date 2020/6/3 14:15 7 | * @Created by limeng 8 | */ 9 | public class ServiceStateException extends RuntimeException { 10 | 11 | private static final long serialVersionUID = -4819737863134593472L; 12 | 13 | public ServiceStateException(String message) { 14 | super(message); 15 | } 16 | 17 | public ServiceStateException(String message, Throwable cause) { 18 | super(message, cause); 19 | } 20 | 21 | public ServiceStateException(Throwable cause) { 22 | super(cause); 23 | } 24 | 25 | public static RuntimeException convert(Throwable fault) { 26 | if (fault instanceof RuntimeException) { 27 | return (RuntimeException) fault; 28 | } else { 29 | return new ServiceStateException(fault); 30 | } 31 | } 32 | 33 | public static RuntimeException convert(String text, Throwable fault) { 34 | if (fault instanceof RuntimeException) { 35 | return (RuntimeException) fault; 36 | } else { 37 | return new ServiceStateException(text, fault); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /core/scheduler/src/main/java/com/knowlegene/parent/scheduler/service/ServiceStateModel.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | /** 4 | * @Classname ServiceStateModel 5 | * @Description TODO 6 | * @Date 2020/6/3 14:07 7 | * @Created by limeng 8 | * 状态机 9 | */ 10 | public class ServiceStateModel { 11 | 12 | private static final boolean[][] statemap= 13 | { 14 | // uninited inited started stopped 15 | /* uninited */ {false, true, false, true}, 16 | /* inited */ {false, true, true, true}, 17 | /* started */ {false, false, true, true}, 18 | /* stopped */ {false, false, false, true}, 19 | }; 20 | 21 | private volatile Service.STATE state; 22 | 23 | private String name; 24 | 25 | public ServiceStateModel(String name){ 26 | this(name,Service.STATE.NOTINITED); 27 | } 28 | 29 | public ServiceStateModel(String name,Service.STATE state) { 30 | this.state = state; 31 | this.name = name; 32 | } 33 | 34 | public Service.STATE getState(){ 35 | return state; 36 | } 37 | 38 | public boolean isInState(Service.STATE proposed){ 39 | return state.equals(proposed); 40 | } 41 | 42 | public void ensureCurrentState(Service.STATE expectedState){ 43 | if(state != expectedState){ 44 | throw new ServiceStateException(name+ ": for this operation, the " + 45 | "current service state must be " 46 | + expectedState 47 | + " instead of " + state); 48 | } 49 | } 50 | 51 | public static boolean isValidStateTransition(Service.STATE current, 52 | Service.STATE proposed){ 53 | boolean[] row = statemap[current.getValue()]; 54 | return row[proposed.getValue()]; 55 | } 56 | 57 | public static void checkStateTransition(String name, 58 | Service.STATE state, 59 | Service.STATE proposed){ 60 | if(!isValidStateTransition(state,proposed)){ 61 | throw new ServiceStateException(name + " cannot enter state " 62 | + proposed + " from state " + state); 63 | } 64 | } 65 | 66 | 67 | public synchronized Service.STATE enterState(Service.STATE proposed){ 68 | checkStateTransition(name,state,proposed); 69 | Service.STATE oldState = state; 70 | 71 | state = proposed; 72 | return oldState; 73 | } 74 | 75 | 76 | 77 | @Override 78 | public String toString() { 79 | return (name.isEmpty() ? "" : ((name) + ": ")) 80 | + state.toString(); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /core/scheduler/src/test/java/com/knowlegene/parent/scheduler/service/TestCacheManager.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | import com.knowlegene.parent.scheduler.utils.CacheManager; 4 | 5 | /** 6 | * @Classname TestCacheManager 7 | * @Description TODO 8 | * @Date 2020/6/10 13:58 9 | * @Created by limeng 10 | */ 11 | public class TestCacheManager { 12 | public static void main(String[] args) { 13 | 14 | CacheManager.setCache("test","value"); 15 | CacheManager.setCache("test2","value2"); 16 | CacheManager.setCache("test3","value4"); 17 | 18 | Object value = CacheManager.getCache("test"); 19 | System.out.println(value); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /core/scheduler/src/test/java/com/knowlegene/parent/scheduler/service/TestJobEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 4 | 5 | /** 6 | * @Classname TestJobEvent 7 | * @Description TODO 8 | * @Date 2020/6/5 16:41 9 | * @Created by limeng 10 | */ 11 | public class TestJobEvent extends AbstractEvent { 12 | private String jobID; 13 | 14 | public TestJobEvent (String jobID, TestJobEventType type) { 15 | super(type); 16 | this.jobID = jobID; 17 | } 18 | 19 | public TestJobEvent(TestJobEventType testJobEventType) { 20 | super(testJobEventType); 21 | } 22 | 23 | public TestJobEvent(TestJobEventType testJobEventType, long timestamp) { 24 | super(testJobEventType, timestamp); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /core/scheduler/src/test/java/com/knowlegene/parent/scheduler/service/TestJobEventType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | /** 4 | * @Classname TestJobEventType 5 | * @Description TODO 6 | * @Date 2020/6/5 16:42 7 | * @Created by limeng 8 | */ 9 | public enum TestJobEventType { 10 | JOB_KILL, 11 | JOB_INIT, 12 | JOB_START 13 | } 14 | -------------------------------------------------------------------------------- /core/scheduler/src/test/java/com/knowlegene/parent/scheduler/service/TestSimpleMRAppMaster.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | import com.knowlegene.parent.scheduler.event.AsyncDispatcher; 4 | import com.knowlegene.parent.scheduler.event.Dispatcher; 5 | import com.knowlegene.parent.scheduler.event.EventHandler; 6 | 7 | /** 8 | * @Classname TestSimpleMRAppMaster 9 | * @Description TODO 10 | * @Date 2020/6/5 16:43 11 | * @Created by limeng 12 | */ 13 | public class TestSimpleMRAppMaster extends CompositeService { 14 | private Dispatcher dispatcher;//中央异步调度器 15 | private String jobID; 16 | private int taskNumber;//该作业包含的任务数目 17 | private String[] taskIDs;//该作业内部包含的所有任务 18 | 19 | 20 | 21 | public TestSimpleMRAppMaster(String name, String jobID, int taskNumber) { 22 | super(name); 23 | this.jobID = jobID; 24 | this.taskNumber = taskNumber; 25 | this.taskIDs = new String[taskNumber]; 26 | for (int i = 0; i < taskNumber; i++) { 27 | taskIDs[i] = new String(jobID+"_task_"+i); 28 | } 29 | } 30 | 31 | @Override 32 | public void serviceInit() throws Exception { 33 | dispatcher = new AsyncDispatcher(); 34 | //注册Job和Task事件调度器 35 | dispatcher.register(TestJobEventType.class,new JobEventDispatcher()); 36 | dispatcher.register(TestTaskEventType.class,new TaskEventDispatcher()); 37 | addService((Service)dispatcher); 38 | super.serviceInit(); 39 | } 40 | 41 | 42 | 43 | public Dispatcher getDispatcher() { 44 | return dispatcher; 45 | } 46 | 47 | private class JobEventDispatcher implements EventHandler { 48 | 49 | 50 | public void handle(TestJobEvent event) { 51 | if(event.getType() == TestJobEventType.JOB_KILL){ 52 | System.out.println("Receive JOB_KILL event, killing all the tasks"); 53 | for (int i = 0; i < taskNumber; i++) { 54 | dispatcher.getEventHandler().handle(new TestTaskEvent(taskIDs[i], TestTaskEventType.T_KILL)); 55 | } 56 | }else if(event.getType() == TestJobEventType.JOB_INIT){ 57 | System.out.println("Receive JOB_INIT event, scheduling tasks"); 58 | for (int i = 0; i < taskNumber; i++) { 59 | dispatcher.getEventHandler().handle(new TestTaskEvent(taskIDs[i], TestTaskEventType.T_SCHEDULE)); 60 | } 61 | } 62 | } 63 | } 64 | 65 | private class TaskEventDispatcher implements EventHandler { 66 | 67 | public void handle(TestTaskEvent event) { 68 | if (event.getType() == TestTaskEventType.T_KILL) { 69 | System.out.println("Receive T_KILL event of task" + event.getTaskID()); 70 | } else if (event.getType() == TestTaskEventType.T_SCHEDULE) { 71 | System.out.println("Receive T_SCHEDULE of task" + event.getTaskID()); 72 | } 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /core/scheduler/src/test/java/com/knowlegene/parent/scheduler/service/TestSimpleMRAppMasterRun.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 4 | 5 | /** 6 | * @Classname TestSimpleMRAppMaster2 7 | * @Description TODO 8 | * @Date 2020/6/5 16:44 9 | * @Created by limeng 10 | */ 11 | public class TestSimpleMRAppMasterRun { 12 | public static void main(String[] args) throws Exception { 13 | String jobID = "job_2020"; 14 | TestSimpleMRAppMaster appMaster = new TestSimpleMRAppMaster("Simple MRAppMaster", jobID, 5); 15 | 16 | appMaster.serviceInit(); 17 | appMaster.serviceStart(); 18 | appMaster.serviceStop(); 19 | /** 20 | * Receive JOB_INIT event, scheduling tasks 21 | * Receive JOB_KILL event, killing all the tasks 22 | * Receive T_SCHEDULE of taskjob_2020_task_0 23 | * Receive T_SCHEDULE of taskjob_2020_task_1 24 | * Receive T_SCHEDULE of taskjob_2020_task_2 25 | * Receive T_SCHEDULE of taskjob_2020_task_3 26 | * Receive T_SCHEDULE of taskjob_2020_task_4 27 | * Receive T_KILL event of taskjob_2020_task_0 28 | * Receive T_KILL event of taskjob_2020_task_1 29 | * Receive T_KILL event of taskjob_2020_task_2 30 | * Receive T_KILL event of taskjob_2020_task_3 31 | * Receive T_KILL event of taskjob_2020_task_4 32 | */ 33 | //TestTaskEvent 34 | appMaster.getDispatcher().getEventHandler().handle((AbstractEvent)new TestTaskEvent(jobID, TestTaskEventType.T_SCHEDULE)); 35 | //appMaster.getDispatcher().getEventHandler().handle(new TestTaskEvent(jobID, TestTaskEventType.JOB_KILL)); 36 | 37 | 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /core/scheduler/src/test/java/com/knowlegene/parent/scheduler/service/TestTaskEvent.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | import com.knowlegene.parent.scheduler.event.AbstractEvent; 4 | 5 | /** 6 | * @Classname TestTaskEvent 7 | * @Description TODO 8 | * @Date 2020/6/5 16:42 9 | * @Created by limeng 10 | */ 11 | public class TestTaskEvent extends AbstractEvent { 12 | private String taskID; 13 | public String getTaskID() { 14 | return taskID; 15 | } 16 | 17 | public TestTaskEvent (String taskID, TestTaskEventType type) { 18 | super(type); 19 | this.taskID = taskID; 20 | } 21 | 22 | public TestTaskEvent(TestTaskEventType testTaskEventType) { 23 | super(testTaskEventType); 24 | } 25 | 26 | public TestTaskEvent(TestTaskEventType testTaskEventType, long timestamp) { 27 | super(testTaskEventType, timestamp); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /core/scheduler/src/test/java/com/knowlegene/parent/scheduler/service/TestTaskEventType.java: -------------------------------------------------------------------------------- 1 | package com.knowlegene.parent.scheduler.service; 2 | 3 | /** 4 | * @Classname TestTaskEventType 5 | * @Description TODO 6 | * @Date 2020/6/5 16:42 7 | * @Created by limeng 8 | */ 9 | public enum TestTaskEventType { 10 | T_KILL, 11 | T_SCHEDULE 12 | } 13 | -------------------------------------------------------------------------------- /docs/img/clentsource.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlimeng/exchange-parent/53ade8d76e5ad2a11951f5785d28027a76605a41/docs/img/clentsource.png -------------------------------------------------------------------------------- /docs/img/source.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dlimeng/exchange-parent/53ade8d76e5ad2a11951f5785d28027a76605a41/docs/img/source.png -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | client/kd-process 7 | client/kd-config 8 | core/scheduler 9 | 10 | 11 | com.knowlegene.parent 12 | exchange-parent 13 | 0.1.0 14 | exchange-parent 15 | 16 | pom 17 | 18 | 19 | UTF-8 20 | UTF-8 21 | 1.8 22 | 8.0.27 23 | 11.2.0.1.0 24 | 1.8 25 | 2.15.2 26 | 2.11 27 | 0.1.0 28 | 29 | 30 | 31 | 32 | 33 | 34 | mysql 35 | mysql-connector-java 36 | ${mysql.version} 37 | 38 | 39 | 40 | 41 | com.knowlegene.parent.scheduler 42 | scheduler 43 | ${kd.project.version} 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | org.apache.commons 52 | commons-lang3 53 | 3.7 54 | 55 | 56 | org.projectlombok 57 | lombok 58 | compile 59 | 1.18.2 60 | true 61 | 62 | 63 | 64 | com.alibaba 65 | fastjson 66 | 1.2.61 67 | 68 | 69 | 70 | commons-beanutils 71 | commons-beanutils 72 | 1.9.2 73 | 74 | 75 | 76 | com.google.auto.value 77 | auto-value-annotations 78 | 1.6 79 | 80 | 81 | com.google.auto.value 82 | auto-value 83 | 1.6 84 | provided 85 | 86 | 87 | 88 | 89 | junit 90 | junit 91 | 4.13.1 92 | 93 | 94 | 95 | commons-codec 96 | commons-codec 97 | 1.13 98 | 99 | 100 | 101 | com.google.guava 102 | guava 103 | 20.0 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | --------------------------------------------------------------------------------