├── bd-kafka ├── src │ ├── main │ │ ├── resources │ │ │ ├── zookeeper.properties │ │ │ ├── producer.properties │ │ │ └── consumer.properties │ │ └── java │ │ │ └── org │ │ │ └── bd │ │ │ └── kafka │ │ │ ├── ATest03.java │ │ │ ├── ProducerRunner.java │ │ │ ├── ConsumerRunner.java │ │ │ ├── ATest02.java │ │ │ ├── ATest.java │ │ │ ├── utils │ │ │ └── PropertiesUtil.java │ │ │ └── ProducerClient.java │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── kafka │ │ └── AppTest.java └── pom.xml ├── bd-mapreduce ├── src │ ├── main │ │ └── resources │ │ │ ├── math.txt │ │ │ ├── chinese.txt │ │ │ └── english.txt │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── mapreduce │ │ └── AppTest.java └── pom.xml ├── bd-zk ├── src │ ├── main │ │ ├── resources │ │ │ └── zookeeper.properties │ │ └── java │ │ │ └── org │ │ │ └── bd │ │ │ └── zk │ │ │ ├── callback │ │ │ ├── Test.java │ │ │ ├── ZkLinkAsyncCallback.java │ │ │ └── A.java │ │ │ ├── ZkPaths.java │ │ │ ├── ZkException.java │ │ │ ├── AtestClient.java │ │ │ ├── watcher │ │ │ └── ZkLinkWatcher.java │ │ │ └── utils │ │ │ └── PropertiesUtil.java │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── zk │ │ └── AppTest.java └── pom.xml ├── bd-hdfs ├── src │ ├── main │ │ ├── resources │ │ │ └── hdfs.properties │ │ └── java │ │ │ └── org │ │ │ └── bd │ │ │ └── hdfs │ │ │ ├── utils │ │ │ ├── Consts.java │ │ │ ├── SysVarsUtils.java │ │ │ └── PropertiesUtil.java │ │ │ ├── HdfsClientException.java │ │ │ └── Test.java │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── hdfs │ │ └── AppTest.java └── pom.xml ├── bd-kylin ├── src │ ├── main │ │ ├── resources │ │ │ └── kylin.properties │ │ └── java │ │ │ └── org │ │ │ └── bd │ │ │ └── kylin │ │ │ ├── cube │ │ │ ├── Notify_list.java │ │ │ ├── Dimensions.java │ │ │ ├── Rowkey.java │ │ │ ├── Hbase_mapping.java │ │ │ ├── Aggregation_groups.java │ │ │ └── Measures.java │ │ │ ├── utils │ │ │ ├── Consts.java │ │ │ ├── SysVarsUtils.java │ │ │ └── PropertiesUtil.java │ │ │ ├── CubeBuildTypeEnum.java │ │ │ ├── CubeException.java │ │ │ ├── rest │ │ │ ├── ProjectRest.java │ │ │ ├── response │ │ │ │ ├── ProjectRestResponse.java │ │ │ │ └── QueryRestResponse.java │ │ │ ├── TableRest.java │ │ │ ├── QueryRest.java │ │ │ └── ModelRest.java │ │ │ ├── model │ │ │ ├── Dimensions.java │ │ │ ├── Partition_desc.java │ │ │ ├── Lookups.java │ │ │ └── ModelJson.java │ │ │ ├── response │ │ │ ├── Realizations.java │ │ │ ├── CubeResponse.java │ │ │ └── HBaseResponse.java │ │ │ ├── JobTimeFilterEnum.java │ │ │ ├── JobStatusEnum.java │ │ │ └── request │ │ │ ├── JobBuildRequest.java │ │ │ ├── SQLRequest.java │ │ │ ├── ModelRequest.java │ │ │ ├── JobListRequest.java │ │ │ └── CubeRequest.java │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── kylin │ │ └── AppTest.java └── pom.xml ├── bd-spark └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── bd │ │ │ └── spark │ │ │ ├── mllib │ │ │ └── T.java │ │ │ ├── enums │ │ │ ├── Consts.java │ │ │ ├── DbmsType.java │ │ │ └── FormatType.java │ │ │ ├── utils │ │ │ ├── SysVarsUtils.java │ │ │ └── JdbcDriveUtil.java │ │ │ └── SessionDrive.java │ └── resources │ │ ├── jdbc.properties │ │ └── spark.properties │ └── test │ └── java │ └── org │ └── bd │ └── spark │ ├── kafka │ ├── ConsumerTest.java │ ├── ProducerTest.java │ ├── UserKafkaProducer.java │ └── UserKafkaConsumer.java │ ├── stream │ ├── AppStreamingTest.java │ ├── TextFileStreamTest.java │ └── SocketStreamTest.java │ ├── write │ ├── WriteTextTest.java │ ├── WriteCsvTest.java │ ├── WriteJsonTest.java │ ├── WriteParquetTest.java │ ├── WriteMysqlTest.java │ └── AppWriteTest.java │ └── read │ ├── ReadJsonTest.java │ ├── ReadParquetTest.java │ ├── ReadCsvTest.java │ ├── ReadTextTest.java │ ├── AppReadTest.java │ ├── ReadMysqlTest.java │ └── ReadHiveTest.java ├── bd-hive-udf ├── src │ ├── main │ │ ├── java │ │ │ └── org │ │ │ │ └── bd │ │ │ │ └── hive │ │ │ │ ├── udaf │ │ │ │ ├── HiveAvg.java │ │ │ │ └── Att.java │ │ │ │ └── udf │ │ │ │ ├── UpperCase.java │ │ │ │ └── LowerCase.java │ │ └── resources │ │ │ └── package.xml │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── hive │ │ └── udf │ │ └── AppTest.java └── pom.xml ├── bd-flink └── src │ ├── main │ ├── java │ │ └── org │ │ │ └── bd │ │ │ └── flink │ │ │ ├── idgenerate │ │ │ ├── IdGeneratorType.java │ │ │ ├── ESIdGenerator.java │ │ │ ├── AutoIdGenerator.java │ │ │ ├── IdGeneratorBuilder.java │ │ │ └── MD5XdrIdGenerator.java │ │ │ ├── XDREnrichedMessage.java │ │ │ ├── XDRRawMessage.java │ │ │ ├── JsonToXDRRawMessage.java │ │ │ └── XDREnrichedMessageSinkFunction.java │ └── resources │ │ └── log4j.properties │ └── test │ └── java │ └── org │ └── bd │ └── flink │ └── AppTest.java ├── bd-hbase ├── src │ ├── main │ │ ├── resources │ │ │ └── hbase.properties │ │ └── java │ │ │ └── org │ │ │ └── bd │ │ │ └── hbase │ │ │ ├── utils │ │ │ ├── Consts.java │ │ │ ├── SysVarsUtils.java │ │ │ ├── HBaseUtils.java │ │ │ └── PropertiesUtil.java │ │ │ ├── Test.java │ │ │ └── HBaseConfig.java │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── hbase │ │ └── AppTest.java └── pom.xml ├── bd-elasticsearch ├── src │ └── main │ │ └── resources │ │ └── log4j2.properties └── pom.xml ├── bd-hive ├── src │ ├── main │ │ ├── resources │ │ │ └── hive.properties │ │ └── java │ │ │ └── org │ │ │ └── bd │ │ │ └── hive │ │ │ ├── HiveException.java │ │ │ ├── utils │ │ │ ├── Consts.java │ │ │ ├── SysVarsUtils.java │ │ │ └── PropertiesUtil.java │ │ │ └── DbCloseUtils.java │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── hive │ │ └── AppTest.java └── pom.xml ├── bd-datax ├── src │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── bd │ │ │ └── datax │ │ │ ├── bean │ │ │ ├── Read.java │ │ │ ├── Write.java │ │ │ ├── Script.java │ │ │ ├── Job.java │ │ │ ├── Content.java │ │ │ ├── Transformer.java │ │ │ ├── SettingBean.java │ │ │ ├── Setting.java │ │ │ └── TransformerBean.java │ │ │ ├── DataxSourceType.java │ │ │ ├── DataxException.java │ │ │ ├── DataxBean.java │ │ │ ├── DataxPara.java │ │ │ ├── mysql │ │ │ ├── MysqlPara.java │ │ │ └── MysqlRead.java │ │ │ ├── sqlserver │ │ │ ├── SqlServerPara.java │ │ │ └── SqlServerWrite.java │ │ │ ├── oracle │ │ │ └── OraclePara.java │ │ │ ├── hdfs │ │ │ └── HdfsPara.java │ │ │ └── ftp │ │ │ └── FtpPara.java │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── datax │ │ └── AppTest.java └── pom.xml ├── bd-impala └── src │ ├── main │ ├── resources │ │ └── impala.properties │ └── java │ │ └── org │ │ └── bd │ │ └── impala │ │ ├── ImpalaException.java │ │ └── utils │ │ ├── SysVarsUtils.java │ │ ├── Consts.java │ │ ├── ImpaIaTableColumn.java │ │ └── PropertiesUtil.java │ └── test │ └── java │ └── org │ └── bd │ └── impala │ └── AppTest.java ├── README.md ├── bd-yarn ├── src │ └── main │ │ └── java │ │ └── org │ │ └── bd │ │ └── yarn │ │ ├── exception │ │ └── YarnException.java │ │ └── utils │ │ └── YarnUtil.java └── pom.xml ├── bd-solr ├── src │ └── test │ │ └── java │ │ └── org │ │ └── bd │ │ └── solr │ │ └── AppTest.java └── pom.xml └── pom.xml /bd-kafka/src/main/resources/zookeeper.properties: -------------------------------------------------------------------------------- 1 | zookeeper.connect=localhost:2181 -------------------------------------------------------------------------------- /bd-mapreduce/src/main/resources/math.txt: -------------------------------------------------------------------------------- 1 | Zhao 38 2 | Qian 45 3 | Sun 23 4 | Li 43 -------------------------------------------------------------------------------- /bd-mapreduce/src/main/resources/chinese.txt: -------------------------------------------------------------------------------- 1 | Zhao 98 2 | Qian 9 3 | Sun 67 4 | Li 23 -------------------------------------------------------------------------------- /bd-mapreduce/src/main/resources/english.txt: -------------------------------------------------------------------------------- 1 | Zhao 93 2 | Qian 42 3 | Sun 87 4 | Li 54 -------------------------------------------------------------------------------- /bd-zk/src/main/resources/zookeeper.properties: -------------------------------------------------------------------------------- 1 | zookeeper.connect=localhost:2181 2 | sessionTimeout=10000 -------------------------------------------------------------------------------- /bd-hdfs/src/main/resources/hdfs.properties: -------------------------------------------------------------------------------- 1 | fs.defaultFS = hdfs://lab-southeastasia-jupyter-vm-hdp-01:8020/ 2 | -------------------------------------------------------------------------------- /bd-kylin/src/main/resources/kylin.properties: -------------------------------------------------------------------------------- 1 | kylin.path = http://10.10.10.23:7070/kylin/api 2 | kylin.project = demo -------------------------------------------------------------------------------- /bd-spark/src/main/java/org/bd/spark/mllib/T.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.mllib; 2 | 3 | public class T { 4 | 5 | } 6 | -------------------------------------------------------------------------------- /bd-hive-udf/src/main/java/org/bd/hive/udaf/HiveAvg.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive.udaf; 2 | 3 | 4 | public class HiveAvg { 5 | 6 | } 7 | -------------------------------------------------------------------------------- /bd-flink/src/main/java/org/bd/flink/idgenerate/IdGeneratorType.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink.idgenerate; 2 | 3 | public enum IdGeneratorType { 4 | AUTO , 5 | MD5, 6 | PREFIX; 7 | } 8 | -------------------------------------------------------------------------------- /bd-hbase/src/main/resources/hbase.properties: -------------------------------------------------------------------------------- 1 | hbase.zookeeper.quorum = 10.10.10.22,10.10.10.23,10.10.10.24 2 | hbase.zookeeper.property.clientPort = 2181 3 | hbase.master = 10.10.10.22:600000 -------------------------------------------------------------------------------- /bd-elasticsearch/src/main/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | appender.console.type = Console 2 | appender.console.name = console 3 | appender.console.layout.type = PatternLayout 4 | 5 | rootLogger.level = info 6 | rootLogger.appenderRef.console.ref = console -------------------------------------------------------------------------------- /bd-flink/src/main/java/org/bd/flink/idgenerate/ESIdGenerator.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink.idgenerate; 2 | 3 | import java.io.Serializable; 4 | 5 | public interface ESIdGenerator extends Serializable { 6 | String generate(final String msg, final int idLength); 7 | } 8 | -------------------------------------------------------------------------------- /bd-flink/src/main/java/org/bd/flink/idgenerate/AutoIdGenerator.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink.idgenerate; 2 | 3 | public class AutoIdGenerator implements ESIdGenerator { 4 | @Override 5 | public String generate(String msg, int idLength) { 6 | return null; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /bd-hive/src/main/resources/hive.properties: -------------------------------------------------------------------------------- 1 | hive.driver=org.apache.hive.jdbc.HiveDriver 2 | hive.url=jdbc:hive2://10.10.10.23:10000 3 | hive.user= 4 | hive.password= 5 | initialsize=5 6 | maxactive=50 7 | minEvictableIdleTimeMillis=30000 8 | removeAbandonedTimeout=1080 9 | timeBetweenEvictionRunsMillis=30000 -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/bean/Read.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.bean; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: datax读取抽象类
6 | * 版本历史: 7 | * @author wpk | 2017年9月20日 下午9:47:38 |创建 8 | */ 9 | public abstract class Read { 10 | 11 | } 12 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/bean/Write.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.bean; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: datax写入抽象类
6 | * 版本历史: 7 | * @author wpk | 2017年9月20日 下午9:47:46 |创建 8 | */ 9 | public abstract class Write { 10 | 11 | } 12 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/cube/Notify_list.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.cube; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 通知渠道配置
6 | * 版本历史: 7 | * @author wpk | 2017年9月14日 下午12:2:26 |创建 8 | */ 9 | public class Notify_list { 10 | 11 | } 12 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/kafka/ConsumerTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.kafka; 2 | 3 | public class ConsumerTest { 4 | 5 | public static void main(String[] args) { 6 | UserKafkaConsumer consumerThread = new UserKafkaConsumer("topic1"); 7 | consumerThread.start(); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/kafka/ProducerTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.kafka; 2 | 3 | public class ProducerTest { 4 | 5 | public static void main(String[] args) { 6 | UserKafkaProducer producerThread = new UserKafkaProducer("topic1"); 7 | producerThread.start(); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /bd-impala/src/main/resources/impala.properties: -------------------------------------------------------------------------------- 1 | impala.driver=org.apache.hive.jdbc.HiveDriver 2 | impala.url=jdbc:hive2://192.168.1.73:21050/default;auth=noSasl 3 | impala_user= 4 | impala.password= 5 | initialsize=5 6 | maxactive=50 7 | minEvictableIdleTimeMillis=30000 8 | removeAbandonedTimeout=1080 9 | timeBetweenEvictionRunsMillis=30000 -------------------------------------------------------------------------------- /bd-spark/src/main/resources/jdbc.properties: -------------------------------------------------------------------------------- 1 | mysql.jdbc.driver=com.mysql.jdbc.Driver 2 | mysql.jdbc.url=jdbc:mysql://localhost:3306/test?useUnicode=true&characterEncoding=utf8 3 | mysql.jdbc.user=root 4 | mysql.jdbc.password=root 5 | 6 | spark.jdbc.driver=org.apache.hive.jdbc.HiveDriver 7 | spark.jdbc.url=jdbc:hive2://192.168.1.71:10000 8 | spark.jdbc.user= 9 | spark.jdbc.password= 10 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/utils/Consts.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.utils; 2 | 3 | 4 | /** 5 | * 版权信息: big data module
6 | * 功能描述: 常量类
7 | * 版本历史: 8 | * @author wpk | 2017年8月23日 上午11:18:51 |创建 9 | */ 10 | public class Consts { 11 | 12 | //属性配置文件名称 13 | public static final String KYLIN_PROPERTIES = "kylin.properties"; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /bd-flink/src/test/java/org/bd/flink/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink; 2 | 3 | import static org.junit.Assert.assertTrue; 4 | 5 | import org.junit.Test; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | { 12 | /** 13 | * Rigorous Test :-) 14 | */ 15 | @Test 16 | public void shouldAnswerWithTrue() 17 | { 18 | assertTrue( true ); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bd-utils 2 | 大数据工具封装 3 | 4 | 该项目主要涉及到的组件有 5 | 6 | datax(数据采集工具接口封装) 7 | 8 | hbase(非关系型数据库的接口封装) 9 | 10 | hdfs(hdfs文件系统接口封装) 11 | 12 | hive(hive查询接口封装) 13 | 14 | hive-udf(hive自定义函数例子，及打包上传运行) 15 | 16 | impala(impala查询接口封装) 17 | 18 | kafka(消息中间件的接口封装) 19 | 20 | kylin(数据预处理的接口封装) 21 | 22 | mapreduce(mapreduce例子，及打包上传运行) 23 | 24 | solr(全文搜索客户端接口封装) 25 | 26 | spark(包括sparkSQL,sparkStreaming接口封装) 27 | 28 | zookeeper(客户端接口封装) 29 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/bean/Script.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.bean; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: datax脚本
6 | * 版本历史: 7 | * @author wpk | 2017年10月16日 下午4:18:21 |创建 8 | */ 9 | public class Script { 10 | 11 | private Job job; 12 | 13 | public Job getJob() { 14 | return job; 15 | } 16 | 17 | public void setJob(Job job) { 18 | this.job = job; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /bd-flink/src/main/java/org/bd/flink/XDREnrichedMessage.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink; 2 | 3 | import lombok.*; 4 | 5 | import java.io.Serializable; 6 | 7 | @Data 8 | @Builder 9 | @NoArgsConstructor 10 | @AllArgsConstructor 11 | public class XDREnrichedMessage implements Serializable { 12 | private String apn; 13 | private String cell_id; 14 | private String country; 15 | private String enterprise_name; 16 | private String timestamp_event; 17 | } 18 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/CubeBuildTypeEnum.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述:
6 | * 版本历史: 7 | * @author wpk | 2017年11月21日 上午11:43:42 |创建 8 | */ 9 | public enum CubeBuildTypeEnum { 10 | /** 11 | * 重建节段或者增量节段 12 | */ 13 | BUILD, 14 | /** 15 | * 合并节段 16 | */ 17 | MERGE, 18 | 19 | /** 20 | * 刷新节段 21 | */ 22 | REFRESH 23 | } 24 | -------------------------------------------------------------------------------- /bd-flink/src/main/java/org/bd/flink/XDRRawMessage.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink; 2 | 3 | import lombok.*; 4 | 5 | @NoArgsConstructor 6 | @AllArgsConstructor 7 | @Getter 8 | @Setter 9 | @Builder 10 | public class XDRRawMessage { 11 | private String apn; //1 12 | private String cell_id; //2 13 | private String country; //3 14 | private String enterprise_name; //4 15 | private String timestamp_event; //30 16 | } 17 | -------------------------------------------------------------------------------- /bd-flink/src/main/java/org/bd/flink/idgenerate/IdGeneratorBuilder.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink.idgenerate; 2 | 3 | public class IdGeneratorBuilder { 4 | static public ESIdGenerator build(IdGeneratorType generatorType) { 5 | switch (generatorType) { 6 | case AUTO: 7 | return new AutoIdGenerator(); 8 | case MD5: 9 | return MD5XdrIdGenerator.builder().build(); 10 | } 11 | return null; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /bd-hdfs/src/main/java/org/bd/hdfs/utils/Consts.java: -------------------------------------------------------------------------------- 1 | package org.bd.hdfs.utils; 2 | 3 | 4 | /** 5 | * 版权信息: big data module
6 | * 功能描述: 常量类
7 | * 版本历史: 8 | * @author wpk | 2017年11月15日 上午10:24:21 |创建 9 | */ 10 | public class Consts { 11 | 12 | //impala驱动类 13 | public static final String FS_DEFAULTFS = "fs.defaultFS"; 14 | 15 | //属性配置文件名称 16 | public static final String HDFS_PROPERTIES = "hdfs.properties"; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /bd-kafka/src/main/java/org/bd/kafka/ATest03.java: -------------------------------------------------------------------------------- 1 | package org.bd.kafka; 2 | 3 | import java.util.UUID; 4 | import java.util.concurrent.ExecutionException; 5 | 6 | import org.apache.kafka.clients.consumer.ConsumerRecords; 7 | 8 | import net.sf.json.JSONArray; 9 | import net.sf.json.JSONObject; 10 | 11 | public class ATest03 { 12 | 13 | public static void main(String[] arg){ 14 | final ConsumerClient cc = new ConsumerClient("rta_xdr"); 15 | ConsumerRecords cr = cc.getRecords(); 16 | 17 | System.out.println(cr); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/DataxSourceType.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 读写类型
6 | * 版本历史: 7 | * @author wpk | 2017年8月7日 下午3:22:45 |创建 8 | */ 9 | public class DataxSourceType { 10 | 11 | public final static String ORACLE = "1"; 12 | public final static String SQLSERVER = "2"; 13 | public final static String MYSQL = "3"; 14 | public final static String HDFS = "4"; 15 | public final static String FTP = "5"; 16 | public final static String HBASE = "6"; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /bd-zk/src/main/java/org/bd/zk/callback/Test.java: -------------------------------------------------------------------------------- 1 | package org.bd.zk.callback; 2 | 3 | import java.io.Serializable; 4 | 5 | public class Test implements Serializable{ 6 | 7 | /** 8 | * 9 | */ 10 | private static final long serialVersionUID = 1L; 11 | private int id; 12 | private String name; 13 | public int getId() { 14 | return id; 15 | } 16 | public void setId(int id) { 17 | this.id = id; 18 | } 19 | public String getName() { 20 | return name; 21 | } 22 | public void setName(String name) { 23 | this.name = name; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/stream/AppStreamingTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.stream; 2 | 3 | 4 | public class AppStreamingTest { 5 | 6 | public static void main(String[] args) { 7 | try { 8 | System.setProperty("hadoop.home.dir", "F:\\hadoop-common-2.2.0-bin-master"); 9 | 10 | SocketStreamTest.socketStreaming();//socket流式处理 11 | 12 | TextFileStreamTest.TextFileStreaming();//监控指定目录下流式处理 13 | 14 | // KafkaStreamTest.kafkaStreaming();//kafka流式处理 15 | } catch (Exception e) { 16 | e.printStackTrace(); 17 | } 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/CubeException.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin; 2 | 3 | public class CubeException extends RuntimeException { 4 | 5 | private static final long serialVersionUID = 1L; 6 | 7 | public CubeException() { 8 | super(); 9 | } 10 | 11 | public CubeException(String message) { 12 | super(message); 13 | } 14 | 15 | public CubeException(String message, Throwable cause) { 16 | super(message, cause); 17 | } 18 | 19 | public CubeException(Throwable cause) { 20 | super(cause); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /bd-yarn/src/main/java/org/bd/yarn/exception/YarnException.java: -------------------------------------------------------------------------------- 1 | package org.bd.yarn.exception; 2 | 3 | public class YarnException extends RuntimeException { 4 | 5 | private static final long serialVersionUID = -1520131049273172806L; 6 | 7 | public YarnException() { 8 | super(); 9 | } 10 | 11 | public YarnException(String message) { 12 | super(message); 13 | } 14 | 15 | public YarnException(String message, Throwable cause) { 16 | super(message, cause); 17 | } 18 | 19 | public YarnException(Throwable cause) { 20 | super(cause); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /bd-hdfs/src/main/java/org/bd/hdfs/HdfsClientException.java: -------------------------------------------------------------------------------- 1 | package org.bd.hdfs; 2 | 3 | public class HdfsClientException extends RuntimeException { 4 | 5 | private static final long serialVersionUID = 1L; 6 | 7 | public HdfsClientException() { 8 | super(); 9 | } 10 | 11 | public HdfsClientException(String message) { 12 | super(message); 13 | } 14 | 15 | public HdfsClientException(String message, Throwable cause) { 16 | super(message, cause); 17 | } 18 | 19 | public HdfsClientException(Throwable cause) { 20 | super(cause); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /bd-zk/src/main/java/org/bd/zk/callback/ZkLinkAsyncCallback.java: -------------------------------------------------------------------------------- 1 | package org.bd.zk.callback; 2 | 3 | import org.apache.zookeeper.AsyncCallback; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: zookeeper连接异步回调
8 | * 版本历史: 9 | * @author wpk | 2017年12月4日 下午5:52:50 |创建 10 | */ 11 | public class ZkLinkAsyncCallback implements AsyncCallback.StringCallback{ 12 | 13 | @Override 14 | public void processResult(int rc, String path, Object ctx, String name) { 15 | System.out.println("Create path result: [" + rc + ", " + path + ", " + ctx + ", real path name: " + name); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /bd-hbase/src/main/java/org/bd/hbase/utils/Consts.java: -------------------------------------------------------------------------------- 1 | package org.bd.hbase.utils; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 常量类
6 | * 版本历史: 7 | * @author wpk | 2017年10月31日 下午2:27:48 |创建 8 | */ 9 | public class Consts { 10 | 11 | public static final String HBASE_ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; 12 | public static final String HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT = "hbase.zookeeper.property.clientPort"; 13 | public static final String HBASE_MASTER = "hbase.master"; 14 | 15 | //属性配置文件名称 16 | public static final String HBASE_PROPERTIES = "hbase.properties"; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /bd-hive-udf/src/main/java/org/bd/hive/udf/UpperCase.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive.udf; 2 | 3 | import org.apache.hadoop.hive.ql.exec.UDF; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: 字符串转大写类
8 | * 版本历史: 9 | * @author wpk | 2017年9月4日 下午9:02:22 |创建 10 | */ 11 | public class UpperCase extends UDF { 12 | 13 | /** 14 | * 描述： 接受单行输入，并产生单行输出 15 | * @author wpk | 2017年9月4日 下午9:06:27 |创建 16 | * @param s 17 | * @return String 18 | */ 19 | public String evaluate(String s) { 20 | if (s == null) { 21 | return null; 22 | } 23 | return s.toUpperCase(); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/rest/ProjectRest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.rest; 2 | 3 | import org.bd.kylin.RestRequstHandle; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: kylin项目接口
8 | * 版本历史: 9 | * @author wpk | 2017年11月21日 下午2:37:07 |创建 10 | */ 11 | public class ProjectRest extends RestRequstHandle{ 12 | 13 | /** 14 | * 描述： 获取项目信息 15 | * @author wpk | 2017年11月21日 下午2:38:51 |创建 16 | * @return String 17 | */ 18 | public static String getProjects(){ 19 | String para = "projects"; 20 | String result = request(para, RequestMethod.GET); 21 | return result; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /bd-hive-udf/src/main/java/org/bd/hive/udf/LowerCase.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive.udf; 2 | 3 | import org.apache.hadoop.hive.ql.exec.UDF; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: 字符串转小写类
8 | * 版本历史: 9 | * @author wpk | 2017年9月4日 下午9:04:19 |创建 10 | */ 11 | public class LowerCase extends UDF { 12 | 13 | /** 14 | * 描述： 接受单行输入，并产生单行输出 15 | * @author wpk | 2017年9月4日 下午9:06:10 |创建 16 | * @param s 17 | * @return String 18 | */ 19 | public String evaluate(String s) { 20 | if (s == null) { 21 | return null; 22 | } 23 | return s.toLowerCase(); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /bd-flink/src/main/java/org/bd/flink/JsonToXDRRawMessage.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink; 2 | 3 | import lombok.Builder; 4 | import org.apache.flink.api.common.functions.MapFunction; 5 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 6 | 7 | @Builder 8 | public class JsonToXDRRawMessage implements MapFunction { 9 | private transient ObjectMapper mapper; 10 | 11 | @Override 12 | public XDRRawMessage map(String s) throws Exception { 13 | if (mapper == null) { 14 | mapper = new ObjectMapper(); 15 | } 16 | 17 | return mapper.readValue(s, XDRRawMessage.class); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /bd-spark/src/main/resources/spark.properties: -------------------------------------------------------------------------------- 1 | spark.driver=org.apache.hive.jdbc.HiveDriver 2 | spark.url=jdbc:hive2://192.168.1.71:10000 3 | spark.user= 4 | spark.password= 5 | 6 | 7 | #\u8FDE\u63A5\u6C60\u521D\u59CB\u5316\u5927\u5C0F 8 | initialsize=5 9 | #\u8FDE\u63A5\u6C60\u6700\u5927\u8FDE\u63A5\u6570 10 | maxactive=50 11 | #30s\u6700\u5C0F\u751F\u5B58\u65F6\u95F430 * 1000 12 | minEvictableIdleTimeMillis=30000 13 | #\u8FDE\u63A5\u5E9F\u5F03\u8D85\u8FC71\u5C0F\u65F6\u672A\u5173\u95ED\uFF0C\u5C31\u4F1A\u88AB\u5F3A\u884C\u56DE\u6536 14 | removeAbandonedTimeout=1080 15 | #30\u79D2\u68C0\u6D4B\u4E00\u6B21\u9700\u8981\u5F3A\u884C\u56DE\u6536\u7684\u8FDE\u63A5 16 | timeBetweenEvictionRunsMillis=30000 -------------------------------------------------------------------------------- /bd-zk/src/main/java/org/bd/zk/ZkPaths.java: -------------------------------------------------------------------------------- 1 | package org.bd.zk; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: zookeeper树状路径(组合模式)
8 | * 版本历史: 9 | * @author wpk | 2017年12月4日 下午3:40:19 |创建 10 | */ 11 | public class ZkPaths { 12 | 13 | private String name; 14 | private List children; 15 | 16 | public String getName() { 17 | return name; 18 | } 19 | public void setName(String name) { 20 | this.name = name; 21 | } 22 | public List getChildren() { 23 | return children; 24 | } 25 | public void setChildren(List children) { 26 | this.children = children; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/model/Dimensions.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.model; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: 维度
8 | * 版本历史: 9 | * @author wpk | 2017年9月14日 下午3:33:38 |创建 10 | */ 11 | public class Dimensions { 12 | 13 | private String table; 14 | private List columns; 15 | 16 | public String getTable() { 17 | return table; 18 | } 19 | public void setTable(String table) { 20 | this.table = table; 21 | } 22 | public List getColumns() { 23 | return columns; 24 | } 25 | public void setColumns(List columns) { 26 | this.columns = columns; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/bean/Job.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.bean; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: datax作业配置
8 | * 版本历史: 9 | * @author wpk | 2017年9月20日 下午9:07:52 |创建 10 | */ 11 | public class Job { 12 | 13 | private Setting setting; 14 | private List content; 15 | 16 | public Setting getSetting() { 17 | return setting; 18 | } 19 | public void setSetting(Setting setting) { 20 | this.setting = setting; 21 | } 22 | public List getContent() { 23 | return content; 24 | } 25 | public void setContent(List content) { 26 | this.content = content; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /bd-spark/src/main/java/org/bd/spark/enums/Consts.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.enums; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 常量类
6 | * 版本历史: 7 | * @author wpk | 2017年7月20日上午11:41:38 |创建 8 | */ 9 | public class Consts { 10 | 11 | /** spark驱动类*/ 12 | public static final String spark_driver = "spark.driver"; 13 | /** spark连接地址*/ 14 | public static final String spark_url = "spark.url"; 15 | /** spark连接用户*/ 16 | public static final String spark_user = "spark.user"; 17 | /** spark连接密码*/ 18 | public static final String spark_password = "spark.password"; 19 | 20 | /** 属性配置文件名称*/ 21 | public static final String spark_properties = "spark.properties"; 22 | 23 | } 24 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/response/Realizations.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.response; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述:
6 | * 版本历史: 7 | * @author wpk | 2017年11月21日 下午3:35:24 |创建 8 | */ 9 | public class Realizations { 10 | 11 | private String type; 12 | private String realization; 13 | 14 | public String getType() { 15 | return type; 16 | } 17 | public void setType(String type) { 18 | this.type = type; 19 | } 20 | public String getRealization() { 21 | return realization; 22 | } 23 | public void setRealization(String realization) { 24 | this.realization = realization; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /bd-zk/src/main/java/org/bd/zk/ZkException.java: -------------------------------------------------------------------------------- 1 | package org.bd.zk; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: zookeeper运行异常捕获类
6 | * 版本历史: 7 | * @author wpk | 2017年12月4日 下午9:15:25 |创建 8 | */ 9 | public class ZkException extends RuntimeException { 10 | 11 | private static final long serialVersionUID = 1L; 12 | 13 | public ZkException() { 14 | super(); 15 | } 16 | 17 | public ZkException(String message) { 18 | super(message); 19 | } 20 | 21 | public ZkException(String message, Throwable cause) { 22 | super(message, cause); 23 | } 24 | 25 | public ZkException(Throwable cause) { 26 | super(cause); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /bd-hive/src/main/java/org/bd/hive/HiveException.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: hive运行异常捕获类
6 | * 版本历史: 7 | * @author wpk | 2017年10月31日 下午2:48:35 |创建 8 | */ 9 | public class HiveException extends RuntimeException { 10 | 11 | private static final long serialVersionUID = 1L; 12 | 13 | public HiveException() { 14 | super(); 15 | } 16 | 17 | public HiveException(String message) { 18 | super(message); 19 | } 20 | 21 | public HiveException(String message, Throwable cause) { 22 | super(message, cause); 23 | } 24 | 25 | public HiveException(Throwable cause) { 26 | super(cause); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/write/WriteTextTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.write; 2 | 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Row; 5 | import org.bd.spark.WriteComm; 6 | 7 | /** 8 | * 版权信息: big data module
9 | * 功能描述: 写入text文件
10 | * 版本历史: 11 | * @author wpk | 2018年1月16日 上午11:40:55 |创建 12 | */ 13 | public class WriteTextTest { 14 | 15 | public static void writeTxt(Dataset ds) throws Exception{ 16 | Dataset txt = WriteComm.getInstance().writeToTxt(ds,"C:/Users/Administrator/Desktop/sparkFile/role.txt"); 17 | txt.show(); 18 | } 19 | 20 | public static void main(String[] args) { 21 | // TODO Auto-generated method stub 22 | 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/DataxException.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: datax异常类
6 | * 版本历史: 7 | * @author wpk | 2017年10月13日 下午4:44:46 |创建 8 | */ 9 | public class DataxException extends RuntimeException { 10 | 11 | private static final long serialVersionUID = -1520131049273172806L; 12 | 13 | public DataxException() { 14 | super(); 15 | } 16 | 17 | public DataxException(String message) { 18 | super(message); 19 | } 20 | 21 | public DataxException(String message, Throwable cause) { 22 | super(message, cause); 23 | } 24 | 25 | public DataxException(Throwable cause) { 26 | super(cause); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /bd-impala/src/main/java/org/bd/impala/ImpalaException.java: -------------------------------------------------------------------------------- 1 | package org.bd.impala; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: hive运行异常捕获类
6 | * 版本历史: 7 | * @author wpk | 2017年10月31日 下午2:48:35 |创建 8 | */ 9 | public class ImpalaException extends RuntimeException { 10 | 11 | private static final long serialVersionUID = 1L; 12 | 13 | public ImpalaException() { 14 | super(); 15 | } 16 | 17 | public ImpalaException(String message) { 18 | super(message); 19 | } 20 | 21 | public ImpalaException(String message, Throwable cause) { 22 | super(message, cause); 23 | } 24 | 25 | public ImpalaException(Throwable cause) { 26 | super(cause); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /bd-kafka/src/main/resources/producer.properties: -------------------------------------------------------------------------------- 1 | # 2 | #zookeeper.connect=localhost:2181 3 | # 序列号类，默认kafka.serializer.DefaultEncoder 4 | #serializer.class=kafka.serializer.StringEncoder 5 | # 启动时producer查询brokers的列表，可以是集群中所有brokers的一个子集。 6 | # 注意，这个参数只是用来获取topic的元信息用，producer会从元信息中挑选合适的broker并与之建立socket连接。 7 | # 格式是：host1:port1,host2:port2。 8 | #metadata.broker.list=localhost:9092 9 | 10 | ################################新版本#################################################### 11 | bootstrap.servers=localhost:9092 12 | 13 | acks=all 14 | 15 | retries=0 16 | 17 | batch.size=16384 18 | 19 | linger.ms=1 20 | 21 | buffer.memory=33554432 22 | 23 | value.serializer=org.apache.kafka.common.serialization.StringSerializer 24 | 25 | key.serializer=org.apache.kafka.common.serialization.StringSerializer -------------------------------------------------------------------------------- /bd-hive/src/test/java/org/bd/hive/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive; 2 | 3 | import java.sql.ResultSet; 4 | import java.sql.ResultSetMetaData; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | 9 | /** 10 | * Unit test for simple App. 11 | */ 12 | public class AppTest { 13 | 14 | public static void main( String[] arg ) throws Exception { 15 | IHiveClient ihc = new HiveClient(); 16 | ResultSet rs = ihc.executeQuery("show partitions wpk_test.test11"); 17 | ResultSetMetaData metaData = rs.getMetaData(); 18 | int count = metaData.getColumnCount(); 19 | List list = new ArrayList(); 20 | for(int i=0;i版权信息: big data module
10 | * 功能描述: datax参数对象
11 | * 版本历史: 12 | * @author wpk | 2017年10月13日 下午10:22:17 |创建 13 | */ 14 | @SuppressWarnings("rawtypes") 15 | public interface DataxBean { 16 | 17 | /** 18 | * 描述： 获取读取对象 19 | * @author wpk | 2017年10月13日 下午10:24:24 |创建 20 | * @param map 21 | * @return Read 22 | */ 23 | public Read readBean(Map map); 24 | 25 | /** 26 | * 描述： 获取写入对象 27 | * @author wpk | 2017年10月13日 下午10:25:20 |创建 28 | * @param map 29 | * @return Write 30 | */ 31 | public Write writeBean(Map map); 32 | } 33 | -------------------------------------------------------------------------------- /bd-kafka/src/main/resources/consumer.properties: -------------------------------------------------------------------------------- 1 | #组，每个组只支持一个消费者，如果存在多个消费者，则只有一个消费到信息，其它消费者将不能消费到信息 2 | group.id=group5 3 | #参数earliest表示获取所有信息，但要确保每个组都是最新的组；默认是latest，表示获取最新信息 4 | auto.offset.reset=earliest 5 | #通过指定一个或多个代理的列表来使用配置bootstrap.server来实现对集群的连接 6 | bootstrap.servers=localhost:9092 7 | #默认为true,true表示偏移量是由配置自动;false表示偏移量由开发者自己去控制 8 | enable.auto.commit=false 9 | #每n秒提交一次 10 | auto.commit.interval.ms=1000 11 | #通过使用心跳机制，代理将自动检测测试组中的失败流程。消费者会定期自动ping集群，让集群知道它还活着。请注意，使用者是单线程的，因此只有在调用了poll(long)时才可以发送周期性心跳。只要消费者能够做到这一点，它就被认为是活的，并保留从分配给它的分区中消费的权利。如果它停止了呼叫(长时间)的心跳，超过了session . timeout。然后，它将被认为是死的，它的分区将被分配给另一个进程。 12 | session.timeout.ms=30000 13 | 14 | key.deserializer=org.apache.kafka.common.serialization.StringDeserializer 15 | 16 | value.deserializer=org.apache.kafka.common.serialization.StringDeserializer 17 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/write/WriteCsvTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.write; 2 | 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Row; 5 | import org.bd.spark.WriteComm; 6 | import org.bd.spark.enums.FormatType; 7 | 8 | /** 9 | * 版权信息: big data module
10 | * 功能描述: 写入csv文件
11 | * 版本历史: 12 | * @author wpk | 2018年1月16日 上午11:39:20 |创建 13 | */ 14 | public class WriteCsvTest { 15 | 16 | public static void writeCsv(Dataset ds) throws Exception{ 17 | Dataset parquet = WriteComm.getInstance().writeToFile(ds, FormatType.CSV, "C:/Users/Administrator/Desktop/sparkFile/role.csv"); 18 | parquet.show(); 19 | } 20 | 21 | public static void main(String[] args) { 22 | // TODO Auto-generated method stub 23 | 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/write/WriteJsonTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.write; 2 | 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Row; 5 | import org.bd.spark.WriteComm; 6 | import org.bd.spark.enums.FormatType; 7 | 8 | /** 9 | * 版权信息: big data module
10 | * 功能描述: 写入json文件
11 | * 版本历史: 12 | * @author wpk | 2018年1月16日 上午11:38:03 |创建 13 | */ 14 | public class WriteJsonTest { 15 | 16 | public static void writeJson(Dataset ds) throws Exception{ 17 | Dataset parquet = WriteComm.getInstance().writeToFile(ds, FormatType.JSON, "C:/Users/Administrator/Desktop/sparkFile/role.json"); 18 | parquet.show(); 19 | } 20 | 21 | public static void main(String[] args) { 22 | // TODO Auto-generated method stub 23 | 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /bd-zk/src/test/java/org/bd/zk/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.zk; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-hdfs/src/test/java/org/bd/hdfs/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.hdfs; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-solr/src/test/java/org/bd/solr/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.solr; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/write/WriteParquetTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.write; 2 | 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Row; 5 | import org.bd.spark.WriteComm; 6 | import org.bd.spark.enums.FormatType; 7 | 8 | /** 9 | * 版权信息: big data module
10 | * 功能描述: 写入parquet文件
11 | * 版本历史: 12 | * @author wpk | 2018年1月16日 上午11:42:34 |创建 13 | */ 14 | public class WriteParquetTest { 15 | 16 | public static void writeParquet(Dataset ds) throws Exception{ 17 | Dataset parquet = WriteComm.getInstance().writeToFile(ds, FormatType.PARQUET, "C:/Users/Administrator/Desktop/sparkFile/role.parquet"); 18 | parquet.show(); 19 | } 20 | 21 | 22 | public static void main(String[] args) { 23 | // TODO Auto-generated method stub 24 | 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /bd-datax/src/test/java/org/bd/datax/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-hbase/src/test/java/org/bd/hbase/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.hbase; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-impala/src/test/java/org/bd/impala/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.impala; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-kafka/src/test/java/org/bd/kafka/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kafka; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-kylin/src/test/java/org/bd/kylin/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-hive-udf/src/test/java/org/bd/hive/udf/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive.udf; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/read/ReadJsonTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.read; 2 | 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Row; 5 | import org.apache.spark.sql.SparkSession; 6 | import org.bd.spark.ReadComm; 7 | import org.bd.spark.enums.FormatType; 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 从json读取数据
12 | * 版本历史: 13 | * @author wpk | 2018年1月16日 上午11:20:56 |创建 14 | */ 15 | public class ReadJsonTest { 16 | 17 | public static void readJson(SparkSession spark) throws Exception{ 18 | Dataset json = ReadComm.getInstance(). 19 | readSource(spark, FormatType.JSON, "D:/wpk/devToll/workspace/nkzjProject1/idata-spark/test-file/employees.json"); 20 | json.show(); 21 | } 22 | 23 | public static void main(String[] arg){ 24 | 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /bd-mapreduce/src/test/java/org/bd/mapreduce/AppTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.mapreduce; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/JobTimeFilterEnum.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 作业的时间过滤对象
6 | * 版本历史: 7 | * @author wpk | 2017年11月21日 上午10:25:39 |创建 8 | */ 9 | public enum JobTimeFilterEnum { 10 | 11 | LAST_ONE_DAY(0), LAST_ONE_WEEK(1), LAST_ONE_MONTH(2), LAST_ONE_YEAR(3), ALL(4); 12 | 13 | private final int code; 14 | 15 | private JobTimeFilterEnum(int code) { 16 | this.code = code; 17 | } 18 | 19 | public static JobTimeFilterEnum getByCode(int code) { 20 | for (JobTimeFilterEnum timeFilter : values()) { 21 | if (timeFilter.getCode() == code) { 22 | return timeFilter; 23 | } 24 | } 25 | 26 | return null; 27 | } 28 | 29 | public int getCode() { 30 | return code; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /bd-spark/src/main/java/org/bd/spark/enums/DbmsType.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.enums; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 数据库类型
6 | * 版本历史: 7 | * @author wpk | 2017年7月25日下午3:06:44 |创建 8 | */ 9 | public enum DbmsType { 10 | 11 | /** mysq数据库*/ 12 | MYSQL{public String value(){return "mysql";}}, 13 | /** oracle数据库*/ 14 | ORACLE{public String value(){return "oracle";}}, 15 | /** spark的jdbc连接服务*/ 16 | SPARK{public String value(){return "spark";}}; 17 | 18 | public abstract String value(); 19 | 20 | /** 21 | * 描述：获取枚举类的对象 22 | * @author wpk | 2017年7月25日下午6:23:23 |创建 23 | * @param value 24 | * @return 25 | */ 26 | public static DbmsType getByValue(String value){ 27 | for(DbmsType ot : values()){ 28 | if((ot.value()).equals(value)) 29 | return ot; 30 | } 31 | return null; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /bd-spark/src/main/java/org/bd/spark/enums/FormatType.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.enums; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 读\写资源格式
6 | * 版本历史: 7 | * @author wpk | 2017年7月25日上午11:19:10 |创建 8 | */ 9 | public enum FormatType { 10 | 11 | /** txt文件类型*/ 12 | TEXT("1", "text"), 13 | /** json文件类型*/ 14 | JSON("2", "json"), 15 | /** csv文件类型*/ 16 | CSV("3", "csv"), 17 | /** parquet文件类型*/ 18 | PARQUET("4", "parquet"), 19 | /** 数据库类型*/ 20 | JDBC("5", "jdbc"); 21 | 22 | private final String state; 23 | private final String value; 24 | 25 | private FormatType(String state, String value){ 26 | this.state = state; 27 | this.value = value; 28 | } 29 | 30 | public String getState() { 31 | return this.state; 32 | } 33 | 34 | public String getValue() { 35 | return this.value; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/read/ReadParquetTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.read; 2 | 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Row; 5 | import org.apache.spark.sql.SparkSession; 6 | import org.bd.spark.ReadComm; 7 | import org.bd.spark.enums.FormatType; 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 从parquet读取数据
12 | * 版本历史: 13 | * @author wpk | 2018年1月16日 上午11:26:11 |创建 14 | */ 15 | public class ReadParquetTest { 16 | 17 | public static void readParquet(SparkSession spark) throws Exception{ 18 | Dataset parquet = ReadComm.getInstance(). 19 | readSource(spark, FormatType.PARQUET, "D:/wpk/devToll/workspace/nkzjProject1/idata-spark/test-file/users.parquet"); 20 | parquet.show(); 21 | } 22 | 23 | public static void main(String[] args) { 24 | // TODO Auto-generated method stub 25 | 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/bean/Content.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.bean; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: datax内容设置类
8 | * 版本历史: 9 | * @author wpk | 2017年9月20日 下午9:18:32 |创建 10 | */ 11 | public class Content { 12 | 13 | private Read reader; 14 | private List transformer; 15 | private Write writer; 16 | 17 | public Read getReader() { 18 | return reader; 19 | } 20 | public void setReader(Read reader) { 21 | this.reader = reader; 22 | } 23 | public List getTransformer() { 24 | return transformer; 25 | } 26 | public void setTransformer(List transformer) { 27 | this.transformer = transformer; 28 | } 29 | public Write getWriter() { 30 | return writer; 31 | } 32 | public void setWriter(Write writer) { 33 | this.writer = writer; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /bd-zk/src/main/java/org/bd/zk/AtestClient.java: -------------------------------------------------------------------------------- 1 | package org.bd.zk; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.zookeeper.KeeperException; 6 | import org.apache.zookeeper.ZooKeeper; 7 | import org.apache.zookeeper.data.Stat; 8 | 9 | public class AtestClient { 10 | 11 | public static void main(String[] arg) throws IOException, KeeperException, InterruptedException{ 12 | ZkClient zkClient = new ZkClient(); 13 | ZooKeeper zk = zkClient.getZk(); 14 | System.out.println("获取节点数据:"+zk.getData("/wpk", false, new Stat())); 15 | System.out.println("获取节点设置后的状态信息:"+zk.setData("/wpk", "nihao123".getBytes(), -1)); 16 | 17 | zkClient.registerPersistentWatcher("/wpk"); 18 | 19 | /*ZooKeeper zk = new ZooKeeper("localhost:2181", 30000, new Watcher(){ 20 | @Override 21 | public void process(WatchedEvent event){ 22 | System.out.println("Receive watched event :" + event); 23 | } 24 | });*/ 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /bd-zk/src/main/java/org/bd/zk/callback/A.java: -------------------------------------------------------------------------------- 1 | package org.bd.zk.callback; 2 | 3 | import org.I0Itec.zkclient.IZkDataListener; 4 | import org.I0Itec.zkclient.ZkClient; 5 | 6 | 7 | public class A { 8 | 9 | public static void main(String[] args) { 10 | // TODO Auto-generated method stub 11 | ZkClient zk = new ZkClient("localhost:2181", 30000); 12 | zk.subscribeDataChanges("/wpk", new IZkDataListener(){ 13 | @Override 14 | public void handleDataChange(String dataPath, Object data) throws Exception { 15 | System.out.println("数据变更"); 16 | } 17 | @Override 18 | public void handleDataDeleted(String dataPath) throws Exception { 19 | System.out.println("数据删除"); 20 | } 21 | 22 | }); 23 | Test t = new Test(); 24 | t.setId(1); 25 | t.setName("张鹏"); 26 | zk.writeData("/wpk", t); 27 | Test sd = zk.readData("/wpk"); 28 | System.out.println(sd.getId()+"\t"+sd.getName()); 29 | } 30 | 31 | 32 | } 33 | -------------------------------------------------------------------------------- /bd-hive-udf/src/main/resources/package.xml: -------------------------------------------------------------------------------- 1 | 3 | package 4 | 5 | jar 6 | 7 | true 8 | 18 | 19 | 20 | lib 21 | runtime 22 | 23 | 24 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.bd 5 | wpk-manage 6 | 0.0.1-SNAPSHOT 7 | pom 8 | 9 | bd-datax 10 | bd-elasticsearch 11 | bd-flink 12 | bd-hbase 13 | bd-hdfs 14 | bd-hive 15 | bd-hive-udf 16 | bd-impala 17 | bd-kafka 18 | bd-kylin 19 | bd-mapreduce 20 | bd-solr 21 | bd-spark 22 | bd-yarn 23 | bd-zk 24 | 25 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/cube/Dimensions.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.cube; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 维度
6 | * 版本历史: 7 | * @author wpk | 2017年9月14日 下午12:13:42 |创建 8 | */ 9 | public class Dimensions { 10 | 11 | private String table; 12 | private String column; 13 | private String name; 14 | private String derived = null; 15 | 16 | public String getTable() { 17 | return table; 18 | } 19 | public void setTable(String table) { 20 | this.table = table; 21 | } 22 | public String getColumn() { 23 | return column; 24 | } 25 | public void setColumn(String column) { 26 | this.column = column; 27 | } 28 | public String getName() { 29 | return name; 30 | } 31 | public void setName(String name) { 32 | this.name = name; 33 | } 34 | public String getDerived() { 35 | return derived; 36 | } 37 | public void setDerived(String derived) { 38 | this.derived = derived; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /bd-yarn/src/main/java/org/bd/yarn/utils/YarnUtil.java: -------------------------------------------------------------------------------- 1 | package org.bd.yarn.utils; 2 | 3 | import java.io.IOException; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.apache.hadoop.yarn.client.api.YarnClient; 7 | 8 | /** 9 | * 版权信息: big data module
10 | * 功能描述: yarn工具类
11 | * 版本历史: 12 | * @author wpk | 2018年9月1日 下午4:20:37 |创建 13 | */ 14 | public class YarnUtil { 15 | 16 | private YarnClient client; 17 | private Configuration conf; 18 | 19 | public YarnClient getClient(){ 20 | if(client == null){ 21 | Configuration conf = new Configuration(); 22 | client = YarnClient.createYarnClient(); 23 | client.init(conf); 24 | client.start(); 25 | } 26 | return client; 27 | } 28 | 29 | public Configuration getConfig(){ 30 | return conf; 31 | } 32 | 33 | public void close(){ 34 | try { 35 | if(client == null){ 36 | client.close(); 37 | } 38 | } catch (IOException e) { 39 | e.printStackTrace(); 40 | } 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/JobStatusEnum.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 作业状态
6 | * 版本历史: 7 | * @author wpk | 2017年11月21日 上午10:21:31 |创建 8 | */ 9 | public enum JobStatusEnum { 10 | 11 | NEW(0), PENDING(1), RUNNING(2), FINISHED(4), ERROR(8), DISCARDED(16), STOPPED(32); 12 | 13 | private final int code; 14 | 15 | private JobStatusEnum(int statusCode) { 16 | this.code = statusCode; 17 | } 18 | 19 | public static JobStatusEnum getByCode(int statusCode) { 20 | for (JobStatusEnum status : values()) { 21 | if (status.getCode() == statusCode) { 22 | return status; 23 | } 24 | } 25 | 26 | return null; 27 | } 28 | 29 | public int getCode() { 30 | return this.code; 31 | } 32 | 33 | public boolean isComplete() { 34 | return code == JobStatusEnum.FINISHED.getCode() || code == JobStatusEnum.ERROR.getCode() || code == JobStatusEnum.DISCARDED.getCode(); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/read/ReadCsvTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.read; 2 | 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Row; 5 | import org.apache.spark.sql.SparkSession; 6 | import org.bd.spark.ReadComm; 7 | import org.bd.spark.enums.FormatType; 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 从csv读取数据
12 | * 版本历史: 13 | * @author wpk | 2018年1月16日 上午11:22:57 |创建 14 | */ 15 | public class ReadCsvTest { 16 | 17 | public static void readCsv(SparkSession spark) throws Exception{ 18 | /*CSVWriter writer =new CSVWriter(new FileWriter("D:/wpk/devToll/workspace/nkzjProject1/idata-spark/test-file/people.csv"),','); 19 | CSVReader reader =new CSVReader(new Reader(),'\t');*/ 20 | 21 | Dataset csv = ReadComm.getInstance(). 22 | readSource(spark, FormatType.CSV, "D:/wpk/devToll/workspace/nkzjProject1/idata-spark/test-file/people.csv"); 23 | csv.show(); 24 | } 25 | 26 | public static void main(String[] args) { 27 | // TODO Auto-generated method stub 28 | 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/DataxPara.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: datax键值对参数配置
6 | * 版本历史: 7 | * @author wpk | 2017年8月7日 下午10:41:24 |创建 8 | */ 9 | public class DataxPara { 10 | 11 | public final static String job = "job"; 12 | 13 | public final static String setting = "setting";//全局设置 14 | public final static String speed = "speed";//速度设置 15 | public final static String channel = "channel"; 16 | public final static String byt = "byte"; 17 | public final static String errorLimit = "errorLimit";//错误限制 18 | public final static String record = "record"; 19 | public final static String percentage = "percentage"; 20 | 21 | public final static String content = "content";//内容设置 22 | public final static String reader = "reader";//读取设置 23 | public final static String transformer = "transformer";//转换设置 24 | public final static String writer = "writer";//写入设置 25 | 26 | public final static String name = "name"; 27 | public final static String parameter = "parameter"; 28 | 29 | } 30 | -------------------------------------------------------------------------------- /bd-zk/src/main/java/org/bd/zk/watcher/ZkLinkWatcher.java: -------------------------------------------------------------------------------- 1 | package org.bd.zk.watcher; 2 | 3 | import org.apache.zookeeper.WatchedEvent; 4 | import org.apache.zookeeper.Watcher; 5 | import org.apache.zookeeper.Watcher.Event.KeeperState; 6 | 7 | /** 8 | * 版权信息: big data module
9 | * 功能描述: zookeeper连接同步观察
10 | * 版本历史: 11 | * @author wpk | 2017年12月3日 上午10:54:59 |创建 12 | */ 13 | public class ZkLinkWatcher implements Watcher{ 14 | 15 | @Override 16 | public void process(WatchedEvent event) { 17 | if (event.getState() == KeeperState.Disconnected) {//断开连接 18 | System.out.print("====断开连接===="); 19 | } else if(event.getState() == KeeperState.SyncConnected) {//同步连接 20 | System.out.print("====同步连接===="); 21 | } else if(event.getState() == KeeperState.Expired) {//过期 22 | System.out.print("====过期===="); 23 | } else if(event.getState() == KeeperState.AuthFailed){//验证失败 24 | System.out.print("====验证失败===="); 25 | } 26 | System.out.println("Receive watched event :" + event); 27 | } 28 | 29 | 30 | } 31 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/model/Partition_desc.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.model; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 分区设置
6 | * 版本历史: 7 | * @author wpk | 2017年9月14日 下午3:38:51 |创建 8 | */ 9 | public class Partition_desc { 10 | 11 | private String partition_date_column; 12 | private String partition_date_format = "yyyy-MM-dd"; 13 | private String partition_type = "APPEND"; 14 | 15 | public String getPartition_date_column() { 16 | return partition_date_column; 17 | } 18 | public void setPartition_date_column(String partition_date_column) { 19 | this.partition_date_column = partition_date_column; 20 | } 21 | public String getPartition_date_format() { 22 | return partition_date_format; 23 | } 24 | public void setPartition_date_format(String partition_date_format) { 25 | this.partition_date_format = partition_date_format; 26 | } 27 | public String getPartition_type() { 28 | return partition_type; 29 | } 30 | public void setPartition_type(String partition_type) { 31 | this.partition_type = partition_type; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/read/ReadTextTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.read; 2 | 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Row; 5 | import org.apache.spark.sql.SparkSession; 6 | import org.bd.spark.ReadComm; 7 | import org.bd.spark.enums.FormatType; 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 从test读取数据
12 | * 版本历史: 13 | * @author wpk | 2018年1月16日 上午11:24:28 |创建 14 | */ 15 | public class ReadTextTest { 16 | 17 | public static void readText(SparkSession spark) throws Exception{ 18 | Dataset text = ReadComm.getInstance(). 19 | readSource(spark, FormatType.TEXT, "D:/wpk/devToll/workspace/nkzjProject1/idata-spark/test-file/people.txt"); 20 | // Dataset text = SparkReadCommon.getInstance(). 21 | // readSource(spark, ReadFormat.TEXT.value(), "D:/wpk/devToll/workspace/nkzjProject1/idata-spark/test-file/people.txt", 22 | // "D:/wpk/devToll/workspace/nkzjProject1/idata-spark/test-file/people2.txt"); 23 | 24 | text.show(); 25 | } 26 | 27 | public static void main(String[] args) { 28 | // TODO Auto-generated method stub 29 | 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /bd-hbase/src/main/java/org/bd/hbase/Test.java: -------------------------------------------------------------------------------- 1 | package org.bd.hbase; 2 | 3 | import java.io.IOException; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: hbase客户端
8 | * 版本历史: 9 | * @author wpk | 2017年11月8日 上午8:54:06 |创建 10 | */ 11 | public class Test { 12 | 13 | public static void main(String[] arg) throws IOException{ 14 | // createTable("wpk_test1","col1"); 15 | 16 | /*List list = new ArrayList(); 17 | list.add("col1"); 18 | list.add("col2"); 19 | list.add("COL3"); 20 | list.add("CoL4"); 21 | HBaseClient.createTable("wpk_test", list); 22 | System.out.println("新增结束..."); 23 | HBaseClient.createTable("wpk_test", "codls5");*/ 24 | 25 | // HBaseClient.deleteColumnFamily("wpk_test", "codls5"); 26 | 27 | // HBaseClient.deleteTable("member"); 28 | 29 | HBaseClient.listTableNames(); 30 | 31 | /*Collection c = HBaseClient.getHColumnDescriptors("wpk_test"); 32 | for(HColumnDescriptor d : c){ 33 | System.out.println(d.getNameAsString()); 34 | }*/ 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/request/JobBuildRequest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.request; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 创建cube任务请求参数对象
6 | * 版本历史: 7 | * @author wpk | 2017年11月21日 上午9:44:21 |创建 8 | */ 9 | public class JobBuildRequest { 10 | 11 | private long startTime; 12 | 13 | private long endTime; 14 | 15 | private String buildType; 16 | 17 | // private boolean force; 18 | 19 | public long getStartTime() { 20 | return startTime; 21 | } 22 | 23 | public void setStartTime(long startTime) { 24 | this.startTime = startTime; 25 | } 26 | 27 | public long getEndTime() { 28 | return endTime; 29 | } 30 | 31 | public void setEndTime(long endTime) { 32 | this.endTime = endTime; 33 | } 34 | 35 | public String getBuildType() { 36 | return buildType; 37 | } 38 | 39 | public void setBuildType(String buildType) { 40 | this.buildType = buildType; 41 | } 42 | 43 | /*public boolean isForce() { 44 | return force; 45 | } 46 | 47 | public void setForce(boolean force) { 48 | this.force = force; 49 | }*/ 50 | 51 | } 52 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/write/WriteMysqlTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.write; 2 | 3 | import org.apache.spark.SparkConf; 4 | import org.apache.spark.sql.Dataset; 5 | import org.apache.spark.sql.Row; 6 | import org.apache.spark.sql.SparkSession; 7 | import org.bd.spark.SessionDrive; 8 | import org.bd.spark.WriteComm; 9 | import org.bd.spark.enums.DbmsType; 10 | 11 | /** 12 | * 版权信息: big data module
13 | * 功能描述: 写入mysql数据库
14 | * 版本历史: 15 | * @author wpk | 2018年1月16日 上午11:36:42 |创建 16 | */ 17 | public class WriteMysqlTest { 18 | 19 | public static void writeMysql(Dataset ds) throws Exception{ 20 | SparkConf conf = new SparkConf(); 21 | conf.setAppName("wirteMysql"); 22 | conf.setMaster("local[2]"); 23 | conf.set("spark.some.config.option", "some-value"); 24 | SparkSession spark = SessionDrive.getInstance().getSparkSession(conf); 25 | Dataset jdbcDF = WriteComm.getInstance().writeToJDBC(ds, DbmsType.MYSQL, "a_spark_text1"); 26 | jdbcDF.show(); 27 | 28 | spark.stop(); 29 | } 30 | 31 | public static void main(String[] args) { 32 | // TODO Auto-generated method stub 33 | 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/bean/Transformer.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.bean; 2 | 3 | import java.util.List; 4 | 5 | 6 | /** 7 | * 版权信息: big data module
8 | * 功能描述: datax字段处理类
9 | * 版本历史: 10 | * @author wpk | 2017年9月20日 下午9:47:30 |创建 11 | */ 12 | public class Transformer { 13 | 14 | private String name; 15 | private Parameter parameter; 16 | 17 | public class Parameter{ 18 | private List extraPackage; 19 | private String code; 20 | 21 | public List getExtraPackage() { 22 | return extraPackage; 23 | } 24 | public void setExtraPackage(List extraPackage) { 25 | this.extraPackage = extraPackage; 26 | } 27 | public String getCode() { 28 | return code; 29 | } 30 | public void setCode(String code) { 31 | this.code = code; 32 | } 33 | 34 | } 35 | 36 | public String getName() { 37 | return name; 38 | } 39 | 40 | public void setName(String name) { 41 | this.name = name; 42 | } 43 | 44 | public Parameter getParameter() { 45 | return parameter; 46 | } 47 | 48 | public void setParameter(Parameter parameter) { 49 | this.parameter = parameter; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /bd-hive/src/main/java/org/bd/hive/utils/Consts.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive.utils; 2 | 3 | 4 | /** 5 | * 版权信息: big data module
6 | * 功能描述: 常量类
7 | * 版本历史: 8 | * @author wpk | 2017年10月31日 下午2:27:48 |创建 9 | */ 10 | public class Consts { 11 | 12 | //驱动类 13 | public static final String impala_driver = "hive.driver"; 14 | //url连接地址 15 | public static final String impala_url = "hive.url"; 16 | //连接用户 17 | public static final String impala_user = "hive.user"; 18 | //连接密码 19 | public static final String impala_password = "hive.password"; 20 | //初始化连接池连接数 21 | public static final String initialsize = "initialsize"; 22 | //连接池最大连接数 23 | public static final String maxactive = "maxactive"; 24 | //最小生存时间 25 | public static final String minEvictableIdleTimeMillis = "minEvictableIdleTimeMillis"; 26 | //开启强行回收功能 27 | public static final String removeAbandoned = "removeAbandoned"; 28 | //连接废弃超过3小时未关闭，就会被强行回收 29 | public static final String removeAbandonedTimeout = "removeAbandonedTimeout"; 30 | //30秒检测一次需要强行回收的连接 31 | public static final String timeBetweenEvictionRunsMillis = "timeBetweenEvictionRunsMillis"; 32 | 33 | //属性配置文件名称 34 | public static final String hive_properties = "hive.properties"; 35 | 36 | } 37 | -------------------------------------------------------------------------------- /bd-zk/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | 11 | bd-zk 12 | bd-zk 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | com.github.sgroschupf 28 | zkclient 29 | 0.1 30 | 31 | 32 | org.apache.zookeeper 33 | zookeeper 34 | 3.4.6 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /bd-kafka/src/main/java/org/bd/kafka/ProducerRunner.java: -------------------------------------------------------------------------------- 1 | package org.bd.kafka; 2 | 3 | import java.util.concurrent.ExecutionException; 4 | import java.util.concurrent.TimeUnit; 5 | 6 | import org.apache.kafka.clients.producer.RecordMetadata; 7 | 8 | /** 9 | * 版权信息: big data module
10 | * 功能描述: 生产者线程
11 | * 版本历史: 12 | * @author wpk | 2017年11月26日 下午3:29:05 |创建 13 | */ 14 | public class ProducerRunner implements Runnable { 15 | 16 | private ProducerClient producer; 17 | 18 | public void run() { 19 | int i=0; 20 | producer = new ProducerClient("topic1"); 21 | while(true){ 22 | try { 23 | RecordMetadata rm = producer.sendMsg("message: " + i++); 24 | System.out.println(rm.toString()); 25 | TimeUnit.SECONDS.sleep(2); 26 | } catch (InterruptedException e) { 27 | e.printStackTrace(); 28 | } catch (ExecutionException e) { 29 | e.printStackTrace(); 30 | } 31 | } 32 | } 33 | 34 | public static void main(String[] args) { 35 | ProducerRunner item = new ProducerRunner(); 36 | Thread t = new Thread(item); 37 | t.start(); 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /bd-hdfs/src/main/java/org/bd/hdfs/utils/SysVarsUtils.java: -------------------------------------------------------------------------------- 1 | package org.bd.hdfs.utils; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.lang.StringUtils; 7 | 8 | /** 9 | * 版权信息: big data module
10 | * 功能描述: 获取系统常量配置参数
11 | * 版本历史: 12 | * @author wpk | 2017年11月15日 上午10:25:15 |创建 13 | */ 14 | public class SysVarsUtils { 15 | 16 | private Map configCache = new HashMap(); 17 | 18 | private static SysVarsUtils instance; 19 | 20 | public static SysVarsUtils getInstance(){ 21 | if(instance==null)instance = new SysVarsUtils(); 22 | return instance; 23 | } 24 | 25 | /**清空缓存*/ 26 | public void clearCache(){ 27 | configCache.clear(); 28 | } 29 | 30 | /**根据参数名取得参数值 */ 31 | public String getVarByName(String name){ 32 | if(StringUtils.isEmpty(name)){ 33 | return null; 34 | } 35 | String config = configCache.get(name); 36 | if(config != null) { 37 | return config; 38 | } 39 | PropertiesUtil propertiesUtil = PropertiesUtil.getInstance(); 40 | config = propertiesUtil.getPropertyValue(Consts.HDFS_PROPERTIES, name); 41 | if(config != null){ 42 | configCache.put(name, config); 43 | } 44 | return config; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /bd-flink/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | log4j.rootLogger=INFO, console 20 | 21 | log4j.appender.console=org.apache.log4j.ConsoleAppender 22 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n 24 | -------------------------------------------------------------------------------- /bd-hive/src/main/java/org/bd/hive/utils/SysVarsUtils.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive.utils; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.lang.StringUtils; 7 | 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 获取系统常量配置参数
12 | * 版本历史: 13 | * @author wpk | 2017年10月31日 下午2:30:27 |创建 14 | */ 15 | public class SysVarsUtils { 16 | 17 | private Map configCache = new HashMap(); 18 | 19 | private static SysVarsUtils instance; 20 | 21 | public static SysVarsUtils getInstance(){ 22 | if(instance==null)instance = new SysVarsUtils(); 23 | return instance; 24 | } 25 | 26 | /**清空缓存*/ 27 | public void clearCache(){ 28 | configCache.clear(); 29 | } 30 | 31 | /**根据参数名取得参数值 */ 32 | public String getVarByName(String name){ 33 | if(StringUtils.isEmpty(name)){ 34 | return null; 35 | } 36 | String config = configCache.get(name); 37 | if(config != null) { 38 | return config; 39 | } 40 | PropertiesUtil propertiesUtil = PropertiesUtil.getInstance(); 41 | config = propertiesUtil.getPropertyValue(Consts.hive_properties, name); 42 | if(config != null){ 43 | configCache.put(name, config); 44 | } 45 | return config; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /bd-hbase/src/main/java/org/bd/hbase/utils/SysVarsUtils.java: -------------------------------------------------------------------------------- 1 | package org.bd.hbase.utils; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.lang.StringUtils; 7 | 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 获取系统常量配置参数
12 | * 版本历史: 13 | * @author wpk | 2017年10月31日 下午2:30:27 |创建 14 | */ 15 | public class SysVarsUtils { 16 | 17 | private Map configCache = new HashMap(); 18 | 19 | private static SysVarsUtils instance; 20 | 21 | public static SysVarsUtils getInstance(){ 22 | if(instance==null)instance = new SysVarsUtils(); 23 | return instance; 24 | } 25 | 26 | /**清空缓存*/ 27 | public void clearCache(){ 28 | configCache.clear(); 29 | } 30 | 31 | /**根据参数名取得参数值 */ 32 | public String getVarByName(String name){ 33 | if(StringUtils.isEmpty(name)){ 34 | return null; 35 | } 36 | String config = configCache.get(name); 37 | if(config != null) { 38 | return config; 39 | } 40 | PropertiesUtil propertiesUtil = PropertiesUtil.getInstance(); 41 | config = propertiesUtil.getPropertyValue(Consts.HBASE_PROPERTIES, name); 42 | if(config != null){ 43 | configCache.put(name, config); 44 | } 45 | return config; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /bd-impala/src/main/java/org/bd/impala/utils/SysVarsUtils.java: -------------------------------------------------------------------------------- 1 | package org.bd.impala.utils; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.lang.StringUtils; 7 | 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 获取系统常量配置参数
12 | * 版本历史: 13 | * @author wpk | 2017年8月23日 上午11:21:03 |创建 14 | */ 15 | public class SysVarsUtils { 16 | 17 | private Map configCache = new HashMap(); 18 | 19 | private static SysVarsUtils instance; 20 | 21 | public static SysVarsUtils getInstance(){ 22 | if(instance==null)instance = new SysVarsUtils(); 23 | return instance; 24 | } 25 | 26 | /**清空缓存*/ 27 | public void clearCache(){ 28 | configCache.clear(); 29 | } 30 | 31 | /**根据参数名取得参数值 */ 32 | public String getVarByName(String name){ 33 | if(StringUtils.isEmpty(name)){ 34 | return null; 35 | } 36 | String config = configCache.get(name); 37 | if(config != null) { 38 | return config; 39 | } 40 | PropertiesUtil propertiesUtil = PropertiesUtil.getInstance(); 41 | config = propertiesUtil.getPropertyValue(Consts.impala_properties, name); 42 | if(config != null){ 43 | configCache.put(name, config); 44 | } 45 | return config; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/utils/SysVarsUtils.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.utils; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.lang3.StringUtils; 7 | 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 获取系统常量配置参数
12 | * 版本历史: 13 | * @author wpk | 2017年8月23日 上午11:21:03 |创建 14 | */ 15 | public class SysVarsUtils { 16 | 17 | private Map configCache = new HashMap(); 18 | 19 | private static SysVarsUtils instance; 20 | 21 | public static SysVarsUtils getInstance(){ 22 | if(instance==null)instance = new SysVarsUtils(); 23 | return instance; 24 | } 25 | 26 | /**清空缓存*/ 27 | public void clearCache(){ 28 | configCache.clear(); 29 | } 30 | 31 | /**根据参数名取得参数值 */ 32 | public String getVarByName(String name){ 33 | if(StringUtils.isEmpty(name)){ 34 | return null; 35 | } 36 | String config = configCache.get(name); 37 | if(config != null) { 38 | return config; 39 | } 40 | PropertiesUtil propertiesUtil = PropertiesUtil.getInstance(); 41 | config = propertiesUtil.getPropertyValue(Consts.KYLIN_PROPERTIES, name); 42 | if(config != null){ 43 | configCache.put(name, config); 44 | } 45 | return config; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /bd-impala/src/main/java/org/bd/impala/utils/Consts.java: -------------------------------------------------------------------------------- 1 | package org.bd.impala.utils; 2 | 3 | 4 | /** 5 | * 版权信息: big data module
6 | * 功能描述: 常量类
7 | * 版本历史: 8 | * @author wpk | 2017年8月23日 上午11:18:51 |创建 9 | */ 10 | public class Consts { 11 | 12 | /**impala 连接配置信息参数*/ 13 | //impala驱动类 14 | public static final String impala_driver = "impala.driver"; 15 | //impala url连接地址 16 | public static final String impala_url = "impala.url"; 17 | //impala连接用户 18 | public static final String impala_user = "impala_user"; 19 | //impala连接密码 20 | public static final String impala_password = "impala.password"; 21 | //初始化连接池连接数 22 | public static final String initialsize = "initialsize"; 23 | //连接池最大连接数 24 | public static final String maxactive = "maxactive"; 25 | //最小生存时间 26 | public static final String minEvictableIdleTimeMillis = "minEvictableIdleTimeMillis"; 27 | //开启强行回收功能 28 | public static final String removeAbandoned = "removeAbandoned"; 29 | //连接废弃超过3小时未关闭，就会被强行回收 30 | public static final String removeAbandonedTimeout = "removeAbandonedTimeout"; 31 | //30秒检测一次需要强行回收的连接 32 | public static final String timeBetweenEvictionRunsMillis = "timeBetweenEvictionRunsMillis"; 33 | 34 | //属性配置文件名称 35 | public static final String impala_properties = "impala.properties"; 36 | 37 | } 38 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/read/AppReadTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.read; 2 | 3 | import java.sql.SQLException; 4 | 5 | import org.apache.spark.SparkConf; 6 | import org.apache.spark.sql.SparkSession; 7 | import org.bd.spark.SessionDrive; 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 从不同库、文件读取数据测试
12 | * 版本历史: 13 | * @author wpk | 2017年7月25日上午9:43:09 |创建 14 | */ 15 | public class AppReadTest { 16 | 17 | public static void main(String[] args) { 18 | try { 19 | System.setProperty("hadoop.home.dir", "F:\\hadoop-common-2.2.0-bin-master"); 20 | SparkConf conf = new SparkConf(); 21 | conf.setAppName("readTest"); 22 | conf.setMaster("local[2]"); 23 | conf.set("spark.some.config.option", "some-value"); 24 | SparkSession spark = SessionDrive.getInstance().getSparkSession(conf); 25 | 26 | ReadJsonTest.readJson(spark);//读取json文件 27 | ReadCsvTest.readCsv(spark);//读取csv文件 28 | ReadTextTest.readText(spark);//读取text文件 29 | ReadParquetTest.readParquet(spark);//读取parquet文件 30 | spark.stop(); 31 | 32 | ReadMysqlTest.readMysql();//读取数据库 33 | 34 | ReadHiveTest.readHive();//读取hive表 35 | } catch (SQLException e) { 36 | e.printStackTrace(); 37 | } catch (Exception e){ 38 | e.printStackTrace(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/model/Lookups.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.model; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: 查询表
8 | * 版本历史: 9 | * @author wpk | 2017年9月14日 下午3:33:51 |创建 10 | */ 11 | public class Lookups { 12 | 13 | private String table; 14 | private Join join; 15 | 16 | public class Join{ 17 | private String type; 18 | private List primary_key; 19 | private List foreign_key; 20 | 21 | public String getType() { 22 | return type; 23 | } 24 | public void setType(String type) { 25 | this.type = type; 26 | } 27 | public List getPrimary_key() { 28 | return primary_key; 29 | } 30 | public void setPrimary_key(List primary_key) { 31 | this.primary_key = primary_key; 32 | } 33 | public List getForeign_key() { 34 | return foreign_key; 35 | } 36 | public void setForeign_key(List foreign_key) { 37 | this.foreign_key = foreign_key; 38 | } 39 | } 40 | 41 | public String getTable() { 42 | return table; 43 | } 44 | 45 | public void setTable(String table) { 46 | this.table = table; 47 | } 48 | 49 | public Join getJoin() { 50 | return join; 51 | } 52 | 53 | public void setJoin(Join join) { 54 | this.join = join; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/read/ReadMysqlTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.read; 2 | 3 | import org.apache.spark.SparkConf; 4 | import org.apache.spark.sql.Dataset; 5 | import org.apache.spark.sql.Row; 6 | import org.apache.spark.sql.SparkSession; 7 | import org.bd.spark.ReadComm; 8 | import org.bd.spark.SessionDrive; 9 | import org.bd.spark.enums.DbmsType; 10 | 11 | /** 12 | * 版权信息: big data module
13 | * 功能描述: 从mysql上读取数据
14 | * 版本历史: 15 | * @author wpk | 2018年1月16日 上午11:27:10 |创建 16 | */ 17 | public class ReadMysqlTest { 18 | 19 | public static Dataset readMysql() throws Exception{ 20 | SparkConf conf = new SparkConf(); 21 | conf.setAppName("readMysql"); 22 | conf.setMaster("local[2]"); 23 | conf.set("spark.some.config.option", "some-value"); 24 | conf.set("spark.sql.warehouse.dir","file:///");//不加这一句，启动时会报错Error while instantiating 'org.apache.spark.sql.internal.SessionState' 25 | SparkSession spark = SessionDrive.getInstance().getSparkSession(conf); 26 | Dataset jdbcDF = ReadComm.getInstance().readByJDBC(spark, DbmsType.MYSQL, "a_spark_text"); 27 | jdbcDF.show(); 28 | 29 | // spark.stop(); 30 | return jdbcDF; 31 | } 32 | 33 | public static void main(String[] args) { 34 | // TODO Auto-generated method stub 35 | 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/read/ReadHiveTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.read; 2 | 3 | import org.apache.spark.SparkConf; 4 | import org.apache.spark.sql.Dataset; 5 | import org.apache.spark.sql.Row; 6 | import org.apache.spark.sql.SparkSession; 7 | import org.bd.spark.SessionDrive; 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 从hive读取数据
12 | * 版本历史: 13 | * @author wpk | 2018年1月16日 上午11:28:05 |创建 14 | */ 15 | public class ReadHiveTest { 16 | 17 | public static Dataset readHive() throws Exception{ 18 | SparkConf conf = new SparkConf(); 19 | conf.setAppName("SparkPostHive"); 20 | conf.setMaster("local[2]"); 21 | conf.set("support.type", "hive"); 22 | conf.set("spark.sql.warehouse.dir", "./spark-warehouse");//不加这一句，启动时会报错Error while instantiating 'org.apache.spark.sql.internal.SessionState' 23 | SparkSession spark = SessionDrive.getInstance().getSparkSession(conf); 24 | 25 | spark.sql("select count(1) from syeas.t_gl_voucherassistrecord").show(); 26 | 27 | Dataset ds1 = spark.sql("select * from syeas.t_gl_voucherassistrecord limit 5"); 28 | ds1.show(); 29 | 30 | //关闭程序 31 | // spark.stop(); 32 | 33 | return ds1; 34 | } 35 | 36 | public static void main(String[] args) { 37 | // TODO Auto-generated method stub 38 | 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/write/AppWriteTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.write; 2 | 3 | import java.sql.SQLException; 4 | 5 | import org.apache.spark.sql.Dataset; 6 | import org.apache.spark.sql.Row; 7 | import org.apache.spark.sql.SparkSession; 8 | import org.bd.spark.read.ReadMysqlTest; 9 | 10 | /** 11 | * 版权信息: big data module
12 | * 功能描述: 写入不同库、文件测试
13 | * 版本历史: 14 | * @author wpk | 2017年7月25日下午3:23:06 |创建 15 | */ 16 | public class AppWriteTest { 17 | 18 | public static void main(String[] args) { 19 | try { 20 | System.setProperty("hadoop.home.dir", "F:\\hadoop-common-2.2.0-bin-master"); 21 | Dataset mysql = ReadMysqlTest.readMysql();//从mysql读取 22 | 23 | //创建临时表用于sql查询 24 | mysql.createOrReplaceTempView("role"); 25 | SparkSession spark = mysql.sparkSession(); 26 | Dataset sqlDF = spark.sql("select id,name,code,operateStatus from role"); 27 | sqlDF.show(); 28 | 29 | WriteJsonTest.writeJson(sqlDF);//写入json文件 30 | WriteCsvTest.writeCsv(sqlDF);//写入csv文件 31 | WriteTextTest.writeTxt(sqlDF);//写入text文件 32 | WriteParquetTest.writeParquet(sqlDF);//写入parquet文件 33 | 34 | //写入数据库 35 | // WriteMysqlTest.writeMysql(sqlDF); 36 | } catch (SQLException e) { 37 | e.printStackTrace(); 38 | } catch (Exception e){ 39 | e.printStackTrace(); 40 | } 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /bd-elasticsearch/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.bd 5 | wpk-manage 6 | 0.0.1-SNAPSHOT 7 | 8 | 9 | com.bd.es 10 | bd-elasticsearch 11 | 12 | 13 | 14 | org.elasticsearch.client 15 | transport 16 | 6.2.2 17 | 18 | 19 | org.apache.logging.log4j 20 | log4j-api 21 | 2.9.1 22 | 23 | 24 | org.apache.logging.log4j 25 | log4j-core 26 | 2.9.1 27 | 28 | 29 | com.fasterxml.jackson.core 30 | jackson-core 31 | 2.9.6 32 | 33 | 34 | com.fasterxml.jackson.core 35 | jackson-databind 36 | 2.9.6 37 | 38 | 39 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/rest/response/ProjectRestResponse.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.rest.response; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import org.bd.kylin.response.ProjectInfo; 7 | import org.bd.kylin.rest.ProjectRest; 8 | import org.bd.kylin.utils.JsonBinder; 9 | 10 | import com.fasterxml.jackson.core.JsonParseException; 11 | import com.fasterxml.jackson.core.type.TypeReference; 12 | import com.fasterxml.jackson.databind.JsonMappingException; 13 | import com.fasterxml.jackson.databind.ObjectMapper; 14 | 15 | /** 16 | * 版权信息: big data module
17 | * 功能描述: kylin项目接口
18 | * 版本历史: 19 | * @author wpk | 2017年11月21日 下午3:23:13 |创建 20 | */ 21 | public class ProjectRestResponse extends ProjectRest{ 22 | 23 | /** 24 | * 描述： 获取项目信息 25 | * @author wpk | 2017年11月21日 下午3:24:05 |创建 26 | * @return List 27 | * @throws IOException 28 | * @throws JsonMappingException 29 | * @throws JsonParseException 30 | */ 31 | @SuppressWarnings("unchecked") 32 | public static List getProjectsResp() throws JsonParseException, JsonMappingException, IOException{ 33 | String result = getProjects(); 34 | ObjectMapper om = JsonBinder.buildNonNullBinder().getMapper(); 35 | List list = (List)om.readValue(result, new TypeReference>() {}); 36 | return list; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/request/SQLRequest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.request; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: 查询语句请求参数
8 | * 版本历史: 9 | * @author wpk | 2017年11月20日 下午9:54:14 |创建 10 | */ 11 | public class SQLRequest implements Serializable { 12 | 13 | protected static final long serialVersionUID = 1L; 14 | 15 | private String sql; 16 | private String project; 17 | private Integer offset = 0; 18 | private Integer limit = 0; 19 | private boolean acceptPartial = true; 20 | 21 | public SQLRequest() { 22 | } 23 | 24 | public String getSql() { 25 | return sql; 26 | } 27 | 28 | public void setSql(String sql) { 29 | this.sql = sql; 30 | } 31 | 32 | public String getProject() { 33 | return project; 34 | } 35 | 36 | public void setProject(String project) { 37 | this.project = project; 38 | } 39 | 40 | public Integer getOffset() { 41 | return offset; 42 | } 43 | 44 | public void setOffset(Integer offset) { 45 | this.offset = offset; 46 | } 47 | 48 | public Integer getLimit() { 49 | return limit; 50 | } 51 | 52 | public void setLimit(Integer limit) { 53 | this.limit = limit; 54 | } 55 | 56 | public boolean getAcceptPartial() { 57 | return acceptPartial; 58 | } 59 | 60 | public void setAcceptPartial(boolean acceptPartial) { 61 | this.acceptPartial = acceptPartial; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/bean/SettingBean.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.bean; 2 | 3 | import java.util.Map; 4 | 5 | import org.bd.datax.DataxPara; 6 | import org.bd.datax.bean.Setting.ErrorLimit; 7 | import org.bd.datax.bean.Setting.Speed; 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: setting转换类
12 | * 版本历史: 13 | * @author wpk | 2017年10月13日 下午10:54:36 |创建 14 | */ 15 | @SuppressWarnings("rawtypes") 16 | public class SettingBean { 17 | 18 | /** 19 | * 描述： 获取全局对象 20 | * @author wpk | 2017年10月13日 下午10:55:03 |创建 21 | * @param map 22 | * @return 23 | * @return Setting 24 | */ 25 | public Setting getBean(Map map){ 26 | Setting bean = new Setting(); 27 | Speed speed = bean.new Speed(); 28 | ErrorLimit errorLimit = bean.new ErrorLimit(); 29 | 30 | if(map.containsKey(DataxPara.channel) && map.get(DataxPara.channel)!=null){ 31 | speed.setChannel(Integer.parseInt(map.get(DataxPara.channel).toString())); 32 | } 33 | 34 | if(map.containsKey(DataxPara.record) && map.get(DataxPara.record)!=null){ 35 | errorLimit.setRecord(Integer.parseInt(map.get(DataxPara.record).toString())); 36 | } 37 | if(map.containsKey(DataxPara.percentage) && map.get(DataxPara.percentage)!=null){ 38 | errorLimit.setPercentage(Double.parseDouble(map.get(DataxPara.percentage).toString())); 39 | } 40 | 41 | bean.setSpeed(speed); 42 | bean.setErrorLimit(errorLimit); 43 | return bean; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/cube/Rowkey.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.cube; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: rowkey配置
8 | * 版本历史: 9 | * @author wpk | 2017年9月14日 下午12:24:26 |创建 10 | */ 11 | public class Rowkey { 12 | 13 | private List rowkey_columns; 14 | 15 | public class Rowkey_columns{ 16 | private String column; 17 | private String encoding = "dict"; 18 | private Integer valueLength = 0; 19 | private String isShardBy = "false"; 20 | 21 | public String getColumn() { 22 | return column; 23 | } 24 | public void setColumn(String column) { 25 | this.column = column; 26 | } 27 | public String getEncoding() { 28 | return encoding; 29 | } 30 | public void setEncoding(String encoding) { 31 | this.encoding = encoding; 32 | } 33 | public Integer getValueLength() { 34 | return valueLength; 35 | } 36 | public void setValueLength(Integer valueLength) { 37 | this.valueLength = valueLength; 38 | } 39 | public String getIsShardBy() { 40 | return isShardBy; 41 | } 42 | public void setIsShardBy(String isShardBy) { 43 | this.isShardBy = isShardBy; 44 | } 45 | } 46 | 47 | public List getRowkey_columns() { 48 | return rowkey_columns; 49 | } 50 | 51 | public void setRowkey_columns(List rowkey_columns) { 52 | this.rowkey_columns = rowkey_columns; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /bd-hbase/src/main/java/org/bd/hbase/utils/HBaseUtils.java: -------------------------------------------------------------------------------- 1 | package org.bd.hbase.utils; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import javax.activation.UnsupportedDataTypeException; 7 | 8 | /** 9 | * 版权信息: big data module
10 | * 功能描述: hbase工具类
11 | * 版本历史: 12 | * @author wpk | 2017年11月9日 下午2:23:45 |创建 13 | */ 14 | public class HBaseUtils { 15 | 16 | /** 17 | * 描述： 参数转换 18 | * @author wpk | 2017年11月9日 下午2:24:31 |创建 19 | * @param obj 20 | * @throws UnsupportedDataTypeException 21 | * @return List 22 | */ 23 | @SuppressWarnings("unchecked") 24 | public static List familyDataTypeConver(Object obj) throws UnsupportedDataTypeException{ 25 | List list = new ArrayList(); 26 | if(obj instanceof String){ 27 | list.add((String)obj); 28 | }else if(obj instanceof List){ 29 | List objList = (List)obj; 30 | if(null == objList || objList.size() == 0) return list; 31 | Object _obj= objList.get(0); 32 | boolean flag = _obj instanceof String; 33 | if(!flag){ 34 | String str = String.format("参数不支持%s类型转%s类型", _obj.getClass().getName(), String.class.getName()); 35 | throw new UnsupportedDataTypeException(str); 36 | }else{ 37 | list.addAll((List)obj); 38 | } 39 | }else{ 40 | String str = String.format("参数不支持%s类型", obj.getClass().getName()); 41 | throw new UnsupportedDataTypeException(str); 42 | } 43 | return list; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/cube/Hbase_mapping.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.cube; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: hbase表映射
8 | * 版本历史: 9 | * @author wpk | 2017年9月14日 下午12:34:42 |创建 10 | */ 11 | public class Hbase_mapping { 12 | 13 | private List column_family; 14 | 15 | public class Column_family{ 16 | private String name = "F1"; 17 | private List columns; 18 | 19 | public String getName() { 20 | return name; 21 | } 22 | public void setName(String name) { 23 | this.name = name; 24 | } 25 | public List getColumns() { 26 | return columns; 27 | } 28 | public void setColumns(List columns) { 29 | this.columns = columns; 30 | } 31 | } 32 | 33 | public class Columns{ 34 | private String qualifier = "M"; 35 | private List measure_refs; 36 | 37 | public String getQualifier() { 38 | return qualifier; 39 | } 40 | public void setQualifier(String qualifier) { 41 | this.qualifier = qualifier; 42 | } 43 | public List getMeasure_refs() { 44 | return measure_refs; 45 | } 46 | public void setMeasure_refs(List measure_refs) { 47 | this.measure_refs = measure_refs; 48 | } 49 | } 50 | 51 | public List getColumn_family() { 52 | return column_family; 53 | } 54 | 55 | public void setColumn_family(List column_family) { 56 | this.column_family = column_family; 57 | } 58 | 59 | } 60 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/mysql/MysqlPara.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.mysql; 2 | 3 | import org.bd.datax.DataxPara; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: mysql键值对参数配置
8 | * 版本历史: 9 | * @author wpk | 2017年8月8日 下午3:12:28 |创建 10 | */ 11 | public class MysqlPara extends DataxPara{ 12 | 13 | //=======================读写共同参数================================ 14 | public final static String connection = "connection"; 15 | /** 连接路径（多条连接以逗号分隔）*/ 16 | public final static String jdbcUrl = "jdbcUrl"; 17 | /** 连接用户名*/ 18 | public final static String username = "username"; 19 | /** 密码*/ 20 | public final static String password = "password"; 21 | /** 表（多个表以逗号分隔）*/ 22 | public final static String table = "table"; 23 | /** 列*/ 24 | public final static String column = "column"; 25 | //=======================读取参数================================ 26 | /** 分片字段*/ 27 | public final static String splitPk = "splitPk"; 28 | /** 筛选条件*/ 29 | public final static String where = "where"; 30 | /** 执行语句，该值将忽略table、column、where条件的配置*/ 31 | public final static String querySql = "querySql"; 32 | //=======================写入参数================================ 33 | /** 写入前执行该语句（多条语句以分号分隔）*/ 34 | public final static String preSql = "preSql"; 35 | /** 写入后执行该语句（多条语句以分号分隔）*/ 36 | public final static String postSql = "postSql"; 37 | /** 定义一次性批量写入数据量，注意该值过大将出现OOM情况*/ 38 | public final static String batchSize = "batchSize"; 39 | /** 写入模式，如替换、更新、插入等*/ 40 | public final static String writeMode = "writeMode"; 41 | 42 | } 43 | -------------------------------------------------------------------------------- /bd-yarn/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | 4 | com.bd 5 | wpk-manage 6 | 0.0.1-SNAPSHOT 7 | 8 | 9 | bd-yarn 10 | 11 | 12 | 13 | junit 14 | junit 15 | 3.8.1 16 | test 17 | 18 | 19 | com.bd 20 | bd-hdfs 21 | 0.0.1-SNAPSHOT 22 | 23 | 24 | 25 | org.apache.hadoop 26 | hadoop-yarn-common 27 | 3.1.1 28 | 29 | 30 | org.apache.hadoop 31 | hadoop-yarn-client 32 | 3.1.1 33 | 34 | 35 | org.apache.hadoop 36 | hadoop-yarn-server-resourcemanager 37 | 3.1.1 38 | 39 | 40 | org.apache.hadoop 41 | hadoop-yarn-applications-distributedshell 42 | 3.1.1 43 | provided 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /bd-solr/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | 11 | bd-solr 12 | bd-solr 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | org.apache.solr 28 | solr-solrj 29 | 5.3.1 30 | 31 | 32 | com.github.pagehelper 33 | pagehelper 34 | 3.7.5 35 | 36 | 37 | org.apache.commons 38 | commons-lang3 39 | 3.5 40 | 41 | 42 | org.apache.ibatis 43 | ibatis-core 44 | 3.0 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/sqlserver/SqlServerPara.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.sqlserver; 2 | 3 | import org.bd.datax.DataxPara; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: sqlserver键值对参数配置
8 | * 版本历史: 9 | * @author wpk | 2017年8月8日 下午2:38:37 |创建 10 | */ 11 | public class SqlServerPara extends DataxPara{ 12 | 13 | //=======================读写共同参数================================ 14 | public final static String connection = "connection"; 15 | /** 连接路径（多条连接以逗号分隔）*/ 16 | public final static String jdbcUrl = "jdbcUrl"; 17 | /** 连接用户名*/ 18 | public final static String username = "username"; 19 | /** 密码*/ 20 | public final static String password = "password"; 21 | /** 表（多个表以逗号分隔）*/ 22 | public final static String table = "table"; 23 | /** 列*/ 24 | public final static String column = "column"; 25 | //=======================读取参数================================ 26 | /** 分片字段*/ 27 | public final static String splitPk = "splitPk"; 28 | /** 筛选条件*/ 29 | public final static String where = "where"; 30 | /** 执行语句，该值将忽略table、column、where条件的配置*/ 31 | public final static String querySql = "querySql"; 32 | /** 定义一次性批量获取数据量，注意该值过大将出现OOM情况*/ 33 | public final static String fetchSize = "fetchSize"; 34 | //=======================写入参数================================ 35 | /** 写入前执行该语句（多条语句以分号分隔）*/ 36 | public final static String preSql = "preSql"; 37 | /** 写入后执行该语句（多条语句以分号分隔）*/ 38 | public final static String postSql = "postSql"; 39 | /** 定义一次性批量写入数据量，注意该值过大将出现OOM情况*/ 40 | public final static String batchSize = "batchSize"; 41 | 42 | } 43 | -------------------------------------------------------------------------------- /bd-flink/src/main/java/org/bd/flink/idgenerate/MD5XdrIdGenerator.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink.idgenerate; 2 | 3 | import lombok.Builder; 4 | import lombok.NonNull; 5 | 6 | import java.security.MessageDigest; 7 | import java.security.NoSuchAlgorithmException; 8 | 9 | @Builder 10 | public class MD5XdrIdGenerator implements ESIdGenerator { 11 | @Override 12 | public String generate(final String msg, final int idLength) { 13 | try { 14 | MessageDigest md = MessageDigest.getInstance("MD5"); 15 | md.update(msg.getBytes(), 0, msg.length()); 16 | 17 | byte messageDigest[] = md.digest(); 18 | 19 | char[] hexDigits = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; 20 | char[] resultCharArray = new char[messageDigest.length * 2]; 21 | int index = 0; 22 | for (byte b : messageDigest) { 23 | resultCharArray[index++] = hexDigits[b >>> 4 & 0xf]; 24 | resultCharArray[index++] = hexDigits[b & 0xf]; 25 | } 26 | String result = new String(resultCharArray); 27 | int startIndex = 0; 28 | int endIndex = 32; 29 | if(idLength <= 24 && idLength > 0){ 30 | startIndex = 8; 31 | endIndex = 8 + idLength; 32 | }else if(idLength > 24 && idLength <=32){ 33 | endIndex = idLength; 34 | } 35 | return result.substring(startIndex, endIndex); 36 | 37 | } catch (NoSuchAlgorithmException e) { 38 | return null; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /bd-datax/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | 11 | bd-datax 12 | bd-datax 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | 28 | com.fasterxml.jackson.core 29 | jackson-core 30 | 2.6.5 31 | 32 | 33 | com.fasterxml.jackson.core 34 | jackson-databind 35 | 2.6.5 36 | compile 37 | 38 | 39 | org.apache.commons 40 | commons-lang3 41 | 3.5 42 | 43 | 44 | org.json 45 | json 46 | 20160810 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /bd-impala/src/main/java/org/bd/impala/utils/ImpaIaTableColumn.java: -------------------------------------------------------------------------------- 1 | package org.bd.impala.utils; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: impala表的结构对象
8 | * 版本历史: 9 | * @author wpk | 2017年8月23日 上午11:21:36 |创建 10 | */ 11 | @Deprecated 12 | public class ImpaIaTableColumn { 13 | 14 | public final static String COLUMN_NAME = "name"; 15 | public final static String COLUMN_TYPE = "type"; 16 | public final static String COLUMN_COMMENT = "comment"; 17 | 18 | private String name = ""; 19 | private String type = "string"; 20 | private String comment = ""; 21 | 22 | public ImpaIaTableColumn(){ 23 | } 24 | public ImpaIaTableColumn(String name){ 25 | this.name = name.trim(); 26 | } 27 | public ImpaIaTableColumn(String name, String type){ 28 | this.name = name.trim(); 29 | this.type = type.trim(); 30 | } 31 | public ImpaIaTableColumn(String name, String type, String comment){ 32 | this.name = name.trim(); 33 | this.type = type.trim(); 34 | this.comment = comment.trim(); 35 | } 36 | public String getName() { 37 | return name; 38 | } 39 | public void setName(String name) { 40 | this.name = name.trim(); 41 | } 42 | public String getType() { 43 | return type; 44 | } 45 | public void setType(String type) { 46 | if(StringUtils.isNotEmpty(type.trim())){ 47 | this.type = type.trim(); 48 | } 49 | } 50 | public String getComment() { 51 | return comment; 52 | } 53 | public void setComment(String comment) { 54 | this.comment = comment.trim(); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/cube/Aggregation_groups.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.cube; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述:
8 | * 版本历史: 9 | * @author wpk | 2017年9月14日 下午3:15:47 |创建 10 | */ 11 | public class Aggregation_groups { 12 | 13 | private List includes; 14 | private Select_rule select_rule; 15 | 16 | public class Select_rule{ 17 | private List mandatory_dims; 18 | private List> hierarchy_dims; 19 | private List> joint_dims; 20 | 21 | public List getMandatory_dims() { 22 | return mandatory_dims; 23 | } 24 | public void setMandatory_dims(List mandatory_dims) { 25 | this.mandatory_dims = mandatory_dims; 26 | } 27 | public List> getHierarchy_dims() { 28 | return hierarchy_dims; 29 | } 30 | public void setHierarchy_dims(List> hierarchy_dims) { 31 | this.hierarchy_dims = hierarchy_dims; 32 | } 33 | public List> getJoint_dims() { 34 | return joint_dims; 35 | } 36 | public void setJoint_dims(List> joint_dims) { 37 | this.joint_dims = joint_dims; 38 | } 39 | } 40 | 41 | public List getIncludes() { 42 | return includes; 43 | } 44 | 45 | public void setIncludes(List includes) { 46 | this.includes = includes; 47 | } 48 | 49 | public Select_rule getSelect_rule() { 50 | return select_rule; 51 | } 52 | 53 | public void setSelect_rule(Select_rule select_rule) { 54 | this.select_rule = select_rule; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/oracle/OraclePara.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.oracle; 2 | 3 | import org.bd.datax.DataxPara; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: oracle键值对参数配置
8 | * 版本历史: 9 | * @author wpk | 2017年8月8日 上午9:18:20 |创建 10 | */ 11 | public class OraclePara extends DataxPara{ 12 | 13 | //=======================读写共同参数================================ 14 | public final static String connection = "connection"; 15 | /** 连接路径（多条连接以逗号分隔）*/ 16 | public final static String jdbcUrl = "jdbcUrl"; 17 | /** 连接用户名*/ 18 | public final static String username = "username"; 19 | /** 密码*/ 20 | public final static String password = "password"; 21 | /** 表（多个表以逗号分隔）*/ 22 | public final static String table = "table"; 23 | /** 列*/ 24 | public final static String column = "column"; 25 | /** 连接配置*/ 26 | public final static String session = "session"; 27 | //=======================读取参数================================ 28 | /** 分片字段*/ 29 | public final static String splitPk = "splitPk"; 30 | /** 筛选条件*/ 31 | public final static String where = "where"; 32 | /** 执行语句，该值将忽略table、column、where条件的配置*/ 33 | public final static String querySql = "querySql"; 34 | /** 定义一次性批量获取数据量，注意该值过大将出现OOM情况*/ 35 | public final static String fetchSize = "fetchSize"; 36 | //=======================写入参数================================ 37 | /** 写入前执行该语句（多条语句以分号分隔）*/ 38 | public final static String preSql = "preSql"; 39 | /** 写入后执行该语句（多条语句以分号分隔）*/ 40 | public final static String postSql = "postSql"; 41 | /** 定义一次性批量写入数据量，注意该值过大将出现OOM情况*/ 42 | public final static String batchSize = "batchSize"; 43 | 44 | } 45 | -------------------------------------------------------------------------------- /bd-spark/src/main/java/org/bd/spark/utils/SysVarsUtils.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.utils; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.lang.StringUtils; 7 | import org.bd.spark.enums.Consts; 8 | 9 | 10 | /** 11 | * 版权信息: big data module
12 | * 功能描述: 获取系统常量配置参数
13 | * 版本历史: 14 | * @author wpk | 2017年7月20日下午3:00:24 |创建 15 | */ 16 | public class SysVarsUtils { 17 | 18 | private Map configCache = new HashMap(); 19 | 20 | private volatile static SysVarsUtils instance; 21 | 22 | /** 23 | * 描述：获取对象 24 | * @author wpk | 2017年7月25日下午6:27:50 |创建 25 | * @return 26 | */ 27 | public static SysVarsUtils getInstance(){ 28 | if(instance==null){ 29 | synchronized (SysVarsUtils.class) { 30 | if(instance == null){ 31 | instance = new SysVarsUtils(); 32 | } 33 | } 34 | } 35 | return instance; 36 | } 37 | 38 | /**清空缓存*/ 39 | public void clearCache(){ 40 | configCache.clear(); 41 | } 42 | 43 | /** 44 | * 描述：根据参数名取得参数值 45 | * @author wpk | 2017年7月25日下午6:28:24 |创建 46 | * @param name 47 | * @return 48 | */ 49 | public String getVarByName(String name){ 50 | if(StringUtils.isEmpty(name)){ 51 | return null; 52 | } 53 | String config = configCache.get(name); 54 | if(config != null) { 55 | return config; 56 | } 57 | PropertiesUtil propertiesUtil = PropertiesUtil.getInstance(); 58 | config = propertiesUtil.getPropertyValue(Consts.spark_properties, name); 59 | if(config != null){ 60 | configCache.put(name, config); 61 | } 62 | return config; 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /bd-kafka/src/main/java/org/bd/kafka/ConsumerRunner.java: -------------------------------------------------------------------------------- 1 | package org.bd.kafka; 2 | 3 | import java.util.concurrent.atomic.AtomicBoolean; 4 | 5 | import org.apache.kafka.clients.consumer.ConsumerRecord; 6 | import org.apache.kafka.clients.consumer.ConsumerRecords; 7 | import org.apache.kafka.common.errors.WakeupException; 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 消费者线程
12 | * 版本历史: 13 | * @author wpk | 2017年11月26日 下午3:18:25 |创建 14 | */ 15 | public class ConsumerRunner implements Runnable { 16 | 17 | private AtomicBoolean closed = new AtomicBoolean(false); 18 | private ConsumerClient consumer; 19 | 20 | public void run() { 21 | ConsumerClient item = new ConsumerClient("topic1"); 22 | try { 23 | while (!closed.get()) { 24 | ConsumerRecords records = item.getRecords(); 25 | for (ConsumerRecord record : records){ 26 | System.out.printf("topic = %s, partition = %s offset = %d, key = %s, value = %s\n",record.topic(), record.partition(), record.offset(), record.key(), record.value()); 27 | } 28 | } 29 | } catch (WakeupException e) { 30 | if (!closed.get()) throw e; 31 | } finally { 32 | item.close(); 33 | } 34 | } 35 | 36 | public void shutdown() { 37 | closed.set(true); 38 | consumer.wakeup(); 39 | } 40 | 41 | public static void main(String[] args) { 42 | ConsumerRunner item = new ConsumerRunner();// 使用kafka集群中创建好的主题 test 43 | Thread t = new Thread(item); 44 | t.start(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/request/ModelRequest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.request; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 创建模型请求参数
6 | * 版本历史: 7 | * @author wpk | 2017年11月21日 上午9:49:32 |创建 8 | */ 9 | public class ModelRequest { 10 | 11 | private String project; 12 | private String modelDescData; 13 | /*private String uuid; 14 | private String modelName; 15 | private boolean successful; 16 | private String message;*/ 17 | 18 | public ModelRequest() {} 19 | 20 | /*public ModelRequest(String modelName, String modelDescData) { 21 | this.modelName = modelName; 22 | this.modelDescData = modelDescData; 23 | }*/ 24 | 25 | public String getProject() { 26 | return project; 27 | } 28 | 29 | public void setProject(String project) { 30 | this.project = project; 31 | } 32 | 33 | public String getModelDescData() { 34 | return modelDescData; 35 | } 36 | 37 | public void setModelDescData(String modelDescData) { 38 | this.modelDescData = modelDescData; 39 | } 40 | 41 | /*public String getUuid() { 42 | return uuid; 43 | } 44 | 45 | public void setUuid(String uuid) { 46 | this.uuid = uuid; 47 | } 48 | 49 | public String getModelName() { 50 | return modelName; 51 | } 52 | 53 | public void setModelName(String modelName) { 54 | this.modelName = modelName; 55 | } 56 | 57 | public boolean isSuccessful() { 58 | return successful; 59 | } 60 | 61 | public void setSuccessful(boolean successful) { 62 | this.successful = successful; 63 | } 64 | 65 | public String getMessage() { 66 | return message; 67 | } 68 | 69 | public void setMessage(String message) { 70 | this.message = message; 71 | }*/ 72 | 73 | } 74 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/bean/Setting.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.bean; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: data全局参数设置类
6 | * 版本历史: 7 | * @author wpk | 2017年9月20日 下午9:08:06 |创建 8 | */ 9 | public class Setting { 10 | 11 | private Speed speed;//设置传输速度 byte/s 尽量逼近这个速度但是不高于它 12 | private ErrorLimit errorLimit;//出错限制 13 | 14 | public class Speed{ 15 | private Integer channel = 1;//表示通道数量，byte表示通道速度，如果单通道速度1MB，配置byte为1048576表示一个channel 16 | // private Integer _byte = 1048576; 17 | 18 | public Integer getChannel() { 19 | return channel; 20 | } 21 | public void setChannel(Integer channel) { 22 | this.channel = channel; 23 | } 24 | /*public Integer get_byte() { 25 | return _byte; 26 | } 27 | public void set_byte(Integer _byte) { 28 | this._byte = _byte; 29 | }*/ 30 | } 31 | 32 | public class ErrorLimit{ 33 | private Integer record = 0;//先选择record 34 | private Double percentage = 0.02;//百分比 1表示100% 35 | 36 | public Integer getRecord() { 37 | return record; 38 | } 39 | public void setRecord(Integer record) { 40 | this.record = record; 41 | } 42 | public Double getPercentage() { 43 | return percentage; 44 | } 45 | public void setPercentage(Double percentage) { 46 | this.percentage = percentage; 47 | } 48 | } 49 | 50 | public Speed getSpeed() { 51 | return speed; 52 | } 53 | 54 | public void setSpeed(Speed speed) { 55 | this.speed = speed; 56 | } 57 | 58 | public ErrorLimit getErrorLimit() { 59 | return errorLimit; 60 | } 61 | 62 | public void setErrorLimit(ErrorLimit errorLimit) { 63 | this.errorLimit = errorLimit; 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/rest/TableRest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.rest; 2 | 3 | import org.bd.kylin.RestRequstHandle; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: 获取hive表信息
8 | * 版本历史: 9 | * @author wpk | 2017年11月21日 上午8:57:10 |创建 10 | */ 11 | public class TableRest extends RestRequstHandle{ 12 | 13 | /** 14 | * 描述： 获取hive所有数据库 15 | * @author wpk | 2017年11月21日 上午8:58:49 |创建 16 | * @return String 17 | */ 18 | public static String showHiveDatabases(){ 19 | String para = "tables/hive"; 20 | String result = request(para, RequestMethod.GET); 21 | return result; 22 | } 23 | 24 | /** 25 | * 描述： 获取hive指定库下的所有表 26 | * @author wpk | 2017年11月21日 上午9:01:00 |创建 27 | * @param database 数据库名 28 | * @return String 29 | */ 30 | public static String showHiveTables(String database){ 31 | String para = "tables/hive/"+database; 32 | String result = request(para, RequestMethod.GET); 33 | return result; 34 | } 35 | 36 | /** 37 | * 描述： 获取kylin上存在的所有hive表 38 | * @author wpk | 2017年11月21日 上午9:07:04 |创建 39 | * @param project kylin指定项目名 40 | * @return String 41 | */ 42 | public static String getTableDesc(String project){ 43 | return getTableDesc(project, false); 44 | } 45 | 46 | /** 47 | * 描述： 获取kylin上存在的所有hive表 48 | * @author wpk | 2017年11月21日 上午9:33:18 |创建 49 | * @param project kylin指定项目名 50 | * @param ext 表存储的详细信息，true表示展示，false表示隐藏 51 | * @return String 52 | */ 53 | public static String getTableDesc(String project, boolean ext){ 54 | String para = "tables?project="+project+"&ext="+ext; 55 | String result = request(para, RequestMethod.GET); 56 | return result; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /bd-hive/src/main/java/org/bd/hive/DbCloseUtils.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive; 2 | 3 | import java.sql.Connection; 4 | import java.sql.ResultSet; 5 | import java.sql.Statement; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * 版权信息: big data module
12 | * 功能描述: 数据库连接关闭类
13 | * 版本历史: 14 | * @author wpk | 2017年10月31日 上午10:05:10 |创建 15 | */ 16 | public class DbCloseUtils { 17 | 18 | private static final Logger logger = LoggerFactory.getLogger(DbCloseUtils.class); 19 | 20 | /** 21 | * 描述： 关闭结果集 22 | * @author wpk | 2017年10月31日 上午10:05:53 |创建 23 | * @param rs 24 | * @return void 25 | */ 26 | public static void close(final ResultSet rs) { 27 | close((AutoCloseable) rs); 28 | } 29 | 30 | /** 31 | * 描述： 关闭声明 32 | * @author wpk | 2017年10月31日 上午10:06:27 |创建 33 | * @param stmt 34 | * @return void 35 | */ 36 | public static void close(final Statement stmt) { 37 | close((AutoCloseable) stmt); 38 | } 39 | 40 | /** 41 | * 描述： 关闭连接 42 | * @author wpk | 2017年10月31日 上午10:06:41 |创建 43 | * @param conn 44 | * @return void 45 | */ 46 | public static void close(final Connection conn) { 47 | close((AutoCloseable) conn); 48 | } 49 | 50 | /** 51 | * 描述： 52 | * @author wpk | 2017年10月31日 上午10:07:03 |创建 53 | * @param closeable 54 | * @return void 55 | */ 56 | public static void close(final AutoCloseable closeable) { 57 | try { 58 | if (closeable != null) { 59 | closeable.close(); 60 | } 61 | } catch (final Exception ioe) { 62 | logger.error("", ioe); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/kafka/UserKafkaProducer.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.kafka; 2 | 3 | import java.util.Properties; 4 | 5 | import org.apache.kafka.clients.producer.KafkaProducer; 6 | import org.apache.kafka.clients.producer.ProducerRecord; 7 | 8 | public class UserKafkaProducer extends Thread { 9 | private final KafkaProducer producer; 10 | private final String topic; 11 | private final Properties props = new Properties(); 12 | public UserKafkaProducer(String topic) 13 | { 14 | props.put("metadata.broker.list", "192.168.32.115:9092"); 15 | props.put("bootstrap.servers", "192.168.32.115:9092"); 16 | props.put("acks", "all"); 17 | props.put("retries", 0); 18 | props.put("batch.size", 16384); 19 | props.put("linger.ms", 1); 20 | props.put("buffer.memory", 33554432); 21 | props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 22 | props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); 23 | producer = new KafkaProducer(props); 24 | this.topic = topic; 25 | } 26 | 27 | @Override 28 | public void run() { 29 | int messageNo = 1; 30 | while (true) 31 | { 32 | String messageStr = new String("Message_" + messageNo); 33 | System.out.println("Send:" + messageStr); 34 | producer.send(new ProducerRecord(topic, messageStr)); 35 | messageNo++; 36 | try { 37 | sleep(3000); 38 | } catch (InterruptedException e) { 39 | e.printStackTrace(); 40 | } 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/hdfs/HdfsPara.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.hdfs; 2 | 3 | import org.bd.datax.DataxPara; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: HDFS键值对参数配置
8 | * 版本历史: 9 | * @author wpk | 2017年8月8日 上午10:26:39 |创建 10 | */ 11 | public class HdfsPara extends DataxPara{ 12 | 13 | //=======================读写共同参数================================ 14 | /** 文件路径*/ 15 | public final static String path = "path"; 16 | /** hdfs文件系统namenode节点地址*/ 17 | public final static String defaultFS = "defaultFS"; 18 | /** 文件类型*/ 19 | public final static String fileType = "fileType"; 20 | /** 列*/ 21 | public final static String column = "column"; 22 | /** 字段分隔符*/ 23 | public final static String fieldDelimiter = "fieldDelimiter"; 24 | /** 当前fileType（文件类型）为csv下的文件压缩方式*/ 25 | public final static String compress = "compress"; 26 | /** 文件编码*/ 27 | public final static String encoding = "encoding"; 28 | /** 空指针定义*/ 29 | public final static String nullFormat = "nullFormat"; 30 | /** 是否有Kerberos认证*/ 31 | public final static String haveKerberos = "haveKerberos"; 32 | /** Kerberos认证 keytab文件路径，绝对路径*/ 33 | public final static String kerberosKeytabFilePath = "kerberosKeytabFilePath"; 34 | /** Kerberos认证Principal名*/ 35 | public final static String kerberosPrincipal = "kerberosPrincipal"; 36 | 37 | //======================读取参数================================= 38 | /** 读取CSV类型文件参数配置*/ 39 | public final static String csvReaderConfig = "csvReaderConfig"; 40 | 41 | //=======================写入参数================================ 42 | /** 文件名*/ 43 | public final static String fileName = "fileName"; 44 | /** hdfswriter写入前数据清理处理模式*/ 45 | public final static String writeMode = "writeMode"; 46 | 47 | } 48 | -------------------------------------------------------------------------------- /bd-kafka/src/main/java/org/bd/kafka/ATest02.java: -------------------------------------------------------------------------------- 1 | package org.bd.kafka; 2 | 3 | import java.util.UUID; 4 | import java.util.concurrent.ExecutionException; 5 | 6 | import net.sf.json.JSONArray; 7 | import net.sf.json.JSONObject; 8 | 9 | public class ATest02 { 10 | 11 | public static void main(String[] arg){ 12 | final ProducerClient pc = new ProducerClient("topic3"); 13 | Thread t = new Thread(new Runnable() { 14 | @Override 15 | public void run() { 16 | try { 17 | JSONArray jArray = getInfor(); 18 | for (int i = 0; i < jArray.size(); i++) { 19 | Thread.sleep(2000); 20 | JSONObject jo = (JSONObject)jArray.get(i); 21 | System.out.println(jo); 22 | pc.sendMsg(jo.toString()); 23 | } 24 | } catch (InterruptedException e) { 25 | e.printStackTrace(); 26 | } catch (ExecutionException e) { 27 | e.printStackTrace(); 28 | } 29 | } 30 | }); 31 | t.start(); 32 | } 33 | 34 | public static JSONArray getInfor(){ 35 | JSONArray ja = new JSONArray(); 36 | JSONObject jo1 = new JSONObject(); 37 | jo1.put("aaa", "d316"); 38 | jo1.put("bbb", "d318"); 39 | jo1.put("ccc", "0"); 40 | jo1.put("ddd", "d316-0"); 41 | 42 | JSONObject jo2 = new JSONObject(); 43 | jo2.put("aaa", "d3161"); 44 | jo2.put("bbb", "d3181"); 45 | jo2.put("ccc", "1"); 46 | jo2.put("ddd", "d316-2"); 47 | 48 | JSONObject jo3 = new JSONObject(); 49 | jo3.put("aaa", "d3162"); 50 | jo3.put("bbb", "d3182"); 51 | jo3.put("ccc", "2"); 52 | jo3.put("ddd", "d316-3"); 53 | 54 | JSONObject jo4 = new JSONObject(); 55 | jo4.put("aaa", "d3163"); 56 | jo4.put("bbb", "d3183"); 57 | jo4.put("ccc", "3"); 58 | jo4.put("ddd", "d316-1"); 59 | 60 | ja.add(jo1); 61 | ja.add(jo2); 62 | ja.add(jo3); 63 | ja.add(jo4); 64 | return ja; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /bd-hbase/src/main/java/org/bd/hbase/HBaseConfig.java: -------------------------------------------------------------------------------- 1 | package org.bd.hbase; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.hbase.HBaseConfiguration; 5 | import org.bd.hbase.utils.Consts; 6 | import org.bd.hbase.utils.SysVarsUtils; 7 | 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 连接hbase数据库配置
12 | * 版本历史: 13 | * @author wpk | 2017年11月8日 下午4:22:54 |创建 14 | */ 15 | public class HBaseConfig{ 16 | 17 | private static final ThreadLocal configThreadLocal = new ThreadLocal(); 18 | 19 | /** 20 | * 描述： 创建hbase连接配置 21 | * @author wpk | 2017年11月8日 下午4:23:23 |创建 22 | * @return Configuration 23 | */ 24 | private static Configuration createHBaseConfiguration() { 25 | SysVarsUtils sysVarsUtils = SysVarsUtils.getInstance(); 26 | 27 | Configuration conf = HBaseConfiguration.create(); 28 | 29 | conf.set("hbase.zookeeper.property.clientPort", sysVarsUtils.getVarByName(Consts.HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT)); 30 | conf.set("hbase.zookeeper.quorum", sysVarsUtils.getVarByName(Consts.HBASE_ZOOKEEPER_QUORUM)); 31 | conf.set("hbase.master", sysVarsUtils.getVarByName(Consts.HBASE_MASTER)); 32 | 33 | return conf; 34 | } 35 | 36 | /** 37 | * 描述： 获取hbase连接配置 38 | * @author wpk | 2017年11月8日 下午4:23:44 |创建 39 | * @return Configuration 40 | */ 41 | public synchronized static Configuration getHBaseConfiguration() { 42 | if (configThreadLocal.get() == null) { 43 | Configuration config = createHBaseConfiguration(); 44 | configThreadLocal.set(config); 45 | } 46 | return configThreadLocal.get(); 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/stream/TextFileStreamTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.stream; 2 | 3 | import java.util.Arrays; 4 | 5 | import org.apache.spark.SparkConf; 6 | import org.apache.spark.streaming.Durations; 7 | import org.apache.spark.streaming.api.java.JavaDStream; 8 | import org.apache.spark.streaming.api.java.JavaPairDStream; 9 | import org.apache.spark.streaming.api.java.JavaStreamingContext; 10 | 11 | import scala.Tuple2; 12 | 13 | /** 14 | * 版权信息: big data module
15 | * 功能描述: 实时获取文件流处理
16 | * 版本历史: 17 | * @author wpk | 2018年1月16日 下午4:24:21 |创建 18 | */ 19 | public class TextFileStreamTest { 20 | 21 | public static void TextFileStreaming() throws Exception{ 22 | SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("TextFileStreaming"); 23 | JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5));//每5秒计算一批线程 24 | JavaDStream lines = jssc.textFileStream("C:/Users/Administrator/Desktop/sparkFile/TextFileStream"); 25 | JavaDStream words = lines.flatMap(x -> Arrays.asList(x.split(" ")).iterator()); 26 | // Count each word in each batch 27 | JavaPairDStream pairs = words.mapToPair(s -> new Tuple2<>(s, 1)); 28 | JavaPairDStream wordCounts = pairs.reduceByKey((i1, i2) -> i1 + i2); 29 | 30 | // Print the first ten elements of each RDD generated in this DStream to the console 31 | wordCounts.print(); 32 | 33 | jssc.start(); // Start the computation 34 | jssc.awaitTermination(); // Wait for the computation to terminate 35 | } 36 | 37 | 38 | public static void main(String[] arg) throws Exception{ 39 | System.setProperty("hadoop.home.dir", "F:\\hadoop-common-2.2.0-bin-master"); 40 | TextFileStreaming(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /bd-spark/src/main/java/org/bd/spark/utils/JdbcDriveUtil.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.utils; 2 | 3 | import java.sql.Connection; 4 | import java.sql.DriverManager; 5 | import java.sql.SQLException; 6 | 7 | import org.bd.spark.enums.Consts; 8 | 9 | 10 | /** 11 | * 版权信息: big data module
12 | * 功能描述: spark的jdbc驱动类
13 | * 版本历史: 14 | * @author wpk | 2017年7月20日上午11:45:44 |创建 15 | */ 16 | public class JdbcDriveUtil{ 17 | 18 | private volatile static JdbcDriveUtil sc = null; 19 | 20 | /** 默认构造类*/ 21 | public JdbcDriveUtil() { 22 | try { 23 | if(sc == null){ 24 | SysVarsUtils sysVarsUtils = SysVarsUtils.getInstance(); 25 | String driver = sysVarsUtils.getVarByName(Consts.spark_driver); 26 | Class.forName(driver); 27 | } 28 | } catch(ClassNotFoundException e) { 29 | System.out.println(String.format("spark的jdbc驱动加载异常：(%s)",e.getMessage())); 30 | e.printStackTrace(); 31 | } 32 | } 33 | 34 | /** 35 | * 描述：接口 36 | * @author wpk | 2017年7月24日下午4:54:22 |创建 37 | * @return 38 | */ 39 | public static JdbcDriveUtil getInstance(){ 40 | if(sc == null){ 41 | synchronized (JdbcDriveUtil.class) { 42 | if(sc == null){ 43 | sc = new JdbcDriveUtil(); 44 | } 45 | } 46 | } 47 | return sc; 48 | } 49 | 50 | /** 51 | * 描述：获取JDBC连接 52 | * @author wpk | 2017年7月24日下午5:00:59 |创建 53 | * @return 54 | * @throws SQLException 55 | */ 56 | public Connection getConn() throws SQLException{ 57 | SysVarsUtils sysVarsUtils = SysVarsUtils.getInstance(); 58 | String url = sysVarsUtils.getVarByName(Consts.spark_url); 59 | String user = sysVarsUtils.getVarByName(Consts.spark_user); 60 | String pwd = sysVarsUtils.getVarByName(Consts.spark_password); 61 | Connection conn = DriverManager.getConnection(url,user,pwd); 62 | return conn; 63 | } 64 | } -------------------------------------------------------------------------------- /bd-hive-udf/src/main/java/org/bd/hive/udaf/Att.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive.udaf; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStreamReader; 6 | 7 | public class Att { 8 | 9 | private static final String ROOT_PATH = System.getProperty("user.dir"); 10 | 11 | public static void main(String[] args) throws IOException { 12 | 13 | System.out.println(ROOT_PATH); 14 | String command = runtimeCmd()+"#cd "+ ROOT_PATH + " && mvn clean package"; 15 | System.out.println(command); 16 | String[] commands = command.split("#"); 17 | execCmd(commands); 18 | 19 | } 20 | 21 | public static String runtimeCmd() { 22 | String os = System.getProperty("os.name").toLowerCase(); 23 | if(os.indexOf("windows")>=0) { 24 | return "cmd#/c" ; 25 | }else if(os.indexOf("linux")>=0) { 26 | return "/bin/sh#-c"; 27 | } 28 | return null; 29 | } 30 | 31 | public static int execCmd(String[] commands) { 32 | BufferedReader br = null; 33 | Process process = null; 34 | try { 35 | process = Runtime.getRuntime().exec(commands); 36 | br = new BufferedReader(new InputStreamReader(process.getInputStream())); 37 | String line = null; 38 | while ((line = br.readLine()) != null || "".equals(line = br.readLine())) { 39 | System.out.println(line); 40 | } 41 | BufferedReader errors = new BufferedReader(new InputStreamReader(process.getErrorStream())); 42 | String line2 = null; 43 | while ((line2 = errors.readLine()) != null || "".equals(line2 = errors.readLine())) { 44 | System.out.print(line2); 45 | } 46 | return process.waitFor(); 47 | } catch (Exception e) { 48 | e.printStackTrace(); 49 | System.out.print(e.getMessage()); 50 | } finally { 51 | process.destroy(); 52 | if (br != null) { 53 | try { 54 | br.close(); 55 | } catch (Exception e) { 56 | e.printStackTrace(); 57 | } 58 | } 59 | } 60 | return -1; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /bd-kylin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | bd-kylin 11 | bd-kylin 12 | http://maven.apache.org 13 | 14 | 15 | UTF-8 16 | 17 | 18 | 19 | 20 | junit 21 | junit 22 | 3.8.1 23 | test 24 | 25 | 26 | 27 | com.fasterxml.jackson.core 28 | jackson-core 29 | 2.6.5 30 | 31 | 32 | com.fasterxml.jackson.core 33 | jackson-databind 34 | 2.6.5 35 | compile 36 | 37 | 38 | org.apache.commons 39 | commons-lang3 40 | 3.5 41 | 42 | 43 | org.json 44 | json 45 | 20160810 46 | 47 | 48 | commons-io 49 | commons-io 50 | 2.5 51 | 52 | 53 | 54 | log4j 55 | log4j 56 | 1.2.17 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/stream/SocketStreamTest.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.stream; 2 | 3 | import java.util.Arrays; 4 | 5 | import org.apache.spark.SparkConf; 6 | import org.apache.spark.storage.StorageLevel; 7 | import org.apache.spark.streaming.Durations; 8 | import org.apache.spark.streaming.api.java.JavaDStream; 9 | import org.apache.spark.streaming.api.java.JavaPairDStream; 10 | import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; 11 | import org.apache.spark.streaming.api.java.JavaStreamingContext; 12 | 13 | import scala.Tuple2; 14 | 15 | /** 16 | * 版权信息: big data module
17 | * 功能描述: socket流处理
18 | * 版本历史: 19 | * @author wpk | 2018年1月16日 上午11:52:42 |创建 20 | */ 21 | public class SocketStreamTest { 22 | 23 | public static void socketStreaming() throws Exception{ 24 | SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount"); 25 | JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(5));//每5秒计算一批线程 26 | JavaReceiverInputDStream lines = jssc.socketTextStream("172.16.3.132", 9999); 27 | JavaDStream words = lines.flatMap(x -> Arrays.asList(x.split(" ")).iterator()); 28 | // Count each word in each batch 29 | JavaPairDStream pairs = words.mapToPair(s -> new Tuple2<>(s, 1)); 30 | JavaPairDStream wordCounts = pairs.reduceByKey((i1, i2) -> i1 + i2); 31 | 32 | // Print the first ten elements of each RDD generated in this DStream to the console 33 | // wordCounts.print(); 34 | 35 | wordCounts.dstream().saveAsTextFiles("C:/Users/Administrator/Desktop/sparkFile/wordCount/", "socketStream"); 36 | 37 | jssc.start(); // Start the computation 38 | jssc.awaitTermination(); // Wait for the computation to terminate 39 | } 40 | 41 | 42 | public static void main(String[] arg){ 43 | 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/response/CubeResponse.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.response; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: cube响应对象
6 | * 版本历史: 7 | * @author wpk | 2017年11月20日 下午5:44:20 |创建 8 | */ 9 | public class CubeResponse { 10 | 11 | private String uuid; 12 | private long last_modified; 13 | private String version; 14 | private String name; 15 | private String owner; 16 | private String descriptor; 17 | private int cost; 18 | private String status; 19 | private long create_time_utc; 20 | 21 | public String getUuid() { 22 | return uuid; 23 | } 24 | public void setUuid(String uuid) { 25 | this.uuid = uuid; 26 | } 27 | public long getLast_modified() { 28 | return last_modified; 29 | } 30 | public void setLast_modified(long last_modified) { 31 | this.last_modified = last_modified; 32 | } 33 | public String getVersion() { 34 | return version; 35 | } 36 | public void setVersion(String version) { 37 | this.version = version; 38 | } 39 | public String getName() { 40 | return name; 41 | } 42 | public void setName(String name) { 43 | this.name = name; 44 | } 45 | public String getOwner() { 46 | return owner; 47 | } 48 | public void setOwner(String owner) { 49 | this.owner = owner; 50 | } 51 | public String getDescriptor() { 52 | return descriptor; 53 | } 54 | public void setDescriptor(String descriptor) { 55 | this.descriptor = descriptor; 56 | } 57 | public int getCost() { 58 | return cost; 59 | } 60 | public void setCost(int cost) { 61 | this.cost = cost; 62 | } 63 | public String getStatus() { 64 | return status; 65 | } 66 | public void setStatus(String status) { 67 | this.status = status; 68 | } 69 | public long getCreate_time_utc() { 70 | return create_time_utc; 71 | } 72 | public void setCreate_time_utc(long create_time_utc) { 73 | this.create_time_utc = create_time_utc; 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/cube/Measures.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.cube; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 度量
6 | * 版本历史: 7 | * @author wpk | 2017年9月14日 下午12:14:26 |创建 8 | */ 9 | public class Measures { 10 | 11 | private String name; 12 | private Function function; 13 | 14 | public class Function{ 15 | private String expression; 16 | private String returntype; 17 | private Parameter parameter; 18 | 19 | public String getExpression() { 20 | return expression; 21 | } 22 | public void setExpression(String expression) { 23 | this.expression = expression; 24 | } 25 | public String getReturntype() { 26 | return returntype; 27 | } 28 | public void setReturntype(String returntype) { 29 | this.returntype = returntype; 30 | } 31 | public Parameter getParameter() { 32 | return parameter; 33 | } 34 | public void setParameter(Parameter parameter) { 35 | this.parameter = parameter; 36 | } 37 | } 38 | 39 | public class Parameter{ 40 | private String type; 41 | private String value; 42 | private String next_parameter = null; 43 | 44 | public String getType() { 45 | return type; 46 | } 47 | public void setType(String type) { 48 | this.type = type; 49 | } 50 | public String getValue() { 51 | return value; 52 | } 53 | public void setValue(String value) { 54 | this.value = value; 55 | } 56 | public String getNext_parameter() { 57 | return next_parameter; 58 | } 59 | public void setNext_parameter(String next_parameter) { 60 | this.next_parameter = next_parameter; 61 | } 62 | } 63 | 64 | public String getName() { 65 | return name; 66 | } 67 | 68 | public void setName(String name) { 69 | this.name = name; 70 | } 71 | 72 | public Function getFunction() { 73 | return function; 74 | } 75 | 76 | public void setFunction(Function function) { 77 | this.function = function; 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /bd-spark/src/test/java/org/bd/spark/kafka/UserKafkaConsumer.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark.kafka; 2 | 3 | import java.util.HashMap; 4 | import java.util.List; 5 | import java.util.Map; 6 | import java.util.Properties; 7 | 8 | import kafka.consumer.ConsumerConfig; 9 | //import kafka.consumer.ConsumerConnector; 10 | import kafka.consumer.ConsumerIterator; 11 | import kafka.consumer.KafkaStream; 12 | import kafka.javaapi.consumer.ConsumerConnector; 13 | 14 | public class UserKafkaConsumer extends Thread { 15 | private final ConsumerConnector consumer; 16 | private final String topic; 17 | public UserKafkaConsumer(String topic) 18 | { 19 | consumer = kafka.consumer.Consumer.createJavaConsumerConnector( 20 | createConsumerConfig()); 21 | this.topic = topic; 22 | } 23 | private static ConsumerConfig createConsumerConfig() 24 | { 25 | Properties props = new Properties(); 26 | props.put("zookeeper.connect", "192.168.32.115:2181"); 27 | props.put("group.id", "group1"); 28 | props.put("zookeeper.session.timeout.ms", "40000"); 29 | props.put("zookeeper.sync.time.ms", "200"); 30 | props.put("auto.commit.interval.ms", "1000"); 31 | return new ConsumerConfig(props); 32 | } 33 | 34 | @Override 35 | public void run() { 36 | Map topicCountMap = new HashMap(); 37 | topicCountMap.put(topic, new Integer(1)); 38 | Map>> consumerMap = consumer.createMessageStreams(topicCountMap); 39 | KafkaStream stream = consumerMap.get(topic).get(0); 40 | ConsumerIterator it = stream.iterator(); 41 | while (it.hasNext()) { 42 | System.out.println("receive：" + new String(it.next().message())); 43 | try { 44 | sleep(3000); 45 | } catch (InterruptedException e) { 46 | e.printStackTrace(); 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/ftp/FtpPara.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.ftp; 2 | 3 | import org.bd.datax.DataxPara; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: ftp键值对参数配置
8 | * 版本历史: 9 | * @author wpk | 2017年8月8日 下午3:12:28 |创建 10 | */ 11 | public class FtpPara extends DataxPara{ 12 | 13 | //=======================读写共同参数================================ 14 | /** ftp服务器协议*/ 15 | public final static String protocol = "protocol"; 16 | /** ftp服务器地址*/ 17 | public final static String host = "host"; 18 | /** ftp服务器端口*/ 19 | public final static String port = "port"; 20 | /** 连接ftp服务器连接超时时间*/ 21 | public final static String timeout = "timeout"; 22 | /** ftp服务器访问用户名*/ 23 | public final static String username = "username"; 24 | /** ftp服务器访问密码*/ 25 | public final static String password = "password"; 26 | /** 远程FTP文件系统的路径信息*/ 27 | public final static String path = "path"; 28 | /** 字段分隔符*/ 29 | public final static String fieldDelimiter = "fieldDelimiter"; 30 | /** 文件的编码配置*/ 31 | public final static String encoding = "encoding"; 32 | /** 指定值转为null空字符*/ 33 | public final static String nullFormat = "nullFormat"; 34 | 35 | //=======================读取参数================================ 36 | /** 连接模式*/ 37 | public final static String connectPattern = "connectPattern"; 38 | /** 读取字段列表*/ 39 | public final static String column = "column"; 40 | /** 文本压缩类型*/ 41 | public final static String compress = "compress"; 42 | /** 跳过文件表头标题*/ 43 | public final static String skipHeader = "skipHeader"; 44 | /** 允许遍历文件夹的最大层数*/ 45 | public final static String maxTraversalLevel = "maxTraversalLevel"; 46 | 47 | //=======================写入参数================================ 48 | /** 写入文件名*/ 49 | public final static String fileName = "fileName"; 50 | /** 写入模式*/ 51 | public final static String writeMode = "writeMode"; 52 | /** 格式化日期*/ 53 | public final static String dateFormat = "dateFormat"; 54 | /** 文件写出的格式*/ 55 | public final static String fileFormat = "fileFormat"; 56 | /** txt写出时的表头*/ 57 | public final static String header = "header"; 58 | 59 | } 60 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/request/JobListRequest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.request; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: job请求参数对象
8 | * 版本历史: 9 | * @author wpk | 2017年11月21日 上午10:16:39 |创建 10 | */ 11 | public class JobListRequest { 12 | 13 | /** 14 | * 状态
15 | * job状态可以参考com.idata.bd.kylin.JobStatusEnum类
16 | * 默认获取所有状态 17 | */ 18 | private List status; 19 | /** 20 | * cube名称 21 | */ 22 | private String cubeName; 23 | /** 24 | * 项目名称 25 | */ 26 | private String projectName; 27 | /** 28 | * 从第几行开始获取（用于分页下标） 29 | */ 30 | private Integer offset; 31 | /** 32 | * 指定job数（用于分页） 33 | */ 34 | private Integer limit; 35 | /** 36 | * 时间过滤
37 | * job时间过滤可以参考com.idata.bd.kylin.JobTimeFilterEnum类
38 | * 默认获取上个星期的作业 39 | */ 40 | private Integer timeFilter; 41 | 42 | public JobListRequest() {} 43 | 44 | public List getStatus() { 45 | return status; 46 | } 47 | 48 | public void setStatus(List status) { 49 | this.status = status; 50 | } 51 | 52 | public String getCubeName() { 53 | return cubeName; 54 | } 55 | 56 | public void setCubeName(String cubeName) { 57 | this.cubeName = cubeName; 58 | } 59 | 60 | public String getProjectName() { 61 | return projectName; 62 | } 63 | 64 | public void setProjectName(String projectName) { 65 | this.projectName = projectName; 66 | } 67 | 68 | public Integer getOffset() { 69 | return offset; 70 | } 71 | 72 | public void setOffset(Integer offset) { 73 | this.offset = offset; 74 | } 75 | 76 | public Integer getLimit() { 77 | return limit; 78 | } 79 | 80 | public void setLimit(Integer limit) { 81 | this.limit = limit; 82 | } 83 | 84 | public Integer getTimeFilter() { 85 | return timeFilter; 86 | } 87 | 88 | public void setTimeFilter(Integer timeFilter) { 89 | this.timeFilter = timeFilter; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /bd-kafka/src/main/java/org/bd/kafka/ATest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kafka; 2 | 3 | import java.util.UUID; 4 | import java.util.concurrent.ExecutionException; 5 | 6 | import net.sf.json.JSONObject; 7 | 8 | public class ATest { 9 | 10 | public static void main(String[] arg){ 11 | test02(); 12 | } 13 | 14 | public static void test02(){ 15 | final ProducerClient pc = new ProducerClient("rta_xdr"); 16 | Thread t = new Thread(new Runnable() { 17 | @Override 18 | public void run() { 19 | int i = 1; 20 | while (true) { 21 | try { 22 | pc.sendMsg(i+""); 23 | System.out.println("==="+i); 24 | i++; 25 | Thread.sleep(5000); 26 | } catch (InterruptedException e) { 27 | e.printStackTrace(); 28 | } catch (ExecutionException e) { 29 | e.printStackTrace(); 30 | } 31 | } 32 | } 33 | }); 34 | t.start(); 35 | } 36 | 37 | public static void test01(){ 38 | final ProducerClient pc = new ProducerClient("rta_xdr"); 39 | Thread t = new Thread(new Runnable() { 40 | @Override 41 | public void run() { 42 | int i = 1; 43 | StringBuffer sb = new StringBuffer(); 44 | while (true) { 45 | try { 46 | int c = (int) (Math.random() * 100); 47 | if(c > 30){ 48 | pc.sendMsg(getInfor(c)); 49 | sb.append(c).append(","); 50 | System.out.println(getInfor(c)); 51 | if (i == 3) { 52 | i = 1; 53 | System.out.println(sb.toString()); 54 | sb.setLength(0); 55 | } else { 56 | i++; 57 | } 58 | } 59 | Thread.sleep(5000); 60 | //System.out.println(getInfor(i++)); 61 | } catch (InterruptedException e) { 62 | e.printStackTrace(); 63 | } catch (ExecutionException e) { 64 | e.printStackTrace(); 65 | } 66 | } 67 | } 68 | }); 69 | t.start(); 70 | } 71 | 72 | public static String getInfor(int i){ 73 | JSONObject jo = new JSONObject(); 74 | jo.put("apn", "apn"); 75 | jo.put("cell_id", i); 76 | jo.put("country", "CN"); 77 | jo.put("map_errorcode", (int) (Math.random() * 100)); 78 | jo.put("enterprise_name", UUID.randomUUID().toString()); 79 | jo.put("timestamp_event", System.currentTimeMillis()+""); 80 | return jo.toString(); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/model/ModelJson.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.model; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * 版权信息: big data module
7 | * 功能描述: cube模型脚本
8 | * 版本历史: 9 | * @author wpk | 2017年9月14日 下午3:30:24 |创建 10 | */ 11 | public class ModelJson { 12 | 13 | private String name; 14 | private String description; 15 | private String fact_table; 16 | private String capacity = "MEDIUM"; 17 | private List lookups; 18 | private List dimensions; 19 | private List metrics; 20 | private String filter_condition; 21 | private Partition_desc partition_desc; 22 | 23 | public String getDescription() { 24 | return description; 25 | } 26 | public void setDescription(String description) { 27 | this.description = description; 28 | } 29 | public String getName() { 30 | return name; 31 | } 32 | public void setName(String name) { 33 | this.name = name; 34 | } 35 | public String getFact_table() { 36 | return fact_table; 37 | } 38 | public void setFact_table(String fact_table) { 39 | this.fact_table = fact_table; 40 | } 41 | public String getCapacity() { 42 | return capacity; 43 | } 44 | public void setCapacity(String capacity) { 45 | this.capacity = capacity; 46 | } 47 | public List getLookups() { 48 | return lookups; 49 | } 50 | public void setLookups(List lookups) { 51 | this.lookups = lookups; 52 | } 53 | public List getDimensions() { 54 | return dimensions; 55 | } 56 | public void setDimensions(List dimensions) { 57 | this.dimensions = dimensions; 58 | } 59 | public List getMetrics() { 60 | return metrics; 61 | } 62 | public void setMetrics(List metrics) { 63 | this.metrics = metrics; 64 | } 65 | public String getFilter_condition() { 66 | return filter_condition; 67 | } 68 | public void setFilter_condition(String filter_condition) { 69 | this.filter_condition = filter_condition; 70 | } 71 | public Partition_desc getPartition_desc() { 72 | return partition_desc; 73 | } 74 | public void setPartition_desc(Partition_desc partition_desc) { 75 | this.partition_desc = partition_desc; 76 | } 77 | 78 | } -------------------------------------------------------------------------------- /bd-hdfs/src/main/java/org/bd/hdfs/Test.java: -------------------------------------------------------------------------------- 1 | package org.bd.hdfs; 2 | 3 | import org.apache.hadoop.fs.FileStatus; 4 | 5 | 6 | 7 | public class Test { 8 | 9 | public static void main(String[] args) { 10 | HdfsClient item = new HdfsClient(); 11 | /*boolean flag1 = item.isDirExists("/user/udf_prog/idata-udf.jar/idata-udf.jar"); 12 | if(flag1){ 13 | System.out.println("路径存在"); 14 | }else{ 15 | System.out.println("路径不存在"); 16 | }*/ 17 | /*boolean falg2 = item.mkDir("/backup"); 18 | if(falg2){ 19 | System.out.println("创建成功"); 20 | }else{ 21 | System.out.println("创建失败"); 22 | }*/ 23 | item.uploadFile("C:\\Users\\ennwpae\\Desktop\\jupyter1.html", "/backup"); 24 | /*boolean flag3 = item.deleteDir("/backup"); 25 | if(flag3){ 26 | System.out.println("删除成功"); 27 | }else{ 28 | System.out.println("删除失败"); 29 | }*/ 30 | /*FileStatus[] list = item.dirList("/backup"); 31 | for (FileStatus f : list) { 32 | System.out.printf("name: %s, folder: %s, size: %d, %s, %s, %d\n", f.getPath(), f.isDirectory(), f.getLen(), f.getGroup(), f.getOwner(), f.getBlockSize()); 33 | }*/ 34 | /*boolean flag4 = item.renameDir("/user/udf_prog/output/test11", "/user/udf_prog/output/test1"); 35 | if(flag4){ 36 | System.out.println("重命名成功"); 37 | }else{ 38 | System.out.println("重命名失败"); 39 | }*/ 40 | // item.uploadFile("C:\\Users\\Administrator\\Desktop/idata.sql", "/user/udf_prog/output/test1"); 41 | // item.downloadFile("C:\\Users\\Administrator\\Desktop\\ceshi", "/user/udf_prog/output/test1/test.sql"); 42 | // item.createFile("/user/udf_prog/output/test1/test.txt", "fsadfsadfsadfsadf"); 43 | /*boolean flag3 = item.deleteDir("/user/udf_prog/output/test1/test.txt"); 44 | if(flag3){ 45 | System.out.println("success"); 46 | }else{ 47 | System.out.println("fail"); 48 | }*/ 49 | 50 | /*String look = item.lookFile("/user/udf_prog/output/test1/test.sql"); 51 | System.out.println(look);*/ 52 | 53 | /*FileStatus[] paths = item.dirListByLike("/user/udf_prog/output/test1","t"); 54 | for(FileStatus p : paths){ 55 | System.out.println(p.getPath()); 56 | }*/ 57 | System.exit(0); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/rest/QueryRest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.rest; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | import org.bd.kylin.CubeException; 5 | import org.bd.kylin.RestRequstHandle; 6 | import org.bd.kylin.request.SQLRequest; 7 | import org.bd.kylin.utils.JsonBinder; 8 | 9 | import com.fasterxml.jackson.core.JsonProcessingException; 10 | import com.fasterxml.jackson.databind.ObjectMapper; 11 | 12 | /** 13 | * 版权信息: big data module
14 | * 功能描述: kylin语句查询
15 | * 版本历史: 16 | * @author wpk | 2017年11月21日 上午9:15:43 |创建 17 | */ 18 | public class QueryRest extends RestRequstHandle{ 19 | 20 | /** 21 | * 描述： 获取kylin上的所有表元数据信息
22 | * 所有表指的是用于创建模型和cube的表 23 | * @author wpk | 2017年11月21日 上午9:19:29 |创建 24 | * @return String 25 | */ 26 | public static String getMetadatas(String project){ 27 | String para = "tables_and_columns?project="+project; 28 | String result = request(para, RequestMethod.GET); 29 | return result; 30 | } 31 | 32 | /** 33 | * 描述： 查询语句 34 | * @author wpk | 2017年11月21日 下午3:56:10 |创建 35 | * @return String 36 | * @throws JsonProcessingException 37 | */ 38 | public static String query(SQLRequest sr) throws JsonProcessingException{ 39 | String para = "query"; 40 | if(null == sr){ 41 | throw new CubeException("传入[SQLRequest]对象为空!"); 42 | } 43 | if(StringUtils.isBlank(sr.getProject())){ 44 | throw new CubeException("传入[project]参数为空!"); 45 | } 46 | if(StringUtils.isBlank(sr.getSql())){ 47 | throw new CubeException("传入[sql]参数为空!"); 48 | } 49 | 50 | ObjectMapper om = JsonBinder.buildNonNullBinder().getMapper(); 51 | String json = om.writeValueAsString(sr); 52 | 53 | String result = request(para, RequestMethod.POST, json); 54 | return result; 55 | } 56 | 57 | /** 58 | * 描述： 查询语句 59 | * @author wpk | 2017年11月21日 下午5:24:25 |创建 60 | * @param project 61 | * @param sql 62 | * @throws JsonProcessingException 63 | * @return String 64 | */ 65 | public static String query(String project, String sql) throws JsonProcessingException{ 66 | SQLRequest item = new SQLRequest(); 67 | item.setProject(project); 68 | item.setSql(sql); 69 | return query(item); 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /bd-hive-udf/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | 11 | bd-hive-udf 12 | bd-hive-udf 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | org.apache.hive 28 | hive-exec 29 | 1.2.1 30 | 31 | 32 | 33 | jdk.tools 34 | jdk.tools 35 | 1.7 36 | system 37 | ${JAVA_HOME}/lib/tools.jar 38 | 39 | 40 | 41 | 42 | 43 | 44 | maven-compiler-plugin 45 | 2.3.2 46 | 47 | 1.7 48 | 1.7 49 | 50 | 51 | 52 | maven-assembly-plugin 53 | 54 | false 55 | 56 | src/main/resources/package.xml 57 | 58 | 61 | 62 | 63 | 64 | make-assembly 65 | package 66 | 67 | single 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | -------------------------------------------------------------------------------- /bd-zk/src/main/java/org/bd/zk/utils/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package org.bd.zk.utils; 2 | 3 | import java.io.FileInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.util.Hashtable; 7 | import java.util.Properties; 8 | 9 | import org.apache.log4j.Logger; 10 | 11 | /** 12 | * 版权信息: big data module
13 | * 功能描述: 读取属性配置文件工具类
14 | * 版本历史: 15 | * @author wpk | 2017年11月24日 下午6:01:53 |创建 16 | */ 17 | public class PropertiesUtil { 18 | 19 | static PropertiesUtil pu;// 创建对象pu 20 | private static Hashtable register = new Hashtable(); 21 | private static Logger log = Logger.getLogger(PropertiesUtil.class); 22 | 23 | private PropertiesUtil() { 24 | super(); 25 | } 26 | 27 | /** 28 | * 取得PropertiesUtil的一个实例 29 | */ 30 | public static PropertiesUtil getInstance() { 31 | if (pu == null) 32 | pu = new PropertiesUtil(); 33 | return pu; 34 | } 35 | 36 | /** 37 | * 读取配置文件 38 | */ 39 | public Properties getProperties(String fileName) { 40 | InputStream is = null; 41 | Properties p = null; 42 | try { 43 | p = (Properties) register.get(fileName); 44 | if (p == null) { 45 | try { 46 | is = new FileInputStream(fileName); 47 | } catch (Exception e) { 48 | if (fileName.startsWith("/")) 49 | is = PropertiesUtil.class.getResourceAsStream(fileName); 50 | else 51 | is = PropertiesUtil.class.getResourceAsStream("/" + fileName); 52 | } 53 | if (is == null) { 54 | log.info("未找到名称为" + fileName + "的资源！"); 55 | } 56 | p = new Properties(); 57 | p.load(is); 58 | register.put(fileName, p); 59 | is.close(); 60 | } 61 | } catch (Exception e) { 62 | log.error("读取properties时异常", e); 63 | } finally { 64 | if (is != null) 65 | try { 66 | is.close(); 67 | } catch (IOException e) { 68 | log.error("读取properties关闭流时异常", e); 69 | } 70 | } 71 | return p; 72 | } 73 | 74 | public String getValue(String fileName, String strKey) { 75 | Properties p = getProperties(fileName); 76 | try { 77 | return p.getProperty(strKey); 78 | } catch (Exception e) { 79 | log.error("读取properties时异常", e); 80 | } 81 | return null; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /bd-kafka/src/main/java/org/bd/kafka/utils/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package org.bd.kafka.utils; 2 | 3 | import java.io.FileInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.util.Hashtable; 7 | import java.util.Properties; 8 | 9 | import org.apache.log4j.Logger; 10 | 11 | /** 12 | * 版权信息: big data module
13 | * 功能描述: 读取属性配置文件工具类
14 | * 版本历史: 15 | * @author wpk | 2017年11月24日 下午6:01:53 |创建 16 | */ 17 | public class PropertiesUtil { 18 | 19 | static PropertiesUtil pu;// 创建对象pu 20 | private static Hashtable register = new Hashtable(); 21 | private static Logger log = Logger.getLogger(PropertiesUtil.class); 22 | 23 | private PropertiesUtil() { 24 | super(); 25 | } 26 | 27 | /** 28 | * 取得PropertiesUtil的一个实例 29 | */ 30 | public static PropertiesUtil getInstance() { 31 | if (pu == null) 32 | pu = new PropertiesUtil(); 33 | return pu; 34 | } 35 | 36 | /** 37 | * 读取配置文件 38 | */ 39 | public Properties getProperties(String fileName) { 40 | InputStream is = null; 41 | Properties p = null; 42 | try { 43 | p = (Properties) register.get(fileName); 44 | if (p == null) { 45 | try { 46 | is = new FileInputStream(fileName); 47 | } catch (Exception e) { 48 | if (fileName.startsWith("/")) 49 | is = PropertiesUtil.class.getResourceAsStream(fileName); 50 | else 51 | is = PropertiesUtil.class.getResourceAsStream("/" + fileName); 52 | } 53 | if (is == null) { 54 | log.info("未找到名称为" + fileName + "的资源！"); 55 | } 56 | p = new Properties(); 57 | p.load(is); 58 | register.put(fileName, p); 59 | is.close(); 60 | } 61 | } catch (Exception e) { 62 | log.error("读取properties时异常", e); 63 | } finally { 64 | if (is != null) 65 | try { 66 | is.close(); 67 | } catch (IOException e) { 68 | log.error("读取properties关闭流时异常", e); 69 | } 70 | } 71 | return p; 72 | } 73 | 74 | public String getValue(String fileName, String strKey) { 75 | Properties p = getProperties(fileName); 76 | try { 77 | return p.getProperty(strKey); 78 | } catch (Exception e) { 79 | log.error("读取properties时异常", e); 80 | } 81 | return null; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /bd-flink/src/main/java/org/bd/flink/XDREnrichedMessageSinkFunction.java: -------------------------------------------------------------------------------- 1 | package org.bd.flink; 2 | 3 | import lombok.Builder; 4 | import lombok.NonNull; 5 | 6 | import org.apache.flink.api.common.functions.RuntimeContext; 7 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException; 8 | import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper; 9 | import org.apache.flink.streaming.connectors.elasticsearch.ElasticsearchSinkFunction; 10 | import org.apache.flink.streaming.connectors.elasticsearch.RequestIndexer; 11 | import org.bd.flink.idgenerate.ESIdGenerator; 12 | import org.bd.flink.idgenerate.IdGeneratorBuilder; 13 | import org.bd.flink.idgenerate.IdGeneratorType; 14 | import org.elasticsearch.action.index.IndexRequest; 15 | import org.elasticsearch.client.Requests; 16 | import org.elasticsearch.common.xcontent.XContentType; 17 | 18 | @Builder 19 | public class XDREnrichedMessageSinkFunction implements ElasticsearchSinkFunction { 20 | @NonNull private String esIndex; 21 | @NonNull private IdGeneratorType idGeneratorType = IdGeneratorType.AUTO; 22 | 23 | public IndexRequest createIndexRequest(XDREnrichedMessage element) { 24 | final ObjectMapper mapper = new ObjectMapper(); 25 | ESIdGenerator idGenerator = IdGeneratorBuilder.build(idGeneratorType); 26 | 27 | try { 28 | IndexRequest req = Requests.indexRequest() 29 | .index(esIndex) 30 | .type("_doc") 31 | .source(mapper.writeValueAsString(element), XContentType.JSON); 32 | 33 | String id = idGenerator.generate(element.getTimestamp_event() 34 | + element.getCell_id() + element.getApn(), 50); 35 | if (id != null) { 36 | return req.id(id); 37 | } 38 | 39 | return req; 40 | } catch (JsonProcessingException e) { 41 | e.printStackTrace(); 42 | return null; 43 | } 44 | } 45 | 46 | @Override 47 | public void process(XDREnrichedMessage xdrEnrichedMessage, RuntimeContext runtimeContext, RequestIndexer requestIndexer) { 48 | IndexRequest r = createIndexRequest(xdrEnrichedMessage); 49 | if (r != null) { 50 | requestIndexer.add(r); 51 | } 52 | } 53 | } -------------------------------------------------------------------------------- /bd-hive/src/main/java/org/bd/hive/utils/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package org.bd.hive.utils; 2 | 3 | import java.io.FileInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.util.Hashtable; 7 | import java.util.Properties; 8 | 9 | import org.apache.log4j.Logger; 10 | 11 | /** 12 | * 版权信息: big data module
13 | * 功能描述: 读取属性配置文件工具类
14 | * 版本历史: 15 | * @author wpk | 2017年8月23日 上午11:20:19 |创建 16 | */ 17 | public class PropertiesUtil { 18 | 19 | static PropertiesUtil pu;// 创建对象pu 20 | private static Hashtable register = new Hashtable(); 21 | private static Logger log = Logger.getLogger(PropertiesUtil.class); 22 | 23 | private PropertiesUtil() { 24 | super(); 25 | } 26 | 27 | /** 28 | * 取得PropertiesUtil的一个实例 29 | */ 30 | public static PropertiesUtil getInstance() { 31 | if (pu == null) 32 | pu = new PropertiesUtil(); 33 | return pu; 34 | } 35 | 36 | /** 37 | * 读取配置文件 38 | */ 39 | private Properties getProperties(String fileName) { 40 | InputStream is = null; 41 | Properties p = null; 42 | try { 43 | p = (Properties) register.get(fileName); 44 | if (p == null) { 45 | try { 46 | is = new FileInputStream(fileName); 47 | } catch (Exception e) { 48 | if (fileName.startsWith("/")) 49 | is = PropertiesUtil.class.getResourceAsStream(fileName); 50 | else 51 | is = PropertiesUtil.class.getResourceAsStream("/" + fileName); 52 | } 53 | if (is == null) { 54 | log.info("未找到名称为" + fileName + "的资源！"); 55 | } 56 | p = new Properties(); 57 | p.load(is); 58 | register.put(fileName, p); 59 | is.close(); 60 | } 61 | } catch (Exception e) { 62 | log.error("读取properties时异常", e); 63 | } finally { 64 | if (is != null) 65 | try { 66 | is.close(); 67 | } catch (IOException e) { 68 | log.error("读取properties关闭流时异常", e); 69 | } 70 | } 71 | return p; 72 | } 73 | 74 | public String getPropertyValue(String fileName, String strKey) { 75 | Properties p = getProperties(fileName); 76 | try { 77 | return p.getProperty(strKey); 78 | } catch (Exception e) { 79 | log.error("读取properties时异常", e); 80 | } 81 | return null; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /bd-hbase/src/main/java/org/bd/hbase/utils/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package org.bd.hbase.utils; 2 | 3 | import java.io.FileInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.util.Hashtable; 7 | import java.util.Properties; 8 | 9 | import org.apache.log4j.Logger; 10 | 11 | /** 12 | * 版权信息: big data module
13 | * 功能描述: 读取属性配置文件工具类
14 | * 版本历史: 15 | * @author wpk | 2017年8月23日 上午11:20:19 |创建 16 | */ 17 | public class PropertiesUtil { 18 | 19 | static PropertiesUtil pu;// 创建对象pu 20 | private static Hashtable register = new Hashtable(); 21 | private static Logger log = Logger.getLogger(PropertiesUtil.class); 22 | 23 | private PropertiesUtil() { 24 | super(); 25 | } 26 | 27 | /** 28 | * 取得PropertiesUtil的一个实例 29 | */ 30 | public static PropertiesUtil getInstance() { 31 | if (pu == null) 32 | pu = new PropertiesUtil(); 33 | return pu; 34 | } 35 | 36 | /** 37 | * 读取配置文件 38 | */ 39 | private Properties getProperties(String fileName) { 40 | InputStream is = null; 41 | Properties p = null; 42 | try { 43 | p = (Properties) register.get(fileName); 44 | if (p == null) { 45 | try { 46 | is = new FileInputStream(fileName); 47 | } catch (Exception e) { 48 | if (fileName.startsWith("/")) 49 | is = PropertiesUtil.class.getResourceAsStream(fileName); 50 | else 51 | is = PropertiesUtil.class.getResourceAsStream("/" + fileName); 52 | } 53 | if (is == null) { 54 | log.info("未找到名称为" + fileName + "的资源！"); 55 | } 56 | p = new Properties(); 57 | p.load(is); 58 | register.put(fileName, p); 59 | is.close(); 60 | } 61 | } catch (Exception e) { 62 | log.error("读取properties时异常", e); 63 | } finally { 64 | if (is != null) 65 | try { 66 | is.close(); 67 | } catch (IOException e) { 68 | log.error("读取properties关闭流时异常", e); 69 | } 70 | } 71 | return p; 72 | } 73 | 74 | public String getPropertyValue(String fileName, String strKey) { 75 | Properties p = getProperties(fileName); 76 | try { 77 | return p.getProperty(strKey); 78 | } catch (Exception e) { 79 | log.error("读取properties时异常", e); 80 | } 81 | return null; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /bd-hdfs/src/main/java/org/bd/hdfs/utils/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package org.bd.hdfs.utils; 2 | 3 | import java.io.FileInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.util.Hashtable; 7 | import java.util.Properties; 8 | 9 | import org.apache.log4j.Logger; 10 | 11 | /** 12 | * 版权信息: big data module
13 | * 功能描述: 读取属性配置文件工具类
14 | * 版本历史: 15 | * @author wpk | 2017年11月15日 上午10:25:41 |创建 16 | */ 17 | public class PropertiesUtil { 18 | 19 | static PropertiesUtil pu;// 创建对象pu 20 | private static Hashtable register = new Hashtable(); 21 | private static Logger log = Logger.getLogger(PropertiesUtil.class); 22 | 23 | private PropertiesUtil() { 24 | super(); 25 | } 26 | 27 | /** 28 | * 取得PropertiesUtil的一个实例 29 | */ 30 | public static PropertiesUtil getInstance() { 31 | if (pu == null) 32 | pu = new PropertiesUtil(); 33 | return pu; 34 | } 35 | 36 | /** 37 | * 读取配置文件 38 | */ 39 | private Properties getProperties(String fileName) { 40 | InputStream is = null; 41 | Properties p = null; 42 | try { 43 | p = (Properties) register.get(fileName); 44 | if (p == null) { 45 | try { 46 | is = new FileInputStream(fileName); 47 | } catch (Exception e) { 48 | if (fileName.startsWith("/")) 49 | is = PropertiesUtil.class.getResourceAsStream(fileName); 50 | else 51 | is = PropertiesUtil.class.getResourceAsStream("/" + fileName); 52 | } 53 | if (is == null) { 54 | log.info("未找到名称为" + fileName + "的资源！"); 55 | } 56 | p = new Properties(); 57 | p.load(is); 58 | register.put(fileName, p); 59 | is.close(); 60 | } 61 | } catch (Exception e) { 62 | log.error("读取properties时异常", e); 63 | } finally { 64 | if (is != null) 65 | try { 66 | is.close(); 67 | } catch (IOException e) { 68 | log.error("读取properties关闭流时异常", e); 69 | } 70 | } 71 | return p; 72 | } 73 | 74 | public String getPropertyValue(String fileName, String strKey) { 75 | Properties p = getProperties(fileName); 76 | try { 77 | return p.getProperty(strKey); 78 | } catch (Exception e) { 79 | log.error("读取properties时异常", e); 80 | } 81 | return null; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/utils/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.utils; 2 | 3 | import java.io.FileInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.util.Hashtable; 7 | import java.util.Properties; 8 | 9 | import org.apache.log4j.Logger; 10 | 11 | /** 12 | * 版权信息: big data module
13 | * 功能描述: 读取属性配置文件工具类
14 | * 版本历史: 15 | * @author wpk | 2017年8月23日 上午11:20:19 |创建 16 | */ 17 | public class PropertiesUtil { 18 | 19 | static PropertiesUtil pu;// 创建对象pu 20 | private static Hashtable register = new Hashtable(); 21 | private static Logger log = Logger.getLogger(PropertiesUtil.class); 22 | 23 | private PropertiesUtil() { 24 | super(); 25 | } 26 | 27 | /** 28 | * 取得PropertiesUtil的一个实例 29 | */ 30 | public static PropertiesUtil getInstance() { 31 | if (pu == null) 32 | pu = new PropertiesUtil(); 33 | return pu; 34 | } 35 | 36 | /** 37 | * 读取配置文件 38 | */ 39 | private Properties getProperties(String fileName) { 40 | InputStream is = null; 41 | Properties p = null; 42 | try { 43 | p = (Properties) register.get(fileName); 44 | if (p == null) { 45 | try { 46 | is = new FileInputStream(fileName); 47 | } catch (Exception e) { 48 | if (fileName.startsWith("/")) 49 | is = PropertiesUtil.class.getResourceAsStream(fileName); 50 | else 51 | is = PropertiesUtil.class.getResourceAsStream("/" + fileName); 52 | } 53 | if (is == null) { 54 | log.info("未找到名称为" + fileName + "的资源！"); 55 | } 56 | p = new Properties(); 57 | p.load(is); 58 | register.put(fileName, p); 59 | is.close(); 60 | } 61 | } catch (Exception e) { 62 | log.error("读取properties时异常", e); 63 | } finally { 64 | if (is != null) 65 | try { 66 | is.close(); 67 | } catch (IOException e) { 68 | log.error("读取properties关闭流时异常", e); 69 | } 70 | } 71 | return p; 72 | } 73 | 74 | public String getPropertyValue(String fileName, String strKey) { 75 | Properties p = getProperties(fileName); 76 | try { 77 | return p.getProperty(strKey); 78 | } catch (Exception e) { 79 | log.error("读取properties时异常", e); 80 | } 81 | return null; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /bd-impala/src/main/java/org/bd/impala/utils/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package org.bd.impala.utils; 2 | 3 | import java.io.FileInputStream; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.util.Hashtable; 7 | import java.util.Properties; 8 | 9 | import org.apache.log4j.Logger; 10 | 11 | /** 12 | * 版权信息: big data module
13 | * 功能描述: 读取属性配置文件工具类
14 | * 版本历史: 15 | * @author wpk | 2017年8月23日 上午11:20:19 |创建 16 | */ 17 | public class PropertiesUtil { 18 | 19 | static PropertiesUtil pu;// 创建对象pu 20 | private static Hashtable register = new Hashtable(); 21 | private static Logger log = Logger.getLogger(PropertiesUtil.class); 22 | 23 | private PropertiesUtil() { 24 | super(); 25 | } 26 | 27 | /** 28 | * 取得PropertiesUtil的一个实例 29 | */ 30 | public static PropertiesUtil getInstance() { 31 | if (pu == null) 32 | pu = new PropertiesUtil(); 33 | return pu; 34 | } 35 | 36 | /** 37 | * 读取配置文件 38 | */ 39 | private Properties getProperties(String fileName) { 40 | InputStream is = null; 41 | Properties p = null; 42 | try { 43 | p = (Properties) register.get(fileName); 44 | if (p == null) { 45 | try { 46 | is = new FileInputStream(fileName); 47 | } catch (Exception e) { 48 | if (fileName.startsWith("/")) 49 | is = PropertiesUtil.class.getResourceAsStream(fileName); 50 | else 51 | is = PropertiesUtil.class.getResourceAsStream("/" + fileName); 52 | } 53 | if (is == null) { 54 | log.info("未找到名称为" + fileName + "的资源！"); 55 | } 56 | p = new Properties(); 57 | p.load(is); 58 | register.put(fileName, p); 59 | is.close(); 60 | } 61 | } catch (Exception e) { 62 | log.error("读取properties时异常", e); 63 | } finally { 64 | if (is != null) 65 | try { 66 | is.close(); 67 | } catch (IOException e) { 68 | log.error("读取properties关闭流时异常", e); 69 | } 70 | } 71 | return p; 72 | } 73 | 74 | public String getPropertyValue(String fileName, String strKey) { 75 | Properties p = getProperties(fileName); 76 | try { 77 | return p.getProperty(strKey); 78 | } catch (Exception e) { 79 | log.error("读取properties时异常", e); 80 | } 81 | return null; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /bd-mapreduce/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | 11 | bd-mapreduce 12 | bd-mapreduce 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | 28 | org.apache.hadoop 29 | hadoop-common 30 | 2.6.0 31 | 32 | 33 | 34 | org.apache.hadoop 35 | hadoop-mapreduce-client-core 36 | 2.6.0 37 | 38 | 39 | 40 | jdk.tools 41 | jdk.tools 42 | 1.7 43 | system 44 | ${JAVA_HOME}/lib/tools.jar 45 | 46 | 47 | 48 | 49 | 50 | 51 | maven-compiler-plugin 52 | 2.3.2 53 | 54 | 1.7 55 | 1.7 56 | 57 | 58 | 59 | maven-assembly-plugin 60 | 61 | 62 | jar-with-dependencies 63 | 64 | 65 | 66 | 67 | make-assembly 68 | package 69 | 70 | single 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/bean/TransformerBean.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.bean; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | import org.json.JSONArray; 8 | 9 | import org.bd.datax.bean.Transformer.Parameter; 10 | import org.bd.datax.DataxException; 11 | 12 | /** 13 | * 版权信息: big data module
14 | * 功能描述: Transformer转换实体类
15 | * 版本历史: 16 | * @author wpk | 2017年10月15日 上午11:01:30 |创建 17 | */ 18 | @SuppressWarnings("rawtypes") 19 | public class TransformerBean { 20 | 21 | public final static String column = "column"; 22 | public final static String code = "code"; 23 | public final static String conf = "字段转换配置"; 24 | 25 | /** 26 | * 描述： 获取转换对象 27 | * @author wpk | 2017年10月15日 上午11:02:29 |创建 28 | * @param map 29 | * @return Transformer 30 | */ 31 | public Transformer getBean(Map map){ 32 | Transformer bean = new Transformer(); 33 | Parameter para = bean.new Parameter(); 34 | if(map.containsKey(column) && map.get(column) != null){ 35 | if(map.get(column) instanceof JSONArray){ 36 | JSONArray ja = (JSONArray)map.get(column); 37 | String code = getCode(ja); 38 | if(null == code){ 39 | throw new DataxException(String.format("%s【%s】不存在或者为空值", conf, code)); 40 | } 41 | para.setCode(code); 42 | List list = new ArrayList(); 43 | list.add("import groovy.json.JsonSlurper;"); 44 | para.setExtraPackage(list); 45 | }else{ 46 | throw new DataxException(String.format("%s【%s】类型错误,请输入%s类型", conf, column, JSONArray.class)); 47 | } 48 | } 49 | 50 | bean.setName("dx_groovy"); 51 | bean.setParameter(para); 52 | return bean; 53 | } 54 | 55 | /** 56 | * 描述： 获取code 57 | * @author wpk | 2017年10月15日 上午11:18:48 |创建 58 | * @param ja 59 | * @return String 60 | */ 61 | private String getCode(JSONArray ja){ 62 | StringBuffer sb = new StringBuffer(); 63 | for(int i=0;i0){ 69 | sb.append("return record;"); 70 | return sb.toString(); 71 | } 72 | return null; 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /bd-kafka/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | 11 | bd-kafka 12 | bd-kafka 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | org.apache.kafka 28 | kafka_2.11 29 | 0.10.1.0 30 | 31 | 32 | org.apache.kafka 33 | kafka-clients 34 | 0.10.1.0 35 | 36 | 37 | log4j 38 | log4j 39 | 1.2.17 40 | 41 | 42 | 43 | 44 | net.sf.json-lib 45 | json-lib 46 | 2.4 47 | jdk15 48 | 49 | 50 | 51 | 52 | 53 | 54 | maven-compiler-plugin 55 | 2.3.2 56 | 57 | 1.8 58 | 1.8 59 | 60 | 61 | 62 | maven-assembly-plugin 63 | 64 | 65 | jar-with-dependencies 66 | 67 | 68 | 69 | 70 | make-assembly 71 | package 72 | 73 | single 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/request/CubeRequest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.request; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述: 创建cube请求参数对象
6 | * 版本历史: 7 | * @author wpk | 2017年11月21日 上午9:48:09 |创建 8 | */ 9 | public class CubeRequest { 10 | 11 | private String project; 12 | private String cubeDescData; 13 | /*private String uuid; 14 | private String cubeName; 15 | private String streamingData; 16 | private String kafkaData; 17 | private boolean successful; 18 | private String message; 19 | private String streamingCube;*/ 20 | 21 | public CubeRequest() {} 22 | 23 | /*public CubeRequest(String cubeName, String cubeDescData) { 24 | this.cubeName = cubeName; 25 | this.cubeDescData = cubeDescData; 26 | }*/ 27 | 28 | public String getProject() { 29 | return project; 30 | } 31 | 32 | public void setProject(String project) { 33 | this.project = project; 34 | } 35 | 36 | public String getCubeDescData() { 37 | return cubeDescData; 38 | } 39 | 40 | public void setCubeDescData(String cubeDescData) { 41 | this.cubeDescData = cubeDescData; 42 | } 43 | 44 | /*public String getUuid() { 45 | return uuid; 46 | } 47 | 48 | public void setUuid(String uuid) { 49 | this.uuid = uuid; 50 | } 51 | 52 | public String getCubeName() { 53 | return cubeName; 54 | } 55 | 56 | public void setCubeName(String cubeName) { 57 | this.cubeName = cubeName; 58 | } 59 | 60 | public String getStreamingData() { 61 | return streamingData; 62 | } 63 | 64 | public void setStreamingData(String streamingData) { 65 | this.streamingData = streamingData; 66 | } 67 | 68 | public String getKafkaData() { 69 | return kafkaData; 70 | } 71 | 72 | public void setKafkaData(String kafkaData) { 73 | this.kafkaData = kafkaData; 74 | } 75 | 76 | public boolean isSuccessful() { 77 | return successful; 78 | } 79 | 80 | public void setSuccessful(boolean successful) { 81 | this.successful = successful; 82 | } 83 | 84 | public String getMessage() { 85 | return message; 86 | } 87 | 88 | public void setMessage(String message) { 89 | this.message = message; 90 | } 91 | 92 | public String getStreamingCube() { 93 | return streamingCube; 94 | } 95 | 96 | public void setStreamingCube(String streamingCube) { 97 | this.streamingCube = streamingCube; 98 | }*/ 99 | 100 | } 101 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/rest/response/QueryRestResponse.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.rest.response; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import org.bd.kylin.request.SQLRequest; 7 | import org.bd.kylin.response.SQLResponse; 8 | import org.bd.kylin.response.TableMeta; 9 | import org.bd.kylin.rest.QueryRest; 10 | import org.bd.kylin.utils.JsonBinder; 11 | 12 | import com.fasterxml.jackson.core.JsonParseException; 13 | import com.fasterxml.jackson.core.type.TypeReference; 14 | import com.fasterxml.jackson.databind.JsonMappingException; 15 | import com.fasterxml.jackson.databind.ObjectMapper; 16 | 17 | /** 18 | * 版权信息: big data module
19 | * 功能描述: kylin语句查询
20 | * 版本历史: 21 | * @author wpk | 2017年11月21日 下午3:00:27 |创建 22 | */ 23 | public class QueryRestResponse extends QueryRest{ 24 | 25 | /** 26 | * 描述： 获取kylin上的所有表元数据信息
27 | * 所有表指的是用于创建模型和cube的表 28 | * @author wpk | 2017年11月21日 下午3:01:11 |创建 29 | * @param project 30 | * @throws JsonParseException 31 | * @throws JsonMappingException 32 | * @throws IOException 33 | * @return List 34 | */ 35 | @SuppressWarnings("unchecked") 36 | public static List getMetadatasResp(String project) throws JsonParseException, JsonMappingException, IOException{ 37 | String result = getMetadatas(project); 38 | ObjectMapper om = JsonBinder.buildNonNullBinder().getMapper(); 39 | List list = (List)om.readValue(result, new TypeReference>() {}); 40 | return list; 41 | } 42 | 43 | /** 44 | * 描述： 查询语句 45 | * @author wpk | 2017年11月21日 下午5:06:21 |创建 46 | * @param sr 47 | * @throws IOException 48 | * @return SQLResponse 49 | */ 50 | public static SQLResponse queryResp(SQLRequest sr) throws IOException{ 51 | String result = query(sr); 52 | ObjectMapper om = JsonBinder.buildNonNullBinder().getMapper(); 53 | SQLResponse sqlResult = om.readValue(result, SQLResponse.class); 54 | return sqlResult; 55 | } 56 | 57 | /** 58 | * 描述： 查询语句 59 | * @author wpk | 2017年11月21日 下午5:22:47 |创建 60 | * @param project 61 | * @param sql 62 | * @throws IOException 63 | * @return SQLResponse 64 | */ 65 | public static SQLResponse queryResp(String project, String sql) throws IOException{ 66 | SQLRequest item = new SQLRequest(); 67 | item.setProject(project); 68 | item.setSql(sql); 69 | return queryResp(item); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/rest/ModelRest.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.rest; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | import org.apache.log4j.Logger; 5 | import org.bd.kylin.RestRequstHandle; 6 | 7 | /** 8 | * 版权信息: big data module
9 | * 功能描述: model请求rest接口类
10 | * 版本历史: 11 | * @author wpk | 2017年10月30日 下午2:33:52 |创建 12 | */ 13 | public class ModelRest extends RestRequstHandle{ 14 | 15 | private static final Logger logger = Logger.getLogger(ModelRest.class); 16 | 17 | /** 18 | * 描述： 通过模型名称获取模型信息 19 | * @author wpk | 2017年10月30日 下午2:36:25 |创建 20 | * @param modelName 21 | * @throws Exception 22 | * @return String 23 | */ 24 | public static String getModel(String modelName) { 25 | String para = "model/"+modelName; 26 | String result = request(para, RequestMethod.GET); 27 | return result; 28 | } 29 | 30 | /** 31 | * 描述： 提交模型 32 | * @author wpk | 2017年10月30日 下午2:36:34 |创建 33 | * @param json 34 | * @throws Exception 35 | * @return String 36 | */ 37 | public static String submitModel(String json) { 38 | String para = "models/"; 39 | String result = request(para, RequestMethod.POST, json); 40 | return result; 41 | } 42 | 43 | /** 44 | * 描述： 删除model（注意，先删除model下的所有cube后再删除model） 45 | * @author wpk | 2017年10月30日 下午2:36:58 |创建 46 | * @param modelName 47 | * @throws Exception 48 | * @return String 49 | */ 50 | public static void cancelModel(String modelName) { 51 | String para = "models/"+modelName; 52 | try { 53 | request(para, RequestMethod.DELETE); 54 | } catch (Exception e) { 55 | String str = "Data Model with name "+modelName+" not found"; 56 | if(!e.getMessage().contains(str)){ 57 | logger.equals(e.getMessage()); 58 | } 59 | } 60 | } 61 | 62 | /** 63 | * 描述： 获取kylin上所有的模型 64 | * @author wpk | 2017年11月21日 上午10:00:09 |创建 65 | * @return String 66 | */ 67 | public static String getModels(){ 68 | return getModels(null); 69 | } 70 | 71 | /** 72 | * 描述： 获取指定项目下的所有模型 73 | * @author wpk | 2017年11月21日 上午10:01:22 |创建 74 | * @param project 75 | * @return String 76 | */ 77 | public static String getModels(String project){ 78 | StringBuilder para = new StringBuilder(); 79 | para.append("models"); 80 | if(StringUtils.isNotBlank(project)){ 81 | para.append("?project=").append(project); 82 | } 83 | String result = request(para.toString(), RequestMethod.GET); 84 | return result; 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /bd-kafka/src/main/java/org/bd/kafka/ProducerClient.java: -------------------------------------------------------------------------------- 1 | package org.bd.kafka; 2 | 3 | import java.util.List; 4 | import java.util.Properties; 5 | import java.util.concurrent.ExecutionException; 6 | import java.util.concurrent.Future; 7 | 8 | import org.apache.kafka.clients.producer.KafkaProducer; 9 | import org.apache.kafka.clients.producer.ProducerRecord; 10 | import org.apache.kafka.clients.producer.RecordMetadata; 11 | import org.apache.kafka.common.PartitionInfo; 12 | import org.bd.kafka.utils.PropertiesUtil; 13 | 14 | /** 15 | * 版权信息: big data module
16 | * 功能描述: kafka生产者客户端
17 | * 版本历史: 18 | * @author wpk | 2017年11月26日 下午1:29:19 |创建 19 | */ 20 | public class ProducerClient { 21 | 22 | private KafkaProducer producer = null; 23 | private String topic; 24 | 25 | public ProducerClient(String topic){ 26 | this.topic = topic; 27 | init(); 28 | } 29 | 30 | /** 31 | * 描述： 初始化生产者 32 | * @author wpk | 2017年11月26日 下午1:32:09 |创建 33 | * @return void 34 | */ 35 | public void init(){ 36 | if(null == producer){ 37 | Properties properties = PropertiesUtil.getInstance().getProperties("producer.properties"); 38 | producer = new KafkaProducer(properties); 39 | } 40 | } 41 | 42 | /** 43 | * 描述： 获取分区信息 44 | * @author wpk | 2017年11月26日 下午1:32:01 |创建 45 | * @return List 46 | */ 47 | public List getPartition(){ 48 | List list = producer.partitionsFor(topic); 49 | for(PartitionInfo p : list){ 50 | System.out.println(p.toString()); 51 | } 52 | return list; 53 | } 54 | 55 | /** 56 | * 描述： 发送信息 57 | * @author wpk | 2017年11月26日 下午1:34:23 |创建 58 | * @param msg 59 | * @return Future 60 | * @throws ExecutionException 61 | * @throws InterruptedException 62 | */ 63 | public RecordMetadata sendMsg(String msg) throws InterruptedException, ExecutionException{ 64 | Future fture = producer.send(new ProducerRecord(topic, msg)); 65 | RecordMetadata rm = fture.get(); 66 | return rm; 67 | } 68 | 69 | /** 70 | * 描述： 关闭生产者 71 | * @author wpk | 2017年11月26日 下午1:42:35 |创建 72 | * @return void 73 | */ 74 | public void close(){ 75 | if(null != producer){ 76 | producer.close(); 77 | } 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /bd-spark/src/main/java/org/bd/spark/SessionDrive.java: -------------------------------------------------------------------------------- 1 | package org.bd.spark; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.spark.SparkConf; 7 | import org.apache.spark.sql.SparkSession; 8 | 9 | /** 10 | * 版权信息: big data module
11 | * 功能描述: 创建spark程序起点
12 | * 版本历史: 13 | * @author wpk | 2017年7月21日下午2:13:27 |创建 14 | */ 15 | public class SessionDrive { 16 | 17 | public volatile static SessionDrive ssd; 18 | public final static String OTHER = "other"; 19 | public final static String HIVE = "hive"; 20 | public final static String SUPPORT_TYPE = "support.type"; 21 | 22 | public static Map ssMap = new HashMap(); 23 | 24 | /** 默认构造类*/ 25 | public SessionDrive(){} 26 | 27 | /** 28 | * 描述：获取实例 29 | * @author wpk | 2017年7月21日下午2:16:24 |创建 30 | * @return 31 | */ 32 | public static SessionDrive getInstance() { 33 | if (ssd == null){ 34 | synchronized (SessionDrive.class) { 35 | if(ssd == null){ 36 | ssd = new SessionDrive(); 37 | } 38 | } 39 | } 40 | return ssd; 41 | } 42 | 43 | /** 44 | * 描述：获取SparkSession 45 | * @author wpk | 2017年7月21日下午2:41:44 |创建 46 | * @param conf 47 | * @return 48 | * @throws Exception 49 | */ 50 | public SparkSession getSparkSession(SparkConf conf) throws Exception{ 51 | SparkSession spark = null; 52 | try { 53 | if(!conf.contains(SUPPORT_TYPE)){ 54 | if(!ssMap.containsKey(OTHER)){ 55 | spark = SparkSession.builder().config(conf).getOrCreate(); 56 | }else{ 57 | spark = ssMap.get(OTHER); 58 | } 59 | }else{ 60 | String opType = conf.get(SUPPORT_TYPE); 61 | if(HIVE.equals(opType)){ 62 | if(!ssMap.containsKey(HIVE)){ 63 | spark = SparkSession.builder().config(conf).enableHiveSupport().getOrCreate(); 64 | }else{ 65 | spark = ssMap.get(HIVE); 66 | } 67 | }else{ 68 | throw new Exception(String.format("获取SparkSeesion失败,传入的类型[%s]不存在。", opType)); 69 | } 70 | } 71 | } catch (Exception e) { 72 | throw new Exception(e); 73 | } 74 | return spark; 75 | } 76 | 77 | /** 78 | * 描述：关闭SparkSession 79 | * @author wpk | 2017年7月24日下午5:11:49 |创建 80 | * @param opType 81 | * @throws Exception 82 | */ 83 | public void stop(String opType) throws Exception{ 84 | try { 85 | if(ssMap.containsKey(opType)){ 86 | ssMap.get(opType).stop();//停止 87 | ssMap.remove(opType);//移除 88 | } 89 | } catch (Exception e) { 90 | throw new Exception(e); 91 | } 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /bd-hbase/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | 11 | bd-hbase 12 | bd-hbase 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | 28 | org.apache.hbase 29 | hbase-client 30 | 1.1.1 31 | 32 | 47 | 48 | 49 | jdk.tools 50 | jdk.tools 51 | 1.7 52 | system 53 | ${JAVA_HOME}/lib/tools.jar 54 | 55 | 56 | 57 | 58 | 59 | 60 | maven-compiler-plugin 61 | 2.3.2 62 | 63 | 1.8 64 | 1.8 65 | 66 | 67 | 68 | maven-assembly-plugin 69 | 70 | 71 | jar-with-dependencies 72 | 73 | 74 | 75 | 76 | make-assembly 77 | package 78 | 79 | single 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /bd-hdfs/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | 11 | bd-hdfs 12 | bd-hdfs 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 27 | 28 | org.apache.hadoop 29 | hadoop-common 30 | 3.1.1 31 | 32 | 33 | 34 | 35 | org.apache.hadoop 36 | hadoop-hdfs 37 | 3.1.1 38 | 39 | 40 | 41 | org.apache.hadoop 42 | hadoop-client 43 | 3.1.1 44 | 45 | 46 | 47 | 48 | 49 | jdk.tools 50 | jdk.tools 51 | 1.7 52 | system 53 | ${JAVA_HOME}/lib/tools.jar 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | maven-compiler-plugin 62 | 2.3.2 63 | 64 | 1.8 65 | 1.8 66 | 67 | 68 | 69 | maven-assembly-plugin 70 | 71 | 72 | jar-with-dependencies 73 | 74 | 75 | 76 | 77 | make-assembly 78 | package 79 | 80 | single 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | -------------------------------------------------------------------------------- /bd-kylin/src/main/java/org/bd/kylin/response/HBaseResponse.java: -------------------------------------------------------------------------------- 1 | package org.bd.kylin.response; 2 | 3 | /** 4 | * 版权信息: big data module
5 | * 功能描述:
6 | * 版本历史: 7 | * @author wpk | 2017年11月21日 上午9:53:30 |创建 8 | */ 9 | public class HBaseResponse { 10 | 11 | private String segmentName; 12 | private String segmentStatus; 13 | private String tableName; 14 | private long tableSize; 15 | private int regionCount; 16 | private long dateRangeStart; 17 | private long dateRangeEnd; 18 | private long sourceOffsetStart; 19 | private long sourceOffsetEnd; 20 | private long sourceCount; 21 | 22 | public HBaseResponse() { 23 | } 24 | 25 | public String getTableName() { 26 | return tableName; 27 | } 28 | 29 | public void setTableName(String tableName) { 30 | this.tableName = tableName; 31 | } 32 | 33 | public long getTableSize() { 34 | return tableSize; 35 | } 36 | 37 | public void setTableSize(long tableSize) { 38 | this.tableSize = tableSize; 39 | } 40 | 41 | public int getRegionCount() { 42 | return regionCount; 43 | } 44 | 45 | public void setRegionCount(int regionCount) { 46 | this.regionCount = regionCount; 47 | } 48 | 49 | public long getDateRangeStart() { 50 | return dateRangeStart; 51 | } 52 | 53 | public void setDateRangeStart(long dateRangeStart) { 54 | this.dateRangeStart = dateRangeStart; 55 | } 56 | 57 | public long getDateRangeEnd() { 58 | return dateRangeEnd; 59 | } 60 | 61 | public void setDateRangeEnd(long dateRangeEnd) { 62 | this.dateRangeEnd = dateRangeEnd; 63 | } 64 | 65 | public String getSegmentName() { 66 | return segmentName; 67 | } 68 | 69 | public void setSegmentName(String segmentName) { 70 | this.segmentName = segmentName; 71 | } 72 | 73 | public String getSegmentStatus() { 74 | return segmentStatus; 75 | } 76 | 77 | public void setSegmentStatus(String segmentStatus) { 78 | this.segmentStatus = segmentStatus; 79 | } 80 | 81 | public long getSourceOffsetStart() { 82 | return sourceOffsetStart; 83 | } 84 | 85 | public void setSourceOffsetStart(long sourceOffsetStart) { 86 | this.sourceOffsetStart = sourceOffsetStart; 87 | } 88 | 89 | public long getSourceOffsetEnd() { 90 | return sourceOffsetEnd; 91 | } 92 | 93 | public void setSourceOffsetEnd(long sourceOffsetEnd) { 94 | this.sourceOffsetEnd = sourceOffsetEnd; 95 | } 96 | 97 | public long getSourceCount() { 98 | return sourceCount; 99 | } 100 | 101 | public void setSourceCount(long sourceCount) { 102 | this.sourceCount = sourceCount; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /bd-hive/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.bd 7 | wpk-manage 8 | 0.0.1-SNAPSHOT 9 | 10 | 11 | bd-hive 12 | bd-hive 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 3.8.1 24 | test 25 | 26 | 31 | 41 | 42 | org.apache.hive 43 | hive-jdbc 44 | 1.1.0 45 | 46 | 47 | org.apache.hadoop 48 | hadoop-common 49 | 2.6.0 50 | 51 | 52 | 53 | com.alibaba 54 | druid 55 | 1.0.20 56 | 57 | 58 | jdk.tools 59 | jdk.tools 60 | 1.7 61 | system 62 | ${JAVA_HOME}/lib/tools.jar 63 | 64 | 65 | 66 | 67 | 68 | 69 | maven-compiler-plugin 70 | 2.3.2 71 | 72 | 1.8 73 | 1.8 74 | 75 | 76 | 77 | maven-assembly-plugin 78 | 79 | 80 | jar-with-dependencies 81 | 82 | 83 | 84 | 85 | make-assembly 86 | package 87 | 88 | single 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/mysql/MysqlRead.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.mysql; 2 | 3 | import java.util.List; 4 | 5 | import org.bd.datax.bean.Read; 6 | 7 | /** 8 | * 版权信息: big data module
9 | * 功能描述: mysql读取配置
10 | * 版本历史: 11 | * @author wpk | 2017年9月20日 下午9:20:02 |创建 12 | */ 13 | public class MysqlRead extends Read{ 14 | 15 | private String name = "mysqlreader"; 16 | private Parameter parameter; 17 | 18 | public class Parameter{ 19 | private String username;//必填-用户名 20 | private String password;//必填-密码 21 | //目的表需要读取数据的字段,字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。 22 | //如果要依次读取全部列，使用*表示, 例如: "column": ["*"] 23 | private List column;//必填 24 | private String splitPk;//对字段进行分片，仅支持整数型、字符串型的字段 25 | private String where;//筛选条件 26 | private List connection;//必填 27 | 28 | public class Connection{ 29 | private List jdbcUrl;//必填-jdbc连接信息 30 | private List table;//必填-同步表 31 | private List querySql;//自定义执行语句，注意（当用户配置querySql时，直接忽略table、column、where条件的配置） 32 | 33 | public List getJdbcUrl() { 34 | return jdbcUrl; 35 | } 36 | public void setJdbcUrl(List jdbcUrl) { 37 | this.jdbcUrl = jdbcUrl; 38 | } 39 | public List getTable() { 40 | return table; 41 | } 42 | public void setTable(List table) { 43 | this.table = table; 44 | } 45 | public List getQuerySql() { 46 | return querySql; 47 | } 48 | public void setQuerySql(List querySql) { 49 | this.querySql = querySql; 50 | } 51 | } 52 | 53 | public String getUsername() { 54 | return username; 55 | } 56 | public void setUsername(String username) { 57 | this.username = username; 58 | } 59 | public String getPassword() { 60 | return password; 61 | } 62 | public void setPassword(String password) { 63 | this.password = password; 64 | } 65 | public List getColumn() { 66 | return column; 67 | } 68 | public void setColumn(List column) { 69 | this.column = column; 70 | } 71 | public String getSplitPk() { 72 | return splitPk; 73 | } 74 | public void setSplitPk(String splitPk) { 75 | this.splitPk = splitPk; 76 | } 77 | public String getWhere() { 78 | return where; 79 | } 80 | public void setWhere(String where) { 81 | this.where = where; 82 | } 83 | public List getConnection() { 84 | return connection; 85 | } 86 | public void setConnection(List connection) { 87 | this.connection = connection; 88 | } 89 | } 90 | 91 | public String getName() { 92 | return name; 93 | } 94 | 95 | public void setName(String name) { 96 | this.name = name; 97 | } 98 | 99 | public Parameter getParameter() { 100 | return parameter; 101 | } 102 | 103 | public void setParameter(Parameter parameter) { 104 | this.parameter = parameter; 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /bd-datax/src/main/java/org/bd/datax/sqlserver/SqlServerWrite.java: -------------------------------------------------------------------------------- 1 | package org.bd.datax.sqlserver; 2 | 3 | import java.util.List; 4 | 5 | import org.bd.datax.bean.Write; 6 | 7 | /** 8 | * 版权信息: big data module
9 | * 功能描述: 写入配置
10 | * 版本历史: 11 | * @author wpk | 2017年10月13日 上午11:31:58 |创建 12 | */ 13 | public class SqlServerWrite extends Write{ 14 | 15 | private String name = "sqlserverwriter"; 16 | private WriteParameter parameter; 17 | 18 | public class WriteParameter{ 19 | private String username;//必填-用户名 20 | private String password;//必填-密码 21 | //目的表需要写入数据的字段,字段之间用英文逗号分隔。例如: "column": ["id","name","age"]。 22 | //如果要依次写入全部列，使用*表示, 例如: "column": ["*"] 23 | private List column;//必填 24 | private List preSql;//写入前执行的语句 25 | private List postSql;//写入后执行的语句 26 | private Integer batchSize;//一次性批量提交的记录数大小，默认”1024“ 27 | private List connection;//必填 28 | 29 | public class WriteConnection{ 30 | private String jdbcUrl;//必填-jdbc连接信息 31 | private List table;//必填-同步表 32 | 33 | public String getJdbcUrl() { 34 | return jdbcUrl; 35 | } 36 | public void setJdbcUrl(String jdbcUrl) { 37 | this.jdbcUrl = jdbcUrl; 38 | } 39 | public List getTable() { 40 | return table; 41 | } 42 | public void setTable(List table) { 43 | this.table = table; 44 | } 45 | } 46 | 47 | public String getUsername() { 48 | return username; 49 | } 50 | public void setUsername(String username) { 51 | this.username = username; 52 | } 53 | public String getPassword() { 54 | return password; 55 | } 56 | public void setPassword(String password) { 57 | this.password = password; 58 | } 59 | public List getColumn() { 60 | return column; 61 | } 62 | public void setColumn(List column) { 63 | this.column = column; 64 | } 65 | public List getPreSql() { 66 | return preSql; 67 | } 68 | public void setPreSql(List preSql) { 69 | this.preSql = preSql; 70 | } 71 | public List getPostSql() { 72 | return postSql; 73 | } 74 | public void setPostSql(List postSql) { 75 | this.postSql = postSql; 76 | } 77 | public Integer getBatchSize() { 78 | return batchSize; 79 | } 80 | public void setBatchSize(Integer batchSize) { 81 | this.batchSize = batchSize; 82 | } 83 | public List getConnection() { 84 | return connection; 85 | } 86 | public void setConnection(List connection) { 87 | this.connection = connection; 88 | } 89 | } 90 | 91 | public String getName() { 92 | return name; 93 | } 94 | 95 | public void setName(String name) { 96 | this.name = name; 97 | } 98 | 99 | public WriteParameter getParameter() { 100 | return parameter; 101 | } 102 | 103 | public void setParameter(WriteParameter parameter) { 104 | this.parameter = parameter; 105 | } 106 | 107 | } 108 | --------------------------------------------------------------------------------