├── .gitignore ├── LICENSE ├── README.md ├── pom.xml ├── yongqing-bigdata-examples ├── README.md ├── kafka-connector-example │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── yongqing │ │ │ │ └── kafka │ │ │ │ └── connector │ │ │ │ └── example │ │ │ │ ├── sink │ │ │ │ ├── ExampleSinkConnector.java │ │ │ │ └── ExampleSinkTask.java │ │ │ │ └── source │ │ │ │ ├── ExampleSourceConnector.java │ │ │ │ └── ExampleSourceTask.java │ │ └── resources │ │ │ ├── kafka.properties │ │ │ └── log4j2.xml │ │ └── test │ │ └── java │ │ └── com │ │ └── yongqing │ │ └── kafka │ │ └── connector │ │ └── example │ │ └── Test.java └── pom.xml └── yongqing-bigdata-tools ├── README.md ├── pom.xml ├── yongqing-bigdata-business-log-flume ├── docs │ └── jiagou1.png ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── log │ │ │ └── flume │ │ │ ├── notice │ │ │ ├── AbstractNotice.java │ │ │ └── Notice.java │ │ │ └── sink │ │ │ ├── AbstractBigDataSink.java │ │ │ ├── ElasticSearchSink.java │ │ │ ├── HbaseSink.java │ │ │ ├── HiveByNoTransactionSink.java │ │ │ ├── HiveSink.java │ │ │ ├── KafkaSink.java │ │ │ ├── MongoDbSink.java │ │ │ ├── SinkConstants.java │ │ │ ├── bean │ │ │ └── DataBaseConfig.java │ │ │ └── log │ │ │ └── process │ │ │ ├── AbstractEventTo.java │ │ │ ├── EventExec.java │ │ │ ├── EventTo.java │ │ │ ├── EventToEs.java │ │ │ ├── EventToHbase.java │ │ │ ├── EventToHive.java │ │ │ ├── EventToHiveByNoTransaction.java │ │ │ ├── EventToKafka.java │ │ │ └── EventToMongoDb.java │ └── resources │ │ ├── flume-conf.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── log │ └── flume │ └── Test.java ├── yongqing-bigdata-tools-common ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── common │ │ │ └── bigdata │ │ │ └── tool │ │ │ ├── Base64Utils.java │ │ │ ├── BigDataCommonUtil.java │ │ │ ├── BigDataHttpClient.java │ │ │ ├── DateCommonUtil.java │ │ │ ├── DayUtil.java │ │ │ ├── ExcelUtil.java │ │ │ ├── FileStreamUtils.java │ │ │ ├── GsonUtil.java │ │ │ ├── GuidUtil.java │ │ │ ├── HtmlParser.java │ │ │ ├── HtmlToPdf.java │ │ │ ├── ListUtil.java │ │ │ ├── MD5Utils.java │ │ │ ├── MonthUtil.java │ │ │ ├── NullStringToEmptyAdapterFactory.java │ │ │ ├── PdfUtil.java │ │ │ ├── ReflectionUtils.java │ │ │ ├── UUIDGenerator.java │ │ │ └── WeekUtil.java │ └── resources │ │ ├── common.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── common │ └── Test.java ├── yongqing-crawler-analysis ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── yongqing │ │ └── crawler │ │ └── analysis │ │ ├── TaxCrawlerAbStractAnalysis.java │ │ ├── TaxCrawlerAnalysis.java │ │ ├── constants │ │ └── TaxCrawlerConstant.java │ │ ├── exception │ │ └── CrawlerExcption.java │ │ └── tax │ │ └── bean │ │ ├── TaxCrawlerAnalysisBean.java │ │ ├── TaxPdfTable.java │ │ └── TaxTable.java │ └── test │ └── java │ └── com │ └── yongqing │ └── crawler │ └── analysis │ └── Test.java ├── yongqing-elasticsearch-tool ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── elasticsearch │ │ │ └── client │ │ │ ├── Client.java │ │ │ ├── DefaultSingletonElasticSearchClient.java │ │ │ ├── DefaultSingletonElasticSearchPoolClient.java │ │ │ ├── ElasticSearchAbstractClient.java │ │ │ ├── ElasticSearchClient.java │ │ │ ├── ElasticSearchPoolClient.java │ │ │ ├── bean │ │ │ └── IndexBean.java │ │ │ └── exception │ │ │ └── ElasticsearchException.java │ └── resources │ │ ├── elasticsearch.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── elasticsearch │ └── Test.java ├── yongqing-etcd-tool-v2 ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── etcd │ │ │ ├── action │ │ │ ├── Action.java │ │ │ ├── DefaultSingletonEtcdSubject.java │ │ │ ├── EtcdSubject.java │ │ │ └── Observer.java │ │ │ ├── exec │ │ │ └── Exec.java │ │ │ └── tools │ │ │ ├── EtcdUtil.java │ │ │ └── core │ │ │ └── CommonUtil.java │ └── resources │ │ ├── etcd.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── etcd │ └── Test.java ├── yongqing-etcd-tool ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── etcd │ │ │ ├── action │ │ │ ├── Action.java │ │ │ ├── DefaultSingletonEtcdSubject.java │ │ │ ├── EtcdSubject.java │ │ │ └── Observer.java │ │ │ ├── exec │ │ │ └── Exec.java │ │ │ └── tools │ │ │ ├── EtcdUtil.java │ │ │ └── core │ │ │ └── CommonUtil.java │ └── resources │ │ ├── etcd.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── etcd │ └── Test.java ├── yongqing-flume-canal-source ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── flume │ │ │ └── canal │ │ │ └── source │ │ │ └── CanalSource.java │ └── resources │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── flume │ └── canal │ └── source │ └── Test.java ├── yongqing-hbase-tool ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── hbase │ │ │ ├── exception │ │ │ └── HbaseException.java │ │ │ └── utils │ │ │ ├── Client.java │ │ │ ├── HbaseAbstractClient.java │ │ │ └── HbaseClient.java │ └── resources │ │ ├── hbase.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── hbase │ └── Test.java ├── yongqing-hdfs-tool ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── hdfs │ │ │ └── tool │ │ │ ├── AbstractHdfsClient.java │ │ │ ├── Clinet.java │ │ │ ├── DefaultHdfsClient.java │ │ │ ├── HdfsClient.java │ │ │ ├── constant │ │ │ └── Constants.java │ │ │ └── pojo │ │ │ └── ParameterPojo.java │ └── resources │ │ ├── hdfs.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── hdfs │ └── tool │ └── Test.java ├── yongqing-hive-tool ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── hive │ │ │ └── tool │ │ │ ├── DefaultSingletonHiveWrite.java │ │ │ ├── Event.java │ │ │ ├── HiveDataSerializer.java │ │ │ ├── HiveDataSerializerImpl.java │ │ │ ├── HiveException.java │ │ │ ├── HiveJsonDataSerializerImpl.java │ │ │ ├── HiveWrite.java │ │ │ ├── HiveWriter.java │ │ │ ├── client │ │ │ ├── Client.java │ │ │ ├── DefaultHiveMetaStoreClient.java │ │ │ └── HiveMetaStoreClient.java │ │ │ ├── constant │ │ │ └── Constants.java │ │ │ ├── hcatalog │ │ │ └── HCataLogClient.java │ │ │ ├── pojo │ │ │ ├── DefaultEvent.java │ │ │ ├── FieldDealPojo.java │ │ │ └── ParameterPojo.java │ │ │ └── queue │ │ │ └── EventQueue.java │ └── resources │ │ ├── hive.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── hive │ └── tool │ └── Test.java ├── yongqing-jdbc-common ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── yongqing │ └── jdbc │ └── common │ ├── JdbcConnection.java │ └── JdbcConnectionPool.java ├── yongqing-kafka-tool ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── kafka │ │ │ ├── consumer │ │ │ └── MyKafkaConsumer.java │ │ │ └── producer │ │ │ ├── KafkaAbstractProducer.java │ │ │ ├── KafkaPartitioner.java │ │ │ ├── KafkaProducer.java │ │ │ └── MyKafkaProducer.java │ └── resources │ │ ├── kafka.properties │ │ ├── log4j.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── kafka │ └── Test.java ├── yongqing-log-output ├── README.md ├── docs │ └── jiagou1.png ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── log │ │ │ └── output │ │ │ ├── factory │ │ │ ├── LogFactory.java │ │ │ └── adapter │ │ │ │ └── NullStringToEmptyAdapterFactory.java │ │ │ └── utils │ │ │ ├── DateUtil.java │ │ │ ├── LogPrint.java │ │ │ ├── UUIDGenerator.java │ │ │ ├── WebToolUtil.java │ │ │ └── gson │ │ │ └── GsonUtil.java │ └── resources │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── log │ └── output │ └── Test.java ├── yongqing-log-processor ├── README.md ├── docs │ └── jiagou1.png ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── yongqing │ │ └── processor │ │ └── log │ │ ├── Processor.java │ │ └── bean │ │ ├── BusinessLog.java │ │ ├── CrawlerLog.java │ │ ├── MysqlBinLog.java │ │ ├── NginxLog.java │ │ ├── ProcessorBean.java │ │ ├── SqlLog.java │ │ └── SystemRequestLog.java │ └── test │ └── java │ └── com │ └── yongqing │ └── processor │ └── log │ └── Test.java ├── yongqing-mongodb-jdbc ├── README.md ├── pom.xml └── src │ └── main │ ├── java │ └── mongodb │ │ ├── conn │ │ ├── MongoClientConnection.java │ │ ├── MongoExecutor.java │ │ ├── ServerConnection.java │ │ └── Subtable.java │ │ ├── jdbc │ │ ├── MongoArray.java │ │ ├── MongoConnection.java │ │ ├── MongoDataSource.java │ │ ├── MongoDataSourceFactory.java │ │ ├── MongoDatabaseMetaData.java │ │ ├── MongoDriver.java │ │ ├── MongoPooledConnection.java │ │ ├── MongoPreparedStatement.java │ │ ├── MongoResultSet.java │ │ ├── MongoResultSetMetaData.java │ │ ├── MongoStatement.java │ │ ├── MongoTableData.java │ │ ├── MongoXAConnection.java │ │ ├── MongoXADataSource.java │ │ └── MongoXAResource.java │ │ └── query │ │ ├── MongoBuilder.java │ │ ├── MongoBuilderFatalException.java │ │ ├── MongoBuilderUpstreamException.java │ │ ├── MongoCreateIndex.java │ │ ├── MongoDeleteQuery.java │ │ ├── MongoDropIndex.java │ │ ├── MongoDropTable.java │ │ ├── MongoExpression.java │ │ ├── MongoInsertQuery.java │ │ ├── MongoQuery.java │ │ ├── MongoSelectQuery.java │ │ ├── MongoUpdateQuery.java │ │ └── MongoUpsertQuery.java │ └── resources │ └── mongo │ ├── MongoConnection.properties │ ├── MongoDriver.properties │ ├── MongoPreparedStatement.properties │ ├── MongoResultSet.properties │ ├── MongoResultSetMetaData.properties │ ├── MongoStatement.properties │ └── ServerConnection.properties ├── yongqing-mongodb-tool ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── mongodb │ │ │ └── client │ │ │ ├── Client.java │ │ │ ├── MongodbAbstractBaseClient.java │ │ │ ├── MongodbAbstractClient.java │ │ │ ├── MongodbAbstractPoolClient.java │ │ │ ├── MongodbClient.java │ │ │ ├── MongodbClientEtcdAction.java │ │ │ └── MongodbPoolClient.java │ └── resources │ │ ├── log4j2.xml │ │ └── mongodb.properties │ └── test │ └── java │ └── com │ └── yongqing │ └── mongodb │ └── Test.java ├── yongqing-mybatis-sqlOutput-plugin ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── mybatis │ │ │ └── sqloutput │ │ │ └── plugin │ │ │ └── MybatisSqlOutputPlugin.java │ └── resources │ │ ├── log4j2.xml │ │ └── yongqing-mybatis-sqlOutput-plugin.properties │ └── test │ └── java │ └── com │ └── yongqing │ └── mybatis │ └── sqloutput │ └── plugin │ └── Test.java ├── yongqing-presto-hbase ├── pom.xml └── src │ ├── main │ ├── java │ │ ├── com │ │ │ └── yongqing │ │ │ │ └── presto │ │ │ │ └── hbase │ │ │ │ ├── HbaseClient.java │ │ │ │ ├── HbaseConnector.java │ │ │ │ ├── HbaseConnectorFactory.java │ │ │ │ ├── HbaseConnectorId.java │ │ │ │ ├── HbaseErrorCode.java │ │ │ │ ├── HbaseHandleResolver.java │ │ │ │ ├── HbaseMetadata.java │ │ │ │ ├── HbaseMetadataFactory.java │ │ │ │ ├── HbaseModule.java │ │ │ │ ├── HbasePlugin.java │ │ │ │ ├── HbaseSplitManager.java │ │ │ │ ├── HbaseTableManager.java │ │ │ │ ├── Types.java │ │ │ │ ├── conf │ │ │ │ ├── HbaseConfig.java │ │ │ │ ├── HbaseSessionProperties.java │ │ │ │ └── HbaseTableProperties.java │ │ │ │ ├── io │ │ │ │ ├── HbasePageSink.java │ │ │ │ ├── HbasePageSinkProvider.java │ │ │ │ ├── HbaseRecordCursor.java │ │ │ │ ├── HbaseRecordSet.java │ │ │ │ └── HbaseRecordSetProvider.java │ │ │ │ ├── metadata │ │ │ │ ├── HbaseTable.java │ │ │ │ ├── HbaseView.java │ │ │ │ └── ZooKeeperMetadataManager.java │ │ │ │ ├── model │ │ │ │ ├── HbaseColumnConstraint.java │ │ │ │ ├── HbaseColumnHandle.java │ │ │ │ ├── HbaseSplit.java │ │ │ │ ├── HbaseTableHandle.java │ │ │ │ ├── HbaseTableLayoutHandle.java │ │ │ │ └── HbaseTransactionHandle.java │ │ │ │ └── serializers │ │ │ │ └── HbaseRowSerializerUtil.java │ │ └── org │ │ │ └── apache │ │ │ └── hadoop │ │ │ └── hbase │ │ │ └── mapreduce │ │ │ └── TabletSplitMetadata.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── com.facebook.presto.spi.Plugin │ └── test │ └── resources │ └── log4j.properties ├── yongqing-rocketmq-flume-tool ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── rocketmq │ │ │ ├── alyun │ │ │ └── listen │ │ │ │ └── AliyunRocketmqMessageListener.java │ │ │ ├── elasticsearch │ │ │ ├── ContentBuilderUtil.java │ │ │ ├── DefaultIndexNameBuilder.java │ │ │ ├── ElasticSearchEventSerializer.java │ │ │ ├── ElasticSearchRocketMqEventSerializer.java │ │ │ ├── ElasticSearchSinkConstants.java │ │ │ ├── HbaseSinkConstants.java │ │ │ ├── IndexNameBuilder.java │ │ │ ├── TimeBasedIndexNameBuilder.java │ │ │ ├── TimestampedEvent.java │ │ │ └── client │ │ │ │ ├── Client.java │ │ │ │ ├── ElasticSearchClient.java │ │ │ │ ├── ElasticSearchHttpClient.java │ │ │ │ └── RoundRobinList.java │ │ │ ├── flume │ │ │ ├── sink │ │ │ │ ├── ElasticsearchSink.java │ │ │ │ └── HbaseSink.java │ │ │ └── source │ │ │ │ └── RocketmqSource.java │ │ │ └── queue │ │ │ └── FlumeMq.java │ └── resources │ │ ├── flume-conf.properties │ │ └── log4j2.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── rocketmq │ └── Test.java ├── yongqing-spring-mybatis-sqlOutput ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── spring │ │ │ └── mybatis │ │ │ └── sqloutput │ │ │ ├── annotation │ │ │ └── SqlOutput.java │ │ │ ├── aspect │ │ │ └── SqlOutputAdvice.java │ │ │ ├── autoconfigure │ │ │ └── SelfAutoConfig.java │ │ │ ├── mapper │ │ │ └── SqlBaseMapper.java │ │ │ ├── plugin │ │ │ └── MybatisSqlOutputPlugin.java │ │ │ └── util │ │ │ └── SqlUtil.java │ └── resources │ │ ├── META-INF │ │ └── spring.factories │ │ ├── log4j2.xml │ │ └── yongqing-mybatis-sqlOutput-plugin.properties │ └── test │ └── java │ └── com │ └── yongqing │ └── spring │ └── mybatis │ └── sqloutput │ └── Test.java ├── yongqing-sql-analyse ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── sql │ │ │ └── analyse │ │ │ ├── MybatisSessionFactory.java │ │ │ ├── bean │ │ │ └── SqlOutput.java │ │ │ ├── exception │ │ │ └── SqlAnalyseException.java │ │ │ ├── sqlexec │ │ │ ├── DefaultSqlExec.java │ │ │ ├── JdbcSqlExec.java │ │ │ ├── MybatisSqlExec.java │ │ │ └── SqlExec.java │ │ │ └── util │ │ │ ├── ParseUpdateSqlUtil.java │ │ │ └── SqlLogUtil.java │ └── resources │ │ ├── log4j2.xml │ │ └── mybatis-config.xml │ └── test │ └── java │ └── com │ └── yongqing │ └── sql │ └── analyse │ └── Test.java ├── yongqing-thread-pool-tool ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── yongqing │ │ │ └── thread │ │ │ └── pool │ │ │ └── tool │ │ │ ├── AbstractPoolExec.java │ │ │ ├── CallableFunction.java │ │ │ ├── DefaultPoolExec.java │ │ │ ├── Exec.java │ │ │ ├── FutureFunction.java │ │ │ └── PoolExec.java │ └── resources │ │ ├── log4j2.xml │ │ └── threadPool.properties │ └── test │ └── java │ └── com │ └── yongqing │ └── thread │ └── pool │ └── tool │ └── Test.java └── yongqing-zookeeper-tool ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── yongqing │ │ └── zookeeper │ │ └── tool │ │ ├── client │ │ ├── AbstractZKClient.java │ │ ├── DefaultZKClient.java │ │ └── ZKClient.java │ │ ├── distributed │ │ └── DistributedLock.java │ │ └── exec │ │ └── Exec.java └── resources │ └── log4j2.xml └── test └── java └── com └── yongqing └── zookeeper └── tool └── Test.java /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | *.iml 10 | 11 | # Mobile Tools for Java (J2ME) 12 | .mtj.tmp/ 13 | 14 | # Package Files # 15 | *.jar 16 | *.war 17 | *.nar 18 | *.ear 19 | *.zip 20 | *.tar.gz 21 | *.rar 22 | 23 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 24 | hs_err_pid* 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # yongqing 2 | 张永清的开放大数据工具包项目 3 |
4 | yongqing-jdbc-common 5 |
6 | yongqing-bigdata-business-log-flume 7 |
8 | yongqing-bigdata-tools-common 9 |
10 | yongqing-crawler-analysis 11 |
12 | yongqing-elasticsearch-tool 13 |
14 | yongqing-etcd-tool 15 |
16 | yongqing-etcd-tool-v2 17 |
18 | yongqing-flume-canal-source 19 |
20 | yongqing-hbase-tool 21 |
22 | yongqing-hdfs-tool 23 |
24 | yongqing-hive-tool 25 |
26 | yongqing-kafka-tool 27 |
28 | yongqing-log-output 29 |
30 | yongqing-log-processor 31 |
32 | yongqing-mongodb-jdbc 33 |
34 | yongqing-mongodb-tool 35 |
36 | yongqing-mybatis-sqlOutput-plugin 37 |
38 | yongqing-presto-hbase 39 |
40 | yongqing-rocketmq-flume-tool 41 |
42 | yongqing-spring-mybatis-sqlOutput 43 |
44 | yongqing-sql-analyse 45 |
46 | yongqing-thread-pool-tool 47 |
48 | yongqing-zookeeper-tool 49 |
-------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | yongqing.bigdata.tools 6 | bigdata-tool-parent 7 | pom 8 | 1.0 9 | 10 | UTF-8 11 | 1.8 12 | 13 | 14 | yongqing-bigdata-tools 15 | yongqing-bigdata-examples 16 | 17 | 18 | 19 | 20 | org.apache.maven.plugins 21 | maven-compiler-plugin 22 | 23 | ${java.version} 24 | ${java.version} 25 | true 26 | ${project.build.sourceEncoding} 27 | 28 | 29 | 30 | compile 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /yongqing-bigdata-examples/README.md: -------------------------------------------------------------------------------- 1 | # yongqing-bigdata-examples 2 | 3 | 大数据相关操作的示例工程 -------------------------------------------------------------------------------- /yongqing-bigdata-examples/kafka-connector-example/src/main/java/com/yongqing/kafka/connector/example/sink/ExampleSinkConnector.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.kafka.connector.example.sink; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | import org.apache.kafka.common.utils.AppInfoParser; 5 | import org.apache.kafka.connect.connector.Task; 6 | import org.apache.kafka.connect.sink.SinkConnector; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | /** 12 | * 13 | */ 14 | public class ExampleSinkConnector extends SinkConnector{ 15 | @Override 16 | public void start(Map map) { 17 | 18 | } 19 | //指定Task执行的类 20 | @Override 21 | public Class taskClass() { 22 | return ExampleSinkTask.class; 23 | } 24 | //task对应的config 25 | @Override 26 | public List> taskConfigs(int i) { 27 | return null; 28 | } 29 | 30 | @Override 31 | public void stop() { 32 | 33 | } 34 | //配置定义 35 | @Override 36 | public ConfigDef config() { 37 | return null; 38 | } 39 | 40 | @Override 41 | public String version() { 42 | return AppInfoParser.getVersion(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /yongqing-bigdata-examples/kafka-connector-example/src/main/java/com/yongqing/kafka/connector/example/sink/ExampleSinkTask.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.kafka.connector.example.sink; 2 | 3 | import org.apache.kafka.clients.consumer.OffsetAndMetadata; 4 | import org.apache.kafka.common.TopicPartition; 5 | import org.apache.kafka.connect.sink.SinkRecord; 6 | import org.apache.kafka.connect.sink.SinkTask; 7 | 8 | import java.util.Collection; 9 | import java.util.Map; 10 | 11 | /** 12 | * 13 | */ 14 | public class ExampleSinkTask extends SinkTask { 15 | @Override 16 | public String version() { 17 | return new ExampleSinkConnector().version(); 18 | } 19 | 20 | @Override 21 | public void start(Map map) { 22 | 23 | } 24 | 25 | @Override 26 | public void put(Collection collection) { 27 | 28 | } 29 | @Override 30 | public void flush(Map offsets){ 31 | 32 | } 33 | @Override 34 | public void stop() { 35 | 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /yongqing-bigdata-examples/kafka-connector-example/src/main/java/com/yongqing/kafka/connector/example/source/ExampleSourceConnector.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.kafka.connector.example.source; 2 | 3 | import org.apache.kafka.common.config.ConfigDef; 4 | import org.apache.kafka.common.utils.AppInfoParser; 5 | import org.apache.kafka.connect.connector.Task; 6 | import org.apache.kafka.connect.source.SourceConnector; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | /** 12 | * 13 | */ 14 | public class ExampleSourceConnector extends SourceConnector{ 15 | @Override 16 | public void start(Map map) { 17 | 18 | } 19 | //返回需要指定的TASK 20 | @Override 21 | public Class taskClass() { 22 | return ExampleSourceTask.class; 23 | } 24 | //TASK的配置 25 | @Override 26 | public List> taskConfigs(int i) { 27 | return null; 28 | } 29 | 30 | @Override 31 | public void stop() { 32 | 33 | } 34 | 35 | @Override 36 | public ConfigDef config() { 37 | return null; 38 | } 39 | 40 | @Override 41 | public String version() { 42 | return AppInfoParser.getVersion(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /yongqing-bigdata-examples/kafka-connector-example/src/main/java/com/yongqing/kafka/connector/example/source/ExampleSourceTask.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.kafka.connector.example.source; 2 | 3 | import org.apache.kafka.connect.source.SourceRecord; 4 | import org.apache.kafka.connect.source.SourceTask; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | /** 10 | * 11 | */ 12 | public class ExampleSourceTask extends SourceTask { 13 | @Override 14 | public String version() { 15 | return new ExampleSourceConnector().version(); 16 | } 17 | //任务启动 18 | @Override 19 | public void start(Map map) { 20 | 21 | } 22 | //需要发送到kafka的数据。 23 | @Override 24 | public List poll() throws InterruptedException { 25 | return null; 26 | } 27 | //任务停止 28 | @Override 29 | public void stop() { 30 | 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /yongqing-bigdata-examples/kafka-connector-example/src/main/resources/kafka.properties: -------------------------------------------------------------------------------- 1 | endpoints=http://47.111.122.xxx:2379 -------------------------------------------------------------------------------- /yongqing-bigdata-examples/kafka-connector-example/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | kafka-connector-example 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-examples/kafka-connector-example/src/test/java/com/yongqing/kafka/connector/example/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.kafka.connector.example; 2 | 3 | /** 4 | * 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-examples/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | yongqing.bigdata.tools 7 | bigdata-tool-parent 8 | 1.0 9 | 10 | yongqing-bigdata-examples 11 | pom 12 | 1.0 13 | 14 | 15 | 16 | 17 | 18 | kafka-connector-example 19 | 20 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/README.md: -------------------------------------------------------------------------------- 1 | # yongqing-bigdata-tools 2 | 3 | 大数据工具包 4 |
5 | yongqing-bigdata-business-log-flume 6 |
7 | yongqing-bigdata-tools-common 8 |
9 | yongqing-crawler-analysis 10 |
11 | yongqing-elasticsearch-tool 12 |
13 | yongqing-etcd-tool 14 |
15 | yongqing-etcd-tool-v2 16 |
17 | yongqing-flume-canal-source 18 |
19 | yongqing-hbase-tool 20 |
21 | yongqing-hdfs-tool 22 |
23 | yongqing-hive-tool 24 |
25 | yongqing-kafka-tool 26 |
27 | yongqing-log-output 28 |
29 | yongqing-log-processor 30 |
31 | yongqing-mongodb-jdbc 32 |
33 | yongqing-mongodb-tool 34 |
35 | yongqing-mybatis-sqlOutput-plugin 36 |
37 | yongqing-presto-hbase 38 |
39 | yongqing-rocketmq-flume-tool 40 |
41 | yongqing-spring-mybatis-sqlOutput 42 |
43 | yongqing-sql-analyse 44 |
45 | yongqing-thread-pool-tool 46 |
47 | yongqing-zookeeper-tool 48 |
-------------------------------------------------------------------------------- /yongqing-bigdata-tools/pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | yongqing.bigdata.tools 7 | bigdata-tool-parent 8 | 1.0 9 | 10 | yongqing-bigdata-tools 11 | pom 12 | 1.0 13 | 14 | 15 | 16 | 17 | 18 | yongqing-jdbc-common 19 | yongqing-log-output 20 | yongqing-etcd-tool 21 | yongqing-etcd-tool-v2 22 | yongqing-hbase-tool 23 | yongqing-rocketmq-flume-tool 24 | yongqing-elasticsearch-tool 25 | yongqing-bigdata-tools-common 26 | yongqing-bigdata-business-log-flume 27 | yongqing-mongodb-tool 28 | yongqing-kafka-tool 29 | yongqing-mongodb-jdbc 30 | yongqing-log-processor 31 | yongqing-thread-pool-tool 32 | yongqing-mybatis-sqlOutput-plugin 33 | yongqing-spring-mybatis-sqlOutput 34 | yongqing-sql-analyse 35 | yongqing-crawler-analysis 36 | yongqing-hdfs-tool 37 | yongqing-hive-tool 38 | yongqing-flume-canal-source 39 | yongqing-zookeeper-tool 40 | yongqing-presto-hbase 41 | 42 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/docs/jiagou1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/597365581/bigdata_tools/c82f36db5dc4a557bdcf611736a8d3dc34a4ff0b/yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/docs/jiagou1.png -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/src/main/java/com/yongqing/log/flume/notice/AbstractNotice.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.flume.notice; 2 | 3 | import com.yongqing.common.bigdata.tool.BigDataHttpClient; 4 | import com.yongqing.common.bigdata.tool.GsonUtil; 5 | import com.yongqing.etcd.tools.EtcdUtil; 6 | 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | /** 11 | * 12 | */ 13 | public abstract class AbstractNotice implements Notice{ 14 | 15 | @Override 16 | public void noticePostLog(List> noticeMsg) { 17 | BigDataHttpClient.postJsonData(EtcdUtil.getLocalPropertie("noticeUrl"), GsonUtil.gson.toJson(noticeMsg)); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/src/main/java/com/yongqing/log/flume/notice/Notice.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.flume.notice; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | /** 7 | * 8 | */ 9 | public interface Notice { 10 | void noticePostLog(String logType); 11 | void noticePostLog(List> noticeMsg); 12 | } 13 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/src/main/java/com/yongqing/log/flume/sink/SinkConstants.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.flume.sink; 2 | 3 | import com.google.gson.reflect.TypeToken; 4 | import com.yongqing.common.bigdata.tool.GsonUtil; 5 | import com.yongqing.etcd.action.Action; 6 | import com.yongqing.etcd.tools.EtcdUtil; 7 | import com.yongqing.processor.log.Processor; 8 | import com.yongqing.processor.log.bean.ProcessorBean; 9 | 10 | import java.util.List; 11 | import java.util.Properties; 12 | 13 | /** 14 | * 15 | */ 16 | public class SinkConstants implements Action { 17 | 18 | public static final String BATCH_SIZE = "batchSize"; 19 | public static final String LOG_TYPE = "logType"; 20 | public static final String KAFKA_TOPIC = "kafkaTopic"; 21 | public static final String KAFKA_PARTITION_NUMBER = "kafkaPartitionNumber"; 22 | public static final String ETCD_CONFIG = "etcdConfig"; 23 | public static volatile List processorBeanList; 24 | 25 | @Override 26 | public void doAction(Properties oldProp, Properties newProp) { 27 | synchronized (SinkConstants.class) { 28 | if ((null == oldProp.getProperty("processors") && null != newProp.getProperty("processors")) || (null != oldProp.getProperty("processors") && null != newProp.getProperty("processors") && !oldProp.getProperty("processors").equals(newProp.getProperty("processors")))) { 29 | processorBeanList = GsonUtil.gson.fromJson(EtcdUtil.getLocalPropertie("processors"), new TypeToken>() { 30 | }.getType()); 31 | processorBeanList.forEach(processorBean -> { 32 | try { 33 | Processor processor = (Processor) Class.forName(processorBean.getProcessor()).newInstance(); 34 | processorBean.setProcessorInstance(processor); 35 | } catch (Throwable e) { 36 | e.printStackTrace(); 37 | } 38 | }); 39 | } else if (null != oldProp.getProperty("processors") && null == newProp.getProperty("processors")) { 40 | if (null != processorBeanList) { 41 | processorBeanList.clear(); 42 | processorBeanList = null; 43 | } 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/src/main/java/com/yongqing/log/flume/sink/bean/DataBaseConfig.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.flume.sink.bean; 2 | 3 | /** 4 | * 5 | */ 6 | public class DataBaseConfig { 7 | private String businessId; 8 | private String databaseName; 9 | private String tableName; 10 | private String primaryKeyField; 11 | private String filterFields; 12 | 13 | public String getFilterFields() { 14 | return filterFields; 15 | } 16 | 17 | public void setFilterFields(String filterFields) { 18 | this.filterFields = filterFields; 19 | } 20 | 21 | public String getBusinessId() { 22 | return businessId; 23 | } 24 | 25 | public void setBusinessId(String businessId) { 26 | this.businessId = businessId; 27 | } 28 | 29 | public String getDatabaseName() { 30 | return databaseName; 31 | } 32 | 33 | public void setDatabaseName(String databaseName) { 34 | this.databaseName = databaseName; 35 | } 36 | 37 | public String getTableName() { 38 | return tableName; 39 | } 40 | 41 | public void setTableName(String tableName) { 42 | this.tableName = tableName; 43 | } 44 | 45 | public String getPrimaryKeyField() { 46 | return primaryKeyField; 47 | } 48 | 49 | public void setPrimaryKeyField(String primaryKeyField) { 50 | this.primaryKeyField = primaryKeyField; 51 | } 52 | 53 | @Override 54 | public String toString() { 55 | return "DataBaseConfig{" + 56 | "businessId='" + businessId + '\'' + 57 | ", databaseName='" + databaseName + '\'' + 58 | ", tableName='" + tableName + '\'' + 59 | ", primaryKeyField='" + primaryKeyField + '\'' + 60 | ", filterFields='" + filterFields + '\'' + 61 | '}'; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/src/main/java/com/yongqing/log/flume/sink/log/process/EventExec.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.flume.sink.log.process; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * 7 | */ 8 | @FunctionalInterface 9 | public interface EventExec { 10 | void exec(String docId, Map customizeField, String dataBaseName, String tableName); 11 | } 12 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/src/main/java/com/yongqing/log/flume/sink/log/process/EventTo.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.flume.sink.log.process; 2 | 3 | import org.apache.flume.Event; 4 | 5 | /** 6 | * 7 | */ 8 | public interface EventTo { 9 | void addEvent(Event event, String logType); 10 | void execute(String logType); 11 | } 12 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/src/main/java/com/yongqing/log/flume/sink/log/process/EventToEs.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.flume.sink.log.process; 2 | 3 | 4 | import com.yongqing.elasticsearch.client.DefaultSingletonElasticSearchClient; 5 | import com.yongqing.etcd.action.Action; 6 | import com.yongqing.etcd.tools.EtcdUtil; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | import java.util.*; 10 | 11 | 12 | 13 | /** 14 | * 15 | */ 16 | public class EventToEs extends AbstractEventTo implements Action { 17 | private static final Logger logger = LoggerFactory 18 | .getLogger(EventToEs.class); 19 | 20 | public EventToEs() { 21 | super(); 22 | logList = new ArrayList(); 23 | DefaultSingletonElasticSearchClient.getInstance(EtcdUtil.getLocalPropertie("esHosts").split(",")); 24 | } 25 | public EventToEs(List logList) { 26 | this.logList = logList; 27 | DefaultSingletonElasticSearchClient.getInstance(EtcdUtil.getLocalPropertie("esHosts").split(",")); 28 | } 29 | public void execute(String logType){ 30 | super.execute(logType,(String docId, Map customizeField, String dataBaseName, String tableName)->{ 31 | try { 32 | DefaultSingletonElasticSearchClient.getInstance(EtcdUtil.getLocalPropertie("esHosts").split(",")).upDdateocAsUpsert(dataBaseName, tableName, docId, customizeField); 33 | } catch (Throwable e) { 34 | logger.info("elasticSearchClient upDdateocAsUpsert cause Exception", e); 35 | } 36 | } ); 37 | } 38 | @Override 39 | public void doAction(Properties properties, Properties properties1) { 40 | if (null != properties.getProperty("esHosts") && null != properties1.getProperty("esHosts") && !properties.getProperty("esHosts").equals(properties1.getProperty("esHosts"))) { 41 | synchronized (this) { 42 | try { 43 | DefaultSingletonElasticSearchClient.close(); 44 | } catch (Exception e) { 45 | logger.error("es connection close cause Exception", e); 46 | } 47 | DefaultSingletonElasticSearchClient.getInstance(EtcdUtil.getLocalPropertie("esHosts").split(",")); 48 | } 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | log-flume 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-business-log-flume/src/test/java/com/yongqing/log/flume/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.flume; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-tools-common/src/main/java/com/yongqing/common/bigdata/tool/ExcelUtil.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.common.bigdata.tool; 2 | 3 | import lombok.extern.log4j.Log4j2; 4 | import org.apache.poi.hssf.usermodel.HSSFWorkbook; 5 | import org.apache.poi.ss.usermodel.Workbook; 6 | import org.apache.poi.xssf.usermodel.XSSFWorkbook; 7 | 8 | import java.io.File; 9 | import java.io.FileInputStream; 10 | import java.io.IOException; 11 | import java.io.InputStream; 12 | 13 | /** 14 | * excel 工具类 15 | */ 16 | @Log4j2 17 | public class ExcelUtil { 18 | private ExcelUtil(){ 19 | 20 | } 21 | public static boolean isExcel(File file) { 22 | return file.getName().toLowerCase().endsWith("xlsx") || file.getName().toLowerCase().endsWith("xls"); 23 | } 24 | 25 | public static Workbook getReadWorkbook(File excelFile) throws Exception { 26 | InputStream inputStream = null; 27 | try { 28 | if(!excelFile.exists()){ 29 | throw new RuntimeException("excelFile:"+excelFile+" is not exist"); 30 | } 31 | inputStream = new FileInputStream(excelFile); 32 | if (excelFile.getName().toLowerCase().endsWith("xlsx")) { 33 | return new XSSFWorkbook(inputStream); 34 | } else if (excelFile.getName().toLowerCase().endsWith("xls")) { 35 | return new HSSFWorkbook(inputStream); 36 | } else { 37 | throw new RuntimeException("Invalid excel file"); 38 | } 39 | } catch (IOException e) { 40 | log.error("getReadWorkbook cause Exception",e); 41 | throw new RuntimeException(e.getMessage()); 42 | } finally { 43 | if(null!=inputStream){ 44 | inputStream.close(); 45 | } 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-tools-common/src/main/java/com/yongqing/common/bigdata/tool/GsonUtil.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.common.bigdata.tool; 2 | 3 | 4 | import com.google.gson.Gson; 5 | import com.google.gson.GsonBuilder; 6 | import com.google.gson.LongSerializationPolicy; 7 | 8 | 9 | /** 10 | * json相关处理的工具类 11 | */ 12 | 13 | public class GsonUtil { 14 | 15 | //定义gson对像，对象的String属性为null时也支持序列化 16 | public static final Gson gson = new GsonBuilder().setLongSerializationPolicy(LongSerializationPolicy.STRING).disableHtmlEscaping().registerTypeAdapterFactory(new NullStringToEmptyAdapterFactory()).setDateFormat("yyyy-MM-dd HH:mm:ss.SSS").create(); 17 | } 18 | 19 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-tools-common/src/main/java/com/yongqing/common/bigdata/tool/ListUtil.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.common.bigdata.tool; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | /** 7 | * 8 | */ 9 | public class ListUtil { 10 | public static List getPagingList(List list,Integer start,Integer length){ 11 | start = start<0?0:start; 12 | //默认为10 13 | length = length<=0?10:length; 14 | Integer size = list.size(); 15 | if(start>size){ 16 | start = size; 17 | } 18 | Integer toIndex = (start+length-1)>=size?size:(start+length-1); 19 | if(toIndex<=0){ 20 | toIndex = size; 21 | } 22 | return list.subList(start,toIndex); 23 | } 24 | public static List> averageAssign(List source, int n) { 25 | if(source.size()> result = new ArrayList>(); 29 | int remainder = source.size() % n; //(先计算出余数) 30 | int number = source.size() / n; //然后是商 31 | int offset = 0;//偏移量 32 | for (int i = 0; i < n; i++) { 33 | List value = null; 34 | if (remainder > 0) { 35 | value = source.subList(i * number + offset, (i + 1) * number + offset + 1); 36 | remainder--; 37 | offset++; 38 | } else { 39 | value = source.subList(i * number + offset, (i + 1) * number + offset); 40 | } 41 | result.add(value); 42 | } 43 | return result; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-tools-common/src/main/java/com/yongqing/common/bigdata/tool/MD5Utils.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.common.bigdata.tool; 2 | 3 | import lombok.extern.log4j.Log4j2; 4 | import org.apache.commons.codec.binary.Base64; 5 | 6 | import java.security.MessageDigest; 7 | import java.security.NoSuchAlgorithmException; 8 | 9 | /** 10 | * 11 | */ 12 | @Log4j2 13 | public class MD5Utils { 14 | private static MessageDigest md5 =null; 15 | static { 16 | try { 17 | md5 = MessageDigest.getInstance("MD5"); 18 | } catch (NoSuchAlgorithmException e) { 19 | log.error("MessageDigest.getInstance(\"MD5\") cause Exception",e); 20 | } 21 | } 22 | //24位 23 | public static String getMD5StrByBase64String(String strValue) throws Exception { 24 | return Base64.encodeBase64String(md5.digest(strValue.getBytes("UTF-8"))); 25 | } 26 | //32位 27 | public static String getMD5Str(String strValue) throws Exception { 28 | String result = ""; 29 | md5.update((strValue).getBytes("UTF-8")); 30 | int i; 31 | StringBuffer buf = new StringBuffer(""); 32 | byte b[] = md5.digest(); 33 | for(int offset=0; offset { 27 | 28 | @Override 29 | public String read(JsonReader reader) throws IOException { 30 | if (reader.peek() == JsonToken.NULL) { 31 | reader.nextNull(); 32 | return ""; 33 | } 34 | return reader.nextString(); 35 | } 36 | 37 | @Override 38 | public void write(JsonWriter writer, String value) throws IOException { 39 | if (value == null) { 40 | writer.value(""); 41 | return; 42 | } 43 | writer.value(value); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-tools-common/src/main/java/com/yongqing/common/bigdata/tool/UUIDGenerator.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.common.bigdata.tool; 2 | 3 | import java.text.SimpleDateFormat; 4 | import java.util.UUID; 5 | 6 | /** 7 | * 生成唯一ID 8 | */ 9 | public class UUIDGenerator { 10 | /** 11 | * 12 | * 功能描述:
13 | * 〈UUID获取〉 14 | * 15 | * @return 16 | * @see [相关类/方法](可选) 17 | * @since [产品/模块版本](可选) 18 | */ 19 | private static final ThreadLocal threadRequestID = new ThreadLocal(); 20 | private static final ThreadLocal threadNumID = new ThreadLocal(); 21 | private static int guid=100; 22 | public static String getRequestID() { 23 | String requestID = (String) threadRequestID.get(); 24 | if (requestID == null) { 25 | requestID = UUIDGenerator.getUUID(); 26 | threadRequestID.set(requestID); 27 | } 28 | return requestID; 29 | } 30 | public static String getNumID() { 31 | String numID = (String) threadNumID.get(); 32 | if (numID == null) { 33 | numID = UUIDGenerator.getGuid(); 34 | threadNumID.set(numID); 35 | } 36 | return numID; 37 | } 38 | public static void destroyRequestID() { 39 | threadRequestID.set(null); 40 | } 41 | public static void destroyNumID() { 42 | threadNumID.set(null); 43 | } 44 | public static String getUUID() { 45 | // 取两组UUID拼接字符串 46 | String s = UUID.randomUUID().toString() + UUID.randomUUID().toString(); 47 | // 去除UUID中的'-',并截取成48位 48 | return s.substring(0, 8) + s.substring(9, 13) + s.substring(14, 18) + s.substring(19, 23)+s.substring(24,36) ; 49 | 50 | } 51 | 52 | public static String getGuid(){ 53 | guid+=1; 54 | long now = System.currentTimeMillis(); 55 | //获取4位年份数字 56 | SimpleDateFormat dateFormat=new SimpleDateFormat("yyyy"); 57 | //获取时间戳 58 | String time=dateFormat.format(now); 59 | String info=Long.toString(now)+""; 60 | int ran=0; 61 | if(guid>999){ 62 | guid=100; 63 | } 64 | ran=guid; 65 | 66 | return time+info.substring(2, info.length())+ran; 67 | } 68 | 69 | 70 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-tools-common/src/main/resources/common.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/597365581/bigdata_tools/c82f36db5dc4a557bdcf611736a8d3dc34a4ff0b/yongqing-bigdata-tools/yongqing-bigdata-tools-common/src/main/resources/common.properties -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-tools-common/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | yongqing-bigdata-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-bigdata-tools-common/src/test/java/com/yongqing/common/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.common; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-crawler-analysis/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | yongqing.bigdata.tools 8 | yongqing-bigdata-tools 9 | 1.0 10 | 11 | yongqing-crawler-analysis 12 | jar 13 | 1.0-SNAPSHOT 14 | 15 | 16 | 17 | 18 | yongqing.bigdata.tools 19 | yongqing-etcd-tool 20 | 1.0-SNAPSHOT 21 | 22 | 23 | 24 | 25 | 26 | 27 | org.apache.maven.plugins 28 | maven-compiler-plugin 29 | 30 | ${java.version} 31 | ${java.version} 32 | true 33 | ${project.build.sourceEncoding} 34 | 35 | 36 | 37 | org.apache.maven.plugins 38 | maven-jar-plugin 39 | 3.0.2 40 | 41 | 42 | log4j2.xml 43 | 44 | 45 | 46 | 47 | compile 48 | 49 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-crawler-analysis/src/main/java/com/yongqing/crawler/analysis/TaxCrawlerAbStractAnalysis.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.crawler.analysis; 2 | 3 | import com.yongqing.crawler.analysis.tax.bean.TaxPdfTable; 4 | import com.yongqing.crawler.analysis.tax.bean.TaxTable; 5 | 6 | /** 7 | * 8 | */ 9 | public abstract class TaxCrawlerAbStractAnalysis implements TaxCrawlerAnalysis{ 10 | @Override 11 | public TaxTable analysisTaxTable(TaxPdfTable taxPdfTable, String taxTableType){ 12 | TaxTable taxTable = new TaxTable(); 13 | taxTable.setDeclarationDate(taxPdfTable.getDeclarationDate()); 14 | taxTable.setDeclarationTableName(taxPdfTable.getDeclarationTableName()); 15 | taxTable.setStartTaxDate(taxPdfTable.getStartTaxDate()); 16 | taxTable.setEndTaxDate(taxPdfTable.getEndTaxDate()); 17 | taxTable.setTaxBalance(taxPdfTable.getTaxBalance()); 18 | taxTable.setDocId(taxPdfTable.getDocId()); 19 | taxTable.setRemark(taxPdfTable.getRemark()); 20 | taxTable.setType(taxPdfTable.getType()); 21 | return taxTable; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-crawler-analysis/src/main/java/com/yongqing/crawler/analysis/TaxCrawlerAnalysis.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.crawler.analysis; 2 | 3 | import com.yongqing.crawler.analysis.exception.CrawlerExcption; 4 | import com.yongqing.crawler.analysis.tax.bean.TaxPdfTable; 5 | import com.yongqing.crawler.analysis.tax.bean.TaxTable; 6 | 7 | /** 8 | * 9 | */ 10 | public interface TaxCrawlerAnalysis { 11 | TaxTable analysisTaxTable(TaxPdfTable taxPdfTable, String taxTableType) throws CrawlerExcption; 12 | } 13 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-crawler-analysis/src/main/java/com/yongqing/crawler/analysis/exception/CrawlerExcption.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.crawler.analysis.exception; 2 | 3 | /** 4 | * 5 | */ 6 | public class CrawlerExcption extends RuntimeException{ 7 | public CrawlerExcption(String s, Exception e) { 8 | super(s, e); 9 | } 10 | public CrawlerExcption(String s){ 11 | super(s); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-crawler-analysis/src/main/java/com/yongqing/crawler/analysis/tax/bean/TaxCrawlerAnalysisBean.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.crawler.analysis.tax.bean; 2 | 3 | import com.yongqing.crawler.analysis.TaxCrawlerAnalysis; 4 | 5 | /** 6 | * 7 | */ 8 | public class TaxCrawlerAnalysisBean { 9 | private TaxCrawlerAnalysis taxCrawlerAnalysisInstance; 10 | 11 | private String taxCrawlerAnalysis; 12 | 13 | private String type; 14 | 15 | private String provinceCity; 16 | 17 | public TaxCrawlerAnalysis getTaxCrawlerAnalysisInstance() { 18 | return taxCrawlerAnalysisInstance; 19 | } 20 | 21 | public void setTaxCrawlerAnalysisInstance(TaxCrawlerAnalysis taxCrawlerAnalysisInstance) { 22 | this.taxCrawlerAnalysisInstance = taxCrawlerAnalysisInstance; 23 | } 24 | 25 | public String getTaxCrawlerAnalysis() { 26 | return taxCrawlerAnalysis; 27 | } 28 | 29 | public void setTaxCrawlerAnalysis(String taxCrawlerAnalysis) { 30 | this.taxCrawlerAnalysis = taxCrawlerAnalysis; 31 | } 32 | 33 | public String getType() { 34 | return type; 35 | } 36 | 37 | public void setType(String type) { 38 | this.type = type; 39 | } 40 | 41 | public String getProvinceCity() { 42 | return provinceCity; 43 | } 44 | 45 | public void setProvinceCity(String provinceCity) { 46 | this.provinceCity = provinceCity; 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | return "TaxCrawlerAnalysisBean{" + 52 | "taxCrawlerAnalysisInstance=" + taxCrawlerAnalysisInstance + 53 | ", taxCrawlerAnalysis='" + taxCrawlerAnalysis + '\'' + 54 | ", type='" + type + '\'' + 55 | ", provinceCity='" + provinceCity + '\'' + 56 | '}'; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-crawler-analysis/src/test/java/com/yongqing/crawler/analysis/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.crawler.analysis; 2 | 3 | /** 4 | * 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-elasticsearch-tool/src/main/java/com/yongqing/elasticsearch/client/Client.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.elasticsearch.client; 2 | 3 | import org.elasticsearch.client.RestHighLevelClient; 4 | 5 | /** 6 | * 7 | */ 8 | public interface Client { 9 | RestHighLevelClient getClient(); 10 | void close(); 11 | } 12 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-elasticsearch-tool/src/main/java/com/yongqing/elasticsearch/client/DefaultSingletonElasticSearchClient.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.elasticsearch.client; 2 | 3 | /** 4 | * 5 | */ 6 | public class DefaultSingletonElasticSearchClient { 7 | 8 | private static volatile ElasticSearchClient elasticSearchClient = null; 9 | 10 | private DefaultSingletonElasticSearchClient() { 11 | super(); 12 | } 13 | 14 | public synchronized static ElasticSearchClient getInstance(String[] hostsAndPorts) { 15 | if (null == elasticSearchClient) { 16 | elasticSearchClient = new ElasticSearchClient(hostsAndPorts); 17 | } 18 | return elasticSearchClient; 19 | } 20 | public synchronized static void close(){ 21 | if (null != elasticSearchClient) { 22 | elasticSearchClient.close(); 23 | elasticSearchClient = null; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-elasticsearch-tool/src/main/java/com/yongqing/elasticsearch/client/DefaultSingletonElasticSearchPoolClient.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.elasticsearch.client; 2 | 3 | /** 4 | * 5 | */ 6 | public class DefaultSingletonElasticSearchPoolClient { 7 | 8 | private static volatile ElasticSearchPoolClient elasticSearchPoolClient = null; 9 | 10 | private DefaultSingletonElasticSearchPoolClient(){ 11 | super(); 12 | } 13 | 14 | public synchronized static ElasticSearchPoolClient getInstance(String[] hostsAndPorts, Integer poolSize) { 15 | if (null == elasticSearchPoolClient) { 16 | elasticSearchPoolClient = new ElasticSearchPoolClient(hostsAndPorts,poolSize); 17 | } 18 | return elasticSearchPoolClient; 19 | } 20 | 21 | public synchronized static void close(){ 22 | if (null != elasticSearchPoolClient) { 23 | elasticSearchPoolClient.close(); 24 | elasticSearchPoolClient = null; 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-elasticsearch-tool/src/main/java/com/yongqing/elasticsearch/client/ElasticSearchClient.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.elasticsearch.client; 2 | 3 | import org.apache.http.HttpHost; 4 | import org.elasticsearch.client.RestClient; 5 | import org.elasticsearch.client.RestClientBuilder; 6 | import org.elasticsearch.client.RestHighLevelClient; 7 | 8 | import java.io.IOException; 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | 12 | /** 13 | * es操作 14 | * https://www.cnblogs.com/reycg-blog/p/9931482.html 15 | */ 16 | public class ElasticSearchClient extends ElasticSearchAbstractClient { 17 | private String[] hostsAndPorts; 18 | private volatile RestHighLevelClient client; 19 | private volatile RestClientBuilder restClientBuilder; 20 | 21 | public ElasticSearchClient(String[] hostsAndPorts) { 22 | this.hostsAndPorts = hostsAndPorts; 23 | } 24 | 25 | public RestHighLevelClient getClient() { 26 | // RestHighLevelClient client = null; 27 | synchronized (this) { 28 | if (null == client) { 29 | List httpHosts = new ArrayList(); 30 | if (hostsAndPorts.length > 0) { 31 | for (String hostsAndPort : hostsAndPorts) { 32 | String[] hp = hostsAndPort.split(":"); 33 | httpHosts.add(new HttpHost(hp[0], Integer.valueOf(hp[1]), "http")); 34 | } 35 | if (null == restClientBuilder) { 36 | restClientBuilder = RestClient.builder(httpHosts.toArray(new HttpHost[0])); 37 | } 38 | 39 | } else { 40 | if (null == restClientBuilder) { 41 | restClientBuilder = RestClient.builder(new HttpHost("127.0.0.1", 9200, "http")); 42 | } 43 | } 44 | client = new RestHighLevelClient(restClientBuilder); 45 | } 46 | } 47 | return client; 48 | } 49 | 50 | @Override 51 | public synchronized void close() { 52 | if (null != client) { 53 | try { 54 | client.close(); 55 | } catch (IOException e) { 56 | e.printStackTrace(); 57 | } 58 | client = null; 59 | restClientBuilder = null; 60 | } 61 | } 62 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-elasticsearch-tool/src/main/java/com/yongqing/elasticsearch/client/bean/IndexBean.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.elasticsearch.client.bean; 2 | 3 | /** 4 | * 5 | */ 6 | public class IndexBean { 7 | //index name 8 | private String index; 9 | //index type 10 | private String indexType; 11 | //index doc id 12 | private String docId; 13 | // 1 IndexRequest 2 UpdateRequest 3 DeleteRequest 14 | private String operateType; 15 | 16 | public String getOperateType() { 17 | return operateType; 18 | } 19 | 20 | public void setOperateType(String operateType) { 21 | this.operateType = operateType; 22 | } 23 | 24 | public String getIndex() { 25 | return index; 26 | } 27 | 28 | public void setIndex(String index) { 29 | this.index = index; 30 | } 31 | 32 | public String getIndexType() { 33 | return indexType; 34 | } 35 | 36 | public void setIndexType(String indexType) { 37 | this.indexType = indexType; 38 | } 39 | 40 | public String getDocId() { 41 | return docId; 42 | } 43 | 44 | public void setDocId(String docId) { 45 | this.docId = docId; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-elasticsearch-tool/src/main/java/com/yongqing/elasticsearch/client/exception/ElasticsearchException.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.elasticsearch.client.exception; 2 | 3 | /** 4 | * 自定义的es异常类 5 | */ 6 | public class ElasticsearchException extends RuntimeException { 7 | public ElasticsearchException(String s, Exception e) { 8 | super(s, e); 9 | } 10 | public ElasticsearchException(String s){ 11 | super(s); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-elasticsearch-tool/src/main/resources/elasticsearch.properties: -------------------------------------------------------------------------------- 1 | elasticsearchHostAndPort=10.100.81.xx:9200 -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-elasticsearch-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | elasticsearch-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-elasticsearch-tool/src/test/java/com/yongqing/elasticsearch/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.elasticsearch; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool-v2/src/main/java/com/yongqing/etcd/action/Action.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.action; 2 | 3 | import java.util.Properties; 4 | 5 | /** 6 | *在etcd配置发生变化后，需要去触发的操作。 7 | */ 8 | public interface Action { 9 | /** 10 | * 需要执行的操作 11 | * @param oldProp 老的配置属性 12 | * @param newProp 新的配置属性 13 | */ 14 | void doAction(Properties oldProp, Properties newProp); 15 | } 16 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool-v2/src/main/java/com/yongqing/etcd/action/DefaultSingletonEtcdSubject.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.action; 2 | 3 | /** 4 | * 5 | */ 6 | public class DefaultSingletonEtcdSubject { 7 | 8 | private DefaultSingletonEtcdSubject(){} 9 | 10 | public static class SingletonEtcdSubject{ 11 | public static EtcdSubject etcdSubject = new EtcdSubject(); 12 | } 13 | public static EtcdSubject getDefaultSingletonEtcdSubject(){ 14 | return SingletonEtcdSubject.etcdSubject; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool-v2/src/main/java/com/yongqing/etcd/action/EtcdSubject.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.action; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Properties; 6 | 7 | /** 8 | * 9 | */ 10 | public class EtcdSubject { 11 | 12 | private List observerList = new ArrayList<>(); 13 | private Properties oldProp; 14 | private Properties newProp; 15 | 16 | public Properties getNewProperties() { 17 | return newProp; 18 | } 19 | 20 | public Properties getOldProperties() { 21 | return oldProp; 22 | } 23 | 24 | public void setProperties(Properties oldProp, Properties newProp) { 25 | this.oldProp = oldProp; 26 | this.newProp = newProp; 27 | notifyAllObservers(oldProp, newProp); 28 | } 29 | 30 | public void addObserver(Observer observer) { 31 | observerList.add(observer); 32 | } 33 | 34 | public void notifyAllObservers(Properties oldProp, Properties newProp) { 35 | for (Observer observer : observerList) { 36 | observer.doAction(oldProp, newProp); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool-v2/src/main/java/com/yongqing/etcd/action/Observer.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.action; 2 | 3 | /** 4 | * 5 | */ 6 | public abstract class Observer implements Action { 7 | protected EtcdSubject etcdSubject; 8 | 9 | public Observer(EtcdSubject etcdSubject) { 10 | this.etcdSubject = etcdSubject; 11 | this.etcdSubject.addObserver(this); 12 | } 13 | public Observer(){ 14 | this.etcdSubject = DefaultSingletonEtcdSubject.getDefaultSingletonEtcdSubject(); 15 | this.etcdSubject.addObserver(this); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool-v2/src/main/java/com/yongqing/etcd/exec/Exec.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.exec; 2 | 3 | /** 4 | * 分布式执行 5 | */ 6 | @FunctionalInterface 7 | public interface Exec { 8 | R exec(T parameters); 9 | } 10 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool-v2/src/main/java/com/yongqing/etcd/tools/core/CommonUtil.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.tools.core; 2 | 3 | import com.yongqing.common.bigdata.tool.BigDataCommonUtil; 4 | 5 | /** 6 | * 7 | */ 8 | public class CommonUtil extends BigDataCommonUtil { 9 | 10 | 11 | } 12 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool-v2/src/main/resources/etcd.properties: -------------------------------------------------------------------------------- 1 | endpoints=http://10.1.30.xxx:2379 -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool-v2/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | etcd-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool-v2/src/test/java/com/yongqing/etcd/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/README.md: -------------------------------------------------------------------------------- 1 | # yongqing-etcd-tool 2 | 3 | 大数据工具包-etcd的操作 4 |

**API**

5 | 6 | **工具类:EtcdUtil.class** 7 | 8 |

public static void initListen(String key);

9 |

etcd的监听，监听指定的key，当key 发生变化后，监听自动感知到变化。 key发生变化后，会自动更新本地缓存数据

10 |

参数：需要监听的key值

11 | 12 |

每个项目在etcd中只有一个唯一的key，建议key以系统+war包来命名

13 |

etcd中每个key的value中以key/value的形式存储值

14 |

示例(value中的值以换行分隔，每一行是一个key和value，以=号隔开，某一行如果以#开头，代表为注释，在代码中不起任何的作用，将不会读入本地缓存):

15 |

aa=123456

16 |

bb=654321

17 |

dd=23456

18 |

eee=23456

19 |

ggg=12345

20 |

hhhh=123456789

21 | 22 |

public static String getLocalPropertie(String localKey) ;

23 | 24 |

需要从本地属性缓存中读取的key的值

25 |

参数：本地属性缓存中的key的值

26 | 27 | **高级选项用法** 28 | 29 |

当本地缓存中发生变化后，可以自己决定缓存变化后，需要自动完成事情

30 |

自己定义class实现Action这个接口中的方法即可。 31 |

/**

32 |

*在etcd配置发生变化后，需要去触发的操作。

33 |

*/

34 |

public interface Action {

35 | /**

36 | * 需要执行的操作

37 | * @param oldProp 老的配置属性

38 | * @param newProp 新的配置属性

39 | */

40 |

void doAction(Properties oldProp, Properties newProp);

41 |

}

42 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/src/main/java/com/yongqing/etcd/action/Action.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.action; 2 | 3 | import java.util.Properties; 4 | 5 | /** 6 | *在etcd配置发生变化后，需要去触发的操作。 7 | */ 8 | public interface Action { 9 | /** 10 | * 需要执行的操作 11 | * @param oldProp 老的配置属性 12 | * @param newProp 新的配置属性 13 | */ 14 | void doAction(Properties oldProp, Properties newProp); 15 | } 16 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/src/main/java/com/yongqing/etcd/action/DefaultSingletonEtcdSubject.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.action; 2 | 3 | /** 4 | * 5 | */ 6 | public class DefaultSingletonEtcdSubject { 7 | 8 | private DefaultSingletonEtcdSubject(){} 9 | 10 | public static class SingletonEtcdSubject{ 11 | public static EtcdSubject etcdSubject = new EtcdSubject(); 12 | } 13 | public static EtcdSubject getDefaultSingletonEtcdSubject(){ 14 | return SingletonEtcdSubject.etcdSubject; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/src/main/java/com/yongqing/etcd/action/EtcdSubject.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.action; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Properties; 6 | 7 | /** 8 | * 9 | */ 10 | public class EtcdSubject { 11 | 12 | private List observerList = new ArrayList<>(); 13 | private Properties oldProp; 14 | private Properties newProp; 15 | 16 | public Properties getNewProperties() { 17 | return newProp; 18 | } 19 | 20 | public Properties getOldProperties() { 21 | return oldProp; 22 | } 23 | 24 | public void setProperties(Properties oldProp, Properties newProp) { 25 | this.oldProp = oldProp; 26 | this.newProp = newProp; 27 | notifyAllObservers(oldProp, newProp); 28 | } 29 | 30 | public void addObserver(Observer observer) { 31 | observerList.add(observer); 32 | } 33 | 34 | public void notifyAllObservers(Properties oldProp, Properties newProp) { 35 | for (Observer observer : observerList) { 36 | observer.doAction(oldProp, newProp); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/src/main/java/com/yongqing/etcd/action/Observer.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.action; 2 | 3 | /** 4 | * 5 | */ 6 | public abstract class Observer implements Action { 7 | protected EtcdSubject etcdSubject; 8 | 9 | public Observer(EtcdSubject etcdSubject) { 10 | this.etcdSubject = etcdSubject; 11 | this.etcdSubject.addObserver(this); 12 | } 13 | public Observer(){ 14 | this.etcdSubject = DefaultSingletonEtcdSubject.getDefaultSingletonEtcdSubject(); 15 | this.etcdSubject.addObserver(this); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/src/main/java/com/yongqing/etcd/exec/Exec.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.exec; 2 | 3 | /** 4 | * 分布式执行 5 | */ 6 | @FunctionalInterface 7 | public interface Exec { 8 | R exec(T parameters); 9 | } 10 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/src/main/java/com/yongqing/etcd/tools/core/CommonUtil.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd.tools.core; 2 | 3 | import com.yongqing.common.bigdata.tool.BigDataCommonUtil; 4 | 5 | /** 6 | * 7 | */ 8 | public class CommonUtil extends BigDataCommonUtil { 9 | 10 | 11 | } 12 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/src/main/resources/etcd.properties: -------------------------------------------------------------------------------- 1 | endpoints=http://47.111.122.xxx:2379 -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | etcd-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-etcd-tool/src/test/java/com/yongqing/etcd/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.etcd; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-flume-canal-source/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | canal-flume 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-flume-canal-source/src/test/java/com/yongqing/flume/canal/source/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.flume.canal.source; 2 | 3 | /** 4 | * 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hbase-tool/src/main/java/com/yongqing/hbase/exception/HbaseException.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hbase.exception; 2 | 3 | /** 4 | * 自定义的hbase异常类 5 | */ 6 | public class HbaseException extends RuntimeException { 7 | public HbaseException(String s, Exception e) { 8 | super(s, e); 9 | } 10 | public HbaseException(String s){ 11 | super(s); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hbase-tool/src/main/java/com/yongqing/hbase/utils/Client.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hbase.utils; 2 | 3 | import org.apache.hadoop.hbase.client.Put; 4 | 5 | import java.io.IOException; 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | /** 10 | * hbase操作 11 | */ 12 | public interface Client { 13 | /** 14 | * 链接初始化接口 对hbase的Connection进行初始化 15 | */ 16 | void initHbaseConnection(); 17 | List> getHbaseData(String queryType, String tableName, List queryStrList, Boolean isNeedReverse) throws IOException; 18 | List> getHbaseDataByScanAndStartRowAndEndRow(String queryType, String tableName, String startRow, String endRow) throws IOException; 19 | List> getHbaseDataByScan(String queryType, String tableName) throws IOException; 20 | void multiplePut(String insertType, List listPut, String tableName) throws IOException; 21 | void close(); 22 | 23 | } 24 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hbase-tool/src/main/resources/hbase.properties: -------------------------------------------------------------------------------- 1 | hadoopUser=root 2 | hadoopUserGroup=root 3 | zkClient=10.100.81.xx,10.100.81.xx,10.100.81.xx,10.100.81.xx,10.100.81.xx 4 | zkClientPort=2181 5 | zkZnodeparent=/hbase 6 | hbaseClientRetriesNumber=3 7 | hbaseClientPause=2000 8 | hbaseRpcTimeOut=20000 9 | hbaseClientScannerTimeOut=60000 10 | hbaseClientOperationTimeout=20000 11 | ipcScoketTimeout=4000 12 | zkRecoveryRetryIntervalmill=200 13 | userHome=/hbase 14 | userDir=/hbase -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hbase-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | hbase-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hbase-tool/src/test/java/com/yongqing/hbase/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hbase; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hdfs-tool/src/main/java/com/yongqing/hdfs/tool/Clinet.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hdfs.tool; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | import org.apache.hadoop.fs.FileSystem; 5 | 6 | import java.io.IOException; 7 | 8 | /** 9 | * 10 | */ 11 | public interface Clinet { 12 | 13 | FileSystem getFileSystem(); 14 | Configuration getConfiguration(); 15 | boolean createHdfspath(String path); 16 | void close() throws IOException; 17 | } 18 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hdfs-tool/src/main/java/com/yongqing/hdfs/tool/constant/Constants.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hdfs.tool.constant; 2 | 3 | /** 4 | * 5 | */ 6 | public interface Constants { 7 | 8 | 9 | 10 | /** 11 | * COLON : 12 | */ 13 | String COLON = ":"; 14 | 15 | /** 16 | * SINGLE_SLASH / 17 | */ 18 | String SINGLE_SLASH = "/"; 19 | 20 | /** 21 | * DOUBLE_SLASH // 22 | */ 23 | String DOUBLE_SLASH = "//"; 24 | 25 | /** 26 | * SEMICOLON ; 27 | */ 28 | String SEMICOLON = ";"; 29 | 30 | /** 31 | * comma , 32 | */ 33 | String COMMA = ","; 34 | 35 | /** 36 | * hadoop configuration 37 | */ 38 | String HADOOP_RM_STATE_ACTIVE = "ACTIVE"; 39 | 40 | String HADOOP_RM_STATE_STANDBY = "STANDBY"; 41 | 42 | String HADOOP_RESOURCE_MANAGER_HTTPADDRESS_PORT = "resource.manager.httpaddress.port"; 43 | 44 | /** 45 | * fs.defaultFS 46 | */ 47 | String FS_DEFAULTFS = "fs.defaultFS"; 48 | 49 | /** 50 | * yarn.resourcemanager.ha.rm.idsfs.defaultFS 51 | */ 52 | String YARN_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm.ids"; 53 | 54 | /** 55 | * yarn.application.status.address 56 | */ 57 | String YARN_APPLICATION_STATUS_ADDRESS = "yarn.application.status.address"; 58 | 59 | /** 60 | * hdfs configuration 61 | * hdfs.root.user 62 | */ 63 | String HDFS_ROOT_USER = "hdfs.root.user"; 64 | /** 65 | * loginUserFromKeytab path 66 | */ 67 | String LOGIN_USER_KEY_TAB_PATH = "login.user.keytab.path"; 68 | /** 69 | * java.security.krb5.conf 70 | */ 71 | String JAVA_SECURITY_KRB5_CONF = "java.security.krb5.conf"; 72 | 73 | /** 74 | * java.security.krb5.conf.path 75 | */ 76 | String JAVA_SECURITY_KRB5_CONF_PATH = "java.security.krb5.conf.path"; 77 | 78 | /** 79 | * hadoop.security.authentication 80 | */ 81 | String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication"; 82 | 83 | /** 84 | * hadoop.security.authentication 85 | */ 86 | String HADOOP_SECURITY_AUTHENTICATION_STARTUP_STATE = "hadoop.security.authentication.startup.state"; 87 | 88 | 89 | /** 90 | * loginUserFromKeytab user 91 | */ 92 | String LOGIN_USER_KEY_TAB_USERNAME = "login.user.keytab.username"; 93 | 94 | String HADOOP_HOME="hadoop.home.dir"; 95 | 96 | } 97 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hdfs-tool/src/main/resources/hdfs.properties: -------------------------------------------------------------------------------- 1 | java.security.krb5.conf=/opt/krb5.conf 2 | java.security.krb5.conf.path=/opt/hdfs.headless.keytab 3 | hadoop.security.authentication=kerberos 4 | hadoop.security.authentication.startup.state=false 5 | login.user.keytab.username=hdfs-mycluster@ESZ.COM 6 | login.user.keytab.path=/opt 7 | yarn.application.status.address=http://10.100.81.xx:8088/ws/v1/cluster/apps/%s 8 | yarn.resourcemanager.ha.rm.ids=10.100.81.xxx,10.100.81.xx 9 | fs.defaultFS=hdfs://ns 10 | resource.manager.httpaddress.port=8088 11 | hdfs.root.user=root 12 | hadoop.home.dir=/ -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hdfs-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | hdfs-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hdfs-tool/src/test/java/com/yongqing/hdfs/tool/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hdfs.tool; 2 | 3 | /** 4 | */ 5 | public class Test { 6 | } 7 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/DefaultSingletonHiveWrite.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool; 2 | 3 | import com.yongqing.hive.tool.pojo.ParameterPojo; 4 | import com.yongqing.hive.tool.pojo.FieldDealPojo; 5 | 6 | /** 7 | * 8 | */ 9 | public class DefaultSingletonHiveWrite { 10 | private static volatile HiveWrite hiveWrite = null; 11 | 12 | private DefaultSingletonHiveWrite() { 13 | super(); 14 | } 15 | 16 | public synchronized static HiveWrite getDefaultSingletonHiveJsonWrite(String metaStoreUri, String proxyUser, String database, String table, String tzName) { 17 | if (null == hiveWrite) { 18 | hiveWrite = new HiveWrite(metaStoreUri, proxyUser, database, table, tzName,null,"JSON"); 19 | } 20 | return hiveWrite; 21 | } 22 | 23 | public synchronized static HiveWrite getDefaultSingletonHiveWrite(String metaStoreUri, String proxyUser, String database, String table, String tzName,FieldDealPojo fieldDealPojo,String dataType) { 24 | if (null == hiveWrite) { 25 | hiveWrite = new HiveWrite(metaStoreUri, proxyUser, database, table, tzName,fieldDealPojo,dataType); 26 | } 27 | return hiveWrite; 28 | } 29 | 30 | public synchronized static HiveWrite getDefaultSingletonHiveWrite(String metaStoreUri, String proxyUser, String database, String table,FieldDealPojo fieldDealPojo,String dataType) { 31 | if (null == hiveWrite) { 32 | hiveWrite = new HiveWrite(metaStoreUri, proxyUser, database, table, null,fieldDealPojo,dataType); 33 | } 34 | return hiveWrite; 35 | } 36 | 37 | public synchronized static HiveWrite getDefaultSingletonHiveWrite(ParameterPojo parameterPojo, String dataType) { 38 | if (null == hiveWrite) { 39 | hiveWrite = new HiveWrite(parameterPojo.getMetaStoreUri(), parameterPojo.getProxyUser(), parameterPojo.getDatabase(), parameterPojo.getTable(), parameterPojo.getHivePartition(),parameterPojo.getHiveTxnsPerBatchAsk(),parameterPojo.getBatchSize(),parameterPojo.getIdleTimeout(),parameterPojo.getCallTimeout(),parameterPojo.getHeartBeatInterval(),parameterPojo.getMaxOpenConnections(),parameterPojo.getAutoCreatePartitions(),parameterPojo.getUseLocalTime(),parameterPojo.getTzName(),parameterPojo.getNeedRounding(),parameterPojo.getRoundUnit(),parameterPojo.getRoundValue(),parameterPojo.getFieldDealPojo(),dataType); 40 | } 41 | return hiveWrite; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/Event.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | */ 7 | public interface Event { 8 | 9 | 10 | 11 | public Map getHeaders(); 12 | 13 | 14 | public void setHeaders(Map headers); 15 | 16 | 17 | public byte[] getBody(); 18 | 19 | 20 | public void setBody(byte[] body); 21 | } 22 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/HiveDataSerializer.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool; 2 | 3 | import org.apache.hive.hcatalog.streaming.HiveEndPoint; 4 | import org.apache.hive.hcatalog.streaming.RecordWriter; 5 | import org.apache.hive.hcatalog.streaming.StreamingException; 6 | import org.apache.hive.hcatalog.streaming.TransactionBatch; 7 | 8 | import java.io.IOException; 9 | import java.util.Collection; 10 | 11 | /** 12 | * 13 | */ 14 | public interface HiveDataSerializer { 15 | 16 | void write(TransactionBatch txnBatch, byte[] e) 17 | throws StreamingException, IOException, InterruptedException; 18 | 19 | void write(TransactionBatch txnBatch, Collection events) 20 | throws StreamingException, IOException, InterruptedException; 21 | 22 | RecordWriter createRecordWriter(HiveEndPoint endPoint) 23 | throws StreamingException, IOException, ClassNotFoundException; 24 | } 25 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/HiveDataSerializerImpl.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool; 2 | 3 | import lombok.extern.log4j.Log4j2; 4 | import org.apache.hive.hcatalog.streaming.*; 5 | 6 | import java.io.IOException; 7 | import java.util.Collection; 8 | 9 | /** 10 | * 11 | */ 12 | @Log4j2 13 | public class HiveDataSerializerImpl implements HiveDataSerializer { 14 | private String[] fieldToColMapping = null; 15 | private Character serdeSeparator = null; 16 | private String delimiter; 17 | 18 | public HiveDataSerializerImpl() { 19 | super(); 20 | } 21 | 22 | public HiveDataSerializerImpl(String delimiter, Character serdeSeparator, String[] fieldToColMapping) { 23 | super(); 24 | this.delimiter = delimiter; 25 | this.fieldToColMapping = fieldToColMapping; 26 | this.serdeSeparator = serdeSeparator; 27 | } 28 | 29 | @Override 30 | public void write(TransactionBatch txnBatch, byte[] e) throws StreamingException, IOException, InterruptedException { 31 | log.info("start to write event...event:{}",new String(e)); 32 | txnBatch.write(e); 33 | } 34 | 35 | @Override 36 | public void write(TransactionBatch txnBatch, Collection events) throws StreamingException, IOException, InterruptedException { 37 | log.info("start to write event...size:{}",events.size()); 38 | txnBatch.write(events); 39 | } 40 | 41 | @Override 42 | public RecordWriter createRecordWriter(HiveEndPoint endPoint) throws StreamingException, IOException, ClassNotFoundException { 43 | if(null==delimiter || null==fieldToColMapping){ 44 | throw new NullPointerException("delimiter or fieldToColMapping can not be null"); 45 | } 46 | if (serdeSeparator == null) { 47 | return new DelimitedInputWriter(fieldToColMapping, delimiter, endPoint); 48 | } 49 | return new DelimitedInputWriter(fieldToColMapping, delimiter, endPoint, null, serdeSeparator); 50 | } 51 | 52 | public void setFieldDeal(String[] fieldToColMapping, Character serdeSeparator, String delimiter) { 53 | this.serdeSeparator = serdeSeparator; 54 | this.fieldToColMapping = fieldToColMapping; 55 | this.delimiter = delimiter; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/HiveException.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool; 2 | 3 | import org.apache.hive.hcatalog.streaming.HiveEndPoint; 4 | 5 | /** 6 | * 自定义的hive异常类 7 | */ 8 | public class HiveException extends RuntimeException { 9 | public HiveException(String s, Exception e) { 10 | super(s, e); 11 | } 12 | public HiveException(String s){ 13 | super(s); 14 | } 15 | 16 | public HiveException(HiveEndPoint endPoint, Long currentTxnId, Throwable e) { 17 | super("Failed writing to : " + endPoint + ". TxnID : " + currentTxnId, e); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/HiveJsonDataSerializerImpl.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool; 2 | 3 | import lombok.extern.log4j.Log4j2; 4 | import org.apache.hive.hcatalog.streaming.*; 5 | 6 | import java.io.IOException; 7 | import java.util.Collection; 8 | 9 | /** 10 | * 11 | */ 12 | @Log4j2 13 | public class HiveJsonDataSerializerImpl implements HiveDataSerializer{ 14 | 15 | 16 | @Override 17 | public void write(TransactionBatch txnBatch, byte[] event) throws StreamingException, IOException, InterruptedException { 18 | log.info("start to write event...size:{},event:{}",event.length,new String(event)); 19 | txnBatch.write(event); 20 | } 21 | 22 | @Override 23 | public void write(TransactionBatch txnBatch, Collection events) 24 | throws StreamingException, IOException, InterruptedException { 25 | log.info("start to write event...size:{}",events.size()); 26 | txnBatch.write(events); 27 | } 28 | 29 | @Override 30 | public RecordWriter createRecordWriter(HiveEndPoint endPoint) 31 | throws StreamingException, IOException, ClassNotFoundException { 32 | return new StrictJsonWriter(endPoint); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/client/Client.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool.client; 2 | 3 | import org.apache.hadoop.hive.metastore.api.MetaException; 4 | 5 | /** 6 | * 7 | */ 8 | public interface Client { 9 | void close(); 10 | void reconnect() throws MetaException; 11 | } 12 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/constant/Constants.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool.constant; 2 | 3 | /** 4 | * 5 | */ 6 | public interface Constants { 7 | int DEFAULT_MAXOPENCONNECTIONS = 500; 8 | int DEFAULT_TXNSPERBATCH = 100; 9 | int DEFAULT_BATCHSIZE = 15; 10 | int DEFAULT_CALLTIMEOUT = 10000; 11 | int DEFAULT_IDLETIMEOUT = 0; 12 | int DEFAULT_HEARTBEATINTERVAL = 240; // seconds 13 | String HIVEMETASTOREURIS = "hive.metastore.uris"; 14 | } 15 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/hcatalog/HCataLogClient.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool.hcatalog; 2 | 3 | import com.yongqing.hdfs.tool.HdfsClient; 4 | import com.yongqing.hive.tool.HiveException; 5 | import org.apache.hadoop.mapreduce.Job; 6 | import org.apache.hive.hcatalog.common.HCatException; 7 | import org.apache.hive.hcatalog.data.schema.HCatSchema; 8 | import org.apache.hive.hcatalog.data.transfer.*; 9 | import org.apache.hive.hcatalog.mapreduce.HCatBaseInputFormat; 10 | import org.apache.hive.hcatalog.mapreduce.HCatBaseOutputFormat; 11 | 12 | import java.io.IOException; 13 | import java.util.Map; 14 | 15 | /** 16 | * 17 | */ 18 | public class HCataLogClient { 19 | 20 | public HCatSchema getHCatSchema(HdfsClient hdfsClient, String type) throws IOException { 21 | if ("1".equals(type)) { 22 | return HCatBaseInputFormat.getTableSchema(hdfsClient.getConfiguration()); 23 | } else if ("2".equals(type)) { 24 | return HCatBaseOutputFormat.getTableSchema(hdfsClient.getConfiguration()); 25 | } else { 26 | throw new HiveException("type is not support,only support type is 1(InputFormar) or 2(OutputFormat)"); 27 | } 28 | } 29 | 30 | public Job getJob(HdfsClient hdfsClient, String jobName) throws IOException { 31 | Job job = Job.getInstance(hdfsClient.getConfiguration()); 32 | job.setJobName(jobName); 33 | return job; 34 | } 35 | 36 | public HCatReader getHCatReader(ReadEntity entity, Map conf) { 37 | return DataTransferFactory.getHCatReader(entity, conf); 38 | } 39 | 40 | public ReaderContext getReaderContext(ReadEntity entity, Map conf) throws HCatException { 41 | return getHCatReader(entity, conf).prepareRead(); 42 | } 43 | 44 | public HCatWriter getHCatWriter(WriteEntity entity, Map conf) { 45 | return DataTransferFactory.getHCatWriter(entity, conf); 46 | } 47 | 48 | public WriterContext getWriterContext(WriteEntity entity, Map conf) throws HCatException { 49 | return getHCatWriter(entity, conf).prepareWrite(); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/pojo/DefaultEvent.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool.pojo; 2 | 3 | import com.yongqing.hive.tool.Event; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * 9 | */ 10 | public class DefaultEvent implements Event { 11 | private Map headers; 12 | private byte[] body; 13 | 14 | @Override 15 | public Map getHeaders() { 16 | return headers; 17 | } 18 | 19 | @Override 20 | public void setHeaders(Map headers) { 21 | this.headers = headers; 22 | } 23 | 24 | @Override 25 | public byte[] getBody() { 26 | return body; 27 | } 28 | 29 | @Override 30 | public void setBody(byte[] body) { 31 | this.body = body; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/pojo/FieldDealPojo.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool.pojo; 2 | 3 | /** 4 | * 5 | */ 6 | public class FieldDealPojo { 7 | private String serdeSeparator; 8 | private String delimiter; 9 | private String fieldNames; 10 | 11 | public String getSerdeSeparator() { 12 | return serdeSeparator; 13 | } 14 | 15 | public void setSerdeSeparator(String serdeSeparator) { 16 | this.serdeSeparator = serdeSeparator; 17 | } 18 | 19 | public String getDelimiter() { 20 | return delimiter; 21 | } 22 | 23 | public void setDelimiter(String delimiter) { 24 | this.delimiter = delimiter; 25 | } 26 | 27 | public String getFieldNames() { 28 | return fieldNames; 29 | } 30 | 31 | public void setFieldNames(String fieldNames) { 32 | this.fieldNames = fieldNames; 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return "FieldDealPojo{" + 38 | "serdeSeparator=" + serdeSeparator + 39 | ", delimiter='" + delimiter + '\'' + 40 | ", fieldNames='" + fieldNames + '\'' + 41 | '}'; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/java/com/yongqing/hive/tool/queue/EventQueue.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool.queue; 2 | 3 | import com.yongqing.hive.tool.Event; 4 | 5 | 6 | import java.util.concurrent.BlockingQueue; 7 | import java.util.concurrent.LinkedBlockingQueue; 8 | 9 | 10 | /** 11 | * 12 | */ 13 | public class EventQueue { 14 | 15 | //阻塞队列 16 | private final static BlockingQueue eventArrayBlockingQueue = new LinkedBlockingQueue(); 17 | 18 | 19 | 20 | public static BlockingQueue getEventQueue() { 21 | return eventArrayBlockingQueue; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/resources/hive.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/597365581/bigdata_tools/c82f36db5dc4a557bdcf611736a8d3dc34a4ff0b/yongqing-bigdata-tools/yongqing-hive-tool/src/main/resources/hive.properties -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | hive-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-hive-tool/src/test/java/com/yongqing/hive/tool/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.hive.tool; 2 | 3 | /** 4 | */ 5 | public class Test { 6 | } 7 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-jdbc-common/src/main/java/com/yongqing/jdbc/common/JdbcConnectionPool.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.jdbc.common; 2 | 3 | import java.sql.Connection; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.Random; 8 | 9 | public class JdbcConnectionPool { 10 | 11 | private final static List jdbcConnectionList = new ArrayList<>(); 12 | 13 | public static synchronized void initPool(int size, String driverName, String jdbcUrl) { 14 | if (size == 0) { 15 | throw new RuntimeException("size must >0"); 16 | } 17 | if (jdbcConnectionList.size() == 0) { 18 | for (int i = 0; i <= size; i++) { 19 | jdbcConnectionList.add(new JdbcConnection(driverName, jdbcUrl)); 20 | } 21 | 22 | } 23 | } 24 | 25 | public synchronized static void initPool(int size, String driverName, String jdbcUrl, String user, String password) { 26 | if (size == 0) { 27 | throw new RuntimeException("size must >0"); 28 | } 29 | if (jdbcConnectionList.size() == 0) { 30 | for (int i = 0; i <= size; i++) { 31 | jdbcConnectionList.add(new JdbcConnection(user, password, driverName, jdbcUrl)); 32 | } 33 | 34 | } 35 | 36 | } 37 | 38 | public static Connection getConnection() throws Exception { 39 | if (jdbcConnectionList.size() == 0) { 40 | throw new RuntimeException("please init JdbcConnectionPool"); 41 | } else { 42 | return jdbcConnectionList.get(new Random().nextInt(jdbcConnectionList.size())).getConnection(); 43 | } 44 | } 45 | 46 | public static JdbcConnection getJdbcConnection() throws Exception { 47 | if (jdbcConnectionList.size() == 0) { 48 | throw new RuntimeException("please init JdbcConnectionPool"); 49 | } else { 50 | return jdbcConnectionList.get(new Random().nextInt(jdbcConnectionList.size())); 51 | } 52 | } 53 | 54 | public static List> executeQuery(String sql) { 55 | if (jdbcConnectionList.size() == 0) { 56 | throw new RuntimeException("please init JdbcConnectionPool"); 57 | } else { 58 | return jdbcConnectionList.get(new Random().nextInt(jdbcConnectionList.size())).executeQuery(sql); 59 | } 60 | 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-kafka-tool/src/main/java/com/yongqing/kafka/producer/KafkaPartitioner.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.kafka.producer; 2 | 3 | 4 | import kafka.utils.VerifiableProperties; 5 | import org.apache.kafka.clients.producer.Partitioner; 6 | import org.apache.kafka.common.Cluster; 7 | import org.apache.kafka.common.PartitionInfo; 8 | import org.apache.kafka.common.utils.Utils; 9 | 10 | import java.util.List; 11 | import java.util.Map; 12 | import java.util.Random; 13 | import java.util.concurrent.ConcurrentHashMap; 14 | import java.util.concurrent.ConcurrentMap; 15 | import java.util.concurrent.ThreadLocalRandom; 16 | import java.util.concurrent.atomic.AtomicInteger; 17 | 18 | /** Kafka发送消息分区策略 19 | * 20 | */ 21 | public class KafkaPartitioner implements Partitioner { 22 | private final ConcurrentMap topicCounterMap = new ConcurrentHashMap<>(); 23 | 24 | 25 | @Override 26 | public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) { 27 | List partitions = cluster.partitionsForTopic(topic); 28 | int numPartitions = partitions.size(); 29 | if (keyBytes == null) { 30 | int nextValue = nextValue(topic); 31 | List availablePartitions = cluster.availablePartitionsForTopic(topic); 32 | if (availablePartitions.size() > 0) { 33 | int part = Utils.toPositive(nextValue) % availablePartitions.size(); 34 | return availablePartitions.get(part).partition(); 35 | } else { 36 | // no partitions are available, give a non-available partition 37 | return Utils.toPositive(nextValue) % numPartitions; 38 | } 39 | } else { 40 | // hash the keyBytes to choose a partition 41 | return Utils.toPositive(Utils.murmur2(keyBytes)) % numPartitions; 42 | } 43 | } 44 | private int nextValue(String topic) { 45 | AtomicInteger counter = topicCounterMap.get(topic); 46 | if (null == counter) { 47 | counter = new AtomicInteger(ThreadLocalRandom.current().nextInt()); 48 | AtomicInteger currentCounter = topicCounterMap.putIfAbsent(topic, counter); 49 | if (currentCounter != null) { 50 | counter = currentCounter; 51 | } 52 | } 53 | return counter.getAndIncrement(); 54 | } 55 | @Override 56 | public void close() { 57 | 58 | } 59 | 60 | @Override 61 | public void configure(Map map) { 62 | 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-kafka-tool/src/main/java/com/yongqing/kafka/producer/KafkaProducer.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.kafka.producer; 2 | 3 | import org.apache.kafka.clients.producer.Callback; 4 | 5 | /** 6 | * 7 | */ 8 | public interface KafkaProducer { 9 | 10 | void sendMessage(String msg,Callback callback); 11 | void sendMessage(String msg,String topicName,Integer kafkaPartitionNumber); 12 | void sendMessage(String msg,String topicName,Integer kafkaPartitionNumber,Callback callback); 13 | void sendMessageWithCatch(String msg); 14 | void sendMessageWithCatch(String msg,Callback callback); 15 | void sendMessageWithCatch(String msg,String topicName,Integer kafkaPartitionNumber); 16 | void sendMessageWithCatch(String msg,String topicName,Integer kafkaPartitionNumber,Callback callback); 17 | void close(); 18 | void flush(); 19 | void close(long timeout, java.util.concurrent.TimeUnit timeUnit); 20 | void sendMessageWithTransactional(String msg); 21 | void sendMessageWithTransactional(String msg,String topicName,Integer kafkaPartitionNumber); 22 | } 23 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-kafka-tool/src/main/resources/kafka.properties: -------------------------------------------------------------------------------- 1 | bootstrap.servers=10.100.81.xx:9092 2 | batch.size=1 3 | linger.ms=5 4 | kafkaPartitionNumber=1 5 | topicName=testzyq 6 | group.id=test -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-kafka-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | kafka-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-kafka-tool/src/test/java/com/yongqing/kafka/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.kafka; 2 | 3 | /** 4 | * 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-output/README.md: -------------------------------------------------------------------------------- 1 | # yongqing-log-output 2 | 3 | 大数据日志采集--日志统一输出 4 | jiagou1.png 5 | 6 | ****1、 目的***** 7 | 8 | 指定统一的业务日志输出规范，便于未来业务日志通过统一采集的方式进入大数据平台，做业务分析和统计。 9 | 10 | ****2、 日志采集类型**** 11 |

系统请求日志:请求调用分析，一般用于对外提供的标准的API接口服务的系统请求日志的采集，一般适合入hive和Hbase

12 |

系统日志:用于ELK分析，用于分析系统异常日志情况

13 |

业务日志:业务数据分析，业务数据的采集

14 |

Nginx日志:接口调用分析，用于分析每个http接口或者页面的调用量，调用的成功率等，这种直接从nginx端采集，可以入Hive和ES

15 | 16 | **2.1、 系统请求日志** 17 | 18 | 固定规范：日志输出时，以json的形式输出. 19 | 20 | **通用字段** 21 | 22 |

请求发起时间 requestInitiationTime 非必填，一般是用户传入，格式：yyyy-MM-dd HH:mm:ss.SSS

23 |

请求接收时间 requestReceiveTime 必填，系统实际接收到请求的时间，格式：yyyy-MM-dd HH:mm:ss.SSS

24 |

请求日志生成时间 requestLogTime 必填，统一获取(日志打印的时间)，格式：yyyy-MM-dd HH:mm:ss.SSS

25 |

请求的IP地址 requestInitiationIp 必填，请求发起的IP地址，统一获取

26 |

请求处理的IP requestHostIp 必填，请求实际被处理的系统的机器IP，统一获取

27 |

请求的服务 requestApi 必填，请求的系统服务，统一获取

28 |

请求的url requestUrl 必填，请求的Url地址，统一获取

29 |

请求的参数 requestParameter 必填，请求的参数，统一获取

30 |

请求处理完成时间 requestCompletionTime 必填，请求处理完成的时间，统一获取，格式：yyyy-MM-dd HH:mm:ss.SSS

31 |

请求的处理系统 requestHostSystem 请求实际被处理的系统(系统的唯一英文简称)

32 |

请求响应的Code requestCompletionCode 必填

33 |

请求结果 requestCompletionResult 必填

34 |

请求id requestId 必填，系统统一生成

35 |

请求的应用ID appId 非必填，主要针对应用接入的情况，比如需要先注册接入，接入后，会分配一个唯一的应用ID，请求时需要传入

36 |

请求的签名 sign 非必填，和appId结合使用，用于接口请求的校验

37 |

自定义字段 customizeField 这是一个子Map，描述的是自定义的字段内容,一般输出的格式：{ “key1”:”value1”,”key2”:”value2” }

38 |

采集标志 collectionSign 必填，采集标志，用于采集判断，不带此标志的数据不采集。固定为systemRequestCollection

39 | 40 | **2.2、 业务日志** 41 | 42 | 固定规范：日志输出时，以json的形式输出. 43 | 44 | **通用字段** 45 | 46 |

系统名称 businessSystem 必填，业务系统(系统的唯一英文简称)

47 |

业务id businessId 必填，业务ID（每个业务不可重复，建议未来统一生成和分配，在接入统一采集）

48 |

业务类型 businessType 必填，业务类型（统一定义）

49 |

业务名称 businessName 必填，业务名称

50 |

业务日志打印时间 businessLogTime 必填，系统统一生成获取(业务日志打印时间)，格式：yyyy-MM-dd HH:mm:ss.SSS

51 |

日志ID businessLogId 必填，系统统一生成获取

52 |

自定义字段 customizeField 这是一个子Map，描述的是自定义的字段内容,一般输出的格式：{ "key1":"value1","key2":"value2" }

53 |

采集标志 collectionSign 必填，采集标志，用于采集判断，不带此标志的数据不采集。固定为 businessCollection

54 | 55 | **2.3、 Nginx日志** 56 | 57 | Nginx日志，所有系统使用统一的nginx.conf日志配置，然后输出的nginx 请求日志字段完全一致。 58 | 59 | **2.4、 系统日志** 60 | 61 | 系统日志只对日志文件名称统一规范，然后生成的日志统一采集入到ELK中，包括Info日志，error和异常等日志。 62 | 63 | 名称规范:系统简称.log，在此格式上按照yyyy-MM-dd统一自动归档。 64 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-output/docs/jiagou1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/597365581/bigdata_tools/c82f36db5dc4a557bdcf611736a8d3dc34a4ff0b/yongqing-bigdata-tools/yongqing-log-output/docs/jiagou1.png -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-output/src/main/java/com/yongqing/log/output/factory/adapter/NullStringToEmptyAdapterFactory.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.output.factory.adapter; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.TypeAdapter; 5 | import com.google.gson.TypeAdapterFactory; 6 | import com.google.gson.reflect.TypeToken; 7 | import com.google.gson.stream.JsonReader; 8 | import com.google.gson.stream.JsonToken; 9 | import com.google.gson.stream.JsonWriter; 10 | 11 | import java.io.IOException; 12 | 13 | 14 | @SuppressWarnings("unchecked") 15 | public class NullStringToEmptyAdapterFactory implements TypeAdapterFactory { 16 | public TypeAdapter create(Gson gson, TypeToken type) { 17 | Class rawType = (Class) type.getRawType(); 18 | if (rawType != String.class) { 19 | return null; 20 | } 21 | return (TypeAdapter) new StringNullAdapter(); 22 | } 23 | } 24 | 25 | // StringNullAdapter.java 26 | class StringNullAdapter extends TypeAdapter { 27 | 28 | @Override 29 | public String read(JsonReader reader) throws IOException { 30 | if (reader.peek() == JsonToken.NULL) { 31 | reader.nextNull(); 32 | return ""; 33 | } 34 | return reader.nextString(); 35 | } 36 | 37 | @Override 38 | public void write(JsonWriter writer, String value) throws IOException { 39 | if (value == null) { 40 | writer.value(""); 41 | return; 42 | } 43 | writer.value(value); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-output/src/main/java/com/yongqing/log/output/utils/DateUtil.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.output.utils; 2 | import com.yongqing.common.bigdata.tool.DateCommonUtil; 3 | 4 | /** 5 | * 日期时间工具类 6 | */ 7 | public class DateUtil extends DateCommonUtil { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-output/src/main/java/com/yongqing/log/output/utils/UUIDGenerator.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.output.utils; 2 | 3 | import java.text.SimpleDateFormat; 4 | import java.util.UUID; 5 | 6 | /** 7 | * 生成唯一ID 8 | */ 9 | public class UUIDGenerator { 10 | /** 11 | * 12 | * 功能描述:
13 | * 〈UUID获取〉 14 | * 15 | * @return 16 | * @see [相关类/方法](可选) 17 | * @since [产品/模块版本](可选) 18 | */ 19 | private static final ThreadLocal threadRequestID = new ThreadLocal(); 20 | private static final ThreadLocal threadNumID = new ThreadLocal(); 21 | private static int guid=100; 22 | public static String getRequestID() { 23 | String requestID = (String) threadRequestID.get(); 24 | if (requestID == null) { 25 | requestID = UUIDGenerator.getUUID(); 26 | threadRequestID.set(requestID); 27 | } 28 | return requestID; 29 | } 30 | public static String getNumID() { 31 | String numID = (String) threadNumID.get(); 32 | if (numID == null) { 33 | numID = UUIDGenerator.getGuid(); 34 | threadNumID.set(numID); 35 | } 36 | return numID; 37 | } 38 | public static void destroyRequestID() { 39 | threadRequestID.set(null); 40 | } 41 | public static void destroyNumID() { 42 | threadNumID.set(null); 43 | } 44 | public static String getUUID() { 45 | // 取两组UUID拼接字符串 46 | String s = UUID.randomUUID().toString() + UUID.randomUUID().toString(); 47 | // 去除UUID中的'-',并截取成48位 48 | return s.substring(0, 8) + s.substring(9, 13) + s.substring(14, 18) + s.substring(19, 23)+s.substring(24,36) ; 49 | 50 | } 51 | 52 | public static String getGuid(){ 53 | guid+=1; 54 | long now = System.currentTimeMillis(); 55 | //获取4位年份数字 56 | SimpleDateFormat dateFormat=new SimpleDateFormat("yyyy"); 57 | //获取时间戳 58 | String time=dateFormat.format(now); 59 | String info=Long.toString(now)+""; 60 | int ran=0; 61 | if(guid>999){ 62 | guid=100; 63 | } 64 | ran=guid; 65 | 66 | return time+info.substring(2, info.length())+ran; 67 | } 68 | 69 | 70 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-output/src/main/java/com/yongqing/log/output/utils/gson/GsonUtil.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.output.utils.gson; 2 | 3 | 4 | import com.google.gson.Gson; 5 | import com.google.gson.GsonBuilder; 6 | import com.google.gson.LongSerializationPolicy; 7 | import com.yongqing.log.output.factory.adapter.NullStringToEmptyAdapterFactory; 8 | 9 | /** 10 | * json相关处理的工具类 11 | */ 12 | 13 | public class GsonUtil { 14 | 15 | //定义gson对像，对象的String属性为null时也支持序列化 16 | public static final Gson gson = new GsonBuilder().setLongSerializationPolicy(LongSerializationPolicy.STRING).disableHtmlEscaping().registerTypeAdapterFactory(new NullStringToEmptyAdapterFactory()).setDateFormat("yyyy-MM-dd HH:mm:ss.SSS").create(); 17 | } 18 | 19 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-output/src/test/java/com/yongqing/log/output/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.log.output; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-processor/README.md: -------------------------------------------------------------------------------- 1 | # yongqing-log-processor 2 | 3 | **日志处理** 4 | 5 | **架构图：** 6 | 7 | jiagou1.png 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-processor/docs/jiagou1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/597365581/bigdata_tools/c82f36db5dc4a557bdcf611736a8d3dc34a4ff0b/yongqing-bigdata-tools/yongqing-log-processor/docs/jiagou1.png -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-processor/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | yongqing.bigdata.tools 8 | yongqing-bigdata-tools 9 | 1.0 10 | 11 | yongqing-log-processor 12 | jar 13 | 1.0-SNAPSHOT 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | org.apache.maven.plugins 22 | maven-compiler-plugin 23 | 24 | ${java.version} 25 | ${java.version} 26 | true 27 | ${project.build.sourceEncoding} 28 | 29 | 30 | 31 | org.apache.maven.plugins 32 | maven-jar-plugin 33 | 3.0.2 34 | 35 | 36 | log4j2.xml 37 | 38 | 39 | 40 | 41 | compile 42 | 43 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-processor/src/main/java/com/yongqing/processor/log/Processor.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.processor.log; 2 | 3 | /** 4 | * 5 | */ 6 | public interface Processor { 7 | T process(T log); 8 | } 9 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-processor/src/main/java/com/yongqing/processor/log/bean/BusinessLog.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.processor.log.bean; 2 | 3 | import java.util.Map; 4 | 5 | /** 6 | * 业务日志 7 | */ 8 | public class BusinessLog { 9 | //系统名称 必填，业务系统(系统的唯一英文简称) 10 | private String businessSystem; 11 | //业务id 必填，业务ID（每个业务不可重复，建议未来统一生成和分配，在接入统一采集） 12 | private String businessId; 13 | //业务类型 必填，业务类型（统一定义） 14 | private String businessType; 15 | //业务名称 必填，业务名称 16 | private String businessName; 17 | //业务日志打印时间 必填，系统统一生成获取(业务日志打印时间) 18 | private String businessLogTime; 19 | //日志ID 必填，系统统一生成获取 20 | private String businessLogId; 21 | //自定义字段这是一个子Map，描述的是自定义的字段内容,一般输出的格式：{ "key1":"value1","key2":"value2" } 22 | private Map customizeField; 23 | //采集标志 必填，采集标志，用于采集判断，不带此标志的数据不采集。固定为 businessCollection 24 | private String collectionSign = "businessCollection"; 25 | 26 | public String getBusinessSystem() { 27 | return businessSystem; 28 | } 29 | 30 | public void setBusinessSystem(String businessSystem) { 31 | this.businessSystem = businessSystem; 32 | } 33 | 34 | public String getBusinessId() { 35 | return businessId; 36 | } 37 | 38 | public void setBusinessId(String businessId) { 39 | this.businessId = businessId; 40 | } 41 | 42 | public String getBusinessType() { 43 | return businessType; 44 | } 45 | 46 | public void setBusinessType(String businessType) { 47 | this.businessType = businessType; 48 | } 49 | 50 | public String getBusinessName() { 51 | return businessName; 52 | } 53 | 54 | public void setBusinessName(String businessName) { 55 | this.businessName = businessName; 56 | } 57 | 58 | public String getBusinessLogTime() { 59 | return businessLogTime; 60 | } 61 | 62 | public void setBusinessLogTime(String businessLogTime) { 63 | this.businessLogTime = businessLogTime; 64 | } 65 | 66 | public String getBusinessLogId() { 67 | return businessLogId; 68 | } 69 | 70 | public void setBusinessLogId(String businessLogId) { 71 | this.businessLogId = businessLogId; 72 | } 73 | 74 | public Map getCustomizeField() { 75 | return customizeField; 76 | } 77 | 78 | public void setCustomizeField(Map customizeField) { 79 | this.customizeField = customizeField; 80 | } 81 | 82 | public String getCollectionSign() { 83 | return collectionSign; 84 | } 85 | 86 | public void setCollectionSign(String collectionSign) { 87 | this.collectionSign = collectionSign; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-processor/src/main/java/com/yongqing/processor/log/bean/ProcessorBean.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.processor.log.bean; 2 | 3 | import com.yongqing.processor.log.Processor; 4 | 5 | /** 6 | * 7 | */ 8 | public class ProcessorBean { 9 | private String processor; 10 | private String logType; 11 | private Processor processorInstance; 12 | 13 | public Processor getProcessorInstance() { 14 | return processorInstance; 15 | } 16 | 17 | public void setProcessorInstance(Processor processorInstance) { 18 | this.processorInstance = processorInstance; 19 | } 20 | 21 | public String getProcessor() { 22 | return processor; 23 | } 24 | 25 | public void setProcessor(String processor) { 26 | this.processor = processor; 27 | } 28 | 29 | public String getLogType() { 30 | return logType; 31 | } 32 | 33 | public void setLogType(String logType) { 34 | this.logType = logType; 35 | } 36 | 37 | @Override 38 | public String toString() { 39 | return "ProcessorBean{" + 40 | "processor='" + processor + '\'' + 41 | ", logType='" + logType + '\'' + 42 | ", processorInstance=" + processorInstance + 43 | '}'; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-log-processor/src/test/java/com/yongqing/processor/log/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.processor.log; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/README.md: -------------------------------------------------------------------------------- 1 | # yongqing-mongodb-jdbc 2 | 3 | MongoDB的JDBC工程项目，支持标准的jdbc。工程代码来源于第三方MongoDB JDBC的jar包（unityjdbc.jar）反编译生成的源码项目。 4 | unityjdbc.jar 参考地址：http://www.unityjdbc.com/mongojdbc/mongo_jdbc.php -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/conn/MongoClientConnection.java: -------------------------------------------------------------------------------- 1 | package mongodb.conn; 2 | 3 | import com.mongodb.MongoClient; 4 | 5 | public class MongoClientConnection 6 | { 7 | public MongoClient client; 8 | public int count; 9 | 10 | public String toString() 11 | { 12 | return "Client: " + this.client.toString() + " SQL connections: " + this.count; 13 | } 14 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/conn/Subtable.java: -------------------------------------------------------------------------------- 1 | package mongodb.conn; 2 | 3 | import unity.annotation.SourceTable; 4 | import unity.engine.Attribute; 5 | 6 | public class Subtable 7 | { 8 | public String name; 9 | public String parentName; 10 | public SourceTable rootTable; 11 | public Subtable parentTable; 12 | public Attribute[] attr; 13 | public String listName; 14 | public int valueType; 15 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/jdbc/MongoDataSourceFactory.java: -------------------------------------------------------------------------------- 1 | package mongodb.jdbc; 2 | 3 | import java.util.Hashtable; 4 | import javax.naming.Context; 5 | import javax.naming.Name; 6 | import javax.naming.RefAddr; 7 | import javax.naming.Reference; 8 | import javax.naming.spi.ObjectFactory; 9 | 10 | public class MongoDataSourceFactory 11 | implements ObjectFactory 12 | { 13 | public Object getObjectInstance(Object obj, Name name, Context nameCtx, Hashtable environment) 14 | throws Exception 15 | { 16 | Reference ref = (Reference)obj; 17 | String className = ref.getClassName(); 18 | 19 | if ((className != null) && ((className.equals("mongodb.jdbc.MongoDataSource")) || (className.equals("mongodb.jdbc.MongoXADataSource")))) 20 | { 21 | MongoDataSource ds = null; 22 | try 23 | { 24 | ds = (MongoDataSource)Class.forName(className).newInstance(); 25 | } catch (Exception ex) { 26 | throw new RuntimeException("Unable to create DataSource of class '" + className + "', reason: " + ex.toString()); 27 | } 28 | 29 | ds.setUrl(getRefString("url", ref)); 30 | ds.setPassword(getRefString("password", ref)); 31 | ds.setUser(getRefString("user", ref)); 32 | 33 | return ds; 34 | } 35 | 36 | return null; 37 | } 38 | 39 | private String getRefString(String name, Reference ref) 40 | { 41 | RefAddr ra = ref.get(name); 42 | return ra != null ? (String)ra.getContent() : null; 43 | } 44 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/jdbc/MongoPooledConnection.java: -------------------------------------------------------------------------------- 1 | package mongodb.jdbc; 2 | 3 | import java.sql.Connection; 4 | import java.sql.SQLException; 5 | import javax.sql.ConnectionEventListener; 6 | import javax.sql.PooledConnection; 7 | import javax.sql.StatementEventListener; 8 | 9 | public class MongoPooledConnection 10 | implements PooledConnection 11 | { 12 | protected Connection conn; 13 | 14 | public MongoPooledConnection(Connection c) 15 | { 16 | this.conn = c; 17 | } 18 | 19 | public void close() 20 | throws SQLException 21 | { 22 | this.conn.close(); 23 | } 24 | 25 | public Connection getConnection() throws SQLException 26 | { 27 | return this.conn; 28 | } 29 | 30 | public void addConnectionEventListener(ConnectionEventListener listener) 31 | { 32 | } 33 | 34 | public void removeConnectionEventListener(ConnectionEventListener listener) 35 | { 36 | } 37 | 38 | public void addStatementEventListener(StatementEventListener listener) 39 | { 40 | } 41 | 42 | public void removeStatementEventListener(StatementEventListener listener) 43 | { 44 | } 45 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/jdbc/MongoResultSet.java: -------------------------------------------------------------------------------- 1 | package mongodb.jdbc; 2 | 3 | import java.io.PrintStream; 4 | import java.sql.SQLException; 5 | import java.util.ResourceBundle; 6 | import unity.engine.Relation; 7 | import unity.engine.TableData; 8 | import unity.engine.Tuple; 9 | import unity.generic.jdbc.ResultSetImpl; 10 | import unity.generic.jdbc.ResultSetMetaDataImpl; 11 | import unity.jdbc.UnityDriver; 12 | 13 | public class MongoResultSet extends ResultSetImpl 14 | { 15 | protected static ResourceBundle resources = ResourceBundle.getBundle("resources/mongo/MongoResultSet", locale); 16 | 17 | public MongoResultSet(TableData rows, Relation relation, int type, MongoStatement stmt) 18 | throws SQLException 19 | { 20 | this.relation = relation; 21 | this.meta = new MongoResultSetMetaData(relation); 22 | this.rows = rows; 23 | this.columnHeaders = this.meta.getColumnHeaders(); 24 | this.stmt = stmt; 25 | this.cursor = 0; 26 | this.currentRow = new Tuple(relation); 27 | this._resultSetType = type; 28 | } 29 | 30 | public boolean next() 31 | throws SQLException 32 | { 33 | if ((UnityDriver.isExpiredTrial()) && (this.lastIndex >= UnityDriver.getMaxResults())) 34 | { 35 | System.out.println(UnityDriver.i18n.getString("ResultSet.MaxRows") + UnityDriver.getMaxResults() + UnityDriver.i18n.getString("ResultSet.TrialVersion") + " " + UnityDriver.getTrialExpiryDate()); 36 | 37 | return false; 38 | } 39 | 40 | return super.next(); 41 | } 42 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/jdbc/MongoResultSetMetaData.java: -------------------------------------------------------------------------------- 1 | package mongodb.jdbc; 2 | 3 | import java.sql.SQLException; 4 | import java.util.ResourceBundle; 5 | import unity.engine.Relation; 6 | import unity.generic.jdbc.ResultSetMetaDataImpl; 7 | 8 | public class MongoResultSetMetaData extends ResultSetMetaDataImpl 9 | { 10 | protected static ResourceBundle resources = ResourceBundle.getBundle("resources/mongo/MongoResultSetMetaData", locale); 11 | 12 | public MongoResultSetMetaData(Relation relation) 13 | throws SQLException 14 | { 15 | super(relation); 16 | } 17 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/jdbc/MongoTableData.java: -------------------------------------------------------------------------------- 1 | package mongodb.jdbc; 2 | 3 | import java.sql.SQLException; 4 | import java.util.ArrayList; 5 | import unity.engine.IServerConnection; 6 | import unity.engine.TableData; 7 | import unity.engine.Tuple; 8 | 9 | public class MongoTableData extends TableData 10 | { 11 | public MongoTableData(IServerConnection connection, boolean buffering, int statementId) 12 | { 13 | super(connection, buffering, statementId); 14 | } 15 | 16 | public void setTuples(ArrayList data) 17 | throws SQLException 18 | { 19 | this.rows = new ArrayList(); 20 | 21 | for (int idx = 0; idx < data.size(); idx++) 22 | { 23 | this.rows.add(Tuple.convertToBytes((Object[])data.get(idx), this.relation)); 24 | } 25 | 26 | this.dataComplete = true; 27 | } 28 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/jdbc/MongoXAConnection.java: -------------------------------------------------------------------------------- 1 | package mongodb.jdbc; 2 | 3 | import java.sql.Connection; 4 | import java.sql.SQLException; 5 | import javax.sql.XAConnection; 6 | import javax.transaction.xa.XAResource; 7 | 8 | public class MongoXAConnection extends MongoPooledConnection 9 | implements XAConnection 10 | { 11 | private XAResource resource; 12 | 13 | public MongoXAConnection(Connection c) 14 | { 15 | super(c); 16 | this.resource = new MongoXAResource(this); 17 | } 18 | 19 | public XAResource getXAResource() 20 | throws SQLException 21 | { 22 | return this.resource; 23 | } 24 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/jdbc/MongoXADataSource.java: -------------------------------------------------------------------------------- 1 | package mongodb.jdbc; 2 | 3 | import java.sql.Connection; 4 | import java.sql.SQLException; 5 | import javax.sql.XAConnection; 6 | import javax.sql.XADataSource; 7 | 8 | public class MongoXADataSource extends MongoDataSource 9 | implements XADataSource 10 | { 11 | private static final long serialVersionUID = 1L; 12 | 13 | public XAConnection getXAConnection() 14 | throws SQLException 15 | { 16 | Connection c = super.getConnection(); 17 | MongoXAConnection xaconn = new MongoXAConnection(c); 18 | return xaconn; 19 | } 20 | 21 | public XAConnection getXAConnection(String user, String password) 22 | throws SQLException 23 | { 24 | Connection c = super.getConnection(user, password); 25 | MongoXAConnection xaconn = new MongoXAConnection(c); 26 | return xaconn; 27 | } 28 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/jdbc/MongoXAResource.java: -------------------------------------------------------------------------------- 1 | package mongodb.jdbc; 2 | 3 | import javax.transaction.xa.XAException; 4 | import javax.transaction.xa.XAResource; 5 | import javax.transaction.xa.Xid; 6 | 7 | public class MongoXAResource 8 | implements XAResource 9 | { 10 | private MongoXAConnection connection; 11 | public static final int DEFAULT_TIMEOUT = 100; 12 | private int transactionTimeout = 100; 13 | 14 | public MongoXAResource(MongoXAConnection conn) 15 | { 16 | this.connection = conn; 17 | } 18 | 19 | public void commit(Xid xid, boolean onePhase) 20 | throws XAException 21 | { 22 | } 23 | 24 | public void end(Xid xid, int flags) 25 | throws XAException 26 | { 27 | } 28 | 29 | public void forget(Xid xid) 30 | throws XAException 31 | { 32 | } 33 | 34 | public int getTransactionTimeout() 35 | throws XAException 36 | { 37 | return this.transactionTimeout; 38 | } 39 | 40 | public boolean isSameRM(XAResource xares) 41 | throws XAException 42 | { 43 | return false; 44 | } 45 | 46 | public int prepare(Xid xid) 47 | throws XAException 48 | { 49 | return 0; 50 | } 51 | 52 | public Xid[] recover(int flag) 53 | throws XAException 54 | { 55 | return null; 56 | } 57 | 58 | public void rollback(Xid xid) 59 | throws XAException 60 | { 61 | } 62 | 63 | public boolean setTransactionTimeout(int seconds) 64 | throws XAException 65 | { 66 | if (seconds == 0) 67 | { 68 | this.transactionTimeout = 100; 69 | } 70 | this.transactionTimeout = seconds; 71 | return true; 72 | } 73 | 74 | public void start(Xid xid, int flags) 75 | throws XAException 76 | { 77 | } 78 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoBuilderFatalException.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | public class MongoBuilderFatalException extends Exception 4 | { 5 | private static final long serialVersionUID = -3471272040234423421L; 6 | 7 | public MongoBuilderFatalException(Throwable cause) 8 | { 9 | super(cause); 10 | } 11 | 12 | public MongoBuilderFatalException(String string) 13 | { 14 | this(new Throwable(string)); 15 | } 16 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoBuilderUpstreamException.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | public class MongoBuilderUpstreamException extends Exception 4 | { 5 | private static final long serialVersionUID = -6123204868374038276L; 6 | 7 | public MongoBuilderUpstreamException(Throwable cause) 8 | { 9 | super(cause); 10 | } 11 | 12 | public MongoBuilderUpstreamException(String string) 13 | { 14 | this(new Throwable(string)); 15 | } 16 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoCreateIndex.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | import com.mongodb.DB; 4 | import com.mongodb.DBCollection; 5 | import com.mongodb.DBObject; 6 | 7 | public class MongoCreateIndex extends MongoQuery 8 | { 9 | private DBObject keys; 10 | private DBObject options; 11 | 12 | public String toMongoString() 13 | throws MongoBuilderFatalException 14 | { 15 | StringBuffer buf = new StringBuffer(); 16 | buf.append("db."); 17 | buf.append(this.collectionName); 18 | buf.append("."); 19 | buf.append("ensureIndex("); 20 | buf.append(this.keys); 21 | buf.append(", "); 22 | buf.append(this.options); 23 | buf.append(")"); 24 | return buf.toString(); 25 | } 26 | 27 | public Object run(DB db) 28 | throws Exception 29 | { 30 | db.getCollection(this.collectionName).createIndex(this.keys, this.options); 31 | return null; 32 | } 33 | 34 | public void setIndex(DBObject keys, DBObject options) 35 | { 36 | this.keys = keys; 37 | this.options = options; 38 | } 39 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoDeleteQuery.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | import com.mongodb.BasicDBObject; 4 | import com.mongodb.DB; 5 | import com.mongodb.DBCollection; 6 | 7 | public class MongoDeleteQuery extends MongoQuery 8 | { 9 | public BasicDBObject query; 10 | 11 | public MongoDeleteQuery() 12 | { 13 | this.query = new BasicDBObject(); 14 | } 15 | 16 | private String removeToString() 17 | { 18 | StringBuffer buf = new StringBuffer(); 19 | buf.append("remove("); 20 | if (this.query.size() > 0) 21 | buf.append(this.query.toString()); 22 | else { 23 | buf.append("{ }"); 24 | } 25 | buf.append(")"); 26 | return buf.toString(); 27 | } 28 | 29 | public String toMongoString() 30 | throws MongoBuilderFatalException 31 | { 32 | StringBuffer buf = new StringBuffer(); 33 | buf.append("db."); 34 | buf.append(this.collectionName); 35 | buf.append("."); 36 | buf.append(removeToString()); 37 | return buf.toString(); 38 | } 39 | 40 | public Object run(DB db) 41 | throws Exception 42 | { 43 | return db.getCollection(this.collectionName).remove(this.query); 44 | } 45 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoDropIndex.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | import com.mongodb.DB; 4 | import com.mongodb.DBCollection; 5 | 6 | public class MongoDropIndex extends MongoQuery 7 | { 8 | private String indexName; 9 | 10 | public MongoDropIndex(String indexName) 11 | { 12 | this.indexName = indexName; 13 | } 14 | 15 | public String toMongoString() 16 | throws MongoBuilderFatalException 17 | { 18 | StringBuffer buf = new StringBuffer(); 19 | buf.append("db."); 20 | buf.append(this.collectionName); 21 | buf.append("."); 22 | buf.append("dropIndex(\""); 23 | buf.append(this.indexName); 24 | buf.append("\")"); 25 | return buf.toString(); 26 | } 27 | 28 | public Object run(DB db) 29 | throws Exception 30 | { 31 | db.getCollection(this.collectionName).dropIndex(this.indexName); 32 | return null; 33 | } 34 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoDropTable.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | import com.mongodb.DB; 4 | import com.mongodb.DBCollection; 5 | 6 | public class MongoDropTable extends MongoQuery 7 | { 8 | public String toMongoString() 9 | throws MongoBuilderFatalException 10 | { 11 | StringBuffer buf = new StringBuffer(); 12 | buf.append("db."); 13 | buf.append(this.collectionName); 14 | buf.append("."); 15 | buf.append("drop()"); 16 | return buf.toString(); 17 | } 18 | 19 | public Object run(DB db) 20 | throws Exception 21 | { 22 | db.getCollection(this.collectionName).drop(); 23 | return null; 24 | } 25 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoInsertQuery.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | import com.mongodb.BasicDBObject; 4 | import com.mongodb.DB; 5 | import com.mongodb.DBCollection; 6 | import com.mongodb.DBObject; 7 | import com.mongodb.DuplicateKeyException; 8 | import com.mongodb.MongoException; 9 | import java.io.PrintStream; 10 | import java.sql.SQLException; 11 | import unity.jdbc.UnityDriver; 12 | 13 | public class MongoInsertQuery extends MongoQuery 14 | { 15 | public BasicDBObject insertFields; 16 | 17 | public MongoInsertQuery() 18 | { 19 | this.insertFields = new BasicDBObject(); 20 | } 21 | 22 | public String toMongoString() 23 | throws MongoBuilderFatalException 24 | { 25 | return "db." + this.collectionName + ".insert(" + this.insertFields.toString() + ")"; 26 | } 27 | 28 | public Object run(DB db) 29 | throws Exception 30 | { 31 | try 32 | { 33 | db.getCollection(this.collectionName).insert(new DBObject[] { this.insertFields }); 34 | return Integer.valueOf(1); 35 | } 36 | catch (DuplicateKeyException e) 37 | { 38 | if (UnityDriver.DEBUG) { 39 | System.out.println("Mongo genererated the following exception: " + e); 40 | } 41 | 42 | String err = e.getMessage().substring(e.getMessage().lastIndexOf("duplicate key error index: ") + "duplicate key error index: ".length()); 43 | String[] fields = err.substring(0, err.indexOf(" ")).split("\\."); 44 | String reason = "The statement was aborted because it would have caused a duplicate key value in a unique or primary key constraint or unique index identified by '" + fields[2] + "' defined on '" + fields[0] + "." + fields[1] + "'."; 45 | 46 | throw new SQLException(reason); 47 | } 48 | catch (MongoException e) 49 | { 50 | String msg = e.toString(); 51 | if (msg.contains("not authorized")) 52 | throw new SQLException("INSERT failed on table " + this.collectionName + ". User does not have write permission. Error returned from Mongo: " + e.toString()); 53 | throw new SQLException(e); 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoQuery.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | import com.mongodb.DB; 4 | import java.util.ArrayList; 5 | import unity.query.LQCondNode; 6 | 7 | public abstract class MongoQuery 8 | { 9 | public String collectionName = null; 10 | protected ArrayList conditions; 11 | 12 | public abstract String toMongoString() 13 | throws MongoBuilderFatalException; 14 | 15 | public abstract Object run(DB paramDB) 16 | throws Exception; 17 | 18 | public String toString() 19 | { 20 | try 21 | { 22 | return toMongoString(); 23 | } 24 | catch (MongoBuilderFatalException e) 25 | { 26 | e.printStackTrace(); 27 | } 28 | return ""; 29 | } 30 | 31 | public void addCondition(LQCondNode cn) 32 | { 33 | if (this.conditions == null) 34 | this.conditions = new ArrayList(1); 35 | this.conditions.add(cn); 36 | } 37 | 38 | public ArrayList getConditions() 39 | { 40 | return this.conditions; 41 | } 42 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoUpdateQuery.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | import com.mongodb.BasicDBObject; 4 | import com.mongodb.DB; 5 | import com.mongodb.DBCollection; 6 | 7 | public class MongoUpdateQuery extends MongoQuery 8 | { 9 | public BasicDBObject query; 10 | public BasicDBObject update; 11 | 12 | public MongoUpdateQuery() 13 | throws MongoBuilderFatalException 14 | { 15 | this.update = new BasicDBObject(); 16 | this.query = new BasicDBObject(); 17 | } 18 | 19 | private String updateToString() 20 | { 21 | StringBuffer buf = new StringBuffer(); 22 | buf.append("update("); 23 | if (this.query.size() > 0) 24 | buf.append(this.query.toString()); 25 | else 26 | buf.append("{ }"); 27 | buf.append(", "); 28 | buf.append(this.update.toString()); 29 | buf.append(")"); 30 | return buf.toString(); 31 | } 32 | 33 | public String toMongoString() 34 | throws MongoBuilderFatalException 35 | { 36 | StringBuffer buf = new StringBuffer(); 37 | buf.append("db."); 38 | buf.append(this.collectionName); 39 | buf.append("."); 40 | buf.append(updateToString()); 41 | return buf.toString(); 42 | } 43 | 44 | public Object run(DB db) 45 | throws Exception 46 | { 47 | return db.getCollection(this.collectionName).update(this.query, this.update, false, true); 48 | } 49 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/java/mongodb/query/MongoUpsertQuery.java: -------------------------------------------------------------------------------- 1 | package mongodb.query; 2 | 3 | import com.mongodb.BasicDBObject; 4 | import com.mongodb.DB; 5 | import com.mongodb.DBCollection; 6 | import com.mongodb.MongoException; 7 | import java.io.PrintStream; 8 | import java.sql.SQLException; 9 | import unity.jdbc.UnityDriver; 10 | 11 | public class MongoUpsertQuery extends MongoInsertQuery 12 | { 13 | public String toMongoString() 14 | throws MongoBuilderFatalException 15 | { 16 | return "db." + this.collectionName + ".save(" + this.insertFields.toString() + ")"; 17 | } 18 | 19 | public Object run(DB db) 20 | throws Exception 21 | { 22 | try 23 | { 24 | db.getCollection(this.collectionName).save(this.insertFields); 25 | return Integer.valueOf(1); 26 | } 27 | catch (MongoException e) 28 | { 29 | if (UnityDriver.DEBUG) { 30 | System.out.println("Mongo genererated the following exception: " + e); 31 | } 32 | throw new SQLException(e.getCause()); 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/resources/mongo/MongoConnection.properties: -------------------------------------------------------------------------------- 1 | commitError = Connection is in auto-commit mode. Calling commit() is invalid. 2 | notsupportedError = Feature is not supported by Mongo JDBC Driver. 3 | nosupportScrollSensitive = No support for TYPE_SCROLL_SENSITIVE Resultset. Use TYPE_SCROLL_INSENSITIVE instead. 4 | nosupportConcurUpdatable = No support for CONCUR_UPDATABLE Resultset. Use CONCUR_READ_ONLY instead. 5 | invalidTimeout = Invalid timeout value of 6 | rollbackError = Connection is in auto-commit mode. Calling rollback() is invalid. 7 | abortExecutorError = Null executor passed to abort. 8 | closedError = Error: Connection is closed. 9 | notSupported = Feature is not supported. -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/resources/mongo/MongoDriver.properties: -------------------------------------------------------------------------------- 1 | registerError = Failed to register Mongo JDBC Driver: 2 | loggerError = Mongo JDBC Driver does not use java.util.logging 3 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/resources/mongo/MongoPreparedStatement.properties: -------------------------------------------------------------------------------- 1 | objectConvertError = Cannot convert to target SQL Type. 2 | serializationError = Cannot extract serialization from object 3 | notsupportedError = Feature is not supported by Mongo JDBC Driver. 4 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/resources/mongo/MongoResultSet.properties: -------------------------------------------------------------------------------- 1 | forwardonly = ERROR: Row navigation not allowed for TYPE_FORWARD_ONLY ResultSets. 2 | unknownColumn = Syntax Error: Unknown column name 3 | bigdecimalError = BigDecimal conversion error: 4 | cursorError = Cursors are not supported by Enterprise Historian. 5 | conversionError = Conversion error: 6 | featureError = Feature is not supported by Enterprise Historian. 7 | invalidCursorLocation = ERROR: Invalid cursor location: 8 | convertError = Type conversion error. 9 | 10 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/resources/mongo/MongoResultSetMetaData.properties: -------------------------------------------------------------------------------- 1 | featureError = Feature is not supported by Mongo JDBC Driver. 2 | 3 | 4 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/resources/mongo/MongoStatement.properties: -------------------------------------------------------------------------------- 1 | featureError = Feature is not supported by Mongo JDBC Driver. 2 | errorExecuteUpdate = ERROR: SELECT statement not allowed with executeUpdate(): 3 | invalidFetchSize = Invalid number of rows provided for fetch size: 4 | errorExecuteQuery = ERROR: Only SELECT statements allowed with executeQuery(): 5 | 6 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-jdbc/src/main/resources/mongo/ServerConnection.properties: -------------------------------------------------------------------------------- 1 | userRequired = User is required for a connection. 2 | streamClosed = Unable to read command due to stream closed. 3 | unexpectedResponse = Unexpected server command response. 4 | noMD5 = MD5 encryption not available. Login failed. 5 | noSchema = ERROR: No schema defined. A schema is needed when performing joins or complex SQL. Put the parameter rebuildschema=true in your JDBC URL at least for the first time connecting to rebuild the schema. Example: jdbc:mongo:localhost:27017?rebuildschema=true. The default schema location is in the file mongo_.xml. Use the schema parameter to set a file location (e.g. schema=mongo.xml) to store the schema. See connection parameters at http://www.unityjdbc.com/mongojdbc/ for more details. 6 | 7 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-tool/src/main/java/com/yongqing/mongodb/client/MongodbAbstractClient.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.mongodb.client; 2 | 3 | import com.mongodb.client.MongoClient; 4 | import com.mongodb.client.MongoDatabase; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.function.Consumer; 9 | 10 | /** 11 | * 12 | */ 13 | public abstract class MongodbAbstractClient extends MongodbAbstractBaseClient { 14 | //mongoClient连接 15 | protected static volatile MongoClient mongoClient; 16 | 17 | @Override 18 | public synchronized void close() { 19 | if (null != mongoClient) { 20 | mongoClient.close(); 21 | mongoClient = null; 22 | } 23 | } 24 | 25 | 26 | public MongoDatabase getDatabase(String dataBaseName) { 27 | return mongoClient.getDatabase(dataBaseName); 28 | } 29 | 30 | public List listCollectionNames(String dataBaseName) { 31 | List stringList = new ArrayList(); 32 | mongoClient.getDatabase(dataBaseName).listCollectionNames().forEach((Consumer) t -> { 33 | stringList.add(t); 34 | }); 35 | return stringList; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-tool/src/main/java/com/yongqing/mongodb/client/MongodbAbstractPoolClient.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.mongodb.client; 2 | 3 | import com.mongodb.client.MongoClient; 4 | import com.mongodb.client.MongoDatabase; 5 | import lombok.extern.log4j.Log4j2; 6 | 7 | import java.util.ArrayList; 8 | import java.util.List; 9 | import java.util.Random; 10 | import java.util.concurrent.CopyOnWriteArrayList; 11 | import java.util.function.Consumer; 12 | 13 | /** 14 | * 15 | */ 16 | @Log4j2 17 | public abstract class MongodbAbstractPoolClient extends MongodbAbstractBaseClient{ 18 | //mongoClient连接池 19 | protected static volatile CopyOnWriteArrayList mongoClientList; 20 | 21 | protected static Integer poolSize=3; 22 | @Override 23 | public synchronized void close() { 24 | if (null != mongoClientList && mongoClientList.size()>0) { 25 | for(MongoClient mongoClient:mongoClientList){ 26 | try{ 27 | if(null!=mongoClient){ 28 | mongoClient.close(); 29 | mongoClient=null; 30 | } 31 | } 32 | catch (Throwable e){ 33 | log.error("close mongoClient cause Exception",e); 34 | } 35 | } 36 | mongoClientList=null; 37 | } 38 | } 39 | public MongoClient getMongoClient(){ 40 | if(null != mongoClientList && mongoClientList.size()>0){ 41 | return mongoClientList.get(new Random().nextInt(mongoClientList.size())); 42 | } 43 | else { 44 | throw new RuntimeException("mongoClient pool is null or pool size is 0,please check....."); 45 | } 46 | } 47 | public MongoDatabase getDatabase(String dataBaseName) { 48 | return getMongoClient().getDatabase(dataBaseName); 49 | } 50 | public List listCollectionNames(String dataBaseName) { 51 | List stringList = new ArrayList(); 52 | getMongoClient().getDatabase(dataBaseName).listCollectionNames().forEach((Consumer) t -> { 53 | stringList.add(t); 54 | }); 55 | return stringList; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-tool/src/main/java/com/yongqing/mongodb/client/MongodbClientEtcdAction.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.mongodb.client; 2 | 3 | import com.yongqing.etcd.action.Action; 4 | 5 | import java.util.Properties; 6 | 7 | /** 8 | * 9 | */ 10 | public class MongodbClientEtcdAction implements Action { 11 | @Override 12 | public void doAction(Properties properties, Properties properties1) { 13 | if (null != properties.getProperty("mongodbUrl") && null != properties1.getProperty("mongodbUrl") && !properties.getProperty("mongodbUrl").equals(properties1.getProperty("mongodbUrl"))) { 14 | if (null != MongodbClient.mongoClient) { 15 | MongodbClient.getInstanceByEtcd().close(); 16 | } 17 | if (null != MongodbPoolClient.mongoClientList) { 18 | MongodbPoolClient.getInstanceByEtcd().close(); 19 | } 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | mongodb-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-tool/src/main/resources/mongodb.properties: -------------------------------------------------------------------------------- 1 | mongodbUrl=mongodb://10.100.81.xxx:17017 -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mongodb-tool/src/test/java/com/yongqing/mongodb/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.mongodb; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mybatis-sqlOutput-plugin/src/main/resources/yongqing-mybatis-sqlOutput-plugin.properties: -------------------------------------------------------------------------------- 1 | endpoints=http://47.111.122.xxx:2379 -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-mybatis-sqlOutput-plugin/src/test/java/com/yongqing/mybatis/sqloutput/plugin/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.mybatis.sqloutput.plugin; 2 | 3 | /** 4 | * 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/HbaseConnectorFactory.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase; 2 | 3 | import com.facebook.presto.spi.ConnectorHandleResolver; 4 | import com.facebook.presto.spi.NodeManager; 5 | import com.facebook.presto.spi.connector.Connector; 6 | import com.facebook.presto.spi.connector.ConnectorContext; 7 | import com.facebook.presto.spi.connector.ConnectorFactory; 8 | import com.facebook.presto.spi.type.TypeManager; 9 | import com.google.common.base.Throwables; 10 | import com.google.inject.Injector; 11 | import io.airlift.bootstrap.Bootstrap; 12 | 13 | import java.util.Map; 14 | 15 | import static com.google.common.base.Preconditions.checkArgument; 16 | import static com.google.common.base.Strings.isNullOrEmpty; 17 | import static java.util.Objects.requireNonNull; 18 | 19 | public class HbaseConnectorFactory 20 | implements ConnectorFactory 21 | { 22 | private String name; 23 | 24 | public HbaseConnectorFactory(final String name) 25 | { 26 | checkArgument(!isNullOrEmpty(name), "name is null or empty"); 27 | this.name = name; 28 | } 29 | 30 | @Override 31 | public String getName() 32 | { 33 | return name; 34 | } 35 | 36 | @Override 37 | public ConnectorHandleResolver getHandleResolver() 38 | { 39 | return new HbaseHandleResolver(); 40 | } 41 | 42 | @Override 43 | public Connector create(String connectorId, Map config, ConnectorContext context) 44 | { 45 | requireNonNull(connectorId, "connectorId is null"); 46 | requireNonNull(config, "requiredConfig is null"); 47 | requireNonNull(context, "context is null"); 48 | 49 | try { 50 | Bootstrap app = new Bootstrap( 51 | new HbaseModule(), 52 | binder -> { 53 | binder.bind(HbaseConnectorId.class).toInstance(new HbaseConnectorId(connectorId)); 54 | binder.bind(TypeManager.class).toInstance(context.getTypeManager()); 55 | binder.bind(NodeManager.class).toInstance(context.getNodeManager()); 56 | }); 57 | 58 | Injector injector = app 59 | .strictConfig() 60 | .doNotInitializeLogging() 61 | .setRequiredConfigurationProperties(config) 62 | .initialize(); 63 | 64 | return injector.getInstance(HbaseConnector.class); 65 | } 66 | catch (Exception e) { 67 | throw Throwables.propagate(e); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/HbaseConnectorId.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase; 2 | 3 | import java.util.Objects; 4 | 5 | import static java.util.Objects.requireNonNull; 6 | 7 | public final class HbaseConnectorId 8 | { 9 | private final String id; 10 | 11 | public HbaseConnectorId(String id) 12 | { 13 | this.id = requireNonNull(id, "id is null"); 14 | } 15 | 16 | @Override 17 | public String toString() 18 | { 19 | return id; 20 | } 21 | 22 | @Override 23 | public int hashCode() 24 | { 25 | return Objects.hash(id); 26 | } 27 | 28 | @Override 29 | public boolean equals(Object obj) 30 | { 31 | if (this == obj) { 32 | return true; 33 | } 34 | 35 | if ((obj == null) || (getClass() != obj.getClass())) { 36 | return false; 37 | } 38 | 39 | return Objects.equals(this.id, ((HbaseConnectorId) obj).id); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/HbaseErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase; 2 | 3 | import com.facebook.presto.spi.ErrorCode; 4 | import com.facebook.presto.spi.ErrorCodeSupplier; 5 | import com.facebook.presto.spi.ErrorType; 6 | 7 | import static com.facebook.presto.spi.ErrorType.EXTERNAL; 8 | 9 | public enum HbaseErrorCode 10 | implements ErrorCodeSupplier 11 | { 12 | // Thrown when an Hbase error is caught that we were not expecting, 13 | // such as when a create table operation fails (even though we know it will succeed due to our validation steps) 14 | UNEXPECTED_HBASE_ERROR(1, EXTERNAL), 15 | 16 | // Thrown when a ZooKeeper error is caught due to a failed operation 17 | ZOOKEEPER_ERROR(2, EXTERNAL), 18 | 19 | // Thrown when a serialization error occurs when reading/writing data from/to Hbase 20 | IO_ERROR(3, EXTERNAL), 21 | 22 | // Thrown when a table that is expected to exist does not exist 23 | HBASE_TABLE_DNE(4, EXTERNAL), 24 | 25 | HBASE_TABLE_CLOSE_ERR(5, EXTERNAL), 26 | 27 | HBASE_TABLE_EXISTS(6, EXTERNAL); 28 | 29 | private final ErrorCode errorCode; 30 | 31 | HbaseErrorCode(int code, ErrorType type) 32 | { 33 | errorCode = new ErrorCode(code + 0x0103_0000, name(), type); 34 | } 35 | 36 | @Override 37 | public ErrorCode toErrorCode() 38 | { 39 | return errorCode; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/HbaseHandleResolver.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase; 2 | 3 | import com.yongqing.presto.hbase.model.HbaseColumnHandle; 4 | import com.yongqing.presto.hbase.model.HbaseSplit; 5 | import com.yongqing.presto.hbase.model.HbaseTableHandle; 6 | import com.yongqing.presto.hbase.model.HbaseTableLayoutHandle; 7 | import com.yongqing.presto.hbase.model.HbaseTransactionHandle; 8 | import com.facebook.presto.spi.ColumnHandle; 9 | import com.facebook.presto.spi.ConnectorHandleResolver; 10 | import com.facebook.presto.spi.ConnectorInsertTableHandle; 11 | import com.facebook.presto.spi.ConnectorOutputTableHandle; 12 | import com.facebook.presto.spi.ConnectorSplit; 13 | import com.facebook.presto.spi.ConnectorTableHandle; 14 | import com.facebook.presto.spi.ConnectorTableLayoutHandle; 15 | import com.facebook.presto.spi.connector.ConnectorTransactionHandle; 16 | 17 | public class HbaseHandleResolver 18 | implements ConnectorHandleResolver 19 | { 20 | @Override 21 | public Class getTableLayoutHandleClass() 22 | { 23 | return HbaseTableLayoutHandle.class; 24 | } 25 | 26 | @Override 27 | public Class getTableHandleClass() 28 | { 29 | return HbaseTableHandle.class; 30 | } 31 | 32 | @Override 33 | public Class getInsertTableHandleClass() 34 | { 35 | return HbaseTableHandle.class; 36 | } 37 | 38 | @Override 39 | public Class getOutputTableHandleClass() 40 | { 41 | return HbaseTableHandle.class; 42 | } 43 | 44 | @Override 45 | public Class getColumnHandleClass() 46 | { 47 | return HbaseColumnHandle.class; 48 | } 49 | 50 | @Override 51 | public Class getSplitClass() 52 | { 53 | return HbaseSplit.class; 54 | } 55 | 56 | @Override 57 | public Class getTransactionHandleClass() 58 | { 59 | return HbaseTransactionHandle.class; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/HbaseMetadataFactory.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase; 2 | 3 | public class HbaseMetadataFactory 4 | { 5 | } 6 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/HbasePlugin.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase; 2 | 3 | import com.facebook.presto.spi.Plugin; 4 | import com.facebook.presto.spi.connector.ConnectorFactory; 5 | import com.google.common.collect.ImmutableList; 6 | import com.google.common.collect.ImmutableSet; 7 | 8 | import java.util.Set; 9 | 10 | public class HbasePlugin 11 | implements Plugin 12 | { 13 | @Override 14 | public Set> getFunctions() 15 | { 16 | return ImmutableSet.of(); 17 | } 18 | 19 | @Override 20 | public Iterable getConnectorFactories() 21 | { 22 | return ImmutableList.of(new HbaseConnectorFactory("hbase")); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/Types.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase; 2 | 3 | import com.facebook.presto.spi.type.StandardTypes; 4 | import com.facebook.presto.spi.type.Type; 5 | 6 | /** 7 | * Utility class for Presto Type-related functionality. 8 | */ 9 | public final class Types 10 | { 11 | private Types() {} 12 | 13 | public static boolean isArrayType(Type type) 14 | { 15 | return type.getTypeSignature().getBase().equals(StandardTypes.ARRAY); 16 | } 17 | 18 | public static boolean isMapType(Type type) 19 | { 20 | return type.getTypeSignature().getBase().equals(StandardTypes.MAP); 21 | } 22 | 23 | /** 24 | * Gets the element type of the given array type. Does not validate that the given type is an array. 25 | * 26 | * @param type An array type 27 | * @return Element type of the array 28 | * @throws IndexOutOfBoundsException If type is not an array 29 | * @see Types#isArrayType 30 | */ 31 | public static Type getElementType(Type type) 32 | { 33 | return type.getTypeParameters().get(0); 34 | } 35 | 36 | /** 37 | * Gets the key type of the given map type. Does not validate that the given type is a map. 38 | * 39 | * @param type A map type 40 | * @return Key type of the map 41 | * @throws IndexOutOfBoundsException If type is not a map 42 | * @see Types#isMapType 43 | */ 44 | public static Type getKeyType(Type type) 45 | { 46 | return type.getTypeParameters().get(0); 47 | } 48 | 49 | /** 50 | * Gets the value type of the given map type. Does not validate that the given type is a map. 51 | * 52 | * @param type A map type 53 | * @return Value type of the map 54 | * @throws IndexOutOfBoundsException If type is not a map 55 | * @see Types#isMapType 56 | */ 57 | public static Type getValueType(Type type) 58 | { 59 | return type.getTypeParameters().get(1); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/conf/HbaseConfig.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase.conf; 2 | 3 | import io.airlift.configuration.Config; 4 | import io.airlift.configuration.ConfigDescription; 5 | 6 | import javax.validation.constraints.NotNull; 7 | 8 | public class HbaseConfig 9 | { 10 | private String hbaseMaster; 11 | private String zooKeepers; 12 | private String zkMetadataRoot = "/presto-hbase"; 13 | 14 | public String getHbaseMaster() 15 | { 16 | return hbaseMaster; 17 | } 18 | 19 | @Config("hbase.hosts") 20 | @ConfigDescription("IP:PORT where hbase master connect") 21 | public HbaseConfig setHbaseMaster(String hbaseMaster) 22 | { 23 | this.hbaseMaster = hbaseMaster; 24 | return this; 25 | } 26 | 27 | @NotNull 28 | public String getZooKeepers() 29 | { 30 | return this.zooKeepers; 31 | } 32 | 33 | @Config("hbase.zookeepers") 34 | @ConfigDescription("ZooKeeper quorum connect string for Hbase") 35 | public HbaseConfig setZooKeepers(String zooKeepers) 36 | { 37 | this.zooKeepers = zooKeepers; 38 | return this; 39 | } 40 | 41 | @NotNull 42 | public String getZkMetadataRoot() 43 | { 44 | return zkMetadataRoot; 45 | } 46 | 47 | @Config("hbase.zookeeper.metadata.root") 48 | @ConfigDescription("Sets the root znode for metadata storage") 49 | public void setZkMetadataRoot(String zkMetadataRoot) 50 | { 51 | this.zkMetadataRoot = zkMetadataRoot; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/io/HbasePageSinkProvider.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase.io; 2 | 3 | import com.yongqing.presto.hbase.HbaseClient; 4 | import com.yongqing.presto.hbase.model.HbaseTableHandle; 5 | import com.facebook.presto.spi.*; 6 | import com.facebook.presto.spi.connector.ConnectorPageSinkProvider; 7 | import com.facebook.presto.spi.connector.ConnectorTransactionHandle; 8 | import org.apache.hadoop.hbase.client.Connection; 9 | 10 | import javax.inject.Inject; 11 | 12 | import static java.util.Objects.requireNonNull; 13 | 14 | public class HbasePageSinkProvider 15 | implements ConnectorPageSinkProvider 16 | { 17 | private final HbaseClient client; 18 | private final Connection connection; 19 | 20 | @Inject 21 | public HbasePageSinkProvider( 22 | Connection connection, 23 | HbaseClient client) 24 | { 25 | this.client = requireNonNull(client, "client is null"); 26 | this.connection = requireNonNull(connection, "connection is null"); 27 | } 28 | 29 | //TODO 30 | @Override 31 | public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorOutputTableHandle outputTableHandle, PageSinkProperties pageSinkProperties) { 32 | HbaseTableHandle tableHandle = (HbaseTableHandle) outputTableHandle; 33 | return new HbasePageSink(connection, client.getTable(tableHandle.toSchemaTableName())); 34 | } 35 | //TODO 36 | @Override 37 | public ConnectorPageSink createPageSink(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorInsertTableHandle insertTableHandle, PageSinkProperties pageSinkProperties) { 38 | return createPageSink(transactionHandle, session, (ConnectorOutputTableHandle) insertTableHandle, pageSinkProperties); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/io/HbaseRecordSetProvider.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase.io; 2 | 3 | import com.yongqing.presto.hbase.HbaseClient; 4 | import com.yongqing.presto.hbase.HbaseConnectorId; 5 | import com.yongqing.presto.hbase.conf.HbaseConfig; 6 | import com.yongqing.presto.hbase.model.HbaseColumnHandle; 7 | import com.yongqing.presto.hbase.model.HbaseSplit; 8 | import com.facebook.presto.spi.ColumnHandle; 9 | import com.facebook.presto.spi.ConnectorSession; 10 | import com.facebook.presto.spi.ConnectorSplit; 11 | import com.facebook.presto.spi.RecordSet; 12 | import com.facebook.presto.spi.connector.ConnectorRecordSetProvider; 13 | import com.facebook.presto.spi.connector.ConnectorTransactionHandle; 14 | import com.google.common.collect.ImmutableList; 15 | 16 | import javax.inject.Inject; 17 | 18 | import java.util.List; 19 | 20 | import static com.google.common.base.Preconditions.checkArgument; 21 | import static java.util.Objects.requireNonNull; 22 | 23 | public class HbaseRecordSetProvider 24 | implements ConnectorRecordSetProvider 25 | { 26 | private final String connectorId; 27 | private final HbaseClient hbaseClient; 28 | private final HbaseConfig config; 29 | 30 | @Inject 31 | public HbaseRecordSetProvider( 32 | HbaseClient hbaseClient, 33 | HbaseConnectorId connectorId, 34 | HbaseConfig config) 35 | { 36 | this.hbaseClient = requireNonNull(hbaseClient, "hbaseClient is null"); 37 | this.connectorId = requireNonNull(connectorId, "connectorId is null").toString(); 38 | this.config = requireNonNull(config, "config is null"); 39 | } 40 | 41 | @Override 42 | public RecordSet getRecordSet(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorSplit split, List columns) 43 | { 44 | requireNonNull(split, "split is null"); 45 | HbaseSplit hbaseSplit = (HbaseSplit) split; 46 | checkArgument(hbaseSplit.getConnectorId().equals(connectorId), "split is not for this connector"); 47 | 48 | ImmutableList.Builder handles = ImmutableList.builder(); 49 | for (ColumnHandle handle : columns) { 50 | handles.add((HbaseColumnHandle) handle); 51 | } 52 | 53 | return new HbaseRecordSet(hbaseClient, session, hbaseSplit, handles.build()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/model/HbaseTableLayoutHandle.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase.model; 2 | 3 | import com.facebook.presto.spi.ColumnHandle; 4 | import com.facebook.presto.spi.ConnectorTableLayoutHandle; 5 | import com.facebook.presto.spi.predicate.TupleDomain; 6 | import com.fasterxml.jackson.annotation.JsonCreator; 7 | import com.fasterxml.jackson.annotation.JsonProperty; 8 | 9 | import java.util.Objects; 10 | 11 | import static com.google.common.base.MoreObjects.toStringHelper; 12 | import static java.util.Objects.requireNonNull; 13 | 14 | public class HbaseTableLayoutHandle 15 | implements ConnectorTableLayoutHandle 16 | { 17 | private final HbaseTableHandle table; 18 | private final TupleDomain constraint; 19 | 20 | @JsonCreator 21 | public HbaseTableLayoutHandle( 22 | @JsonProperty("table") HbaseTableHandle table, 23 | @JsonProperty("constraint") TupleDomain constraint) 24 | { 25 | this.table = requireNonNull(table, "table is null"); 26 | this.constraint = requireNonNull(constraint, "constraint is null"); 27 | } 28 | 29 | @JsonProperty 30 | public HbaseTableHandle getTable() 31 | { 32 | return table; 33 | } 34 | 35 | @JsonProperty 36 | public TupleDomain getConstraint() 37 | { 38 | return constraint; 39 | } 40 | 41 | @Override 42 | public boolean equals(Object obj) 43 | { 44 | if (this == obj) { 45 | return true; 46 | } 47 | 48 | if (obj == null || getClass() != obj.getClass()) { 49 | return false; 50 | } 51 | 52 | HbaseTableLayoutHandle other = (HbaseTableLayoutHandle) obj; 53 | return Objects.equals(table, other.table) 54 | && Objects.equals(constraint, other.constraint); 55 | } 56 | 57 | @Override 58 | public int hashCode() 59 | { 60 | return Objects.hash(table, constraint); 61 | } 62 | 63 | @Override 64 | public String toString() 65 | { 66 | return toStringHelper(this) 67 | .add("table", table) 68 | .add("constraint", constraint) 69 | .toString(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/java/com/yongqing/presto/hbase/model/HbaseTransactionHandle.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.presto.hbase.model; 2 | 3 | import com.facebook.presto.spi.connector.ConnectorTransactionHandle; 4 | import com.fasterxml.jackson.annotation.JsonCreator; 5 | import com.fasterxml.jackson.annotation.JsonProperty; 6 | 7 | import java.util.Objects; 8 | import java.util.UUID; 9 | 10 | import static com.google.common.base.MoreObjects.toStringHelper; 11 | import static java.util.Objects.requireNonNull; 12 | 13 | public class HbaseTransactionHandle 14 | implements ConnectorTransactionHandle 15 | { 16 | private final UUID uuid; 17 | 18 | public HbaseTransactionHandle() 19 | { 20 | this(UUID.randomUUID()); 21 | } 22 | 23 | @JsonCreator 24 | public HbaseTransactionHandle(@JsonProperty("uuid") UUID uuid) 25 | { 26 | this.uuid = requireNonNull(uuid, "uuid is null"); 27 | } 28 | 29 | @JsonProperty 30 | public UUID getUuid() 31 | { 32 | return uuid; 33 | } 34 | 35 | @Override 36 | public boolean equals(Object obj) 37 | { 38 | if (this == obj) { 39 | return true; 40 | } 41 | if ((obj == null) || (getClass() != obj.getClass())) { 42 | return false; 43 | } 44 | 45 | return Objects.equals(uuid, ((HbaseTransactionHandle) obj).uuid); 46 | } 47 | 48 | @Override 49 | public int hashCode() 50 | { 51 | return Objects.hash(uuid); 52 | } 53 | 54 | @Override 55 | public String toString() 56 | { 57 | return toStringHelper(this).add("uuid", uuid).toString(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/main/resources/META-INF/services/com.facebook.presto.spi.Plugin: -------------------------------------------------------------------------------- 1 | com.yongqing.presto.hbase.HbasePlugin -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-presto-hbase/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # WARNING 3 | # ^^^^^^^ 4 | # This configuration file is for development only and should NOT be used be 5 | # used in production. For example configuration, see the Presto documentation. 6 | # 7 | log4j.rootLogger=INFO, console 8 | 9 | log4j.appender.console=org.apache.log4j.ConsoleAppender 10 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 11 | log4j.appender.console.Target = System.out 12 | log4j.appender.console.Threshold = INFO 13 | log4j.appender.console.layout.ConversionPattern=%d{yy-MM-dd HH:mm:ss} %p %c{1}: %m%n 14 | 15 | com.facebook.presto=INFO 16 | com.sun.jersey.guice.spi.container.GuiceComponentProviderFactory=WARN 17 | com.ning.http.client=WARN 18 | com.facebook.presto.server.PluginManager=DEBUG 19 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/README.md: -------------------------------------------------------------------------------- 1 | # yongqing-rocketmq-flume-tool 2 | 3 | 大数据--flume对rocketmq的操作 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/alyun/listen/AliyunRocketmqMessageListener.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.alyun.listen; 2 | 3 | import com.aliyun.openservices.ons.api.Action; 4 | import com.aliyun.openservices.ons.api.ConsumeContext; 5 | import com.aliyun.openservices.ons.api.Message; 6 | import com.aliyun.openservices.ons.api.MessageListener; 7 | import com.yongqing.rocketmq.queue.FlumeMq; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | /** 12 | * 13 | */ 14 | public class AliyunRocketmqMessageListener implements MessageListener { 15 | public static Logger log = LoggerFactory.getLogger(AliyunRocketmqMessageListener.class); 16 | 17 | @Override 18 | public Action consume(Message message, ConsumeContext consumeContext) { 19 | log.info("Receive:{}",message); 20 | try { 21 | FlumeMq.getFlumeMq().put(message); 22 | } catch (InterruptedException e) { 23 | log.info("message put cause error",e); 24 | } 25 | return Action.CommitMessage; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/elasticsearch/DefaultIndexNameBuilder.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.elasticsearch; 2 | 3 | import org.apache.flume.Context; 4 | import org.apache.flume.Event; 5 | import org.apache.flume.conf.ComponentConfiguration; 6 | import org.apache.flume.formatter.output.BucketPath; 7 | 8 | /** 9 | * Created by zhangyongqing on 2019-10-15. 10 | */ 11 | public class DefaultIndexNameBuilder implements 12 | IndexNameBuilder { 13 | private String indexPrefix; 14 | @Override 15 | public String getIndexName(Event event) { 16 | String realIndexPrefix = BucketPath.escapeString(indexPrefix, event.getHeaders()); 17 | return realIndexPrefix; 18 | } 19 | 20 | @Override 21 | public String getIndexPrefix(Event event) { 22 | return BucketPath.escapeString(indexPrefix, event.getHeaders()); 23 | } 24 | 25 | @Override 26 | public void configure(Context context) { 27 | indexPrefix = context.getString(ElasticSearchSinkConstants.INDEX_NAME); 28 | } 29 | 30 | @Override 31 | public void configure(ComponentConfiguration componentConfiguration) { 32 | 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/elasticsearch/ElasticSearchEventSerializer.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.elasticsearch; 2 | 3 | import org.apache.flume.Event; 4 | import org.apache.flume.conf.Configurable; 5 | import org.apache.flume.conf.ConfigurableComponent; 6 | import org.elasticsearch.common.bytes.BytesReference; 7 | import org.elasticsearch.common.io.stream.BytesStream; 8 | import org.elasticsearch.common.xcontent.XContentBuilder; 9 | 10 | import java.io.IOException; 11 | import java.nio.charset.Charset; 12 | 13 | /** 14 | * 15 | */ 16 | public interface ElasticSearchEventSerializer extends Configurable, ConfigurableComponent { 17 | 18 | Charset charset = Charset.defaultCharset(); 19 | XContentBuilder getContentBuilder(Event event) throws IOException; 20 | } 21 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/elasticsearch/ElasticSearchRocketMqEventSerializer.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.elasticsearch; 2 | 3 | import org.apache.flume.Context; 4 | import org.apache.flume.Event; 5 | import org.apache.flume.conf.ComponentConfiguration; 6 | import org.elasticsearch.common.xcontent.XContentBuilder; 7 | import java.io.IOException; 8 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 9 | /** 10 | * RocketMq消息的Serializer 11 | */ 12 | public class ElasticSearchRocketMqEventSerializer implements ElasticSearchEventSerializer { 13 | @Override 14 | public XContentBuilder getContentBuilder(Event event) throws IOException { 15 | XContentBuilder builder = jsonBuilder().startObject(); 16 | appendHeaders(builder,event); 17 | builder.endObject(); 18 | return builder; 19 | } 20 | private void appendHeaders(XContentBuilder builder, Event event) throws IOException { 21 | ContentBuilderUtil.appendField(builder,"topic",event.getHeaders().get("topic").getBytes(charset)); 22 | ContentBuilderUtil.appendField(builder,"groupId",event.getHeaders().get("groupId").getBytes(charset)); 23 | ContentBuilderUtil.appendField(builder,"message",event.getBody()); 24 | } 25 | 26 | @Override 27 | public void configure(Context context) { 28 | 29 | } 30 | 31 | @Override 32 | public void configure(ComponentConfiguration componentConfiguration) { 33 | 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/elasticsearch/HbaseSinkConstants.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.elasticsearch; 2 | 3 | /** 4 | * 5 | */ 6 | public class HbaseSinkConstants { 7 | 8 | public static final String BATCH_SIZE = "batchSize"; 9 | } 10 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/elasticsearch/IndexNameBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | */ 4 | package com.yongqing.rocketmq.elasticsearch; 5 | 6 | import org.apache.flume.Event; 7 | import org.apache.flume.conf.Configurable; 8 | import org.apache.flume.conf.ConfigurableComponent; 9 | 10 | public interface IndexNameBuilder extends Configurable, 11 | ConfigurableComponent { 12 | /** 13 | * Gets the name of the index to use for an index request 14 | * @param event 15 | * Event which determines index name 16 | * @return index name of the form 'indexPrefix-indexDynamicName' 17 | */ 18 | String getIndexName(Event event); 19 | 20 | /** 21 | * Gets the prefix of index to use for an index request. 22 | * @param event 23 | * Event which determines index name 24 | * @return Index prefix name 25 | */ 26 | String getIndexPrefix(Event event); 27 | } 28 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/elasticsearch/TimestampedEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | */ 4 | package com.yongqing.rocketmq.elasticsearch; 5 | 6 | import com.google.common.collect.Maps; 7 | import org.apache.commons.lang.StringUtils; 8 | import org.apache.flume.Event; 9 | import org.apache.flume.event.SimpleEvent; 10 | import org.joda.time.DateTimeUtils; 11 | 12 | import java.util.Map; 13 | 14 | /** 15 | * {@link org.apache.flume.Event} implementation that has a timestamp. 16 | * The timestamp is taken from (in order of precedence):
    17 | *
  1. The "timestamp" header of the base event, if present
    2. 18 | *
  2. The "@timestamp" header of the base event, if present
    3. 19 | *
  3. The current time in millis, otherwise
    4. 20 | *
21 | */ 22 | final class TimestampedEvent extends SimpleEvent { 23 | 24 | private final long timestamp; 25 | 26 | TimestampedEvent(Event base) { 27 | setBody(base.getBody()); 28 | Map headers = Maps.newHashMap(base.getHeaders()); 29 | String timestampString = headers.get("timestamp"); 30 | if (StringUtils.isBlank(timestampString)) { 31 | timestampString = headers.get("@timestamp"); 32 | } 33 | if (StringUtils.isBlank(timestampString)) { 34 | this.timestamp = DateTimeUtils.currentTimeMillis(); 35 | headers.put("timestamp", String.valueOf(timestamp )); 36 | } else { 37 | this.timestamp = Long.valueOf(timestampString); 38 | } 39 | setHeaders(headers); 40 | } 41 | 42 | long getTimestamp() { 43 | return timestamp; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/elasticsearch/client/Client.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.elasticsearch.client; 2 | 3 | import com.yongqing.rocketmq.elasticsearch.IndexNameBuilder; 4 | import org.apache.flume.Event; 5 | import org.apache.flume.conf.Configurable; 6 | 7 | /** 8 | * 9 | */ 10 | public interface Client extends Configurable { 11 | 12 | void close(); 13 | 14 | void addEvent(Event event, IndexNameBuilder indexNameBuilder, 15 | String indexType, long ttlMs) throws Exception; 16 | 17 | void execute() throws Exception; 18 | } 19 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/elasticsearch/client/ElasticSearchHttpClient.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.elasticsearch.client; 2 | 3 | import com.yongqing.common.bigdata.tool.BigDataHttpClient; 4 | 5 | 6 | /** 7 | * 8 | */ 9 | public class ElasticSearchHttpClient extends BigDataHttpClient { 10 | 11 | } 12 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/elasticsearch/client/RoundRobinList.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.elasticsearch.client; 2 | 3 | import java.util.Collection; 4 | import java.util.Iterator; 5 | 6 | /* 7 | * 8 | */ 9 | 10 | public class RoundRobinList { 11 | 12 | private Iterator iterator; 13 | private final Collection elements; 14 | 15 | public RoundRobinList(Collection elements) { 16 | this.elements = elements; 17 | iterator = this.elements.iterator(); 18 | } 19 | 20 | public synchronized T get() { 21 | if (iterator.hasNext()) { 22 | return iterator.next(); 23 | } else { 24 | iterator = elements.iterator(); 25 | return iterator.next(); 26 | } 27 | } 28 | 29 | public int size() { 30 | return elements.size(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/flume/sink/HbaseSink.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.flume.sink; 2 | 3 | import org.apache.flume.Context; 4 | import org.apache.flume.EventDeliveryException; 5 | import org.apache.flume.Sink; 6 | import org.apache.flume.conf.BatchSizeSupported; 7 | import org.apache.flume.conf.Configurable; 8 | import org.apache.flume.sink.AbstractSink; 9 | 10 | /** 11 | * hbase sink的自定义实现 12 | */ 13 | public class HbaseSink extends AbstractSink implements Configurable, BatchSizeSupported { 14 | @Override 15 | public Status process() throws EventDeliveryException { 16 | return null; 17 | } 18 | 19 | @Override 20 | public long getBatchSize() { 21 | return 0; 22 | } 23 | 24 | @Override 25 | public void configure(Context context) { 26 | 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/java/com/yongqing/rocketmq/queue/FlumeMq.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq.queue; 2 | 3 | import com.aliyun.openservices.ons.api.Message; 4 | 5 | import java.util.concurrent.SynchronousQueue; 6 | 7 | /** 8 | * 9 | */ 10 | public class FlumeMq { 11 | //同步阻塞队列 12 | private final static SynchronousQueue flumeMq = new SynchronousQueue(); 13 | 14 | public static SynchronousQueue getFlumeMq() { 15 | return flumeMq; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | rocketMq-flume 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-rocketmq-flume-tool/src/test/java/com/yongqing/rocketmq/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.rocketmq; 2 | 3 | /** 4 | * Created by zhangyongqing on 2019-10-09. 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-spring-mybatis-sqlOutput/src/main/java/com/yongqing/spring/mybatis/sqloutput/annotation/SqlOutput.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.spring.mybatis.sqloutput.annotation; 2 | 3 | import java.lang.annotation.*; 4 | 5 | 6 | 7 | /** 8 | * 9 | */ 10 | @Documented 11 | @Retention(RetentionPolicy.CLASS) 12 | @Target({ElementType.TYPE, ElementType.METHOD}) 13 | public @interface SqlOutput { 14 | } 15 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-spring-mybatis-sqlOutput/src/main/java/com/yongqing/spring/mybatis/sqloutput/autoconfigure/SelfAutoConfig.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.spring.mybatis.sqloutput.autoconfigure; 2 | 3 | import com.yongqing.spring.mybatis.sqloutput.aspect.SqlOutputAdvice; 4 | import com.yongqing.spring.mybatis.sqloutput.plugin.MybatisSqlOutputPlugin; 5 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | /** 9 | * 10 | */ 11 | @Configuration 12 | //@ComponentScan("com.yongqing.spring.mybatis") 13 | @EnableConfigurationProperties({SqlOutputAdvice.class, MybatisSqlOutputPlugin.class}) 14 | public class SelfAutoConfig { 15 | 16 | } 17 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-spring-mybatis-sqlOutput/src/main/java/com/yongqing/spring/mybatis/sqloutput/mapper/SqlBaseMapper.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.spring.mybatis.sqloutput.mapper; 2 | 3 | import org.springframework.transaction.annotation.Transactional; 4 | 5 | /** 6 | * 7 | */ 8 | @Transactional 9 | public interface SqlBaseMapper { 10 | 11 | 12 | } 13 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-spring-mybatis-sqlOutput/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | # Auto Configure 2 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ 3 | com.yongqing.spring.mybatis.sqloutput.autoconfigure.SelfAutoConfig -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-spring-mybatis-sqlOutput/src/main/resources/yongqing-mybatis-sqlOutput-plugin.properties: -------------------------------------------------------------------------------- 1 | endpoints=http://47.111.122.xxx:2379 -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-spring-mybatis-sqlOutput/src/test/java/com/yongqing/spring/mybatis/sqloutput/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.spring.mybatis.sqloutput; 2 | 3 | /** 4 | * . 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-sql-analyse/src/main/java/com/yongqing/sql/analyse/MybatisSessionFactory.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.sql.analyse; 2 | 3 | import org.apache.ibatis.session.SqlSessionFactory; 4 | import org.apache.ibatis.session.SqlSessionFactoryBuilder; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.io.InputStream; 9 | 10 | /** 11 | * 12 | * sqlsession factory 单例 事务设置为手动提交 13 | */ 14 | public class MybatisSessionFactory { 15 | private static final Logger LOG = LoggerFactory.getLogger(MybatisSessionFactory.class); 16 | private static SqlSessionFactory sqlSessionFactory; 17 | private MybatisSessionFactory(){ 18 | super(); 19 | } 20 | public synchronized static SqlSessionFactory getSqlSessionFactory(){ 21 | if(null==sqlSessionFactory){ 22 | InputStream inputStream=null; 23 | try{ 24 | inputStream = MybatisSessionFactory.class.getClassLoader().getResourceAsStream("mybatis-config.xml"); 25 | sqlSessionFactory = new SqlSessionFactoryBuilder().build(inputStream); 26 | } 27 | catch (Exception e){ 28 | LOG.error("create MybatisSessionFactory read mybatis-config.xml cause Exception",e); 29 | } 30 | if(null!=sqlSessionFactory){ 31 | LOG.info("get Mybatis sqlsession sucessed...."); 32 | } 33 | else { 34 | LOG.info("get Mybatis sqlsession failed...."); 35 | } 36 | } 37 | return sqlSessionFactory; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-sql-analyse/src/main/java/com/yongqing/sql/analyse/bean/SqlOutput.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.sql.analyse.bean; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | /** 7 | * 8 | */ 9 | public class SqlOutput { 10 | 11 | private String sql; 12 | //query update delete all insert 13 | private String sqlType; 14 | 15 | private List> sqlResult; 16 | 17 | private String execStatus; 18 | 19 | private String execMessage; 20 | private Integer effectRowNum; 21 | 22 | public Integer getEffectRowNum() { 23 | return effectRowNum; 24 | } 25 | 26 | public void setEffectRowNum(Integer effectRowNum) { 27 | this.effectRowNum = effectRowNum; 28 | } 29 | 30 | public String getExecStatus() { 31 | return execStatus; 32 | } 33 | 34 | public void setExecStatus(String execStatus) { 35 | this.execStatus = execStatus; 36 | } 37 | 38 | public String getExecMessage() { 39 | return execMessage; 40 | } 41 | 42 | public void setExecMessage(String execMessage) { 43 | this.execMessage = execMessage; 44 | } 45 | 46 | public String getSql() { 47 | return sql; 48 | } 49 | 50 | public void setSql(String sql) { 51 | this.sql = sql; 52 | } 53 | 54 | public String getSqlType() { 55 | return sqlType; 56 | } 57 | 58 | public void setSqlType(String sqlType) { 59 | this.sqlType = sqlType; 60 | } 61 | 62 | public List> getSqlResult() { 63 | return sqlResult; 64 | } 65 | 66 | public void setSqlResult(List> sqlResult) { 67 | this.sqlResult = sqlResult; 68 | } 69 | 70 | @Override 71 | public String toString() { 72 | return "SqlOutput{" + 73 | "sql='" + sql + '\'' + 74 | ", sqlType='" + sqlType + '\'' + 75 | ", sqlResult=" + sqlResult + 76 | ", execStatus='" + execStatus + '\'' + 77 | ", execMessage='" + execMessage + '\'' + 78 | ", effectRowNum=" + effectRowNum + 79 | '}'; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-sql-analyse/src/main/java/com/yongqing/sql/analyse/exception/SqlAnalyseException.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.sql.analyse.exception; 2 | 3 | /** 4 | * 5 | */ 6 | public class SqlAnalyseException extends RuntimeException { 7 | public SqlAnalyseException(String s) { 8 | super(s); 9 | } 10 | public SqlAnalyseException(String s,Exception e) { 11 | super(s,e); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-sql-analyse/src/main/java/com/yongqing/sql/analyse/sqlexec/SqlExec.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.sql.analyse.sqlexec; 2 | 3 | import com.yongqing.sql.analyse.bean.SqlOutput; 4 | 5 | import java.sql.Connection; 6 | import java.util.List; 7 | 8 | /** 9 | * 10 | */ 11 | public interface SqlExec { 12 | 13 | 14 | SqlOutput executeQuery(Connection connection, String sql); 15 | 16 | SqlOutput executeQuery(Connection connection, String sql, List columnList); 17 | 18 | /** 19 | * 执行SQL查询 SELECT 20 | * ResultSet executeQuery(String sql); 执行SQL查询，并返回ResultSet 对象。 21 | * 22 | * @param sql 要查询的SQL语句 23 | * @param separator 列分隔符 24 | * @param start 记录开始 25 | * @param length 返回记录数 26 | * @return 执行结果 27 | */ 28 | 29 | String exportQuery(Connection connection, String sql, String separator, int start, int length); 30 | 31 | SqlOutput executeUpdate(Connection connection, String sql); 32 | 33 | SqlOutput execute(Connection connection, String sql); 34 | } 35 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-sql-analyse/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | sql-analyse 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-sql-analyse/src/main/resources/mybatis-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-sql-analyse/src/test/java/com/yongqing/sql/analyse/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.sql.analyse; 2 | 3 | /** 4 | * 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-thread-pool-tool/src/main/java/com/yongqing/thread/pool/tool/CallableFunction.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.thread.pool.tool; 2 | import java.util.List; 3 | import java.util.concurrent.Callable; 4 | import java.util.function.Consumer; 5 | 6 | /** 7 | * 8 | */ 9 | @FunctionalInterface 10 | public interface CallableFunction { 11 | Callable getCallableObject(Q data); 12 | } 13 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-thread-pool-tool/src/main/java/com/yongqing/thread/pool/tool/DefaultPoolExec.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.thread.pool.tool; 2 | import java.util.concurrent.ExecutorService; 3 | 4 | /** 5 | * 6 | */ 7 | public class DefaultPoolExec extends AbstractPoolExec{ 8 | public DefaultPoolExec(){ 9 | super(); 10 | } 11 | public DefaultPoolExec(int poolSize){ 12 | this.poolSize = poolSize; 13 | } 14 | public DefaultPoolExec(ExecutorService executorService){ 15 | this.executorService = executorService; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-thread-pool-tool/src/main/java/com/yongqing/thread/pool/tool/Exec.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.thread.pool.tool; 2 | 3 | import com.google.common.util.concurrent.ListenableFuture; 4 | import com.google.common.util.concurrent.ListeningExecutorService; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * 10 | */ 11 | public interface Exec { 12 | ListenableFuture> exec(ListeningExecutorService executorService, CallableFunction function , List listData,Integer slicesNum) throws Exception; 13 | ListenableFuture exec(ListeningExecutorService executorService,List threads); 14 | } 15 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-thread-pool-tool/src/main/java/com/yongqing/thread/pool/tool/FutureFunction.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.thread.pool.tool; 2 | 3 | 4 | /** 5 | * 6 | */ 7 | @FunctionalInterface 8 | public interface FutureFunction { 9 | void futuresAdd(E e,Q q,L l, D data); 10 | } 11 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-thread-pool-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | thread-pool-tools 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-thread-pool-tool/src/main/resources/threadPool.properties: -------------------------------------------------------------------------------- 1 | endpoints=http://47.111.122.xxx:2379 -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-thread-pool-tool/src/test/java/com/yongqing/thread/pool/tool/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.thread.pool.tool; 2 | 3 | /** 4 | * 5 | */ 6 | public class Test { 7 | } 8 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-zookeeper-tool/src/main/java/com/yongqing/zookeeper/tool/client/DefaultZKClient.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.zookeeper.tool.client; 2 | import org.apache.curator.RetryPolicy; 3 | 4 | 5 | /** 6 | * 7 | */ 8 | public class DefaultZKClient { 9 | 10 | public volatile static AbstractZKClient abstractZKClient; 11 | 12 | public synchronized static AbstractZKClient getAbstractZKClient(String zookeeperConnectionString, RetryPolicy retryPolicy) { 13 | if (null == abstractZKClient) { 14 | abstractZKClient = new AbstractZKClient(zookeeperConnectionString, retryPolicy); 15 | } 16 | return abstractZKClient; 17 | } 18 | 19 | public synchronized static AbstractZKClient getAbstractZKClient(String zookeeperConnectionString, Integer zookeeperRetrySleep, Integer zookeeperRetryMaxtime) { 20 | if (null == abstractZKClient) { 21 | abstractZKClient = new AbstractZKClient(zookeeperConnectionString, zookeeperRetrySleep, zookeeperRetryMaxtime); 22 | } 23 | return abstractZKClient; 24 | } 25 | 26 | public synchronized static AbstractZKClient getAbstractZKClient(String zookeeperConnectionString, Integer zookeeperRetrySleep, Integer zookeeperRetryMaxtime, Integer zookeeperSessionTimeout, Integer zookeeperConnectionTimeout) { 27 | if (null == abstractZKClient) { 28 | abstractZKClient = new AbstractZKClient(zookeeperConnectionString, zookeeperRetrySleep, zookeeperRetryMaxtime, zookeeperSessionTimeout, zookeeperConnectionTimeout); 29 | } 30 | return abstractZKClient; 31 | } 32 | } 33 | 34 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-zookeeper-tool/src/main/java/com/yongqing/zookeeper/tool/client/ZKClient.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.zookeeper.tool.client; 2 | 3 | /** 4 | * 5 | */ 6 | public interface ZKClient { 7 | void start(); 8 | 9 | void close(); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-zookeeper-tool/src/main/java/com/yongqing/zookeeper/tool/distributed/DistributedLock.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.zookeeper.tool.distributed; 2 | 3 | import com.yongqing.zookeeper.tool.exec.Exec; 4 | 5 | import java.util.concurrent.TimeUnit; 6 | 7 | /** 8 | * 9 | */ 10 | public interface DistributedLock { 11 | R distributedLockExec(String lockPath, Exec exec, T parameters); 12 | R distributedLockExec(String lockPath, Exec exec, T parameters, Long time, TimeUnit unit); 13 | void distributedLockExecNoReturn(String lockPath, Exec exec); 14 | void distributedLockExecNoReturn(String lockPath, Exec exec, T parameters); 15 | R distributedLockExec(String lockPath, Exec exec); 16 | void distributedLockExecNoReturn(String lockPath, Exec exec,Long time, TimeUnit unit); 17 | R distributedLockExec(String lockPath, Exec exec, Long time, TimeUnit unit); 18 | void distributedLockExecNoReturn(String lockPath, Exec exec, T parameters, Long time, TimeUnit unit); 19 | } 20 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-zookeeper-tool/src/main/java/com/yongqing/zookeeper/tool/exec/Exec.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.zookeeper.tool.exec; 2 | 3 | /** 4 | * 5 | */ 6 | @FunctionalInterface 7 | public interface Exec { 8 | R exec(T parameters); 9 | } 10 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-zookeeper-tool/src/main/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | /opt/logs 5 | yongqing-zookeeper-tool 6 | /opt/logs 7 | %clr{${sys:PID}} 8 | %d{yy-MM-dd HH:mm:ss} %-5level %class{1.} %L - %m%n%xwEx 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /yongqing-bigdata-tools/yongqing-zookeeper-tool/src/test/java/com/yongqing/zookeeper/tool/Test.java: -------------------------------------------------------------------------------- 1 | package com.yongqing.zookeeper.tool; 2 | 3 | /** 4 | * 5 | */ 6 | public class Test { 7 | } 8 | --------------------------------------------------------------------------------