├── .github └── workflows │ └── release.yml ├── .gitignore ├── README.md ├── adbpgwriter ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── doc │ └── adbpgwriter.md │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── adbpgwriter │ │ ├── AdbpgWriter.java │ │ ├── copy │ │ ├── Adb4pgClientProxy.java │ │ └── AdbProxy.java │ │ ├── package-info.java │ │ └── util │ │ ├── Adb4pgUtil.java │ │ ├── Constant.java │ │ └── Key.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── adswriter ├── doc │ └── adswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── adswriter │ │ ├── AdsException.java │ │ ├── AdsWriter.java │ │ ├── AdsWriterErrorCode.java │ │ ├── ads │ │ ├── ColumnDataType.java │ │ ├── ColumnInfo.java │ │ ├── TableInfo.java │ │ └── package-info.java │ │ ├── insert │ │ ├── AdsClientProxy.java │ │ ├── AdsInsertProxy.java │ │ ├── AdsInsertUtil.java │ │ ├── AdsProxy.java │ │ └── OperationType.java │ │ ├── load │ │ ├── AdsHelper.java │ │ ├── TableMetaHelper.java │ │ └── TransferProjectConf.java │ │ ├── odps │ │ ├── DataType.java │ │ ├── FieldSchema.java │ │ ├── TableMeta.java │ │ └── package-info.java │ │ ├── package-info.java │ │ └── util │ │ ├── AdsUtil.java │ │ ├── Constant.java │ │ └── Key.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── build.sh ├── cassandrareader ├── doc │ └── cassandrareader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── cassandrareader │ │ ├── CassandraReader.java │ │ ├── CassandraReaderErrorCode.java │ │ ├── CassandraReaderHelper.java │ │ ├── Key.java │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ └── LocalStrings_zh_TW.properties │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── cassandrawriter ├── doc │ └── cassandrawriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── cassandrawriter │ │ ├── CassandraWriter.java │ │ ├── CassandraWriterErrorCode.java │ │ ├── CassandraWriterHelper.java │ │ ├── Key.java │ │ ├── LocalStrings.properties │ │ ├── LocalStrings_en_US.properties │ │ ├── LocalStrings_ja_JP.properties │ │ ├── LocalStrings_zh_CN.properties │ │ ├── LocalStrings_zh_HK.properties │ │ └── LocalStrings_zh_TW.properties │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── clickhousewriter ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── clickhousewriter │ │ ├── ClickhouseWriter.java │ │ └── ClickhouseWriterErrorCode.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── common ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── alibaba │ └── datax │ └── common │ ├── base │ └── BaseObject.java │ ├── constant │ ├── CommonConstant.java │ └── PluginType.java │ ├── element │ ├── BoolColumn.java │ ├── BytesColumn.java │ ├── Column.java │ ├── ColumnCast.java │ ├── DateColumn.java │ ├── DoubleColumn.java │ ├── LongColumn.java │ ├── OverFlowUtil.java │ ├── Record.java │ └── StringColumn.java │ ├── exception │ ├── CommonErrorCode.java │ ├── DataXException.java │ └── ExceptionTracker.java │ ├── plugin │ ├── AbstractJobPlugin.java │ ├── AbstractPlugin.java │ ├── AbstractTaskPlugin.java │ ├── JobPluginCollector.java │ ├── PluginCollector.java │ ├── Pluginable.java │ ├── RecordReceiver.java │ ├── RecordSender.java │ └── TaskPluginCollector.java │ ├── spi │ ├── ErrorCode.java │ ├── Hook.java │ ├── Reader.java │ └── Writer.java │ ├── statistics │ ├── PerfRecord.java │ ├── PerfTrace.java │ └── VMInfo.java │ └── util │ ├── Configuration.java │ ├── FilterUtil.java │ ├── HostUtils.java │ ├── ListUtil.java │ ├── RangeSplitUtil.java │ ├── RetryUtil.java │ └── StrUtil.java ├── core ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── bin │ ├── datax.py │ ├── dxprof.py │ └── perftrace.py │ ├── conf │ ├── .secret.properties │ ├── core.json │ └── logback.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ ├── core │ │ ├── AbstractContainer.java │ │ ├── Engine.java │ │ ├── container │ │ │ └── util │ │ │ │ ├── HookInvoker.java │ │ │ │ └── JobAssignUtil.java │ │ ├── job │ │ │ ├── JobContainer.java │ │ │ ├── meta │ │ │ │ ├── ExecuteMode.java │ │ │ │ └── State.java │ │ │ └── scheduler │ │ │ │ ├── AbstractScheduler.java │ │ │ │ └── processinner │ │ │ │ ├── ProcessInnerScheduler.java │ │ │ │ └── StandAloneScheduler.java │ │ ├── statistics │ │ │ ├── communication │ │ │ │ ├── Communication.java │ │ │ │ ├── CommunicationTool.java │ │ │ │ └── LocalTGCommunicationManager.java │ │ │ ├── container │ │ │ │ ├── collector │ │ │ │ │ ├── AbstractCollector.java │ │ │ │ │ └── ProcessInnerCollector.java │ │ │ │ ├── communicator │ │ │ │ │ ├── AbstractContainerCommunicator.java │ │ │ │ │ ├── job │ │ │ │ │ │ └── StandAloneJobContainerCommunicator.java │ │ │ │ │ └── taskgroup │ │ │ │ │ │ ├── AbstractTGContainerCommunicator.java │ │ │ │ │ │ └── StandaloneTGContainerCommunicator.java │ │ │ │ └── report │ │ │ │ │ ├── AbstractReporter.java │ │ │ │ │ └── ProcessInnerReporter.java │ │ │ └── plugin │ │ │ │ ├── DefaultJobPluginCollector.java │ │ │ │ └── task │ │ │ │ ├── AbstractTaskPluginCollector.java │ │ │ │ ├── HttpPluginCollector.java │ │ │ │ ├── StdoutPluginCollector.java │ │ │ │ └── util │ │ │ │ └── DirtyRecord.java │ │ ├── taskgroup │ │ │ ├── TaskGroupContainer.java │ │ │ ├── TaskMonitor.java │ │ │ └── runner │ │ │ │ ├── AbstractRunner.java │ │ │ │ ├── ReaderRunner.java │ │ │ │ ├── TaskGroupContainerRunner.java │ │ │ │ └── WriterRunner.java │ │ ├── transport │ │ │ ├── channel │ │ │ │ ├── Channel.java │ │ │ │ └── memory │ │ │ │ │ └── MemoryChannel.java │ │ │ ├── exchanger │ │ │ │ ├── BufferedRecordExchanger.java │ │ │ │ ├── BufferedRecordTransformerExchanger.java │ │ │ │ ├── RecordExchanger.java │ │ │ │ └── TransformerExchanger.java │ │ │ ├── record │ │ │ │ ├── DefaultRecord.java │ │ │ │ └── TerminateRecord.java │ │ │ └── transformer │ │ │ │ ├── ComplexTransformerProxy.java │ │ │ │ ├── FilterTransformer.java │ │ │ │ ├── GroovyTransformer.java │ │ │ │ ├── GroovyTransformerStaticUtil.java │ │ │ │ ├── PadTransformer.java │ │ │ │ ├── ReplaceTransformer.java │ │ │ │ ├── SubstrTransformer.java │ │ │ │ ├── TransformerErrorCode.java │ │ │ │ ├── TransformerExecution.java │ │ │ │ ├── TransformerExecutionParas.java │ │ │ │ ├── TransformerInfo.java │ │ │ │ └── TransformerRegistry.java │ │ └── util │ │ │ ├── ClassSize.java │ │ │ ├── ClassUtil.java │ │ │ ├── ConfigParser.java │ │ │ ├── ConfigurationValidate.java │ │ │ ├── ErrorRecordChecker.java │ │ │ ├── ExceptionTracker.java │ │ │ ├── FrameworkErrorCode.java │ │ │ ├── HttpClientUtil.java │ │ │ ├── SecretUtil.java │ │ │ ├── TransformerUtil.java │ │ │ └── container │ │ │ ├── ClassLoaderSwapper.java │ │ │ ├── CoreConstant.java │ │ │ ├── JarLoader.java │ │ │ └── LoadUtil.java │ │ └── dataxservice │ │ └── face │ │ └── domain │ │ └── enums │ │ ├── EnumStrVal.java │ │ ├── EnumVal.java │ │ ├── ExecuteMode.java │ │ └── State.java │ ├── job │ └── job.json │ ├── log │ └── datax.log │ ├── script │ └── Readme.md │ └── tmp │ └── readme.txt ├── datax-opensource-dingding.png ├── dataxPluginDev.md ├── drdsreader ├── doc │ └── drdsreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── drdsreader │ │ ├── DrdsReader.java │ │ ├── DrdsReaderErrorCode.java │ │ └── DrdsReaderSplitUtil.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── drdswriter ├── doc │ └── drdswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── drdswriter │ │ └── DrdsWriter.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── elasticsearchwriter ├── README.md ├── build.sh ├── doc │ └── elasticsearchwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── elasticsearchwriter │ │ ├── ESClient.java │ │ ├── ESColumn.java │ │ ├── ESFieldType.java │ │ ├── ESWriter.java │ │ ├── ESWriterErrorCode.java │ │ └── Key.java │ └── resources │ └── plugin.json ├── ftpreader ├── doc │ └── ftpreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── ftpreader │ │ ├── Constant.java │ │ ├── FtpHelper.java │ │ ├── FtpReader.java │ │ ├── FtpReaderErrorCode.java │ │ ├── Key.java │ │ ├── SftpHelper.java │ │ └── StandardFtpHelper.java │ └── resources │ ├── plugin-template.json │ ├── plugin.json │ └── plugin_job_template.json ├── ftpwriter ├── doc │ ├── .gitkeep │ └── ftpwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── ftpwriter │ │ ├── FtpWriter.java │ │ ├── FtpWriterErrorCode.java │ │ ├── Key.java │ │ └── util │ │ ├── Constant.java │ │ ├── IFtpHelper.java │ │ ├── SftpHelperImpl.java │ │ └── StandardFtpHelperImpl.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── gdbreader ├── doc │ └── gdbreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── gdbreader │ │ ├── GdbReader.java │ │ ├── GdbReaderErrorCode.java │ │ ├── Key.java │ │ ├── mapping │ │ ├── DefaultGdbMapper.java │ │ ├── MappingRule.java │ │ ├── MappingRuleFactory.java │ │ └── ValueType.java │ │ ├── model │ │ ├── AbstractGdbGraph.java │ │ ├── GdbElement.java │ │ ├── GdbGraph.java │ │ └── ScriptGdbGraph.java │ │ └── util │ │ └── ConfigHelper.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── gdbwriter ├── doc │ └── gdbwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── gdbwriter │ │ ├── GdbWriter.java │ │ ├── GdbWriterErrorCode.java │ │ ├── Key.java │ │ ├── client │ │ ├── GdbGraphManager.java │ │ └── GdbWriterConfig.java │ │ ├── mapping │ │ ├── DefaultGdbMapper.java │ │ ├── GdbMapper.java │ │ ├── MapperConfig.java │ │ ├── MappingRule.java │ │ ├── MappingRuleFactory.java │ │ └── ValueType.java │ │ ├── model │ │ ├── AbstractGdbGraph.java │ │ ├── GdbEdge.java │ │ ├── GdbElement.java │ │ ├── GdbGraph.java │ │ ├── GdbVertex.java │ │ └── ScriptGdbGraph.java │ │ └── util │ │ ├── ConfigHelper.java │ │ └── GdbDuplicateIdException.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── hbase094xreader ├── doc │ ├── .gitkeep │ └── hbase094xreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── hbase094xreader │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── Hbase094xHelper.java │ │ ├── Hbase094xReader.java │ │ ├── Hbase094xReaderErrorCode.java │ │ ├── HbaseAbstractTask.java │ │ ├── HbaseColumnCell.java │ │ ├── Key.java │ │ ├── ModeType.java │ │ ├── MultiVersionFixedColumnTask.java │ │ ├── MultiVersionTask.java │ │ └── NormalTask.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── hbase094xwriter ├── doc │ ├── .gitkeep │ └── hbase094xwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── hbase094xwriter │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── Hbase094xHelper.java │ │ ├── Hbase094xWriter.java │ │ ├── Hbase094xWriterErrorCode.java │ │ ├── HbaseAbstractTask.java │ │ ├── Key.java │ │ ├── ModeType.java │ │ ├── NormalTask.java │ │ └── NullModeType.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── hbase11xreader ├── doc │ ├── .gitkeep │ └── hbase11xreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── hbase11xreader │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── Hbase11xHelper.java │ │ ├── Hbase11xReader.java │ │ ├── Hbase11xReaderErrorCode.java │ │ ├── HbaseAbstractTask.java │ │ ├── HbaseColumnCell.java │ │ ├── Key.java │ │ ├── ModeType.java │ │ ├── MultiVersionDynamicColumnTask.java │ │ ├── MultiVersionFixedColumnTask.java │ │ ├── MultiVersionTask.java │ │ └── NormalTask.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── hbase11xsqlreader ├── doc │ └── hbase11xsqlreader.md ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── package.xml │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── datax │ │ │ └── plugin │ │ │ └── reader │ │ │ └── hbase11xsqlreader │ │ │ ├── HadoopSerializationUtil.java │ │ │ ├── HbaseSQLHelper.java │ │ │ ├── HbaseSQLReader.java │ │ │ ├── HbaseSQLReaderConfig.java │ │ │ ├── HbaseSQLReaderErrorCode.java │ │ │ ├── HbaseSQLReaderTask.java │ │ │ ├── Key.java │ │ │ ├── LocalStrings.properties │ │ │ ├── LocalStrings_en_US.properties │ │ │ ├── LocalStrings_ja_JP.properties │ │ │ └── LocalStrings_zh_CN.properties │ └── resources │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test │ └── java │ └── com │ └── alibaba │ └── datax │ └── plugin │ └── reader │ └── hbase11xsqlreader │ ├── HbaseSQLHelperTest.java │ └── HbaseSQLReaderTaskTest.java ├── hbase11xsqlwriter ├── doc │ └── hbase11xsqlwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── hbase11xsqlwriter │ │ ├── Constant.java │ │ ├── HbaseSQLHelper.java │ │ ├── HbaseSQLWriter.java │ │ ├── HbaseSQLWriterConfig.java │ │ ├── HbaseSQLWriterErrorCode.java │ │ ├── HbaseSQLWriterTask.java │ │ ├── Key.java │ │ ├── NullModeType.java │ │ └── ThinClientPTable.java │ └── resources │ └── plugin.json ├── hbase11xwriter ├── doc │ ├── .gitkeep │ └── hbase11xwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── hbase11xwriter │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── Hbase11xHelper.java │ │ ├── Hbase11xWriter.java │ │ ├── Hbase11xWriterErrorCode.java │ │ ├── HbaseAbstractTask.java │ │ ├── Key.java │ │ ├── ModeType.java │ │ ├── MultiVersionTask.java │ │ ├── NormalTask.java │ │ └── NullModeType.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── hbase20xsqlreader ├── doc │ └── hbase20xsqlreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── hbase20xsqlreader │ │ ├── Constant.java │ │ ├── HBase20SQLReaderHelper.java │ │ ├── HBase20xSQLReader.java │ │ ├── HBase20xSQLReaderErrorCode.java │ │ ├── HBase20xSQLReaderTask.java │ │ └── Key.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── hbase20xsqlwriter ├── doc │ └── hbase20xsqlwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── hbase20xsqlwriter │ │ ├── Constant.java │ │ ├── HBase20xSQLHelper.java │ │ ├── HBase20xSQLWriter.java │ │ ├── HBase20xSQLWriterErrorCode.java │ │ ├── HBase20xSQLWriterTask.java │ │ ├── Key.java │ │ └── NullModeType.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── hdfsreader ├── doc │ └── hdfsreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── hdfsreader │ │ ├── Constant.java │ │ ├── DFSUtil.java │ │ ├── HdfsFileType.java │ │ ├── HdfsReader.java │ │ ├── HdfsReaderErrorCode.java │ │ └── Key.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── hdfswriter ├── doc │ └── hdfswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── hdfswriter │ │ ├── Constant.java │ │ ├── HdfsHelper.java │ │ ├── HdfsWriter.java │ │ ├── HdfsWriterErrorCode.java │ │ ├── Key.java │ │ └── SupportHiveDataType.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── images ├── DataX-logo.jpg ├── DataX开源用户交流群.jpg ├── DataX开源用户交流群2.jpg ├── DataX开源用户交流群3.jpg ├── DataX开源用户交流群4.jpg ├── DataX开源用户交流群5.jpg ├── datax-enterprise-users.jpg ├── datax-opensource-dingding.png ├── dingding.jpg ├── plugin_dev_guide_1.png ├── plugin_dev_guide_2.png └── plugin_dev_guide_3.png ├── introduction.md ├── kingbaseesreader ├── doc │ └── kingbaseesreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── kingbaseesreader │ │ ├── Constant.java │ │ └── KingbaseesReader.java │ ├── libs │ └── kingbase8-8.2.0.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── kingbaseeswriter ├── doc │ └── kingbaseeswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── kingbaseeswriter │ │ └── KingbaseesWriter.java │ ├── libs │ └── kingbase8-8.2.0.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── kuduwriter ├── README.md ├── doc │ ├── image-20200901193148188.png │ └── kuduwirter.md ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── package.xml │ ├── java │ │ └── com │ │ │ └── q1 │ │ │ ├── datax │ │ │ └── plugin │ │ │ │ └── writer │ │ │ │ └── kudu11xwriter │ │ │ │ ├── ColumnType.java │ │ │ │ ├── Constant.java │ │ │ │ ├── InsertModeType.java │ │ │ │ ├── Key.java │ │ │ │ ├── Kudu11xHelper.java │ │ │ │ ├── Kudu11xWriter.java │ │ │ │ ├── Kudu11xWriterErrorcode.java │ │ │ │ └── KuduWriterTask.java │ │ │ └── kudu │ │ │ └── conf │ │ │ └── KuduConfig.java │ └── resources │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test │ └── java │ └── com │ └── dai │ └── test.java ├── license.txt ├── mongodbreader ├── doc │ └── mongodbreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── mongodbreader │ │ ├── KeyConstant.java │ │ ├── MongoDBReader.java │ │ ├── MongoDBReaderErrorCode.java │ │ └── util │ │ ├── CollectionSplitUtil.java │ │ └── MongoUtil.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── mongodbwriter ├── doc │ └── mongodbwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── mongodbwriter │ │ ├── KeyConstant.java │ │ ├── MongoDBWriter.java │ │ ├── MongoDBWriterErrorCode.java │ │ └── util │ │ └── MongoUtil.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── mysqlreader ├── doc │ └── mysqlreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── mysqlreader │ │ ├── MysqlReader.java │ │ └── MysqlReaderErrorCode.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── mysqlwriter ├── doc │ └── mysqlwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── mysqlwriter │ │ └── MysqlWriter.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── ocswriter ├── doc │ └── ocswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── ocswriter │ │ ├── Key.java │ │ ├── OcsWriter.java │ │ └── utils │ │ ├── CommonUtils.java │ │ ├── ConfigurationChecker.java │ │ └── OcsWriterErrorCode.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── odpsreader ├── doc │ └── odpsreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── odpsreader │ │ ├── ColumnType.java │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── OdpsReader.java │ │ ├── OdpsReaderErrorCode.java │ │ ├── ReaderProxy.java │ │ └── util │ │ ├── DESCipher.java │ │ ├── IdAndKeyUtil.java │ │ ├── OdpsExceptionMsg.java │ │ ├── OdpsSplitUtil.java │ │ └── OdpsUtil.java │ ├── libs │ └── bcprov-jdk15on-1.52.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── odpswriter ├── doc │ └── odpswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── odpswriter │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── OdpsWriter.java │ │ ├── OdpsWriterErrorCode.java │ │ ├── OdpsWriterProxy.java │ │ └── util │ │ ├── DESCipher.java │ │ ├── IdAndKeyUtil.java │ │ ├── OdpsExceptionMsg.java │ │ └── OdpsUtil.java │ ├── libs │ └── bcprov-jdk15on-1.52.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── opentsdbreader ├── doc │ └── opentsdbreader.md ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── package.xml │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── datax │ │ │ └── plugin │ │ │ └── reader │ │ │ ├── conn │ │ │ ├── CliQuery.java │ │ │ ├── Connection4TSDB.java │ │ │ ├── DataPoint4TSDB.java │ │ │ ├── DumpSeries.java │ │ │ ├── OpenTSDBConnection.java │ │ │ └── OpenTSDBDump.java │ │ │ ├── opentsdbreader │ │ │ ├── Constant.java │ │ │ ├── Key.java │ │ │ ├── OpenTSDBReader.java │ │ │ └── OpenTSDBReaderErrorCode.java │ │ │ └── util │ │ │ ├── HttpUtils.java │ │ │ ├── TSDBUtils.java │ │ │ └── TimeUtils.java │ └── resources │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test │ └── java │ └── com │ └── alibaba │ └── datax │ └── plugin │ └── reader │ ├── conn │ └── OpenTSDBConnectionTest.java │ └── util │ ├── Const.java │ ├── HttpUtilsTest.java │ ├── TSDBTest.java │ └── TimeUtilsTest.java ├── oraclereader ├── doc │ └── oraclereader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── oraclereader │ │ ├── Constant.java │ │ ├── OracleReader.java │ │ └── OracleReaderErrorCode.java │ ├── lib │ └── ojdbc6-11.2.0.3.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── oraclewriter ├── doc │ └── oraclewriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── oraclewriter │ │ ├── OracleWriter.java │ │ └── OracleWriterErrorCode.java │ ├── lib │ └── ojdbc6-11.2.0.3.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── oscarwriter ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── oscarwriter │ │ ├── OscarWriter.java │ │ └── OscarWriterErrorCode.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── ossreader ├── doc │ └── ossreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── ossreader │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── OssReader.java │ │ ├── OssReaderErrorCode.java │ │ └── util │ │ └── OssUtil.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── osswriter ├── doc │ └── osswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── osswriter │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── OssWriter.java │ │ ├── OssWriterErrorCode.java │ │ └── util │ │ └── OssUtil.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── otsreader ├── doc │ └── otsreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── otsreader │ │ ├── Key.java │ │ ├── OtsReader.java │ │ ├── OtsReaderError.java │ │ ├── OtsReaderMasterProxy.java │ │ ├── OtsReaderSlaveProxy.java │ │ ├── adaptor │ │ ├── OTSColumnAdaptor.java │ │ └── PrimaryKeyValueAdaptor.java │ │ ├── callable │ │ ├── GetFirstRowPrimaryKeyCallable.java │ │ ├── GetRangeCallable.java │ │ └── GetTableMetaCallable.java │ │ ├── model │ │ ├── OTSColumn.java │ │ ├── OTSConf.java │ │ ├── OTSConst.java │ │ ├── OTSPrimaryKeyColumn.java │ │ └── OTSRange.java │ │ └── utils │ │ ├── Common.java │ │ ├── DefaultNoRetry.java │ │ ├── GsonParser.java │ │ ├── ParamChecker.java │ │ ├── RangeSplit.java │ │ ├── ReaderModelParser.java │ │ └── RetryHelper.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── otsstreamreader ├── README.md ├── pom.xml ├── src │ └── main │ │ ├── assembly │ │ └── package.xml │ │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── datax │ │ │ └── plugin │ │ │ └── reader │ │ │ └── otsstreamreader │ │ │ └── internal │ │ │ ├── OTSReaderError.java │ │ │ ├── OTSStreamReader.java │ │ │ ├── OTSStreamReaderException.java │ │ │ ├── OTSStreamReaderMasterProxy.java │ │ │ ├── OTSStreamReaderSlaveProxy.java │ │ │ ├── config │ │ │ ├── Mode.java │ │ │ ├── OTSRetryStrategyForStreamReader.java │ │ │ ├── OTSStreamReaderConfig.java │ │ │ ├── OTSStreamReaderConstants.java │ │ │ └── StatusTableConstants.java │ │ │ ├── core │ │ │ ├── CheckpointTimeTracker.java │ │ │ ├── IStreamRecordSender.java │ │ │ ├── MultiVerModeRecordSender.java │ │ │ ├── OTSStreamReaderChecker.java │ │ │ ├── RecordProcessor.java │ │ │ ├── ShardStatusChecker.java │ │ │ └── SingleVerAndUpOnlyModeRecordSender.java │ │ │ ├── model │ │ │ ├── OTSErrorCode.java │ │ │ ├── ShardCheckpoint.java │ │ │ └── StreamJob.java │ │ │ └── utils │ │ │ ├── ColumnValueTransformHelper.java │ │ │ ├── GsonParser.java │ │ │ ├── OTSHelper.java │ │ │ ├── ParamChecker.java │ │ │ └── TimeUtils.java │ │ └── resources │ │ ├── log4j2.xml │ │ └── plugin.json └── tools │ ├── config.json │ ├── tablestore_streamreader_console.py │ └── tabulate.py ├── otswriter ├── doc │ └── otswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── otswriter │ │ ├── Key.java │ │ ├── OtsWriter.java │ │ ├── OtsWriterError.java │ │ ├── OtsWriterMasterProxy.java │ │ ├── OtsWriterSlaveProxy.java │ │ ├── WriterRetryPolicy.java │ │ ├── callable │ │ └── GetTableMetaCallable.java │ │ ├── model │ │ ├── LogExceptionManager.java │ │ ├── OTSAttrColumn.java │ │ ├── OTSConf.java │ │ ├── OTSConst.java │ │ ├── OTSErrorMessage.java │ │ ├── OTSOpType.java │ │ ├── OTSPKColumn.java │ │ ├── OTSRowPrimaryKey.java │ │ ├── RowDeleteChangeWithRecord.java │ │ ├── RowPutChangeWithRecord.java │ │ ├── RowUpdateChangeWithRecord.java │ │ └── WithRecord.java │ │ └── utils │ │ ├── ColumnConversion.java │ │ ├── Common.java │ │ ├── GsonParser.java │ │ ├── ParamChecker.java │ │ ├── RetryHelper.java │ │ └── WriterModelParser.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── package.xml ├── plugin-rdbms-util ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── alibaba │ └── datax │ └── plugin │ └── rdbms │ ├── reader │ ├── CommonRdbmsReader.java │ ├── Constant.java │ ├── Key.java │ ├── ResultSetReadProxy.java │ └── util │ │ ├── HintUtil.java │ │ ├── OriginalConfPretreatmentUtil.java │ │ ├── PreCheckTask.java │ │ ├── ReaderSplitUtil.java │ │ └── SingleTableSplitUtil.java │ ├── util │ ├── ConnectionFactory.java │ ├── Constant.java │ ├── DBUtil.java │ ├── DBUtilErrorCode.java │ ├── DataBaseType.java │ ├── JdbcConnectionFactory.java │ ├── RdbmsException.java │ ├── RdbmsRangeSplitWrap.java │ └── TableExpandUtil.java │ └── writer │ ├── CommonRdbmsWriter.java │ ├── Constant.java │ ├── Key.java │ ├── MysqlWriterErrorCode.java │ └── util │ ├── OriginalConfPretreatmentUtil.java │ └── WriterUtil.java ├── plugin-unstructured-storage-util ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── alibaba │ └── datax │ └── plugin │ └── unstructuredstorage │ ├── reader │ ├── ColumnEntry.java │ ├── Constant.java │ ├── ExpandLzopInputStream.java │ ├── Key.java │ ├── UnstructuredStorageReaderErrorCode.java │ ├── UnstructuredStorageReaderUtil.java │ └── ZipCycleInputStream.java │ └── writer │ ├── Constant.java │ ├── Key.java │ ├── TextCsvWriterManager.java │ ├── UnstructuredStorageWriterErrorCode.java │ ├── UnstructuredStorageWriterUtil.java │ └── UnstructuredWriter.java ├── pom.xml ├── postgresqlreader ├── doc │ └── postgresqlreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── postgresqlreader │ │ ├── Constant.java │ │ └── PostgresqlReader.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── postgresqlwriter ├── doc │ └── postgresqlwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── postgresqlwriter │ │ └── PostgresqlWriter.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── rdbmsreader ├── doc │ └── rdbmsreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── rdbmsreader │ │ ├── Constant.java │ │ ├── RdbmsReader.java │ │ └── SubCommonRdbmsReader.java │ ├── libs │ ├── Dm7JdbcDriver16.jar │ ├── db2jcc4.jar │ ├── edb-jdbc16.jar │ └── jconn3-1.0.0-SNAPSHOT.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── rdbmswriter ├── doc │ └── rdbmswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── rdbmswriter │ │ ├── RdbmsWriter.java │ │ └── SubCommonRdbmsWriter.java │ ├── libs │ ├── Dm7JdbcDriver16.jar │ ├── db2jcc4.jar │ ├── edb-jdbc16.jar │ └── jconn3-1.0.0-SNAPSHOT.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── rpm ├── t_dp_dw_datax_3_core_all-build.sh ├── t_dp_dw_datax_3_core_all.spec ├── t_dp_dw_datax_3_hook_dqc-build.sh └── t_dp_dw_datax_3_hook_dqc.spec ├── sqlserverreader ├── doc │ └── sqlserverreader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── sqlserverreader │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── SqlServerReader.java │ │ └── SqlServerReaderErrorCode.java │ ├── lib │ └── sqljdbc4-4.0.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── sqlserverwriter ├── doc │ └── sqlserverwriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── sqlserverwriter │ │ ├── SqlServerWriter.java │ │ └── SqlServerWriterErrorCode.java │ ├── lib │ └── sqljdbc4-4.0.jar │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── starrockswriter ├── doc │ └── starrockswriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── starrocks │ │ └── connector │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── starrockswriter │ │ ├── StarRocksWriter.java │ │ ├── StarRocksWriterOptions.java │ │ ├── manager │ │ ├── StarRocksFlushTuple.java │ │ ├── StarRocksStreamLoadFailedException.java │ │ ├── StarRocksStreamLoadVisitor.java │ │ └── StarRocksWriterManager.java │ │ ├── row │ │ ├── StarRocksBaseSerializer.java │ │ ├── StarRocksCsvSerializer.java │ │ ├── StarRocksDelimiterParser.java │ │ ├── StarRocksISerializer.java │ │ ├── StarRocksJsonSerializer.java │ │ └── StarRocksSerializerFactory.java │ │ └── util │ │ └── StarRocksWriterUtil.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── streamreader ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── streamreader │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── StreamReader.java │ │ └── StreamReaderErrorCode.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── streamwriter ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── streamwriter │ │ ├── Key.java │ │ ├── StreamWriter.java │ │ └── StreamWriterErrorCode.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── transformer ├── doc │ ├── .gitkeep │ └── transformer.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ └── java │ └── com │ └── alibaba │ └── datax │ └── transformer │ ├── ComplexTransformer.java │ └── Transformer.java ├── tsdbreader ├── doc │ └── tsdbreader.md ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── package.xml │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── datax │ │ │ └── plugin │ │ │ └── reader │ │ │ └── tsdbreader │ │ │ ├── Constant.java │ │ │ ├── Key.java │ │ │ ├── TSDBReader.java │ │ │ ├── TSDBReaderErrorCode.java │ │ │ ├── conn │ │ │ ├── Connection4TSDB.java │ │ │ ├── DataPoint4MultiFieldsTSDB.java │ │ │ ├── DataPoint4TSDB.java │ │ │ ├── MultiFieldQueryResult.java │ │ │ ├── QueryResult.java │ │ │ ├── TSDBConnection.java │ │ │ └── TSDBDump.java │ │ │ └── util │ │ │ ├── HttpUtils.java │ │ │ ├── TSDBUtils.java │ │ │ └── TimeUtils.java │ └── resources │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test │ └── java │ └── com │ └── alibaba │ └── datax │ └── plugin │ └── reader │ └── tsdbreader │ ├── conn │ └── TSDBConnectionTest.java │ └── util │ ├── Const.java │ ├── HttpUtilsTest.java │ └── TimeUtilsTest.java ├── tsdbwriter ├── doc │ └── tsdbhttpwriter.md ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── package.xml │ ├── java │ │ └── com │ │ │ └── alibaba │ │ │ └── datax │ │ │ └── plugin │ │ │ └── writer │ │ │ ├── conn │ │ │ ├── Connection4TSDB.java │ │ │ ├── DataPoint4TSDB.java │ │ │ └── TSDBConnection.java │ │ │ ├── tsdbwriter │ │ │ ├── Constant.java │ │ │ ├── Key.java │ │ │ ├── TSDBWriter.java │ │ │ └── TSDBWriterErrorCode.java │ │ │ └── util │ │ │ ├── HttpUtils.java │ │ │ └── TSDBUtils.java │ └── resources │ │ ├── plugin.json │ │ └── plugin_job_template.json │ └── test │ └── java │ └── com │ └── alibaba │ └── datax │ └── plugin │ └── writer │ ├── conn │ └── TSDBConnectionTest.java │ └── util │ ├── Const.java │ ├── HttpUtilsTest.java │ └── TSDBTest.java ├── txtfilereader ├── doc │ └── txtfilereader.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── reader │ │ └── txtfilereader │ │ ├── Constant.java │ │ ├── Key.java │ │ ├── TxtFileReader.java │ │ └── TxtFileReaderErrorCode.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json ├── txtfilewriter ├── doc │ └── txtfilewriter.md ├── pom.xml └── src │ └── main │ ├── assembly │ └── package.xml │ ├── java │ └── com │ │ └── alibaba │ │ └── datax │ │ └── plugin │ │ └── writer │ │ └── txtfilewriter │ │ ├── Key.java │ │ ├── TxtFileWriter.java │ │ └── TxtFileWriterErrorCode.java │ └── resources │ ├── plugin.json │ └── plugin_job_template.json └── userGuid.md /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*.*.*" 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Set up Maven Central Repository 13 | uses: actions/setup-java@v2 14 | with: 15 | java-version: '8' 16 | distribution: 'adopt' 17 | - name: Checkout 18 | uses: actions/checkout@v2 19 | with: 20 | fetch-depth: 0 21 | - run: | 22 | git checkout main && git pull origin main 23 | rm -rf target 24 | mvn -U -pl starrockswriter -am clean package assembly:assembly -Dmaven.test.skip=true 25 | cd target/datax/datax/plugin/writer/ 26 | tar -czvf starrockswriter.tar.gz starrockswriter 27 | - name: Release 28 | uses: softprops/action-gh-release@v1 29 | with: 30 | files: | 31 | target/datax/datax/plugin/writer/starrockswriter.tar.gz 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StarRocks writer for DataX 2 | 3 | This is a repo forked from [DataX](https://github.com/alibaba/DataX), and maintained by [StarRocks](https://www.starrocks.com) (starrockswriter). 4 | 5 | ## How to use 6 | 7 | 1. Run `./build.sh` to gennerate the `starrockswriter.tar.gz`, then untar it into your own [DataX release](https://github.com/alibaba/DataX) directory(which will be `datax/plugin/writer/`). 8 | 2. Create a `job.json` to define the reader and writer. More details about the configurations, please refer to the [Documentations](https://docs.starrocks.com/en-us/main/loading/DataX-starrocks-writer). 9 | 3. Run `python datax/bin/datax.py --jvm="-Xms6G -Xmx6G" --loglevel=debug job.json` to start a job. 10 | -------------------------------------------------------------------------------- /adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/copy/AdbProxy.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.adbpgwriter.copy; 2 | 3 | import com.alibaba.datax.common.plugin.RecordReceiver; 4 | 5 | import java.sql.Connection; 6 | /** 7 | * @author yuncheng 8 | */ 9 | public interface AdbProxy { 10 | public abstract void startWriteWithConnection(RecordReceiver recordReceiver, Connection connection); 11 | 12 | public void closeResource(); 13 | } 14 | -------------------------------------------------------------------------------- /adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Greenplum Writer. 3 | * 4 | * @since 0.0.1 5 | */ 6 | package com.alibaba.datax.plugin.writer.adbpgwriter; 7 | 8 | 9 | -------------------------------------------------------------------------------- /adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.adbpgwriter.util; 2 | /** 3 | * @author yuncheng 4 | */ 5 | public class Constant { 6 | public static final int DEFAULT_RETRY_TIMES = 3; 7 | 8 | public static final String COLUMN_QUOTE_CHARACTER = "\""; 9 | 10 | 11 | 12 | } 13 | -------------------------------------------------------------------------------- /adbpgwriter/src/main/java/com/alibaba/datax/plugin/writer/adbpgwriter/util/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.adbpgwriter.util; 2 | /** 3 | * @author yuncheng 4 | */ 5 | public class Key { 6 | 7 | public final static String COLUMN = "column"; 8 | public final static String IS_INSERTINGORE = "insertIgnore"; 9 | public final static String HOST = "host"; 10 | public final static String PORT = "port"; 11 | public final static String DATABASE = "database"; 12 | public final static String SCHEMA = "schema"; 13 | public final static String EMPTY_AS_NULL = "emptyAsNull"; 14 | 15 | public final static String IGNORE_INSERT = "ignoreInsert"; 16 | 17 | public final static String RETRY_CONNECTION_TIME = "retryTimes"; 18 | 19 | public final static String RETRY_INTERVAL_TIME = "retryIntervalTime"; 20 | 21 | public final static String COMMIT_SIZE = "commitSize"; 22 | 23 | public final static String PARALLEL_NUMBER = "parallelNumber"; 24 | 25 | public final static String SHARED_DATASOURCE = "shareDataSource"; 26 | } 27 | -------------------------------------------------------------------------------- /adbpgwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adbpgwriter", 3 | "class": "com.alibaba.datax.plugin.writer.adbpgwriter.AdbpgWriter", 4 | "description": "", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /adbpgwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adbpgwriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "host": "", 7 | "port": "", 8 | "database": "", 9 | "schema": "", 10 | "table": "", 11 | "column": ["*"] 12 | } 13 | } -------------------------------------------------------------------------------- /adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/ads/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ADS meta and service. 3 | * 4 | * @since 0.0.1 5 | */ 6 | package com.alibaba.datax.plugin.writer.adswriter.ads; -------------------------------------------------------------------------------- /adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/insert/AdsProxy.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.adswriter.insert; 2 | 3 | import com.alibaba.datax.common.plugin.RecordReceiver; 4 | 5 | import java.sql.Connection; 6 | 7 | public interface AdsProxy { 8 | public abstract void startWriteWithConnection(RecordReceiver recordReceiver, Connection connection, 9 | int columnNumber); 10 | 11 | public void closeResource(); 12 | } 13 | -------------------------------------------------------------------------------- /adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/odps/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ODPS meta. 3 | * 4 | * @since 0.0.1 5 | */ 6 | package com.alibaba.datax.plugin.writer.adswriter.odps; -------------------------------------------------------------------------------- /adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * ADS Writer. 3 | * 4 | * @since 0.0.1 5 | */ 6 | package com.alibaba.datax.plugin.writer.adswriter; -------------------------------------------------------------------------------- /adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/util/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.adswriter.util; 2 | 3 | public class Constant { 4 | 5 | public static final String LOADMODE = "load"; 6 | 7 | public static final String INSERTMODE = "insert"; 8 | 9 | public static final String DELETEMODE = "delete"; 10 | 11 | public static final String REPLACEMODE = "replace"; 12 | 13 | public static final String STREAMMODE = "stream"; 14 | 15 | public static final int DEFAULT_BATCH_SIZE = 32; 16 | 17 | public static final long DEFAULT_SOCKET_TIMEOUT = 3600000L; 18 | 19 | public static final int DEFAULT_RETRY_TIMES = 3; 20 | 21 | public static final String INSERT_TEMPLATE = "insert into %s ( %s ) values "; 22 | 23 | public static final String DELETE_TEMPLATE = "delete from %s where "; 24 | 25 | public static final String ADS_TABLE_INFO = "adsTableInfo"; 26 | 27 | public static final String ADS_QUOTE_CHARACTER = "`"; 28 | 29 | } 30 | -------------------------------------------------------------------------------- /adswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adswriter", 3 | "class": "com.alibaba.datax.plugin.writer.adswriter.AdsWriter", 4 | "description": "", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /adswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adswriter", 3 | "parameter": { 4 | "url": "", 5 | "username": "", 6 | "password": "", 7 | "schema": "", 8 | "table": "", 9 | "partition": "", 10 | "overWrite": "", 11 | "lifeCycle": 2 12 | } 13 | } -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | ShellDir="$( cd "$( dirname "$0" )" && pwd )" 2 | cd $ShellDir 3 | rm -rf target 4 | mvn -U -pl starrockswriter -am clean package assembly:assembly -Dmaven.test.skip=true 5 | rm -f starrockswriter.tar.gz 6 | cd target/datax/datax/plugin/writer/ 7 | tar -czvf starrockswriter.tar.gz starrockswriter 8 | mv starrockswriter.tar.gz $ShellDir 9 | -------------------------------------------------------------------------------- /cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/CassandraReaderErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.cassandrareader; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum CassandraReaderErrorCode implements ErrorCode { 6 | CONF_ERROR("CassandraReader-00", "配置错误."), 7 | ; 8 | 9 | private final String code; 10 | private final String description; 11 | 12 | private CassandraReaderErrorCode(String code, String description) { 13 | this.code = code; 14 | this.description = description; 15 | } 16 | 17 | @Override 18 | public String getCode() { 19 | return this.code; 20 | } 21 | 22 | @Override 23 | public String getDescription() { 24 | return this.description; 25 | } 26 | 27 | @Override 28 | public String toString() { 29 | return String.format("Code:[%s], Description:[%s]. ", this.code, 30 | this.description); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF -------------------------------------------------------------------------------- /cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_en_US.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_en_US.properties -------------------------------------------------------------------------------- /cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_ja_JP.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF -------------------------------------------------------------------------------- /cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_CN.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF -------------------------------------------------------------------------------- /cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_HK.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF -------------------------------------------------------------------------------- /cassandrareader/src/main/java/com/alibaba/datax/plugin/reader/cassandrareader/LocalStrings_zh_TW.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF -------------------------------------------------------------------------------- /cassandrareader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cassandrareader", 3 | "class": "com.alibaba.datax.plugin.reader.cassandrareader.CassandraReader", 4 | "description": "useScene: prod. mechanism: execute select cql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /cassandrareader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cassandrareader", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "host": "", 7 | "port": "", 8 | "useSSL": false, 9 | "keyspace": "", 10 | "table": "", 11 | "column": [ 12 | "c1","c2","c3" 13 | ] 14 | } 15 | } -------------------------------------------------------------------------------- /cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/CassandraWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.cassandrawriter; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | /** 6 | * Created by mazhenlin on 2019/8/19. 7 | */ 8 | public enum CassandraWriterErrorCode implements ErrorCode { 9 | CONF_ERROR("CassandraWriter-00", "配置错误."), 10 | WRITE_DATA_ERROR("CassandraWriter-01", "写入数据时失败."), 11 | ; 12 | 13 | private final String code; 14 | private final String description; 15 | 16 | private CassandraWriterErrorCode(String code, String description) { 17 | this.code = code; 18 | this.description = description; 19 | } 20 | 21 | @Override 22 | public String getCode() { 23 | return this.code; 24 | } 25 | 26 | @Override 27 | public String getDescription() { 28 | return this.description; 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return String.format("Code:[%s], Description:[%s].", this.code, this.description); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. 2 | errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 -------------------------------------------------------------------------------- /cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_en_US.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=Error in parameter configuration. 2 | errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 -------------------------------------------------------------------------------- /cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_ja_JP.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. 2 | errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 -------------------------------------------------------------------------------- /cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_CN.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. 2 | errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 -------------------------------------------------------------------------------- /cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_HK.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. 2 | errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 -------------------------------------------------------------------------------- /cassandrawriter/src/main/java/com/alibaba/datax/plugin/writer/cassandrawriter/LocalStrings_zh_TW.properties: -------------------------------------------------------------------------------- 1 | errorcode.config_invalid_exception=\u914D\u7F6E\u9519\u8BEF. 2 | errorcode.write_failed_exception=\u5199\u5165\u6570\u636E\u65F6\u5931\u8D25 -------------------------------------------------------------------------------- /cassandrawriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cassandrawriter", 3 | "class": "com.alibaba.datax.plugin.writer.cassandrawriter.CassandraWriter", 4 | "description": "useScene: prod. mechanism: use datax driver, execute insert sql.", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /cassandrawriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cassandrawriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "host": "", 7 | "port": "", 8 | "useSSL": false, 9 | "keyspace": "", 10 | "table": "", 11 | "column": [ 12 | "c1","c2","c3" 13 | ] 14 | } 15 | } -------------------------------------------------------------------------------- /clickhousewriter/src/main/java/com/alibaba/datax/plugin/writer/clickhousewriter/ClickhouseWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.clickhousewriter; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum ClickhouseWriterErrorCode implements ErrorCode { 6 | TUPLE_NOT_SUPPORTED_ERROR("ClickhouseWriter-00", "不支持TUPLE类型导入."), 7 | ; 8 | 9 | private final String code; 10 | private final String description; 11 | 12 | private ClickhouseWriterErrorCode(String code, String description) { 13 | this.code = code; 14 | this.description = description; 15 | } 16 | 17 | @Override 18 | public String getCode() { 19 | return this.code; 20 | } 21 | 22 | @Override 23 | public String getDescription() { 24 | return this.description; 25 | } 26 | 27 | @Override 28 | public String toString() { 29 | return String.format("Code:[%s], Description:[%s].", this.code, this.description); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /clickhousewriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "clickhousewriter", 3 | "class": "com.alibaba.datax.plugin.writer.clickhousewriter.ClickhouseWriter", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql.", 5 | "developer": "jiye.tjy" 6 | } -------------------------------------------------------------------------------- /clickhousewriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "clickhousewriter", 3 | "parameter": { 4 | "username": "username", 5 | "password": "password", 6 | "column": ["col1", "col2", "col3"], 7 | "connection": [ 8 | { 9 | "jdbcUrl": "jdbc:clickhouse://:[/]", 10 | "table": ["table1", "table2"] 11 | } 12 | ], 13 | "preSql": [], 14 | "postSql": [], 15 | 16 | "batchSize": 65536, 17 | "batchByteSize": 134217728, 18 | "dryRun": false, 19 | "writeMode": "insert" 20 | } 21 | } -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/base/BaseObject.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.base; 2 | 3 | import org.apache.commons.lang3.builder.EqualsBuilder; 4 | import org.apache.commons.lang3.builder.HashCodeBuilder; 5 | import org.apache.commons.lang3.builder.ToStringBuilder; 6 | import org.apache.commons.lang3.builder.ToStringStyle; 7 | 8 | public class BaseObject { 9 | 10 | @Override 11 | public int hashCode() { 12 | return HashCodeBuilder.reflectionHashCode(this, false); 13 | } 14 | 15 | @Override 16 | public boolean equals(Object object) { 17 | return EqualsBuilder.reflectionEquals(this, object, false); 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return ToStringBuilder.reflectionToString(this, 23 | ToStringStyle.MULTI_LINE_STYLE); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/constant/CommonConstant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.constant; 2 | 3 | public final class CommonConstant { 4 | /** 5 | * 用于插件对自身 split 的每个 task 标识其使用的资源，以告知core 对 reader/writer split 之后的 task 进行拼接时需要根据资源标签进行更有意义的 shuffle 操作 6 | */ 7 | public static String LOAD_BALANCE_RESOURCE_MARK = "loadBalanceResourceMark"; 8 | 9 | } 10 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/constant/PluginType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.constant; 2 | 3 | /** 4 | * Created by jingxing on 14-8-31. 5 | */ 6 | public enum PluginType { 7 | //pluginType还代表了资源目录，很难扩展,或者说需要足够必要才扩展。先mark Handler（其实和transformer一样），再讨论 8 | READER("reader"), TRANSFORMER("transformer"), WRITER("writer"), HANDLER("handler"); 9 | 10 | private String pluginType; 11 | 12 | private PluginType(String pluginType) { 13 | this.pluginType = pluginType; 14 | } 15 | 16 | @Override 17 | public String toString() { 18 | return this.pluginType; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/element/Record.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.element; 2 | 3 | /** 4 | * Created by jingxing on 14-8-24. 5 | */ 6 | 7 | public interface Record { 8 | 9 | public void addColumn(Column column); 10 | 11 | public void setColumn(int i, final Column column); 12 | 13 | public Column getColumn(int i); 14 | 15 | public String toString(); 16 | 17 | public int getColumnNumber(); 18 | 19 | public int getByteSize(); 20 | 21 | public int getMemorySize(); 22 | 23 | } 24 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/exception/ExceptionTracker.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.exception; 2 | 3 | import java.io.PrintWriter; 4 | import java.io.StringWriter; 5 | 6 | public final class ExceptionTracker { 7 | public static final int STRING_BUFFER = 1024; 8 | 9 | public static String trace(Throwable ex) { 10 | StringWriter sw = new StringWriter(STRING_BUFFER); 11 | PrintWriter pw = new PrintWriter(sw); 12 | ex.printStackTrace(pw); 13 | return sw.toString(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/plugin/AbstractJobPlugin.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.plugin; 2 | 3 | /** 4 | * Created by jingxing on 14-8-24. 5 | */ 6 | public abstract class AbstractJobPlugin extends AbstractPlugin { 7 | /** 8 | * @return the jobPluginCollector 9 | */ 10 | public JobPluginCollector getJobPluginCollector() { 11 | return jobPluginCollector; 12 | } 13 | 14 | /** 15 | * @param jobPluginCollector 16 | * the jobPluginCollector to set 17 | */ 18 | public void setJobPluginCollector( 19 | JobPluginCollector jobPluginCollector) { 20 | this.jobPluginCollector = jobPluginCollector; 21 | } 22 | 23 | private JobPluginCollector jobPluginCollector; 24 | 25 | } 26 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/plugin/AbstractTaskPlugin.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.plugin; 2 | 3 | /** 4 | * Created by jingxing on 14-8-24. 5 | */ 6 | public abstract class AbstractTaskPlugin extends AbstractPlugin { 7 | 8 | //TaskPlugin 应该具备taskId 9 | private int taskGroupId; 10 | private int taskId; 11 | private TaskPluginCollector taskPluginCollector; 12 | 13 | public TaskPluginCollector getTaskPluginCollector() { 14 | return taskPluginCollector; 15 | } 16 | 17 | public void setTaskPluginCollector( 18 | TaskPluginCollector taskPluginCollector) { 19 | this.taskPluginCollector = taskPluginCollector; 20 | } 21 | 22 | public int getTaskId() { 23 | return taskId; 24 | } 25 | 26 | public void setTaskId(int taskId) { 27 | this.taskId = taskId; 28 | } 29 | 30 | public int getTaskGroupId() { 31 | return taskGroupId; 32 | } 33 | 34 | public void setTaskGroupId(int taskGroupId) { 35 | this.taskGroupId = taskGroupId; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/plugin/JobPluginCollector.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.plugin; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | /** 7 | * Created by jingxing on 14-9-9. 8 | */ 9 | public interface JobPluginCollector extends PluginCollector { 10 | 11 | /** 12 | * 从Task获取自定义收集信息 13 | * 14 | * */ 15 | Map> getMessage(); 16 | 17 | /** 18 | * 从Task获取自定义收集信息 19 | * 20 | * */ 21 | List getMessage(String key); 22 | } 23 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/plugin/PluginCollector.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.plugin; 2 | 3 | 4 | /** 5 | * 这里只是一个标示类 6 | * */ 7 | public interface PluginCollector { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/plugin/Pluginable.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.plugin; 2 | 3 | import com.alibaba.datax.common.util.Configuration; 4 | 5 | public interface Pluginable { 6 | String getDeveloper(); 7 | 8 | String getDescription(); 9 | 10 | void setPluginConf(Configuration pluginConf); 11 | 12 | void init(); 13 | 14 | void destroy(); 15 | 16 | String getPluginName(); 17 | 18 | Configuration getPluginJobConf(); 19 | 20 | Configuration getPeerPluginJobConf(); 21 | 22 | public String getPeerPluginName(); 23 | 24 | void setPluginJobConf(Configuration jobConf); 25 | 26 | void setPeerPluginJobConf(Configuration peerPluginJobConf); 27 | 28 | public void setPeerPluginName(String peerPluginName); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/plugin/RecordReceiver.java: -------------------------------------------------------------------------------- 1 | /** 2 | * (C) 2010-2013 Alibaba Group Holding Limited. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alibaba.datax.common.plugin; 18 | 19 | import com.alibaba.datax.common.element.Record; 20 | 21 | public interface RecordReceiver { 22 | 23 | public Record getFromReader(); 24 | 25 | public void shutdown(); 26 | } 27 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/plugin/RecordSender.java: -------------------------------------------------------------------------------- 1 | /** 2 | * (C) 2010-2013 Alibaba Group Holding Limited. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.alibaba.datax.common.plugin; 18 | 19 | import com.alibaba.datax.common.element.Record; 20 | 21 | public interface RecordSender { 22 | 23 | public Record createRecord(); 24 | 25 | public void sendToWriter(Record record); 26 | 27 | public void flush(); 28 | 29 | public void terminate(); 30 | 31 | public void shutdown(); 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/spi/ErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.spi; 2 | 3 | /** 4 | * 尤其注意：最好提供toString()实现。例如： 5 | * 6 | * 
 7 |  * 
 8 |  * @Override
 9 |  * public String toString() {
10 |  * 	return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
11 |  * }
12 |  *
13 | * 14 | */ 15 | public interface ErrorCode { 16 | // 错误码编号 17 | String getCode(); 18 | 19 | // 错误码描述 20 | String getDescription(); 21 | 22 | /** 必须提供toString的实现 23 | * 24 | * 
25 | 	 * @Override
26 | 	 * public String toString() {
27 | 	 * 	return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
28 | 	 * }
29 | 	 *
30 | * 31 | */ 32 | String toString(); 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/com/alibaba/datax/common/spi/Hook.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.common.spi; 2 | 3 | import com.alibaba.datax.common.util.Configuration; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * Created by xiafei.qiuxf on 14/12/17. 9 | */ 10 | public interface Hook { 11 | 12 | /** 13 | * 返回名字 14 | * 15 | * @return 16 | */ 17 | public String getName(); 18 | 19 | /** 20 | * TODO 文档 21 | * 22 | * @param jobConf 23 | * @param msg 24 | */ 25 | public void invoke(Configuration jobConf, Map msg); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/conf/.secret.properties: -------------------------------------------------------------------------------- 1 | #ds basicAuth config 2 | auth.user= 3 | auth.pass= 4 | current.keyVersion= 5 | current.publicKey= 6 | current.privateKey= 7 | current.service.username= 8 | current.service.password= 9 | 10 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/job/meta/ExecuteMode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.job.meta; 2 | 3 | /** 4 | * Created by liupeng on 15/12/21. 5 | */ 6 | public enum ExecuteMode { 7 | STANDALONE("standalone"), ; 8 | 9 | String value; 10 | 11 | private ExecuteMode(String value) { 12 | this.value = value; 13 | } 14 | 15 | public String value() { 16 | return this.value; 17 | } 18 | 19 | public String getValue() { 20 | return this.value; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/job/meta/State.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.job.meta; 2 | 3 | /** 4 | * Created by liupeng on 15/12/21. 5 | */ 6 | public enum State { 7 | SUBMITTING(10), 8 | WAITING(20), 9 | RUNNING(30), 10 | KILLING(40), 11 | KILLED(50), 12 | FAILED(60), 13 | SUCCEEDED(70), ; 14 | 15 | int value; 16 | 17 | private State(int value) { 18 | this.value = value; 19 | } 20 | 21 | public int value() { 22 | return this.value; 23 | } 24 | 25 | public boolean isFinished() { 26 | return this == KILLED || this == FAILED || this == SUCCEEDED; 27 | } 28 | 29 | public boolean isRunning() { 30 | return !this.isFinished(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/job/scheduler/processinner/StandAloneScheduler.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.job.scheduler.processinner; 2 | 3 | import com.alibaba.datax.core.statistics.container.communicator.AbstractContainerCommunicator; 4 | 5 | /** 6 | * Created by hongjiao.hj on 2014/12/22. 7 | */ 8 | public class StandAloneScheduler extends ProcessInnerScheduler{ 9 | 10 | public StandAloneScheduler(AbstractContainerCommunicator containerCommunicator) { 11 | super(containerCommunicator); 12 | } 13 | 14 | @Override 15 | protected boolean isJobKilling(Long jobId) { 16 | return false; 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/statistics/container/collector/ProcessInnerCollector.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.statistics.container.collector; 2 | 3 | import com.alibaba.datax.core.statistics.communication.Communication; 4 | import com.alibaba.datax.core.statistics.communication.LocalTGCommunicationManager; 5 | 6 | public class ProcessInnerCollector extends AbstractCollector { 7 | 8 | public ProcessInnerCollector(Long jobId) { 9 | super.setJobId(jobId); 10 | } 11 | 12 | @Override 13 | public Communication collectFromTaskGroup() { 14 | return LocalTGCommunicationManager.getJobCommunication(); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/statistics/container/communicator/taskgroup/StandaloneTGContainerCommunicator.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.statistics.container.communicator.taskgroup; 2 | 3 | import com.alibaba.datax.common.util.Configuration; 4 | import com.alibaba.datax.core.statistics.container.report.ProcessInnerReporter; 5 | import com.alibaba.datax.core.statistics.communication.Communication; 6 | 7 | public class StandaloneTGContainerCommunicator extends AbstractTGContainerCommunicator { 8 | 9 | public StandaloneTGContainerCommunicator(Configuration configuration) { 10 | super(configuration); 11 | super.setReporter(new ProcessInnerReporter()); 12 | } 13 | 14 | @Override 15 | public void report(Communication communication) { 16 | super.getReporter().reportTGCommunication(super.taskGroupId, communication); 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/statistics/container/report/AbstractReporter.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.statistics.container.report; 2 | 3 | import com.alibaba.datax.core.statistics.communication.Communication; 4 | 5 | public abstract class AbstractReporter { 6 | 7 | public abstract void reportJobCommunication(Long jobId, Communication communication); 8 | 9 | public abstract void reportTGCommunication(Integer taskGroupId, Communication communication); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/statistics/container/report/ProcessInnerReporter.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.statistics.container.report; 2 | 3 | import com.alibaba.datax.core.statistics.communication.Communication; 4 | import com.alibaba.datax.core.statistics.communication.LocalTGCommunicationManager; 5 | 6 | public class ProcessInnerReporter extends AbstractReporter { 7 | 8 | @Override 9 | public void reportJobCommunication(Long jobId, Communication communication) { 10 | // do nothing 11 | } 12 | 13 | @Override 14 | public void reportTGCommunication(Integer taskGroupId, Communication communication) { 15 | LocalTGCommunicationManager.updateTaskGroupCommunication(taskGroupId, communication); 16 | } 17 | } -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/statistics/plugin/task/HttpPluginCollector.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.statistics.plugin.task; 2 | 3 | import com.alibaba.datax.common.constant.PluginType; 4 | import com.alibaba.datax.common.element.Record; 5 | import com.alibaba.datax.common.util.Configuration; 6 | import com.alibaba.datax.core.statistics.communication.Communication; 7 | 8 | /** 9 | * Created by jingxing on 14-9-9. 10 | */ 11 | public class HttpPluginCollector extends AbstractTaskPluginCollector { 12 | public HttpPluginCollector(Configuration configuration, Communication Communication, 13 | PluginType type) { 14 | super(configuration, Communication, type); 15 | } 16 | 17 | @Override 18 | public void collectDirtyRecord(Record dirtyRecord, Throwable t, 19 | String errorMessage) { 20 | super.collectDirtyRecord(dirtyRecord, t, errorMessage); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/transport/record/TerminateRecord.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.transport.record; 2 | 3 | import com.alibaba.datax.common.element.Column; 4 | import com.alibaba.datax.common.element.Record; 5 | 6 | /** 7 | * 作为标示 生产者已经完成生产的标志 8 | * 9 | */ 10 | public class TerminateRecord implements Record { 11 | private final static TerminateRecord SINGLE = new TerminateRecord(); 12 | 13 | private TerminateRecord() { 14 | } 15 | 16 | public static TerminateRecord get() { 17 | return SINGLE; 18 | } 19 | 20 | @Override 21 | public void addColumn(Column column) { 22 | } 23 | 24 | @Override 25 | public Column getColumn(int i) { 26 | return null; 27 | } 28 | 29 | @Override 30 | public int getColumnNumber() { 31 | return 0; 32 | } 33 | 34 | @Override 35 | public int getByteSize() { 36 | return 0; 37 | } 38 | 39 | @Override 40 | public int getMemorySize() { 41 | return 0; 42 | } 43 | 44 | @Override 45 | public void setColumn(int i, Column column) { 46 | return; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/transport/transformer/ComplexTransformerProxy.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.transport.transformer; 2 | 3 | import com.alibaba.datax.common.element.Record; 4 | import com.alibaba.datax.transformer.ComplexTransformer; 5 | import com.alibaba.datax.transformer.Transformer; 6 | 7 | import java.util.Map; 8 | 9 | /** 10 | * no comments. 11 | * Created by liqiang on 16/3/8. 12 | */ 13 | public class ComplexTransformerProxy extends ComplexTransformer { 14 | private Transformer realTransformer; 15 | 16 | public ComplexTransformerProxy(Transformer transformer) { 17 | setTransformerName(transformer.getTransformerName()); 18 | this.realTransformer = transformer; 19 | } 20 | 21 | @Override 22 | public Record evaluate(Record record, Map tContext, Object... paras) { 23 | return this.realTransformer.evaluate(record, paras); 24 | } 25 | 26 | public Transformer getRealTransformer() { 27 | return realTransformer; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/transport/transformer/GroovyTransformerStaticUtil.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.transport.transformer; 2 | 3 | /** 4 | * GroovyTransformer的帮助类，供groovy代码使用，必须全是static的方法 5 | * Created by liqiang on 16/3/4. 6 | */ 7 | public class GroovyTransformerStaticUtil { 8 | 9 | 10 | } 11 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/transport/transformer/TransformerInfo.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.transport.transformer; 2 | 3 | import com.alibaba.datax.transformer.ComplexTransformer; 4 | 5 | /** 6 | * 单实例. 7 | * Created by liqiang on 16/3/9. 8 | */ 9 | public class TransformerInfo { 10 | 11 | /** 12 | * function基本信息 13 | */ 14 | private ComplexTransformer transformer; 15 | private ClassLoader classLoader; 16 | private boolean isNative; 17 | 18 | 19 | public ComplexTransformer getTransformer() { 20 | return transformer; 21 | } 22 | 23 | public ClassLoader getClassLoader() { 24 | return classLoader; 25 | } 26 | 27 | public boolean isNative() { 28 | return isNative; 29 | } 30 | 31 | public void setTransformer(ComplexTransformer transformer) { 32 | this.transformer = transformer; 33 | } 34 | 35 | public void setClassLoader(ClassLoader classLoader) { 36 | this.classLoader = classLoader; 37 | } 38 | 39 | public void setIsNative(boolean isNative) { 40 | this.isNative = isNative; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/util/ClassUtil.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.util; 2 | 3 | import java.lang.reflect.Constructor; 4 | 5 | public final class ClassUtil { 6 | 7 | /** 8 | * 通过反射构造类对象 9 | * 10 | * @param className 11 | * 反射的类名称 12 | * @param t 13 | * 反射类的类型Class对象 14 | * @param args 15 | * 构造参数 16 | * 17 | * */ 18 | @SuppressWarnings({ "rawtypes", "unchecked" }) 19 | public static T instantiate(String className, Class t, 20 | Object... args) { 21 | try { 22 | Constructor constructor = (Constructor) Class.forName(className) 23 | .getConstructor(ClassUtil.toClassType(args)); 24 | return (T) constructor.newInstance(args); 25 | } catch (Exception e) { 26 | throw new IllegalArgumentException(e); 27 | } 28 | } 29 | 30 | private static Class[] toClassType(Object[] args) { 31 | Class[] clazzs = new Class[args.length]; 32 | 33 | for (int i = 0, length = args.length; i < length; i++) { 34 | clazzs[i] = args[i].getClass(); 35 | } 36 | 37 | return clazzs; 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/util/ConfigurationValidate.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.util; 2 | 3 | import com.alibaba.datax.common.util.Configuration; 4 | import org.apache.commons.lang.Validate; 5 | 6 | /** 7 | * Created by jingxing on 14-9-16. 8 | * 9 | * 对配置文件做整体检查 10 | */ 11 | public class ConfigurationValidate { 12 | public static void doValidate(Configuration allConfig) { 13 | Validate.isTrue(allConfig!=null, ""); 14 | 15 | coreValidate(allConfig); 16 | 17 | pluginValidate(allConfig); 18 | 19 | jobValidate(allConfig); 20 | } 21 | 22 | private static void coreValidate(Configuration allconfig) { 23 | return; 24 | } 25 | 26 | private static void pluginValidate(Configuration allConfig) { 27 | return; 28 | } 29 | 30 | private static void jobValidate(Configuration allConfig) { 31 | return; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/core/util/ExceptionTracker.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.core.util; 2 | 3 | import java.io.PrintWriter; 4 | import java.io.StringWriter; 5 | 6 | public class ExceptionTracker { 7 | public static final int STRING_BUFFER = 4096; 8 | 9 | public static String trace(Throwable ex) { 10 | StringWriter sw = new StringWriter(STRING_BUFFER); 11 | PrintWriter pw = new PrintWriter(sw); 12 | ex.printStackTrace(pw); 13 | return sw.toString(); 14 | } 15 | } -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/EnumStrVal.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.dataxservice.face.domain.enums; 2 | 3 | public interface EnumStrVal { 4 | public String value(); 5 | } 6 | -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/EnumVal.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.dataxservice.face.domain.enums; 2 | 3 | public interface EnumVal { 4 | public int value(); 5 | } -------------------------------------------------------------------------------- /core/src/main/java/com/alibaba/datax/dataxservice/face/domain/enums/State.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.dataxservice.face.domain.enums; 2 | 3 | public enum State implements EnumVal { 4 | 5 | SUBMITTING(10), 6 | WAITING(20), 7 | RUNNING(30), 8 | KILLING(40), 9 | KILLED(50), 10 | FAILED(60), 11 | SUCCEEDED(70); 12 | 13 | 14 | /* 一定会被初始化的 */ 15 | int value; 16 | 17 | State(int value) { 18 | this.value = value; 19 | } 20 | 21 | @Override 22 | public int value() { 23 | return value; 24 | } 25 | 26 | 27 | public boolean isFinished() { 28 | return this == KILLED || this == FAILED || this == SUCCEEDED; 29 | } 30 | 31 | public boolean isRunning() { 32 | return !isFinished(); 33 | } 34 | 35 | } -------------------------------------------------------------------------------- /core/src/main/log/datax.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/core/src/main/log/datax.log -------------------------------------------------------------------------------- /core/src/main/script/Readme.md: -------------------------------------------------------------------------------- 1 | some script here. -------------------------------------------------------------------------------- /core/src/main/tmp/readme.txt: -------------------------------------------------------------------------------- 1 | If you are developing DataX Plugin, In your Plugin you can use this directory to store temporary resources . 2 | 3 | NOTE: 4 | Each time install DataX, this directory will be cleaned up ! -------------------------------------------------------------------------------- /datax-opensource-dingding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/datax-opensource-dingding.png -------------------------------------------------------------------------------- /drdsreader/src/main/java/com/alibaba/datax/plugin/reader/drdsreader/DrdsReaderErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.drdsreader; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum DrdsReaderErrorCode implements ErrorCode { 6 | GET_TOPOLOGY_FAILED("DrdsReader-01", "获取 drds 表的拓扑结构失败."),; 7 | 8 | private final String code; 9 | private final String description; 10 | 11 | private DrdsReaderErrorCode(String code, String description) { 12 | this.code = code; 13 | this.description = description; 14 | } 15 | 16 | @Override 17 | public String getCode() { 18 | return this.code; 19 | } 20 | 21 | @Override 22 | public String getDescription() { 23 | return this.description; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return String.format("Code:[%s], Description:[%s]. ", this.code, 29 | this.description); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /drdsreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "drdsreader", 3 | "class": "com.alibaba.datax.plugin.reader.drdsreader.DrdsReader", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /drdsreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "drdsreader", 3 | "parameter": { 4 | "jdbcUrl": "", 5 | "username": "", 6 | "password": "", 7 | "table": "", 8 | "column": [], 9 | "where": "" 10 | } 11 | } -------------------------------------------------------------------------------- /drdswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "drdswriter", 3 | "class": "com.alibaba.datax.plugin.writer.drdswriter.DrdsWriter", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /drdswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "drdswriter", 3 | "parameter": { 4 | "jdbcUrl": "", 5 | "username": "", 6 | "password": "", 7 | "table": "", 8 | "column": [], 9 | "writeMode": "", 10 | "preSql": [], 11 | "postSql": [] 12 | } 13 | } -------------------------------------------------------------------------------- /elasticsearchwriter/README.md: -------------------------------------------------------------------------------- 1 | 本插件仅在Elasticsearch 5.x上测试 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /elasticsearchwriter/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | SCRIPT_HOME=$(cd $(dirname $0); pwd) 4 | cd $SCRIPT_HOME/.. 5 | mvn clean package -DskipTests assembly:assembly 6 | 7 | cd $SCRIPT_HOME/target/datax/plugin/writer/ 8 | 9 | if [ -d "eswriter" ]; then 10 | tar -zcvf eswriter.tgz eswriter 11 | cp eswriter.tgz $SCRIPT_HOME 12 | cd $SCRIPT_HOME 13 | ansible-playbook -i hosts main.yml -u vagrant -k 14 | fi 15 | 16 | 17 | 18 | 19 | -------------------------------------------------------------------------------- /elasticsearchwriter/src/main/java/com/alibaba/datax/plugin/writer/elasticsearchwriter/ESFieldType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.elasticsearchwriter; 2 | 3 | /** 4 | * Created by xiongfeng.bxf on 17/3/1. 5 | */ 6 | public enum ESFieldType { 7 | ID, 8 | STRING, 9 | TEXT, 10 | KEYWORD, 11 | LONG, 12 | INTEGER, 13 | SHORT, 14 | BYTE, 15 | DOUBLE, 16 | FLOAT, 17 | DATE, 18 | BOOLEAN, 19 | BINARY, 20 | INTEGER_RANGE, 21 | FLOAT_RANGE, 22 | LONG_RANGE, 23 | DOUBLE_RANGE, 24 | DATE_RANGE, 25 | GEO_POINT, 26 | GEO_SHAPE, 27 | 28 | IP, 29 | COMPLETION, 30 | TOKEN_COUNT, 31 | 32 | ARRAY, 33 | OBJECT, 34 | NESTED; 35 | 36 | public static ESFieldType getESFieldType(String type) { 37 | if (type == null) { 38 | return null; 39 | } 40 | for (ESFieldType f : ESFieldType.values()) { 41 | if (f.name().compareTo(type.toUpperCase()) == 0) { 42 | return f; 43 | } 44 | } 45 | return null; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /elasticsearchwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "elasticsearchwriter", 3 | "class": "com.alibaba.datax.plugin.writer.elasticsearchwriter.ESWriter", 4 | "description": "适用于: 生产环境. 原理: TODO", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.ftpreader; 2 | 3 | 4 | public class Constant { 5 | public static final String SOURCE_FILES = "sourceFiles"; 6 | 7 | public static final int DEFAULT_FTP_PORT = 21; 8 | public static final int DEFAULT_SFTP_PORT = 22; 9 | public static final int DEFAULT_TIMEOUT = 60000; 10 | public static final int DEFAULT_MAX_TRAVERSAL_LEVEL = 100; 11 | public static final String DEFAULT_FTP_CONNECT_PATTERN = "PASV"; 12 | 13 | 14 | } 15 | -------------------------------------------------------------------------------- /ftpreader/src/main/java/com/alibaba/datax/plugin/reader/ftpreader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.ftpreader; 2 | 3 | public class Key { 4 | public static final String PROTOCOL = "protocol"; 5 | public static final String HOST = "host"; 6 | public static final String USERNAME = "username"; 7 | public static final String PASSWORD = "password"; 8 | public static final String PORT = "port"; 9 | public static final String TIMEOUT = "timeout"; 10 | public static final String CONNECTPATTERN = "connectPattern"; 11 | public static final String PATH = "path"; 12 | public static final String MAXTRAVERSALLEVEL = "maxTraversalLevel"; 13 | } 14 | -------------------------------------------------------------------------------- /ftpreader/src/main/resources/plugin-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ftpreader", 3 | "parameter": { 4 | "host": "", 5 | "port": "", 6 | "username": "", 7 | "password": "", 8 | "protocol": "", 9 | "path": [ 10 | "" 11 | ], 12 | "encoding": "UTF-8", 13 | "column": [ 14 | { 15 | "index": 0, 16 | "type": "long" 17 | }, 18 | { 19 | "index": 1, 20 | "type": "boolean" 21 | }, 22 | { 23 | "index": 2, 24 | "type": "double" 25 | }, 26 | { 27 | "index": 3, 28 | "type": "string" 29 | }, 30 | { 31 | "index": 4, 32 | "type": "date", 33 | "format": "yyyy.MM.dd" 34 | } 35 | ], 36 | "fieldDelimiter": "," 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /ftpreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ftpreader", 3 | "class": "com.alibaba.datax.plugin.reader.ftpreader.FtpReader", 4 | "description": "useScene: test. mechanism: use datax framework to transport data from txt file. warn: The more you know about the data, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /ftpreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ftpreader", 3 | "parameter": { 4 | "host": "", 5 | "protocol": "sftp", 6 | "port":"", 7 | "username": "", 8 | "password": "", 9 | "path": [], 10 | "column": [ 11 | { 12 | "index": 0, 13 | "type": "" 14 | } 15 | ], 16 | "fieldDelimiter": ",", 17 | "encoding": "UTF-8" 18 | } 19 | } -------------------------------------------------------------------------------- /ftpwriter/doc/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/ftpwriter/doc/.gitkeep -------------------------------------------------------------------------------- /ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.ftpwriter; 2 | 3 | public class Key { 4 | public static final String PROTOCOL = "protocol"; 5 | 6 | public static final String HOST = "host"; 7 | 8 | public static final String USERNAME = "username"; 9 | 10 | public static final String PASSWORD = "password"; 11 | 12 | public static final String PORT = "port"; 13 | 14 | public static final String TIMEOUT = "timeout"; 15 | 16 | public static final String CONNECTPATTERN = "connectPattern"; 17 | 18 | public static final String PATH = "path"; 19 | } 20 | -------------------------------------------------------------------------------- /ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.ftpwriter.util; 2 | 3 | 4 | public class Constant { 5 | public static final int DEFAULT_FTP_PORT = 21; 6 | 7 | public static final int DEFAULT_SFTP_PORT = 22; 8 | 9 | public static final int DEFAULT_TIMEOUT = 60000; 10 | 11 | public static final int DEFAULT_MAX_TRAVERSAL_LEVEL = 100; 12 | 13 | public static final String DEFAULT_FTP_CONNECT_PATTERN = "PASV"; 14 | 15 | public static final String CONTROL_ENCODING = "utf8"; 16 | } 17 | -------------------------------------------------------------------------------- /ftpwriter/src/main/java/com/alibaba/datax/plugin/writer/ftpwriter/util/IFtpHelper.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.ftpwriter.util; 2 | 3 | import java.io.OutputStream; 4 | import java.util.Set; 5 | 6 | public interface IFtpHelper { 7 | 8 | //使用被动方式 9 | public void loginFtpServer(String host, String username, String password, int port, int timeout); 10 | 11 | public void logoutFtpServer(); 12 | 13 | /** 14 | * warn: 不支持递归创建, 比如 mkdir -p 15 | * */ 16 | public void mkdir(String directoryPath); 17 | 18 | /** 19 | * 支持目录递归创建 20 | */ 21 | public void mkDirRecursive(String directoryPath); 22 | 23 | public OutputStream getOutputStream(String filePath); 24 | 25 | public String getRemoteFileContent(String filePath); 26 | 27 | public Set getAllFilesInDir(String dir, String prefixFileName); 28 | 29 | /** 30 | * warn: 不支持文件夹删除, 比如 rm -rf 31 | * */ 32 | public void deleteFiles(Set filesToDelete); 33 | 34 | public void completePendingCommand(); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /ftpwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ftpwriter", 3 | "class": "com.alibaba.datax.plugin.writer.ftpwriter.FtpWriter", 4 | "description": "useScene: test. mechanism: use datax framework to transport data from ftp txt file. warn: The more you know about the data, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /ftpwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ftpwriter", 3 | "parameter": { 4 | "name": "ftpwriter", 5 | "parameter": { 6 | "protocol": "", 7 | "host": "", 8 | "port": "", 9 | "username": "", 10 | "password": "", 11 | "timeout": "", 12 | "connectPattern": "", 13 | "path": "", 14 | "fileName": "", 15 | "writeMode": "", 16 | "fieldDelimiter": "", 17 | "encoding": "", 18 | "nullFormat": "", 19 | "dateFormat": "", 20 | "fileFormat": "", 21 | "header": [] 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /gdbreader/src/main/java/com/alibaba/datax/plugin/reader/gdbreader/model/GdbElement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) 2019-present Alibaba Group Holding Limited. 3 | * 4 | * This program is free software; you can redistribute it and/or modify 5 | * it under the terms of the GNU General Public License version 2 as 6 | * published by the Free Software Foundation. 7 | */ 8 | package com.alibaba.datax.plugin.reader.gdbreader.model; 9 | 10 | import lombok.Data; 11 | 12 | import java.util.HashMap; 13 | import java.util.Map; 14 | 15 | /** 16 | * @author : Liu Jianping 17 | * @date : 2019/9/6 18 | */ 19 | 20 | @Data 21 | public class GdbElement { 22 | String id = null; 23 | String label = null; 24 | String to = null; 25 | String from = null; 26 | String toLabel = null; 27 | String fromLabel = null; 28 | 29 | Map properties = new HashMap<>(); 30 | 31 | public GdbElement() { 32 | } 33 | 34 | public GdbElement(String id, String label) { 35 | this.id = id; 36 | this.label = label; 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /gdbreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gdbreader", 3 | "class": "com.alibaba.datax.plugin.reader.gdbreader.GdbReader", 4 | "description": "useScene: prod. mechanism: connect GDB with gremlin-client, execute 'g.V().propertyMap() or g.E().propertyMap()' to get record", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/GdbWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.gdbwriter; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum GdbWriterErrorCode implements ErrorCode { 6 | BAD_CONFIG_VALUE("GdbWriter-00", "您配置的值不合法."), 7 | CONFIG_ITEM_MISS("GdbWriter-01", "您配置项缺失."), 8 | FAIL_CLIENT_CONNECT("GdbWriter-02", "GDB连接异常."),; 9 | 10 | private final String code; 11 | private final String description; 12 | 13 | private GdbWriterErrorCode(String code, String description) { 14 | this.code = code; 15 | this.description = description; 16 | } 17 | 18 | @Override 19 | public String getCode() { 20 | return this.code; 21 | } 22 | 23 | @Override 24 | public String getDescription() { 25 | return this.description; 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); 31 | } 32 | } -------------------------------------------------------------------------------- /gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/GdbMapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.alibaba.datax.plugin.writer.gdbwriter.mapping; 5 | 6 | import java.util.function.Function; 7 | 8 | import com.alibaba.datax.common.element.Record; 9 | import com.alibaba.datax.plugin.writer.gdbwriter.model.GdbElement; 10 | 11 | /** 12 | * @author jerrywang 13 | * 14 | */ 15 | public interface GdbMapper { 16 | Function getMapper(MappingRule rule); 17 | } 18 | -------------------------------------------------------------------------------- /gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/mapping/MappingRule.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.alibaba.datax.plugin.writer.gdbwriter.mapping; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | import com.alibaba.datax.plugin.writer.gdbwriter.Key.ImportType; 10 | import com.alibaba.datax.plugin.writer.gdbwriter.Key.PropertyType; 11 | 12 | import lombok.Data; 13 | 14 | /** 15 | * @author jerrywang 16 | * 17 | */ 18 | @Data 19 | public class MappingRule { 20 | private String id = null; 21 | 22 | private String label = null; 23 | 24 | private ImportType importType = null; 25 | 26 | private String from = null; 27 | 28 | private String to = null; 29 | 30 | private List properties = new ArrayList<>(); 31 | 32 | private String propertiesJsonStr = null; 33 | 34 | private boolean numPattern = false; 35 | 36 | @Data 37 | public static class PropertyMappingRule { 38 | private String key = null; 39 | 40 | private String value = null; 41 | 42 | private ValueType valueType = null; 43 | 44 | private PropertyType pType = PropertyType.single; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbGraph.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.alibaba.datax.plugin.writer.gdbwriter.model; 5 | 6 | import java.util.List; 7 | 8 | import com.alibaba.datax.common.element.Record; 9 | 10 | import groovy.lang.Tuple2; 11 | 12 | /** 13 | * @author jerrywang 14 | * 15 | */ 16 | public interface GdbGraph extends AutoCloseable { 17 | List> add(List> records); 18 | 19 | @Override 20 | void close(); 21 | } 22 | -------------------------------------------------------------------------------- /gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/model/GdbVertex.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.alibaba.datax.plugin.writer.gdbwriter.model; 5 | 6 | import lombok.EqualsAndHashCode; 7 | import lombok.ToString; 8 | 9 | /** 10 | * @author jerrywang 11 | * 12 | */ 13 | @EqualsAndHashCode(callSuper = true) 14 | @ToString(callSuper = true) 15 | public class GdbVertex extends GdbElement { 16 | 17 | } 18 | -------------------------------------------------------------------------------- /gdbwriter/src/main/java/com/alibaba/datax/plugin/writer/gdbwriter/util/GdbDuplicateIdException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * (C) 2019-present Alibaba Group Holding Limited. 3 | * 4 | * This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public 5 | * License version 2 as published by the Free Software Foundation. 6 | */ 7 | package com.alibaba.datax.plugin.writer.gdbwriter.util; 8 | 9 | /** 10 | * @author : Liu Jianping 11 | * @date : 2019/8/3 12 | */ 13 | 14 | public class GdbDuplicateIdException extends Exception { 15 | public GdbDuplicateIdException(Exception e) { 16 | super(e); 17 | } 18 | 19 | public GdbDuplicateIdException() { 20 | super(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /gdbwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gdbwriter", 3 | "class": "com.alibaba.datax.plugin.writer.gdbwriter.GdbWriter", 4 | "description": "useScene: prod. mechanism: connect GDB with gremlin-client, execute DSL as 'g.addV() or g.addE()' to write record", 5 | "developer": "alibaba" 6 | } 7 | -------------------------------------------------------------------------------- /hbase094xreader/doc/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/hbase094xreader/doc/.gitkeep -------------------------------------------------------------------------------- /hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hbase094xreader; 2 | 3 | public final class Constant { 4 | public static final String RANGE = "range"; 5 | 6 | public static final String ROWKEY_FLAG = "rowkey"; 7 | 8 | public static final String DEFAULT_ENCODING = "UTF-8"; 9 | 10 | public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; 11 | 12 | public static final int DEFAULT_SCAN_CACHE_SIZE = 256; 13 | 14 | public static final int DEFAULT_SCAN_BATCH_SIZE = 100; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/ModeType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hbase094xreader; 2 | 3 | import com.alibaba.datax.common.exception.DataXException; 4 | 5 | import java.util.Arrays; 6 | 7 | public enum ModeType { 8 | Normal("normal"), 9 | MultiVersionFixedColumn("multiVersionFixedColumn") 10 | ; 11 | 12 | private String mode; 13 | 14 | ModeType(String mode) { 15 | this.mode = mode.toLowerCase(); 16 | } 17 | 18 | public static ModeType getByTypeName(String modeName) { 19 | for (ModeType modeType : values()) { 20 | if (modeType.mode.equalsIgnoreCase(modeName)) { 21 | return modeType; 22 | } 23 | } 24 | 25 | throw DataXException.asDataXException(Hbase094xReaderErrorCode.ILLEGAL_VALUE, 26 | String.format("HbaseReader 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values()))); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /hbase094xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase094xreader/MultiVersionFixedColumnTask.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hbase094xreader; 2 | 3 | import com.alibaba.datax.common.util.Configuration; 4 | import org.apache.hadoop.hbase.client.Scan; 5 | import org.apache.hadoop.hbase.util.Bytes; 6 | 7 | import java.util.Map; 8 | 9 | public class MultiVersionFixedColumnTask extends MultiVersionTask { 10 | 11 | public MultiVersionFixedColumnTask(Configuration configuration) { 12 | super(configuration); 13 | } 14 | 15 | @Override 16 | public void initScan(Scan scan) { 17 | for (Map aColumn : column) { 18 | String columnName = aColumn.get(Key.NAME); 19 | if(!Hbase094xHelper.isRowkeyColumn(columnName)){ 20 | String[] cfAndQualifier = columnName.split(":"); 21 | scan.addColumn(Bytes.toBytes(cfAndQualifier[0].trim()), Bytes.toBytes(cfAndQualifier[1].trim())); 22 | } 23 | } 24 | super.setMaxVersions(scan); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /hbase094xreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase094xreader", 3 | "class": "com.alibaba.datax.plugin.reader.hbase094xreader.Hbase094xReader", 4 | "description": "useScene: prod. mechanism: Scan to read data.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /hbase094xreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase094xreader", 3 | "parameter": { 4 | "hbaseConfig": {}, 5 | "table": "", 6 | "encoding": "", 7 | "mode": "", 8 | "column": [], 9 | "range": { 10 | "startRowkey": "", 11 | "endRowkey": "", 12 | "isBinaryRowkey": true 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /hbase094xwriter/doc/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/hbase094xwriter/doc/.gitkeep -------------------------------------------------------------------------------- /hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase094xwriter; 2 | 3 | public final class Constant { 4 | public static final String DEFAULT_ENCODING = "UTF-8"; 5 | public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; 6 | public static final String DEFAULT_NULL_MODE = "skip"; 7 | public static final long DEFAULT_WRITE_BUFFER_SIZE = 8 * 1024 * 1024; 8 | } 9 | -------------------------------------------------------------------------------- /hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/ModeType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase094xwriter; 2 | 3 | import com.alibaba.datax.common.exception.DataXException; 4 | 5 | import java.util.Arrays; 6 | 7 | public enum ModeType { 8 | Normal("normal"), 9 | MultiVersion("multiVersion") 10 | ; 11 | 12 | private String mode; 13 | 14 | 15 | ModeType(String mode) { 16 | this.mode = mode.toLowerCase(); 17 | } 18 | 19 | public String getMode() { 20 | return mode; 21 | } 22 | 23 | public static ModeType getByTypeName(String modeName) { 24 | for (ModeType modeType : values()) { 25 | if (modeType.mode.equalsIgnoreCase(modeName)) { 26 | return modeType; 27 | } 28 | } 29 | throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE, 30 | String.format("Hbasewriter 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values()))); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hbase094xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase094xwriter/NullModeType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase094xwriter; 2 | 3 | import com.alibaba.datax.common.exception.DataXException; 4 | 5 | import java.util.Arrays; 6 | 7 | public enum NullModeType { 8 | Skip("skip"), 9 | Empty("empty") 10 | ; 11 | 12 | private String mode; 13 | 14 | 15 | NullModeType(String mode) { 16 | this.mode = mode.toLowerCase(); 17 | } 18 | 19 | public String getMode() { 20 | return mode; 21 | } 22 | 23 | public static NullModeType getByTypeName(String modeName) { 24 | for (NullModeType modeType : values()) { 25 | if (modeType.mode.equalsIgnoreCase(modeName)) { 26 | return modeType; 27 | } 28 | } 29 | throw DataXException.asDataXException(Hbase094xWriterErrorCode.ILLEGAL_VALUE, 30 | String.format("Hbasewriter 不支持该 nullMode 类型:%s, 目前支持的 nullMode 类型是:%s", modeName, Arrays.asList(values()))); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hbase094xwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase094xwriter", 3 | "class": "com.alibaba.datax.plugin.writer.hbase094xwriter.Hbase094xWriter", 4 | "description": "use put: prod. mechanism: use hbase java api put data.", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /hbase094xwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase094xwriter", 3 | "parameter": { 4 | "hbaseConfig": {}, 5 | "table": "", 6 | "mode": "", 7 | "rowkeyColumn": [ 8 | ], 9 | "column": [ 10 | ], 11 | "versionColumn":{ 12 | "index": "", 13 | "value":"" 14 | }, 15 | "encoding": "" 16 | } 17 | } -------------------------------------------------------------------------------- /hbase11xreader/doc/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/hbase11xreader/doc/.gitkeep -------------------------------------------------------------------------------- /hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hbase11xreader; 2 | 3 | public final class Constant { 4 | public static final String RANGE = "range"; 5 | 6 | public static final String ROWKEY_FLAG = "rowkey"; 7 | 8 | public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; 9 | 10 | public static final String DEFAULT_ENCODING = "UTF-8"; 11 | 12 | public static final int DEFAULT_SCAN_CACHE_SIZE = 256; 13 | 14 | public static final int DEFAULT_SCAN_BATCH_SIZE = 100; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/ModeType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hbase11xreader; 2 | 3 | import com.alibaba.datax.common.exception.DataXException; 4 | 5 | import java.util.Arrays; 6 | 7 | public enum ModeType { 8 | Normal("normal"), 9 | MultiVersionFixedColumn("multiVersionFixedColumn") 10 | ; 11 | 12 | private String mode; 13 | 14 | 15 | ModeType(String mode) { 16 | this.mode = mode.toLowerCase(); 17 | } 18 | 19 | public String getMode() { 20 | return mode; 21 | } 22 | 23 | public static ModeType getByTypeName(String modeName) { 24 | for (ModeType modeType : values()) { 25 | if (modeType.mode.equalsIgnoreCase(modeName)) { 26 | return modeType; 27 | } 28 | } 29 | throw DataXException.asDataXException(Hbase11xReaderErrorCode.ILLEGAL_VALUE, 30 | String.format("HbaseReader 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values()))); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/MultiVersionDynamicColumnTask.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hbase11xreader; 2 | 3 | import com.alibaba.datax.common.util.Configuration; 4 | import org.apache.hadoop.hbase.client.Scan; 5 | import org.apache.hadoop.hbase.util.Bytes; 6 | 7 | import java.util.List; 8 | 9 | public class MultiVersionDynamicColumnTask extends MultiVersionTask { 10 | private List columnFamilies = null; 11 | 12 | public MultiVersionDynamicColumnTask(Configuration configuration){ 13 | super(configuration); 14 | 15 | this.columnFamilies = configuration.getList(Key.COLUMN_FAMILY, String.class); 16 | } 17 | 18 | @Override 19 | public void initScan(Scan scan) { 20 | for (String columnFamily : columnFamilies) { 21 | scan.addFamily(Bytes.toBytes(columnFamily.trim())); 22 | } 23 | 24 | super.setMaxVersions(scan); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /hbase11xreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xreader/MultiVersionFixedColumnTask.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hbase11xreader; 2 | 3 | import com.alibaba.datax.common.util.Configuration; 4 | import org.apache.hadoop.hbase.client.Scan; 5 | import org.apache.hadoop.hbase.util.Bytes; 6 | 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | public class MultiVersionFixedColumnTask extends MultiVersionTask { 11 | 12 | public MultiVersionFixedColumnTask(Configuration configuration) { 13 | super(configuration); 14 | } 15 | 16 | @Override 17 | public void initScan(Scan scan) { 18 | for (Map aColumn : column) { 19 | String columnName = aColumn.get(Key.NAME); 20 | if(!Hbase11xHelper.isRowkeyColumn(columnName)){ 21 | String[] cfAndQualifier = columnName.split(":"); 22 | scan.addColumn(Bytes.toBytes(cfAndQualifier[0].trim()), Bytes.toBytes(cfAndQualifier[1].trim())); 23 | } 24 | } 25 | super.setMaxVersions(scan); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /hbase11xreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase11xreader", 3 | "class": "com.alibaba.datax.plugin.reader.hbase11xreader.Hbase11xReader", 4 | "description": "useScene: prod. mechanism: Scan to read data.", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /hbase11xreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase11xreader", 3 | "parameter": { 4 | "hbaseConfig": {}, 5 | "table": "", 6 | "encoding": "", 7 | "mode": "", 8 | "column": [], 9 | "range": { 10 | "startRowkey": "", 11 | "endRowkey": "", 12 | "isBinaryRowkey": true 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/HadoopSerializationUtil.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hbase11xsqlreader; 2 | 3 | 4 | import java.io.ByteArrayInputStream; 5 | import java.io.ByteArrayOutputStream; 6 | import java.io.DataInputStream; 7 | import java.io.DataOutputStream; 8 | import java.io.IOException; 9 | 10 | import org.apache.hadoop.io.Writable; 11 | 12 | public class HadoopSerializationUtil { 13 | 14 | public static byte[] serialize(Writable writable) throws IOException { 15 | ByteArrayOutputStream out = new ByteArrayOutputStream(); 16 | DataOutputStream dataout = new DataOutputStream(out); 17 | writable.write(dataout); 18 | dataout.close(); 19 | return out.toByteArray(); 20 | } 21 | 22 | public static void deserialize(Writable writable, byte[] bytes) throws Exception { 23 | 24 | ByteArrayInputStream in = new ByteArrayInputStream(bytes); 25 | DataInputStream datain = new DataInputStream(in); 26 | writable.readFields(datain); 27 | datain.close(); 28 | } 29 | 30 | 31 | } -------------------------------------------------------------------------------- /hbase11xsqlreader/src/main/java/com/alibaba/datax/plugin/reader/hbase11xsqlreader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hbase11xsqlreader; 2 | 3 | import org.apache.hadoop.hbase.HConstants; 4 | 5 | public final class Key { 6 | 7 | public final static String MOCK_JOBID_IDENTIFIER = "phoenixreader"; 8 | public final static int MOCK_JOBID = 1; 9 | public final static String SPLIT_KEY = "phoenixsplit"; 10 | 11 | /** 12 | * 【必选】hbase集群配置，连接一个hbase集群需要的最小配置只有两个：zk和znode 13 | */ 14 | public final static String HBASE_CONFIG = "hbaseConfig"; 15 | public final static String HBASE_ZK_QUORUM = HConstants.ZOOKEEPER_QUORUM; 16 | public final static String HBASE_ZNODE_PARENT = HConstants.ZOOKEEPER_ZNODE_PARENT; 17 | 18 | /** 19 | * 【必选】writer要写入的表的表名 20 | */ 21 | public final static String TABLE = "table"; 22 | 23 | /** 24 | * 【必选】列配置 25 | */ 26 | public final static String COLUMN = "column"; 27 | 28 | } 29 | -------------------------------------------------------------------------------- /hbase11xsqlreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase11xsqlreader", 3 | "class": "com.alibaba.datax.plugin.reader.hbase11xsqlreader.HbaseSQLReader", 4 | "description": "useScene: prod. mechanism: Scan to read data.", 5 | "developer": "liwei.li, bug reported to : liwei.li@alibaba-inc.com" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /hbase11xsqlreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase11sqlreader", 3 | "parameter": { 4 | "hbaseConfig": { 5 | "hbase.zookeeper.quorum": "hb-proxy-pub-xxx-001.hbase.rds.aliyuncs.com,hb-proxy-pub-xxx-002.hbase.rds.aliyuncs.com,hb-proxy-pub-xxx-003.hbase.rds.aliyuncs.com" 6 | }, 7 | "table": "TABLE1", 8 | "column": [ 9 | "ID", 10 | "COL1" 11 | ] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /hbase11xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xsqlwriter/NullModeType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase11xsqlwriter; 2 | 3 | import com.alibaba.datax.common.exception.DataXException; 4 | 5 | import java.util.Arrays; 6 | 7 | public enum NullModeType { 8 | Skip("skip"), 9 | Empty("empty") 10 | ; 11 | 12 | private String mode; 13 | 14 | 15 | NullModeType(String mode) { 16 | this.mode = mode.toLowerCase(); 17 | } 18 | 19 | public String getMode() { 20 | return mode; 21 | } 22 | 23 | public static NullModeType getByTypeName(String modeName) { 24 | for (NullModeType modeType : values()) { 25 | if (modeType.mode.equalsIgnoreCase(modeName)) { 26 | return modeType; 27 | } 28 | } 29 | throw DataXException.asDataXException(HbaseSQLWriterErrorCode.ILLEGAL_VALUE, 30 | "Hbasewriter 不支持该 nullMode 类型:" + modeName + ", 目前支持的 nullMode 类型是:" + Arrays.asList(values())); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hbase11xsqlwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase11xsqlwriter", 3 | "class": "com.alibaba.datax.plugin.writer.hbase11xsqlwriter.HbaseSQLWriter", 4 | "description": "useScene: prod. mechanism: use hbase sql UPSERT to put data, index tables will be updated too.", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /hbase11xwriter/doc/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/hbase11xwriter/doc/.gitkeep -------------------------------------------------------------------------------- /hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase11xwriter; 2 | 3 | public final class Constant { 4 | public static final String DEFAULT_ENCODING = "UTF-8"; 5 | public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; 6 | public static final String DEFAULT_NULL_MODE = "skip"; 7 | public static final long DEFAULT_WRITE_BUFFER_SIZE = 8 * 1024 * 1024; 8 | } 9 | -------------------------------------------------------------------------------- /hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/ModeType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase11xwriter; 2 | 3 | import com.alibaba.datax.common.exception.DataXException; 4 | 5 | import java.util.Arrays; 6 | 7 | public enum ModeType { 8 | Normal("normal"), 9 | MultiVersion("multiVersion") 10 | ; 11 | 12 | private String mode; 13 | 14 | 15 | ModeType(String mode) { 16 | this.mode = mode.toLowerCase(); 17 | } 18 | 19 | public String getMode() { 20 | return mode; 21 | } 22 | 23 | public static ModeType getByTypeName(String modeName) { 24 | for (ModeType modeType : values()) { 25 | if (modeType.mode.equalsIgnoreCase(modeName)) { 26 | return modeType; 27 | } 28 | } 29 | throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, 30 | String.format("Hbasewriter 不支持该 mode 类型:%s, 目前支持的 mode 类型是:%s", modeName, Arrays.asList(values()))); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hbase11xwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase11xwriter/NullModeType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase11xwriter; 2 | 3 | import com.alibaba.datax.common.exception.DataXException; 4 | 5 | import java.util.Arrays; 6 | 7 | public enum NullModeType { 8 | Skip("skip"), 9 | Empty("empty") 10 | ; 11 | 12 | private String mode; 13 | 14 | 15 | NullModeType(String mode) { 16 | this.mode = mode.toLowerCase(); 17 | } 18 | 19 | public String getMode() { 20 | return mode; 21 | } 22 | 23 | public static NullModeType getByTypeName(String modeName) { 24 | for (NullModeType modeType : values()) { 25 | if (modeType.mode.equalsIgnoreCase(modeName)) { 26 | return modeType; 27 | } 28 | } 29 | throw DataXException.asDataXException(Hbase11xWriterErrorCode.ILLEGAL_VALUE, 30 | String.format("Hbasewriter 不支持该 nullMode 类型:%s, 目前支持的 nullMode 类型是:%s", modeName, Arrays.asList(values()))); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hbase11xwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase11xwriter", 3 | "class": "com.alibaba.datax.plugin.writer.hbase11xwriter.Hbase11xWriter", 4 | "description": "use put: prod. mechanism: use hbase java api put data.", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /hbase11xwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase11xwriter", 3 | "parameter": { 4 | "hbaseConfig": { 5 | "hbase.rootdir": "", 6 | "hbase.cluster.distributed": "", 7 | "hbase.zookeeper.quorum": "" 8 | }, 9 | "table": "", 10 | "mode": "", 11 | "rowkeyColumn": [ 12 | ], 13 | "column": [ 14 | ], 15 | "versionColumn":{ 16 | "index": "", 17 | "value":"" 18 | }, 19 | "encoding": "" 20 | } 21 | } -------------------------------------------------------------------------------- /hbase20xsqlreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase20xsqlreader", 3 | "class": "com.alibaba.datax.plugin.reader.hbase20xsqlreader.HBase20xSQLReader", 4 | "description": "useScene: prod. mechanism: read data from phoenix through queryserver.", 5 | "developer": "bake" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /hbase20xsqlreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase20xsqlreader", 3 | "parameter": { 4 | "queryserverAddress": "", 5 | "serialization": "PROTOBUF", 6 | "schema": "", 7 | "table": "TABLE1", 8 | "column": ["ID", "NAME"], 9 | "splitKey": "rowkey", 10 | "splitPoint":[], 11 | "where": "" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /hbase20xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase20xsqlwriter/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase20xsqlwriter; 2 | 3 | public final class Constant { 4 | public static final String DEFAULT_NULL_MODE = "skip"; 5 | public static final String DEFAULT_SERIALIZATION = "PROTOBUF"; 6 | public static final int DEFAULT_BATCH_ROW_COUNT = 256; // 默认一次写256行 7 | 8 | public static final int TYPE_UNSIGNED_TINYINT = 11; 9 | public static final int TYPE_UNSIGNED_SMALLINT = 13; 10 | public static final int TYPE_UNSIGNED_INTEGER = 9; 11 | public static final int TYPE_UNSIGNED_LONG = 10; 12 | public static final int TYPE_UNSIGNED_FLOAT = 14; 13 | public static final int TYPE_UNSIGNED_DOUBLE = 15; 14 | public static final int TYPE_UNSIGNED_DATE = 19; 15 | public static final int TYPE_UNSIGNED_TIME = 18; 16 | public static final int TYPE_UNSIGNED_TIMESTAMP = 20; 17 | } 18 | -------------------------------------------------------------------------------- /hbase20xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase20xsqlwriter/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase20xsqlwriter; 2 | 3 | public class Key { 4 | 5 | /** 6 | * 【必选】writer要写入的表的表名 7 | */ 8 | public final static String TABLE = "table"; 9 | /** 10 | * 【必选】writer要写入哪些列 11 | */ 12 | public final static String COLUMN = "column"; 13 | /** 14 | * 【必选】Phoenix QueryServer服务地址 15 | */ 16 | public final static String QUERYSERVER_ADDRESS = "queryServerAddress"; 17 | /** 18 | * 【可选】序列化格式，默认为PROTOBUF 19 | */ 20 | public static final String SERIALIZATION_NAME = "serialization"; 21 | 22 | /** 23 | * 【可选】批量写入的最大行数，默认100行 24 | */ 25 | public static final String BATCHSIZE = "batchSize"; 26 | 27 | /** 28 | * 【可选】遇到空值默认跳过 29 | */ 30 | public static final String NULLMODE = "nullMode"; 31 | /** 32 | * 【可选】Phoenix表所属schema，默认为空 33 | */ 34 | public static final String SCHEMA = "schema"; 35 | 36 | } 37 | -------------------------------------------------------------------------------- /hbase20xsqlwriter/src/main/java/com/alibaba/datax/plugin/writer/hbase20xsqlwriter/NullModeType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hbase20xsqlwriter; 2 | 3 | import com.alibaba.datax.common.exception.DataXException; 4 | 5 | import java.util.Arrays; 6 | 7 | public enum NullModeType { 8 | Skip("skip"), 9 | Empty("empty") 10 | ; 11 | 12 | private String mode; 13 | 14 | 15 | NullModeType(String mode) { 16 | this.mode = mode.toLowerCase(); 17 | } 18 | 19 | public String getMode() { 20 | return mode; 21 | } 22 | 23 | public static NullModeType getByTypeName(String modeName) { 24 | for (NullModeType modeType : values()) { 25 | if (modeType.mode.equalsIgnoreCase(modeName)) { 26 | return modeType; 27 | } 28 | } 29 | throw DataXException.asDataXException(HBase20xSQLWriterErrorCode.ILLEGAL_VALUE, 30 | "Hbasewriter 不支持该 nullMode 类型:" + modeName + ", 目前支持的 nullMode 类型是:" + Arrays.asList(values())); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /hbase20xsqlwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase20xsqlwriter", 3 | "class": "com.alibaba.datax.plugin.writer.hbase20xsqlwriter.HBase20xSQLWriter", 4 | "description": "useScene: prod. mechanism: use hbase sql UPSERT to put data, index tables will be updated too.", 5 | "developer": "bake" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /hbase20xsqlwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hbase20xsqlwriter", 3 | "parameter": { 4 | "queryServerAddress": "", 5 | "table": "", 6 | "serialization": "PROTOBUF", 7 | "column": [ 8 | ], 9 | "batchSize": "100", 10 | "nullMode": "skip", 11 | "schema": "" 12 | } 13 | } -------------------------------------------------------------------------------- /hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hdfsreader; 2 | 3 | /** 4 | * Created by mingya.wmy on 2015/8/14. 5 | */ 6 | public class Constant { 7 | public static final String SOURCE_FILES = "sourceFiles"; 8 | public static final String TEXT = "TEXT"; 9 | public static final String ORC = "ORC"; 10 | public static final String CSV = "CSV"; 11 | public static final String SEQ = "SEQ"; 12 | public static final String RC = "RC"; 13 | } 14 | -------------------------------------------------------------------------------- /hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/HdfsFileType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hdfsreader; 2 | 3 | /** 4 | * Created by mingya.wmy on 2015/8/22. 5 | * 6 | */ 7 | public enum HdfsFileType { 8 | ORC, SEQ, RC, CSV, TEXT, 9 | } 10 | -------------------------------------------------------------------------------- /hdfsreader/src/main/java/com/alibaba/datax/plugin/reader/hdfsreader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.hdfsreader; 2 | 3 | public final class Key { 4 | 5 | /** 6 | * 此处声明插件用到的需要插件使用者提供的配置项 7 | */ 8 | public final static String PATH = "path"; 9 | public final static String DEFAULT_FS = "defaultFS"; 10 | public static final String FILETYPE = "fileType"; 11 | public static final String HADOOP_CONFIG = "hadoopConfig"; 12 | public static final String HAVE_KERBEROS = "haveKerberos"; 13 | public static final String KERBEROS_KEYTAB_FILE_PATH = "kerberosKeytabFilePath"; 14 | public static final String KERBEROS_PRINCIPAL = "kerberosPrincipal"; 15 | } 16 | -------------------------------------------------------------------------------- /hdfsreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hdfsreader", 3 | "class": "com.alibaba.datax.plugin.reader.hdfsreader.HdfsReader", 4 | "description": "useScene: test. mechanism: use datax framework to transport data from hdfs. warn: The more you know about the data, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /hdfsreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hdfsreader", 3 | "parameter": { 4 | "path": "", 5 | "defaultFS": "", 6 | "column": [], 7 | "fileType": "orc", 8 | "encoding": "UTF-8", 9 | "fieldDelimiter": "," 10 | } 11 | } -------------------------------------------------------------------------------- /hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hdfswriter; 2 | 3 | public class Constant { 4 | 5 | public static final String DEFAULT_ENCODING = "UTF-8"; 6 | public static final String DEFAULT_NULL_FORMAT = "\\N"; 7 | } 8 | -------------------------------------------------------------------------------- /hdfswriter/src/main/java/com/alibaba/datax/plugin/writer/hdfswriter/SupportHiveDataType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.hdfswriter; 2 | 3 | public enum SupportHiveDataType { 4 | TINYINT, 5 | SMALLINT, 6 | INT, 7 | BIGINT, 8 | FLOAT, 9 | DOUBLE, 10 | 11 | TIMESTAMP, 12 | DATE, 13 | 14 | STRING, 15 | VARCHAR, 16 | CHAR, 17 | 18 | BOOLEAN 19 | } 20 | -------------------------------------------------------------------------------- /hdfswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hdfswriter", 3 | "class": "com.alibaba.datax.plugin.writer.hdfswriter.HdfsWriter", 4 | "description": "useScene: prod. mechanism: via FileSystem connect HDFS write data concurrent.", 5 | "developer": "alibaba" 6 | } 7 | -------------------------------------------------------------------------------- /hdfswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hdfswriter", 3 | "parameter": { 4 | "defaultFS": "", 5 | "fileType": "", 6 | "path": "", 7 | "fileName": "", 8 | "column": [], 9 | "writeMode": "", 10 | "fieldDelimiter": "", 11 | "compress":"" 12 | } 13 | } -------------------------------------------------------------------------------- /images/DataX-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/DataX-logo.jpg -------------------------------------------------------------------------------- /images/DataX开源用户交流群.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/DataX开源用户交流群.jpg -------------------------------------------------------------------------------- /images/DataX开源用户交流群2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/DataX开源用户交流群2.jpg -------------------------------------------------------------------------------- /images/DataX开源用户交流群3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/DataX开源用户交流群3.jpg -------------------------------------------------------------------------------- /images/DataX开源用户交流群4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/DataX开源用户交流群4.jpg -------------------------------------------------------------------------------- /images/DataX开源用户交流群5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/DataX开源用户交流群5.jpg -------------------------------------------------------------------------------- /images/datax-enterprise-users.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/datax-enterprise-users.jpg -------------------------------------------------------------------------------- /images/datax-opensource-dingding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/datax-opensource-dingding.png -------------------------------------------------------------------------------- /images/dingding.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/dingding.jpg -------------------------------------------------------------------------------- /images/plugin_dev_guide_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/plugin_dev_guide_1.png -------------------------------------------------------------------------------- /images/plugin_dev_guide_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/plugin_dev_guide_2.png -------------------------------------------------------------------------------- /images/plugin_dev_guide_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/images/plugin_dev_guide_3.png -------------------------------------------------------------------------------- /kingbaseesreader/src/main/java/com/alibaba/datax/plugin/reader/kingbaseesreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.kingbaseesreader; 2 | 3 | public class Constant { 4 | 5 | public static final int DEFAULT_FETCH_SIZE = 1000; 6 | 7 | } 8 | -------------------------------------------------------------------------------- /kingbaseesreader/src/main/libs/kingbase8-8.2.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/kingbaseesreader/src/main/libs/kingbase8-8.2.0.jar -------------------------------------------------------------------------------- /kingbaseesreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kingbaseesreader", 3 | "class": "com.alibaba.datax.plugin.reader.kingbaseesreader.KingbaseesReader", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /kingbaseesreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kingbaseesreader", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "connection": [ 7 | { 8 | "table": [], 9 | "jdbcUrl": [] 10 | } 11 | ] 12 | } 13 | } -------------------------------------------------------------------------------- /kingbaseeswriter/src/main/libs/kingbase8-8.2.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/kingbaseeswriter/src/main/libs/kingbase8-8.2.0.jar -------------------------------------------------------------------------------- /kingbaseeswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kingbaseeswriter", 3 | "class": "com.alibaba.datax.plugin.writer.kingbaseeswriter.KingbaseesWriter", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /kingbaseeswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kingbaseeswriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "column": [], 7 | "preSql": [], 8 | "connection": [ 9 | { 10 | "jdbcUrl": "", 11 | "table": [] 12 | } 13 | ], 14 | "preSql": [], 15 | "postSql": [] 16 | } 17 | } -------------------------------------------------------------------------------- /kuduwriter/README.md: -------------------------------------------------------------------------------- 1 | # datax-kudu-plugin 2 | datax kudu的writer插件 3 | 4 | 5 | 6 | 仅在kudu11进行过测试 7 | -------------------------------------------------------------------------------- /kuduwriter/doc/image-20200901193148188.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/kuduwriter/doc/image-20200901193148188.png -------------------------------------------------------------------------------- /kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/Constant.java: -------------------------------------------------------------------------------- 1 | package com.q1.datax.plugin.writer.kudu11xwriter; 2 | 3 | /** 4 | * @author daizihao 5 | * @create 2020-08-31 14:42 6 | **/ 7 | public class Constant { 8 | public static final String DEFAULT_ENCODING = "UTF-8"; 9 | // public static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; 10 | 11 | public static final String COMPRESSION = "DEFAULT_COMPRESSION"; 12 | public static final String ENCODING = "AUTO_ENCODING"; 13 | public static final Long ADMIN_TIMEOUTMS = 60000L; 14 | public static final Long SESSION_TIMEOUTMS = 60000L; 15 | 16 | 17 | public static final String INSERT_MODE = "upsert"; 18 | public static final long DEFAULT_WRITE_BATCH_SIZE = 512L; 19 | public static final long DEFAULT_MUTATION_BUFFER_SPACE = 3072L; 20 | 21 | } 22 | -------------------------------------------------------------------------------- /kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/InsertModeType.java: -------------------------------------------------------------------------------- 1 | package com.q1.datax.plugin.writer.kudu11xwriter; 2 | 3 | import com.alibaba.datax.common.exception.DataXException; 4 | 5 | import java.util.Arrays; 6 | 7 | /** 8 | * @author daizihao 9 | * @create 2020-08-31 14:47 10 | **/ 11 | public enum InsertModeType { 12 | Insert("insert"), 13 | Upsert("upsert"), 14 | Update("update"); 15 | private String mode; 16 | 17 | InsertModeType(String mode) { 18 | this.mode = mode.toLowerCase(); 19 | } 20 | 21 | public String getMode() { 22 | return mode; 23 | } 24 | 25 | public static InsertModeType getByTypeName(String modeName) { 26 | for (InsertModeType modeType : values()) { 27 | if (modeType.mode.equalsIgnoreCase(modeName)) { 28 | return modeType; 29 | } 30 | } 31 | throw DataXException.asDataXException(Kudu11xWriterErrorcode.ILLEGAL_VALUE, 32 | String.format("Kuduwriter does not support the mode :[%s], currently supported mode types are :%s", modeName, Arrays.asList(values()))); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /kuduwriter/src/main/java/com/q1/kudu/conf/KuduConfig.java: -------------------------------------------------------------------------------- 1 | package com.q1.kudu.conf; 2 | 3 | /** 4 | * @author daizihao 5 | * @create 2020-09-16 11:39 6 | **/ 7 | public class KuduConfig { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /kuduwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kuduwriter", 3 | "class": "com.q1.datax.plugin.writer.kudu11xwriter.Kudu11xWriter", 4 | "description": "use put: prod. mechanism: use kudu java api put data.", 5 | "developer": "com.q1.daizihao" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | Copyright 1999-2017 Alibaba Group Holding Ltd. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /mongodbreader/src/main/java/com/alibaba/datax/plugin/reader/mongodbreader/MongoDBReaderErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.mongodbreader; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | /** 6 | * Created by jianying.wcj on 2015/3/19 0019. 7 | */ 8 | public enum MongoDBReaderErrorCode implements ErrorCode { 9 | 10 | ILLEGAL_VALUE("ILLEGAL_PARAMETER_VALUE","参数不合法"), 11 | ILLEGAL_ADDRESS("ILLEGAL_ADDRESS","不合法的Mongo地址"), 12 | UNEXCEPT_EXCEPTION("UNEXCEPT_EXCEPTION","未知异常"); 13 | 14 | private final String code; 15 | 16 | private final String description; 17 | 18 | private MongoDBReaderErrorCode(String code,String description) { 19 | this.code = code; 20 | this.description = description; 21 | } 22 | 23 | @Override 24 | public String getCode() { 25 | return code; 26 | } 27 | 28 | @Override 29 | public String getDescription() { 30 | return description; 31 | } 32 | } 33 | 34 | -------------------------------------------------------------------------------- /mongodbreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongodbreader", 3 | "class": "com.alibaba.datax.plugin.reader.mongodbreader.MongoDBReader", 4 | "description": "useScene: prod. mechanism: via mongoclient connect mongodb reader data concurrent.", 5 | "developer": "alibaba" 6 | } 7 | -------------------------------------------------------------------------------- /mongodbreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongodbreader", 3 | "parameter": { 4 | "address": [], 5 | "userName": "", 6 | "userPassword": "", 7 | "dbName": "", 8 | "collectionName": "", 9 | "column": [] 10 | } 11 | } -------------------------------------------------------------------------------- /mongodbwriter/src/main/java/com/alibaba/datax/plugin/writer/mongodbwriter/MongoDBWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.mongodbwriter; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum MongoDBWriterErrorCode implements ErrorCode { 6 | 7 | ILLEGAL_VALUE("ILLEGAL_PARAMETER_VALUE","参数不合法"), 8 | ILLEGAL_ADDRESS("ILLEGAL_ADDRESS","不合法的Mongo地址"), 9 | JSONCAST_EXCEPTION("JSONCAST_EXCEPTION","json类型转换异常"), 10 | UNEXCEPT_EXCEPTION("UNEXCEPT_EXCEPTION","未知异常"); 11 | 12 | private final String code; 13 | 14 | private final String description; 15 | 16 | private MongoDBWriterErrorCode(String code,String description) { 17 | this.code = code; 18 | this.description = description; 19 | } 20 | 21 | @Override 22 | public String getCode() { 23 | return code; 24 | } 25 | 26 | @Override 27 | public String getDescription() { 28 | return description; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /mongodbwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongodbwriter", 3 | "class": "com.alibaba.datax.plugin.writer.mongodbwriter.MongoDBWriter", 4 | "description": "useScene: prod. mechanism: via mongoclient connect mongodb write data concurrent.", 5 | "developer": "alibaba" 6 | } 7 | -------------------------------------------------------------------------------- /mongodbwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongodbwriter", 3 | "parameter": { 4 | "address": [], 5 | "userName": "", 6 | "userPassword": "", 7 | "dbName": "", 8 | "collectionName": "", 9 | "column": [], 10 | "upsertInfo": { 11 | "isUpsert": "", 12 | "upsertKey": "" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /mysqlreader/src/main/java/com/alibaba/datax/plugin/reader/mysqlreader/MysqlReaderErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.mysqlreader; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum MysqlReaderErrorCode implements ErrorCode { 6 | ; 7 | 8 | private final String code; 9 | private final String description; 10 | 11 | private MysqlReaderErrorCode(String code, String description) { 12 | this.code = code; 13 | this.description = description; 14 | } 15 | 16 | @Override 17 | public String getCode() { 18 | return this.code; 19 | } 20 | 21 | @Override 22 | public String getDescription() { 23 | return this.description; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return String.format("Code:[%s], Description:[%s]. ", this.code, 29 | this.description); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /mysqlreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mysqlreader", 3 | "class": "com.alibaba.datax.plugin.reader.mysqlreader.MysqlReader", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /mysqlreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mysqlreader", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "column": [], 7 | "connection": [ 8 | { 9 | "jdbcUrl": [], 10 | "table": [] 11 | } 12 | ], 13 | "where": "" 14 | } 15 | } -------------------------------------------------------------------------------- /mysqlwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mysqlwriter", 3 | "class": "com.alibaba.datax.plugin.writer.mysqlwriter.MysqlWriter", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /mysqlwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mysqlwriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "writeMode": "", 7 | "column": [], 8 | "session": [], 9 | "preSql": [], 10 | "connection": [ 11 | { 12 | "jdbcUrl": "", 13 | "table": [] 14 | } 15 | ] 16 | } 17 | } -------------------------------------------------------------------------------- /ocswriter/src/main/java/com/alibaba/datax/plugin/writer/ocswriter/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.ocswriter; 2 | 3 | public final class Key { 4 | public final static String USER = "username"; 5 | 6 | public final static String PASSWORD = "password"; 7 | 8 | public final static String PROXY = "proxy"; 9 | 10 | public final static String PORT = "port"; 11 | 12 | public final static String WRITE_MODE = "writeMode"; 13 | 14 | public final static String WRITE_FORMAT = "writeFormat"; 15 | 16 | public final static String FIELD_DELIMITER = "fieldDelimiter"; 17 | 18 | public final static String EXPIRE_TIME = "expireTime"; 19 | 20 | public final static String BATCH_SIZE = "batchSize"; 21 | 22 | public final static String INDEXES = "indexes"; 23 | } 24 | -------------------------------------------------------------------------------- /ocswriter/src/main/java/com/alibaba/datax/plugin/writer/ocswriter/utils/CommonUtils.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.ocswriter.utils; 2 | 3 | public class CommonUtils { 4 | 5 | public static void sleepInMs(long time) { 6 | try{ 7 | Thread.sleep(time); 8 | } catch (InterruptedException e) { 9 | // 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /ocswriter/src/main/java/com/alibaba/datax/plugin/writer/ocswriter/utils/OcsWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.ocswriter.utils; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum OcsWriterErrorCode implements ErrorCode { 6 | REQUIRED_VALUE("OcsWriterErrorCode-000", "参数不能为空"), 7 | ILLEGAL_PARAM_VALUE("OcsWriterErrorCode-001", "参数不合法"), 8 | HOST_UNREACHABLE("OcsWriterErrorCode-002", "服务不可用"), 9 | OCS_INIT_ERROR("OcsWriterErrorCode-003", "初始化ocs client失败"), 10 | DIRTY_RECORD("OcsWriterErrorCode-004", "脏数据"), 11 | SHUTDOWN_FAILED("OcsWriterErrorCode-005", "关闭ocs client失败"), 12 | COMMIT_FAILED("OcsWriterErrorCode-006", "提交数据到ocs失败"); 13 | 14 | private final String code; 15 | private final String description; 16 | 17 | private OcsWriterErrorCode(String code, String description) { 18 | this.code = code; 19 | this.description = description; 20 | } 21 | 22 | @Override 23 | public String getCode() { 24 | return null; 25 | } 26 | 27 | @Override 28 | public String getDescription() { 29 | return null; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /ocswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ocswriter", 3 | "class": "com.alibaba.datax.plugin.writer.ocswriter.OcsWriter", 4 | "description": "set|add|replace|append|prepend record into ocs.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /ocswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ocswriter", 3 | "parameter": { 4 | "proxy": "", 5 | "port": "", 6 | "userName": "", 7 | "password": "", 8 | "writeMode": "", 9 | "writeFormat": "", 10 | "fieldDelimiter": "", 11 | "expireTime": "", 12 | "indexes": "", 13 | "batchSize": "" 14 | } 15 | } -------------------------------------------------------------------------------- /odpsreader/src/main/java/com/alibaba/datax/plugin/reader/odpsreader/ColumnType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.odpsreader; 2 | 3 | public enum ColumnType { 4 | PARTITION, NORMAL, CONSTANT, UNKNOWN, ; 5 | 6 | @Override 7 | public String toString() { 8 | switch (this) { 9 | case PARTITION: 10 | return "partition"; 11 | case NORMAL: 12 | return "normal"; 13 | case CONSTANT: 14 | return "constant"; 15 | default: 16 | return "unknown"; 17 | } 18 | } 19 | 20 | public static ColumnType asColumnType(String columnTypeString) { 21 | if ("partition".equals(columnTypeString)) { 22 | return PARTITION; 23 | } else if ("normal".equals(columnTypeString)) { 24 | return NORMAL; 25 | } else if ("constant".equals(columnTypeString)) { 26 | return CONSTANT; 27 | } else { 28 | return UNKNOWN; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /odpsreader/src/main/java/com/alibaba/datax/plugin/reader/odpsreader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.odpsreader; 2 | 3 | public class Key { 4 | 5 | public final static String ACCESS_ID = "accessId"; 6 | 7 | public final static String ACCESS_KEY = "accessKey"; 8 | 9 | public static final String PROJECT = "project"; 10 | 11 | public final static String TABLE = "table"; 12 | 13 | public final static String PARTITION = "partition"; 14 | 15 | public final static String ODPS_SERVER = "odpsServer"; 16 | 17 | // 线上环境不需要填写，线下环境必填 18 | public final static String TUNNEL_SERVER = "tunnelServer"; 19 | 20 | public final static String COLUMN = "column"; 21 | 22 | // 当值为：partition 则只切分到分区；当值为：record，则当按照分区切分后达不到adviceNum时，继续按照record切分 23 | public final static String SPLIT_MODE = "splitMode"; 24 | 25 | // 账号类型，默认为aliyun，也可能为taobao等其他类型 26 | public final static String ACCOUNT_TYPE = "accountType"; 27 | 28 | public final static String PACKAGE_AUTHORIZED_PROJECT = "packageAuthorizedProject"; 29 | 30 | public final static String IS_COMPRESS = "isCompress"; 31 | 32 | public final static String MAX_RETRY_TIME = "maxRetryTime"; 33 | 34 | } 35 | -------------------------------------------------------------------------------- /odpsreader/src/main/java/com/alibaba/datax/plugin/reader/odpsreader/util/OdpsExceptionMsg.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.odpsreader.util; 2 | 3 | /** 4 | * Created by hongjiao.hj on 2015/6/9. 5 | */ 6 | public class OdpsExceptionMsg { 7 | 8 | public static final String ODPS_PROJECT_NOT_FOUNT = "ODPS-0420111: Project not found"; 9 | 10 | public static final String ODPS_TABLE_NOT_FOUNT = "ODPS-0130131:Table not found"; 11 | 12 | public static final String ODPS_ACCESS_KEY_ID_NOT_FOUND = "ODPS-0410051:Invalid credentials - accessKeyId not found"; 13 | 14 | public static final String ODPS_ACCESS_KEY_INVALID = "ODPS-0410042:Invalid signature value - User signature dose not match"; 15 | 16 | public static final String ODPS_ACCESS_DENY = "ODPS-0420095: Access Denied - Authorization Failed [4002], You doesn't exist in project"; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /odpsreader/src/main/libs/bcprov-jdk15on-1.52.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/odpsreader/src/main/libs/bcprov-jdk15on-1.52.jar -------------------------------------------------------------------------------- /odpsreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "odpsreader", 3 | "class": "com.alibaba.datax.plugin.reader.odpsreader.OdpsReader", 4 | "description": { 5 | "useScene": "prod.", 6 | "mechanism": "TODO", 7 | "warn": "TODO" 8 | }, 9 | "developer": "alibaba" 10 | } -------------------------------------------------------------------------------- /odpsreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "odpsreader", 3 | "parameter": { 4 | "accessId": "", 5 | "accessKey": "", 6 | "project": "", 7 | "table": "", 8 | "partition": [], 9 | "column": [], 10 | "packageAuthorizedProject": "", 11 | "splitMode": "", 12 | "odpsServer": "" 13 | } 14 | } -------------------------------------------------------------------------------- /odpswriter/src/main/java/com/alibaba/datax/plugin/writer/odpswriter/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.odpswriter; 2 | 3 | 4 | public class Constant { 5 | public static final String SKYNET_ACCESSID = "SKYNET_ACCESSID"; 6 | 7 | public static final String SKYNET_ACCESSKEY = "SKYNET_ACCESSKEY"; 8 | 9 | public static final String DEFAULT_ACCOUNT_TYPE = "aliyun"; 10 | 11 | public static final String TAOBAO_ACCOUNT_TYPE = "taobao"; 12 | 13 | public static final String COLUMN_POSITION = "columnPosition"; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /odpswriter/src/main/java/com/alibaba/datax/plugin/writer/odpswriter/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.odpswriter; 2 | 3 | 4 | public final class Key { 5 | 6 | public final static String ODPS_SERVER = "odpsServer"; 7 | 8 | public final static String TUNNEL_SERVER = "tunnelServer"; 9 | 10 | public final static String ACCESS_ID = "accessId"; 11 | 12 | public final static String ACCESS_KEY = "accessKey"; 13 | 14 | public final static String PROJECT = "project"; 15 | 16 | public final static String TABLE = "table"; 17 | 18 | public final static String PARTITION = "partition"; 19 | 20 | public final static String COLUMN = "column"; 21 | 22 | public final static String TRUNCATE = "truncate"; 23 | 24 | public final static String MAX_RETRY_TIME = "maxRetryTime"; 25 | 26 | public final static String BLOCK_SIZE_IN_MB = "blockSizeInMB"; 27 | 28 | //boolean 类型，default:false 29 | public final static String EMPTY_AS_NULL = "emptyAsNull"; 30 | 31 | public final static String ACCOUNT_TYPE = "accountType"; 32 | 33 | public final static String IS_COMPRESS = "isCompress"; 34 | } 35 | -------------------------------------------------------------------------------- /odpswriter/src/main/java/com/alibaba/datax/plugin/writer/odpswriter/util/OdpsExceptionMsg.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.odpswriter.util; 2 | 3 | /** 4 | * Created by hongjiao.hj on 2015/6/9. 5 | */ 6 | public class OdpsExceptionMsg { 7 | 8 | public static final String ODPS_PROJECT_NOT_FOUNT = "ODPS-0420111: Project not found"; 9 | 10 | public static final String ODPS_TABLE_NOT_FOUNT = "ODPS-0130131:Table not found"; 11 | 12 | public static final String ODPS_ACCESS_KEY_ID_NOT_FOUND = "ODPS-0410051:Invalid credentials - accessKeyId not found"; 13 | 14 | public static final String ODPS_ACCESS_KEY_INVALID = "ODPS-0410042:Invalid signature value - User signature dose not match"; 15 | 16 | public static final String ODPS_ACCESS_DENY = "ODPS-0420095: Access Denied - Authorization Failed [4002], You doesn't exist in project"; 17 | 18 | } 19 | -------------------------------------------------------------------------------- /odpswriter/src/main/libs/bcprov-jdk15on-1.52.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/odpswriter/src/main/libs/bcprov-jdk15on-1.52.jar -------------------------------------------------------------------------------- /odpswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "odpswriter", 3 | "class": "com.alibaba.datax.plugin.writer.odpswriter.OdpsWriter", 4 | "description": { 5 | "useScene": "prod.", 6 | "mechanism": "TODO", 7 | "warn": "TODO" 8 | }, 9 | "developer": "alibaba" 10 | } -------------------------------------------------------------------------------- /odpswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "odpswriter", 3 | "parameter": { 4 | "project": "", 5 | "table": "", 6 | "partition":"", 7 | "column": [], 8 | "accessId": "", 9 | "accessKey": "", 10 | "truncate": true, 11 | "odpsServer": "", 12 | "tunnelServer": "" 13 | } 14 | } -------------------------------------------------------------------------------- /opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.opentsdbreader; 2 | 3 | /** 4 | * Copyright @ 2019 alibaba.com 5 | * All right reserved. 6 | * Function：Key 7 | * 8 | * @author Benedict Jin 9 | * @since 2019-04-18 10 | */ 11 | public final class Constant { 12 | 13 | static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; 14 | } 15 | -------------------------------------------------------------------------------- /opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.opentsdbreader; 2 | 3 | /** 4 | * Copyright @ 2019 alibaba.com 5 | * All right reserved. 6 | * Function：Key 7 | * 8 | * @author Benedict Jin 9 | * @since 2019-04-18 10 | */ 11 | public class Key { 12 | 13 | static final String ENDPOINT = "endpoint"; 14 | static final String COLUMN = "column"; 15 | static final String BEGIN_DATE_TIME = "beginDateTime"; 16 | static final String END_DATE_TIME = "endDateTime"; 17 | } 18 | -------------------------------------------------------------------------------- /opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/opentsdbreader/OpenTSDBReaderErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.opentsdbreader; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | /** 6 | * Copyright @ 2019 alibaba.com 7 | * All right reserved. 8 | * Function：OpenTSDB Reader Error Code 9 | * 10 | * @author Benedict Jin 11 | * @since 2019-04-18 12 | */ 13 | public enum OpenTSDBReaderErrorCode implements ErrorCode { 14 | 15 | REQUIRED_VALUE("OpenTSDBReader-00", "缺失必要的值"), 16 | ILLEGAL_VALUE("OpenTSDBReader-01", "值非法"); 17 | 18 | private final String code; 19 | private final String description; 20 | 21 | OpenTSDBReaderErrorCode(String code, String description) { 22 | this.code = code; 23 | this.description = description; 24 | } 25 | 26 | @Override 27 | public String getCode() { 28 | return this.code; 29 | } 30 | 31 | @Override 32 | public String getDescription() { 33 | return this.description; 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /opentsdbreader/src/main/java/com/alibaba/datax/plugin/reader/util/TimeUtils.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.util; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | 5 | /** 6 | * Copyright @ 2019 alibaba.com 7 | * All right reserved. 8 | * Function：TimeUtils 9 | * 10 | * @author Benedict Jin 11 | * @since 2019-04-22 12 | */ 13 | public final class TimeUtils { 14 | 15 | private TimeUtils() { 16 | } 17 | 18 | private static final long SECOND_MASK = 0xFFFFFFFF00000000L; 19 | private static final long HOUR_IN_MILL = TimeUnit.HOURS.toMillis(1); 20 | 21 | /** 22 | * Weather the timestamp is second. 23 | * 24 | * @param ts timestamp 25 | */ 26 | public static boolean isSecond(long ts) { 27 | return (ts & SECOND_MASK) == 0; 28 | } 29 | 30 | /** 31 | * Get the hour. 32 | * 33 | * @param ms time in millisecond 34 | */ 35 | public static long getTimeInHour(long ms) { 36 | return ms - ms % HOUR_IN_MILL; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /opentsdbreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "opentsdbreader", 3 | "class": "com.alibaba.datax.plugin.reader.opentsdbreader.OpenTSDBReader", 4 | "description": { 5 | "useScene": "从 OpenTSDB 中摄取数据点", 6 | "mechanism": "根据时间和 metric 直连底层 HBase 存储，从而 Scan 出符合条件的数据点", 7 | "warn": "指定起止时间会自动忽略分钟和秒，转为整点时刻，例如 2019-4-18 的 [3:35, 4:55) 会被转为 [3:00, 4:00)" 8 | }, 9 | "developer": "Benedict Jin" 10 | } 11 | -------------------------------------------------------------------------------- /opentsdbreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "opentsdbreader", 3 | "parameter": { 4 | "endpoint": "http://localhost:8242", 5 | "column": [ 6 | "m" 7 | ], 8 | "startTime": "2019-01-01 00:00:00", 9 | "endTime": "2019-01-01 01:00:00" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /opentsdbreader/src/test/java/com/alibaba/datax/plugin/reader/conn/OpenTSDBConnectionTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.conn; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Ignore; 5 | import org.junit.Test; 6 | 7 | /** 8 | * Copyright @ 2019 alibaba.com 9 | * All right reserved. 10 | * Function：OpenTSDB Connection4TSDB Test 11 | * 12 | * @author Benedict Jin 13 | * @since 2019-03-29 14 | */ 15 | @Ignore 16 | public class OpenTSDBConnectionTest { 17 | 18 | private static final String OPENTSDB_ADDRESS = "http://localhost:8242"; 19 | 20 | @Test 21 | public void testVersion() { 22 | String version = new OpenTSDBConnection(OPENTSDB_ADDRESS).version(); 23 | Assert.assertNotNull(version); 24 | } 25 | 26 | @Test 27 | public void testIsSupported() { 28 | Assert.assertTrue(new OpenTSDBConnection(OPENTSDB_ADDRESS).isSupported()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /opentsdbreader/src/test/java/com/alibaba/datax/plugin/reader/util/Const.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.util; 2 | 3 | /** 4 | * Copyright @ 2019 alibaba.com 5 | * All right reserved. 6 | * Function：Const 7 | * 8 | * @author Benedict Jin 9 | * @since 2019-03-29 10 | */ 11 | final class Const { 12 | 13 | private Const() { 14 | } 15 | 16 | static final String OPENTSDB_ADDRESS = "http://localhost:8242"; 17 | static final String TSDB_ADDRESS = "http://localhost:8240"; 18 | } 19 | -------------------------------------------------------------------------------- /opentsdbreader/src/test/java/com/alibaba/datax/plugin/reader/util/HttpUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.util; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Ignore; 5 | import org.junit.Test; 6 | 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | /** 11 | * Copyright @ 2019 alibaba.com 12 | * All right reserved. 13 | * Function：HttpUtils Test 14 | * 15 | * @author Benedict Jin 16 | * @since 2019-03-29 17 | */ 18 | @Ignore 19 | public class HttpUtilsTest { 20 | 21 | @Test 22 | public void testSimpleCase() throws Exception { 23 | String url = "https://httpbin.org/post"; 24 | Map params = new HashMap(); 25 | params.put("foo", "bar"); 26 | 27 | String rsp = HttpUtils.post(url, params); 28 | System.out.println(rsp); 29 | Assert.assertNotNull(rsp); 30 | } 31 | 32 | @Test 33 | public void testGet() throws Exception { 34 | String url = String.format("%s/api/version", Const.OPENTSDB_ADDRESS); 35 | String rsp = HttpUtils.get(url); 36 | System.out.println(rsp); 37 | Assert.assertNotNull(rsp); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /opentsdbreader/src/test/java/com/alibaba/datax/plugin/reader/util/TSDBTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.util; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Ignore; 5 | import org.junit.Test; 6 | 7 | /** 8 | * Copyright @ 2019 alibaba.com 9 | * All right reserved. 10 | * Function：TSDB Test 11 | * 12 | * @author Benedict Jin 13 | * @since 2019-04-11 14 | */ 15 | @Ignore 16 | public class TSDBTest { 17 | 18 | @Test 19 | public void testVersion() { 20 | String version = TSDBUtils.version(Const.TSDB_ADDRESS); 21 | Assert.assertNotNull(version); 22 | System.out.println(version); 23 | 24 | version = TSDBUtils.version(Const.OPENTSDB_ADDRESS); 25 | Assert.assertNotNull(version); 26 | System.out.println(version); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /opentsdbreader/src/test/java/com/alibaba/datax/plugin/reader/util/TimeUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.util; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | import java.text.ParseException; 7 | import java.text.SimpleDateFormat; 8 | import java.util.Date; 9 | 10 | /** 11 | * Copyright @ 2019 alibaba.com 12 | * All right reserved. 13 | * Function：com.alibaba.datax.common.util 14 | * 15 | * @author Benedict Jin 16 | * @since 2019-04-22 17 | */ 18 | public class TimeUtilsTest { 19 | 20 | @Test 21 | public void testIsSecond() { 22 | Assert.assertFalse(TimeUtils.isSecond(System.currentTimeMillis())); 23 | Assert.assertTrue(TimeUtils.isSecond(System.currentTimeMillis() / 1000)); 24 | } 25 | 26 | @Test 27 | public void testGetTimeInHour() throws ParseException { 28 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 29 | Date date = sdf.parse("2019-04-18 15:32:33"); 30 | long timeInHour = TimeUtils.getTimeInHour(date.getTime()); 31 | Assert.assertEquals("2019-04-18 15:00:00", sdf.format(timeInHour)); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /oraclereader/src/main/java/com/alibaba/datax/plugin/reader/oraclereader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.oraclereader; 2 | 3 | public class Constant { 4 | 5 | public static final int DEFAULT_FETCH_SIZE = 1024; 6 | 7 | } 8 | -------------------------------------------------------------------------------- /oraclereader/src/main/java/com/alibaba/datax/plugin/reader/oraclereader/OracleReaderErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.oraclereader; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum OracleReaderErrorCode implements ErrorCode { 6 | HINT_ERROR("Oraclereader-00", "您的 Hint 配置出错."), 7 | 8 | ; 9 | 10 | private final String code; 11 | private final String description; 12 | 13 | private OracleReaderErrorCode(String code, String description) { 14 | this.code = code; 15 | this.description = description; 16 | } 17 | 18 | @Override 19 | public String getCode() { 20 | return this.code; 21 | } 22 | 23 | @Override 24 | public String getDescription() { 25 | return this.description; 26 | } 27 | 28 | @Override 29 | public String toString() { 30 | return String.format("Code:[%s], Description:[%s]. ", this.code, 31 | this.description); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /oraclereader/src/main/lib/ojdbc6-11.2.0.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/oraclereader/src/main/lib/ojdbc6-11.2.0.3.jar -------------------------------------------------------------------------------- /oraclereader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "oraclereader", 3 | "class": "com.alibaba.datax.plugin.reader.oraclereader.OracleReader", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /oraclereader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "oraclereader", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "column": [], 7 | "connection": [ 8 | { 9 | "table": [], 10 | "jdbcUrl": [] 11 | } 12 | ] 13 | } 14 | } -------------------------------------------------------------------------------- /oraclewriter/src/main/java/com/alibaba/datax/plugin/writer/oraclewriter/OracleWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.oraclewriter; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum OracleWriterErrorCode implements ErrorCode { 6 | ; 7 | 8 | private final String code; 9 | private final String describe; 10 | 11 | private OracleWriterErrorCode(String code, String describe) { 12 | this.code = code; 13 | this.describe = describe; 14 | } 15 | 16 | @Override 17 | public String getCode() { 18 | return this.code; 19 | } 20 | 21 | @Override 22 | public String getDescription() { 23 | return this.describe; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return String.format("Code:[%s], Describe:[%s]. ", this.code, 29 | this.describe); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /oraclewriter/src/main/lib/ojdbc6-11.2.0.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/oraclewriter/src/main/lib/ojdbc6-11.2.0.3.jar -------------------------------------------------------------------------------- /oraclewriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "oraclewriter", 3 | "class": "com.alibaba.datax.plugin.writer.oraclewriter.OracleWriter", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /oraclewriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "oraclewriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "column": [], 7 | "preSql": [], 8 | "connection": [ 9 | { 10 | "jdbcUrl": "", 11 | "table": [] 12 | } 13 | ] 14 | } 15 | } -------------------------------------------------------------------------------- /oscarwriter/src/main/java/com/alibaba/datax/plugin/writer/oscarwriter/OscarWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.oscarwriter; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum OscarWriterErrorCode implements ErrorCode { 6 | ; 7 | 8 | private final String code; 9 | private final String describe; 10 | 11 | private OscarWriterErrorCode(String code, String describe) { 12 | this.code = code; 13 | this.describe = describe; 14 | } 15 | 16 | @Override 17 | public String getCode() { 18 | return this.code; 19 | } 20 | 21 | @Override 22 | public String getDescription() { 23 | return this.describe; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return String.format("Code:[%s], Describe:[%s]. ", this.code, 29 | this.describe); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /oscarwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "oscarwriter", 3 | "class": "com.alibaba.datax.plugin.writer.oscarwriter.OscarWriter", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "linjiayu" 6 | } -------------------------------------------------------------------------------- /oscarwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "oscarwriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "column": [], 7 | "preSql": [], 8 | "connection": [ 9 | { 10 | "jdbcUrl": "", 11 | "table": [] 12 | } 13 | ] 14 | } 15 | } -------------------------------------------------------------------------------- /ossreader/src/main/java/com/alibaba/datax/plugin/reader/ossreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.ossreader; 2 | 3 | /** 4 | * Created by mengxin.liumx on 2014/12/7. 5 | */ 6 | public class Constant { 7 | 8 | public static final String OBJECT = "object"; 9 | public static final int SOCKETTIMEOUT = 5000000; 10 | } 11 | -------------------------------------------------------------------------------- /ossreader/src/main/java/com/alibaba/datax/plugin/reader/ossreader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.ossreader; 2 | 3 | /** 4 | * Created by mengxin.liumx on 2014/12/7. 5 | */ 6 | public class Key { 7 | public static final String ENDPOINT = "endpoint"; 8 | 9 | public static final String ACCESSID = "accessId"; 10 | 11 | public static final String ACCESSKEY = "accessKey"; 12 | 13 | public static final String ENCODING = "encoding"; 14 | 15 | public static final String BUCKET = "bucket"; 16 | 17 | public static final String OBJECT = "object"; 18 | 19 | public static final String CNAME = "cname"; 20 | 21 | } 22 | -------------------------------------------------------------------------------- /ossreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ossreader", 3 | "class": "com.alibaba.datax.plugin.reader.ossreader.OssReader", 4 | "description": "", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /ossreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ossreader", 3 | "parameter": { 4 | "endpoint": "", 5 | "accessId": "", 6 | "accessKey": "", 7 | "bucket": "", 8 | "object": [], 9 | "column": [], 10 | "encoding": "", 11 | "fieldDelimiter": "", 12 | "compress": "" 13 | } 14 | } -------------------------------------------------------------------------------- /osswriter/src/main/java/com/alibaba/datax/plugin/writer/osswriter/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.osswriter; 2 | 3 | /** 4 | * Created by haiwei.luo on 15-02-09. 5 | */ 6 | public class Constant { 7 | public static final String OBJECT = "object"; 8 | public static final int SOCKETTIMEOUT = 5000000; 9 | } 10 | -------------------------------------------------------------------------------- /osswriter/src/main/java/com/alibaba/datax/plugin/writer/osswriter/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.osswriter; 2 | 3 | /** 4 | * Created by haiwei.luo on 15-02-09. 5 | */ 6 | public class Key { 7 | public static final String ENDPOINT = "endpoint"; 8 | 9 | public static final String ACCESSID = "accessId"; 10 | 11 | public static final String ACCESSKEY = "accessKey"; 12 | 13 | public static final String BUCKET = "bucket"; 14 | 15 | public static final String OBJECT = "object"; 16 | 17 | public static final String CNAME = "cname"; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /osswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "osswriter", 3 | "class": "com.alibaba.datax.plugin.writer.osswriter.OssWriter", 4 | "description": "", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /osswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "osswriter", 3 | "parameter": { 4 | "endpoint": "", 5 | "accessId": "", 6 | "accessKey": "", 7 | "bucket": "", 8 | "object": "", 9 | "encoding": "", 10 | "fieldDelimiter": "", 11 | "writeMode": "" 12 | } 13 | } -------------------------------------------------------------------------------- /otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/callable/GetTableMetaCallable.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsreader.callable; 2 | 3 | import java.util.concurrent.Callable; 4 | 5 | import com.aliyun.openservices.ots.OTSClient; 6 | import com.aliyun.openservices.ots.model.DescribeTableRequest; 7 | import com.aliyun.openservices.ots.model.DescribeTableResult; 8 | import com.aliyun.openservices.ots.model.TableMeta; 9 | 10 | public class GetTableMetaCallable implements Callable{ 11 | 12 | private OTSClient ots = null; 13 | private String tableName = null; 14 | 15 | public GetTableMetaCallable(OTSClient ots, String tableName) { 16 | this.ots = ots; 17 | this.tableName = tableName; 18 | } 19 | 20 | @Override 21 | public TableMeta call() throws Exception { 22 | DescribeTableRequest describeTableRequest = new DescribeTableRequest(); 23 | describeTableRequest.setTableName(tableName); 24 | DescribeTableResult result = ots.describeTable(describeTableRequest); 25 | TableMeta tableMeta = result.getTableMeta(); 26 | return tableMeta; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSConst.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsreader.model; 2 | 3 | public class OTSConst { 4 | // Reader support type 5 | public final static String TYPE_STRING = "STRING"; 6 | public final static String TYPE_INTEGER = "INT"; 7 | public final static String TYPE_DOUBLE = "DOUBLE"; 8 | public final static String TYPE_BOOLEAN = "BOOL"; 9 | public final static String TYPE_BINARY = "BINARY"; 10 | public final static String TYPE_INF_MIN = "INF_MIN"; 11 | public final static String TYPE_INF_MAX = "INF_MAX"; 12 | 13 | // Column 14 | public final static String NAME = "name"; 15 | public final static String TYPE = "type"; 16 | public final static String VALUE = "value"; 17 | 18 | public final static String OTS_CONF = "OTS_CONF"; 19 | public final static String OTS_RANGE = "OTS_RANGE"; 20 | public final static String OTS_DIRECTION = "OTS_DIRECTION"; 21 | 22 | // options 23 | public final static String RETRY = "maxRetryTime"; 24 | public final static String SLEEP_IN_MILLI_SECOND = "retrySleepInMillionSecond"; 25 | } 26 | -------------------------------------------------------------------------------- /otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSPrimaryKeyColumn.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsreader.model; 2 | 3 | import com.aliyun.openservices.ots.model.PrimaryKeyType; 4 | 5 | public class OTSPrimaryKeyColumn { 6 | private String name; 7 | private PrimaryKeyType type; 8 | 9 | public String getName() { 10 | return name; 11 | } 12 | public void setName(String name) { 13 | this.name = name; 14 | } 15 | public PrimaryKeyType getType() { 16 | return type; 17 | } 18 | public void setType(PrimaryKeyType type) { 19 | this.type = type; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/model/OTSRange.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsreader.model; 2 | 3 | import com.aliyun.openservices.ots.model.RowPrimaryKey; 4 | 5 | public class OTSRange { 6 | 7 | private RowPrimaryKey begin = null; 8 | private RowPrimaryKey end = null; 9 | 10 | public OTSRange() {} 11 | 12 | public OTSRange(RowPrimaryKey begin, RowPrimaryKey end) { 13 | this.begin = begin; 14 | this.end = end; 15 | } 16 | 17 | public RowPrimaryKey getBegin() { 18 | return begin; 19 | } 20 | public void setBegin(RowPrimaryKey begin) { 21 | this.begin = begin; 22 | } 23 | public RowPrimaryKey getEnd() { 24 | return end; 25 | } 26 | public void setEnd(RowPrimaryKey end) { 27 | this.end = end; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /otsreader/src/main/java/com/alibaba/datax/plugin/reader/otsreader/utils/DefaultNoRetry.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsreader.utils; 2 | 3 | import com.aliyun.openservices.ots.internal.OTSDefaultRetryStrategy; 4 | 5 | public class DefaultNoRetry extends OTSDefaultRetryStrategy { 6 | 7 | @Override 8 | public boolean shouldRetry(String action, Exception ex, int retries) { 9 | return false; 10 | } 11 | 12 | } -------------------------------------------------------------------------------- /otsreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "otsreader", 3 | "class": "com.alibaba.datax.plugin.reader.otsreader.OtsReader", 4 | "description": "", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /otsreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "otsreader", 3 | "parameter": { 4 | "endpoint":"", 5 | "accessId":"", 6 | "accessKey":"", 7 | "instanceName":"", 8 | "column":[], 9 | "range":{ 10 | "begin":[], 11 | "end":[] 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/OTSReaderError.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public class OTSReaderError implements ErrorCode { 6 | 7 | private String code; 8 | 9 | private String description; 10 | 11 | public final static OTSReaderError ERROR = new OTSReaderError("OTSStreamReaderError", "OTS Stream Reader Error"); 12 | 13 | public final static OTSReaderError INVALID_PARAM = new OTSReaderError( 14 | "OTSStreamReaderInvalidParameter", "OTS Stream Reader Invalid Parameter"); 15 | 16 | public OTSReaderError(String code, String description) { 17 | this.code = code; 18 | this.description = description; 19 | } 20 | 21 | public String getCode() { 22 | return this.code; 23 | } 24 | 25 | public String getDescription() { 26 | return this.description; 27 | } 28 | 29 | public String toString() { 30 | return "[ code:" + this.code + ", message" + this.description + "]"; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/OTSStreamReaderException.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal; 2 | 3 | public class OTSStreamReaderException extends RuntimeException { 4 | 5 | public OTSStreamReaderException(String message) { 6 | super(message); 7 | } 8 | 9 | public OTSStreamReaderException(String message, Exception cause) { 10 | super(message, cause); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/config/Mode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal.config; 2 | 3 | public enum Mode { 4 | 5 | MULTI_VERSION, 6 | 7 | SINGLE_VERSION_AND_UPDATE_ONLY 8 | } 9 | -------------------------------------------------------------------------------- /otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/core/IStreamRecordSender.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal.core; 2 | 3 | import com.alicloud.openservices.tablestore.model.StreamRecord; 4 | 5 | public interface IStreamRecordSender { 6 | 7 | void sendToDatax(StreamRecord streamRecord); 8 | 9 | } 10 | -------------------------------------------------------------------------------- /otsstreamreader/src/main/java/com/alibaba/datax/plugin/reader/otsstreamreader/internal/utils/ParamChecker.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.otsstreamreader.internal.utils; 2 | 3 | import com.alibaba.datax.common.util.Configuration; 4 | 5 | public class ParamChecker { 6 | 7 | private static void throwNotExistException() { 8 | throw new IllegalArgumentException("missing the key."); 9 | } 10 | 11 | private static void throwStringLengthZeroException() { 12 | throw new IllegalArgumentException("input the key is empty string."); 13 | } 14 | 15 | public static String checkStringAndGet(Configuration param, String key, boolean isTrim) { 16 | try { 17 | String value = param.getString(key); 18 | if (isTrim) { 19 | value = value != null ? value.trim() : null; 20 | } 21 | if (null == value) { 22 | throwNotExistException(); 23 | } else if (value.length() == 0) { 24 | throwStringLengthZeroException(); 25 | } 26 | return value; 27 | } catch(RuntimeException e) { 28 | throw e; 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /otsstreamreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "otsstreamreader", 3 | "class": "com.alibaba.datax.plugin.reader.otsstreamreader.internal.OTSStreamReader", 4 | "description": "", 5 | "developer": "zhaofeng.zhou@alibaba-inc.com" 6 | } 7 | -------------------------------------------------------------------------------- /otsstreamreader/tools/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "endpoint" : "", 3 | "accessId" : "", 4 | "accessKey" : "", 5 | "instanceName" : "", 6 | "statusTable" : "" 7 | } 8 | -------------------------------------------------------------------------------- /otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/WriterRetryPolicy.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.otswriter; 2 | 3 | import com.alibaba.datax.plugin.writer.otswriter.model.OTSConf; 4 | import com.aliyun.openservices.ots.internal.OTSRetryStrategy; 5 | 6 | public class WriterRetryPolicy implements OTSRetryStrategy { 7 | OTSConf conf; 8 | 9 | public WriterRetryPolicy(OTSConf conf) { 10 | this.conf = conf; 11 | } 12 | 13 | @Override 14 | public boolean shouldRetry(String action, Exception ex, int retries) { 15 | return retries <= conf.getRetry(); 16 | } 17 | 18 | @Override 19 | public long getPauseDelay(String action, Exception ex, int retries) { 20 | if (retries <= 0) { 21 | return 0; 22 | } 23 | 24 | int sleepTime = conf.getSleepInMillisecond() * retries; 25 | return sleepTime > 30000 ? 30000 : sleepTime; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/callable/GetTableMetaCallable.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.otswriter.callable; 2 | 3 | import java.util.concurrent.Callable; 4 | 5 | import com.aliyun.openservices.ots.OTSClient; 6 | import com.aliyun.openservices.ots.model.DescribeTableRequest; 7 | import com.aliyun.openservices.ots.model.DescribeTableResult; 8 | import com.aliyun.openservices.ots.model.TableMeta; 9 | 10 | public class GetTableMetaCallable implements Callable{ 11 | 12 | private OTSClient ots = null; 13 | private String tableName = null; 14 | 15 | public GetTableMetaCallable(OTSClient ots, String tableName) { 16 | this.ots = ots; 17 | this.tableName = tableName; 18 | } 19 | 20 | @Override 21 | public TableMeta call() throws Exception { 22 | DescribeTableRequest describeTableRequest = new DescribeTableRequest(); 23 | describeTableRequest.setTableName(tableName); 24 | DescribeTableResult result = ots.describeTable(describeTableRequest); 25 | TableMeta tableMeta = result.getTableMeta(); 26 | return tableMeta; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/OTSAttrColumn.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.otswriter.model; 2 | 3 | import com.aliyun.openservices.ots.model.ColumnType; 4 | 5 | public class OTSAttrColumn { 6 | private String name; 7 | private ColumnType type; 8 | 9 | public OTSAttrColumn(String name, ColumnType type) { 10 | this.name = name; 11 | this.type = type; 12 | } 13 | 14 | public String getName() { 15 | return name; 16 | } 17 | 18 | public ColumnType getType() { 19 | return type; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/OTSOpType.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.otswriter.model; 2 | 3 | public enum OTSOpType { 4 | PUT_ROW, 5 | UPDATE_ROW, 6 | DELETE_ROW 7 | } 8 | -------------------------------------------------------------------------------- /otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/OTSPKColumn.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.otswriter.model; 2 | 3 | import com.aliyun.openservices.ots.model.PrimaryKeyType; 4 | 5 | public class OTSPKColumn { 6 | private String name; 7 | private PrimaryKeyType type; 8 | 9 | public OTSPKColumn(String name, PrimaryKeyType type) { 10 | this.name = name; 11 | this.type = type; 12 | } 13 | 14 | public PrimaryKeyType getType() { 15 | return type; 16 | } 17 | 18 | public String getName() { 19 | return name; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/RowDeleteChangeWithRecord.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.otswriter.model; 2 | 3 | import com.alibaba.datax.common.element.Record; 4 | 5 | public class RowDeleteChangeWithRecord extends com.aliyun.openservices.ots.model.RowDeleteChange implements WithRecord { 6 | 7 | private Record record; 8 | 9 | public RowDeleteChangeWithRecord(String tableName) { 10 | super(tableName); 11 | } 12 | 13 | @Override 14 | public Record getRecord() { 15 | return record; 16 | } 17 | 18 | @Override 19 | public void setRecord(Record record) { 20 | this.record = record; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/RowPutChangeWithRecord.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.otswriter.model; 2 | 3 | import com.alibaba.datax.common.element.Record; 4 | 5 | public class RowPutChangeWithRecord extends com.aliyun.openservices.ots.model.RowPutChange implements WithRecord { 6 | 7 | private Record record; 8 | 9 | public RowPutChangeWithRecord(String tableName) { 10 | super(tableName); 11 | } 12 | 13 | @Override 14 | public Record getRecord() { 15 | return record; 16 | } 17 | 18 | @Override 19 | public void setRecord(Record record) { 20 | this.record = record; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/RowUpdateChangeWithRecord.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.otswriter.model; 2 | 3 | import com.alibaba.datax.common.element.Record; 4 | 5 | public class RowUpdateChangeWithRecord extends com.aliyun.openservices.ots.model.RowUpdateChange implements WithRecord { 6 | 7 | private Record record; 8 | 9 | public RowUpdateChangeWithRecord(String tableName) { 10 | super(tableName); 11 | } 12 | 13 | @Override 14 | public Record getRecord() { 15 | return record; 16 | } 17 | 18 | @Override 19 | public void setRecord(Record record) { 20 | this.record = record; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /otswriter/src/main/java/com/alibaba/datax/plugin/writer/otswriter/model/WithRecord.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.otswriter.model; 2 | 3 | import com.alibaba.datax.common.element.Record; 4 | 5 | public interface WithRecord { 6 | Record getRecord(); 7 | 8 | void setRecord(Record record); 9 | } 10 | -------------------------------------------------------------------------------- /otswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "otswriter", 3 | "class": "com.alibaba.datax.plugin.writer.otswriter.OtsWriter", 4 | "description": "", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /otswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "otswriter", 3 | "parameter": { 4 | "endpoint":"", 5 | "accessId":"", 6 | "accessKey":"", 7 | "instanceName":"", 8 | "table":"", 9 | "primaryKey" : [], 10 | "column" : [], 11 | "writeMode" : "" 12 | } 13 | } -------------------------------------------------------------------------------- /plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/reader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.rdbms.reader; 2 | 3 | public final class Constant { 4 | public static final String PK_TYPE = "pkType"; 5 | 6 | public static final Object PK_TYPE_STRING = "pkTypeString"; 7 | 8 | public static final Object PK_TYPE_LONG = "pkTypeLong"; 9 | 10 | public static final Object PK_TYPE_MONTECARLO = "pkTypeMonteCarlo"; 11 | 12 | public static final String SPLIT_MODE_RANDOMSAMPLE = "randomSampling"; 13 | 14 | public static String CONN_MARK = "connection"; 15 | 16 | public static String TABLE_NUMBER_MARK = "tableNumber"; 17 | 18 | public static String IS_TABLE_MODE = "isTableMode"; 19 | 20 | public final static String FETCH_SIZE = "fetchSize"; 21 | 22 | public static String QUERY_SQL_TEMPLATE_WITHOUT_WHERE = "select %s from %s "; 23 | 24 | public static String QUERY_SQL_TEMPLATE = "select %s from %s where (%s)"; 25 | 26 | public static String TABLE_NAME_PLACEHOLDER = "@table"; 27 | 28 | public static Integer SPLIT_FACTOR = 5; 29 | 30 | } 31 | -------------------------------------------------------------------------------- /plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/util/ConnectionFactory.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.rdbms.util; 2 | 3 | import java.sql.Connection; 4 | 5 | /** 6 | * Date: 15/3/16 下午2:17 7 | */ 8 | public interface ConnectionFactory { 9 | 10 | public Connection getConnecttion(); 11 | 12 | public Connection getConnecttionWithoutRetry(); 13 | 14 | public String getConnectionInfo(); 15 | 16 | } 17 | -------------------------------------------------------------------------------- /plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.rdbms.writer; 2 | 3 | /** 4 | * 用于插件解析用户配置时，需要进行标识（MARK）的常量的声明. 5 | */ 6 | public final class Constant { 7 | public static final int DEFAULT_BATCH_SIZE = 2048; 8 | 9 | public static final int DEFAULT_BATCH_BYTE_SIZE = 32 * 1024 * 1024; 10 | 11 | public static String TABLE_NAME_PLACEHOLDER = "@table"; 12 | 13 | public static String CONN_MARK = "connection"; 14 | 15 | public static String TABLE_NUMBER_MARK = "tableNumber"; 16 | 17 | public static String INSERT_OR_REPLACE_TEMPLATE_MARK = "insertOrReplaceTemplate"; 18 | 19 | public static final String OB10_SPLIT_STRING = "||_dsc_ob10_dsc_||"; 20 | public static final String OB10_SPLIT_STRING_PATTERN = "\\|\\|_dsc_ob10_dsc_\\|\\|"; 21 | 22 | } 23 | -------------------------------------------------------------------------------- /plugin-rdbms-util/src/main/java/com/alibaba/datax/plugin/rdbms/writer/MysqlWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.rdbms.writer; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | //TODO 后续考虑与 util 包种的 DBUTilErrorCode 做合并.（区分读和写的错误码） 6 | public enum MysqlWriterErrorCode implements ErrorCode { 7 | ; 8 | 9 | private final String code; 10 | private final String describe; 11 | 12 | private MysqlWriterErrorCode(String code, String describe) { 13 | this.code = code; 14 | this.describe = describe; 15 | } 16 | 17 | @Override 18 | public String getCode() { 19 | return this.code; 20 | } 21 | 22 | @Override 23 | public String getDescription() { 24 | return this.describe; 25 | } 26 | 27 | @Override 28 | public String toString() { 29 | return String.format("Code:[%s], Describe:[%s]. ", this.code, 30 | this.describe); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/reader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.unstructuredstorage.reader; 2 | 3 | public class Constant { 4 | public static final String DEFAULT_ENCODING = "UTF-8"; 5 | 6 | public static final char DEFAULT_FIELD_DELIMITER = ','; 7 | 8 | public static final boolean DEFAULT_SKIP_HEADER = false; 9 | 10 | public static final String DEFAULT_NULL_FORMAT = "\\N"; 11 | 12 | public static final Integer DEFAULT_BUFFER_SIZE = 8192; 13 | } 14 | -------------------------------------------------------------------------------- /plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/reader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.unstructuredstorage.reader; 2 | 3 | /** 4 | * Created by haiwei.luo on 14-12-5. 5 | */ 6 | public class Key { 7 | public static final String COLUMN = "column"; 8 | 9 | public static final String ENCODING = "encoding"; 10 | 11 | public static final String FIELD_DELIMITER = "fieldDelimiter"; 12 | 13 | public static final String SKIP_HEADER = "skipHeader"; 14 | 15 | public static final String TYPE = "type"; 16 | 17 | public static final String FORMAT = "format"; 18 | 19 | public static final String INDEX = "index"; 20 | 21 | public static final String VALUE = "value"; 22 | 23 | public static final String COMPRESS = "compress"; 24 | 25 | public static final String NULL_FORMAT = "nullFormat"; 26 | 27 | public static final String BUFFER_SIZE = "bufferSize"; 28 | 29 | public static final String CSV_READER_CONFIG = "csvReaderConfig"; 30 | 31 | } 32 | -------------------------------------------------------------------------------- /plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.unstructuredstorage.writer; 2 | 3 | public class Constant { 4 | 5 | public static final String DEFAULT_ENCODING = "UTF-8"; 6 | 7 | public static final char DEFAULT_FIELD_DELIMITER = ','; 8 | 9 | public static final String DEFAULT_NULL_FORMAT = "\\N"; 10 | 11 | public static final String FILE_FORMAT_CSV = "csv"; 12 | 13 | public static final String FILE_FORMAT_TEXT = "text"; 14 | 15 | //每个分块10MB，最大10000个分块 16 | public static final Long MAX_FILE_SIZE = 1024 * 1024 * 10 * 10000L; 17 | 18 | public static final String DEFAULT_SUFFIX = ""; 19 | } 20 | -------------------------------------------------------------------------------- /plugin-unstructured-storage-util/src/main/java/com/alibaba/datax/plugin/unstructuredstorage/writer/UnstructuredWriter.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.unstructuredstorage.writer; 2 | 3 | import java.io.Closeable; 4 | import java.io.IOException; 5 | import java.util.List; 6 | 7 | public interface UnstructuredWriter extends Closeable { 8 | 9 | public void writeOneRecord(List splitedRows) throws IOException; 10 | 11 | public void flush() throws IOException; 12 | 13 | public void close() throws IOException; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /postgresqlreader/src/main/java/com/alibaba/datax/plugin/reader/postgresqlreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.postgresqlreader; 2 | 3 | public class Constant { 4 | 5 | public static final int DEFAULT_FETCH_SIZE = 1000; 6 | 7 | } 8 | -------------------------------------------------------------------------------- /postgresqlreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "postgresqlreader", 3 | "class": "com.alibaba.datax.plugin.reader.postgresqlreader.PostgresqlReader", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /postgresqlreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "postgresqlreader", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "connection": [ 7 | { 8 | "table": [], 9 | "jdbcUrl": [] 10 | } 11 | ] 12 | } 13 | } -------------------------------------------------------------------------------- /postgresqlwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "postgresqlwriter", 3 | "class": "com.alibaba.datax.plugin.writer.postgresqlwriter.PostgresqlWriter", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /postgresqlwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "postgresqlwriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "column": [], 7 | "preSql": [], 8 | "connection": [ 9 | { 10 | "jdbcUrl": "", 11 | "table": [] 12 | } 13 | ], 14 | "preSql": [], 15 | "postSql": [] 16 | } 17 | } -------------------------------------------------------------------------------- /rdbmsreader/src/main/java/com/alibaba/datax/plugin/reader/rdbmsreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.rdbmsreader; 2 | 3 | public class Constant { 4 | 5 | public static final int DEFAULT_FETCH_SIZE = 1000; 6 | 7 | } 8 | -------------------------------------------------------------------------------- /rdbmsreader/src/main/libs/Dm7JdbcDriver16.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/rdbmsreader/src/main/libs/Dm7JdbcDriver16.jar -------------------------------------------------------------------------------- /rdbmsreader/src/main/libs/db2jcc4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/rdbmsreader/src/main/libs/db2jcc4.jar -------------------------------------------------------------------------------- /rdbmsreader/src/main/libs/edb-jdbc16.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/rdbmsreader/src/main/libs/edb-jdbc16.jar -------------------------------------------------------------------------------- /rdbmsreader/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/rdbmsreader/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar -------------------------------------------------------------------------------- /rdbmsreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rdbmsreader", 3 | "class": "com.alibaba.datax.plugin.reader.rdbmsreader.RdbmsReader", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba", 6 | "drivers":["dm.jdbc.driver.DmDriver", "com.sybase.jdbc3.jdbc.SybDriver", "com.edb.Driver"] 7 | } 8 | -------------------------------------------------------------------------------- /rdbmsreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rdbmsreader", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "column": [], 7 | "connection": [ 8 | { 9 | "jdbcUrl": [], 10 | "table": [] 11 | } 12 | ], 13 | "where": "" 14 | } 15 | } -------------------------------------------------------------------------------- /rdbmswriter/src/main/libs/Dm7JdbcDriver16.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/rdbmswriter/src/main/libs/Dm7JdbcDriver16.jar -------------------------------------------------------------------------------- /rdbmswriter/src/main/libs/db2jcc4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/rdbmswriter/src/main/libs/db2jcc4.jar -------------------------------------------------------------------------------- /rdbmswriter/src/main/libs/edb-jdbc16.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/rdbmswriter/src/main/libs/edb-jdbc16.jar -------------------------------------------------------------------------------- /rdbmswriter/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/rdbmswriter/src/main/libs/jconn3-1.0.0-SNAPSHOT.jar -------------------------------------------------------------------------------- /rdbmswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rdbmswriter", 3 | "class": "com.alibaba.datax.plugin.reader.rdbmswriter.RdbmsWriter", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute select sql, retrieve data from the ResultSet. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba", 6 | "drivers":["dm.jdbc.driver.DmDriver", "com.sybase.jdbc3.jdbc.SybDriver", "com.edb.Driver"] 7 | } 8 | -------------------------------------------------------------------------------- /rdbmswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rdbmswriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "writeMode": "", 7 | "column": [], 8 | "session": [], 9 | "preSql": [], 10 | "connection": [ 11 | { 12 | "jdbcUrl": "", 13 | "table": [] 14 | } 15 | ] 16 | } 17 | } -------------------------------------------------------------------------------- /rpm/t_dp_dw_datax_3_core_all-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export PATH=/home/tops/bin/:${PATH} 3 | export temppath=$1 4 | cd $temppath/rpm 5 | sed -i "s/^Release:.*$/Release: "$4"/" $2.spec 6 | sed -i "s/^Version:.*$/Version: "$3"/" $2.spec 7 | sed -i "s/UNKNOWN_DATAX_VERSION/$3-$4/g" ../core/src/main/bin/datax.py 8 | sed -i "s/UNKNOWN_DATAX_VERSION/$3-$4/g" ../core/src/main/bin/perftrace.py 9 | export TAGS=TAG:`svn info|grep "URL"|cut -d ":" -f 2-|sed "s/^ //g"|awk -F "trunk|tags|branche" '{print $1}'`tags/$2_A_`echo $3|tr "." "_"`_$4 10 | sed -i "s#%description#%description \n $TAGS#g" $2.spec 11 | /usr/local/bin/rpm_create -p /home/admin -v $3 -r $4 $2.spec -k 12 | mv `find . -name $2-$3-$4*rpm` . 13 | -------------------------------------------------------------------------------- /rpm/t_dp_dw_datax_3_hook_dqc-build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export PATH=/home/tops/bin/:${PATH} 3 | export temppath=$1 4 | cd $temppath/rpm 5 | sed -i "s/^Release:.*$/Release: "$4"/" $2.spec 6 | sed -i "s/^Version:.*$/Version: "$3"/" $2.spec 7 | export TAGS=TAG:`svn info|grep "URL"|cut -d ":" -f 2-|sed "s/^ //g"|awk -F "trunk|tags|branche" '{print $1}'`tags/$2_A_`echo $3|tr "." "_"`_$4 8 | sed -i "s#%description#%description \n $TAGS#g" $2.spec 9 | /usr/local/bin/rpm_create -p /home/admin -v $3 -r $4 $2.spec -k 10 | mv `find . -name $2-$3-$4*rpm` . 11 | -------------------------------------------------------------------------------- /sqlserverreader/src/main/java/com/alibaba/datax/plugin/reader/sqlserverreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.sqlserverreader; 2 | 3 | public class Constant { 4 | 5 | public static final int DEFAULT_FETCH_SIZE = 1024; 6 | 7 | } 8 | -------------------------------------------------------------------------------- /sqlserverreader/src/main/java/com/alibaba/datax/plugin/reader/sqlserverreader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.sqlserverreader; 2 | 3 | public class Key { 4 | 5 | public static final String FETCH_SIZE = "fetchSize"; 6 | } 7 | -------------------------------------------------------------------------------- /sqlserverreader/src/main/java/com/alibaba/datax/plugin/reader/sqlserverreader/SqlServerReaderErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.sqlserverreader; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum SqlServerReaderErrorCode implements ErrorCode { 6 | ; 7 | 8 | private String code; 9 | private String description; 10 | 11 | private SqlServerReaderErrorCode(String code, String description) { 12 | this.code = code; 13 | this.description = description; 14 | } 15 | 16 | @Override 17 | public String getCode() { 18 | return this.code; 19 | } 20 | 21 | @Override 22 | public String getDescription() { 23 | return this.description; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /sqlserverreader/src/main/lib/sqljdbc4-4.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/sqlserverreader/src/main/lib/sqljdbc4-4.0.jar -------------------------------------------------------------------------------- /sqlserverreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqlserverreader", 3 | "class": "com.alibaba.datax.plugin.reader.sqlserverreader.SqlServerReader", 4 | "description": "useScene: test. mechanism: use datax framework to transport data from SQL Server. warn: The more you know about the data, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } 7 | 8 | -------------------------------------------------------------------------------- /sqlserverreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqlserverreader", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "connection": [ 7 | { 8 | "table": [], 9 | "jdbcUrl": [] 10 | } 11 | ] 12 | } 13 | } -------------------------------------------------------------------------------- /sqlserverwriter/src/main/java/com/alibaba/datax/plugin/writer/sqlserverwriter/SqlServerWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.sqlserverwriter; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum SqlServerWriterErrorCode implements ErrorCode { 6 | ; 7 | 8 | private final String code; 9 | private final String describe; 10 | 11 | private SqlServerWriterErrorCode(String code, String describe) { 12 | this.code = code; 13 | this.describe = describe; 14 | } 15 | 16 | @Override 17 | public String getCode() { 18 | return this.code; 19 | } 20 | 21 | @Override 22 | public String getDescription() { 23 | return this.describe; 24 | } 25 | 26 | @Override 27 | public String toString() { 28 | return String.format("Code:[%s], Describe:[%s]. ", this.code, 29 | this.describe); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /sqlserverwriter/src/main/lib/sqljdbc4-4.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/sqlserverwriter/src/main/lib/sqljdbc4-4.0.jar -------------------------------------------------------------------------------- /sqlserverwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqlserverwriter", 3 | "class": "com.alibaba.datax.plugin.writer.sqlserverwriter.SqlServerWriter", 4 | "description": "useScene: prod. mechanism: Jdbc connection using the database, execute insert sql. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /sqlserverwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqlserverwriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "column": [], 7 | "preSql": [], 8 | "connection": [ 9 | { 10 | "jdbcUrl": "", 11 | "table": [] 12 | } 13 | ], 14 | "preSql": [], 15 | "postSql": [] 16 | } 17 | } -------------------------------------------------------------------------------- /starrockswriter/src/main/java/com/starrocks/connector/datax/plugin/writer/starrockswriter/manager/StarRocksFlushTuple.java: -------------------------------------------------------------------------------- 1 | package com.starrocks.connector.datax.plugin.writer.starrockswriter.manager; 2 | 3 | import java.util.List; 4 | 5 | public class StarRocksFlushTuple { 6 | 7 | private String label; 8 | private Long bytes; 9 | private List rows; 10 | 11 | public StarRocksFlushTuple(String label, Long bytes, List rows) { 12 | this.label = label; 13 | this.bytes = bytes; 14 | this.rows = rows; 15 | } 16 | 17 | public String getLabel() { return label; } 18 | public void setLabel(String label) { this.label = label; } 19 | public Long getBytes() { return bytes; } 20 | public List getRows() { return rows; } 21 | } -------------------------------------------------------------------------------- /starrockswriter/src/main/java/com/starrocks/connector/datax/plugin/writer/starrockswriter/manager/StarRocksStreamLoadFailedException.java: -------------------------------------------------------------------------------- 1 | package com.starrocks.connector.datax.plugin.writer.starrockswriter.manager; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | 6 | 7 | public class StarRocksStreamLoadFailedException extends IOException { 8 | 9 | static final long serialVersionUID = 1L; 10 | 11 | private final Map response; 12 | private boolean reCreateLabel; 13 | 14 | public StarRocksStreamLoadFailedException(String message, Map response) { 15 | super(message); 16 | this.response = response; 17 | } 18 | 19 | public StarRocksStreamLoadFailedException(String message, Map response, boolean reCreateLabel) { 20 | super(message); 21 | this.response = response; 22 | this.reCreateLabel = reCreateLabel; 23 | } 24 | 25 | public Map getFailedResponse() { 26 | return response; 27 | } 28 | 29 | public boolean needReCreateLabel() { 30 | return reCreateLabel; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /starrockswriter/src/main/java/com/starrocks/connector/datax/plugin/writer/starrockswriter/row/StarRocksBaseSerializer.java: -------------------------------------------------------------------------------- 1 | package com.starrocks.connector.datax.plugin.writer.starrockswriter.row; 2 | 3 | import com.alibaba.datax.common.element.Column; 4 | import com.alibaba.datax.common.element.Column.Type; 5 | 6 | public class StarRocksBaseSerializer { 7 | 8 | protected String fieldConvertion(Column col) { 9 | if (null == col.getRawData() || Type.NULL == col.getType()) { 10 | return null; 11 | } 12 | if (Type.BOOL == col.getType()) { 13 | return String.valueOf(col.asLong()); 14 | } 15 | if (Type.BYTES == col.getType()) { 16 | byte[] bts = (byte[])col.getRawData(); 17 | long value = 0; 18 | for (int i = 0; i < bts.length; i++) { 19 | value += (bts[bts.length - i - 1] & 0xffL) << (8 * i); 20 | } 21 | return String.valueOf(value); 22 | } 23 | return col.asString(); 24 | } 25 | 26 | } -------------------------------------------------------------------------------- /starrockswriter/src/main/java/com/starrocks/connector/datax/plugin/writer/starrockswriter/row/StarRocksCsvSerializer.java: -------------------------------------------------------------------------------- 1 | package com.starrocks.connector.datax.plugin.writer.starrockswriter.row; 2 | 3 | import java.io.StringWriter; 4 | 5 | import com.alibaba.datax.common.element.Record; 6 | 7 | import com.google.common.base.Strings; 8 | 9 | public class StarRocksCsvSerializer extends StarRocksBaseSerializer implements StarRocksISerializer { 10 | 11 | private static final long serialVersionUID = 1L; 12 | 13 | private final String columnSeparator; 14 | 15 | public StarRocksCsvSerializer(String sp) { 16 | this.columnSeparator = StarRocksDelimiterParser.parse(sp, "\t"); 17 | } 18 | 19 | @Override 20 | public String serialize(Record row) { 21 | StringBuilder sb = new StringBuilder(); 22 | for (int i = 0; i < row.getColumnNumber(); i++) { 23 | String value = fieldConvertion(row.getColumn(i)); 24 | sb.append(null == value ? "\\N" : value); 25 | if (i < row.getColumnNumber() - 1) { 26 | sb.append(columnSeparator); 27 | } 28 | } 29 | return sb.toString(); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /starrockswriter/src/main/java/com/starrocks/connector/datax/plugin/writer/starrockswriter/row/StarRocksISerializer.java: -------------------------------------------------------------------------------- 1 | package com.starrocks.connector.datax.plugin.writer.starrockswriter.row; 2 | 3 | import java.io.Serializable; 4 | 5 | import com.alibaba.datax.common.element.Record; 6 | 7 | public interface StarRocksISerializer extends Serializable { 8 | 9 | String serialize(Record row); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /starrockswriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "starrockswriter", 3 | "class": "com.starrocks.connector.datax.plugin.writer.starrockswriter.StarRocksWriter", 4 | "description": "useScene: prod. mechanism: StarRocksStreamLoad. warn: The more you know about the database, the less problems you encounter.", 5 | "developer": "starrocks" 6 | } -------------------------------------------------------------------------------- /starrockswriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "starrockswriter", 3 | "parameter": { 4 | "username": "", 5 | "password": "", 6 | "database": "", 7 | "table": "", 8 | "column": [], 9 | "preSql": [], 10 | "postSql": [], 11 | "jdbcUrl": "", 12 | "loadUrl": [] 13 | } 14 | } -------------------------------------------------------------------------------- /streamreader/src/main/java/com/alibaba/datax/plugin/reader/streamreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.streamreader; 2 | 3 | public class Constant { 4 | 5 | public static final String TYPE = "type"; 6 | 7 | public static final String VALUE = "value"; 8 | 9 | public static final String RANDOM = "random"; 10 | 11 | 12 | 13 | public static final String DATE_FORMAT_MARK = "dateFormat"; 14 | 15 | public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; 16 | 17 | public static final String HAVE_MIXUP_FUNCTION = "hasMixupFunction"; 18 | public static final String MIXUP_FUNCTION_PATTERN = "\\s*(.*)\\s*,\\s*(.*)\\s*"; 19 | public static final String MIXUP_FUNCTION_PARAM1 = "mixupParam1"; 20 | public static final String MIXUP_FUNCTION_PARAM2 = "mixupParam2"; 21 | 22 | 23 | } 24 | -------------------------------------------------------------------------------- /streamreader/src/main/java/com/alibaba/datax/plugin/reader/streamreader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.streamreader; 2 | 3 | public class Key { 4 | 5 | /** 6 | * should look like:[{"value":"123","type":"int"},{"value":"hello","type":"string"}] 7 | */ 8 | public static final String COLUMN = "column"; 9 | 10 | public static final String SLICE_RECORD_COUNT = "sliceRecordCount"; 11 | 12 | } 13 | -------------------------------------------------------------------------------- /streamreader/src/main/java/com/alibaba/datax/plugin/reader/streamreader/StreamReaderErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.streamreader; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum StreamReaderErrorCode implements ErrorCode { 6 | REQUIRED_VALUE("StreamReader-00", "缺失必要的值"), 7 | ILLEGAL_VALUE("StreamReader-01", "值非法"), 8 | NOT_SUPPORT_TYPE("StreamReader-02", "不支持的column类型"),; 9 | 10 | 11 | private final String code; 12 | private final String description; 13 | 14 | private StreamReaderErrorCode(String code, String description) { 15 | this.code = code; 16 | this.description = description; 17 | } 18 | 19 | @Override 20 | public String getCode() { 21 | return this.code; 22 | } 23 | 24 | @Override 25 | public String getDescription() { 26 | return this.description; 27 | } 28 | 29 | @Override 30 | public String toString() { 31 | return String.format("Code:[%s], Description:[%s]. ", this.code, 32 | this.description); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /streamreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "streamreader", 3 | "class": "com.alibaba.datax.plugin.reader.streamreader.StreamReader", 4 | "description": { 5 | "useScene": "only for developer test.", 6 | "mechanism": "use datax framework to transport data from stream.", 7 | "warn": "Never use it in your real job." 8 | }, 9 | "developer": "alibaba" 10 | } -------------------------------------------------------------------------------- /streamreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "streamreader", 3 | "parameter": { 4 | "sliceRecordCount": "", 5 | "column": [] 6 | } 7 | } -------------------------------------------------------------------------------- /streamwriter/src/main/java/com/alibaba/datax/plugin/writer/streamwriter/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.streamwriter; 2 | 3 | public class Key { 4 | public static final String FIELD_DELIMITER = "fieldDelimiter"; 5 | 6 | public static final String PRINT = "print"; 7 | 8 | public static final String PATH = "path"; 9 | 10 | public static final String FILE_NAME = "fileName"; 11 | 12 | public static final String RECORD_NUM_BEFORE_SLEEP = "recordNumBeforeSleep"; 13 | 14 | public static final String SLEEP_TIME = "sleepTime"; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /streamwriter/src/main/java/com/alibaba/datax/plugin/writer/streamwriter/StreamWriterErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.streamwriter; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | public enum StreamWriterErrorCode implements ErrorCode { 6 | RUNTIME_EXCEPTION("StreamWriter-00", "运行时异常"), 7 | ILLEGAL_VALUE("StreamWriter-01", "您填写的参数值不合法."), 8 | CONFIG_INVALID_EXCEPTION("StreamWriter-02", "您的参数配置错误."), 9 | SECURITY_NOT_ENOUGH("TxtFileWriter-03", "您缺少权限执行相应的文件写入操作."); 10 | 11 | 12 | 13 | private final String code; 14 | private final String description; 15 | 16 | private StreamWriterErrorCode(String code, String description) { 17 | this.code = code; 18 | this.description = description; 19 | } 20 | 21 | @Override 22 | public String getCode() { 23 | return this.code; 24 | } 25 | 26 | @Override 27 | public String getDescription() { 28 | return this.description; 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return String.format("Code:[%s], Description:[%s]. ", this.code, 34 | this.description); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /streamwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "streamwriter", 3 | "class": "com.alibaba.datax.plugin.writer.streamwriter.StreamWriter", 4 | "description": { 5 | "useScene": "only for developer test.", 6 | "mechanism": "use datax framework to transport data to stream.", 7 | "warn": "Never use it in your real job." 8 | }, 9 | "developer": "alibaba" 10 | } -------------------------------------------------------------------------------- /streamwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "streamwriter", 3 | "parameter": { 4 | "encoding": "", 5 | "print": true 6 | } 7 | } -------------------------------------------------------------------------------- /transformer/doc/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StarRocks/DataX/676af18f6ffbfed1e52bbf7a04c1076eac0124d3/transformer/doc/.gitkeep -------------------------------------------------------------------------------- /transformer/src/main/assembly/package.xml: -------------------------------------------------------------------------------- 1 | 5 | 6 | 7 | dir 8 | 9 | false 10 | 11 | 12 | target/ 13 | 14 | datax-transformer-0.0.1-SNAPSHOT.jar 15 | 16 | /lib 17 | 18 | 19 | 20 | 21 | 22 | false 23 | /lib 24 | runtime 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /transformer/src/main/java/com/alibaba/datax/transformer/ComplexTransformer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.transformer; 2 | 3 | import com.alibaba.datax.common.element.Record; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * no comments. 9 | * Created by liqiang on 16/3/3. 10 | */ 11 | public abstract class ComplexTransformer { 12 | //transformerName的唯一性在datax中检查，或者提交到插件中心检查。 13 | private String transformerName; 14 | 15 | 16 | public String getTransformerName() { 17 | return transformerName; 18 | } 19 | 20 | public void setTransformerName(String transformerName) { 21 | this.transformerName = transformerName; 22 | } 23 | 24 | /** 25 | * @param record 行记录，UDF进行record的处理后，更新相应的record 26 | * @param tContext transformer运行的配置项 27 | * @param paras transformer函数参数 28 | */ 29 | abstract public Record evaluate(Record record, Map tContext, Object... paras); 30 | } 31 | -------------------------------------------------------------------------------- /transformer/src/main/java/com/alibaba/datax/transformer/Transformer.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.transformer; 2 | 3 | import com.alibaba.datax.common.element.Record; 4 | 5 | 6 | /** 7 | * no comments. 8 | * Created by liqiang on 16/3/3. 9 | */ 10 | public abstract class Transformer { 11 | //transformerName的唯一性在datax中检查，或者提交到插件中心检查。 12 | private String transformerName; 13 | 14 | 15 | public String getTransformerName() { 16 | return transformerName; 17 | } 18 | 19 | public void setTransformerName(String transformerName) { 20 | this.transformerName = transformerName; 21 | } 22 | 23 | /** 24 | * @param record 行记录，UDF进行record的处理后，更新相应的record 25 | * @param paras transformer函数参数 26 | */ 27 | abstract public Record evaluate(Record record, Object... paras); 28 | } 29 | -------------------------------------------------------------------------------- /tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.tsdbreader; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | 6 | /** 7 | * Copyright @ 2019 alibaba.com 8 | * All right reserved. 9 | * Function：Constant 10 | * 11 | * @author Benedict Jin 12 | * @since 2019-10-21 13 | */ 14 | public final class Constant { 15 | 16 | static final String DEFAULT_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss"; 17 | 18 | public static final String METRIC_SPECIFY_KEY = "__metric__"; 19 | public static final String TS_SPECIFY_KEY = "__ts__"; 20 | public static final String VALUE_SPECIFY_KEY = "__value__"; 21 | 22 | static final Set MUST_CONTAINED_SPECIFY_KEYS = new HashSet<>(); 23 | 24 | static { 25 | MUST_CONTAINED_SPECIFY_KEYS.add(METRIC_SPECIFY_KEY); 26 | MUST_CONTAINED_SPECIFY_KEYS.add(TS_SPECIFY_KEY); 27 | // __value__ 在多值场景下，可以不指定 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/TSDBReaderErrorCode.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.tsdbreader; 2 | 3 | import com.alibaba.datax.common.spi.ErrorCode; 4 | 5 | /** 6 | * Copyright @ 2019 alibaba.com 7 | * All right reserved. 8 | * Function：TSDB Reader Error Code 9 | * 10 | * @author Benedict Jin 11 | * @since 2019-10-21 12 | */ 13 | public enum TSDBReaderErrorCode implements ErrorCode { 14 | 15 | REQUIRED_VALUE("TSDBReader-00", "缺失必要的值"), 16 | ILLEGAL_VALUE("TSDBReader-01", "值非法"); 17 | 18 | private final String code; 19 | private final String description; 20 | 21 | TSDBReaderErrorCode(String code, String description) { 22 | this.code = code; 23 | this.description = description; 24 | } 25 | 26 | @Override 27 | public String getCode() { 28 | return this.code; 29 | } 30 | 31 | @Override 32 | public String getDescription() { 33 | return this.description; 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return String.format("Code:[%s], Description:[%s]. ", this.code, this.description); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /tsdbreader/src/main/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtils.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.tsdbreader.util; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | 5 | /** 6 | * Copyright @ 2019 alibaba.com 7 | * All right reserved. 8 | * Function：TimeUtils 9 | * 10 | * @author Benedict Jin 11 | * @since 2019-10-21 12 | */ 13 | public final class TimeUtils { 14 | 15 | private TimeUtils() { 16 | } 17 | 18 | private static final long SECOND_MASK = 0xFFFFFFFF00000000L; 19 | private static final long HOUR_IN_MILL = TimeUnit.HOURS.toMillis(1); 20 | 21 | /** 22 | * Weather the timestamp is second. 23 | * 24 | * @param ts timestamp 25 | */ 26 | public static boolean isSecond(long ts) { 27 | return (ts & SECOND_MASK) == 0; 28 | } 29 | 30 | /** 31 | * Get the hour. 32 | * 33 | * @param ms time in millisecond 34 | */ 35 | public static long getTimeInHour(long ms) { 36 | return ms - ms % HOUR_IN_MILL; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /tsdbreader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tsdbreader", 3 | "class": "com.alibaba.datax.plugin.reader.tsdbreader.TSDBReader", 4 | "description": { 5 | "useScene": "从 TSDB 中摄取数据点", 6 | "mechanism": "通过 /api/query 接口查询出符合条件的数据点", 7 | "warn": "指定起止时间会自动忽略分钟和秒，转为整点时刻，例如 2019-4-18 的 [3:35, 4:55) 会被转为 [3:00, 4:00)" 8 | }, 9 | "developer": "Benedict Jin" 10 | } 11 | -------------------------------------------------------------------------------- /tsdbreader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tsdbreader", 3 | "parameter": { 4 | "sinkDbType": "RDB", 5 | "endpoint": "http://localhost:8242", 6 | "column": [ 7 | "__metric__", 8 | "__ts__", 9 | "app", 10 | "cluster", 11 | "group", 12 | "ip", 13 | "zone", 14 | "__value__" 15 | ], 16 | "metric": [ 17 | "m" 18 | ], 19 | "tag": { 20 | "m": { 21 | "app": "a1", 22 | "cluster": "c1" 23 | } 24 | }, 25 | "splitIntervalMs": 60000, 26 | "beginDateTime": "2019-01-01 00:00:00", 27 | "endDateTime": "2019-01-01 01:00:00" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/conn/TSDBConnectionTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.tsdbreader.conn; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Ignore; 5 | import org.junit.Test; 6 | 7 | /** 8 | * Copyright @ 2019 alibaba.com 9 | * All right reserved. 10 | * Function：TSDB Connection4TSDB Test 11 | * 12 | * @author Benedict Jin 13 | * @since 2019-10-21 14 | */ 15 | @Ignore 16 | public class TSDBConnectionTest { 17 | 18 | private static final String TSDB_ADDRESS = "http://localhost:8242"; 19 | 20 | @Test 21 | public void testVersion() { 22 | String version = new TSDBConnection(TSDB_ADDRESS).version(); 23 | Assert.assertNotNull(version); 24 | } 25 | 26 | @Test 27 | public void testIsSupported() { 28 | Assert.assertTrue(new TSDBConnection(TSDB_ADDRESS).isSupported()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/Const.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.tsdbreader.util; 2 | 3 | /** 4 | * Copyright @ 2019 alibaba.com 5 | * All right reserved. 6 | * Function：Const 7 | * 8 | * @author Benedict Jin 9 | * @since 2019-10-21 10 | */ 11 | final class Const { 12 | 13 | private Const() { 14 | } 15 | 16 | static final String TSDB_ADDRESS = "http://localhost:8242"; 17 | } 18 | -------------------------------------------------------------------------------- /tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/HttpUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.tsdbreader.util; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Ignore; 5 | import org.junit.Test; 6 | 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | /** 11 | * Copyright @ 2019 alibaba.com 12 | * All right reserved. 13 | * Function：HttpUtils Test 14 | * 15 | * @author Benedict Jin 16 | * @since 2019-10-21 17 | */ 18 | @Ignore 19 | public class HttpUtilsTest { 20 | 21 | @Test 22 | public void testSimpleCase() throws Exception { 23 | String url = "https://httpbin.org/post"; 24 | Map params = new HashMap<>(); 25 | params.put("foo", "bar"); 26 | 27 | String rsp = HttpUtils.post(url, params); 28 | System.out.println(rsp); 29 | Assert.assertNotNull(rsp); 30 | } 31 | 32 | @Test 33 | public void testGet() throws Exception { 34 | String url = String.format("%s/api/version", Const.TSDB_ADDRESS); 35 | String rsp = HttpUtils.get(url); 36 | System.out.println(rsp); 37 | Assert.assertNotNull(rsp); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tsdbreader/src/test/java/com/alibaba/datax/plugin/reader/tsdbreader/util/TimeUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.tsdbreader.util; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | import java.text.ParseException; 7 | import java.text.SimpleDateFormat; 8 | import java.util.Date; 9 | 10 | /** 11 | * Copyright @ 2019 alibaba.com 12 | * All right reserved. 13 | * Function：com.alibaba.datax.common.util 14 | * 15 | * @author Benedict Jin 16 | * @since 2019-10-21 17 | */ 18 | public class TimeUtilsTest { 19 | 20 | @Test 21 | public void testIsSecond() { 22 | Assert.assertFalse(TimeUtils.isSecond(System.currentTimeMillis())); 23 | Assert.assertTrue(TimeUtils.isSecond(System.currentTimeMillis() / 1000)); 24 | } 25 | 26 | @Test 27 | public void testGetTimeInHour() throws ParseException { 28 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 29 | Date date = sdf.parse("2019-04-18 15:32:33"); 30 | long timeInHour = TimeUtils.getTimeInHour(date.getTime()); 31 | Assert.assertEquals("2019-04-18 15:00:00", sdf.format(timeInHour)); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.tsdbwriter; 2 | 3 | /** 4 | * Copyright @ 2019 alibaba.com 5 | * All right reserved. 6 | * Function：Key 7 | * 8 | * @author Benedict Jin 9 | * @since 2019-04-18 10 | */ 11 | public final class Constant { 12 | 13 | static final int DEFAULT_BATCH_SIZE = 100; 14 | static final int DEFAULT_TRY_SIZE = 3; 15 | static final boolean DEFAULT_IGNORE_WRITE_ERROR = false; 16 | } 17 | -------------------------------------------------------------------------------- /tsdbwriter/src/main/java/com/alibaba/datax/plugin/writer/tsdbwriter/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.tsdbwriter; 2 | 3 | /** 4 | * Copyright @ 2019 alibaba.com 5 | * All right reserved. 6 | * Function：Key 7 | * 8 | * @author Benedict Jin 9 | * @since 2019-04-18 10 | */ 11 | public class Key { 12 | 13 | static final String ENDPOINT = "endpoint"; 14 | static final String BATCH_SIZE = "batchSize"; 15 | static final String MAX_RETRY_TIME = "maxRetryTime"; 16 | static final String IGNORE_WRITE_ERROR = "ignoreWriteError"; 17 | } 18 | -------------------------------------------------------------------------------- /tsdbwriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tsdbwriter", 3 | "class": "com.alibaba.datax.plugin.writer.tsdbwriter.TSDBWriter", 4 | "description": { 5 | "useScene": "往 TSDB 中摄入数据点", 6 | "mechanism": "调用 TSDB 的 /api/put 接口，实现数据点的写入", 7 | "warn": "" 8 | }, 9 | "developer": "Benedict Jin" 10 | } 11 | -------------------------------------------------------------------------------- /tsdbwriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tsdbwriter", 3 | "parameter": { 4 | "endpoint": "http://localhost:8242" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/conn/TSDBConnectionTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.conn; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Ignore; 5 | import org.junit.Test; 6 | 7 | /** 8 | * Copyright @ 2019 alibaba.com 9 | * All right reserved. 10 | * Function：TSDBConnection Test 11 | * 12 | * @author Benedict Jin 13 | * @since 2019-03-29 14 | */ 15 | @Ignore 16 | public class TSDBConnectionTest { 17 | 18 | private static final String TSDB_ADDRESS = "http://localhost:8240"; 19 | 20 | @Test 21 | public void testVersion() { 22 | String version = new TSDBConnection(TSDB_ADDRESS).version(); 23 | Assert.assertNotNull(version); 24 | } 25 | 26 | @Test 27 | public void testIsSupported() { 28 | Assert.assertTrue(new TSDBConnection(TSDB_ADDRESS).isSupported()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/Const.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.util; 2 | 3 | /** 4 | * Copyright @ 2019 alibaba.com 5 | * All right reserved. 6 | * Function：Const 7 | * 8 | * @author Benedict Jin 9 | * @since 2019-03-29 10 | */ 11 | final class Const { 12 | 13 | private Const() { 14 | } 15 | 16 | static final String OPENTSDB_ADDRESS = "http://localhost:8242"; 17 | static final String TSDB_ADDRESS = "http://localhost:8240"; 18 | } 19 | -------------------------------------------------------------------------------- /tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/HttpUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.util; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Ignore; 5 | import org.junit.Test; 6 | 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | /** 11 | * Copyright @ 2019 alibaba.com 12 | * All right reserved. 13 | * Function：HttpUtils Test 14 | * 15 | * @author Benedict Jin 16 | * @since 2019-03-29 17 | */ 18 | @Ignore 19 | public class HttpUtilsTest { 20 | 21 | @Test 22 | public void testSimpleCase() throws Exception { 23 | String url = "https://httpbin.org/post"; 24 | Map params = new HashMap(); 25 | params.put("foo", "bar"); 26 | 27 | String rsp = HttpUtils.post(url, params); 28 | System.out.println(rsp); 29 | Assert.assertNotNull(rsp); 30 | } 31 | 32 | @Test 33 | public void testGet() throws Exception { 34 | String url = String.format("%s/api/version", Const.OPENTSDB_ADDRESS); 35 | String rsp = HttpUtils.get(url); 36 | System.out.println(rsp); 37 | Assert.assertNotNull(rsp); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tsdbwriter/src/test/java/com/alibaba/datax/plugin/writer/util/TSDBTest.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.util; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Ignore; 5 | import org.junit.Test; 6 | 7 | /** 8 | * Copyright @ 2019 alibaba.com 9 | * All right reserved. 10 | * Function：TSDB Test 11 | * 12 | * @author Benedict Jin 13 | * @since 2019-04-11 14 | */ 15 | @Ignore 16 | public class TSDBTest { 17 | 18 | @Test 19 | public void testVersion() { 20 | String version = TSDBUtils.version(Const.TSDB_ADDRESS); 21 | Assert.assertNotNull(version); 22 | System.out.println(version); 23 | 24 | version = TSDBUtils.version(Const.OPENTSDB_ADDRESS); 25 | Assert.assertNotNull(version); 26 | System.out.println(version); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /txtfilereader/src/main/java/com/alibaba/datax/plugin/reader/txtfilereader/Constant.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.txtfilereader; 2 | 3 | /** 4 | * Created by haiwei.luo on 14-9-20. 5 | */ 6 | public class Constant { 7 | public static final String SOURCE_FILES = "sourceFiles"; 8 | 9 | } 10 | -------------------------------------------------------------------------------- /txtfilereader/src/main/java/com/alibaba/datax/plugin/reader/txtfilereader/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.reader.txtfilereader; 2 | 3 | /** 4 | * Created by haiwei.luo on 14-9-20. 5 | */ 6 | public class Key { 7 | public static final String PATH = "path"; 8 | } 9 | -------------------------------------------------------------------------------- /txtfilereader/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "txtfilereader", 3 | "class": "com.alibaba.datax.plugin.reader.txtfilereader.TxtFileReader", 4 | "description": "useScene: test. mechanism: use datax framework to transport data from txt file. warn: The more you know about the data, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /txtfilereader/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "txtfilereader", 3 | "parameter": { 4 | "path": [], 5 | "encoding": "", 6 | "column": [], 7 | "fieldDelimiter": "" 8 | } 9 | } -------------------------------------------------------------------------------- /txtfilewriter/src/main/java/com/alibaba/datax/plugin/writer/txtfilewriter/Key.java: -------------------------------------------------------------------------------- 1 | package com.alibaba.datax.plugin.writer.txtfilewriter; 2 | 3 | /** 4 | * Created by haiwei.luo on 14-9-17. 5 | */ 6 | public class Key { 7 | // must have 8 | public static final String PATH = "path"; 9 | } 10 | -------------------------------------------------------------------------------- /txtfilewriter/src/main/resources/plugin.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "txtfilewriter", 3 | "class": "com.alibaba.datax.plugin.writer.txtfilewriter.TxtFileWriter", 4 | "description": "useScene: test. mechanism: use datax framework to transport data to txt file. warn: The more you know about the data, the less problems you encounter.", 5 | "developer": "alibaba" 6 | } -------------------------------------------------------------------------------- /txtfilewriter/src/main/resources/plugin_job_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "txtfilewriter", 3 | "parameter": { 4 | "path": "", 5 | "fileName": "", 6 | "writeMode": "", 7 | "fieldDelimiter":"", 8 | "dateFormat": "" 9 | } 10 | } --------------------------------------------------------------------------------