├── .gitignore ├── README.md ├── dataroad-client ├── dataroad-client-flink │ └── pom.xml └── pom.xml ├── dataroad-common ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── leonside │ └── dataroad │ └── common │ ├── config │ ├── BaseConfig.java │ ├── ConfigKey.java │ ├── ConfigKeyBinder.java │ ├── Options.java │ └── Validation.java │ ├── constant │ ├── JobCommonConstant.java │ └── JobConfigKeyConstants.java │ ├── context │ ├── ComponentHolder.java │ ├── ErrorLimitConfig.java │ ├── ExecuteContext.java │ ├── JobSetting.java │ ├── LogConfig.java │ ├── RestoreConfig.java │ └── SpeedConfig.java │ ├── domain │ ├── ColumnType.java │ └── MetaColumn.java │ ├── enums │ ├── DatabaseType.java │ ├── FieldType.java │ └── WriteMode.java │ ├── exception │ ├── JobConfigException.java │ ├── JobException.java │ ├── JobFlowException.java │ ├── ScriptExecuteException.java │ └── WriteRecordException.java │ ├── extension │ ├── Activate.java │ ├── ActivateComparator.java │ ├── ConcurrentHashSet.java │ ├── ExtensionLoader.java │ ├── Holder.java │ └── SPI.java │ ├── script │ ├── ScriptEvaluator.java │ ├── ScriptEvaluatorFactory.java │ ├── bsh │ │ └── BshScriptEvaluator.java │ ├── fel │ │ └── FelScriptEvaluator.java │ ├── groovy │ │ └── GroovyScriptEvaluator.java │ └── javascript │ │ └── JavaScriptEvaluator.java │ ├── spi │ ├── Component.java │ ├── ItemProcessor.java │ ├── ItemReader.java │ ├── ItemUnionProcessor.java │ ├── ItemWriter.java │ └── JobExecutionListener.java │ └── utils │ ├── Asserts.java │ ├── ClassUtil.java │ ├── ConfigBeanUtils.java │ ├── DateUtil.java │ ├── EnumUtils.java │ ├── ExceptionUtil.java │ ├── HttpUtil.java │ ├── IdGenerator.java │ ├── JsonUtil.java │ ├── ParameterUtils.java │ ├── ParameterizedTypeUtils.java │ ├── RetryUtil.java │ ├── StringUtil.java │ └── TelnetUtil.java ├── dataroad-config ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── leonside │ └── dataroad │ └── config │ ├── ComponentFactory.java │ ├── JobCreator.java │ ├── JobSchemaParser.java │ ├── domain │ ├── Content.java │ ├── GenericComponentConfig.java │ ├── JobConfig.java │ └── JobConfigs.java │ └── job │ ├── JsonJobCreator.java │ └── JsonJobSchemaParser.java ├── dataroad-core ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── leonside │ │ └── dataroad │ │ └── core │ │ ├── Job.java │ │ ├── aggregations │ │ ├── AggerationEnum.java │ │ ├── NumberFunction.java │ │ ├── config │ │ │ ├── BaseWindowConfig.java │ │ │ ├── CountWindowConfig.java │ │ │ ├── CountWindowConfigKey.java │ │ │ ├── SlidingWindowConfig.java │ │ │ ├── SlidingWindowConfigKey.java │ │ │ ├── TumblingWindowConfig.java │ │ │ └── TumblingWindowConfigKey.java │ │ └── response │ │ │ ├── Aggeration.java │ │ │ ├── Aggerations.java │ │ │ ├── Avg.java │ │ │ ├── Count.java │ │ │ ├── Max.java │ │ │ ├── Min.java │ │ │ ├── MultiValue.java │ │ │ ├── MultiValueAggeration.java │ │ │ ├── SingleValueAggeration.java │ │ │ ├── Stats.java │ │ │ ├── Sum.java │ │ │ └── TopHits.java │ │ ├── builder │ │ ├── AggerationBuilder.java │ │ ├── BaseJobFlowBuilder.java │ │ ├── JobBuilder.java │ │ ├── JobFlowBuilder.java │ │ ├── MultiJobFlowBuilder.java │ │ └── Time.java │ │ ├── component │ │ ├── ComponentInitialization.java │ │ ├── ComponentNameAutoGenerator.java │ │ ├── ComponentNameSupport.java │ │ ├── ComponentType.java │ │ └── JobExtensionLoader.java │ │ ├── flow │ │ ├── JobFlow.java │ │ ├── MultiJobFlow.java │ │ ├── SimpleJobFlow.java │ │ └── Task.java │ │ ├── predicate │ │ ├── ExecuteStatus.java │ │ ├── ExecuteStatusPredicate.java │ │ └── OtherwisePredicate.java │ │ ├── spi │ │ ├── ItemAggregationProcessor.java │ │ ├── ItemDeciderProcessor.java │ │ ├── ItemLookupProcessor.java │ │ ├── JobEngineProvider.java │ │ ├── JobExecutionDecider.java │ │ └── JobPredicate.java │ │ └── support │ │ └── LoggerHelper.java │ └── resources │ └── META-INF │ └── services │ └── com.leonside.dataroad.core.spi.JobPredicate ├── dataroad-dashboard ├── Dockerfile ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── leonside │ │ │ └── dataroad │ │ │ └── dashboard │ │ │ ├── DashboardApplication.java │ │ │ ├── builder │ │ │ └── ComponentParameterBuilder.java │ │ │ ├── configuration │ │ │ ├── CorsConfiguuration.java │ │ │ └── DataroadProperties.java │ │ │ ├── controller │ │ │ ├── ComponentController.java │ │ │ ├── JobFlowController.java │ │ │ └── JobFlowTaskController.java │ │ │ ├── converter │ │ │ ├── JobFlowConverter.java │ │ │ └── JobFlowDesignerWrapper.java │ │ │ ├── domian │ │ │ ├── CodeRecord.java │ │ │ ├── ComponentParameter.java │ │ │ ├── JobFlowConfig.java │ │ │ ├── JobFlowConfigs.java │ │ │ ├── JobFlowDesigner.java │ │ │ ├── JobFlowSchema.java │ │ │ ├── JobFlowSchemas.java │ │ │ ├── JobJarsReponse.java │ │ │ ├── JobRequestParam.java │ │ │ ├── JobSubmitParam.java │ │ │ ├── JobSubmitReponse.java │ │ │ ├── JobUploadReponse.java │ │ │ └── ResponseStatus.java │ │ │ ├── listener │ │ │ ├── ApplicationEnvPreparedInitializer.java │ │ │ └── ApplicationSampleInitListener.java │ │ │ ├── repository │ │ │ ├── JobFlowRepository.java │ │ │ └── XmlFileJobFlowRepository.java │ │ │ ├── service │ │ │ ├── JobFlowService.java │ │ │ ├── JobFlowTaskService.java │ │ │ └── impl │ │ │ │ ├── JobFlowServiceImpl.java │ │ │ │ └── JobFlowTaskServiceImpl.java │ │ │ └── utils │ │ │ ├── HomeFolderUtils.java │ │ │ ├── NetUtil.java │ │ │ ├── ObjectUtils.java │ │ │ ├── PageUtils.java │ │ │ └── ZipUtils.java │ └── resources │ │ ├── META-INF │ │ └── spring.factories │ │ ├── application.yml │ │ ├── sampledata │ │ └── jobflowConfig.xml │ │ └── static │ │ ├── designer.html │ │ ├── index.html │ │ ├── jobscheduleList.html │ │ ├── jobsubmitList.html │ │ ├── js │ │ ├── flow-config.js │ │ ├── flow-desinger.js │ │ ├── flow-display.js │ │ ├── go.js │ │ ├── goSamples.js │ │ ├── jquery │ │ │ └── jquery.min.js │ │ ├── layer │ │ │ ├── layer.js │ │ │ └── skin │ │ │ │ ├── default │ │ │ │ ├── icon-ext.png │ │ │ │ ├── icon.png │ │ │ │ ├── loading-0.gif │ │ │ │ ├── loading-1.gif │ │ │ │ └── loading-2.gif │ │ │ │ └── layer.css │ │ └── layui │ │ │ ├── css │ │ │ ├── layui.css │ │ │ └── modules │ │ │ │ ├── code.css │ │ │ │ ├── laydate │ │ │ │ └── default │ │ │ │ │ └── laydate.css │ │ │ │ └── layer │ │ │ │ └── default │ │ │ │ ├── icon-ext.png │ │ │ │ ├── icon.png │ │ │ │ ├── layer.css │ │ │ │ ├── loading-0.gif │ │ │ │ ├── loading-1.gif │ │ │ │ └── loading-2.gif │ │ │ ├── font │ │ │ ├── iconfont.eot │ │ │ ├── iconfont.svg │ │ │ ├── iconfont.ttf │ │ │ ├── iconfont.woff │ │ │ └── iconfont.woff2 │ │ │ └── layui.js │ │ └── release │ │ ├── go-debug.js │ │ ├── go.js │ │ └── goJS.d.ts │ └── test │ └── java │ └── com │ └── leonside │ └── dataroad │ └── JobFlowConverterTest.java ├── dataroad-engine ├── dataroad-engine-flink │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── leonside │ │ │ └── dataroad │ │ │ └── flink │ │ │ ├── FlinkJob.java │ │ │ ├── FlinkJobProvider.java │ │ │ ├── LocalStreamEnvironment.java │ │ │ ├── config │ │ │ ├── ScriptExpressionConfig.java │ │ │ └── ScriptExpressionConfigKey.java │ │ │ ├── context │ │ │ └── FlinkExecuteContext.java │ │ │ ├── metric │ │ │ ├── AccumulatorCollector.java │ │ │ ├── BaseMetric.java │ │ │ ├── ByteRateLimiter.java │ │ │ ├── ErrorLimiter.java │ │ │ ├── Metrics.java │ │ │ ├── SimpleAccumulatorGauge.java │ │ │ └── SimpleLongCounterMeterView.java │ │ │ ├── predicate │ │ │ ├── ExpressionPredicate.java │ │ │ └── TrueExpressionPredicate.java │ │ │ ├── processor │ │ │ ├── aggeration │ │ │ │ ├── CountWindowAggerationProcessor.java │ │ │ │ ├── SlidingWindowAggerationProcessor.java │ │ │ │ ├── TumblingWindowAggerationProcessor.java │ │ │ │ └── function │ │ │ │ │ ├── CountWindowAggerationFunction.java │ │ │ │ │ ├── GenericAggregateFunction.java │ │ │ │ │ ├── SlidingWindowAggerationFunction.java │ │ │ │ │ ├── TumblingWindowAggerationFunction.java │ │ │ │ │ └── WindowAggerationFunction.java │ │ │ ├── decider │ │ │ │ └── OutputTagDeciderProcessor.java │ │ │ ├── filter │ │ │ │ ├── ScriptFilterProcessor.java │ │ │ │ └── ScriptTransformerProcessor.java │ │ │ ├── lookup │ │ │ │ ├── DirectAllLookupProcessor.java │ │ │ │ ├── config │ │ │ │ │ ├── BaseLookupConfig.java │ │ │ │ │ └── BaseLookupConfigKey.java │ │ │ │ └── function │ │ │ │ │ ├── AbstractAllLookupFunction.java │ │ │ │ │ ├── AbstractLruLookupFunction.java │ │ │ │ │ └── DirectAllLookupFunction.java │ │ │ ├── sql │ │ │ │ ├── SqlTransformerProcessor.java │ │ │ │ └── config │ │ │ │ │ ├── SqlTransformerConfig.java │ │ │ │ │ └── SqlTransformerConfigKey.java │ │ │ └── union │ │ │ │ └── GenericItemUnionProcessor.java │ │ │ ├── reader │ │ │ ├── BaseItemReader.java │ │ │ └── inputformat │ │ │ │ ├── ErrorInputSplit.java │ │ │ │ ├── GenericInputFormatSourceFunction.java │ │ │ │ ├── GenericRichInputFormat.java │ │ │ │ └── GenericRichInputFormatBuilder.java │ │ │ ├── restore │ │ │ └── FormatState.java │ │ │ ├── utils │ │ │ ├── FieldAccessorExtendFactory.java │ │ │ ├── PluginJarHelper.java │ │ │ ├── RawTypeConverter.java │ │ │ ├── RawTypeUtils.java │ │ │ ├── RowFieldAccessor.java │ │ │ ├── RowUtils.java │ │ │ └── UrlUtil.java │ │ │ └── writer │ │ │ ├── BaseItemWriter.java │ │ │ ├── PrintItemWriter.java │ │ │ └── outputformat │ │ │ ├── GenericOutputFormatSinkFunction.java │ │ │ ├── GenericRichOutputFormat.java │ │ │ └── GenericRichOutputFormatBuilder.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.leonside.dataroad.common.spi.ItemProcessor │ │ ├── com.leonside.dataroad.common.spi.ItemUnionProcessor │ │ ├── com.leonside.dataroad.common.spi.ItemWriter │ │ ├── com.leonside.dataroad.core.spi.ItemAggregationProcessor │ │ ├── com.leonside.dataroad.core.spi.ItemDeciderProcessor │ │ ├── com.leonside.dataroad.core.spi.ItemLookupProcessor │ │ ├── com.leonside.dataroad.core.spi.JobEngineProvider │ │ ├── com.leonside.dataroad.core.spi.JobExecutionDecider │ │ └── com.leonside.dataroad.core.spi.JobPredicate └── pom.xml ├── dataroad-plugin ├── dataroad-plugin-es │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── leonside │ │ │ └── dataroad │ │ │ └── plugin │ │ │ └── es │ │ │ ├── EsUtil.java │ │ │ ├── config │ │ │ ├── EsConstants.java │ │ │ ├── EsReaderConfig.java │ │ │ ├── EsReaderConfigKey.java │ │ │ ├── EsWriterConfig.java │ │ │ └── EsWriterConfigKey.java │ │ │ ├── reader │ │ │ ├── EsInputFormat.java │ │ │ ├── EsInputFormatBuilder.java │ │ │ ├── EsInputSplit.java │ │ │ └── EsReader.java │ │ │ └── writer │ │ │ ├── EsOutputFormat.java │ │ │ ├── EsOutputFormatBuilder.java │ │ │ └── EsWriter.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.leonside.dataroad.common.spi.ItemReader │ │ └── com.leonside.dataroad.common.spi.ItemWriter ├── dataroad-plugin-hive │ └── pom.xml ├── dataroad-plugin-jdbc │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── leonside │ │ └── dataroad │ │ └── plugin │ │ └── jdbc │ │ ├── BaseDatabaseDialect.java │ │ ├── DataSource.java │ │ ├── DatabaseDialect.java │ │ ├── ParameterValuesProvider.java │ │ ├── lookup │ │ ├── config │ │ │ ├── JdbcLookupConfig.java │ │ │ └── JdbcLookupKey.java │ │ ├── datasource │ │ │ └── DruidDataSourceProvider.java │ │ └── function │ │ │ ├── JdbcAllLookupFunction.java │ │ │ └── JdbcLruLookupFunction.java │ │ ├── reader │ │ ├── GenericJdbcReader.java │ │ ├── config │ │ │ ├── JdbcReaderConfig.java │ │ │ └── JdbcReaderConfigKey.java │ │ ├── inputformat │ │ │ ├── BigIntegerMaximum.java │ │ │ ├── GenericJdbcInputFormat.java │ │ │ ├── GenericJdbcInputFormatBuilder.java │ │ │ ├── IncrementConfig.java │ │ │ ├── JdbcInputSplit.java │ │ │ └── StringAccumulator.java │ │ └── support │ │ │ └── QuerySqlBuilder.java │ │ ├── type │ │ └── TypeConverterInterface.java │ │ ├── utils │ │ └── DbUtil.java │ │ └── writer │ │ ├── GenericJdbcWriter.java │ │ ├── config │ │ ├── JdbcWriterConfig.java │ │ └── JdbcWriterConfigKey.java │ │ └── outputformat │ │ ├── GenericJdbcOutputFormat.java │ │ ├── GenericJdbcOutputFormatBuilder.java │ │ └── GenericStreamJdbcOutputFormat.java ├── dataroad-plugin-kafka │ └── pom.xml ├── dataroad-plugin-mysql │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── leonside │ │ │ └── dataroad │ │ │ └── plugin │ │ │ └── mysql │ │ │ ├── MySqlDatabaseDialect.java │ │ │ ├── MysqlRawTypeConverter.java │ │ │ ├── lookup │ │ │ └── MysqlLookupProcessor.java │ │ │ ├── reader │ │ │ ├── MysqlJdbcInputFormat.java │ │ │ └── MysqlJdbcReader.java │ │ │ └── writer │ │ │ ├── MysqlJdbcOutputFormat.java │ │ │ ├── MysqlJdbcWriter.java │ │ │ └── MysqlStreamJdbcOutputFormat.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.leonside.dataroad.common.spi.ItemReader │ │ ├── com.leonside.dataroad.common.spi.ItemWriter │ │ └── com.leonside.dataroad.core.spi.ItemLookupProcessor ├── dataroad-plugin-mysqlstream │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── leonside │ │ │ └── dataroad │ │ │ └── plugin │ │ │ └── mysqlstream │ │ │ ├── config │ │ │ ├── MysqlStreamReaderConfig.java │ │ │ └── MysqlStreamReaderConfigKey.java │ │ │ ├── reader │ │ │ └── MysqlStreamJdbcReader.java │ │ │ └── writer │ │ │ ├── MysqlStreamJdbcOutputFormat.java │ │ │ └── MysqlStreamJdbcWriter.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.leonside.dataroad.common.spi.ItemReader │ │ └── com.leonside.dataroad.common.spi.ItemWriter ├── dataroad-plugin-oracle │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── leonside │ │ │ └── dataroad │ │ │ └── plugin │ │ │ └── oracle │ │ │ ├── OracleDatabaseDialect.java │ │ │ ├── converter │ │ │ ├── BlobType.java │ │ │ ├── ClobType.java │ │ │ └── OracleRawTypeConverter.java │ │ │ ├── reader │ │ │ ├── OracleJdbcInputFormat.java │ │ │ └── OracleJdbcReader.java │ │ │ └── writer │ │ │ ├── OracleJdbcOutputFormat.java │ │ │ └── OracleJdbcWriter.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.leonside.dataroad.common.spi.ItemReader │ │ └── com.leonside.dataroad.common.spi.ItemWriter ├── dataroad-plugin-postgresql │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── leonside │ │ │ └── dataroad │ │ │ └── plugin │ │ │ └── postgresql │ │ │ ├── PostgresqlDatabaseDialect.java │ │ │ ├── PostgresqlRawTypeConverter.java │ │ │ ├── PostgresqlTypeConverter.java │ │ │ ├── reader │ │ │ ├── PostgresqlJdbcInputFormat.java │ │ │ ├── PostgresqlJdbcReader.java │ │ │ └── PostgresqlQuerySqlBuilder.java │ │ │ └── writer │ │ │ ├── PostgresqlJdbcOutputFormat.java │ │ │ └── PostgresqlJdbcWriter.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.leonside.dataroad.common.spi.ItemReader │ │ └── com.leonside.dataroad.common.spi.ItemWriter ├── dataroad-plugin-redis │ └── pom.xml ├── dataroad-plugin-sqlserver │ └── pom.xml └── pom.xml ├── dataroad ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── leonside │ │ │ └── dataroad │ │ │ ├── DataroadEngine.java │ │ │ └── DataroadEngine_test.java │ └── resources │ │ ├── log4j.properties │ │ ├── mysql_aggcount_es.json │ │ ├── mysql_aggsliding_es.json │ │ ├── mysql_aggtumbling_es.json │ │ ├── mysql_customsql_incrpolling_filter_writer_fromcheckpoint.json │ │ ├── mysql_decider_mysql.json │ │ ├── mysql_decider_union.json │ │ ├── mysql_filter_mysql.json │ │ ├── mysql_filter_print_restore.json │ │ ├── mysql_incrpolling_writer_restore_fromcheckpoint.json │ │ ├── mysql_splitpk_filter_mysql.json │ │ ├── mysql_splitpk_lookuplru_es.json │ │ ├── mysql_sqltrans_lookup_es.json │ │ ├── mysqlsplitpk_filter_mysqlbatch_restore.json │ │ ├── mysqlstream_directlookup_es_fromcheckpoint.json │ │ ├── mysqlstream_es.json │ │ ├── mysqlstream_lookuplru_es.json │ │ ├── mysqlstream_mysqlstreamwriter_postsql_fromcheckpoint.json │ │ ├── oracle_filter_oracle.json │ │ ├── oracle_incr_filter_oracle_restore.json │ │ └── oracle_incrpoll_filter_oracle.json │ └── test │ └── java │ └── com │ └── leonside │ └── dataroad │ └── common │ ├── GroovyScriptTest.java │ └── TestMain.java ├── doc ├── dashboard-guide.md ├── flow-designer.md ├── images │ ├── dashboard-commit.png │ ├── dashboard-create.png │ ├── dashboard-designer.png │ ├── dashboard-guide-commit.png │ ├── dashboard-guide-designer-line.png │ ├── dashboard-guide-designer-node.png │ ├── dashboard-guide-designer.png │ ├── dashboard-guide-download.png │ ├── dashboard-guide-json.png │ ├── dashboard-guide-upload.png │ ├── dashboard-json.png │ ├── dataroad-arch.png │ ├── dataroad-designer-flow.png │ └── designer-json.png └── plugin │ ├── agg-countWindowAgg.md │ ├── agg-slidingWindowAgg.md │ ├── agg-tumblingWindowAgg.md │ ├── elasticsearch-reader.md │ ├── elasticsearch-writer.md │ ├── flow-forkjoin.md │ ├── lookup-direct.md │ ├── lookup-mysql.md │ ├── mysql-reader.md │ ├── mysql-stream.md │ ├── mysql-writer.md │ ├── oracle-reader.md │ ├── oracle-writer.md │ ├── postgresql-reader.md │ ├── postgresql-writer.md │ ├── scriptfilter.md │ ├── scripttrans.md │ └── sqltrans.md ├── jars ├── bsh-1.0.jar ├── install_jars.sh └── ojdbc6-11.2.0.1.0.jar └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # Project exclude paths 2 | /dataroad/target/ 3 | /dataroad-common/target/ 4 | /dataroad-config/target/ 5 | /dataroad-core/target/ 6 | /dataroad-engine/dataroad-engine-flink/target/ 7 | /dataroad-plugin/dataroad-plugin-mysql/dataroad-plugin-mysql-reader/target/ 8 | /dataroad-plugin/dataroad-plugin-mysql/dataroad-plugin-mysql-writer/target/ 9 | /dataroad-plugin/dataroad-plugin-rdb/dataroad-plugin-rdb-core/target/ 10 | /dataroad-plugin/dataroad-plugin-rdb/dataroad-plugin-rdb-reader/target/ 11 | /dataroad-plugin/dataroad-plugin-rdb/dataroad-plugin-rdb-writer/target/ -------------------------------------------------------------------------------- /dataroad-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | dataroad-all 7 | io.github.leonside 8 | 0.5 9 | 10 | 4.0.0 11 | 12 | dataroad-client 13 | pom 14 | 15 | 16 | 17 | 18 | 19 | 8 20 | 8 21 | 22 | 23 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/config/BaseConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.config; 2 | 3 | import com.leonside.dataroad.common.utils.ConfigBeanUtils; 4 | import lombok.Data; 5 | 6 | import java.io.Serializable; 7 | import java.util.Map; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | @Data 13 | public abstract class BaseConfig implements ConfigKeyBinder, Validation, Serializable { 14 | 15 | private Map parameter; 16 | 17 | public BaseConfig(Map parameter){ 18 | this.parameter = parameter; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/config/ConfigKey.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.config; 2 | 3 | import com.leonside.dataroad.common.enums.FieldType; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public interface ConfigKey { 9 | /** 10 | * 是否必填 11 | * @return 12 | */ 13 | boolean isRequired(); 14 | 15 | /** 16 | * 名称 17 | * @return 18 | */ 19 | String getName(); 20 | 21 | /** 22 | * 获取中文名称 23 | * @return 24 | */ 25 | String getCnName(); 26 | 27 | /** 28 | * 描述 29 | * @return 30 | */ 31 | String getDesc(); 32 | 33 | /** 34 | * 默认值 35 | * @return 36 | */ 37 | String getDefaultValue(); 38 | 39 | /** 40 | * 字段类型，主要用于前端展示 41 | * @return 42 | */ 43 | FieldType getFieldType(); 44 | } 45 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/config/ConfigKeyBinder.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.config; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public interface ConfigKeyBinder { 7 | 8 | Class bindConfigKey(); 9 | 10 | } 11 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/config/Validation.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.config; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public interface Validation extends Serializable { 9 | 10 | default boolean validate(){ 11 | return true; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/constant/JobCommonConstant.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.constant; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public class JobCommonConstant { 7 | 8 | public static final String SCRIPT_CONTEXT_KEY_ROW = "row"; 9 | 10 | public static final String SINGLE_QUOTE_MARK_SYMBOL = "'"; 11 | public static final String DOUBLE_QUOTE_MARK_SYMBOL = "\""; 12 | 13 | public static final String LEFT_PARENTHESIS_SYMBOL = "("; 14 | public static final String RIGHT_PARENTHESIS_SYMBOL = ")"; 15 | public static final String SINGLE_SLASH_SYMBOL = "/"; 16 | public static final String DOUBLE_SLASH_SYMBOL = "//"; 17 | public static final String KEY_HTTP = "http"; 18 | public static final String PROTOCOL_HTTP = "http://"; 19 | 20 | public static final String STAR_SYMBOL = "*"; 21 | public static final String POINT_SYMBOL = "."; 22 | 23 | //JDBC config param 24 | public static final String JDBC_PROTOCOL_MYSQL = "jdbc:mysql://"; 25 | 26 | public static final int MAX_BATCH_SIZE = 200000; 27 | 28 | 29 | public static final String JOBSCHEMA_PATH_PREFIX_CLASSPATH = "classpath:"; 30 | public static final String JOBSCHEMA_PATH_PREFIX_FILESYSTEM = "file:"; 31 | public static final String JOBSCHEMA_PATH_PREFIX_HTTP = "http:"; 32 | public static final String JOBSCHEMA_PATH_PREFIX_HTTPS = "https:"; 33 | /** 34 | * Jdbc获取主键列名 35 | */ 36 | public static final String PRIMARYKEY_COLUMN_NAME = "COLUMN_NAME"; 37 | } 38 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/constant/JobConfigKeyConstants.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.constant; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public abstract class JobConfigKeyConstants { 7 | 8 | public static final String KEY_PREDICATE_OTHERWISE = "otherwise"; 9 | 10 | public static final String COMPONENT_PREFIX_OUTPUTTAG_DECIDER = "outputTag-" ; 11 | 12 | public static final String COMPONENT_PREFIX_UNION_PROCESSOR = "union-" ; 13 | 14 | // public static final String COMPONENT_NAME_PREFIX_AGGERATION = "aggeration-" ; 15 | } 16 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/context/ComponentHolder.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.context; 2 | 3 | import lombok.Data; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | @Data 11 | public class ComponentHolder implements Serializable { 12 | 13 | private String type; 14 | private String pluginName; 15 | 16 | public ComponentHolder(String type, String pluginName){ 17 | this.type = type; 18 | this.pluginName = pluginName; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/context/ErrorLimitConfig.java: -------------------------------------------------------------------------------- 1 | 2 | package com.leonside.dataroad.common.context; 3 | 4 | import com.fasterxml.jackson.annotation.JsonInclude; 5 | 6 | import java.io.Serializable; 7 | 8 | @JsonInclude(JsonInclude.Include.NON_NULL) 9 | public class ErrorLimitConfig implements Serializable { 10 | 11 | public static final int DEFAULT_ERROR_RECORD_LIMIT = 0; 12 | public static final double DEFAULT_ERROR_PERCENTAGE_LIMIT = 0.0; 13 | 14 | private int record = 0; 15 | private double percentage = 0.0; 16 | 17 | public static ErrorLimitConfig defaultConfig(){ 18 | return new ErrorLimitConfig(); 19 | } 20 | 21 | public int getRecord() { 22 | return record; 23 | } 24 | 25 | public void setRecord(int record) { 26 | this.record = record; 27 | } 28 | 29 | public double getPercentage() { 30 | return percentage; 31 | } 32 | 33 | public void setPercentage(double percentage) { 34 | this.percentage = percentage; 35 | } 36 | } -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/context/ExecuteContext.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.context; 2 | 3 | import com.leonside.dataroad.common.config.Options; 4 | import com.leonside.dataroad.common.exception.JobFlowException; 5 | import com.leonside.dataroad.common.spi.JobExecutionListener; 6 | 7 | import java.io.Serializable; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import java.util.Map; 11 | import java.util.concurrent.ConcurrentHashMap; 12 | 13 | /** 14 | * Created by Administrator on 2021/4/6. 15 | */ 16 | public class ExecuteContext implements Serializable { 17 | 18 | private JobSetting jobSetting = new JobSetting(); 19 | 20 | private Options options ; 21 | 22 | private List componentHolders = new ArrayList<>(); 23 | 24 | private transient Map jobFlowOutputStore = new ConcurrentHashMap<>(); 25 | 26 | public void putJobFlowOutput(String key, Object value){ 27 | if(jobFlowOutputStore.containsKey(key)){ 28 | throw new JobFlowException("Contains duplicate Jobflows Output[" + key + "]"); 29 | } 30 | jobFlowOutputStore.put(key, value); 31 | } 32 | 33 | public Object getJobFlowOutput(String key){ 34 | return jobFlowOutputStore.get(key); 35 | } 36 | 37 | public void setOptions(Options options) { 38 | this.options = options; 39 | } 40 | 41 | public Options getOptions() { 42 | return options; 43 | } 44 | 45 | public JobSetting getJobSetting() { 46 | return jobSetting; 47 | } 48 | 49 | public List getComponentHolders() { 50 | return componentHolders; 51 | } 52 | 53 | public void setComponentHolders(List componentHolders) { 54 | this.componentHolders = componentHolders; 55 | } 56 | 57 | public void setJobSetting(JobSetting jobSetting) { 58 | this.jobSetting = jobSetting; 59 | } 60 | 61 | public static ExecuteContext of(JobSetting jobSetting,List componentHolders, Options options){ 62 | ExecuteContext executeContext = new ExecuteContext(); 63 | executeContext.setJobSetting(jobSetting); 64 | executeContext.setOptions(options); 65 | executeContext.setComponentHolders(componentHolders); 66 | return executeContext; 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/context/JobSetting.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.context; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import com.leonside.dataroad.common.config.Validation; 5 | import lombok.Data; 6 | 7 | import java.io.Serializable; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | @Data 13 | @JsonInclude(JsonInclude.Include.NON_NULL) 14 | public class JobSetting implements Serializable , Validation { 15 | 16 | private String name = "defaultJob"; 17 | 18 | private boolean isLocal = false; 19 | // private String monitorUrls; 20 | 21 | private LogConfig log = LogConfig.defaultConfig(); 22 | 23 | private RestoreConfig restore = RestoreConfig.defaultConfig(); 24 | 25 | private SpeedConfig speed = SpeedConfig.defaultConfig(); 26 | 27 | private ErrorLimitConfig errorLimit = ErrorLimitConfig.defaultConfig(); 28 | 29 | 30 | public void setIsLocal(boolean local) { 31 | isLocal = local; 32 | } 33 | 34 | @Override 35 | public boolean validate() { 36 | 37 | restore.validate(); 38 | 39 | return true; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/context/LogConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.context; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import lombok.Data; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | @Data 12 | @JsonInclude(JsonInclude.Include.NON_NULL) 13 | public class LogConfig implements Serializable { 14 | 15 | private boolean isLogger = false; 16 | private String level = "info"; 17 | 18 | public static LogConfig defaultConfig() { 19 | return new LogConfig(); 20 | } 21 | // private String path = null; 22 | // private String pattern = null; 23 | 24 | public void setIsLogger(boolean logger) { 25 | isLogger = logger; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/context/SpeedConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.context; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import lombok.Data; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | @Data 12 | @JsonInclude(JsonInclude.Include.NON_NULL) 13 | public class SpeedConfig implements Serializable { 14 | 15 | private long bytes = Long.MAX_VALUE; 16 | 17 | public static final int CHANNEL_DEFAULT = 1; 18 | private int channel = -1; 19 | private int readerChannel = -1; 20 | private int writerChannel = -1; 21 | 22 | public static SpeedConfig defaultConfig() { 23 | return new SpeedConfig(); 24 | } 25 | // private boolean rebalance = false; 26 | } 27 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/enums/DatabaseType.java: -------------------------------------------------------------------------------- 1 | 2 | package com.leonside.dataroad.common.enums; 3 | 4 | /** 5 | * Database type 6 | * 7 | */ 8 | public enum DatabaseType { 9 | 10 | /** 11 | * database type 12 | */ 13 | MySQL, 14 | SQLServer, 15 | Oracle, 16 | PostgreSQL, 17 | Greenplum, 18 | DB2, 19 | MongoDB, 20 | Redis, 21 | ES, 22 | dm, 23 | /** 24 | * contains ftp and sftp 25 | */ 26 | Hbase, 27 | ODPS, 28 | STREAM, 29 | clickhouse, 30 | Phoenix 31 | } 32 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/enums/FieldType.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.enums; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public enum FieldType { 7 | 8 | STRING, 9 | BOOLEAN, 10 | NUMBER, 11 | ENUM, 12 | OBJECT; 13 | 14 | } 15 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/enums/WriteMode.java: -------------------------------------------------------------------------------- 1 | 2 | package com.leonside.dataroad.common.enums; 3 | 4 | /** 5 | * Data write type 6 | */ 7 | public enum WriteMode { 8 | 9 | /** 10 | * insert into 11 | */ 12 | INSERT, 13 | 14 | /** 15 | * insert into ... on duplicate key update 16 | */ 17 | UPDATE, 18 | 19 | /** 20 | * replace into 21 | */ 22 | REPLACE, 23 | 24 | UPSERT, 25 | /** 26 | * insert update delete ,key the row Kind 27 | */ 28 | STREAM 29 | } 30 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/exception/JobConfigException.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.exception; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public class JobConfigException extends JobException{ 7 | public JobConfigException() { 8 | } 9 | 10 | public JobConfigException(String message) { 11 | super(message); 12 | } 13 | 14 | public JobConfigException(String message, Throwable cause) { 15 | super(message, cause); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/exception/JobException.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.exception; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public class JobException extends RuntimeException { 7 | 8 | public JobException() { 9 | super(); 10 | } 11 | 12 | public JobException(String message) { 13 | super(message); 14 | } 15 | 16 | public JobException(String message, Throwable cause) { 17 | super(message, cause); 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/exception/JobFlowException.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.exception; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public class JobFlowException extends JobException{ 7 | public JobFlowException() { 8 | } 9 | 10 | public JobFlowException(String message) { 11 | super(message); 12 | } 13 | 14 | public JobFlowException(String message, Throwable cause) { 15 | super(message, cause); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/exception/ScriptExecuteException.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.exception; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public class ScriptExecuteException extends JobException{ 7 | public ScriptExecuteException() { 8 | } 9 | 10 | public ScriptExecuteException(String message) { 11 | super(message); 12 | } 13 | 14 | public ScriptExecuteException(String message, Throwable cause) { 15 | super(message, cause); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/exception/WriteRecordException.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.exception; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public class WriteRecordException extends JobException{ 7 | 8 | private int columnIdx; 9 | 10 | public WriteRecordException() { 11 | } 12 | 13 | public WriteRecordException(String message) { 14 | super(message); 15 | } 16 | 17 | public WriteRecordException(String message, Throwable cause) { 18 | super(message, cause); 19 | } 20 | 21 | public int getColumnIdx() { 22 | return columnIdx; 23 | } 24 | 25 | public WriteRecordException(int columnIdx, String message) { 26 | super(message); 27 | this.columnIdx = columnIdx; 28 | } 29 | 30 | public WriteRecordException(int columnIdx,String message, Throwable cause) { 31 | super(message, cause); 32 | this.columnIdx = columnIdx; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/extension/Holder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 1999-2011 Alibaba Group. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.leonside.dataroad.common.extension; 17 | 18 | /** 19 | * Helper Class for hold a value. 20 | * 21 | * @author william.liangf 22 | */ 23 | public class Holder { 24 | 25 | private volatile T value; 26 | 27 | public void set(T value) { 28 | this.value = value; 29 | } 30 | 31 | public T get() { 32 | return value; 33 | } 34 | 35 | } -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/extension/SPI.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 1999-2011 Alibaba Group. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.leonside.dataroad.common.extension; 17 | 18 | import java.lang.annotation.*; 19 | 20 | /** 21 | * 扩展点接口的标识。 22 | *

23 | * 扩展点声明配置文件，格式修改。
24 | * 以Protocol示例，配置文件META-INF/dubbo/com.xxx.Protocol内容：
25 | * 由
26 | *


28 | * 改成使用KV格式
29 | *
32 | *
33 | * 原因：
34 | * 当扩展点的static字段或方法签名上引用了三方库， 35 | * 如果三方库不存在，会导致类初始化失败， 36 | * Extension标识Dubbo就拿不到了，异常信息就和配置对应不起来。 37 | *
38 | * 比如: 39 | * Extension("mina")加载失败， 40 | * 当用户配置使用mina时，就会报找不到扩展点， 41 | * 而不是报加载扩展点失败，以及失败原因。 42 | * 43 | * @author william.liangf 44 | * @author ding.lid 45 | * @export 46 | */ 47 | @Documented 48 | @Retention(RetentionPolicy.RUNTIME) 49 | @Target({ElementType.TYPE}) 50 | public @interface SPI { 51 | 52 | /** 53 | * 缺省扩展点名。 54 | */ 55 | String value() default ""; 56 | 57 | } -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/script/ScriptEvaluator.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.script; 2 | 3 | import java.io.Serializable; 4 | import java.util.Map; 5 | 6 | public interface ScriptEvaluator extends Serializable { 7 | 8 | Object evaluate(Object row, Map var2) throws Exception; 9 | } -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/script/ScriptEvaluatorFactory.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.script; 2 | 3 | import com.leonside.dataroad.common.script.bsh.BshScriptEvaluator; 4 | import com.leonside.dataroad.common.script.fel.FelScriptEvaluator; 5 | import com.leonside.dataroad.common.script.groovy.GroovyScriptEvaluator; 6 | import com.leonside.dataroad.common.script.javascript.JavaScriptEvaluator; 7 | import com.leonside.dataroad.common.utils.Asserts; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | public class ScriptEvaluatorFactory { 13 | 14 | public static ScriptEvaluator createScriptEvalutor(ScriptEngine scriptEngine, String scriptSource, String... packages){ 15 | 16 | Asserts.notNull(scriptSource, "scriptSource can not be null"); 17 | 18 | ScriptEvaluator scriptEvaluator = null; 19 | switch (scriptEngine){ 20 | case bsh: 21 | scriptEvaluator = new BshScriptEvaluator(scriptSource); 22 | break; 23 | case groovy: 24 | scriptEvaluator = new GroovyScriptEvaluator(scriptSource,packages); 25 | break; 26 | case javascript: 27 | scriptEvaluator = new JavaScriptEvaluator(scriptSource); 28 | break; 29 | case fel: 30 | scriptEvaluator = new FelScriptEvaluator(scriptSource); 31 | break; 32 | default: 33 | throw new UnsupportedOperationException("unsupport script engine ["+ scriptEngine +"]"); 34 | } 35 | return scriptEvaluator; 36 | } 37 | 38 | public enum ScriptEngine{ 39 | bsh, 40 | javascript, 41 | groovy, 42 | fel; 43 | 44 | 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/script/bsh/BshScriptEvaluator.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.script.bsh; 2 | 3 | import bsh.Interpreter; 4 | import com.leonside.dataroad.common.constant.JobCommonConstant; 5 | import com.leonside.dataroad.common.exception.ScriptExecuteException; 6 | import com.leonside.dataroad.common.script.ScriptEvaluator; 7 | 8 | import java.util.Iterator; 9 | import java.util.Map; 10 | 11 | /** 12 | * @author leon 13 | */ 14 | public class BshScriptEvaluator implements ScriptEvaluator { 15 | 16 | private String scriptSource; 17 | 18 | private Interpreter interpreter; 19 | 20 | public BshScriptEvaluator(String scriptSource){ 21 | this.scriptSource = scriptSource; 22 | interpreter = new Interpreter(); 23 | interpreter.setClassLoader(BshScriptEvaluator.class.getClassLoader()); 24 | } 25 | 26 | 27 | @Override 28 | public Object evaluate(Object record, Map param) throws Exception { 29 | 30 | try { 31 | if (param != null) { 32 | Iterator var4 = param.entrySet().iterator(); 33 | 34 | while(var4.hasNext()) { 35 | Map.Entry entry = (Map.Entry)var4.next(); 36 | interpreter.set((String)entry.getKey(), entry.getValue()); 37 | } 38 | } 39 | 40 | interpreter.set(JobCommonConstant.SCRIPT_CONTEXT_KEY_ROW, record); 41 | 42 | return interpreter.eval(scriptSource); 43 | } catch (Exception var7) { 44 | throw new ScriptExecuteException("script execute failed", var7); 45 | } 46 | 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/script/fel/FelScriptEvaluator.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.script.fel; 2 | 3 | import com.greenpineyu.fel.FelEngine; 4 | import com.greenpineyu.fel.FelEngineImpl; 5 | import com.greenpineyu.fel.context.FelContext; 6 | import com.leonside.dataroad.common.constant.JobCommonConstant; 7 | import com.leonside.dataroad.common.script.ScriptEvaluator; 8 | 9 | import java.util.Map; 10 | 11 | /** 12 | * @author leon 13 | */ 14 | public class FelScriptEvaluator implements ScriptEvaluator { 15 | 16 | private String scriptSource; 17 | 18 | private volatile FelEngine felEngine; 19 | 20 | public FelScriptEvaluator(String scriptSource, String... packages){ 21 | this.scriptSource = scriptSource; 22 | ; 23 | } 24 | 25 | @Override 26 | public Object evaluate(Object row, Map params) throws Exception { 27 | 28 | 29 | if(felEngine == null){ 30 | synchronized(FelScriptEvaluator.class){ 31 | if(felEngine == null){ 32 | felEngine = new FelEngineImpl(); 33 | } 34 | } 35 | } 36 | //获取引擎上下文 37 | FelContext felContext = felEngine.getContext(); 38 | 39 | if(params != null){ 40 | params.forEach((key,value)->{ 41 | felContext.set(key, params); 42 | }); 43 | } 44 | felContext.set(JobCommonConstant.SCRIPT_CONTEXT_KEY_ROW, row); 45 | 46 | 47 | Object eval = felEngine.eval(scriptSource); 48 | return eval; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/script/javascript/JavaScriptEvaluator.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.script.javascript; 2 | 3 | import com.leonside.dataroad.common.constant.JobCommonConstant; 4 | import com.leonside.dataroad.common.exception.ScriptExecuteException; 5 | import com.leonside.dataroad.common.script.ScriptEvaluator; 6 | 7 | import javax.script.*; 8 | import java.util.Map; 9 | 10 | /** 11 | * @author leon 12 | */ 13 | public class JavaScriptEvaluator implements ScriptEvaluator { 14 | 15 | private String scriptSource; 16 | private volatile ScriptEngine scriptEngine ; 17 | 18 | public JavaScriptEvaluator(String scriptSource){ 19 | this.scriptSource = scriptSource; 20 | } 21 | 22 | @Override 23 | public Object evaluate(Object record, Map param) throws ScriptExecuteException { 24 | 25 | if(scriptEngine == null){ 26 | synchronized(JavaScriptEvaluator.class){ 27 | if(scriptEngine == null){ 28 | scriptEngine = new ScriptEngineManager().getEngineByName("nashorn"); 29 | } 30 | } 31 | } 32 | 33 | SimpleBindings simpleBindings = new SimpleBindings(); 34 | simpleBindings.put(JobCommonConstant.SCRIPT_CONTEXT_KEY_ROW, record); 35 | try { 36 | return scriptEngine.eval(scriptSource, simpleBindings); 37 | } catch (ScriptException e) { 38 | throw new ScriptExecuteException("javascript execute exception",e); 39 | } 40 | } 41 | 42 | 43 | } 44 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/spi/Component.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.spi; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public interface Component extends Serializable { 9 | 10 | String getName(); 11 | 12 | default void setName(String name){} 13 | 14 | default String description(){ return "";} 15 | } 16 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/spi/ItemProcessor.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.spi; 2 | 3 | 4 | import com.leonside.dataroad.common.context.ExecuteContext; 5 | import com.leonside.dataroad.common.extension.SPI; 6 | 7 | /** 8 | * Created by Administrator on 2020/10/21. 9 | */ 10 | @SPI 11 | public interface ItemProcessor extends Component { 12 | 13 | public OUT process(T executeContext, IN in); 14 | 15 | } -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/spi/ItemReader.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.spi; 2 | 3 | 4 | import com.leonside.dataroad.common.context.ExecuteContext; 5 | import com.leonside.dataroad.common.extension.SPI; 6 | 7 | /** 8 | * Created by Administrator on 2020/10/13. 9 | */ 10 | @SPI 11 | public interface ItemReader extends Component { 12 | 13 | R read(T executeContext) throws Exception; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/spi/ItemUnionProcessor.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.spi; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.common.extension.SPI; 5 | 6 | /** 7 | * @author leon 8 | */ 9 | @SPI 10 | public interface ItemUnionProcessor extends ItemProcessor { 11 | 12 | void initializeUnionFlowIndex(Integer[] indexs); 13 | } 14 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/spi/ItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.spi; 2 | 3 | 4 | import com.leonside.dataroad.common.context.ExecuteContext; 5 | import com.leonside.dataroad.common.extension.SPI; 6 | 7 | /** 8 | * Created by Administrator on 2020/10/13. 9 | */ 10 | @SPI 11 | public interface ItemWriter extends Component { 12 | 13 | void write(T executeContext, IN items); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/spi/JobExecutionListener.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.spi; 2 | 3 | import com.leonside.dataroad.common.extension.SPI; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | @SPI 11 | public interface JobExecutionListener extends Serializable { 12 | } 13 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/utils/ClassUtil.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.utils; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.sql.DriverManager; 7 | 8 | public class ClassUtil { 9 | 10 | private static final Logger LOG = LoggerFactory.getLogger(ClassUtil.class); 11 | 12 | public final static Object LOCK_STR = new Object(); 13 | 14 | public static void forName(String clazz, ClassLoader classLoader) { 15 | synchronized (LOCK_STR){ 16 | try { 17 | Class.forName(clazz, true, classLoader); 18 | DriverManager.setLoginTimeout(10); 19 | } catch (Exception e) { 20 | throw new RuntimeException(e); 21 | } 22 | } 23 | } 24 | 25 | 26 | public synchronized static void forName(String clazz) { 27 | try { 28 | Class driverClass = Class.forName(clazz); 29 | driverClass.newInstance(); 30 | } catch (Exception e) { 31 | throw new RuntimeException(e); 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/utils/EnumUtils.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.utils; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | import java.util.stream.Collectors; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | public class EnumUtils { 12 | 13 | public static String[] getEnumNameArray(Class enumClass) { 14 | return Arrays.stream(enumClass.getEnumConstants()).map(item-> item.name()).toArray(String[]::new); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/utils/ExceptionUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.leonside.dataroad.common.utils; 20 | 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import java.io.PrintWriter; 25 | import java.io.StringWriter; 26 | 27 | /** 28 | */ 29 | public class ExceptionUtil { 30 | 31 | private static Logger logger = LoggerFactory.getLogger(ExceptionUtil.class); 32 | 33 | public static String getErrorMessage(Throwable e) { 34 | if (null == e) { 35 | return null; 36 | } 37 | 38 | try (StringWriter stringWriter = new StringWriter(); 39 | PrintWriter writer = new PrintWriter(stringWriter)) { 40 | e.printStackTrace(writer); 41 | writer.flush(); 42 | stringWriter.flush(); 43 | StringBuffer buffer = stringWriter.getBuffer(); 44 | return buffer.toString(); 45 | } catch (Throwable ee) { 46 | logger.error("", ee); 47 | } 48 | return null; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/utils/HttpUtil.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.utils; 2 | 3 | import com.leonside.dataroad.common.exception.JobFlowException; 4 | import lombok.extern.slf4j.Slf4j; 5 | import org.apache.commons.lang.StringUtils; 6 | 7 | import java.io.BufferedReader; 8 | import java.io.IOException; 9 | import java.io.InputStreamReader; 10 | import java.io.OutputStreamWriter; 11 | import java.net.HttpURLConnection; 12 | import java.net.MalformedURLException; 13 | import java.net.URL; 14 | 15 | @Slf4j 16 | public class HttpUtil { 17 | 18 | public static String doGet(String url) throws IOException { 19 | StringBuffer sbf = new StringBuffer(); 20 | HttpURLConnection conn = null; 21 | BufferedReader br = null; 22 | String content = null; 23 | try{ 24 | URL u = new URL(url); 25 | conn = (HttpURLConnection)u.openConnection(); 26 | conn.setReadTimeout(50000); 27 | conn.setConnectTimeout(60000); 28 | if(conn.getResponseCode()==200){ 29 | br = new BufferedReader(new InputStreamReader(conn.getInputStream(),"utf-8")); 30 | while((content=br.readLine())!=null){ 31 | sbf.append(content); 32 | } 33 | } 34 | }finally{ 35 | if(br!=null){ 36 | try { 37 | br.close(); 38 | } catch (IOException e) { 39 | log.error("关闭"+ url + "路径异常",e); 40 | } 41 | } 42 | conn.disconnect(); 43 | } 44 | return sbf.toString(); 45 | } 46 | 47 | } -------------------------------------------------------------------------------- /dataroad-common/src/main/java/com/leonside/dataroad/common/utils/IdGenerator.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common.utils; 2 | 3 | import java.util.UUID; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public class IdGenerator { 9 | 10 | public static String generate(String prefix){ 11 | return prefix + UUID.randomUUID(); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /dataroad-config/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | dataroad-all 7 | io.github.leonside 8 | 0.5 9 | 10 | 4.0.0 11 | 12 | dataroad-config 13 | 14 | 15 | 8 16 | 8 17 | 18 | 19 | 20 | 21 | io.github.leonside 22 | dataroad-core 23 | ${project.version} 24 | 25 | 26 | 27 | org.projectlombok 28 | lombok 29 | true 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /dataroad-config/src/main/java/com/leonside/dataroad/config/JobCreator.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.config; 2 | 3 | import com.leonside.dataroad.config.domain.JobConfigs; 4 | import com.leonside.dataroad.core.Job; 5 | 6 | import java.io.IOException; 7 | import java.io.Serializable; 8 | import java.net.URISyntaxException; 9 | import java.util.List; 10 | 11 | /** 12 | * @author leon 13 | */ 14 | public interface JobCreator extends Serializable { 15 | 16 | // List createJob(JobConfigs jobConfigs) throws Exception; 17 | 18 | List createJob() throws Exception; 19 | 20 | } 21 | -------------------------------------------------------------------------------- /dataroad-config/src/main/java/com/leonside/dataroad/config/JobSchemaParser.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.config; 2 | 3 | 4 | import com.fasterxml.jackson.core.JsonProcessingException; 5 | import com.leonside.dataroad.config.domain.JobConfigs; 6 | import com.leonside.dataroad.core.Job; 7 | 8 | import java.io.IOException; 9 | import java.io.Serializable; 10 | import java.net.URISyntaxException; 11 | import java.util.List; 12 | 13 | public interface JobSchemaParser extends Serializable { 14 | 15 | JobConfigs parserJSON(String json) throws JsonProcessingException; 16 | 17 | JobConfigs parserJSONPath(String path) throws IOException, URISyntaxException; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /dataroad-config/src/main/java/com/leonside/dataroad/config/domain/Content.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.config.domain; 2 | 3 | 4 | import lombok.Data; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | @Data 12 | public class Content { 13 | 14 | private Map content ; 15 | 16 | } 17 | -------------------------------------------------------------------------------- /dataroad-config/src/main/java/com/leonside/dataroad/config/domain/JobConfigs.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.config.domain; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | @Data 9 | public class JobConfigs { 10 | 11 | private JobConfig job; 12 | 13 | public static JobConfigs of(JobConfig job){ 14 | JobConfigs jobConfigs = new JobConfigs(); 15 | jobConfigs.setJob(job); 16 | return jobConfigs; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /dataroad-config/src/main/java/com/leonside/dataroad/config/job/JsonJobSchemaParser.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.config.job; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.leonside.dataroad.common.constant.JobCommonConstant; 5 | import com.leonside.dataroad.common.utils.HttpUtil; 6 | import com.leonside.dataroad.common.utils.JsonUtil; 7 | import com.leonside.dataroad.config.JobSchemaParser; 8 | import com.leonside.dataroad.config.domain.JobConfigs; 9 | 10 | import java.io.File; 11 | import java.io.IOException; 12 | import java.net.URISyntaxException; 13 | import java.net.URL; 14 | 15 | /** 16 | * @author leon 17 | */ 18 | public class JsonJobSchemaParser implements JobSchemaParser { 19 | 20 | @Override 21 | public JobConfigs parserJSON(String json) throws JsonProcessingException { 22 | JobConfigs job = JsonUtil.getInstance().readJson(json, JobConfigs.class); 23 | return job; 24 | } 25 | 26 | @Override 27 | public JobConfigs parserJSONPath(String path) throws IOException, URISyntaxException { 28 | JobConfigs job = null; 29 | 30 | if(path.startsWith(JobCommonConstant.JOBSCHEMA_PATH_PREFIX_CLASSPATH)) { 31 | path = path.replaceAll(JobCommonConstant.JOBSCHEMA_PATH_PREFIX_CLASSPATH,""); 32 | URL resource = JsonJobSchemaParser.class.getResource(path); 33 | job = JsonUtil.getInstance().readJson(new File(resource.toURI()), JobConfigs.class); 34 | 35 | }else if(path.startsWith(JobCommonConstant.JOBSCHEMA_PATH_PREFIX_FILESYSTEM)){ 36 | // path = path.replaceAll(JobCommonConstant.JOBSCHEMA_PATH_PREFIX_FILESYSTEM,""); 37 | job = JsonUtil.getInstance().readJson(new File(new URL(path).toURI()), JobConfigs.class); 38 | 39 | }else if(path.startsWith(JobCommonConstant.JOBSCHEMA_PATH_PREFIX_HTTP) ){ 40 | String json = HttpUtil.doGet(path); 41 | job = JsonUtil.getInstance().readJson(json, JobConfigs.class); 42 | 43 | }else{ 44 | throw new UnsupportedOperationException("不支持的文件路径："+ path); 45 | } 46 | return job; 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /dataroad-core/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | dataroad-all 7 | io.github.leonside 8 | 0.5 9 | 10 | 4.0.0 11 | 12 | dataroad-core 13 | 14 | 15 | 8 16 | 8 17 | 18 | 19 | 20 | 21 | io.github.leonside 22 | dataroad-common 23 | ${project.version} 24 | 25 | 26 | 27 | commons-lang 28 | commons-lang 29 | 30 | 31 | 32 | org.projectlombok 33 | lombok 34 | true 35 | 36 | 37 | 38 | com.google.guava 39 | guava 40 | compile 41 | 42 | 43 | 44 | org.slf4j 45 | slf4j-log4j12 46 | runtime 47 | 48 | 49 | log4j 50 | log4j 51 | runtime 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/Job.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | 7 | public interface Job { 8 | 9 | public void execute() throws Exception; 10 | 11 | } 12 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/AggerationEnum.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations; 2 | 3 | import com.leonside.dataroad.common.utils.Asserts; 4 | 5 | import java.util.List; 6 | import java.util.stream.Collectors; 7 | 8 | public enum AggerationEnum { 9 | AVG, 10 | SUM, 11 | COUNT, 12 | MAX, 13 | MIN, 14 | STATS, 15 | TOPHITS, 16 | TOTALHITS, 17 | MULTI; 18 | 19 | public static List of(List aggNames){ 20 | Asserts.notEmpty(aggNames, "aggNames can not be null"); 21 | return aggNames.stream().map(it->AggerationEnum.valueOf(it.toUpperCase())).collect(Collectors.toList()); 22 | } 23 | } -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/config/BaseWindowConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.config; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 5 | import lombok.Data; 6 | import org.apache.commons.collections.MapUtils; 7 | 8 | import java.util.HashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | /** 13 | * @author leon 14 | */ 15 | @Data 16 | public abstract class BaseWindowConfig extends BaseConfig { 17 | 18 | public BaseWindowConfig(Map parameter) { 19 | super(parameter); 20 | } 21 | 22 | /** 23 | * keyBy，支持多个分组字段，如下的agg和keyby数组对应，配置如： 24 | * "keyBy": ["idcard","name], 25 | */ 26 | public String[] keyBy; 27 | 28 | /** 29 | * "agg": { 30 | * "age": ["stats"], 31 | * "sex": ["max"] 32 | * } 33 | */ 34 | public Map> agg; 35 | 36 | /** 37 | *转换后的agg 38 | */ 39 | private Map> aggerations; 40 | 41 | public void setAggerations(Map> aggerations) { 42 | this.aggerations = aggerations; 43 | } 44 | 45 | public Map> getAggerations(){ 46 | 47 | if(aggerations == null){ 48 | 49 | Map> newAggeration = new HashMap<>(); 50 | 51 | agg.entrySet().stream().forEach(itemMap->{ 52 | newAggeration.put(itemMap.getKey(),AggerationEnum.of(itemMap.getValue())); 53 | }); 54 | 55 | this.aggerations = newAggeration; 56 | } 57 | return aggerations; 58 | } 59 | 60 | @Override 61 | public boolean validate() { 62 | if(MapUtils.isEmpty(agg)){ 63 | throw new IllegalArgumentException("agg config cannot be empty "); 64 | } 65 | return super.validate(); 66 | } 67 | 68 | public abstract String windowComponentName(); 69 | 70 | public enum WindowTimeType{ 71 | event, process, ingestion 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/config/CountWindowConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.config; 2 | 3 | import com.leonside.dataroad.common.config.ConfigKey; 4 | import com.leonside.dataroad.common.config.Validation; 5 | import lombok.Data; 6 | 7 | import java.util.Map; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | @Data 13 | public class CountWindowConfig extends BaseWindowConfig { 14 | 15 | public CountWindowConfig(Map parameter) { 16 | super(parameter); 17 | } 18 | 19 | public CountWindowConfig() { 20 | super(null); 21 | } 22 | 23 | public int windowSize; 24 | 25 | @Override 26 | public String windowComponentName() { 27 | return "countWindowAgg"; 28 | } 29 | 30 | @Override 31 | public Class bindConfigKey() { 32 | return CountWindowConfigKey.class; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/config/SlidingWindowConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.config; 2 | 3 | import com.leonside.dataroad.common.config.ConfigKey; 4 | import lombok.Data; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | @Data 12 | public class SlidingWindowConfig extends TumblingWindowConfig { 13 | 14 | public long slideSize; 15 | 16 | public SlidingWindowConfig(Map parameter) { 17 | super(parameter); 18 | } 19 | 20 | public SlidingWindowConfig(){ 21 | super(null); 22 | } 23 | 24 | 25 | @Override 26 | public String windowComponentName() { 27 | return "slidingWindowAgg"; 28 | } 29 | 30 | @Override 31 | public boolean validate() { 32 | 33 | if(slideSize <= 0){ 34 | throw new IllegalArgumentException("slideSize must be configured"); 35 | } 36 | 37 | return super.validate(); 38 | } 39 | 40 | @Override 41 | public Class bindConfigKey() { 42 | return SlidingWindowConfigKey.class; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/config/TumblingWindowConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.config; 2 | 3 | import com.leonside.dataroad.common.config.ConfigKey; 4 | import com.leonside.dataroad.common.config.Validation; 5 | import lombok.Data; 6 | import org.apache.commons.lang.StringUtils; 7 | 8 | import java.util.Map; 9 | import java.util.concurrent.TimeUnit; 10 | 11 | /** 12 | * @author leon 13 | */ 14 | @Data 15 | public class TumblingWindowConfig extends BaseWindowConfig { 16 | 17 | public long timeSize; 18 | 19 | public TimeUnit timeUnit = TimeUnit.SECONDS; 20 | 21 | public BaseWindowConfig.WindowTimeType timeType = WindowTimeType.process; 22 | 23 | public String eventTimeColumn; 24 | 25 | public int outOfOrderness = 0; 26 | 27 | public TumblingWindowConfig(Map parameter) { 28 | super(parameter); 29 | } 30 | 31 | public TumblingWindowConfig(){ 32 | super(null); 33 | } 34 | 35 | @Override 36 | public String windowComponentName() { 37 | return "tumblingWindowAgg"; 38 | } 39 | 40 | @Override 41 | public boolean validate() { 42 | 43 | if(timeType == WindowTimeType.event && StringUtils.isEmpty(eventTimeColumn)){ 44 | throw new IllegalArgumentException("eventTimeColumn cannot be empty when timeType is WindowTimetype.event"); 45 | } 46 | 47 | return super.validate(); 48 | } 49 | 50 | @Override 51 | public Class bindConfigKey() { 52 | return TumblingWindowConfigKey.class; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/Aggeration.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | import com.leonside.dataroad.common.constant.JobConfigKeyConstants; 4 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 5 | import com.leonside.dataroad.core.aggregations.NumberFunction; 6 | import lombok.Data; 7 | 8 | import java.io.Serializable; 9 | import java.util.Date; 10 | import java.util.LinkedHashMap; 11 | import java.util.Map; 12 | 13 | /** 14 | * @author leon 15 | */ 16 | @Data 17 | public abstract class Aggeration implements Serializable { 18 | 19 | public static final String AGGERATION_KEY_AGGFIELD = "aggBy"; 20 | public static final String AGGERATION_KEY_BEGINTIME = "beginTime"; 21 | public static final String AGGERATION_KEY_ENDTIME = "endTime"; 22 | public static final String AGGERATION_KEY_DUMPTIME = "dumpTime"; 23 | 24 | private String aggField; 25 | 26 | // private Date beginTime; 27 | // private Date endTime; 28 | 29 | private long dumpTime = System.currentTimeMillis(); 30 | 31 | private Class valueClass; 32 | private NumberFunction numberFunction; 33 | 34 | public Aggeration(Class valueClass){ 35 | this.valueClass = valueClass; 36 | numberFunction = NumberFunction.getForClass(valueClass); 37 | } 38 | 39 | public abstract void init(); 40 | 41 | public abstract AggerationEnum getType(); 42 | 43 | public String getAggField(){ 44 | return this.aggField; 45 | } 46 | 47 | public abstract Map asMap(); 48 | 49 | public abstract void calculate(IN value); 50 | 51 | public abstract T merge(T aggeration); 52 | 53 | protected Map getBasicRow() { 54 | Map basicMap = new LinkedHashMap<>(); 55 | basicMap.put(AGGERATION_KEY_AGGFIELD, getAggField()); 56 | basicMap.put(AGGERATION_KEY_DUMPTIME, getDumpTime()); 57 | return basicMap; 58 | } 59 | 60 | 61 | } 62 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/Aggerations.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | import com.leonside.dataroad.common.utils.Asserts; 4 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | public class Aggerations { 12 | 13 | public static Aggeration getAggeration(Class clazz, List aggerationEnums){ 14 | Asserts.notNull(aggerationEnums, "aggeration enum can not be null"); 15 | if(aggerationEnums.size() == 1){ 16 | return getAggeration(clazz, aggerationEnums.get(0)); 17 | }else{ 18 | return new MultiValue<>(clazz, aggerationEnums); 19 | } 20 | } 21 | 22 | public static Aggeration getAggeration(Class clazz,AggerationEnum aggerationEnum ){ 23 | Aggeration aggeration = null; 24 | switch (aggerationEnum){ 25 | case AVG: 26 | aggeration = new Avg(clazz); 27 | break; 28 | case SUM: 29 | aggeration = new Sum(clazz); 30 | break; 31 | case COUNT: 32 | aggeration = new Count(clazz); 33 | break; 34 | case MAX: 35 | aggeration = new Max(clazz); 36 | break; 37 | case MIN: 38 | aggeration = new Min(clazz); 39 | break; 40 | case STATS: 41 | aggeration = new Stats(clazz); 42 | break; 43 | case TOPHITS: 44 | //todo 45 | break; 46 | case TOTALHITS: 47 | //todo 48 | break; 49 | default: 50 | throw new UnsupportedOperationException("unsupported aggerationEnum ["+ aggerationEnum +"]"); 51 | } 52 | return aggeration; 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/Avg.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | public class Avg extends SingleValueAggeration { 11 | 12 | private Double avg; 13 | private Count count; 14 | private Sum sum; 15 | 16 | public Avg(Class valueClass) { 17 | super(valueClass); 18 | } 19 | 20 | @Override 21 | public AggerationEnum getType() { 22 | return AggerationEnum.AVG; 23 | } 24 | 25 | @Override 26 | public Map asMap() { 27 | Map row = getBasicRow(); 28 | row.put(AggerationEnum.AVG.name().toLowerCase(), getValue()); 29 | return row; 30 | } 31 | 32 | @Override 33 | public void calculate(IN value) { 34 | count.calculate(value); 35 | sum.calculate(value); 36 | 37 | } 38 | 39 | @Override 40 | public Avg merge(Avg aggeration) { 41 | avg = getNumberFunction().avg(avg, aggeration.getValue());; 42 | return this; 43 | } 44 | 45 | @Override 46 | public Double getValue() { 47 | return getNumberFunction().avgTotal(sum.getValue(), count.getValue()); 48 | } 49 | 50 | @Override 51 | public void init() { 52 | avg = 0.0; 53 | count = new Count(getValueClass()); 54 | sum = new Sum<>(getValueClass()); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/Count.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | public class Count extends SingleValueAggeration { 11 | 12 | private Long count; 13 | 14 | public Count(Class valueClass) { 15 | super(valueClass); 16 | } 17 | 18 | @Override 19 | public AggerationEnum getType() { 20 | return AggerationEnum.COUNT; 21 | } 22 | 23 | @Override 24 | public Map asMap() { 25 | Map row = getBasicRow(); 26 | row.put(AggerationEnum.COUNT.name().toLowerCase(), getValue()); 27 | return row; 28 | } 29 | 30 | @Override 31 | public void calculate(IN value) { 32 | count = count+1; 33 | } 34 | 35 | @Override 36 | public Count merge(Count aggeration) { 37 | count = count + aggeration.getValue(); 38 | return this; 39 | } 40 | 41 | @Override 42 | public Long getValue() { 43 | return count; 44 | } 45 | 46 | @Override 47 | public void init() { 48 | count = 0L; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/Max.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | public class Max extends SingleValueAggeration { 11 | 12 | private OUT max; 13 | 14 | public Max(Class valueClass) { 15 | super(valueClass); 16 | } 17 | 18 | @Override 19 | public AggerationEnum getType() { 20 | return AggerationEnum.MAX; 21 | } 22 | 23 | @Override 24 | public Map asMap() { 25 | Map row = getBasicRow(); 26 | row.put(AggerationEnum.MAX.name().toLowerCase(), getValue()); 27 | return row; 28 | } 29 | 30 | @Override 31 | public void calculate(IN value) { 32 | max = (OUT) getNumberFunction().max(max, value); 33 | } 34 | 35 | @Override 36 | public Max merge(Max aggeration) { 37 | max = (OUT) getNumberFunction().max(max, aggeration.getValue()); 38 | return this; 39 | } 40 | 41 | @Override 42 | public OUT getValue() { 43 | return max; 44 | } 45 | 46 | @Override 47 | public void init() { 48 | max = (OUT) getNumberFunction().create(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/Min.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | public class Min extends SingleValueAggeration { 11 | 12 | private OUT min; 13 | 14 | public Min(Class valueClass) { 15 | super(valueClass); 16 | } 17 | 18 | @Override 19 | public AggerationEnum getType() { 20 | return AggerationEnum.MIN; 21 | } 22 | 23 | @Override 24 | public Map asMap() { 25 | Map row = getBasicRow(); 26 | row.put(AggerationEnum.MIN.name().toLowerCase(), getValue()); 27 | return row; 28 | } 29 | 30 | @Override 31 | public void calculate(IN value) { 32 | if(this.min == null){ 33 | this.min = (OUT)value; 34 | }else { 35 | min = (OUT) getNumberFunction().min(this.min, value); 36 | } 37 | 38 | } 39 | 40 | @Override 41 | public Min merge(Min aggeration) { 42 | min = (OUT) getNumberFunction().max(min, aggeration.getValue()); 43 | return this; 44 | } 45 | 46 | @Override 47 | public OUT getValue() { 48 | return min; 49 | } 50 | 51 | @Override 52 | public void init() { 53 | // min = (OUT) getNumberFunction().create(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/MultiValue.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | import com.leonside.dataroad.common.utils.Asserts; 4 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.concurrent.ConcurrentHashMap; 9 | 10 | 11 | /** 12 | * @author leon 13 | */ 14 | public class MultiValue extends MultiValueAggeration{ 15 | 16 | private Map aggerationMap = new ConcurrentHashMap<>(); 17 | 18 | private List aggerationEnums; 19 | 20 | public MultiValue(Class valueClass, List aggerationEnumList) { 21 | super(valueClass); 22 | Asserts.notNull(aggerationEnumList, " aggerationEnum list can not be null."); 23 | this.aggerationEnums = aggerationEnumList; 24 | init(); 25 | } 26 | 27 | @Override 28 | public void init() { 29 | aggerationEnums.forEach(it->{ 30 | aggerationMap.put(it, (SingleValueAggeration) Aggerations.getAggeration(getValueClass(), it)); 31 | }); 32 | } 33 | 34 | @Override 35 | public AggerationEnum getType() { 36 | return AggerationEnum.MULTI; 37 | } 38 | 39 | @Override 40 | public Map asMap() { 41 | Map row = getBasicRow(); 42 | aggerationMap.forEach((key, value)->{ 43 | row.put(key.name().toLowerCase(), value.getValue()); 44 | }); 45 | return row; 46 | } 47 | 48 | @Override 49 | public void calculate(IN inValue) { 50 | aggerationMap.forEach((key, value)->{ 51 | value.calculate(inValue); 52 | }); 53 | } 54 | 55 | @Override 56 | public MultiValue merge(MultiValue aggeration) { 57 | aggerationMap.forEach((key,value)->{ 58 | value.merge(aggeration); 59 | }); 60 | return this; 61 | } 62 | 63 | @Override 64 | public OUT getValue(AggerationEnum aggerationEnum) { 65 | return (OUT) aggerationMap.get(aggerationEnum).getValue(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/MultiValueAggeration.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public abstract class MultiValueAggeration extends Aggeration { 9 | 10 | public MultiValueAggeration(Class valueClass){ 11 | super(valueClass); 12 | } 13 | 14 | protected abstract OUT getValue(AggerationEnum aggerationEnum); 15 | } 16 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/SingleValueAggeration.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | /** 4 | * @author leon 5 | */ 6 | public abstract class SingleValueAggeration extends Aggeration { 7 | 8 | public SingleValueAggeration(Class valueClass){ 9 | super(valueClass); 10 | init(); 11 | } 12 | 13 | protected abstract OUT getValue(); 14 | } 15 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/Sum.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | public class Sum extends SingleValueAggeration { 11 | 12 | private OUT sum; 13 | 14 | public Sum(Class valueClass) { 15 | super(valueClass); 16 | } 17 | 18 | @Override 19 | public AggerationEnum getType() { 20 | return AggerationEnum.SUM; 21 | } 22 | 23 | @Override 24 | public Map asMap() { 25 | Map row = getBasicRow(); 26 | row.put(AggerationEnum.SUM.name().toLowerCase(), getValue()); 27 | return row; 28 | } 29 | 30 | @Override 31 | public void calculate(IN value) { 32 | sum = (OUT) getNumberFunction().sum(sum, value); 33 | } 34 | 35 | @Override 36 | public Sum merge(Sum aggeration) { 37 | sum = (OUT) getNumberFunction().sum(sum, aggeration.getValue()); 38 | return this; 39 | } 40 | 41 | @Override 42 | public OUT getValue() { 43 | return sum; 44 | } 45 | 46 | @Override 47 | public void init() { 48 | sum = (OUT) getNumberFunction().create(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/aggregations/response/TopHits.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.aggregations.response; 2 | 3 | 4 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 5 | 6 | import java.util.Map; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | public class TopHits extends SingleValueAggeration { 12 | 13 | private double count; 14 | 15 | private int topNum; 16 | 17 | public TopHits(Class valueClass) { 18 | super(valueClass); 19 | } 20 | 21 | @Override 22 | public void init() { 23 | 24 | } 25 | 26 | @Override 27 | public AggerationEnum getType() { 28 | return AggerationEnum.TOPHITS; 29 | } 30 | 31 | @Override 32 | public Map asMap() { 33 | return null; 34 | } 35 | 36 | @Override 37 | public void calculate(Object value) { 38 | 39 | } 40 | 41 | @Override 42 | public Aggeration merge(Aggeration aggeration) { 43 | return null; 44 | } 45 | 46 | @Override 47 | public Object getValue() { 48 | return null; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/builder/JobBuilder.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.builder; 2 | 3 | 4 | import com.leonside.dataroad.common.context.ExecuteContext; 5 | import com.leonside.dataroad.common.context.JobSetting; 6 | import com.leonside.dataroad.common.spi.ItemReader; 7 | import com.leonside.dataroad.common.spi.JobExecutionListener; 8 | import com.leonside.dataroad.common.utils.Asserts; 9 | 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | 13 | /** 14 | * @author leon 15 | */ 16 | public class JobBuilder { 17 | 18 | private List jobExecutionListeners = new ArrayList<>(); 19 | private ExecuteContext executeContext; 20 | 21 | public static JobBuilder newInstance(){ 22 | return new JobBuilder(); 23 | } 24 | 25 | public JobBuilder listener(JobExecutionListener jobExecutionListener){ 26 | jobExecutionListeners.add(jobExecutionListener); 27 | return this; 28 | } 29 | 30 | public JobFlowBuilder reader(ItemReader itemReader){ 31 | Asserts.notNull(itemReader, "itemReader can not be null"); 32 | JobFlowBuilder jobFlowBuilder = JobFlowBuilder.newInstance(this); 33 | return jobFlowBuilder.reader(itemReader); 34 | } 35 | 36 | public ExecuteContext getExecuteContext() { 37 | return executeContext; 38 | } 39 | 40 | public JobBuilder executeContext(ExecuteContext executeContext) { 41 | this.executeContext = executeContext; 42 | return this; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/builder/Time.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.builder; 2 | 3 | import com.leonside.dataroad.common.utils.Asserts; 4 | 5 | import java.io.Serializable; 6 | import java.util.concurrent.TimeUnit; 7 | 8 | public final class Time implements Serializable { 9 | 10 | private static final long serialVersionUID = -350254188460915999L; 11 | 12 | /** 13 | * The time unit for this policy's time interval. 14 | */ 15 | private final TimeUnit unit; 16 | 17 | /** 18 | * The size of the windows generated by this policy. 19 | */ 20 | private final long size; 21 | 22 | /** 23 | * Instantiation only via factory method. 24 | */ 25 | private Time(long size, TimeUnit unit) { 26 | Asserts.notNull(unit, "time unit may not be null"); 27 | this.unit = unit; 28 | this.size = size; 29 | } 30 | 31 | public TimeUnit getUnit() { 32 | return unit; 33 | } 34 | 35 | public long getSize() { 36 | return size; 37 | } 38 | 39 | public static Time of(long size, TimeUnit unit) { 40 | return new Time(size, unit); 41 | } 42 | } -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/component/ComponentInitialization.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.component; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.common.config.Validation; 5 | import com.leonside.dataroad.common.context.ExecuteContext; 6 | import com.leonside.dataroad.common.utils.ConfigBeanUtils; 7 | import com.leonside.dataroad.common.utils.ParameterizedTypeUtils; 8 | 9 | import java.lang.reflect.Type; 10 | import java.util.Map; 11 | 12 | /** 13 | * @author leon 14 | */ 15 | public interface ComponentInitialization extends Validation { 16 | 17 | default void initialize(T executeContext, Map parameter){ 18 | 19 | C config = ConfigBeanUtils.newInstance(configClass(), parameter); 20 | 21 | if(config.bindConfigKey() != null){ 22 | ConfigBeanUtils.copyConfig(config, parameter, config.bindConfigKey()); 23 | } 24 | 25 | config.validate(); 26 | 27 | doInitialize(executeContext, config); 28 | } 29 | 30 | void doInitialize(T executeContext, C config); 31 | 32 | default Class configClass(){ 33 | Class classGenericType = ParameterizedTypeUtils.getClassGenericType(this.getClass(), ComponentInitialization.class, 1); 34 | 35 | if(BaseConfig.class.isAssignableFrom(classGenericType) ){ 36 | return (Class) classGenericType; 37 | }else{ 38 | return BaseConfig.class; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/component/ComponentNameAutoGenerator.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.component; 2 | 3 | 4 | import com.leonside.dataroad.common.utils.IdGenerator; 5 | 6 | /** 7 | * @author leon 8 | */ 9 | public abstract class ComponentNameAutoGenerator extends ComponentNameSupport { 10 | 11 | public ComponentNameAutoGenerator(String identityPrefix) { 12 | this.name = IdGenerator.generate(identityPrefix); 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/component/ComponentNameSupport.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.component; 2 | 3 | import com.leonside.dataroad.common.spi.Component; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public abstract class ComponentNameSupport implements Component { 9 | 10 | protected String name; 11 | 12 | @Override 13 | public String getName() { 14 | return name; 15 | } 16 | 17 | @Override 18 | public void setName(String name) { 19 | this.name = name; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/component/ComponentType.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.component; 2 | 3 | import com.leonside.dataroad.common.spi.ItemProcessor; 4 | import com.leonside.dataroad.common.spi.ItemReader; 5 | import com.leonside.dataroad.common.spi.ItemUnionProcessor; 6 | import com.leonside.dataroad.common.spi.ItemWriter; 7 | import com.leonside.dataroad.core.spi.ItemAggregationProcessor; 8 | import com.leonside.dataroad.core.spi.ItemDeciderProcessor; 9 | import com.leonside.dataroad.core.spi.ItemLookupProcessor; 10 | import com.leonside.dataroad.core.spi.JobPredicate; 11 | 12 | import java.util.Arrays; 13 | import java.util.Optional; 14 | 15 | public enum ComponentType{ 16 | 17 | reader("reader", ItemReader.class), 18 | writer("writer", ItemWriter.class), 19 | processor("processor", ItemProcessor.class), 20 | lookup("lookup", ItemLookupProcessor.class), 21 | agg("agg", ItemAggregationProcessor.class), 22 | decider("decider", ItemDeciderProcessor.class), 23 | deciderOn("deciderOn", JobPredicate.class), 24 | deciderEnd("deciderEnd", ItemDeciderProcessor.class), 25 | union("union", ItemUnionProcessor.class); 26 | private String name; 27 | private Class spi; 28 | 29 | ComponentType(String name, Class spi) { 30 | this.name = name; 31 | this.spi = spi; 32 | } 33 | 34 | public String getName() { 35 | return name; 36 | } 37 | 38 | public void setName(String name) { 39 | this.name = name; 40 | } 41 | 42 | public Class getSpi() { 43 | return spi; 44 | } 45 | 46 | public void setSpi(Class spi) { 47 | this.spi = spi; 48 | } 49 | 50 | public static ComponentType valueOfNotException(String name){ 51 | Optional first = Arrays.stream(ComponentType.values()).filter(it -> it.name.equals(name)).findFirst(); 52 | return first.isPresent() ? first.get() : null; 53 | } 54 | } -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/component/JobExtensionLoader.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.component; 2 | 3 | import com.leonside.dataroad.common.extension.ExtensionLoader; 4 | import com.leonside.dataroad.common.spi.Component; 5 | 6 | /** 7 | * @author leon 8 | */ 9 | public class JobExtensionLoader { 10 | 11 | public static T getExtensionLoader(Class clazz, String componentName){ 12 | ExtensionLoader extensionLoader = ExtensionLoader.getExtensionLoader(clazz); 13 | return (T) extensionLoader.getExtension(componentName); 14 | } 15 | 16 | public static T getComponent(ComponentType componentType, String componentName){ 17 | return (T) getExtensionLoader(componentType.getSpi(),componentName); 18 | } 19 | 20 | public static T getSingleComponent(ComponentType componentType){ 21 | 22 | ExtensionLoader extensionLoader = ExtensionLoader.getExtensionLoader(componentType.getSpi()); 23 | 24 | return (T) extensionLoader.getFirstExtension(); 25 | } 26 | 27 | 28 | } 29 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/flow/SimpleJobFlow.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.flow; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.common.spi.Component; 5 | 6 | import java.util.Optional; 7 | import java.util.function.Function; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | public class SimpleJobFlow extends JobFlow { 13 | 14 | public SimpleJobFlow(Task task) { 15 | super(task); 16 | } 17 | 18 | @Override 19 | public Optional doExecute(JobFlow simpleJobFlow, ExecuteContext context, Function parentOutput) throws Exception { 20 | 21 | Optional output = simpleJobFlow.getTask().execute(context, parentOutput.apply(simpleJobFlow)); 22 | 23 | output.ifPresent(o -> context.putJobFlowOutput(simpleJobFlow.getTask().getComponentName(), o)); 24 | 25 | return output; 26 | } 27 | 28 | public static SimpleJobFlow of(T component){ 29 | return new SimpleJobFlow(new Task(component)); 30 | } 31 | 32 | @Override 33 | public String toString() { 34 | return "SimpleJobFlow{" + 35 | "task=" + task + 36 | "child=" + getChildren() + 37 | '}'; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/flow/Task.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.flow; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.common.exception.JobFlowException; 5 | import com.leonside.dataroad.common.spi.Component; 6 | import com.leonside.dataroad.common.spi.ItemProcessor; 7 | import com.leonside.dataroad.common.spi.ItemReader; 8 | import com.leonside.dataroad.common.spi.ItemWriter; 9 | import com.leonside.dataroad.common.utils.Asserts; 10 | 11 | import java.io.Serializable; 12 | import java.util.Optional; 13 | 14 | /** 15 | * @author leon 16 | */ 17 | public class Task< CTX extends ExecuteContext, IN, OUT> implements Serializable { 18 | 19 | private Component component; 20 | 21 | public Task(Component component) { 22 | this.component = component; 23 | Asserts.notNull(component, "component can not be null"); 24 | } 25 | 26 | public Component getComponent(){ 27 | return component; 28 | } 29 | 30 | public String getComponentName(){ 31 | return component.getName(); 32 | } 33 | 34 | public Optional execute(CTX executeContext, IN in) throws Exception { 35 | 36 | if(component instanceof ItemReader){ 37 | 38 | return (Optional) Optional.of(((ItemReader) component).read(executeContext)); 39 | 40 | }else if (component instanceof ItemWriter){ 41 | //todo 42 | ((ItemWriter) component).write(executeContext, in); 43 | return Optional.empty(); 44 | 45 | }else if(component instanceof ItemProcessor){ 46 | 47 | return (Optional) Optional.of(((ItemProcessor) component).process(executeContext, in)); 48 | 49 | }else{ 50 | throw new JobFlowException("unkown component type [" + component.getName() + "]"); 51 | } 52 | } 53 | 54 | @Override 55 | public String toString() { 56 | return "Task{" + 57 | "component=" + component.getName() + 58 | '}'; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/predicate/ExecuteStatus.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.predicate; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | 5 | import java.io.Serializable; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | public class ExecuteStatus implements Serializable { 11 | 12 | 13 | public String name; 14 | 15 | public boolean ismatch(String matchValue){ 16 | return StringUtils.equals(name, matchValue); 17 | } 18 | 19 | public ExecuteStatus(String name) { 20 | this.name = name; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/predicate/ExecuteStatusPredicate.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.predicate; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.core.flow.JobFlow; 5 | import com.leonside.dataroad.core.spi.JobExecutionDecider; 6 | import com.leonside.dataroad.core.spi.JobPredicate; 7 | 8 | import java.io.Serializable; 9 | 10 | public class ExecuteStatusPredicate implements JobPredicate, Serializable { 11 | 12 | private String name; 13 | private JobExecutionDecider jobExecutionDecider; 14 | private JobFlow parentJobFlow; 15 | 16 | public ExecuteStatusPredicate(String name, JobExecutionDecider jobExecutionDecider, JobFlow parentJobFlow){ 17 | this.name = name; 18 | this.jobExecutionDecider = jobExecutionDecider; 19 | this.parentJobFlow = parentJobFlow; 20 | } 21 | 22 | @Override 23 | public boolean test(ExecuteContext context, Object row) { 24 | return this.jobExecutionDecider.decide(context, row).ismatch(this.name); 25 | } 26 | 27 | public JobExecutionDecider getJobExecutionDecider() { 28 | return jobExecutionDecider; 29 | } 30 | 31 | } -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/predicate/OtherwisePredicate.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.predicate; 2 | 3 | import com.leonside.dataroad.common.constant.JobConfigKeyConstants; 4 | import com.leonside.dataroad.common.context.ExecuteContext; 5 | import com.leonside.dataroad.core.spi.JobPredicate; 6 | 7 | import java.io.Serializable; 8 | 9 | public class OtherwisePredicate implements JobPredicate, Serializable { 10 | 11 | private String name = JobConfigKeyConstants.KEY_PREDICATE_OTHERWISE; 12 | 13 | @Override 14 | public boolean test(ExecuteContext context, Object row) { 15 | return true; 16 | } 17 | 18 | } -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/spi/ItemAggregationProcessor.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.spi; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.common.extension.SPI; 5 | import com.leonside.dataroad.common.spi.ItemProcessor; 6 | import com.leonside.dataroad.core.aggregations.AggerationEnum; 7 | import com.leonside.dataroad.core.aggregations.config.BaseWindowConfig; 8 | import com.leonside.dataroad.core.builder.AggerationBuilder; 9 | import com.leonside.dataroad.core.component.ComponentInitialization; 10 | 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | /** 15 | * @author leon 16 | */ 17 | @SPI 18 | public interface ItemAggregationProcessor extends ItemProcessor { 19 | 20 | void initialize(BaseWindowConfig baseWindowConfig); 21 | 22 | } 23 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/spi/ItemDeciderProcessor.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.spi; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.common.extension.SPI; 5 | import com.leonside.dataroad.common.spi.ItemProcessor; 6 | import com.leonside.dataroad.core.flow.JobFlow; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * @author leon 12 | */ 13 | @SPI 14 | public interface ItemDeciderProcessor extends ItemProcessor { 15 | 16 | Map getJobFlowDeciders(); 17 | 18 | void initialize(Map predicateJobFlowMap); 19 | } 20 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/spi/ItemLookupProcessor.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.spi; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.common.extension.SPI; 5 | import com.leonside.dataroad.common.spi.ItemProcessor; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | @SPI 11 | public interface ItemLookupProcessor extends ItemProcessor { 12 | } 13 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/spi/JobEngineProvider.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.spi; 2 | 3 | import com.leonside.dataroad.common.config.Options; 4 | import com.leonside.dataroad.common.context.ComponentHolder; 5 | import com.leonside.dataroad.common.context.ExecuteContext; 6 | import com.leonside.dataroad.common.context.JobSetting; 7 | import com.leonside.dataroad.common.extension.SPI; 8 | import com.leonside.dataroad.common.spi.JobExecutionListener; 9 | import com.leonside.dataroad.core.Job; 10 | import com.leonside.dataroad.core.flow.SimpleJobFlow; 11 | 12 | import java.util.List; 13 | 14 | @SPI 15 | public interface JobEngineProvider { 16 | 17 | Job createJob(T executeContext, SimpleJobFlow startJobFlow); 18 | 19 | T createExecuteContext(JobSetting jobSetting, List componentHolders, Options options); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/spi/JobExecutionDecider.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.spi; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.common.extension.SPI; 5 | import com.leonside.dataroad.core.predicate.ExecuteStatus; 6 | 7 | import java.io.Serializable; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | @SPI 13 | public interface JobExecutionDecider extends Serializable { 14 | 15 | ExecuteStatus decide(T context, IN in); 16 | 17 | } 18 | 19 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/spi/JobPredicate.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.spi; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.common.extension.SPI; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | @SPI 12 | @FunctionalInterface 13 | public interface JobPredicate extends Serializable { 14 | 15 | public static JobPredicate TRUE_PREDICATE = (context, o) -> true; 16 | 17 | public static JobPredicate FALSE_PREDICATE = (context, o) -> false; 18 | 19 | boolean test(T t, ROW row); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /dataroad-core/src/main/java/com/leonside/dataroad/core/support/LoggerHelper.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.core.support; 2 | 3 | import com.leonside.dataroad.common.context.LogConfig; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public class LoggerHelper { 9 | 10 | private static String level = "info"; 11 | 12 | public static boolean isLogger = false; 13 | 14 | public static void init(LogConfig logConfig){ 15 | LoggerHelper.level = logConfig.getLevel(); 16 | LoggerHelper.isLogger = logConfig.isLogger(); 17 | } 18 | 19 | public static String getLevel(){ 20 | return level; 21 | } 22 | 23 | public static boolean isLogger(){ 24 | return isLogger; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dataroad-core/src/main/resources/META-INF/services/com.leonside.dataroad.core.spi.JobPredicate: -------------------------------------------------------------------------------- 1 | otherwisePredicate=com.leonside.dataroad.core.predicate.OtherwisePredicate -------------------------------------------------------------------------------- /dataroad-dashboard/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM bus.ga/base/dragon-java:jdk8-alpine-x86 2 | #作者 3 | MAINTAINER leon 4 | #声明一个挂载点，容器内此路径会对应宿主机的某个文件夹 5 | #VOLUME /tmp 6 | #复制上下文目录下的target/demo-1.0.0.jar 到容器里 7 | COPY target/dataroad-dashboard-0.5.jar /opt/dataroad-dashboard-0.5.jar 8 | 9 | COPY target/dataroad-dist /opt/dataroad-dist 10 | 11 | #bash方式执行，使demo-0.0.1jar可访问 12 | #RUN新建立一层，在其上执行这些命令，执行结束后， commit 这一层的修改，构成新的镜像。 13 | #RUN bash -c "touch /opt/dataroad-dashboard-0.5.jar" 14 | 15 | #声明运行时容器提供服务端口，这只是一个声明，在运行时并不会因为这个声明应用就会开启这个端口的服务 16 | EXPOSE 8089 17 | 18 | #指定容器启动程序及参数 "" 19 | ENTRYPOINT ["java","-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005","-jar","/opt/dataroad-dashboard-0.5.jar"] -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/DashboardApplication.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | /** 7 | * @author leon 8 | */ 9 | @SpringBootApplication 10 | public class DashboardApplication { 11 | 12 | public static void main(String[] args) { 13 | SpringApplication.run(DashboardApplication.class, args); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/configuration/CorsConfiguuration.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.configuration; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.web.cors.CorsConfiguration; 6 | import org.springframework.web.cors.UrlBasedCorsConfigurationSource; 7 | import org.springframework.web.filter.CorsFilter; 8 | 9 | @Configuration 10 | public class CorsConfiguuration { 11 | 12 | private CorsConfiguration buildConfig() { 13 | CorsConfiguration corsConfiguration = new CorsConfiguration(); 14 | corsConfiguration.addAllowedOrigin("*"); 15 | corsConfiguration.addAllowedHeader("*"); 16 | corsConfiguration.addAllowedMethod("*"); 17 | return corsConfiguration; 18 | } 19 | 20 | @Bean 21 | public CorsFilter corsFilter() { 22 | UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); 23 | source.registerCorsConfiguration("/**", buildConfig()); 24 | return new CorsFilter(source); 25 | } 26 | } -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/converter/JobFlowConverter.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.converter; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.alibaba.fastjson.JSONObject; 5 | import com.alibaba.fastjson.serializer.SerializerFeature; 6 | import com.leonside.dataroad.common.context.JobSetting; 7 | import com.leonside.dataroad.common.utils.JsonUtil; 8 | import com.leonside.dataroad.config.domain.JobConfigs; 9 | import com.leonside.dataroad.dashboard.domian.JobFlowDesigner; 10 | import com.leonside.dataroad.dashboard.domian.JobFlowSchemas; 11 | import org.apache.commons.lang.StringUtils; 12 | 13 | import java.util.Map; 14 | 15 | /** 16 | * @author leon 17 | */ 18 | public class JobFlowConverter { 19 | 20 | private String designerJson; 21 | 22 | private String globalSetting; 23 | 24 | public JobFlowConverter(String designerJson, String globalSetting) { 25 | this.designerJson = designerJson; 26 | this.globalSetting = globalSetting; 27 | } 28 | 29 | public String convert(){ 30 | 31 | JobFlowDesigner jobFlowDesigner = paserJSON(); 32 | 33 | JobFlowSchemas jobConfigs = toJobConfigs(jobFlowDesigner); 34 | 35 | return JsonUtil.getInstance().writeJson(jobConfigs); 36 | } 37 | 38 | private JobFlowDesigner paserJSON() { 39 | 40 | JobFlowDesigner jobFlowDesigner = JsonUtil.getInstance().readJson(designerJson, JobFlowDesigner.class); 41 | 42 | if(StringUtils.isNotEmpty(globalSetting)){ 43 | Map jobSetting = JsonUtil.getInstance().readJson(globalSetting, Map.class); 44 | jobFlowDesigner.setGlobalSettting(jobSetting); 45 | } 46 | 47 | 48 | return jobFlowDesigner; 49 | } 50 | 51 | 52 | private JobFlowSchemas toJobConfigs(JobFlowDesigner jobFlowDesigner){ 53 | JobFlowDesignerWrapper jobFlowDesignerWrapper = new JobFlowDesignerWrapper(jobFlowDesigner); 54 | return jobFlowDesignerWrapper.toJobConfigs(); 55 | } 56 | 57 | 58 | } 59 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/CodeRecord.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | @Data 9 | public class CodeRecord { 10 | 11 | private String code; 12 | private String desc; 13 | 14 | public CodeRecord(){ 15 | 16 | } 17 | public CodeRecord(String code, String desc) { 18 | this.code = code; 19 | this.desc = desc; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/ComponentParameter.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | @Data 9 | public class ComponentParameter { 10 | private String name; 11 | private String cnName; 12 | private String desc; 13 | private boolean required; 14 | private String defaultValue; 15 | //String、boolean、enum、object类型，用于前端页面渲染 16 | private String fieldType; 17 | private String[] fieldEnumList; 18 | 19 | 20 | } 21 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/JobFlowConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import com.fasterxml.jackson.annotation.JsonFilter; 4 | import com.fasterxml.jackson.annotation.JsonIgnore; 5 | import com.leonside.dataroad.common.context.JobSetting; 6 | import com.leonside.dataroad.common.utils.JsonUtil; 7 | import lombok.Data; 8 | import lombok.Getter; 9 | import lombok.Setter; 10 | import org.apache.commons.lang.StringUtils; 11 | 12 | import javax.xml.bind.annotation.XmlAccessType; 13 | import javax.xml.bind.annotation.XmlAccessorType; 14 | import javax.xml.bind.annotation.XmlRootElement; 15 | import javax.xml.bind.annotation.XmlTransient; 16 | 17 | /** 18 | * @author leon 19 | */ 20 | @Getter 21 | @Setter 22 | @XmlRootElement 23 | @XmlAccessorType(XmlAccessType.FIELD) 24 | public class JobFlowConfig { 25 | private String id; 26 | private String description; 27 | private String golbalSetting; 28 | private boolean status; 29 | private String createTime ; 30 | private String designerJson; 31 | 32 | @JsonIgnore 33 | @XmlTransient 34 | private String jobflowJson; 35 | 36 | public boolean checkGlobalSetting(){ 37 | if(StringUtils.isNotEmpty(golbalSetting)){ 38 | JobSetting jobSetting = JsonUtil.getInstance().readJson(golbalSetting, JobSetting.class); 39 | return jobSetting.validate(); 40 | } 41 | return true; 42 | 43 | } 44 | 45 | public void copy(JobFlowConfig jobFlowConfig){ 46 | this.description = jobFlowConfig.getDescription(); 47 | if(!this.status){ 48 | this.status = jobFlowConfig.isStatus(); 49 | } 50 | this.golbalSetting = jobFlowConfig.getGolbalSetting(); 51 | if(StringUtils.isNotEmpty(jobFlowConfig.getCreateTime())){ 52 | this.createTime = jobFlowConfig.getCreateTime(); 53 | } 54 | if(StringUtils.isNotEmpty(jobFlowConfig.getDesignerJson())){ 55 | this.designerJson = jobFlowConfig.getDesignerJson(); 56 | } 57 | if(StringUtils.isNotEmpty(jobFlowConfig.getJobflowJson())){ 58 | this.jobflowJson = jobFlowConfig.getJobflowJson(); 59 | } 60 | 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/JobFlowConfigs.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore; 4 | import lombok.Data; 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | import org.apache.commons.lang.StringUtils; 8 | 9 | import javax.xml.bind.annotation.XmlAccessType; 10 | import javax.xml.bind.annotation.XmlAccessorType; 11 | import javax.xml.bind.annotation.XmlRootElement; 12 | import javax.xml.bind.annotation.XmlSeeAlso; 13 | import java.util.List; 14 | 15 | /** 16 | * @author leon 17 | */ 18 | @Getter 19 | @Setter 20 | @XmlRootElement 21 | @XmlAccessorType(XmlAccessType.FIELD) 22 | public class JobFlowConfigs { 23 | 24 | private List jobFlowConfigs; 25 | 26 | public static JobFlowConfigs of(List jobFlowConfigs){ 27 | JobFlowConfigs newconfigs = new JobFlowConfigs(); 28 | newconfigs.setJobFlowConfigs(jobFlowConfigs); 29 | return newconfigs; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/JobFlowSchema.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import com.fasterxml.jackson.annotation.JsonInclude; 4 | import com.leonside.dataroad.config.domain.GenericComponentConfig; 5 | import lombok.Data; 6 | 7 | import java.util.HashMap; 8 | import java.util.LinkedHashMap; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | /** 13 | * @author leon 14 | */ 15 | @Data 16 | @JsonInclude(JsonInclude.Include.NON_NULL) 17 | public class JobFlowSchema { 18 | 19 | private Map setting = new LinkedHashMap(); 20 | 21 | public List> content; 22 | 23 | } 24 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/JobFlowSchemas.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | @Data 9 | public class JobFlowSchemas { 10 | 11 | private JobFlowSchema job; 12 | 13 | public static JobFlowSchemas of(JobFlowSchema job){ 14 | JobFlowSchemas jobConfigs = new JobFlowSchemas(); 15 | jobConfigs.setJob(job); 16 | return jobConfigs; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/JobJarsReponse.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import com.leonside.dataroad.common.exception.JobFlowException; 4 | import lombok.Data; 5 | import org.apache.commons.collections.CollectionUtils; 6 | 7 | import java.util.List; 8 | import java.util.Optional; 9 | import java.util.stream.Collectors; 10 | 11 | /** 12 | * @author leon 13 | */ 14 | @Data 15 | public class JobJarsReponse { 16 | 17 | private String address; 18 | private String status; 19 | 20 | private List files; 21 | 22 | public List findDataroadJars(){ 23 | List dataroadJars = files.stream().filter(fileJarInfo -> fileJarInfo.getName().startsWith("dataroad")).collect(Collectors.toList()); 24 | return dataroadJars; 25 | } 26 | 27 | public FileJarInfo findFirstJarNotNull(){ 28 | if(CollectionUtils.isEmpty(files)){ 29 | throw new JobFlowException("请先上传dataroad Jar"); 30 | } 31 | 32 | List dataroadJars = findDataroadJars(); 33 | if(CollectionUtils.isEmpty(dataroadJars)){ 34 | throw new JobFlowException("请先上传dataroad Jar"); 35 | } 36 | return dataroadJars.get(0); 37 | } 38 | 39 | @Data 40 | public static class FileJarInfo{ 41 | private String id; 42 | private String name; 43 | private String uploaded; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/JobRequestParam.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.leonside.dataroad.DataroadEngine; 5 | import com.leonside.dataroad.common.config.Options; 6 | import com.leonside.dataroad.common.utils.JsonUtil; 7 | import com.leonside.dataroad.config.domain.JobConfigs; 8 | import com.leonside.dataroad.config.job.JsonJobSchemaParser; 9 | import com.leonside.dataroad.dashboard.configuration.DataroadProperties; 10 | import com.leonside.dataroad.flink.utils.PluginJarHelper; 11 | import lombok.Data; 12 | import org.apache.commons.collections.CollectionUtils; 13 | import org.apache.commons.collections.MapUtils; 14 | import org.apache.commons.lang.StringUtils; 15 | 16 | import java.io.File; 17 | import java.util.HashMap; 18 | import java.util.Map; 19 | import java.util.Set; 20 | import java.util.stream.Collectors; 21 | 22 | /** 23 | * @author leon 24 | */ 25 | @Data 26 | public class JobRequestParam { 27 | 28 | private String jobId; 29 | private Integer parallelism; 30 | private String allowNonRestoredState; 31 | private String savepointPath; 32 | private String confProp; 33 | 34 | } 35 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/JobSubmitReponse.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | @Data 9 | public class JobSubmitReponse { 10 | 11 | private String jobid; 12 | 13 | } 14 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/JobUploadReponse.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | @Data 9 | public class JobUploadReponse { 10 | 11 | private String filename; 12 | private String status; 13 | 14 | } 15 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/domian/ResponseStatus.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.domian; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | @Data 9 | public class ResponseStatus { 10 | private int code; 11 | private String msg; 12 | private int count; 13 | private T data; 14 | 15 | public ResponseStatus(int status, String message) { 16 | this.code = status; 17 | this.msg = message; 18 | } 19 | 20 | public ResponseStatus(int status, String message,int count, T data) { 21 | this.code = status; 22 | this.msg = message; 23 | this.data = data; 24 | this.count = count; 25 | } 26 | 27 | public ResponseStatus() { 28 | } 29 | 30 | public static ResponseStatus success(String message){ 31 | return new ResponseStatus(0, message); 32 | } 33 | 34 | public static ResponseStatus success(String message,int count, T data){ 35 | return new ResponseStatus(0, message, count, data); 36 | } 37 | 38 | public static ResponseStatus success(){ 39 | return new ResponseStatus(0, "操作成功！"); 40 | } 41 | 42 | public static ResponseStatus error(){ 43 | return new ResponseStatus(1, "操作失败！"); 44 | } 45 | 46 | public static ResponseStatus error(String mesage){ 47 | return new ResponseStatus(1, mesage); 48 | } 49 | 50 | public static ResponseStatus error(int code, String mesage){ 51 | return new ResponseStatus(code, mesage); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/listener/ApplicationEnvPreparedInitializer.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.listener; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.springframework.context.ApplicationContextInitializer; 5 | import org.springframework.context.ConfigurableApplicationContext; 6 | 7 | public class ApplicationEnvPreparedInitializer implements ApplicationContextInitializer { 8 | 9 | public static final String ENV_WEB_UI = "WEB_UI"; 10 | public static final String ENV_DATAROAD_DIST = "DATAROAD_DIST"; 11 | public static final String ENV_HOST_ADDRESS = "HOST_ADDRESS"; 12 | public static final String ENV_SAMPLE_ENABLED = "SAMPLE_ENABLED"; 13 | 14 | public ApplicationEnvPreparedInitializer() { 15 | } 16 | 17 | @Override 18 | public void initialize(ConfigurableApplicationContext applicationContext) { 19 | 20 | String webUI = applicationContext.getEnvironment().getProperty(ENV_WEB_UI); 21 | if(StringUtils.isNotEmpty(webUI)){ 22 | applicationContext.getEnvironment().getSystemProperties().put("dataroad.web-ui", webUI); 23 | } 24 | 25 | String dataroadDist = applicationContext.getEnvironment().getProperty(ENV_DATAROAD_DIST); 26 | if(StringUtils.isNotEmpty(dataroadDist)){ 27 | applicationContext.getEnvironment().getSystemProperties().put("dataroad.dataroad-dist", dataroadDist); 28 | } 29 | 30 | String serverAddress = applicationContext.getEnvironment().getProperty(ENV_HOST_ADDRESS); 31 | if(StringUtils.isNotEmpty(serverAddress)){ 32 | applicationContext.getEnvironment().getSystemProperties().put("dataroad.server-address", serverAddress); 33 | } 34 | 35 | String sampleEnabled = applicationContext.getEnvironment().getProperty(ENV_SAMPLE_ENABLED); 36 | if(StringUtils.isNotEmpty(sampleEnabled)){ 37 | applicationContext.getEnvironment().getSystemProperties().put("dataroad.sample-enabled", sampleEnabled); 38 | } 39 | 40 | } 41 | } -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/listener/ApplicationSampleInitListener.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.listener; 2 | 3 | import com.leonside.dataroad.dashboard.configuration.DataroadProperties; 4 | import com.leonside.dataroad.dashboard.service.JobFlowService; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.boot.ApplicationArguments; 8 | import org.springframework.boot.ApplicationRunner; 9 | import org.springframework.stereotype.Component; 10 | 11 | /** 12 | * @author leon 13 | */ 14 | @Slf4j 15 | @Component 16 | public class ApplicationSampleInitListener implements ApplicationRunner { 17 | @Autowired 18 | private DataroadProperties dataroadProperties; 19 | @Autowired 20 | private JobFlowService jobFlowService; 21 | 22 | @Override 23 | public void run(ApplicationArguments args) { 24 | 25 | if(dataroadProperties.isSampleEnabled()){ 26 | try { 27 | jobFlowService.initSampleConfig(); 28 | } catch (Exception e) { 29 | log.error("Failed to initialize sample data",e); 30 | } 31 | } 32 | 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/repository/JobFlowRepository.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.repository; 2 | 3 | import com.leonside.dataroad.dashboard.domian.JobFlowConfig; 4 | import org.springframework.stereotype.Repository; 5 | 6 | import javax.xml.bind.JAXBException; 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.io.InputStream; 10 | import java.io.InputStreamReader; 11 | import java.util.List; 12 | 13 | /** 14 | * @author leon 15 | */ 16 | @Repository 17 | public interface JobFlowRepository { 18 | 19 | void save(JobFlowConfig jobFlowConfig) throws Exception; 20 | 21 | JobFlowConfig get(String id); 22 | 23 | List loadAll() throws Exception; 24 | 25 | boolean containJobFlow(String id); 26 | 27 | void delete(String id) throws Exception; 28 | 29 | void updateJobFlow(JobFlowConfig jobFlowConfig) throws Exception; 30 | 31 | void updateJobFlowBase(JobFlowConfig jobFlowConfig) throws Exception; 32 | 33 | void init(InputStream sampleConfig) throws Exception; 34 | } 35 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/service/JobFlowService.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.service; 2 | 3 | import com.leonside.dataroad.dashboard.domian.JobFlowConfig; 4 | 5 | import java.io.IOException; 6 | import java.util.List; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | public interface JobFlowService { 12 | 13 | boolean containJobFlow(String id); 14 | 15 | void saveJobFlow(JobFlowConfig jobFlowConfig) throws Exception; 16 | 17 | JobFlowConfig loadJobFlowConfig(String id); 18 | 19 | String loadJobFlowJson(String id); 20 | 21 | List loadJobFlowJsons(String[] ids); 22 | 23 | List loadJobFlowConfig() throws Exception; 24 | 25 | void deleteJobFlow(String id) throws Exception; 26 | 27 | void updateJobFlow(JobFlowConfig jobFlowConfig) throws Exception; 28 | 29 | void updateBaseJobFlow(JobFlowConfig jobFlowConfig) throws Exception; 30 | 31 | void initSampleConfig() throws Exception; 32 | } 33 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/service/JobFlowTaskService.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.service; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.leonside.dataroad.dashboard.domian.*; 5 | 6 | import java.io.IOException; 7 | import java.io.UnsupportedEncodingException; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | public interface JobFlowTaskService { 13 | 14 | JobSubmitReponse submitJobFlow(JobRequestParam jobRequestParam) throws JsonProcessingException, UnsupportedEncodingException; 15 | 16 | JobUploadReponse uploadDataroadJar() throws IOException; 17 | 18 | JobJarsReponse listDataroadJar(); 19 | 20 | void checkDataroadJar(); 21 | } 22 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/utils/HomeFolderUtils.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.utils; 2 | 3 | import lombok.AccessLevel; 4 | import lombok.NoArgsConstructor; 5 | 6 | import java.io.File; 7 | 8 | @NoArgsConstructor(access = AccessLevel.PRIVATE) 9 | public final class HomeFolderUtils { 10 | 11 | public static final String STORE_FILE_NAME = "jobflowConfig.xml"; 12 | 13 | private static final String USER_HOME = System.getProperty("user.home"); 14 | 15 | private static final String FILE_SEPARATOR = System.getProperty("file.separator"); 16 | 17 | private static final String CONSOLE_ROOT_FOLDER = ".dataroad-dashboard"; 18 | 19 | private static final String SCHEMA_FOLDER = "schema"; 20 | 21 | /** 22 | * 获取用户目录文件. 23 | * 24 | * @param fileName 文件名 25 | * @return 用户目录所在的文件名 26 | */ 27 | public static String getFilePathInHomeFolder(final String fileName) { 28 | return String.format("%s%s", getHomeFolder(), fileName); 29 | } 30 | 31 | public static String getSchemaFilePathInHomeFolder(final String fileName) { 32 | return String.format("%s%s", getHomeSchemaFolder(), fileName); 33 | } 34 | 35 | /** 36 | * 创建用户目录. 37 | */ 38 | public static void createHomeFolderIfNotExisted() { 39 | File file = new File(getHomeFolder()); 40 | if (!file.exists()) { 41 | file.mkdirs(); 42 | } 43 | } 44 | 45 | public static String getHomeFolder() { 46 | return String.format("%s%s%s%s", USER_HOME, FILE_SEPARATOR, CONSOLE_ROOT_FOLDER, FILE_SEPARATOR); 47 | } 48 | 49 | private static String getHomeSchemaFolder() { 50 | return String.format("%s%s%s%s%s%s", USER_HOME, FILE_SEPARATOR, CONSOLE_ROOT_FOLDER, FILE_SEPARATOR,SCHEMA_FOLDER,FILE_SEPARATOR); 51 | } 52 | } -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/utils/ObjectUtils.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.utils; 2 | 3 | import java.lang.reflect.Array; 4 | import java.util.Collection; 5 | import java.util.Map; 6 | 7 | public class ObjectUtils { 8 | public ObjectUtils() { 9 | } 10 | 11 | public static boolean isNullOrEmptyString(Object o) { 12 | if (o == null) { 13 | return true; 14 | } else { 15 | if (o instanceof String) { 16 | String str = (String)o; 17 | if (str.length() == 0) { 18 | return true; 19 | } 20 | } 21 | 22 | return false; 23 | } 24 | } 25 | 26 | public static boolean isEmpty(Object o) { 27 | if (o == null) { 28 | return true; 29 | } else { 30 | if (o instanceof String) { 31 | if (((String)o).length() == 0) { 32 | return true; 33 | } 34 | } else if (o instanceof Collection) { 35 | if (((Collection)o).isEmpty()) { 36 | return true; 37 | } 38 | } else if (o.getClass().isArray()) { 39 | if (Array.getLength(o) == 0) { 40 | return true; 41 | } 42 | } else { 43 | if (!(o instanceof Map)) { 44 | return false; 45 | } 46 | 47 | if (((Map)o).isEmpty()) { 48 | return true; 49 | } 50 | } 51 | 52 | return false; 53 | } 54 | } 55 | 56 | public static boolean isNotEmpty(Object c) throws IllegalArgumentException { 57 | return !isEmpty(c); 58 | } 59 | } -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/utils/PageUtils.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.utils; 2 | 3 | import com.leonside.dataroad.dashboard.domian.JobFlowConfig; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | public class PageUtils { 11 | public static List subList(List dataList, int page, int limit) { 12 | 13 | int begin = (page - 1) * limit; 14 | int end = (begin + limit) > dataList.size() ? dataList.size() : begin + limit; 15 | return dataList.subList(begin, end); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/java/com/leonside/dataroad/dashboard/utils/ZipUtils.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.dashboard.utils; 2 | 3 | import com.leonside.dataroad.common.exception.JobConfigException; 4 | import com.leonside.dataroad.common.utils.DateUtil; 5 | import lombok.extern.slf4j.Slf4j; 6 | import org.apache.commons.io.IOUtils; 7 | import org.apache.commons.lang3.time.DateUtils; 8 | 9 | import java.io.*; 10 | import java.util.Arrays; 11 | import java.util.Date; 12 | import java.util.zip.ZipEntry; 13 | import java.util.zip.ZipOutputStream; 14 | 15 | /** 16 | * @author leon 17 | */ 18 | @Slf4j 19 | public class ZipUtils { 20 | 21 | public static File zip (String[] fileIds, String zipFileName) { 22 | 23 | String zipFile = HomeFolderUtils.getHomeFolder() + File.separator + File.separator + zipFileName; 24 | 25 | try(ZipOutputStream out = new ZipOutputStream(new FileOutputStream(zipFile))){ 26 | 27 | Arrays.stream(fileIds).forEach(fileName->{ 28 | String schemaFilePathInHomeFolder = HomeFolderUtils.getSchemaFilePathInHomeFolder(fileName); 29 | try { 30 | File file = new File(schemaFilePathInHomeFolder); 31 | out.putNextEntry(new ZipEntry(file.getName())); 32 | try(FileInputStream inputStream = new FileInputStream(file)){ 33 | IOUtils.copy(inputStream, out); 34 | } 35 | } catch (IOException e) { 36 | log.error("download file exception",e); 37 | } 38 | }); 39 | return new File(zipFile); 40 | } catch (IOException e) { 41 | log.error("download file exception",e); 42 | throw new JobConfigException("download file exception",e); 43 | } 44 | } 45 | 46 | // public static void main(String[] args) throws FileNotFoundException { 47 | // zip(new String[]{"es_filter_es.json","mysql_aggcount_es.json"}, "dataroad-"+ DateUtil.dateToStoreDateTimeString(new Date()) + ".zip"); 48 | // } 49 | } 50 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.context.ApplicationContextInitializer=\ 2 | com.leonside.dataroad.dashboard.listener.ApplicationEnvPreparedInitializer -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | server: 2 | port: 8089 3 | 4 | spring: 5 | application: 6 | name: dataroad-dashboard 7 | mvc: 8 | view: 9 | suffix: .html 10 | prefix: / 11 | 12 | dataroad: 13 | #dataroad-dist 包路径地址，docker中默认在/opt/dataroad-dist/下 14 | dataroad-dist: /opt/dataroad-dist/ 15 | #Flink web-ui地址 16 | web-ui: http://bigdata32:8081 17 | #当前服务器IP地址，当不能获取正确的外部IP地址时候可设置此参数 18 | # host-address: 10.11.1.14 19 | #是否初始化示例流程数据 20 | sample-enabled: false 21 | 22 | 23 | logging: 24 | level: 25 | com: 26 | leonside: 27 | dataroad: DEBUG 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/go.js: -------------------------------------------------------------------------------- 1 | document.write(''); 2 | -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layer/skin/default/icon-ext.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layer/skin/default/icon-ext.png -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layer/skin/default/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layer/skin/default/icon.png -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layer/skin/default/loading-0.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layer/skin/default/loading-0.gif -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layer/skin/default/loading-1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layer/skin/default/loading-1.gif -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layer/skin/default/loading-2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layer/skin/default/loading-2.gif -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/css/modules/code.css: -------------------------------------------------------------------------------- 1 | html #layuicss-skincodecss{display:none;position:absolute;width:1989px}.layui-code-h3,.layui-code-view{position:relative;font-size:12px}.layui-code-view{display:block;margin:10px 0;padding:0;border:1px solid #eee;border-left-width:6px;background-color:#FAFAFA;color:#333;font-family:Courier New}.layui-code-h3{padding:0 10px;height:40px;line-height:40px;border-bottom:1px solid #eee}.layui-code-h3 a{position:absolute;right:10px;top:0;color:#999}.layui-code-view .layui-code-ol{position:relative;overflow:auto}.layui-code-view .layui-code-ol li{position:relative;margin-left:45px;line-height:20px;padding:0 10px;border-left:1px solid #e2e2e2;list-style-type:decimal-leading-zero;*list-style-type:decimal;background-color:#fff}.layui-code-view .layui-code-ol li:first-child{padding-top:10px}.layui-code-view .layui-code-ol li:last-child{padding-bottom:10px}.layui-code-view pre{margin:0}.layui-code-notepad{border:1px solid #0C0C0C;border-left-color:#3F3F3F;background-color:#0C0C0C;color:#C2BE9E}.layui-code-notepad .layui-code-h3{border-bottom:none}.layui-code-notepad .layui-code-ol li{background-color:#3F3F3F;border-left:none}.layui-code-demo .layui-code{visibility:visible!important;margin:-15px;border-top:none;border-right:none;border-bottom:none}.layui-code-demo .layui-tab-content{padding:15px;border-top:none} -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/icon-ext.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/icon-ext.png -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/icon.png -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/loading-0.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/loading-0.gif -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/loading-1.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/loading-1.gif -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/loading-2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layui/css/modules/layer/default/loading-2.gif -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/font/iconfont.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layui/font/iconfont.eot -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/font/iconfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layui/font/iconfont.ttf -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/font/iconfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layui/font/iconfont.woff -------------------------------------------------------------------------------- /dataroad-dashboard/src/main/resources/static/js/layui/font/iconfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-dashboard/src/main/resources/static/js/layui/font/iconfont.woff2 -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/FlinkJob.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink; 2 | 3 | import com.leonside.dataroad.core.Job; 4 | import com.leonside.dataroad.core.flow.SimpleJobFlow; 5 | import com.leonside.dataroad.flink.context.FlinkExecuteContext; 6 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | public class FlinkJob implements Job { 12 | 13 | private SimpleJobFlow simpleJobFlow; 14 | 15 | private StreamExecutionEnvironment environment; 16 | 17 | private FlinkExecuteContext flinkExecuteContext; 18 | 19 | public FlinkJob(FlinkExecuteContext context, SimpleJobFlow startJobFlow) { 20 | this.simpleJobFlow = startJobFlow; 21 | this.flinkExecuteContext = context; 22 | this.environment = context.getEnvironment(); 23 | flinkExecuteContext.setStartJobFlow(startJobFlow); 24 | } 25 | 26 | @Override 27 | public void execute() throws Exception { 28 | 29 | simpleJobFlow.execute(flinkExecuteContext); 30 | 31 | environment.execute(flinkExecuteContext.getJobSetting().getName()); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/config/ScriptExpressionConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.config; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.common.config.ConfigKey; 5 | import com.leonside.dataroad.common.script.ScriptEvaluatorFactory; 6 | import com.leonside.dataroad.common.config.Validation; 7 | import lombok.Data; 8 | 9 | import java.util.Map; 10 | 11 | /** 12 | * @author leon 13 | */ 14 | @Data 15 | public class ScriptExpressionConfig extends BaseConfig { 16 | 17 | public ScriptEvaluatorFactory.ScriptEngine language = ScriptEvaluatorFactory.ScriptEngine.fel; 18 | 19 | public String expression; 20 | 21 | public ScriptExpressionConfig(Map parameter) { 22 | super(parameter); 23 | } 24 | 25 | @Override 26 | public boolean validate() { 27 | 28 | return super.validate(); 29 | } 30 | 31 | @Override 32 | public Class bindConfigKey() { 33 | return ScriptExpressionConfigKey.class; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/context/FlinkExecuteContext.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.context; 2 | 3 | import com.leonside.dataroad.common.context.ExecuteContext; 4 | import com.leonside.dataroad.common.domain.MetaColumn; 5 | import com.leonside.dataroad.core.flow.JobFlow; 6 | import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; 7 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 8 | 9 | import java.util.List; 10 | 11 | /** 12 | * @author leon 13 | */ 14 | public class FlinkExecuteContext extends ExecuteContext { 15 | 16 | public static final String CONFIGURATION_KEY_PARALLELISM = "parallelism.default"; 17 | 18 | private transient StreamExecutionEnvironment environment; 19 | 20 | private transient StreamTableEnvironment streamTableEnvironment; 21 | 22 | private transient JobFlow startJobFlow; 23 | 24 | private List startMetaColumn; 25 | 26 | public StreamExecutionEnvironment getEnvironment() { 27 | return environment; 28 | } 29 | 30 | public JobFlow getStartJobFlow() { 31 | return startJobFlow; 32 | } 33 | 34 | public void setStartJobFlow(JobFlow startJobFlow) { 35 | this.startJobFlow = startJobFlow; 36 | } 37 | 38 | public List getStartMetaColumn() { 39 | return startMetaColumn; 40 | } 41 | 42 | public void setStartMetaColumn(List startMetaColumn) { 43 | this.startMetaColumn = startMetaColumn; 44 | } 45 | 46 | public void setEnvironment(StreamExecutionEnvironment environment) { 47 | this.environment = environment; 48 | } 49 | 50 | public StreamTableEnvironment getOrCreateStreamTableEnvironment() { 51 | if(streamTableEnvironment == null){ 52 | streamTableEnvironment = StreamTableEnvironment.create(getEnvironment()); 53 | } 54 | return streamTableEnvironment; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/metric/BaseMetric.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.metric; 2 | 3 | import org.apache.flink.api.common.accumulators.LongCounter; 4 | import org.apache.flink.api.common.functions.RuntimeContext; 5 | import org.apache.flink.metrics.MetricGroup; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | public class BaseMetric { 13 | 14 | protected final Logger LOG = LoggerFactory.getLogger(getClass()); 15 | 16 | private Long delayPeriodMill = 20000L; 17 | 18 | private MetricGroup flinkxOutput; 19 | 20 | private final Map metricCounters = new HashMap<>(); 21 | 22 | public BaseMetric(RuntimeContext runtimeContext) { 23 | flinkxOutput = runtimeContext.getMetricGroup().addGroup(Metrics.METRIC_GROUP_KEY_FLINKX, Metrics.METRIC_GROUP_VALUE_OUTPUT); 24 | } 25 | 26 | public void addMetric(String metricName, LongCounter counter){ 27 | addMetric(metricName, counter, false); 28 | } 29 | 30 | public void addMetric(String metricName, LongCounter counter, boolean meterView){ 31 | metricCounters.put(metricName, counter); 32 | flinkxOutput.gauge(metricName, new SimpleAccumulatorGauge<>(counter)); 33 | if (meterView){ 34 | flinkxOutput.meter(metricName + Metrics.SUFFIX_RATE, new SimpleLongCounterMeterView(counter, 20)); 35 | } 36 | } 37 | 38 | public Map getMetricCounters() { 39 | return metricCounters; 40 | } 41 | 42 | public void waitForReportMetrics() { 43 | try { 44 | Thread.sleep(delayPeriodMill); 45 | } catch (InterruptedException e){ 46 | LOG.warn("Task thread is interrupted"); 47 | } 48 | } 49 | } -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/metric/Metrics.java: -------------------------------------------------------------------------------- 1 | 2 | package com.leonside.dataroad.flink.metric; 3 | 4 | /** 5 | * Statistics Metrics 6 | * 7 | */ 8 | public class Metrics { 9 | 10 | public static final String NUM_ERRORS = "nErrors"; 11 | 12 | public static final String NUM_NULL_ERRORS = "nullErrors"; 13 | 14 | public static final String NUM_DUPLICATE_ERRORS = "duplicateErrors"; 15 | 16 | public static final String NUM_CONVERSION_ERRORS = "conversionErrors"; 17 | 18 | public static final String NUM_OTHER_ERRORS = "otherErrors"; 19 | 20 | public static final String READ_BYTES = "byteRead"; 21 | 22 | public static final String READ_DURATION = "readDuration"; 23 | 24 | public static final String WRITE_BYTES = "byteWrite"; 25 | 26 | public static final String WRITE_DURATION = "writeDuration"; 27 | 28 | public static final String NUM_WRITES = "numWrite"; 29 | 30 | public static final String SNAPSHOT_WRITES = "snapshotWrite"; 31 | 32 | public static final String JOB_NAME = ""; 33 | 34 | public static final String JOB_ID = ""; 35 | 36 | public static final String SUBTASK_INDEX = ""; 37 | 38 | public static final String NUM_READS = "numRead"; 39 | 40 | public static final String END_LOCATION = "endLocation"; 41 | 42 | public static final String START_LOCATION = "startLocation"; 43 | 44 | public static final String TABLE_COL = "tableCol"; 45 | 46 | public static final String MAX_VALUE = "maxValue"; 47 | 48 | public static final String METRIC_GROUP_KEY_FLINKX = "flinkx"; 49 | 50 | public static final String METRIC_GROUP_VALUE_OUTPUT = "output"; 51 | 52 | public static final String LAST_WRITE_LOCATION_PREFIX = "last_write_location"; 53 | 54 | public static final String LAST_WRITE_NUM__PREFIX = "last_write_num"; 55 | 56 | public static final String SUFFIX_RATE = "PerSecond"; 57 | 58 | } 59 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/metric/SimpleAccumulatorGauge.java: -------------------------------------------------------------------------------- 1 | 2 | package com.leonside.dataroad.flink.metric; 3 | 4 | import org.apache.flink.api.common.accumulators.Accumulator; 5 | import org.apache.flink.metrics.Gauge; 6 | 7 | import java.io.Serializable; 8 | 9 | /** 10 | * 11 | */ 12 | public class SimpleAccumulatorGauge implements Gauge { 13 | 14 | private Accumulator accumulator; 15 | 16 | public SimpleAccumulatorGauge(Accumulator accumulator) { 17 | this.accumulator = accumulator; 18 | } 19 | 20 | @Override 21 | public T getValue() { 22 | return accumulator.getLocalValue(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/predicate/ExpressionPredicate.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.predicate; 2 | 3 | import com.leonside.dataroad.common.exception.ScriptExecuteException; 4 | import com.leonside.dataroad.common.script.ScriptEvaluator; 5 | import com.leonside.dataroad.common.script.ScriptEvaluatorFactory; 6 | import com.leonside.dataroad.core.component.ComponentInitialization; 7 | import com.leonside.dataroad.core.component.ComponentNameSupport; 8 | import com.leonside.dataroad.core.spi.JobPredicate; 9 | import com.leonside.dataroad.flink.config.ScriptExpressionConfig; 10 | import com.leonside.dataroad.flink.context.FlinkExecuteContext; 11 | import org.apache.flink.types.Row; 12 | 13 | import java.util.Map; 14 | 15 | /** 16 | * @author leon 17 | */ 18 | public class ExpressionPredicate extends ComponentNameSupport implements ComponentInitialization, JobPredicate { 19 | 20 | private ScriptExpressionConfig scriptExpressionConfig; 21 | 22 | private ScriptEvaluator scriptEvalutor; 23 | 24 | private Map parameter; 25 | 26 | @Override 27 | public boolean test(FlinkExecuteContext executeContext, Row row) { 28 | 29 | try{ 30 | Object evaluate = scriptEvalutor.evaluate(row, parameter); 31 | 32 | if(evaluate == null || ! (evaluate instanceof Boolean)){ 33 | throw new ScriptExecuteException("Boolean must be returned， check the expression is valid. [" + scriptExpressionConfig.getExpression() + "]"); 34 | } 35 | return (Boolean)evaluate; 36 | }catch (Exception exception){ 37 | throw new ScriptExecuteException("Script execution error [" + scriptExpressionConfig.getExpression() + "]",exception); 38 | } 39 | } 40 | 41 | @Override 42 | public void doInitialize(FlinkExecuteContext executeContext, ScriptExpressionConfig config) { 43 | this.parameter = config.getParameter(); 44 | this.scriptExpressionConfig = config ; 45 | scriptEvalutor = ScriptEvaluatorFactory.createScriptEvalutor(scriptExpressionConfig.getLanguage(), scriptExpressionConfig.getExpression()); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/predicate/TrueExpressionPredicate.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.predicate; 2 | 3 | import com.leonside.dataroad.core.component.ComponentInitialization; 4 | import com.leonside.dataroad.core.component.ComponentNameSupport; 5 | import com.leonside.dataroad.core.spi.JobPredicate; 6 | import com.leonside.dataroad.flink.config.ScriptExpressionConfig; 7 | import com.leonside.dataroad.flink.context.FlinkExecuteContext; 8 | import org.apache.flink.types.Row; 9 | 10 | /** 11 | * @author leon 12 | */ 13 | public class TrueExpressionPredicate extends ComponentNameSupport implements ComponentInitialization, JobPredicate { 14 | 15 | public static final String COMPONENT_ID = "trueExpressionPredicate"; 16 | 17 | @Override 18 | public boolean test(FlinkExecuteContext executeContext, Row row) { 19 | 20 | return true; 21 | } 22 | 23 | @Override 24 | public void doInitialize(FlinkExecuteContext executeContext, ScriptExpressionConfig config) { 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/processor/aggeration/function/WindowAggerationFunction.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.processor.aggeration.function; 2 | 3 | import org.apache.flink.streaming.api.datastream.DataStream; 4 | import org.apache.flink.types.Row; 5 | 6 | import java.io.Serializable; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | public interface WindowAggerationFunction extends Serializable { 12 | 13 | DataStream processWindow(DataStream dataStream); 14 | 15 | DataStream processKeyByWindow(DataStream dataStream); 16 | 17 | 18 | 19 | } 20 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/processor/lookup/DirectAllLookupProcessor.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.processor.lookup; 2 | 3 | import com.leonside.dataroad.core.component.ComponentInitialization; 4 | import com.leonside.dataroad.core.component.ComponentNameSupport; 5 | import com.leonside.dataroad.core.spi.ItemLookupProcessor; 6 | import com.leonside.dataroad.flink.context.FlinkExecuteContext; 7 | import com.leonside.dataroad.flink.processor.lookup.config.BaseLookupConfig; 8 | import com.leonside.dataroad.flink.processor.lookup.function.DirectAllLookupFunction; 9 | import org.apache.flink.streaming.api.datastream.DataStream; 10 | import org.apache.flink.types.Row; 11 | 12 | /** 13 | * @author leon 14 | */ 15 | public class DirectAllLookupProcessor extends ComponentNameSupport implements ComponentInitialization, ItemLookupProcessor,DataStream> { 16 | 17 | private BaseLookupConfig lookupConfig; 18 | 19 | @Override 20 | public DataStream process(FlinkExecuteContext executeContext, DataStream dataStream) { 21 | 22 | DirectAllLookupFunction directAllLookupFunction = new DirectAllLookupFunction.DirectLookupFunctionBuilder() 23 | .lookupConfig(lookupConfig) 24 | .build(); 25 | 26 | return dataStream.map(directAllLookupFunction); 27 | } 28 | 29 | @Override 30 | public void doInitialize(FlinkExecuteContext executeContext, BaseLookupConfig baseConfig) { 31 | lookupConfig = baseConfig; 32 | } 33 | 34 | 35 | // @Override 36 | // public Class configClass() { 37 | // return BaseLookupConfig.class; 38 | // } 39 | } 40 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/processor/lookup/config/BaseLookupConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.processor.lookup.config; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.common.config.ConfigKey; 5 | import lombok.Data; 6 | 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | /** 11 | * @author leon 12 | */ 13 | @Data 14 | public class BaseLookupConfig extends BaseConfig { 15 | 16 | public List> directData; 17 | 18 | public Map joinColumns; 19 | 20 | public String[] columns; 21 | 22 | public int cacheMaxrows = -1; 23 | public int cacheTtl; 24 | /** 25 | * 包含 all、lru 26 | */ 27 | public String cacheType; 28 | // private int cachePeriod ; 29 | 30 | 31 | public BaseLookupConfig(Map parameter) { 32 | super(parameter); 33 | } 34 | 35 | @Override 36 | public boolean validate() { 37 | return super.validate(); 38 | } 39 | 40 | @Override 41 | public Class bindConfigKey() { 42 | return BaseLookupConfigKey.class; 43 | } 44 | 45 | public enum CacheType{ 46 | lru, all,none 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/processor/lookup/function/DirectAllLookupFunction.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.processor.lookup.function; 2 | 3 | import com.leonside.dataroad.common.exception.JobConfigException; 4 | import com.leonside.dataroad.flink.processor.lookup.config.BaseLookupConfig; 5 | import com.leonside.dataroad.flink.utils.RowUtils; 6 | import org.apache.commons.collections.CollectionUtils; 7 | import org.apache.flink.configuration.Configuration; 8 | import org.apache.flink.types.Row; 9 | 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | /** 14 | * @author leon 15 | */ 16 | public class DirectAllLookupFunction extends AbstractAllLookupFunction { 17 | 18 | public DirectAllLookupFunction(BaseLookupConfig baseLookupConfig) { 19 | super(baseLookupConfig); 20 | } 21 | 22 | @Override 23 | public void open(Configuration parameters) throws Exception { 24 | super.open(parameters); 25 | 26 | loadAllData(); 27 | } 28 | 29 | @Override 30 | protected void loadAllData() { 31 | List> directData = baseLookupConfig.getDirectData(); 32 | if(CollectionUtils.isEmpty(directData)){ 33 | throw new JobConfigException("DirectData for DirectLookup cannot be null"); 34 | } 35 | 36 | directData.stream().forEach(it->{ 37 | Row row = RowUtils.toRowWithNames(it); 38 | 39 | buildCache(row); 40 | }); 41 | } 42 | 43 | public static class DirectLookupFunctionBuilder { 44 | 45 | private BaseLookupConfig lookupConfig; 46 | 47 | public DirectLookupFunctionBuilder lookupConfig(BaseLookupConfig lookupConfig){ 48 | this.lookupConfig = lookupConfig; 49 | return this; 50 | } 51 | 52 | public DirectAllLookupFunction build() { 53 | return new DirectAllLookupFunction(lookupConfig); 54 | } 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/processor/sql/SqlTransformerProcessor.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.processor.sql; 2 | 3 | import com.leonside.dataroad.common.spi.ItemProcessor; 4 | import com.leonside.dataroad.core.component.ComponentInitialization; 5 | import com.leonside.dataroad.core.component.ComponentNameSupport; 6 | import com.leonside.dataroad.flink.context.FlinkExecuteContext; 7 | import com.leonside.dataroad.flink.processor.sql.config.SqlTransformerConfig; 8 | import org.apache.flink.streaming.api.datastream.DataStream; 9 | import org.apache.flink.table.api.Table; 10 | import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; 11 | import org.apache.flink.types.Row; 12 | 13 | /** 14 | * @author leon 15 | */ 16 | public class SqlTransformerProcessor extends ComponentNameSupport implements ComponentInitialization, ItemProcessor,DataStream> { 17 | 18 | private SqlTransformerConfig sqlTransformerConfig; 19 | 20 | @Override 21 | public DataStream process(FlinkExecuteContext executeContext, DataStream dataStream) { 22 | 23 | StreamTableEnvironment streamTableEnvironment = executeContext.getOrCreateStreamTableEnvironment(); 24 | 25 | streamTableEnvironment.createTemporaryView(sqlTransformerConfig.getTableName(), dataStream); 26 | 27 | Table table = streamTableEnvironment.sqlQuery(sqlTransformerConfig.getSql()); 28 | 29 | return streamTableEnvironment.toDataStream(table); 30 | } 31 | 32 | 33 | @Override 34 | public void doInitialize(FlinkExecuteContext executeContext, SqlTransformerConfig config) { 35 | sqlTransformerConfig = config; 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/processor/sql/config/SqlTransformerConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.processor.sql.config; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.common.config.ConfigKey; 5 | import lombok.Data; 6 | 7 | import java.util.Map; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | @Data 13 | public class SqlTransformerConfig extends BaseConfig { 14 | 15 | public String sql; 16 | 17 | public String tableName; 18 | 19 | public SqlTransformerConfig(Map parameter) { 20 | super(parameter); 21 | } 22 | 23 | @Override 24 | public Class bindConfigKey() { 25 | return SqlTransformerConfigKey.class; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/processor/union/GenericItemUnionProcessor.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.processor.union; 2 | 3 | import com.leonside.dataroad.common.constant.JobConfigKeyConstants; 4 | import com.leonside.dataroad.common.spi.ItemUnionProcessor; 5 | import com.leonside.dataroad.core.component.ComponentNameAutoGenerator; 6 | import com.leonside.dataroad.flink.context.FlinkExecuteContext; 7 | import org.apache.flink.streaming.api.datastream.DataStream; 8 | import org.apache.flink.types.Row; 9 | 10 | import java.util.ArrayList; 11 | import java.util.Arrays; 12 | import java.util.List; 13 | import java.util.Optional; 14 | 15 | /** 16 | * @author leon 17 | */ 18 | public class GenericItemUnionProcessor extends ComponentNameAutoGenerator implements ItemUnionProcessor>, DataStream> { 19 | 20 | public static String COMPONENT_ID = "unionProcessor"; 21 | 22 | private Integer[] flowindexs; 23 | 24 | public GenericItemUnionProcessor(){ 25 | super(JobConfigKeyConstants.COMPONENT_PREFIX_UNION_PROCESSOR); 26 | } 27 | 28 | @Override 29 | public void initializeUnionFlowIndex(Integer[] indexs) { 30 | this.flowindexs = indexs; 31 | } 32 | 33 | @Override 34 | public DataStream process(FlinkExecuteContext executeContext, List> dataStreams) { 35 | 36 | List> unionDataStreamList; 37 | if(flowindexs != null){ 38 | unionDataStreamList = new ArrayList<>(); 39 | Arrays.stream(flowindexs).forEach(it->{ 40 | unionDataStreamList.add(dataStreams.get(it)); 41 | }); 42 | }else{ 43 | unionDataStreamList = dataStreams; 44 | } 45 | 46 | Optional> reduceStream = unionDataStreamList.stream() 47 | .reduce((dataStream, dataStream2) -> dataStream.union(dataStream2)); 48 | 49 | return reduceStream.isPresent() ? reduceStream.get() : null; 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/reader/inputformat/ErrorInputSplit.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | 20 | package com.leonside.dataroad.flink.reader.inputformat; 21 | 22 | import org.apache.flink.core.io.InputSplit; 23 | 24 | /** 25 | * @author jiangbo 26 | * @date 2019/11/21 27 | */ 28 | public class ErrorInputSplit implements InputSplit { 29 | 30 | int splitNumber; 31 | 32 | String errorMessage; 33 | 34 | public ErrorInputSplit(String errorMessage) { 35 | this.errorMessage = errorMessage; 36 | } 37 | 38 | public String getErrorMessage(){ 39 | return errorMessage; 40 | } 41 | 42 | @Override 43 | public int getSplitNumber() { 44 | return splitNumber; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/reader/inputformat/GenericRichInputFormatBuilder.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.reader.inputformat; 2 | 3 | import com.google.common.base.Preconditions; 4 | import com.leonside.dataroad.common.context.RestoreConfig; 5 | import com.leonside.dataroad.common.config.Validation; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | public abstract class GenericRichInputFormatBuilder implements Validation { 13 | 14 | protected final Logger LOG = LoggerFactory.getLogger(getClass()); 15 | 16 | protected T format; 17 | 18 | public R setMonitorUrls(String monitorUrls) { 19 | format.monitorUrls = monitorUrls; 20 | return (R)this; 21 | } 22 | 23 | public R setBytes(long bytes) { 24 | format.bytes = bytes; 25 | return (R) this; 26 | } 27 | 28 | public R setRestoreConfig(RestoreConfig restoreConfig){ 29 | format.restoreConfig = restoreConfig; 30 | return (R) this; 31 | } 32 | 33 | 34 | public T finish() { 35 | Preconditions.checkNotNull(format); 36 | 37 | validate(); 38 | 39 | return format; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/utils/FieldAccessorExtendFactory.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.utils; 2 | 3 | import org.apache.flink.api.common.ExecutionConfig; 4 | import org.apache.flink.api.common.typeinfo.TypeInformation; 5 | import org.apache.flink.api.java.typeutils.RowTypeInfo; 6 | import org.apache.flink.streaming.util.typeutils.FieldAccessor; 7 | import org.apache.flink.streaming.util.typeutils.FieldAccessorFactory; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | public class FieldAccessorExtendFactory { 13 | 14 | public static FieldAccessor getAccessor( 15 | TypeInformation typeInfo, String field, ExecutionConfig config) { 16 | 17 | if(typeInfo instanceof RowTypeInfo){ 18 | return (FieldAccessor) new RowFieldAccessor(field, typeInfo); 19 | }else{ 20 | return org.apache.flink.streaming.util.typeutils.FieldAccessorFactory.getAccessor(typeInfo, field,config); 21 | } 22 | } 23 | 24 | public static FieldAccessor getAccessor( 25 | TypeInformation typeInfo, int pos, ExecutionConfig config) { 26 | 27 | if(typeInfo instanceof RowTypeInfo){ 28 | return (FieldAccessor) new RowFieldAccessor(pos, typeInfo); 29 | }else{ 30 | return FieldAccessorFactory.getAccessor(typeInfo, pos,config); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/utils/RawTypeConverter.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.utils; 2 | 3 | import org.apache.flink.api.common.typeinfo.TypeInformation; 4 | import org.apache.flink.table.types.DataType; 5 | 6 | /** 7 | * Each connector implements. It is used to convert raw type to flink type. 8 | * 9 | *

e.g.: convert string "SHORT" to {@link DataType}. 10 | */ 11 | @FunctionalInterface 12 | public interface RawTypeConverter { 13 | 14 | /** 15 | * @param type raw type string. e.g.: "SHORT", "INT", "TIMESTAMP" 16 | * @return e.g.: DataTypes.INT(), DataTypes.TIMESTAMP(). 17 | */ 18 | TypeInformation apply(String type); 19 | } 20 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/utils/RawTypeUtils.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.utils; 2 | 3 | import com.leonside.dataroad.common.domain.MetaColumn; 4 | import com.leonside.dataroad.common.exception.JobConfigException; 5 | import org.apache.commons.collections.CollectionUtils; 6 | import org.apache.commons.lang.StringUtils; 7 | import org.apache.flink.api.common.typeinfo.TypeInfo; 8 | import org.apache.flink.api.common.typeinfo.TypeInformation; 9 | import org.apache.flink.api.java.typeutils.RowTypeInfo; 10 | import org.apache.flink.table.types.DataType; 11 | import org.apache.flink.table.types.utils.TypeConversions; 12 | 13 | import java.util.List; 14 | 15 | /** 16 | * @author leon 17 | */ 18 | public class RawTypeUtils { 19 | 20 | public static TypeInformation createRowTypeInfo(RawTypeConverter converter, List metaColumns) { 21 | if(CollectionUtils.isEmpty(metaColumns) || StringUtils.isEmpty(metaColumns.get(0).getType())){ 22 | throw new JobConfigException("column and type can not be null"); 23 | } 24 | 25 | TypeInformation[] types = metaColumns.stream().map(it -> converter.apply(it.getType())).toArray(TypeInformation[]::new); 26 | String[] names = metaColumns.stream().map(it -> it.getName()).toArray(String[]::new); 27 | 28 | return new RowTypeInfo(types, names); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/utils/RowFieldAccessor.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.utils; 2 | 3 | import org.apache.commons.lang.StringUtils; 4 | import org.apache.flink.api.common.typeinfo.TypeInformation; 5 | import org.apache.flink.api.java.typeutils.RowTypeInfo; 6 | import org.apache.flink.streaming.util.typeutils.FieldAccessor; 7 | import org.apache.flink.types.Row; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | public class RowFieldAccessor extends FieldAccessor { 13 | 14 | public String field; 15 | 16 | public int pos; 17 | 18 | private TypeInformation typeInfo; 19 | 20 | public RowFieldAccessor(String field, TypeInformation typeInfo) { 21 | this.field = field; 22 | this.fieldType = ((RowTypeInfo) typeInfo).getTypeAt(field); 23 | } 24 | 25 | public RowFieldAccessor(int pos, TypeInformation typeInfo) { 26 | this.pos = pos; 27 | this.fieldType = ((RowTypeInfo) typeInfo).getTypeAt(pos); 28 | } 29 | 30 | @Override 31 | public F get(R record) { 32 | return StringUtils.isNotEmpty(field) ? (F)record.getField(field) : (F)record.getField(pos); 33 | } 34 | 35 | @Override 36 | public R set(R record, F fieldValue) { 37 | if(StringUtils.isNotEmpty(field)){ 38 | record.setField(field, fieldValue); 39 | }else{ 40 | record.setField(pos, fieldValue); 41 | } 42 | return record; 43 | } 44 | 45 | 46 | 47 | } 48 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/writer/PrintItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.flink.writer; 2 | 3 | import com.leonside.dataroad.common.spi.ItemWriter; 4 | import com.leonside.dataroad.core.component.ComponentNameSupport; 5 | import com.leonside.dataroad.flink.context.FlinkExecuteContext; 6 | import org.apache.flink.streaming.api.datastream.DataStream; 7 | import org.apache.flink.types.Row; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | 13 | public class PrintItemWriter extends ComponentNameSupport implements ItemWriter> { 14 | 15 | @Override 16 | public void write(FlinkExecuteContext executeContext, DataStream items) { 17 | items.print(); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/java/com/leonside/dataroad/flink/writer/outputformat/GenericRichOutputFormatBuilder.java: -------------------------------------------------------------------------------- 1 | 2 | package com.leonside.dataroad.flink.writer.outputformat; 3 | 4 | import com.leonside.dataroad.common.constant.JobCommonConstant; 5 | import com.leonside.dataroad.common.context.RestoreConfig; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | /** 10 | * The builder of RichOutputFormat 11 | * 12 | */ 13 | public abstract class GenericRichOutputFormatBuilder { 14 | protected final Logger LOG = LoggerFactory.getLogger(getClass()); 15 | 16 | protected T format; 17 | 18 | public R setRestore(RestoreConfig restore){ 19 | format.setRestoreConfig(restore); 20 | return (R)this; 21 | } 22 | 23 | public R setErrors(Integer errors) { 24 | format.errors = errors; 25 | return (R)this; 26 | } 27 | 28 | public R setErrorRatio(Double errorRatio) { 29 | format.errorRatio = errorRatio; 30 | return (R)this; 31 | } 32 | 33 | public R setMonitorUrls(String monitorUrl) { 34 | format.monitorUrl = monitorUrl; 35 | return (R)this; 36 | } 37 | 38 | public R setBatchInterval(int batchInterval) { 39 | format.batchInterval = batchInterval; 40 | return (R)this; 41 | } 42 | 43 | public R setRestoreConfig(RestoreConfig restoreConfig){ 44 | format.restoreConfig = restoreConfig; 45 | return (R)this; 46 | } 47 | 48 | /** 49 | * Check the value of parameters 50 | */ 51 | protected abstract void checkFormat(); 52 | 53 | public GenericRichOutputFormat finish() { 54 | checkFormat(); 55 | 56 | /** 57 | * 200000条限制的原因： 58 | * 按照目前的使用情况以及部署配置，假设写入字段数量平均为50个，一个单slot的TaskManager内存为1G， 59 | * 在不考虑各插件批量写入对内存特殊要求并且只考虑插件缓存这么多条数据的情况下，batchInterval为400000条时出现fullGC， 60 | * 为了避免fullGC以及OOM，并且保证batchInterval有足够的配置空间，取最大值的一半200000。 61 | */ 62 | if (this.format.getBatchInterval() > JobCommonConstant.MAX_BATCH_SIZE) { 63 | throw new IllegalArgumentException("批量写入条数必须小于[200000]条"); 64 | } 65 | 66 | return format; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemProcessor: -------------------------------------------------------------------------------- 1 | scriptTransformer=com.leonside.dataroad.flink.processor.filter.ScriptTransformerProcessor 2 | scriptFilter=com.leonside.dataroad.flink.processor.filter.ScriptFilterProcessor 3 | sqlTransformer=com.leonside.dataroad.flink.processor.sql.SqlTransformerProcessor -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemUnionProcessor: -------------------------------------------------------------------------------- 1 | unionProcessor=com.leonside.dataroad.flink.processor.union.GenericItemUnionProcessor -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemWriter: -------------------------------------------------------------------------------- 1 | printWriter=com.leonside.dataroad.flink.writer.PrintItemWriter -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.core.spi.ItemAggregationProcessor: -------------------------------------------------------------------------------- 1 | countWindowAgg=com.leonside.dataroad.flink.processor.aggeration.CountWindowAggerationProcessor 2 | slidingWindowAgg=com.leonside.dataroad.flink.processor.aggeration.SlidingWindowAggerationProcessor 3 | tumblingWindowAgg=com.leonside.dataroad.flink.processor.aggeration.TumblingWindowAggerationProcessor -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.core.spi.ItemDeciderProcessor: -------------------------------------------------------------------------------- 1 | flinkOutputTagDeciderProcessor=com.leonside.dataroad.flink.processor.decider.OutputTagDeciderProcessor -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.core.spi.ItemLookupProcessor: -------------------------------------------------------------------------------- 1 | directLookup=com.leonside.dataroad.flink.processor.lookup.DirectAllLookupProcessor -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.core.spi.JobEngineProvider: -------------------------------------------------------------------------------- 1 | flinkJobProvider=com.leonside.dataroad.flink.FlinkJobProvider -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.core.spi.JobExecutionDecider: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.core.spi.JobExecutionDecider -------------------------------------------------------------------------------- /dataroad-engine/dataroad-engine-flink/src/main/resources/META-INF/services/com.leonside.dataroad.core.spi.JobPredicate: -------------------------------------------------------------------------------- 1 | expressionPredicate=com.leonside.dataroad.flink.predicate.ExpressionPredicate 2 | trueExpressionPredicate=com.leonside.dataroad.flink.predicate.TrueExpressionPredicate -------------------------------------------------------------------------------- /dataroad-engine/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | dataroad-all 7 | io.github.leonside 8 | 0.5 9 | 10 | 4.0.0 11 | 12 | dataroad-engine 13 | pom 14 | 15 | dataroad-engine-flink 16 | 17 | 18 | 19 | 8 20 | 8 21 | 22 | 23 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-es/src/main/java/com/leonside/dataroad/plugin/es/config/EsConstants.java: -------------------------------------------------------------------------------- 1 | 2 | package com.leonside.dataroad.plugin.es.config; 3 | 4 | /** 5 | * Configuration Keys for EsDataWriter and EsDataReader 6 | * 7 | */ 8 | public class EsConstants { 9 | 10 | public static final String KEY_TIMEOUT = "timeout"; 11 | 12 | public static final String KEY_PATH_PREFIX = "pathPrefix"; 13 | 14 | } 15 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-es/src/main/java/com/leonside/dataroad/plugin/es/config/EsReaderConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.es.config; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.common.config.ConfigKey; 5 | import com.leonside.dataroad.common.config.Validation; 6 | import lombok.Data; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * @author leon 12 | */ 13 | @Data 14 | public class EsReaderConfig extends BaseConfig { 15 | 16 | public EsReaderConfig(Map parameter){ 17 | super(parameter); 18 | } 19 | 20 | private String address; 21 | private String username; 22 | private String password; 23 | private Object query; 24 | 25 | private String index; 26 | private String indexType; 27 | private Integer batchSize; 28 | 29 | private Integer timeout; 30 | private String pathPrefix; 31 | 32 | @Override 33 | public Class bindConfigKey() { 34 | return EsReaderConfigKey.class; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-es/src/main/java/com/leonside/dataroad/plugin/es/config/EsWriterConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.es.config; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.common.config.ConfigKey; 5 | import com.leonside.dataroad.common.utils.ParameterUtils; 6 | import lombok.Data; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | /** 12 | * @author leon 13 | */ 14 | @Data 15 | public class EsWriterConfig extends BaseConfig { 16 | public EsWriterConfig(Map parameter) { 17 | super(parameter); 18 | } 19 | 20 | public String address; 21 | public String username; 22 | public String password; 23 | public String index; 24 | public String indexType; 25 | public int bulkAction; 26 | 27 | public Integer timeout; 28 | public String pathPrefix; 29 | 30 | 31 | @Override 32 | public Class bindConfigKey() { 33 | return EsWriterConfigKey.class; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-es/src/main/java/com/leonside/dataroad/plugin/es/reader/EsInputSplit.java: -------------------------------------------------------------------------------- 1 | 2 | package com.leonside.dataroad.plugin.es.reader; 3 | 4 | import org.apache.flink.core.io.InputSplit; 5 | 6 | /** 7 | * The Class describing each InputSplit of Elasticsearch 8 | * 9 | */ 10 | public class EsInputSplit implements InputSplit { 11 | 12 | private int from; 13 | private int size; 14 | 15 | public int getFrom() { 16 | return from; 17 | } 18 | 19 | public void setFrom(int from) { 20 | this.from = from; 21 | } 22 | 23 | public int getSize() { 24 | return size; 25 | } 26 | 27 | public void setSize(int size) { 28 | this.size = size; 29 | } 30 | 31 | public EsInputSplit(int from, int size) { 32 | this.from = from; 33 | this.size = size; 34 | } 35 | 36 | @Override 37 | public int getSplitNumber() { 38 | return 0; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-es/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemReader: -------------------------------------------------------------------------------- 1 | esReader=com.leonside.dataroad.plugin.es.reader.EsReader -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-es/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemWriter: -------------------------------------------------------------------------------- 1 | esWriter=com.leonside.dataroad.plugin.es.writer.EsWriter -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-hive/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | dataroad-plugin 7 | io.github.leonside 8 | 0.5 9 | 10 | 4.0.0 11 | 12 | dataroad-plugin-hive 13 | 14 | 15 | 8 16 | 8 17 | 18 | 19 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-jdbc/src/main/java/com/leonside/dataroad/plugin/jdbc/DataSource.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.jdbc; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | * @author jiangbo 7 | */ 8 | public class DataSource implements Serializable { 9 | 10 | private String jdbcUrl; 11 | private String userName; 12 | private String password; 13 | private String table; 14 | private boolean isSplitByKey; 15 | private Object[] parameterValues; 16 | private boolean isFinished; 17 | 18 | public DataSource() { 19 | this.isSplitByKey = false; 20 | this.isFinished = false; 21 | } 22 | 23 | public String getJdbcUrl() { 24 | return jdbcUrl; 25 | } 26 | 27 | public void setJdbcUrl(String jdbcUrl) { 28 | this.jdbcUrl = jdbcUrl; 29 | } 30 | 31 | public String getUserName() { 32 | return userName; 33 | } 34 | 35 | public void setUserName(String userName) { 36 | this.userName = userName; 37 | } 38 | 39 | public String getPassword() { 40 | return password; 41 | } 42 | 43 | public void setPassword(String password) { 44 | this.password = password; 45 | } 46 | 47 | public String getTable() { 48 | return table; 49 | } 50 | 51 | public void setTable(String table) { 52 | this.table = table; 53 | } 54 | 55 | public boolean isSplitByKey() { 56 | return isSplitByKey; 57 | } 58 | 59 | public void setSplitByKey(boolean splitByKey) { 60 | isSplitByKey = splitByKey; 61 | } 62 | 63 | public Object[] getParameterValues() { 64 | return parameterValues; 65 | } 66 | 67 | public void setParameterValues(Object[] parameterValues) { 68 | this.parameterValues = parameterValues; 69 | } 70 | 71 | public boolean isFinished() { 72 | return isFinished; 73 | } 74 | 75 | public void setFinished(boolean finished) { 76 | isFinished = finished; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-jdbc/src/main/java/com/leonside/dataroad/plugin/jdbc/ParameterValuesProvider.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.jdbc; 2 | 3 | import java.io.Serializable; 4 | 5 | /** 6 | */ 7 | public interface ParameterValuesProvider { 8 | 9 | /** 10 | * 获取分片信息 11 | * TODO 优化这部分逻辑 12 | * 13 | * @return 分片信息 14 | */ 15 | Serializable[][] getParameterValues(); 16 | } 17 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-jdbc/src/main/java/com/leonside/dataroad/plugin/jdbc/lookup/config/JdbcLookupConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.jdbc.lookup.config; 2 | 3 | import com.leonside.dataroad.common.config.ConfigKey; 4 | import com.leonside.dataroad.common.exception.JobConfigException; 5 | import com.leonside.dataroad.common.config.Validation; 6 | import com.leonside.dataroad.flink.processor.lookup.config.BaseLookupConfig; 7 | import lombok.Data; 8 | import org.apache.commons.lang.StringUtils; 9 | 10 | import java.io.Serializable; 11 | import java.util.Map; 12 | import java.util.concurrent.ConcurrentHashMap; 13 | 14 | /** 15 | * @author leon 16 | */ 17 | @Data 18 | public class JdbcLookupConfig extends BaseLookupConfig { 19 | public JdbcLookupConfig(Map parameter) { 20 | super(parameter); 21 | } 22 | 23 | public String username; 24 | public String password; 25 | public String jdbcUrl; 26 | public int fetchSize; 27 | public String schema; 28 | public String table; 29 | public String where; 30 | public String customSql; 31 | public Map druidConf = new ConcurrentHashMap<>(); 32 | 33 | /** vertx pool size */ 34 | public int asyncPoolSize = 5; 35 | /** 失败重试次数 */ 36 | public int maxRetryTimes = 3; 37 | public int errorLogPrintNum = 3; 38 | public int dbConnPoolSize = 5; 39 | public int eventLoopPoolSize = 1; 40 | public int taskQueueSize = 100000; 41 | public int parallelism = 1; 42 | 43 | 44 | @Override 45 | public boolean validate() { 46 | if(StringUtils.isEmpty(table) && StringUtils.isEmpty(customSql)){ 47 | throw new JobConfigException("Table and Custom Sql configurations cannot be empty at the same time."); 48 | } 49 | 50 | return super.validate(); 51 | } 52 | 53 | @Override 54 | public Class bindConfigKey() { 55 | return JdbcLookupKey.class; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-jdbc/src/main/java/com/leonside/dataroad/plugin/jdbc/lookup/datasource/DruidDataSourceProvider.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.jdbc.lookup.datasource; 2 | 3 | import com.google.common.base.CaseFormat; 4 | import io.vertx.ext.jdbc.spi.DataSourceProvider; 5 | import com.alibaba.druid.pool.DruidDataSource; 6 | import io.vertx.core.json.JsonObject; 7 | import io.vertx.ext.jdbc.spi.DataSourceProvider; 8 | 9 | import javax.sql.DataSource; 10 | 11 | import java.util.Map; 12 | import java.util.Properties; 13 | 14 | public class DruidDataSourceProvider implements DataSourceProvider { 15 | 16 | @Override 17 | public DataSource getDataSource(JsonObject config) { 18 | DruidDataSource dataSource = new DruidDataSource(); 19 | Properties props = new Properties(); 20 | for (Map.Entry entry : config) { 21 | String key = entry.getKey(); 22 | if (!"provider_class".equals(key)) { 23 | String formattedName = CaseFormat.LOWER_HYPHEN.to(CaseFormat.LOWER_CAMEL, key); 24 | props.setProperty(formattedName, entry.getValue().toString()); 25 | } 26 | } 27 | dataSource.configFromPropety(props); 28 | return dataSource; 29 | } 30 | 31 | @Override 32 | public void close(DataSource dataSource) { 33 | if (dataSource instanceof DruidDataSource) { 34 | ((DruidDataSource) dataSource).close(); 35 | } 36 | } 37 | 38 | @Override 39 | public int maximumPoolSize(DataSource dataSource, JsonObject config) { 40 | if (dataSource instanceof DruidDataSource) { 41 | return ((DruidDataSource) dataSource).getMaxActive(); 42 | } 43 | return -1; 44 | } 45 | } -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-jdbc/src/main/java/com/leonside/dataroad/plugin/jdbc/reader/config/JdbcReaderConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.jdbc.reader.config; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.common.domain.MetaColumn; 5 | import com.leonside.dataroad.plugin.jdbc.reader.inputformat.IncrementConfig; 6 | import lombok.Data; 7 | 8 | import java.io.Serializable; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | /** 13 | * @author leon 14 | */ 15 | @Data 16 | public class JdbcReaderConfig extends BaseConfig { 17 | 18 | protected String username; 19 | protected String password; 20 | protected String jdbcUrl; 21 | 22 | protected String table; 23 | protected String where; 24 | protected String customSql; 25 | protected String orderByColumn; 26 | 27 | protected String splitKey; 28 | protected int fetchSize; 29 | protected int queryTimeOut; 30 | //手动设置 31 | protected List metaColumns; 32 | 33 | protected IncrementConfig incrementConfig; 34 | 35 | public JdbcReaderConfig(Map parameter) { 36 | super(parameter); 37 | } 38 | 39 | @Override 40 | public Class bindConfigKey() { 41 | return JdbcReaderConfigKey.class; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-jdbc/src/main/java/com/leonside/dataroad/plugin/jdbc/reader/inputformat/StringAccumulator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.leonside.dataroad.plugin.jdbc.reader.inputformat; 20 | 21 | import org.apache.flink.api.common.accumulators.Accumulator; 22 | 23 | /** 24 | * @author jiangbo 25 | * @explanation 26 | * @date 2018/12/21 27 | */ 28 | public class StringAccumulator implements Accumulator { 29 | 30 | private String localValue; 31 | 32 | @Override 33 | public void add(String value) { 34 | localValue = value; 35 | } 36 | 37 | @Override 38 | public String getLocalValue() { 39 | return localValue; 40 | } 41 | 42 | @Override 43 | public void resetLocal() { 44 | localValue = null; 45 | } 46 | 47 | @Override 48 | public Accumulator clone() { 49 | StringAccumulator stringAccumulator = new StringAccumulator(); 50 | stringAccumulator.add(localValue); 51 | return stringAccumulator; 52 | } 53 | 54 | @Override 55 | public void merge(Accumulator other) { 56 | // do nothing 57 | } 58 | 59 | @Override 60 | public String toString() { 61 | return "StringData " + this.localValue; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-jdbc/src/main/java/com/leonside/dataroad/plugin/jdbc/type/TypeConverterInterface.java: -------------------------------------------------------------------------------- 1 | 2 | package com.leonside.dataroad.plugin.jdbc.type; 3 | 4 | import java.io.Serializable; 5 | 6 | /** 7 | * Data type converter 8 | * 9 | */ 10 | public interface TypeConverterInterface extends Serializable { 11 | 12 | /** 13 | * 类型转换，将数据库数据某类型的对象转换为对应的Java基本数据对象实例 14 | * @param data 数据记录 15 | * @param typeName 数据类型 16 | * @return 17 | */ 18 | Object convert(Object data,String typeName); 19 | 20 | } 21 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-jdbc/src/main/java/com/leonside/dataroad/plugin/jdbc/writer/config/JdbcWriterConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.jdbc.writer.config; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.common.config.ConfigKey; 5 | import com.leonside.dataroad.common.enums.WriteMode; 6 | import lombok.Data; 7 | 8 | import java.io.Serializable; 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.Map; 12 | 13 | /** 14 | * @author leon 15 | */ 16 | @Data 17 | public class JdbcWriterConfig extends BaseConfig { 18 | 19 | protected String jdbcUrl; 20 | protected String username; 21 | protected String password; 22 | protected List column = new ArrayList<>(); 23 | protected List fullColumn = new ArrayList<>(); 24 | protected String table; 25 | protected List preSql; 26 | protected List postSql; 27 | protected int batchSize; 28 | protected Map> updateKey; 29 | 30 | public String writeMode = WriteMode.INSERT.name(); 31 | 32 | public JdbcWriterConfig(Map parameter) { 33 | super(parameter); 34 | } 35 | 36 | @Override 37 | public Class bindConfigKey() { 38 | return JdbcWriterConfigKey.class; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-kafka/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | dataroad-all 7 | io.github.leonside 8 | 0.5 9 | ../../pom.xml 10 | 11 | 4.0.0 12 | 13 | dataroad-plugin-kafka 14 | 15 | 16 | 8 17 | 8 18 | 19 | 20 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysql/src/main/java/com/leonside/dataroad/plugin/mysql/reader/MysqlJdbcReader.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.mysql.reader; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.plugin.jdbc.DatabaseDialect; 5 | import com.leonside.dataroad.plugin.jdbc.reader.GenericJdbcReader; 6 | import com.leonside.dataroad.plugin.jdbc.reader.inputformat.GenericJdbcInputFormatBuilder; 7 | import com.leonside.dataroad.plugin.mysql.MySqlDatabaseDialect; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | public class MysqlJdbcReader extends GenericJdbcReader { 13 | 14 | @Override 15 | protected GenericJdbcInputFormatBuilder getGenericJdbcInputFormatBuilder() { 16 | return new GenericJdbcInputFormatBuilder(new MysqlJdbcInputFormat()); 17 | } 18 | 19 | @Override 20 | protected DatabaseDialect obtainDatabaseDialect() { 21 | return new MySqlDatabaseDialect(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysql/src/main/java/com/leonside/dataroad/plugin/mysql/writer/MysqlJdbcOutputFormat.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.mysql.writer; 2 | 3 | import com.leonside.dataroad.plugin.jdbc.writer.outputformat.GenericJdbcOutputFormat; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public class MysqlJdbcOutputFormat extends GenericJdbcOutputFormat { 9 | } 10 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysql/src/main/java/com/leonside/dataroad/plugin/mysql/writer/MysqlJdbcWriter.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.mysql.writer; 2 | 3 | import com.leonside.dataroad.plugin.jdbc.DatabaseDialect; 4 | import com.leonside.dataroad.plugin.jdbc.writer.GenericJdbcWriter; 5 | import com.leonside.dataroad.plugin.jdbc.writer.outputformat.GenericJdbcOutputFormatBuilder; 6 | import com.leonside.dataroad.plugin.mysql.MySqlDatabaseDialect; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | public class MysqlJdbcWriter extends GenericJdbcWriter { 12 | @Override 13 | protected GenericJdbcOutputFormatBuilder getBuilder() { 14 | return new GenericJdbcOutputFormatBuilder(new MysqlJdbcOutputFormat()); 15 | } 16 | 17 | @Override 18 | protected DatabaseDialect obtainDatabaseDialect() { 19 | return new MySqlDatabaseDialect(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysql/src/main/java/com/leonside/dataroad/plugin/mysql/writer/MysqlStreamJdbcOutputFormat.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.mysql.writer; 2 | 3 | import com.leonside.dataroad.plugin.jdbc.writer.outputformat.GenericStreamJdbcOutputFormat; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public class MysqlStreamJdbcOutputFormat extends GenericStreamJdbcOutputFormat { 9 | } 10 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysql/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemReader: -------------------------------------------------------------------------------- 1 | mysqlReader=com.leonside.dataroad.plugin.mysql.reader.MysqlJdbcReader -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysql/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemWriter: -------------------------------------------------------------------------------- 1 | mysqlWriter=com.leonside.dataroad.plugin.mysql.writer.MysqlJdbcWriter -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysql/src/main/resources/META-INF/services/com.leonside.dataroad.core.spi.ItemLookupProcessor: -------------------------------------------------------------------------------- 1 | mysqlLookup=com.leonside.dataroad.plugin.mysql.lookup.MysqlLookupProcessor -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysqlstream/src/main/java/com/leonside/dataroad/plugin/mysqlstream/config/MysqlStreamReaderConfig.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.mysqlstream.config; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.common.config.ConfigKey; 5 | import com.leonside.dataroad.common.config.Validation; 6 | import lombok.Data; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * @author leon 12 | */ 13 | @Data 14 | public class MysqlStreamReaderConfig extends BaseConfig { 15 | 16 | protected String hostname; 17 | protected int port; 18 | protected String schema; 19 | 20 | protected String username; 21 | protected String password; 22 | protected String table; 23 | 24 | public MysqlStreamReaderConfig(Map parameter) { 25 | super(parameter); 26 | } 27 | 28 | @Override 29 | public Class bindConfigKey() { 30 | return MysqlStreamReaderConfigKey.class; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysqlstream/src/main/java/com/leonside/dataroad/plugin/mysqlstream/writer/MysqlStreamJdbcOutputFormat.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.mysqlstream.writer; 2 | 3 | 4 | import com.leonside.dataroad.plugin.jdbc.writer.outputformat.GenericStreamJdbcOutputFormat; 5 | 6 | /** 7 | * @author leon 8 | */ 9 | public class MysqlStreamJdbcOutputFormat extends GenericStreamJdbcOutputFormat { 10 | } 11 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysqlstream/src/main/java/com/leonside/dataroad/plugin/mysqlstream/writer/MysqlStreamJdbcWriter.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.mysqlstream.writer; 2 | 3 | import com.leonside.dataroad.common.enums.WriteMode; 4 | import com.leonside.dataroad.common.exception.JobConfigException; 5 | import com.leonside.dataroad.flink.context.FlinkExecuteContext; 6 | import com.leonside.dataroad.plugin.jdbc.DatabaseDialect; 7 | import com.leonside.dataroad.plugin.jdbc.writer.GenericJdbcWriter; 8 | import com.leonside.dataroad.plugin.jdbc.writer.outputformat.GenericJdbcOutputFormatBuilder; 9 | import com.leonside.dataroad.plugin.mysql.MySqlDatabaseDialect; 10 | 11 | import java.util.Map; 12 | 13 | /** 14 | * @author leon 15 | */ 16 | public class MysqlStreamJdbcWriter extends GenericJdbcWriter { 17 | @Override 18 | protected GenericJdbcOutputFormatBuilder getBuilder() { 19 | return new GenericJdbcOutputFormatBuilder(new MysqlStreamJdbcOutputFormat()) ; 20 | } 21 | 22 | @Override 23 | public void initialize(FlinkExecuteContext executeContext, Map parameter) { 24 | super.initialize(executeContext, parameter); 25 | if(mode != null && !mode.equalsIgnoreCase(WriteMode.STREAM.name())){ 26 | throw new JobConfigException("There is no need to set mode or set stream type for Mysql stream writer "); 27 | } 28 | mode = WriteMode.STREAM.name(); 29 | } 30 | 31 | @Override 32 | protected DatabaseDialect obtainDatabaseDialect() { 33 | return new MySqlDatabaseDialect(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysqlstream/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemReader: -------------------------------------------------------------------------------- 1 | mysqlStreamReader=com.leonside.dataroad.plugin.mysqlstream.reader.MysqlStreamJdbcReader -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-mysqlstream/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemWriter: -------------------------------------------------------------------------------- 1 | mysqlStreamWriter=com.leonside.dataroad.plugin.mysqlstream.writer.MysqlStreamJdbcWriter -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-oracle/src/main/java/com/leonside/dataroad/plugin/oracle/reader/OracleJdbcReader.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.oracle.reader; 2 | 3 | import com.leonside.dataroad.plugin.jdbc.DatabaseDialect; 4 | import com.leonside.dataroad.plugin.jdbc.reader.GenericJdbcReader; 5 | import com.leonside.dataroad.plugin.jdbc.reader.inputformat.GenericJdbcInputFormatBuilder; 6 | import com.leonside.dataroad.plugin.oracle.OracleDatabaseDialect; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | public class OracleJdbcReader extends GenericJdbcReader { 12 | @Override 13 | protected GenericJdbcInputFormatBuilder getGenericJdbcInputFormatBuilder() { 14 | return new GenericJdbcInputFormatBuilder(new OracleJdbcInputFormat()); 15 | } 16 | 17 | @Override 18 | protected DatabaseDialect obtainDatabaseDialect() { 19 | return new OracleDatabaseDialect(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-oracle/src/main/java/com/leonside/dataroad/plugin/oracle/writer/OracleJdbcWriter.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.oracle.writer; 2 | 3 | import com.leonside.dataroad.plugin.jdbc.DatabaseDialect; 4 | import com.leonside.dataroad.plugin.jdbc.writer.GenericJdbcWriter; 5 | import com.leonside.dataroad.plugin.jdbc.writer.outputformat.GenericJdbcOutputFormatBuilder; 6 | import com.leonside.dataroad.plugin.oracle.OracleDatabaseDialect; 7 | 8 | /** 9 | * @author leon 10 | */ 11 | public class OracleJdbcWriter extends GenericJdbcWriter { 12 | @Override 13 | protected GenericJdbcOutputFormatBuilder getBuilder() { 14 | return new GenericJdbcOutputFormatBuilder(new OracleJdbcOutputFormat()); 15 | } 16 | 17 | @Override 18 | protected DatabaseDialect obtainDatabaseDialect() { 19 | return new OracleDatabaseDialect(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-oracle/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemReader: -------------------------------------------------------------------------------- 1 | oracleReader=com.leonside.dataroad.plugin.oracle.reader.OracleJdbcReader 2 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-oracle/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemWriter: -------------------------------------------------------------------------------- 1 | oracleWriter=com.leonside.dataroad.plugin.oracle.writer.OracleJdbcWriter -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-postgresql/src/main/java/com/leonside/dataroad/plugin/postgresql/reader/PostgresqlJdbcInputFormat.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.postgresql.reader; 2 | 3 | import com.leonside.dataroad.plugin.jdbc.reader.inputformat.GenericJdbcInputFormat; 4 | import com.leonside.dataroad.plugin.jdbc.utils.DbUtil; 5 | import org.apache.commons.collections.CollectionUtils; 6 | import org.apache.flink.types.Row; 7 | 8 | import java.io.IOException; 9 | 10 | /** 11 | * @author leon 12 | */ 13 | public class PostgresqlJdbcInputFormat extends GenericJdbcInputFormat { 14 | 15 | @Override 16 | public Row doNextRecord(Row row) throws IOException { 17 | if (!hasNext) { 18 | return null; 19 | } 20 | row = Row.withNames(); 21 | 22 | try { 23 | for (int pos = 0; pos < row.getArity(); pos++) { 24 | Object obj = resultSet.getObject(pos + 1); 25 | if(obj != null) { 26 | if(CollectionUtils.isNotEmpty(columnTypeList)) { 27 | obj = typeConverter.convert(obj, columnTypeList.get(pos)); 28 | } 29 | obj = DbUtil.clobToString(obj); 30 | } 31 | 32 | row.setField(metaData.getColumnName(pos + 1), obj); 33 | } 34 | 35 | return super.doNextRecord(row); 36 | }catch (Exception e) { 37 | throw new IOException("Couldn't read data - " + e.getMessage(), e); 38 | } 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-postgresql/src/main/java/com/leonside/dataroad/plugin/postgresql/writer/PostgresqlJdbcWriter.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.plugin.postgresql.writer; 2 | 3 | import com.leonside.dataroad.common.config.BaseConfig; 4 | import com.leonside.dataroad.plugin.jdbc.DatabaseDialect; 5 | import com.leonside.dataroad.plugin.jdbc.writer.GenericJdbcWriter; 6 | import com.leonside.dataroad.plugin.jdbc.writer.outputformat.GenericJdbcOutputFormatBuilder; 7 | import com.leonside.dataroad.plugin.postgresql.PostgresqlDatabaseDialect; 8 | 9 | /** 10 | * @author leon 11 | */ 12 | public class PostgresqlJdbcWriter extends GenericJdbcWriter { 13 | @Override 14 | protected GenericJdbcOutputFormatBuilder getBuilder() { 15 | return new GenericJdbcOutputFormatBuilder(new PostgresqlJdbcOutputFormat()); 16 | } 17 | 18 | @Override 19 | protected DatabaseDialect obtainDatabaseDialect() { 20 | return new PostgresqlDatabaseDialect(); 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-postgresql/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemReader: -------------------------------------------------------------------------------- 1 | postgresqlReader=com.leonside.dataroad.plugin.postgresql.reader.PostgresqlJdbcReader 2 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-postgresql/src/main/resources/META-INF/services/com.leonside.dataroad.common.spi.ItemWriter: -------------------------------------------------------------------------------- 1 | postgresqlWriter=com.leonside.dataroad.plugin.postgresql.writer.PostgresqlJdbcWriter -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-redis/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | dataroad-plugin 7 | io.github.leonside 8 | 0.5 9 | 10 | 4.0.0 11 | 12 | dataroad-plugin-redis 13 | 14 | 15 | 8 16 | 8 17 | 18 | 19 | -------------------------------------------------------------------------------- /dataroad-plugin/dataroad-plugin-sqlserver/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | dataroad-plugin 7 | io.github.leonside 8 | 0.5 9 | 10 | 4.0.0 11 | 12 | dataroad-plugin-sqlserver 13 | 14 | 15 | 8 16 | 8 17 | 18 | 19 | -------------------------------------------------------------------------------- /dataroad-plugin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | dataroad-all 7 | io.github.leonside 8 | 0.5 9 | 10 | 4.0.0 11 | pom 12 | 13 | dataroad-plugin 14 | 15 | 16 | dataroad-plugin-jdbc 17 | dataroad-plugin-mysql 18 | dataroad-plugin-oracle 19 | dataroad-plugin-sqlserver 20 | dataroad-plugin-postgresql 21 | dataroad-plugin-mysqlstream 22 | dataroad-plugin-hive 23 | dataroad-plugin-redis 24 | dataroad-plugin-es 25 | 26 | 27 | 28 | 8 29 | 8 30 | 31 | 32 | -------------------------------------------------------------------------------- /dataroad/src/main/java/com/leonside/dataroad/DataroadEngine.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad; 2 | 3 | import com.leonside.dataroad.config.JobCreator; 4 | import com.leonside.dataroad.common.config.Options; 5 | import com.leonside.dataroad.config.job.JsonJobCreator; 6 | import com.leonside.dataroad.config.job.JsonJobSchemaParser; 7 | import com.leonside.dataroad.core.Job; 8 | import lombok.extern.slf4j.Slf4j; 9 | import org.apache.commons.collections.map.HashedMap; 10 | import org.apache.commons.lang.StringUtils; 11 | import org.apache.flink.api.java.utils.ParameterTool; 12 | 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | /** 18 | * @author leon 19 | */ 20 | @Slf4j 21 | public class DataroadEngine { 22 | 23 | public static void main(String[] args) throws Exception { 24 | 25 | ParameterTool parameters = ParameterTool.fromArgs(args); 26 | 27 | Options options = Options.of(parameters.toMap()); 28 | 29 | if(StringUtils.isEmpty(options.getConf())){ 30 | throw new UnsupportedOperationException("schema path can not be null!"); 31 | } 32 | 33 | log.info("Start job，schema path is [{}]...." , options.getConf() ); 34 | 35 | JobCreator jsonJobCreator = new JsonJobCreator(new JsonJobSchemaParser(), options); 36 | 37 | List jobs = jsonJobCreator.createJob(); 38 | 39 | jobs.forEach(job ->{ 40 | 41 | try { 42 | job.execute(); 43 | } catch (Exception e) { 44 | e.printStackTrace(); 45 | } 46 | 47 | }); 48 | 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /dataroad/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy ofGenericTopN the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | ################################################################################ 18 | 19 | log4j.rootLogger=INFO, console 20 | 21 | log4j.appender.console=org.apache.log4j.ConsoleAppender 22 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n 24 | -------------------------------------------------------------------------------- /dataroad/src/main/resources/mysql_filter_mysql.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "content": [ 4 | { 5 | "mysqlreader": { 6 | "type": "reader", 7 | "pluginName": "mysqlReader", 8 | "parameter": { 9 | "username": "duceap", 10 | "password": "123", 11 | "jdbcUrl": "jdbc:mysql://10.254.10.31:3306/duceap_job_demo?useunicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai", 12 | "table": "job_execution_log", 13 | "column": ["id","job_name","ip"], 14 | "customSql": "", 15 | "splitKey": "", 16 | "queryTimeOut": 1000, 17 | "requestAccumulatorInterval": 2 18 | } 19 | }, 20 | "myprocessor1": { 21 | "type": "processor", 22 | "pluginName": "filterProcessor", 23 | "dependencies": ["mysqlreader"], 24 | "parameter": { 25 | "expression": "row.getField('age')>=20" 26 | } 27 | }, 28 | "mywriter": { 29 | "type": "writer", 30 | "pluginName": "mysqlWriter", 31 | "parameter": { 32 | "username": "duceap", 33 | "password": "123", 34 | "jdbcUrl": "jdbc:mysql://10.254.10.31:3306/duceap_job_demo?useunicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai", 35 | "table": "job_execution_log2", 36 | "writeMode": "insert", 37 | "column": ["id","job_name","ip"], 38 | "batchSize": 1024 39 | } 40 | } 41 | } 42 | ], 43 | "setting":{ 44 | "name": "myJob" 45 | } 46 | } 47 | } 48 | 49 | -------------------------------------------------------------------------------- /dataroad/src/main/resources/mysql_filter_print_restore.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "content": [ 4 | { 5 | "mysqlreader": { 6 | "type": "reader", 7 | "pluginName": "mysqlReader", 8 | "parameter": { 9 | "username": "duceap", 10 | "password": "123", 11 | "jdbcUrl": "jdbc:mysql://10.254.10.31:3306/duceap_job_demo?useunicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai", 12 | "table": [ 13 | "student" 14 | ], 15 | "column": [ 16 | "id", 17 | "name", 18 | "sex", 19 | "age", 20 | "address", 21 | "idcard", 22 | "phone", 23 | "code", 24 | "create_time", 25 | "area_code" 26 | ], 27 | "customSql": "", 28 | "splitKey": "", 29 | "queryTimeOut": 1000, 30 | "requestAccumulatorInterval": 2 31 | } 32 | }, 33 | "myprocessor1": { 34 | "type": "processor", 35 | "pluginName": "filterProcessor", 36 | "dependencies": [ 37 | "mysqlreader" 38 | ], 39 | "parameter": { 40 | "expression": "row.getField('age')>=20" 41 | } 42 | }, 43 | "mywriter": { 44 | "type": "writer", 45 | "pluginName": "printWriter" 46 | } 47 | } 48 | ], 49 | "setting":{ 50 | "speed": { 51 | "channel": 4, 52 | "bytes": 0 53 | }, 54 | "errorLimit": { 55 | "record": 1 56 | }, 57 | "restore": { 58 | "maxRowNumForCheckpoint": 0, 59 | "isRestore": true, 60 | "restoreColumnName": "id", 61 | "restoreColumnIndex": 0 62 | }, 63 | "log" : { 64 | "isLogger": false, 65 | "level" : "debug", 66 | "path" : "", 67 | "pattern":"" 68 | } 69 | } 70 | } 71 | } 72 | 73 | -------------------------------------------------------------------------------- /dataroad/src/main/resources/mysqlstream_es.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "content": [ 4 | { 5 | "mysqlreader": { 6 | "type": "reader", 7 | "pluginName": "mysqlStreamReader", 8 | "parameter": { 9 | "username": "root", 10 | "password": "dragon@1qaz!QAZ", 11 | "hostname": "10.254.10.31", 12 | "port": 3306, 13 | "table": "student", 14 | "schema": "duceap_job_demo" 15 | } 16 | }, 17 | "mywriter": { 18 | "type": "writer", 19 | "pluginName": "esWriter", 20 | "parameter": { 21 | "address": "bigdata33:9200", 22 | "index": "student1", 23 | "username": "", 24 | "password": "", 25 | "type": "_doc", 26 | "bulkAction": 2, 27 | "timeout": 1000, 28 | "idColumn": [ 29 | { 30 | "name": "id", 31 | "type": "integer" 32 | } 33 | ], 34 | "column": [ 35 | {"name": "id","type": "int"}, 36 | {"name": "name","type": "string"}, 37 | {"name": "sex","type": "int"}, 38 | {"name": "age","type": "int"}, 39 | {"name": "address","type": "string"}, 40 | {"name": "idcard","type": "string"}, 41 | {"name": "phone","type": "string"}, 42 | {"name": "code","type": "int"}, 43 | {"name": "create_time","type": "timestamp"}, 44 | {"name": "area_code","type": "string"} 45 | ] 46 | } 47 | } 48 | } 49 | ], 50 | "setting":{ 51 | "monitorUrls": "", 52 | "speed": { 53 | "channel": 1, 54 | "bytes": 0 55 | }, 56 | "errorLimit": { 57 | "record": 1 58 | }, 59 | "restore": { 60 | "isRestore": true, 61 | "savepointPath": "file:///D:\\conf\\checkpoint", 62 | "savepointRestorePath": "D:\\conf\\checkpoint\\b75a0f3ef253b34d2f2e63c8ad3f7c04\\chk-13", 63 | "savepointInterval": 10000 64 | }, 65 | "log" : { 66 | "isLogger": false, 67 | "level" : "debug", 68 | "path" : "", 69 | "pattern":"" 70 | } 71 | } 72 | } 73 | } 74 | 75 | -------------------------------------------------------------------------------- /dataroad/src/main/resources/mysqlstream_mysqlstreamwriter_postsql_fromcheckpoint.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "content": [ 4 | { 5 | "mysqlreader": { 6 | "type": "reader", 7 | "pluginName": "mysqlStreamReader", 8 | "parameter": { 9 | "username": "root", 10 | "password": "dragon@1qaz!QAZ", 11 | "hostname": "10.254.10.31", 12 | "port": 3306, 13 | "table": "student", 14 | "schema": "duceap_job_demo" 15 | } 16 | }, 17 | "mywriter": { 18 | "type": "writer", 19 | "pluginName": "mysqlStreamWriter", 20 | "parameter": { 21 | "username": "duceap", 22 | "password": "123", 23 | "jdbcUrl": "jdbc:mysql://10.254.10.31:3306/duceap_job_demo?useunicode=true&characterEncoding=utf8&zeroDateTimeBehavior=convertToNull&serverTimezone=Asia/Shanghai", 24 | "table": "student1", 25 | "preSql": [], 26 | "postSql": [ 27 | "update student1 set code = '1000';" 28 | ], 29 | "batchSize": 1 30 | } 31 | } 32 | } 33 | ], 34 | "setting":{ 35 | "monitorUrls": "", 36 | "speed": { 37 | "channel": 1, 38 | "bytes": 0 39 | }, 40 | "errorLimit": { 41 | "record": 1 42 | }, 43 | "restore": { 44 | "isRestore": true, 45 | "savepointPath": "file:///D:\\conf\\checkpoint", 46 | "savepointRestorePath": "D:\\conf\\checkpoint\\b75a0f3ef253b34d2f2e63c8ad3f7c04\\chk-13", 47 | "savepointInterval": 10000 48 | }, 49 | "log" : { 50 | "isLogger": false, 51 | "level" : "debug", 52 | "path" : "", 53 | "pattern":"" 54 | } 55 | } 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /dataroad/src/test/java/com/leonside/dataroad/common/GroovyScriptTest.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common; 2 | 3 | import com.leonside.dataroad.common.script.ScriptEvaluator; 4 | import com.leonside.dataroad.common.script.ScriptEvaluatorFactory; 5 | import org.apache.flink.types.Row; 6 | 7 | /** 8 | * @author leon 9 | */ 10 | public class GroovyScriptTest { 11 | 12 | public static void main(String[] args) throws Exception { 13 | String exp = "row.getField('score')>=500 "; 14 | 15 | Row row = Row.withNames(); 16 | row.setField("name","zhangsan"); 17 | row.setField("score", 599); 18 | 19 | long l = System.currentTimeMillis(); 20 | 21 | ScriptEvaluator scriptEvalutor = ScriptEvaluatorFactory.createScriptEvalutor(ScriptEvaluatorFactory.ScriptEngine.groovy, exp, new String[]{"import org.apache.flink.types.Row;"}); 22 | for (int i = 0; i < 10000; i++) { 23 | 24 | Object evaluate = scriptEvalutor.evaluate(row, null); 25 | // System.out.println(evaluate); 26 | } 27 | // 1w耗时：1978 耗时：N久 28 | System.out.println("耗时："+ (System.currentTimeMillis() - l)); 29 | 30 | 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dataroad/src/test/java/com/leonside/dataroad/common/TestMain.java: -------------------------------------------------------------------------------- 1 | package com.leonside.dataroad.common; 2 | 3 | import org.apache.flink.types.Row; 4 | 5 | /** 6 | * @author leon 7 | */ 8 | public class TestMain { 9 | public static void main(String[] args) { 10 | Row row = Row.withNames(); 11 | row.setField("name1","zhangs"); 12 | row.setField("name2","lisi"); 13 | 14 | Row row2 = Row.withNames(); 15 | row2.setField("name3","zhangs2"); 16 | row2.setField("name1","lisi2"); 17 | 18 | Row join = row.copy(row2); 19 | System.out.println(join); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /doc/images/dashboard-commit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-commit.png -------------------------------------------------------------------------------- /doc/images/dashboard-create.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-create.png -------------------------------------------------------------------------------- /doc/images/dashboard-designer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-designer.png -------------------------------------------------------------------------------- /doc/images/dashboard-guide-commit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-guide-commit.png -------------------------------------------------------------------------------- /doc/images/dashboard-guide-designer-line.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-guide-designer-line.png -------------------------------------------------------------------------------- /doc/images/dashboard-guide-designer-node.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-guide-designer-node.png -------------------------------------------------------------------------------- /doc/images/dashboard-guide-designer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-guide-designer.png -------------------------------------------------------------------------------- /doc/images/dashboard-guide-download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-guide-download.png -------------------------------------------------------------------------------- /doc/images/dashboard-guide-json.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-guide-json.png -------------------------------------------------------------------------------- /doc/images/dashboard-guide-upload.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-guide-upload.png -------------------------------------------------------------------------------- /doc/images/dashboard-json.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dashboard-json.png -------------------------------------------------------------------------------- /doc/images/dataroad-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dataroad-arch.png -------------------------------------------------------------------------------- /doc/images/dataroad-designer-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/dataroad-designer-flow.png -------------------------------------------------------------------------------- /doc/images/designer-json.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/doc/images/designer-json.png -------------------------------------------------------------------------------- /doc/plugin/scripttrans.md: -------------------------------------------------------------------------------- 1 | ## Script Transformer脚本转换 2 | 3 | ### 一、插件名称 4 | 类型：**processor**
5 | 名称：**scriptTransformer** 6 | ### 二、参数说明
7 | 8 | - **language** 9 | - 描述：脚本语言类型，支持groovy,fel,bsh,javascript 10 | - 必选：否 11 | - 字段类型：String 12 | - 默认值：默认采用fel语言 13 | 14 | 15 | - **expression** 16 | - 描述：脚本语言表达，无需返回值。 17 | - 注意： 18 | - 可通过"row"变量获取到Row行记录集，例如判断如果性别为空则设置默认值为0： if(row.getField("sex") == null){row.setField("sex",0) ;}。其中sex对应配置的数据库column列名。 19 | - 脚本转换，通过表达式修改row值，无需返回值！ 20 | - 具体的表达式语法可参见各个脚本语言的编写规则。 21 | - 必选：是 22 | - 字段类型：String 23 | - 默认值：无 24 | 25 | 26 | ### 三、配置示例 27 | #### 1、groovy 28 | ```json 29 | { 30 | "type": "processor", 31 | "pluginName": "scriptFilter", 32 | "parameter": { 33 | "expression": "int sex = row.getField('sex'); if(sex==0){row.setField("sex_value", "男");}else{row.setField("sex_value", "女");} " , 34 | "language": "groovy" 35 | } 36 | } 37 | ``` 38 | 39 | 40 | #### 2、fel 41 | 42 | ```json 43 | { 44 | "type": "processor", 45 | "pluginName": "scriptFilter", 46 | "parameter": { 47 | "expression": "row.setField("area_code",row.getField("area_code").substring(0,4)) ", 48 | "language": "fel" 49 | } 50 | } 51 | ``` 52 | 53 | #### 3、javascript 54 | 55 | ```json 56 | { 57 | "type": "processor", 58 | "pluginName": "scriptFilter", 59 | "parameter": { 60 | "expression": " if(row.getField("sex") == null){row.setField("sex",0) ;} " , 61 | "language": "javascript" 62 | } 63 | } 64 | ``` 65 | 66 | #### 4、bsh 67 | ```json 68 | { 69 | "type": "processor", 70 | "pluginName": "scriptFilter", 71 | "parameter": { 72 | "expression": "row.setField("score",row.getField("score")+1); " , 73 | "language": "bsh" 74 | } 75 | } 76 | ``` -------------------------------------------------------------------------------- /doc/plugin/sqltrans.md: -------------------------------------------------------------------------------- 1 | ## SQL Transformer SQL转换 2 | 3 | ### 一、插件名称 4 | 类型：**processor**
5 | 名称：**sqlTransformer** 6 | ### 二、参数说明
7 | 8 | - **tableName** 9 | - 描述：临时表名，如:t1 10 | - 必选：是 11 | - 字段类型：String 12 | - 默认值：否 13 | 14 | 15 | - **sql** 16 | - 描述：转换的SQL语句，此处语法参照Flink SQL语法，例如：select * from t1 where score>=600 and area_code like '3501%' 17 | - 必选：是 18 | - 字段类型：String 19 | - 默认值：无 20 | 21 | 22 | ### 三、配置示例 23 | #### 1、groovy 24 | ```json 25 | { 26 | "type": "processor", 27 | "pluginName": "sqlTransformer", 28 | "parameter": { 29 | "sql": "select * from t1 where score>=600 and area_code like '3501%'" , 30 | "tabelName": "t1" 31 | } 32 | } 33 | ``` 34 | -------------------------------------------------------------------------------- /jars/bsh-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/jars/bsh-1.0.jar -------------------------------------------------------------------------------- /jars/install_jars.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | mvn install:install-file -DgroupId=bsh -DartifactId=bsh -Dversion=1.0 -Dpackaging=jar -Dfile=./bsh-1.0.jar 4 | 5 | ## oracle driver 6 | mvn install:install-file -DgroupId=com.oracle -DartifactId=ojdbc6 -Dversion=11.2.0.1.0 -Dpackaging=jar -Dfile=./ojdbc6-11.2.0.1.0.jar 7 | -------------------------------------------------------------------------------- /jars/ojdbc6-11.2.0.1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/leonside/dataroad-all/92364dfdc4f141a3d6e181186bedc09342a4b7df/jars/ojdbc6-11.2.0.1.0.jar --------------------------------------------------------------------------------