├── .github
└── ISSUE_TEMPLATE
│ ├── bug_report.md
│ ├── feature_request.md
│ └── question.md
├── .gitignore
├── LICENSE
├── README.md
├── bin
├── db
│ └── datax_web.sql
├── install.sh
├── start-all.sh
├── start.sh
├── stop-all.sh
└── stop.sh
├── datax-admin
├── pom.xml
└── src
│ ├── main
│ ├── assembly
│ │ └── deploy.xml
│ ├── bin
│ │ ├── configure.sh
│ │ ├── datax-admin.sh
│ │ └── env.properties
│ ├── java
│ │ └── com
│ │ │ └── wugui
│ │ │ └── datax
│ │ │ └── admin
│ │ │ ├── DataXAdminApplication.java
│ │ │ ├── config
│ │ │ ├── MybatisPlusConfig.java
│ │ │ ├── SecurityConfig.java
│ │ │ ├── ServiceModelToSwagger2MapperImpl.java
│ │ │ ├── SwaggerConfig.java
│ │ │ └── WebConfig.java
│ │ │ ├── controller
│ │ │ ├── BaseController.java
│ │ │ ├── BaseForm.java
│ │ │ ├── DataxJsonController.java
│ │ │ ├── IndexController.java
│ │ │ ├── JobApiController.java
│ │ │ ├── JobCodeController.java
│ │ │ ├── JobDatasourceController.java
│ │ │ ├── JobGroupController.java
│ │ │ ├── JobInfoController.java
│ │ │ ├── JobLogController.java
│ │ │ ├── JobProjectController.java
│ │ │ ├── JobRegistryController.java
│ │ │ ├── JobTemplateController.java
│ │ │ ├── MetadataController.java
│ │ │ └── UserController.java
│ │ │ ├── core
│ │ │ ├── conf
│ │ │ │ └── JobAdminConfig.java
│ │ │ ├── cron
│ │ │ │ └── CronExpression.java
│ │ │ ├── handler
│ │ │ │ ├── AESEncryptHandler.java
│ │ │ │ └── MybatisMetaObjectHandler.java
│ │ │ ├── kill
│ │ │ │ └── KillJob.java
│ │ │ ├── route
│ │ │ │ ├── ExecutorRouteStrategyEnum.java
│ │ │ │ ├── ExecutorRouter.java
│ │ │ │ └── strategy
│ │ │ │ │ ├── ExecutorRouteBusyover.java
│ │ │ │ │ ├── ExecutorRouteConsistentHash.java
│ │ │ │ │ ├── ExecutorRouteFailover.java
│ │ │ │ │ ├── ExecutorRouteFirst.java
│ │ │ │ │ ├── ExecutorRouteLFU.java
│ │ │ │ │ ├── ExecutorRouteLRU.java
│ │ │ │ │ ├── ExecutorRouteLast.java
│ │ │ │ │ ├── ExecutorRouteRandom.java
│ │ │ │ │ └── ExecutorRouteRound.java
│ │ │ ├── scheduler
│ │ │ │ └── JobScheduler.java
│ │ │ ├── thread
│ │ │ │ ├── JobFailMonitorHelper.java
│ │ │ │ ├── JobLogReportHelper.java
│ │ │ │ ├── JobRegistryMonitorHelper.java
│ │ │ │ ├── JobScheduleHelper.java
│ │ │ │ └── JobTriggerPoolHelper.java
│ │ │ ├── trigger
│ │ │ │ ├── JobTrigger.java
│ │ │ │ └── TriggerTypeEnum.java
│ │ │ └── util
│ │ │ │ ├── I18nUtil.java
│ │ │ │ ├── JacksonUtil.java
│ │ │ │ └── LocalCacheUtil.java
│ │ │ ├── dto
│ │ │ ├── DataXBatchJsonBuildDto.java
│ │ │ ├── DataXJsonBuildDto.java
│ │ │ ├── HbaseReaderDto.java
│ │ │ ├── HbaseWriterDto.java
│ │ │ ├── HiveReaderDto.java
│ │ │ ├── HiveWriterDto.java
│ │ │ ├── MongoDBReaderDto.java
│ │ │ ├── MongoDBWriterDto.java
│ │ │ ├── Range.java
│ │ │ ├── RdbmsReaderDto.java
│ │ │ ├── RdbmsWriterDto.java
│ │ │ ├── TaskScheduleDto.java
│ │ │ ├── TriggerJobDto.java
│ │ │ ├── UpsertInfo.java
│ │ │ └── VersionColumn.java
│ │ │ ├── entity
│ │ │ ├── JobDatasource.java
│ │ │ ├── JobGroup.java
│ │ │ ├── JobInfo.java
│ │ │ ├── JobLog.java
│ │ │ ├── JobLogGlue.java
│ │ │ ├── JobLogReport.java
│ │ │ ├── JobPermission.java
│ │ │ ├── JobProject.java
│ │ │ ├── JobRegistry.java
│ │ │ ├── JobRole.java
│ │ │ ├── JobTemplate.java
│ │ │ ├── JobUser.java
│ │ │ ├── JwtUser.java
│ │ │ └── LoginUser.java
│ │ │ ├── exception
│ │ │ ├── GlobalExceptionHandler.java
│ │ │ └── TokenIsExpiredException.java
│ │ │ ├── filter
│ │ │ ├── JWTAuthenticationFilter.java
│ │ │ └── JWTAuthorizationFilter.java
│ │ │ ├── mapper
│ │ │ ├── JobDatasourceMapper.java
│ │ │ ├── JobGroupMapper.java
│ │ │ ├── JobInfoMapper.java
│ │ │ ├── JobLogGlueMapper.java
│ │ │ ├── JobLogMapper.java
│ │ │ ├── JobLogReportMapper.java
│ │ │ ├── JobProjectMapper.java
│ │ │ ├── JobRegistryMapper.java
│ │ │ ├── JobTemplateMapper.java
│ │ │ ├── JobUserMapper.java
│ │ │ └── PermissionMapper.java
│ │ │ ├── service
│ │ │ ├── DatasourceQueryService.java
│ │ │ ├── DataxJsonService.java
│ │ │ ├── JobDatasourceService.java
│ │ │ ├── JobProjectService.java
│ │ │ ├── JobRegistryService.java
│ │ │ ├── JobService.java
│ │ │ ├── JobTemplateService.java
│ │ │ └── impl
│ │ │ │ ├── AdminBizImpl.java
│ │ │ │ ├── DatasourceQueryServiceImpl.java
│ │ │ │ ├── DataxJsonServiceImpl.java
│ │ │ │ ├── JobDatasourceServiceImpl.java
│ │ │ │ ├── JobProjectServiceImpl.java
│ │ │ │ ├── JobRegistryServiceImpl.java
│ │ │ │ ├── JobServiceImpl.java
│ │ │ │ ├── JobTemplateServiceImpl.java
│ │ │ │ └── UserDetailsServiceImpl.java
│ │ │ ├── tool
│ │ │ ├── database
│ │ │ │ ├── ColumnInfo.java
│ │ │ │ ├── DasColumn.java
│ │ │ │ └── TableInfo.java
│ │ │ ├── datax
│ │ │ │ ├── BaseDataxPlugin.java
│ │ │ │ ├── DataxJsonHelper.java
│ │ │ │ ├── DataxJsonInterface.java
│ │ │ │ ├── DataxPluginInterface.java
│ │ │ │ ├── reader
│ │ │ │ │ ├── BaseReaderPlugin.java
│ │ │ │ │ ├── ClickHouseReader.java
│ │ │ │ │ ├── DataxReaderInterface.java
│ │ │ │ │ ├── HBaseReader.java
│ │ │ │ │ ├── HiveReader.java
│ │ │ │ │ ├── MongoDBReader.java
│ │ │ │ │ ├── MysqlReader.java
│ │ │ │ │ ├── OracleReader.java
│ │ │ │ │ ├── PostgresqlReader.java
│ │ │ │ │ └── SqlServerReader.java
│ │ │ │ └── writer
│ │ │ │ │ ├── BaseWriterPlugin.java
│ │ │ │ │ ├── ClickHouseWriter.java
│ │ │ │ │ ├── DataxWriterInterface.java
│ │ │ │ │ ├── HBaseWriter.java
│ │ │ │ │ ├── HiveWriter.java
│ │ │ │ │ ├── MongoDBWriter.java
│ │ │ │ │ ├── MysqlWriter.java
│ │ │ │ │ ├── OraclelWriter.java
│ │ │ │ │ ├── PostgresqllWriter.java
│ │ │ │ │ └── SqlServerlWriter.java
│ │ │ ├── meta
│ │ │ │ ├── BaseDatabaseMeta.java
│ │ │ │ ├── ClickHouseDataBaseMeta.java
│ │ │ │ ├── DatabaseInterface.java
│ │ │ │ ├── DatabaseMetaFactory.java
│ │ │ │ ├── Hbase20xsqlMeta.java
│ │ │ │ ├── HiveDatabaseMeta.java
│ │ │ │ ├── MySQLDatabaseMeta.java
│ │ │ │ ├── OracleDatabaseMeta.java
│ │ │ │ ├── PostgresqlDatabaseMeta.java
│ │ │ │ └── SqlServerDatabaseMeta.java
│ │ │ ├── pojo
│ │ │ │ ├── DataxHbasePojo.java
│ │ │ │ ├── DataxHivePojo.java
│ │ │ │ ├── DataxMongoDBPojo.java
│ │ │ │ └── DataxRdbmsPojo.java
│ │ │ └── query
│ │ │ │ ├── BaseQueryTool.java
│ │ │ │ ├── ClickHouseQueryTool.java
│ │ │ │ ├── HBaseQueryTool.java
│ │ │ │ ├── Hbase20XsqlQueryTool.java
│ │ │ │ ├── HiveQueryTool.java
│ │ │ │ ├── MongoDBQueryTool.java
│ │ │ │ ├── MySQLQueryTool.java
│ │ │ │ ├── OracleQueryTool.java
│ │ │ │ ├── PostgresqlQueryTool.java
│ │ │ │ ├── QueryToolFactory.java
│ │ │ │ ├── QueryToolInterface.java
│ │ │ │ └── SqlServerQueryTool.java
│ │ │ └── util
│ │ │ ├── AESUtil.java
│ │ │ ├── DBUtilErrorCode.java
│ │ │ ├── DataXException.java
│ │ │ ├── DateFormatUtils.java
│ │ │ ├── ErrorCode.java
│ │ │ ├── JSONUtils.java
│ │ │ ├── JdbcConstants.java
│ │ │ ├── JdbcUtils.java
│ │ │ ├── JwtTokenUtils.java
│ │ │ ├── PageUtils.java
│ │ │ ├── RdbmsException.java
│ │ │ ├── ReflectionUtil.java
│ │ │ └── ServletUtils.java
│ ├── lib
│ │ ├── HiveJDBC41.jar
│ │ ├── ojdbc6-11.2.0.3.jar
│ │ └── sqljdbc4-4.0.jar
│ └── resources
│ │ ├── application.yml
│ │ ├── bootstrap.properties
│ │ ├── i18n
│ │ ├── message.properties
│ │ └── message_en.properties
│ │ ├── logback.xml
│ │ ├── mybatis-mapper
│ │ ├── JobDatasourceMapper.xml
│ │ ├── JobGroupMapper.xml
│ │ ├── JobInfoMapper.xml
│ │ ├── JobLogGlueMapper.xml
│ │ ├── JobLogMapper.xml
│ │ ├── JobLogReportMapper.xml
│ │ ├── JobProjectMapper.xml
│ │ ├── JobRegistryMapper.xml
│ │ ├── JobTemplateMapper.xml
│ │ ├── JobUserMapper.xml
│ │ └── PermissionDaoMapper.xml
│ │ └── static
│ │ ├── avatar.jpg
│ │ ├── favicon.ico
│ │ ├── index.html
│ │ └── static
│ │ ├── css
│ │ ├── app.1d80faa6.css
│ │ ├── chunk-03e96171.42d5ee4e.css
│ │ ├── chunk-35cb90ad.1ff60aed.css
│ │ ├── chunk-3a8a15ea.3dacfa09.css
│ │ ├── chunk-46fcab30.030f9169.css
│ │ ├── chunk-4ba76b80.53ac87fa.css
│ │ ├── chunk-4c5a09d8.19618a3b.css
│ │ ├── chunk-4d80a2a0.d08662c8.css
│ │ ├── chunk-52071f51.ebba880b.css
│ │ ├── chunk-5bfc3adc.70564041.css
│ │ ├── chunk-60797987.53ac87fa.css
│ │ ├── chunk-65c33404.c229d2c2.css
│ │ ├── chunk-69a205c2.2b021e6b.css
│ │ ├── chunk-6ace4dde.3c1adc17.css
│ │ ├── chunk-6d2adc59.28c6d689.css
│ │ ├── chunk-7d41a2d3.55a106d0.css
│ │ ├── chunk-7ff5eb73.3e37c774.css
│ │ ├── chunk-b192701e.53ac87fa.css
│ │ ├── chunk-cc2bc43e.53ac87fa.css
│ │ ├── chunk-commons.c2ac5852.css
│ │ ├── chunk-d7a3743e.c4bc57e7.css
│ │ └── chunk-libs.3dfb7769.css
│ │ ├── fonts
│ │ ├── element-icons.535877f5.woff
│ │ └── element-icons.732389de.ttf
│ │ ├── img
│ │ ├── 401.089007e7.gif
│ │ ├── 404.a57b6f31.png
│ │ ├── 404_cloud.0f4bc32b.png
│ │ ├── avatar.1f4ad1c3.jpg
│ │ └── jsoneditor-icons.8e7baace.svg
│ │ └── js
│ │ ├── app.b686a449.js
│ │ ├── chunk-03d38440.5ca4f893.js
│ │ ├── chunk-03e96171.346c650a.js
│ │ ├── chunk-2d0db496.ee27a71b.js
│ │ ├── chunk-2d230fe7.341b11ea.js
│ │ ├── chunk-35cb90ad.ada9722b.js
│ │ ├── chunk-3a8a15ea.2261c4cc.js
│ │ ├── chunk-46fcab30.6267de3e.js
│ │ ├── chunk-4ba76b80.25d2dcba.js
│ │ ├── chunk-4c5a09d8.07c39843.js
│ │ ├── chunk-4d80a2a0.a4f45bbe.js
│ │ ├── chunk-52071f51.0cb43a45.js
│ │ ├── chunk-541ff83b.0585aa55.js
│ │ ├── chunk-5bfc3adc.2ba5f95e.js
│ │ ├── chunk-5c8098c9.c654c73d.js
│ │ ├── chunk-60797987.7c6e2fb0.js
│ │ ├── chunk-65c33404.b8da711e.js
│ │ ├── chunk-69a205c2.ce29f459.js
│ │ ├── chunk-6ace4dde.056ce860.js
│ │ ├── chunk-6d2adc59.843a9ce1.js
│ │ ├── chunk-7d41a2d3.608575cc.js
│ │ ├── chunk-7ff5eb73.80589f68.js
│ │ ├── chunk-b192701e.5d33016a.js
│ │ ├── chunk-cc2bc43e.4be0dd82.js
│ │ ├── chunk-commons.81a9f846.js
│ │ ├── chunk-d7a3743e.554410e0.js
│ │ ├── chunk-elementUI.b47b6259.js
│ │ └── chunk-libs.1b4497f1.js
│ └── test
│ └── java
│ └── com
│ └── wugui
│ ├── admin
│ ├── controller
│ │ └── AbstractSpringMvcTest.java
│ ├── core
│ │ └── util
│ │ │ └── JacksonUtilTest.java
│ ├── dao
│ │ ├── JobGroupMapperTest.java
│ │ ├── JobInfoMapperTest.java
│ │ ├── JobLogGlueMapperTest.java
│ │ ├── JobLogMapperTest.java
│ │ └── JobRegistryMapperTest.java
│ └── util
│ │ ├── I18nUtilTest.java
│ │ └── MinStack.java
│ ├── adminbiz
│ └── AdminBizTest.java
│ ├── datax
│ └── admin
│ │ └── tool
│ │ ├── .DS_Store
│ │ ├── datax
│ │ ├── DataxJsonHelperTest.java
│ │ └── SpringELUtil.java
│ │ └── query
│ │ ├── Hbase11xsqlToolTest.java
│ │ ├── Hbase20xsqlQueryToolTest.java
│ │ ├── KerberosLogin.java
│ │ ├── MySQLQueryToolTest.java
│ │ ├── OracleQueryToolTest.java
│ │ ├── PostgresqlQueryToolTest.java
│ │ ├── QueryTool.java
│ │ └── SqlServerQueryToolTest.java
│ └── executor
│ └── ExecutorBizTest.java
├── datax-assembly
├── package.xml
└── pom.xml
├── datax-core
├── pom.xml
└── src
│ ├── main
│ └── java
│ │ └── com
│ │ └── wugui
│ │ └── datatx
│ │ └── core
│ │ ├── biz
│ │ ├── AdminBiz.java
│ │ ├── ExecutorBiz.java
│ │ ├── client
│ │ │ └── AdminBizClient.java
│ │ ├── impl
│ │ │ └── ExecutorBizImpl.java
│ │ └── model
│ │ │ ├── HandleCallbackParam.java
│ │ │ ├── HandleProcessCallbackParam.java
│ │ │ ├── LogResult.java
│ │ │ ├── RegistryParam.java
│ │ │ ├── ReturnT.java
│ │ │ └── TriggerParam.java
│ │ ├── enums
│ │ ├── ExecutorBlockStrategyEnum.java
│ │ ├── IncrementTypeEnum.java
│ │ └── RegistryConfig.java
│ │ ├── executor
│ │ ├── JobExecutor.java
│ │ └── impl
│ │ │ └── JobSpringExecutor.java
│ │ ├── glue
│ │ ├── GlueFactory.java
│ │ ├── GlueTypeEnum.java
│ │ └── impl
│ │ │ └── SpringGlueFactory.java
│ │ ├── handler
│ │ ├── IJobHandler.java
│ │ ├── annotation
│ │ │ └── JobHandler.java
│ │ └── impl
│ │ │ ├── GlueJobHandler.java
│ │ │ └── ScriptJobHandler.java
│ │ ├── log
│ │ ├── JobFileAppender.java
│ │ └── JobLogger.java
│ │ ├── thread
│ │ ├── ExecutorRegistryThread.java
│ │ ├── JobLogFileCleanThread.java
│ │ ├── JobThread.java
│ │ ├── ProcessCallbackThread.java
│ │ └── TriggerCallbackThread.java
│ │ └── util
│ │ ├── Constants.java
│ │ ├── DateUtil.java
│ │ ├── FileUtil.java
│ │ ├── JobRemotingUtil.java
│ │ ├── Kernel32.java
│ │ ├── OSUtils.java
│ │ ├── ProcessUtil.java
│ │ ├── ScriptUtil.java
│ │ └── ShardingUtil.java
│ └── test
│ └── java
│ └── com
│ └── wugui
│ └── datax
│ └── impl
│ └── ExecutorBizImplTest.java
├── datax-executor
├── pom.xml
└── src
│ ├── main
│ ├── assembly
│ │ └── deploy.xml
│ ├── bin
│ │ ├── configure.sh
│ │ ├── datax-executor.sh
│ │ └── env.properties
│ ├── java
│ │ └── com
│ │ │ └── wugui
│ │ │ └── datax
│ │ │ └── executor
│ │ │ ├── DataXExecutorApplication.java
│ │ │ ├── core
│ │ │ └── config
│ │ │ │ └── DataXConfig.java
│ │ │ ├── service
│ │ │ ├── command
│ │ │ │ └── BuildCommand.java
│ │ │ ├── jobhandler
│ │ │ │ ├── DataXConstant.java
│ │ │ │ ├── ExecutorJobHandler.java
│ │ │ │ └── KillJobHandler.java
│ │ │ └── logparse
│ │ │ │ ├── AnalysisStatistics.java
│ │ │ │ └── LogStatistics.java
│ │ │ └── util
│ │ │ └── SystemUtils.java
│ └── resources
│ │ ├── application.yml
│ │ └── logback.xml
│ └── test
│ └── java
│ └── com
│ └── wugui
│ └── datax
│ └── executor
│ └── test
│ └── DataXExecutorExampleBootApplicationTests.java
├── datax-rpc
├── pom.xml
└── src
│ ├── main
│ ├── java
│ │ └── com
│ │ │ └── wugui
│ │ │ └── datax
│ │ │ └── rpc
│ │ │ ├── old
│ │ │ ├── registry
│ │ │ │ └── impl
│ │ │ │ │ ├── ZkServiceRegistry.java
│ │ │ │ │ ├── test
│ │ │ │ │ ├── ServiceRegistryTest.java
│ │ │ │ │ └── XxlZkClientTest.java
│ │ │ │ │ └── util
│ │ │ │ │ └── XxlZkClient.java
│ │ │ ├── remoting
│ │ │ │ └── net
│ │ │ │ │ ├── NetEnum.java
│ │ │ │ │ └── impl
│ │ │ │ │ ├── jetty
│ │ │ │ │ ├── client
│ │ │ │ │ │ └── JettyClient.java
│ │ │ │ │ └── server
│ │ │ │ │ │ ├── JettyServer.java
│ │ │ │ │ │ └── JettyServerHandler.java
│ │ │ │ │ ├── mina
│ │ │ │ │ ├── client
│ │ │ │ │ │ ├── MinaClient.java
│ │ │ │ │ │ ├── MinaClientHandler.java
│ │ │ │ │ │ └── MinaConnectClient.java
│ │ │ │ │ ├── codec
│ │ │ │ │ │ ├── MinaDecoder.java
│ │ │ │ │ │ └── MinaEncoder.java
│ │ │ │ │ └── server
│ │ │ │ │ │ ├── MinaServer.java
│ │ │ │ │ │ └── MinaServerHandler.java
│ │ │ │ │ └── servlet
│ │ │ │ │ └── server
│ │ │ │ │ └── ServletServerHandler.java
│ │ │ └── serialize
│ │ │ │ └── impl
│ │ │ │ ├── JacksonSerializer.java
│ │ │ │ ├── KryoSerializer.java
│ │ │ │ └── ProtostuffSerializer.java
│ │ │ ├── registry
│ │ │ ├── ServiceRegistry.java
│ │ │ └── impl
│ │ │ │ ├── LocalServiceRegistry.java
│ │ │ │ └── XxlRegistryServiceRegistry.java
│ │ │ ├── remoting
│ │ │ ├── invoker
│ │ │ │ ├── XxlRpcInvokerFactory.java
│ │ │ │ ├── annotation
│ │ │ │ │ └── XxlRpcReference.java
│ │ │ │ ├── call
│ │ │ │ │ ├── CallType.java
│ │ │ │ │ ├── XxlRpcInvokeCallback.java
│ │ │ │ │ └── XxlRpcInvokeFuture.java
│ │ │ │ ├── generic
│ │ │ │ │ └── XxlRpcGenericService.java
│ │ │ │ ├── impl
│ │ │ │ │ └── XxlRpcSpringInvokerFactory.java
│ │ │ │ ├── reference
│ │ │ │ │ ├── XxlRpcReferenceBean.java
│ │ │ │ │ └── impl
│ │ │ │ │ │ └── XxlRpcSpringReferenceBean.java
│ │ │ │ └── route
│ │ │ │ │ ├── LoadBalance.java
│ │ │ │ │ ├── XxlRpcLoadBalance.java
│ │ │ │ │ └── impl
│ │ │ │ │ ├── XxlRpcLoadBalanceConsistentHashStrategy.java
│ │ │ │ │ ├── XxlRpcLoadBalanceLFUStrategy.java
│ │ │ │ │ ├── XxlRpcLoadBalanceLRUStrategy.java
│ │ │ │ │ ├── XxlRpcLoadBalanceRandomStrategy.java
│ │ │ │ │ └── XxlRpcLoadBalanceRoundStrategy.java
│ │ │ ├── net
│ │ │ │ ├── Client.java
│ │ │ │ ├── Server.java
│ │ │ │ ├── common
│ │ │ │ │ ├── ConnectClient.java
│ │ │ │ │ └── NettyConstant.java
│ │ │ │ ├── impl
│ │ │ │ │ ├── netty
│ │ │ │ │ │ ├── client
│ │ │ │ │ │ │ ├── NettyClient.java
│ │ │ │ │ │ │ ├── NettyClientHandler.java
│ │ │ │ │ │ │ └── NettyConnectClient.java
│ │ │ │ │ │ ├── codec
│ │ │ │ │ │ │ ├── NettyDecoder.java
│ │ │ │ │ │ │ └── NettyEncoder.java
│ │ │ │ │ │ └── server
│ │ │ │ │ │ │ ├── NettyServer.java
│ │ │ │ │ │ │ └── NettyServerHandler.java
│ │ │ │ │ └── netty_http
│ │ │ │ │ │ ├── client
│ │ │ │ │ │ ├── NettyHttpClient.java
│ │ │ │ │ │ ├── NettyHttpClientHandler.java
│ │ │ │ │ │ └── NettyHttpConnectClient.java
│ │ │ │ │ │ └── server
│ │ │ │ │ │ ├── NettyHttpServer.java
│ │ │ │ │ │ └── NettyHttpServerHandler.java
│ │ │ │ └── params
│ │ │ │ │ ├── BaseCallback.java
│ │ │ │ │ ├── Beat.java
│ │ │ │ │ ├── XxlRpcFutureResponse.java
│ │ │ │ │ ├── XxlRpcRequest.java
│ │ │ │ │ └── XxlRpcResponse.java
│ │ │ └── provider
│ │ │ │ ├── XxlRpcProviderFactory.java
│ │ │ │ ├── annotation
│ │ │ │ └── XxlRpcService.java
│ │ │ │ └── impl
│ │ │ │ └── XxlRpcSpringProviderFactory.java
│ │ │ ├── serialize
│ │ │ ├── Serializer.java
│ │ │ └── impl
│ │ │ │ ├── Hessian1Serializer.java
│ │ │ │ └── HessianSerializer.java
│ │ │ └── util
│ │ │ ├── ClassUtil.java
│ │ │ ├── IpUtil.java
│ │ │ ├── NetUtil.java
│ │ │ ├── ThreadPoolUtil.java
│ │ │ ├── ThrowableUtil.java
│ │ │ ├── XxlRpcException.java
│ │ │ └── json
│ │ │ ├── BasicJson.java
│ │ │ ├── BasicJsonReader.java
│ │ │ └── BasicJsonwriter.java
│ └── resources
│ │ └── log4j.properties
│ └── test
│ └── java
│ └── com
│ └── wugui
│ └── datax
│ └── rpc
│ └── test
│ ├── IpUtilTest.java
│ └── SerializerTest.java
├── doc
└── datax-web
│ ├── datax-python3
│ ├── datax.py
│ ├── dxprof.py
│ └── perftrace.py
│ ├── datax-web-deploy-V2.1.1.md
│ ├── datax-web-deploy.md
│ ├── idea-start-datax.md
│ ├── increment-desc.md
│ ├── mysql2hive.md
│ ├── partition-dynamic-param.md
│ ├── partition-synchronization.md
│ └── start-configuration.md
├── pom.xml
└── userGuid.md
/.github/ISSUE_TEMPLATE/bug_report.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: "\U0001F41B Bug Report"
3 | about: Create a report to help us improve
4 | title: "[BUG] bug title "
5 | labels: bug
6 | assignees: ''
7 |
8 | ---
9 |
10 | *For better global communication, please give priority to using English description, thx! *
11 |
12 | **Describe the bug**
13 | A clear and concise description of what the bug is.
14 |
15 | **To Reproduce**
16 | Steps to reproduce the behavior, for example:
17 | 1. Go to '...'
18 | 2. Click on '....'
19 | 3. Scroll down to '....'
20 | 4. See error
21 |
22 | **Expected behavior**
23 | A clear and concise description of what you expected to happen.
24 |
25 | **Screenshots**
26 | If applicable, add screenshots to help explain your problem.
27 |
28 |
29 | **Which version of DataX Web:**
30 | -[2.1.1]
31 |
32 | **Additional context**
33 | Add any other context about the problem here.
34 |
35 | **Requirement or improvement
36 | - Please describe about your requirements or improvement suggestions.
37 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/feature_request.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: "\U0001F680 Feature Request"
3 | about: Suggest an idea for this project
4 | title: "[Feature]"
5 | labels: new feature
6 | assignees: ''
7 |
8 | ---
9 |
10 | **Is your feature request related to a problem? Please describe.**
11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
12 |
13 | **Describe the solution you'd like**
14 | A clear and concise description of what you want to happen.
15 |
16 | **Describe alternatives you've considered**
17 | A clear and concise description of any alternative solutions or features you've considered.
18 |
19 | **Additional context**
20 | Add any other context or screenshots about the feature request here.
21 |
--------------------------------------------------------------------------------
/.github/ISSUE_TEMPLATE/question.md:
--------------------------------------------------------------------------------
1 | ---
2 | name: "\U0001F914 Question"
3 | about: have a question wanted to be help
4 | title: "[QUESTION] question title"
5 | labels: question
6 | assignees: ''
7 |
8 | ---
9 |
10 | *For better global communication, please give priority to using English description, thx! *
11 |
12 | **Describe the question**
13 | A clear and concise description of what the question is.
14 |
15 |
16 | **Which version of DataX Web:**
17 | -[2.1.1]
18 |
19 | **Additional context**
20 | Add any other context about the problem here.
21 |
22 | **Requirement or improvement
23 | - Please describe about your requirements or improvement suggestions.
24 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 |
2 | /datax-executor/target/
3 | /datax-executor/datax-executor.iml
4 | /datax-core/datax-core.iml
5 | /datax-core/target/
6 | /datax-admin/datax-admin.iml
7 | /datax-admin/target/
8 | /.idea/
9 | /datax-admin/.idea/
10 | /datax-registry/datax-registry.iml
11 | /datax-registry/target/
12 | /datax-rpc/datax-rpc.iml
13 | /datax-rpc/target/
14 | /datax-all.iml
15 | /logs/
16 | *-dev*
17 | /data/
18 | /build/
19 | /packages/
20 | /datax-assembly/target/
21 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 WeiYe
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/bin/start-all.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | START_MODULES=("datax-admin" "datax-executor" )
4 |
5 | function LOG(){
6 | currentTime=`date "+%Y-%m-%d %H:%M:%S.%3N"`
7 | echo -e "$currentTime [${1}] ($) $2" | tee -a ${SHELL_LOG}
8 | }
9 |
10 | abs_path(){
11 | SOURCE="${BASH_SOURCE[0]}"
12 | while [ -h "${SOURCE}" ]; do
13 | DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
14 | SOURCE="$(readlink "${SOURCE}")"
15 | [[ ${SOURCE} != /* ]] && SOURCE="${DIR}/${SOURCE}"
16 | done
17 | echo "$( cd -P "$( dirname "${SOURCE}" )" && pwd )"
18 | }
19 |
20 | BIN=`abs_path`
21 | SHELL_LOG="${BIN}/console.out"
22 |
23 | LOG INFO "\033[1m Try To Start Modules In Order \033[0m"
24 | for module in ${START_MODULES[@]}
25 | do
26 | ${BIN}/start.sh -m ${module}
27 | if [ $? != 0 ]; then
28 | LOG ERROR "\033[1m Start Modules [${module}] Failed! \033[0m"
29 | exit 1
30 | fi
31 | done
--------------------------------------------------------------------------------
/bin/stop-all.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 |
4 | STOP_MODULES=("datax-admin" "datax-executor" )
5 |
6 |
7 | function LOG(){
8 | currentTime=`date "+%Y-%m-%d %H:%M:%S.%3N"`
9 | echo -e "$currentTime [${1}] ($) $2" | tee -a ${SHELL_LOG}
10 | }
11 |
12 | abs_path(){
13 | SOURCE="${BASH_SOURCE[0]}"
14 | while [ -h "${SOURCE}" ]; do
15 | DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
16 | SOURCE="$(readlink "${SOURCE}")"
17 | [[ ${SOURCE} != /* ]] && SOURCE="${DIR}/${SOURCE}"
18 | done
19 | echo "$( cd -P "$( dirname "${SOURCE}" )" && pwd )"
20 | }
21 |
22 | BIN=`abs_path`
23 | SHELL_LOG="${BIN}/console.out"
24 |
25 | LOG INFO "\033[1m Try to Stop Modules In Order \033[0m"
26 | for module in ${STOP_MODULES[@]}
27 | do
28 | ${BIN}/stop.sh -m ${module}
29 | if [ $? != 0 ]; then
30 | LOG ERROR "\033[1m Stop Modules [${module}] Failed! \033[0m"
31 | exit 1
32 | fi
33 | done
--------------------------------------------------------------------------------
/datax-admin/src/main/bin/env.properties:
--------------------------------------------------------------------------------
1 | # environment variables
2 |
3 | #JAVA_HOME=""
4 |
5 | WEB_LOG_PATH=${BIN}/../logs
6 | WEB_CONF_PATH=${BIN}/../conf
7 |
8 | DATA_PATH=${BIN}/../data
9 | SERVER_PORT=9527
10 |
11 | #PID_FILE_PATH=${BIN}/dataxadmin.pid
12 |
13 |
14 | # mail account
15 | MAIL_USERNAME=""
16 | MAIL_PASSWORD=""
17 |
18 |
19 | #debug
20 | #REMOTE_DEBUG_SWITCH=true
21 | #REMOTE_DEBUG_PORT=7003
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/config/MybatisPlusConfig.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.config;
2 |
3 | import com.baomidou.mybatisplus.core.injector.DefaultSqlInjector;
4 | import com.baomidou.mybatisplus.core.injector.ISqlInjector;
5 | import com.baomidou.mybatisplus.extension.plugins.PaginationInterceptor;
6 | import org.mybatis.spring.annotation.MapperScan;
7 | import org.springframework.context.annotation.Bean;
8 | import org.springframework.context.annotation.Configuration;
9 | import org.springframework.transaction.annotation.EnableTransactionManagement;
10 |
11 | /**
12 | * MybatisPlus配置类 Spring boot方式
13 | *
14 | * @author huzekang
15 | */
16 | @EnableTransactionManagement
17 | @Configuration
18 | @MapperScan("com.wugui.datax.admin.mapper")
19 | public class MybatisPlusConfig {
20 |
21 | /**
22 | * 分页插件
23 | */
24 | @Bean
25 | public PaginationInterceptor paginationInterceptor() {
26 |
27 | PaginationInterceptor paginationInterceptor = new PaginationInterceptor();
28 | return paginationInterceptor.setOverflow(true);
29 | }
30 |
31 | /**
32 | * MyBatisPlus逻辑删除 ，需要在 yml 中配置开启
33 | * 3.0.7.1版本的LogicSqlInjector里面什么都没做只是 extends DefaultSqlInjector
34 | * 以后版本直接去的了LogicSqlInjector
35 | *
36 | * @return
37 | */
38 | @Bean
39 | public ISqlInjector sqlInjector() {
40 | return new DefaultSqlInjector();
41 | }
42 |
43 |
44 | }
45 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/config/SwaggerConfig.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.config;
2 |
3 | import com.github.xiaoymin.swaggerbootstrapui.annotations.EnableSwaggerBootstrapUI;
4 | import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
5 | import org.springframework.context.annotation.Bean;
6 | import org.springframework.context.annotation.Configuration;
7 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
8 | import springfox.documentation.builders.ApiInfoBuilder;
9 | import springfox.documentation.builders.PathSelectors;
10 | import springfox.documentation.builders.RequestHandlerSelectors;
11 | import springfox.documentation.service.ApiInfo;
12 | import springfox.documentation.spi.DocumentationType;
13 | import springfox.documentation.spring.web.plugins.Docket;
14 | import springfox.documentation.swagger2.annotations.EnableSwagger2;
15 |
16 | /**
17 | * Swagger2API文档的配置
18 | * Created by jwk on 2019/07/05.
19 | */
20 | @Configuration
21 | @EnableSwagger2
22 | @EnableSwaggerBootstrapUI
23 | @ConditionalOnWebApplication
24 | public class SwaggerConfig implements WebMvcConfigurer {
25 |
26 | @Bean
27 | public Docket createRestApi() {
28 | return new Docket(DocumentationType.SWAGGER_2).apiInfo(apiInfo()).select()
29 | .apis(RequestHandlerSelectors.basePackage("com.wugui.datax.admin.controller")).paths(PathSelectors.any())
30 | .build();
31 | }
32 |
33 | private ApiInfo apiInfo() {
34 | return new ApiInfoBuilder().title("DataX Web Api Docs").description("DataX Web Api Docs")
35 | .build();
36 | }
37 |
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/config/WebConfig.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.config;
2 |
3 | import org.springframework.context.annotation.Configuration;
4 | import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
5 | import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
6 |
7 | /**
8 | * 前端静态资源访问
9 | *
10 | * @program: com.wugui.datax-all
11 | * @author: huzekang
12 | * @create: 2019-06-17 10:40
13 | **/
14 | @Configuration
15 |
16 | public class WebConfig implements WebMvcConfigurer {
17 |
18 |
19 | @Override
20 | public void addResourceHandlers(ResourceHandlerRegistry registry) {
21 | registry.addResourceHandler("/index.html").addResourceLocations("classpath:/static/index.html");
22 | registry.addResourceHandler("/static/**").addResourceLocations("classpath:/static/static/");
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/controller/BaseController.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.controller;
2 |
3 |
4 | import com.baomidou.mybatisplus.extension.api.ApiController;
5 | import com.wugui.datax.admin.util.JwtTokenUtils;
6 |
7 | import javax.servlet.http.HttpServletRequest;
8 | import java.util.Enumeration;
9 |
10 | import static com.wugui.datatx.core.util.Constants.STRING_BLANK;
11 |
12 | /**
13 | * base controller
14 | */
15 | public class BaseController extends ApiController {
16 |
17 | public Integer getCurrentUserId(HttpServletRequest request) {
18 | Enumeration<String> auth = request.getHeaders(JwtTokenUtils.TOKEN_HEADER);
19 | String token = auth.nextElement().replace(JwtTokenUtils.TOKEN_PREFIX, STRING_BLANK);
20 | return JwtTokenUtils.getUserId(token);
21 | }
22 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/controller/IndexController.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.controller;
2 |
3 | import com.wugui.datatx.core.biz.model.ReturnT;
4 | import com.wugui.datax.admin.service.JobService;
5 | import io.swagger.annotations.Api;
6 | import io.swagger.annotations.ApiOperation;
7 | import org.springframework.beans.propertyeditors.CustomDateEditor;
8 | import org.springframework.web.bind.WebDataBinder;
9 | import org.springframework.web.bind.annotation.*;
10 |
11 | import javax.annotation.Resource;
12 | import java.text.SimpleDateFormat;
13 | import java.util.Date;
14 | import java.util.Map;
15 |
16 | /**
17 | * index controller
18 | *
19 | * @author jingwk 2019-12-22 16:13:16
20 | */
21 | @RestController
22 | @Api(tags = "首页接口")
23 | @RequestMapping("/api")
24 | public class IndexController {
25 |
26 | @Resource
27 | private JobService jobService;
28 |
29 |
30 | @GetMapping("/index")
31 | @ApiOperation("监控图")
32 | public ReturnT<Map<String, Object>> index() {
33 | return new ReturnT<>(jobService.dashboardInfo());
34 | }
35 |
36 | @PostMapping("/chartInfo")
37 | @ApiOperation("图表信息")
38 | public ReturnT<Map<String, Object>> chartInfo() {
39 | return jobService.chartInfo();
40 | }
41 |
42 | @InitBinder
43 | public void initBinder(WebDataBinder binder) {
44 | SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
45 | dateFormat.setLenient(false);
46 | binder.registerCustomEditor(Date.class, new CustomDateEditor(dateFormat, true));
47 | }
48 |
49 | }
50 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/core/handler/AESEncryptHandler.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.core.handler;
2 |
3 | import com.wugui.datax.admin.util.AESUtil;
4 | import org.apache.commons.lang3.StringUtils;
5 | import org.apache.ibatis.type.BaseTypeHandler;
6 | import org.apache.ibatis.type.JdbcType;
7 | import org.apache.ibatis.type.MappedTypes;
8 |
9 | import java.sql.CallableStatement;
10 | import java.sql.PreparedStatement;
11 | import java.sql.ResultSet;
12 | import java.sql.SQLException;
13 |
14 | /**
15 | * @author water
16 | * @date 20-03-17 下午5:38
17 | */
18 | @MappedTypes({String.class})
19 | public class AESEncryptHandler extends BaseTypeHandler<String> {
20 |
21 |
22 | @Override
23 | public void setNonNullParameter(PreparedStatement ps, int i, String parameter, JdbcType jdbcType) throws SQLException {
24 | if(StringUtils.isNotBlank(parameter)){
25 | ps.setString(i, AESUtil.encrypt(parameter));
26 | }else{
27 | ps.setString(i, null);
28 | }
29 | }
30 |
31 | @Override
32 | public String getNullableResult(ResultSet rs, String columnName) throws SQLException {
33 | String columnValue = rs.getString(columnName);
34 | return AESUtil.decrypt(columnValue);
35 | }
36 |
37 | @Override
38 | public String getNullableResult(ResultSet rs, int columnIndex) throws SQLException {
39 | String columnValue = rs.getString(columnIndex);
40 | return AESUtil.decrypt(columnValue);
41 | }
42 |
43 | @Override
44 | public String getNullableResult(CallableStatement cs, int columnIndex)
45 | throws SQLException {
46 | String columnValue = cs.getString(columnIndex);
47 | return AESUtil.decrypt(columnValue);
48 | }
49 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/core/handler/MybatisMetaObjectHandler.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.core.handler;
2 |
3 | import com.baomidou.mybatisplus.core.handlers.MetaObjectHandler;
4 | import lombok.extern.slf4j.Slf4j;
5 | import org.apache.ibatis.reflection.MetaObject;
6 | import org.springframework.security.core.context.SecurityContext;
7 | import org.springframework.security.core.context.SecurityContextHolder;
8 | import org.springframework.stereotype.Component;
9 |
10 | import java.util.Date;
11 |
12 | /**
13 | * 通用的字段填充，如createBy createDate这些字段的自动填充
14 | *
15 | * @author huzekang
16 | */
17 | @Component
18 | @Slf4j
19 | public class MybatisMetaObjectHandler implements MetaObjectHandler {
20 |
21 | @Override
22 | public void insertFill(MetaObject metaObject) {
23 | setFieldValByName("createDate", new Date(), metaObject);
24 | setFieldValByName("createBy", getCurrentUser(), metaObject);
25 | }
26 |
27 | @Override
28 | public void updateFill(MetaObject metaObject) {
29 | setFieldValByName("updateDate", new Date(), metaObject);
30 | setFieldValByName("updateBy", getCurrentUser(), metaObject);
31 | }
32 |
33 | private String getCurrentUser() {
34 | return SecurityContextHolder.getContext().getAuthentication().getPrincipal().toString();
35 | }
36 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/core/kill/KillJob.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.core.kill;
2 |
3 | import com.wugui.datatx.core.biz.model.ReturnT;
4 | import com.wugui.datatx.core.biz.model.TriggerParam;
5 | import com.wugui.datatx.core.enums.ExecutorBlockStrategyEnum;
6 | import com.wugui.datatx.core.glue.GlueTypeEnum;
7 | import com.wugui.datax.admin.core.trigger.JobTrigger;
8 |
9 | import java.util.Date;
10 |
11 | /**
12 | * datax-job trigger
13 | * Created by jingwk on 2019/12/15.
14 | */
15 | public class KillJob {
16 |
17 | /**
18 | * @param logId
19 | * @param address
20 | * @param processId
21 | */
22 | public static ReturnT<String> trigger(long logId, Date triggerTime, String address, String processId) {
23 | ReturnT<String> triggerResult;
24 | TriggerParam triggerParam = new TriggerParam();
25 | triggerParam.setJobId(-1);
26 | triggerParam.setExecutorHandler("killJobHandler");
27 | triggerParam.setProcessId(processId);
28 | triggerParam.setLogId(logId);
29 | triggerParam.setGlueType(GlueTypeEnum.BEAN.getDesc());
30 | triggerParam.setExecutorBlockStrategy(ExecutorBlockStrategyEnum.SERIAL_EXECUTION.getTitle());
31 | triggerParam.setLogDateTime(triggerTime.getTime());
32 | if (address != null) {
33 | triggerResult = JobTrigger.runExecutor(triggerParam, address);
34 | } else {
35 | triggerResult = new ReturnT<>(ReturnT.FAIL_CODE, null);
36 | }
37 | return triggerResult;
38 | }
39 |
40 | }
41 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/core/route/ExecutorRouter.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.core.route;
2 |
3 | import com.wugui.datatx.core.biz.model.ReturnT;
4 | import com.wugui.datatx.core.biz.model.TriggerParam;
5 | import org.slf4j.Logger;
6 | import org.slf4j.LoggerFactory;
7 |
8 | import java.util.List;
9 |
10 | /**
11 | * Created by xuxueli on 17/3/10.
12 | */
13 | public abstract class ExecutorRouter {
14 | protected static Logger logger = LoggerFactory.getLogger(ExecutorRouter.class);
15 |
16 | /**
17 | * route address
18 | *
19 | * @param addressList
20 | * @return ReturnT.content=address
21 | */
22 | public abstract ReturnT<String> route(TriggerParam triggerParam, List<String> addressList);
23 |
24 | }
25 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/core/route/strategy/ExecutorRouteFirst.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.core.route.strategy;
2 |
3 | import com.wugui.datatx.core.biz.model.ReturnT;
4 | import com.wugui.datatx.core.biz.model.TriggerParam;
5 | import com.wugui.datax.admin.core.route.ExecutorRouter;
6 |
7 | import java.util.List;
8 |
9 | /**
10 | * Created by xuxueli on 17/3/10.
11 | */
12 | public class ExecutorRouteFirst extends ExecutorRouter {
13 |
14 | @Override
15 | public ReturnT<String> route(TriggerParam triggerParam, List<String> addressList){
16 | return new ReturnT<String>(addressList.get(0));
17 | }
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/core/route/strategy/ExecutorRouteLast.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.core.route.strategy;
2 |
3 | import com.wugui.datatx.core.biz.model.ReturnT;
4 | import com.wugui.datatx.core.biz.model.TriggerParam;
5 | import com.wugui.datax.admin.core.route.ExecutorRouter;
6 |
7 | import java.util.List;
8 |
9 | /**
10 | * Created by xuxueli on 17/3/10.
11 | */
12 | public class ExecutorRouteLast extends ExecutorRouter {
13 |
14 | @Override
15 | public ReturnT<String> route(TriggerParam triggerParam, List<String> addressList) {
16 | return new ReturnT<String>(addressList.get(addressList.size()-1));
17 | }
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/core/route/strategy/ExecutorRouteRandom.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.core.route.strategy;
2 |
3 | import com.wugui.datatx.core.biz.model.ReturnT;
4 | import com.wugui.datatx.core.biz.model.TriggerParam;
5 | import com.wugui.datax.admin.core.route.ExecutorRouter;
6 |
7 | import java.util.List;
8 | import java.util.Random;
9 |
10 | /**
11 | * Created by xuxueli on 17/3/10.
12 | */
13 | public class ExecutorRouteRandom extends ExecutorRouter {
14 |
15 | private static Random localRandom = new Random();
16 |
17 | @Override
18 | public ReturnT<String> route(TriggerParam triggerParam, List<String> addressList) {
19 | String address = addressList.get(localRandom.nextInt(addressList.size()));
20 | return new ReturnT<String>(address);
21 | }
22 |
23 | }
24 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/core/route/strategy/ExecutorRouteRound.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.core.route.strategy;
2 |
3 | import com.wugui.datatx.core.biz.model.ReturnT;
4 | import com.wugui.datatx.core.biz.model.TriggerParam;
5 | import com.wugui.datax.admin.core.route.ExecutorRouter;
6 |
7 | import java.util.List;
8 | import java.util.Random;
9 | import java.util.concurrent.ConcurrentHashMap;
10 | import java.util.concurrent.ConcurrentMap;
11 |
12 | /**
13 | * Created by xuxueli on 17/3/10.
14 | */
15 | public class ExecutorRouteRound extends ExecutorRouter {
16 |
17 | private static ConcurrentMap<Integer, Integer> routeCountEachJob = new ConcurrentHashMap<Integer, Integer>();
18 | private static long CACHE_VALID_TIME = 0;
19 | private static int count(int jobId) {
20 | // cache clear
21 | if (System.currentTimeMillis() > CACHE_VALID_TIME) {
22 | routeCountEachJob.clear();
23 | CACHE_VALID_TIME = System.currentTimeMillis() + 1000*60*60*24;
24 | }
25 |
26 | // count++
27 | Integer count = routeCountEachJob.get(jobId);
28 | count = (count==null || count>1000000)?(new Random().nextInt(100)):++count; // 初始化时主动Random一次，缓解首次压力
29 | routeCountEachJob.put(jobId, count);
30 | return count;
31 | }
32 |
33 | @Override
34 | public ReturnT<String> route(TriggerParam triggerParam, List<String> addressList) {
35 | String address = addressList.get(count(triggerParam.getJobId())%addressList.size());
36 | return new ReturnT<String>(address);
37 | }
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/core/trigger/TriggerTypeEnum.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.core.trigger;
2 |
3 | import com.wugui.datax.admin.core.util.I18nUtil;
4 |
5 | /**
6 | * trigger type enum
7 | *
8 | * @author xuxueli 2018-09-16 04:56:41
9 | */
10 | public enum TriggerTypeEnum {
11 |
12 | MANUAL(I18nUtil.getString("jobconf_trigger_type_manual")),
13 | CRON(I18nUtil.getString("jobconf_trigger_type_cron")),
14 | RETRY(I18nUtil.getString("jobconf_trigger_type_retry")),
15 | PARENT(I18nUtil.getString("jobconf_trigger_type_parent")),
16 | API(I18nUtil.getString("jobconf_trigger_type_api"));
17 |
18 | private TriggerTypeEnum(String title){
19 | this.title = title;
20 | }
21 | private String title;
22 | public String getTitle() {
23 | return title;
24 | }
25 |
26 | }
27 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/DataXBatchJsonBuildDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 | import java.util.List;
7 |
8 | /**
9 | * 构建json dto
10 | *
11 | * @author jingwk
12 | * @ClassName DataXJsonDto
13 | * @Version 2.1.2
14 | * @since 2020/05/05 17:15
15 | */
16 | @Data
17 | public class DataXBatchJsonBuildDto implements Serializable {
18 |
19 | private Long readerDatasourceId;
20 |
21 | private List<String> readerTables;
22 |
23 | private Long writerDatasourceId;
24 |
25 | private List<String> writerTables;
26 |
27 | private int templateId;
28 |
29 | private RdbmsReaderDto rdbmsReader;
30 |
31 | private RdbmsWriterDto rdbmsWriter;
32 | }
33 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/DataXJsonBuildDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 | import java.util.List;
7 |
8 | /**
9 | * 构建json dto
10 | *
11 | * @author jingwk
12 | * @ClassName DataxJsonDto
13 | * @Version 2.1.1
14 | * @since 2020/03/14 07:15
15 | */
16 | @Data
17 | public class DataXJsonBuildDto implements Serializable {
18 |
19 | private Long readerDatasourceId;
20 |
21 | private List<String> readerTables;
22 |
23 | private List<String> readerColumns;
24 |
25 | private Long writerDatasourceId;
26 |
27 | private List<String> writerTables;
28 |
29 | private List<String> writerColumns;
30 |
31 | private HiveReaderDto hiveReader;
32 |
33 | private HiveWriterDto hiveWriter;
34 |
35 | private HbaseReaderDto hbaseReader;
36 |
37 | private HbaseWriterDto hbaseWriter;
38 |
39 | private RdbmsReaderDto rdbmsReader;
40 |
41 | private RdbmsWriterDto rdbmsWriter;
42 |
43 | private MongoDBReaderDto mongoDBReader;
44 |
45 | private MongoDBWriterDto mongoDBWriter;
46 | }
47 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/HbaseReaderDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 |
7 | @Data
8 | public class HbaseReaderDto implements Serializable {
9 |
10 | private String readerMaxVersion;
11 |
12 | private String readerMode;
13 |
14 | private Range readerRange;
15 |
16 | }
17 |
18 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/HbaseWriterDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 |
7 | @Data
8 | public class HbaseWriterDto implements Serializable {
9 |
10 | private String writeNullMode;
11 |
12 | private String writerMode;
13 |
14 | private String writerRowkeyColumn;
15 |
16 | private VersionColumn writerVersionColumn;
17 | }
18 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/HiveReaderDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 |
7 | /**
8 | * 构建hive reader dto
9 | *
10 | * @author jingwk
11 | * @ClassName hive reader
12 | * @Version 2.0
13 | * @since 2020/01/11 17:15
14 | */
15 | @Data
16 | public class HiveReaderDto implements Serializable {
17 |
18 | private String readerPath;
19 |
20 | private String readerDefaultFS;
21 |
22 | private String readerFileType;
23 |
24 | private String readerFieldDelimiter;
25 |
26 | private Boolean readerSkipHeader;
27 |
28 | }
29 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/HiveWriterDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 |
7 | /**
8 | * 构建hive write dto
9 | *
10 | * @author jingwk
11 | * @ClassName hive write dto
12 | * @Version 2.0
13 | * @since 2020/01/11 17:15
14 | */
15 | @Data
16 | public class HiveWriterDto implements Serializable {
17 |
18 | private String writerDefaultFS;
19 |
20 | private String writerFileType;
21 |
22 | private String writerPath;
23 |
24 | private String writerFileName;
25 |
26 | private String writeMode;
27 |
28 | private String writeFieldDelimiter;
29 | }
30 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/MongoDBReaderDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 |
7 | /**
8 | * 构建mongodb reader dto
9 | *
10 | * @author jingwk
11 | * @ClassName mongodb reader
12 | * @Version 2.1.1
13 | * @since 2020/03/14 07:15
14 | */
15 | @Data
16 | public class MongoDBReaderDto implements Serializable {
17 |
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/MongoDBWriterDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 |
7 | /**
8 | * 构建mongodb write dto
9 | *
10 | * @author jingwk
11 | * @ClassName mongodb write dto
12 | * @Version 2.1.1
13 | * @since 2020/03/14 07:15
14 | */
15 | @Data
16 | public class MongoDBWriterDto implements Serializable {
17 |
18 | private UpsertInfo upsertInfo;
19 |
20 | }
21 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/Range.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 |
7 | @Data
8 | public class Range implements Serializable {
9 |
10 | private String startRowkey;
11 |
12 | private String endRowkey;
13 |
14 | private Boolean isBinaryRowkey;
15 | }
16 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/RdbmsReaderDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 | import java.util.List;
7 |
8 | /**
9 | * 构建json dto
10 | *
11 | * @author jingwk
12 | * @ClassName RdbmsReaderDto
13 | * @Version 2.0
14 | * @since 2020/01/11 17:15
15 | */
16 | @Data
17 | public class RdbmsReaderDto implements Serializable {
18 |
19 | private String readerSplitPk;
20 |
21 | private String whereParams;
22 |
23 | private String querySql;
24 | }
25 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/RdbmsWriterDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 |
7 | /**
8 | * 构建json dto
9 | *
10 | * @author jingwk
11 | * @ClassName RdbmsWriteDto
12 | * @Version 2.0
13 | * @since 2020/01/11 17:15
14 | */
15 | @Data
16 | public class RdbmsWriterDto implements Serializable {
17 |
18 | private String preSql;
19 |
20 | private String postSql;
21 | }
22 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/TaskScheduleDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | public class TaskScheduleDto {
4 |
5 |
6 | /**
7 | * 所选作业类型:
8 | * 1 -> 每天
9 | * 2 -> 每月
10 | * 3 -> 每周
11 | */
12 | Integer jobType;
13 |
14 | /**
15 | * 一周的哪几天
16 | */
17 | Integer[] dayOfWeeks;
18 |
19 | /**
20 | * 一个月的哪几天
21 | */
22 | Integer[] dayOfMonths;
23 |
24 | /**
25 | * 秒
26 | */
27 | Integer second;
28 |
29 | /**
30 | * 分
31 | */
32 | Integer minute;
33 |
34 | /**
35 | * 时
36 | */
37 | Integer hour;
38 |
39 | public Integer getJobType() {
40 | return jobType;
41 | }
42 |
43 | public void setJobType(Integer jobType) {
44 | this.jobType = jobType;
45 | }
46 |
47 | public Integer[] getDayOfWeeks() {
48 | return dayOfWeeks;
49 | }
50 |
51 | public void setDayOfWeeks(Integer[] dayOfWeeks) {
52 | this.dayOfWeeks = dayOfWeeks;
53 | }
54 |
55 | public Integer[] getDayOfMonths() {
56 | return dayOfMonths;
57 | }
58 |
59 | public void setDayOfMonths(Integer[] dayOfMonths) {
60 | this.dayOfMonths = dayOfMonths;
61 | }
62 |
63 | public Integer getSecond() {
64 | return second;
65 | }
66 |
67 | public void setSecond(Integer second) {
68 | this.second = second;
69 | }
70 |
71 | public Integer getMinute() {
72 | return minute;
73 | }
74 |
75 | public void setMinute(Integer minute) {
76 | this.minute = minute;
77 | }
78 |
79 | public Integer getHour() {
80 | return hour;
81 | }
82 |
83 | public void setHour(Integer hour) {
84 | this.hour = hour;
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/TriggerJobDto.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | import java.io.Serializable;
6 |
7 | /**
8 | * 用于启动任务接收的实体
9 | *
10 | * @author jingwk
11 | * @ClassName TriggerJobDto
12 | * @Version 1.0
13 | * @since 2019/12/01 16:12
14 | */
15 | @Data
16 | public class TriggerJobDto implements Serializable {
17 |
18 | private String executorParam;
19 |
20 | private int jobId;
21 | }
22 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/UpsertInfo.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | /**
6 | * Created by mac on 2020/3/16.
7 | */
8 | @Data
9 | public class UpsertInfo {
10 | /**
11 | * 当设置为true时，表示针对相同的upsertKey做更新操作
12 | */
13 | private Boolean isUpsert;
14 | /**
15 | * upsertKey指定了没行记录的业务主键。用来做更新时使用。
16 | */
17 | private String upsertKey;
18 | }
19 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/dto/VersionColumn.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.dto;
2 |
3 | import lombok.Data;
4 |
5 | @Data
6 | public class VersionColumn {
7 |
8 | private Integer index;
9 |
10 | private String value;
11 | }
12 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/entity/JobLogGlue.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.entity;
2 |
3 | import io.swagger.annotations.ApiModelProperty;
4 | import lombok.Data;
5 |
6 | import java.util.Date;
7 |
8 | /**
9 | * xxl-job log for glue, used to track job code process
10 | *
11 | * @author xuxueli 2016-5-19 17:57:46
12 | */
13 | @Data
14 | public class JobLogGlue {
15 |
16 | private int id;
17 |
18 | @ApiModelProperty("任务主键ID")
19 | private int jobId;
20 |
21 | @ApiModelProperty("GLUE类型\t#com.xxl.job.core.glue.GlueTypeEnum")
22 | private String glueType;
23 |
24 | @ApiModelProperty("GLUE源代码")
25 | private String glueSource;
26 |
27 | @ApiModelProperty("GLUE备注")
28 | private String glueRemark;
29 |
30 | private Date addTime;
31 |
32 | private Date updateTime;
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/entity/JobLogReport.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.entity;
2 |
3 | import lombok.Data;
4 |
5 | import java.util.Date;
6 |
7 | @Data
8 | public class JobLogReport {
9 |
10 | private int id;
11 |
12 | private Date triggerDay;
13 |
14 | private int runningCount;
15 | private int sucCount;
16 | private int failCount;
17 | }
18 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/entity/JobPermission.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.entity;
2 |
3 | public class JobPermission {
4 |
5 | private int id;
6 | //权限名称
7 | private String name;
8 |
9 | //权限描述
10 | private String descritpion;
11 |
12 | //授权链接
13 | private String url;
14 |
15 | //父节点id
16 | private int pid;
17 |
18 | public int getId() {
19 | return id;
20 | }
21 |
22 | public void setId(int id) {
23 | this.id = id;
24 | }
25 |
26 | public String getName() {
27 | return name;
28 | }
29 |
30 | public void setName(String name) {
31 | this.name = name;
32 | }
33 |
34 | public String getDescritpion() {
35 | return descritpion;
36 | }
37 |
38 | public void setDescritpion(String descritpion) {
39 | this.descritpion = descritpion;
40 | }
41 |
42 | public String getUrl() {
43 | return url;
44 | }
45 |
46 | public void setUrl(String url) {
47 | this.url = url;
48 | }
49 |
50 | public int getPid() {
51 | return pid;
52 | }
53 |
54 | public void setPid(int pid) {
55 | this.pid = pid;
56 | }
57 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/entity/JobProject.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.entity;
2 |
3 | import com.baomidou.mybatisplus.annotation.TableField;
4 | import com.fasterxml.jackson.annotation.JsonFormat;
5 | import io.swagger.annotations.ApiModelProperty;
6 | import lombok.Data;
7 |
8 | import java.util.ArrayList;
9 | import java.util.Arrays;
10 | import java.util.Date;
11 | import java.util.List;
12 |
13 | /**
14 | * Created by jingwk on 2020/05/24
15 | */
16 | @Data
17 | public class JobProject {
18 |
19 | @ApiModelProperty("项目Id")
20 | private int id;
21 |
22 | @ApiModelProperty("项目名称")
23 | private String name;
24 |
25 | @ApiModelProperty("项目描述")
26 | private String description;
27 |
28 | @ApiModelProperty("用户Id")
29 | private int userId;
30 |
31 | @ApiModelProperty("标记")
32 | private Boolean flag;
33 |
34 | @ApiModelProperty("创建时间")
35 | @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
36 | private Date createTime;
37 |
38 | @ApiModelProperty("更新时间")
39 | private Date updateTime;
40 |
41 | @TableField(exist=false)
42 | private String userName;
43 |
44 | }
45 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/entity/JobRegistry.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.entity;
2 |
3 | import com.fasterxml.jackson.annotation.JsonFormat;
4 | import lombok.Data;
5 |
6 | import java.util.Date;
7 |
8 | /**
9 | * Created by xuxueli on 16/9/30.
10 | */
11 | @Data
12 | public class JobRegistry {
13 |
14 | private int id;
15 | private String registryGroup;
16 | private String registryKey;
17 | private String registryValue;
18 | private double cpuUsage;
19 | private double memoryUsage;
20 | private double loadAverage;
21 | @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
22 | private Date updateTime;
23 | }
24 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/entity/JobRole.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.entity;
2 |
3 | import io.swagger.annotations.ApiModelProperty;
4 |
5 | /**
6 | * @author xuxueli 2019-05-04 16:43:12
7 | */
8 | public class JobRole {
9 |
10 | private int id;
11 | @ApiModelProperty("账号")
12 | private String name;
13 |
14 | public int getId() {
15 | return id;
16 | }
17 |
18 | public void setId(int id) {
19 | this.id = id;
20 | }
21 |
22 |
23 | public String getName() {
24 | return name;
25 | }
26 |
27 | public void setName(String name) {
28 | this.name = name;
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/entity/LoginUser.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.entity;
2 |
3 | import lombok.Data;
4 |
5 | /**
6 | * Created by jingwk on 2019/11/17
7 | */
8 | @Data
9 | public class LoginUser {
10 |
11 | private String username;
12 | private String password;
13 | private Integer rememberMe;
14 |
15 | }
16 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/exception/GlobalExceptionHandler.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.exception;
2 |
3 | import com.baomidou.mybatisplus.extension.api.R;
4 | import lombok.extern.slf4j.Slf4j;
5 | import org.springframework.web.bind.annotation.ExceptionHandler;
6 | import org.springframework.web.bind.annotation.RestControllerAdvice;
7 |
8 | /**
9 | * Created by jwk on 2019/07/05.
10 | * 全局异常处理
11 | * @author Jing WenKai
12 | * @date 2019/07/05 11:57
13 | */
14 | @Slf4j
15 | @RestControllerAdvice
16 | public class GlobalExceptionHandler {
17 |
18 | @ExceptionHandler(Exception.class)
19 | public R handleException(Exception e){
20 | log.error("系统异常{0}",e);
21 | return R.failed(e.getMessage());
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/exception/TokenIsExpiredException.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.exception;
2 |
3 | /**
4 | * @description: 自定义异常
5 | * @author: jingwk
6 | * @date: 2019/11/17 17:21
7 | */
8 | public class TokenIsExpiredException extends Exception{
9 | public TokenIsExpiredException() {
10 | }
11 |
12 | public TokenIsExpiredException(String message) {
13 | super(message);
14 | }
15 |
16 | public TokenIsExpiredException(String message, Throwable cause) {
17 | super(message, cause);
18 | }
19 |
20 | public TokenIsExpiredException(Throwable cause) {
21 | super(cause);
22 | }
23 |
24 | public TokenIsExpiredException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
25 | super(message, cause, enableSuppression, writableStackTrace);
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/mapper/JobDatasourceMapper.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.mapper;
2 |
3 | import com.baomidou.mybatisplus.core.mapper.BaseMapper;
4 | import com.wugui.datax.admin.entity.JobDatasource;
5 | import org.apache.ibatis.annotations.Mapper;
6 |
7 | /**
8 | * jdbc数据源配置表数据库访问层
9 | *
10 | * @author zhouhongfa@gz-yibo.com
11 | * @version v1.0
12 | * @since 2019-07-30
13 | */
14 | @Mapper
15 | public interface JobDatasourceMapper extends BaseMapper<JobDatasource> {
16 | int update(JobDatasource datasource);
17 |
18 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/mapper/JobGroupMapper.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.mapper;
2 |
3 | import com.wugui.datax.admin.entity.JobGroup;
4 | import org.apache.ibatis.annotations.Mapper;
5 | import org.apache.ibatis.annotations.Param;
6 |
7 | import java.util.List;
8 |
9 | /**
10 | * Created by xuxueli on 16/9/30.
11 | */
12 | @Mapper
13 | public interface JobGroupMapper {
14 |
15 | List<JobGroup> findAll();
16 |
17 | List<JobGroup> find(@Param("appName") String appName,
18 | @Param("title") String title,
19 | @Param("addressList") String addressList);
20 |
21 | int save(JobGroup jobGroup);
22 | List<JobGroup> findByAddressType(@Param("addressType") int addressType);
23 |
24 | int update(JobGroup jobGroup);
25 |
26 | int remove(@Param("id") int id);
27 |
28 | JobGroup load(@Param("id") int id);
29 | }
30 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/mapper/JobLogGlueMapper.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.mapper;
2 |
3 | import com.wugui.datax.admin.entity.JobLogGlue;
4 | import org.apache.ibatis.annotations.Mapper;
5 | import org.apache.ibatis.annotations.Param;
6 |
7 | import java.util.List;
8 |
9 | /**
10 | * job log for glue
11 | *
12 | * @author xuxueli 2016-5-19 18:04:56
13 | */
14 | @Mapper
15 | public interface JobLogGlueMapper {
16 |
17 | int save(JobLogGlue jobLogGlue);
18 |
19 | List<JobLogGlue> findByJobId(@Param("jobId") int jobId);
20 |
21 | int removeOld(@Param("jobId") int jobId, @Param("limit") int limit);
22 |
23 | int deleteByJobId(@Param("jobId") int jobId);
24 |
25 | }
26 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/mapper/JobLogReportMapper.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.mapper;
2 |
3 | import com.wugui.datax.admin.entity.JobLogReport;
4 | import org.apache.ibatis.annotations.Mapper;
5 | import org.apache.ibatis.annotations.Param;
6 |
7 | import java.util.Date;
8 | import java.util.List;
9 |
10 | /**
11 | * job log
12 | *
13 | * @author xuxueli 2019-11-22
14 | */
15 | @Mapper
16 | public interface JobLogReportMapper {
17 |
18 | int save(JobLogReport xxlJobLogReport);
19 |
20 | int update(JobLogReport xxlJobLogReport);
21 |
22 | List<JobLogReport> queryLogReport(@Param("triggerDayFrom") Date triggerDayFrom,
23 | @Param("triggerDayTo") Date triggerDayTo);
24 |
25 | JobLogReport queryLogReportTotal();
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/mapper/JobProjectMapper.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.mapper;
2 |
3 | import com.baomidou.mybatisplus.core.mapper.BaseMapper;
4 | import com.baomidou.mybatisplus.core.metadata.IPage;
5 | import com.wugui.datax.admin.entity.JobDatasource;
6 | import com.wugui.datax.admin.entity.JobProject;
7 | import org.apache.ibatis.annotations.Mapper;
8 | import org.apache.ibatis.annotations.Param;
9 |
10 | /**
11 | * Project
12 | *
13 | * @author jingwk
14 | * @version v2.1.12
15 | * @since 2020-05-24
16 | */
17 | @Mapper
18 | public interface JobProjectMapper extends BaseMapper<JobProject> {
19 | /**
20 | * project page
21 | * @param page
22 | * @param searchName
23 | * @return
24 | */
25 | IPage<JobProject> getProjectListPaging(IPage<JobProject> page,
26 | @Param("searchName") String searchName);
27 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/mapper/JobTemplateMapper.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.mapper;
2 |
3 | import com.wugui.datax.admin.entity.JobTemplate;
4 | import org.apache.ibatis.annotations.Mapper;
5 | import org.apache.ibatis.annotations.Param;
6 |
7 | import java.util.List;
8 |
9 |
10 | /**
11 | * job info
12 | * @author xuxueli 2016-1-12 18:03:45
13 | */
14 | @Mapper
15 | public interface JobTemplateMapper {
16 |
17 | public List<JobTemplate> pageList(@Param("offset") int offset,
18 | @Param("pagesize") int pagesize,
19 | @Param("jobGroup") int jobGroup,
20 | @Param("jobDesc") String jobDesc,
21 | @Param("executorHandler") String executorHandler,
22 | @Param("userId") int userId,
23 | @Param("projectIds") Integer[] projectIds);
24 |
25 | public int pageListCount(@Param("offset") int offset,
26 | @Param("pagesize") int pagesize,
27 | @Param("jobGroup") int jobGroup,
28 | @Param("jobDesc") String jobDesc,
29 | @Param("executorHandler") String executorHandler,
30 | @Param("userId") int userId,
31 | @Param("projectIds") Integer[] projectIds);
32 |
33 | public int save(JobTemplate info);
34 |
35 | public JobTemplate loadById(@Param("id") int id);
36 |
37 | public int update(JobTemplate jobTemplate);
38 |
39 | public int delete(@Param("id") long id);
40 |
41 | }
42 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/mapper/JobUserMapper.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.mapper;
2 |
3 | import com.wugui.datax.admin.entity.JobUser;
4 | import org.apache.ibatis.annotations.Mapper;
5 | import org.apache.ibatis.annotations.Param;
6 | import org.springframework.stereotype.Repository;
7 |
8 | import java.util.List;
9 |
10 | /**
11 | * @author xuxueli 2019-05-04 16:44:59
12 | */
13 | @Mapper
14 | @Repository
15 | public interface JobUserMapper {
16 |
17 | List<JobUser> pageList(@Param("offset") int offset,
18 | @Param("pagesize") int pagesize,
19 | @Param("username") String username);
20 |
21 | List<JobUser> findAll(@Param("username") String username);
22 |
23 | int pageListCount(@Param("offset") int offset,
24 | @Param("pagesize") int pagesize,
25 | @Param("username") String username);
26 |
27 | JobUser loadByUserName(@Param("username") String username);
28 |
29 | JobUser getUserById(@Param("id") int id);
30 |
31 | List<JobUser> getUsersByIds(@Param("ids") String[] ids);
32 |
33 | int save(JobUser jobUser);
34 |
35 | int update(JobUser jobUser);
36 |
37 | int delete(@Param("id") int id);
38 |
39 |
40 |
41 | }
42 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/mapper/PermissionMapper.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.mapper;
2 |
3 | import com.wugui.datax.admin.entity.JobPermission;
4 | import org.apache.ibatis.annotations.Mapper;
5 | import org.springframework.stereotype.Repository;
6 |
7 | import java.util.List;
8 |
9 | @Mapper
10 | @Repository
11 | public interface PermissionMapper {
12 |
13 | List<JobPermission> findAll();
14 |
15 | List<JobPermission> findByAdminUserId(int userId);
16 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/service/DatasourceQueryService.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.service;
2 |
3 | import java.io.IOException;
4 | import java.sql.SQLException;
5 | import java.util.List;
6 |
7 | /**
8 | * 数据库查询服务
9 | *
10 | * @author zhouhongfa@gz-yibo.com
11 | * @ClassName JdbcDatasourceQueryService
12 | * @Version 1.0
13 | * @since 2019/7/31 20:50
14 | */
15 | public interface DatasourceQueryService {
16 |
17 | /**
18 | * 获取db列表
19 | * @param id
20 | * @return
21 | */
22 | List<String> getDBs(Long id) throws IOException;
23 |
24 | /**
25 | * 根据数据源表id查询出可用的表
26 | *
27 | * @param id
28 | * @return
29 | */
30 | List<String> getTables(Long id,String tableSchema) throws IOException;
31 |
32 | /**
33 | * 获取CollectionNames
34 | * @param dbName
35 | * @return
36 | */
37 | List<String> getCollectionNames(long id,String dbName) throws IOException;
38 |
39 | /**
40 | * 根据数据源id，表名查询出该表所有字段
41 | *
42 | * @param id
43 | * @return
44 | */
45 | List<String> getColumns(Long id, String tableName) throws IOException;
46 |
47 | /**
48 | * 根据 sql 语句获取字段
49 | *
50 | * @param datasourceId
51 | * @param querySql
52 | * @return
53 | */
54 | List<String> getColumnsByQuerySql(Long datasourceId, String querySql) throws SQLException;
55 |
56 | /**
57 | * 获取PG table schema
58 | * @param id
59 | * @return
60 | */
61 | List<String> getTableSchema(Long id);
62 | }
63 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/service/DataxJsonService.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.service;
2 |
3 | import com.wugui.datax.admin.dto.DataXJsonBuildDto;
4 |
5 | /**
6 | * com.wugui.datax json构建服务层接口
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @version 1.0
10 | * @since 2019/8/1
11 | */
12 | public interface DataxJsonService {
13 |
14 | /**
15 | * build datax json
16 | *
17 | * @param dto
18 | * @return
19 | */
20 | String buildJobJson(DataXJsonBuildDto dto);
21 | }
22 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/service/JobDatasourceService.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.service;
2 |
3 | import com.baomidou.mybatisplus.extension.service.IService;
4 | import com.wugui.datax.admin.entity.JobDatasource;
5 |
6 | import java.io.IOException;
7 | import java.util.List;
8 |
9 | /**
10 | * jdbc数据源配置表服务接口
11 | *
12 | * @author jingwk
13 | * @version v2.0
14 | * @since 2020-01-10
15 | */
16 | public interface JobDatasourceService extends IService<JobDatasource> {
17 | /**
18 | * 测试数据源
19 | * @param jdbcDatasource
20 | * @return
21 | */
22 | Boolean dataSourceTest(JobDatasource jdbcDatasource) throws IOException;
23 |
24 | /**
25 | *更新数据源信息
26 | * @param datasource
27 | * @return
28 | */
29 | int update(JobDatasource datasource);
30 |
31 | /**
32 | * 获取所有数据源
33 | * @return
34 | */
35 | List<JobDatasource> selectAllDatasource();
36 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/service/JobProjectService.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.service;
2 |
3 | import com.baomidou.mybatisplus.core.metadata.IPage;
4 | import com.baomidou.mybatisplus.extension.service.IService;
5 | import com.wugui.datax.admin.entity.JobDatasource;
6 | import com.wugui.datax.admin.entity.JobProject;
7 | import org.apache.ibatis.annotations.Param;
8 |
9 | import java.io.IOException;
10 |
11 | /**
12 | * Job project
13 | *
14 | * @author jingwk
15 | * @version v2.1.2
16 | * @since 2020-05-24
17 | */
18 | public interface JobProjectService extends IService<JobProject> {
19 |
20 | /**
21 | * project page
22 | * @param pageSize
23 | * @param pageNo
24 | * @param searchName
25 | * @return
26 | */
27 |
28 | IPage<JobProject> getProjectListPaging(Integer pageSize, Integer pageNo, String searchName);
29 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/service/JobRegistryService.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.service;
2 |
3 | import com.baomidou.mybatisplus.extension.service.IService;
4 | import com.wugui.datax.admin.entity.JobRegistry;
5 |
6 | /**
7 | * jdbc数据源配置表服务接口
8 | *
9 | * @author jingwk
10 | * @version v2.1.1
11 | * @since 2020-03-15
12 | */
13 | public interface JobRegistryService extends IService<JobRegistry> {
14 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/service/JobTemplateService.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.service;
2 |
3 |
4 | import com.wugui.datatx.core.biz.model.ReturnT;
5 | import com.wugui.datax.admin.entity.JobTemplate;
6 |
7 | import java.util.Map;
8 |
9 | /**
10 | * core job action for datax-web
11 | *
12 | * @author xuxueli 2016-5-28 15:30:33
13 | */
14 | public interface JobTemplateService {
15 |
16 | /**
17 | * page list
18 | *
19 | * @param start
20 | * @param length
21 | * @param jobGroup
22 | * @param jobDesc
23 | * @param executorHandler
24 | * @param userId
25 | * @return
26 | */
27 | Map<String, Object> pageList(int start, int length, int jobGroup, String jobDesc, String executorHandler, int userId,Integer[] projectIds);
28 | /**
29 | * add job
30 | *
31 | * @param jobTemplate
32 | * @return
33 | */
34 | ReturnT<String> add(JobTemplate jobTemplate);
35 |
36 | /**
37 | * update job
38 | *
39 | * @param jobTemplate
40 | * @return
41 | */
42 | ReturnT<String> update(JobTemplate jobTemplate);
43 |
44 | /**
45 | * remove job
46 | * *
47 | * @param id
48 | * @return
49 | */
50 | ReturnT<String> remove(int id);
51 | }
52 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/service/impl/JobProjectServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.service.impl;
2 |
3 | import com.baomidou.mybatisplus.core.metadata.IPage;
4 | import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
5 | import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
6 | import com.wugui.datax.admin.entity.JobProject;
7 | import com.wugui.datax.admin.mapper.JobProjectMapper;
8 | import com.wugui.datax.admin.service.JobProjectService;
9 | import org.springframework.beans.factory.annotation.Autowired;
10 | import org.springframework.stereotype.Service;
11 |
12 | /**
13 | * JobProjectServiceImpl
14 | * @author jingwk
15 | * @since 2019-05-30
16 | * @version v2.1.2
17 | */
18 | @Service("jobProjectService")
19 | public class JobProjectServiceImpl extends ServiceImpl<JobProjectMapper, JobProject> implements JobProjectService {
20 |
21 | @Autowired
22 | private JobProjectMapper jobProjectMapper;
23 |
24 | @Override
25 | public IPage<JobProject> getProjectListPaging(Integer pageSize, Integer pageNo, String searchName) {
26 | Page<JobProject> page = new Page(pageNo, pageSize);
27 | return jobProjectMapper.getProjectListPaging(page, searchName);
28 | }
29 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/service/impl/JobRegistryServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.service.impl;
2 |
3 | import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
4 | import com.wugui.datax.admin.entity.JobRegistry;
5 | import com.wugui.datax.admin.mapper.JobRegistryMapper;
6 | import com.wugui.datax.admin.service.JobRegistryService;
7 | import org.springframework.stereotype.Service;
8 |
9 | /**
10 | * JobRegistryServiceImpl
11 | * @author jingwk
12 | * @since 2019-03-15
13 | * @version v2.1.1
14 | */
15 | @Service("jobRegistryService")
16 | public class JobRegistryServiceImpl extends ServiceImpl<JobRegistryMapper, JobRegistry> implements JobRegistryService {
17 |
18 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/service/impl/UserDetailsServiceImpl.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.service.impl;
2 |
3 | import com.wugui.datax.admin.entity.JobUser;
4 | import com.wugui.datax.admin.entity.JwtUser;
5 | import com.wugui.datax.admin.mapper.JobUserMapper;
6 | import org.springframework.beans.factory.annotation.Autowired;
7 | import org.springframework.security.core.userdetails.UserDetails;
8 | import org.springframework.security.core.userdetails.UserDetailsService;
9 | import org.springframework.security.core.userdetails.UsernameNotFoundException;
10 | import org.springframework.stereotype.Service;
11 |
12 | /**
13 | * UserDetailsServiceImpl
14 | * @author jingwk
15 | * @since 2019-03-15
16 | * @version v2.1.1
17 | */
18 | @Service
19 | public class UserDetailsServiceImpl implements UserDetailsService {
20 |
21 | @Autowired
22 | private JobUserMapper jobUserMapper;
23 |
24 | @Override
25 | public UserDetails loadUserByUsername(String s) throws UsernameNotFoundException {
26 | JobUser user = jobUserMapper.loadByUserName(s);
27 | return new JwtUser(user);
28 | }
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/database/ColumnInfo.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.database;
2 |
3 | import lombok.Data;
4 |
5 | /**
6 | * 字段信息
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @version 1.0
10 | * @since 2019/7/30
11 | */
12 | @Data
13 | public class ColumnInfo {
14 | /**
15 | * 字段名称
16 | */
17 | private String name;
18 | /**
19 | * 注释
20 | */
21 | private String comment;
22 | /**
23 | * 字段类型
24 | */
25 | private String type;
26 |
27 | /**
28 | * 是否是主键列
29 | */
30 | private Boolean ifPrimaryKey;
31 | /**
32 | * 是否可为null 0 不可为空 1 可以为null
33 | */
34 | private int isnull;
35 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/database/DasColumn.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.database;
2 |
3 | import lombok.AllArgsConstructor;
4 | import lombok.Data;
5 | import lombok.NoArgsConstructor;
6 |
7 | /**
8 | * 原始jdbc字段对象
9 | *
10 | * @author zhouhongfa@gz-yibo.com
11 | * @ClassName DasColumn
12 | * @Version 1.0
13 | * @since 2019/7/17 16:29
14 | */
15 | @Data
16 | @AllArgsConstructor
17 | @NoArgsConstructor
18 | public class DasColumn {
19 |
20 | private String columnName;
21 |
22 | private String columnTypeName;
23 |
24 | private String columnClassName;
25 |
26 | private String columnComment;
27 | private int isNull;
28 | private boolean isprimaryKey;
29 | }
30 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/database/TableInfo.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.database;
2 |
3 | import lombok.Data;
4 |
5 | import java.util.List;
6 |
7 | /**
8 | * 表信息
9 | *
10 | * @author zhouhongfa@gz-yibo.com
11 | * @version 1.0
12 | * @since 2019/7/30
13 | */
14 | @Data
15 | public class TableInfo {
16 | /**
17 | * 表名
18 | */
19 | private String name;
20 |
21 | /**
22 | * 注释
23 | */
24 | private String comment;
25 | /**
26 | * 所有列
27 | */
28 | private List<ColumnInfo> columns;
29 | }
30 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/BaseDataxPlugin.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax;
2 |
3 | import org.slf4j.Logger;
4 | import org.slf4j.LoggerFactory;
5 |
6 | /**
7 | * 抽象实现类
8 | *
9 | * @author zhouhongfa@gz-yibo.com
10 | * @ClassName BaseDataxPlugin
11 | * @Version 1.0
12 | * @since 2019/7/31 9:45
13 | */
14 | public abstract class BaseDataxPlugin implements DataxPluginInterface {
15 |
16 | protected Logger logger = LoggerFactory.getLogger(this.getClass());
17 |
18 | }
19 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/DataxJsonInterface.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax;
2 |
3 | import java.util.Map;
4 |
5 | /**
6 | * 构建 com.wugui.datax json的基础接口
7 | *
8 | * @author jingwk
9 | * @ClassName DataxJsonHelper
10 | * @Version 2.1.1
11 | * @since 2020/03/14 12:24
12 | */
13 | public interface DataxJsonInterface {
14 |
15 | Map<String, Object> buildJob();
16 |
17 | Map<String, Object> buildSetting();
18 |
19 | Map<String, Object> buildContent();
20 |
21 | Map<String, Object> buildReader();
22 |
23 | Map<String, Object> buildHiveReader();
24 |
25 | Map<String, Object> buildHiveWriter();
26 |
27 | Map<String, Object> buildHBaseReader();
28 |
29 | Map<String, Object> buildHBaseWriter();
30 |
31 | Map<String, Object> buildMongoDBReader();
32 |
33 | Map<String, Object> buildMongoDBWriter();
34 |
35 | Map<String, Object> buildWriter();
36 | }
37 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/DataxPluginInterface.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax;
2 |
3 | import com.wugui.datax.admin.tool.pojo.DataxHbasePojo;
4 | import com.wugui.datax.admin.tool.pojo.DataxHivePojo;
5 | import com.wugui.datax.admin.tool.pojo.DataxMongoDBPojo;
6 | import com.wugui.datax.admin.tool.pojo.DataxRdbmsPojo;
7 |
8 | import java.util.Map;
9 |
10 | /**
11 | * 插件基础接口
12 | *
13 | * @author zhouhongfa@gz-yibo.com
14 | * @ClassName DataxPluginInterface
15 | * @Version 1.0
16 | * @since 2019/7/30 22:59
17 | */
18 | public interface DataxPluginInterface {
19 | /**
20 | * 获取reader插件名称
21 | *
22 | * @return
23 | */
24 | String getName();
25 |
26 | /**
27 | * 构建
28 | *
29 | * @return dataxPluginPojo
30 | */
31 | Map<String, Object> build(DataxRdbmsPojo dataxPluginPojo);
32 |
33 |
34 | /**
35 | * hive json构建
36 | * @param dataxHivePojo
37 | * @return
38 | */
39 | Map<String, Object> buildHive(DataxHivePojo dataxHivePojo);
40 |
41 | /**
42 | * hbase json构建
43 | * @param dataxHbasePojo
44 | * @return
45 | */
46 | Map<String, Object> buildHbase(DataxHbasePojo dataxHbasePojo);
47 |
48 | /**
49 | * mongodb json构建
50 | * @param dataxMongoDBPojo
51 | * @return
52 | */
53 | Map<String,Object> buildMongoDB(DataxMongoDBPojo dataxMongoDBPojo);
54 |
55 | /**
56 | * 获取示例
57 | *
58 | * @return
59 | */
60 | Map<String, Object> sample();
61 | }
62 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/reader/ClickHouseReader.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.reader;
2 |
3 | import java.util.Map;
4 |
5 | public class ClickHouseReader extends BaseReaderPlugin implements DataxReaderInterface {
6 | @Override
7 | public String getName() {
8 | return "clickhousereader";
9 | }
10 |
11 |
12 | @Override
13 | public Map<String, Object> sample() {
14 | return null;
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/reader/DataxReaderInterface.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.reader;
2 |
3 | import com.wugui.datax.admin.tool.datax.DataxPluginInterface;
4 |
5 | /**
6 | * 用于构建reader的接口
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @version 1.0
10 | * @since 2019/7/30
11 | */
12 | public interface DataxReaderInterface extends DataxPluginInterface {
13 |
14 |
15 | }
16 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/reader/HBaseReader.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.reader;
2 |
3 | import com.google.common.collect.Maps;
4 | import com.wugui.datax.admin.tool.pojo.DataxHbasePojo;
5 | import org.apache.commons.lang3.StringUtils;
6 |
7 | import java.util.Map;
8 |
9 | public class HBaseReader extends BaseReaderPlugin implements DataxReaderInterface {
10 | @Override
11 | public String getName() {
12 | return "hbase11xreader";
13 | }
14 |
15 | @Override
16 | public Map<String, Object> sample() {
17 | return null;
18 | }
19 |
20 | public Map<String, Object> buildHbase(DataxHbasePojo plugin) {
21 | //构建
22 | Map<String, Object> readerObj = Maps.newLinkedHashMap();
23 | readerObj.put("name", getName());
24 | Map<String, Object> parameterObj = Maps.newLinkedHashMap();
25 | Map<String, Object> confige = Maps.newLinkedHashMap();
26 | confige.put("hbase.zookeeper.quorum",plugin.getReaderHbaseConfig());
27 | parameterObj.put("hbaseConfig", confige);
28 | parameterObj.put("table", plugin.getReaderTable());
29 | parameterObj.put("mode", plugin.getReaderMode());
30 | parameterObj.put("column", plugin.getColumns());
31 | if(StringUtils.isNotBlank(plugin.getReaderRange().getStartRowkey()) && StringUtils.isNotBlank(plugin.getReaderRange().getEndRowkey())){
32 | parameterObj.put("range", plugin.getReaderRange());
33 | }
34 | parameterObj.put("maxVersion", plugin.getReaderMaxVersion());
35 | readerObj.put("parameter", parameterObj);
36 | return readerObj;
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/reader/HiveReader.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.reader;
2 |
3 | import com.google.common.collect.Maps;
4 | import com.wugui.datax.admin.tool.pojo.DataxHivePojo;
5 |
6 | import java.util.Map;
7 |
8 | /**
9 | * hive reader 构建类
10 | *
11 | * @author jingwk
12 | * @version 2.0
13 | * @since 2020/01/05
14 | */
15 | public class HiveReader extends BaseReaderPlugin implements DataxReaderInterface {
16 | @Override
17 | public String getName() {
18 | return "hdfsreader";
19 | }
20 |
21 | @Override
22 | public Map<String, Object> sample() {
23 | return null;
24 | }
25 |
26 |
27 | @Override
28 | public Map<String, Object> buildHive(DataxHivePojo plugin) {
29 | //构建
30 | Map<String, Object> readerObj = Maps.newLinkedHashMap();
31 | readerObj.put("name", getName());
32 | Map<String, Object> parameterObj = Maps.newLinkedHashMap();
33 | parameterObj.put("path", plugin.getReaderPath());
34 | parameterObj.put("defaultFS", plugin.getReaderDefaultFS());
35 | parameterObj.put("fileType", plugin.getReaderFileType());
36 | parameterObj.put("fieldDelimiter", plugin.getReaderFieldDelimiter());
37 | parameterObj.put("skipHeader", plugin.getSkipHeader());
38 | parameterObj.put("column", plugin.getColumns());
39 | readerObj.put("parameter", parameterObj);
40 | return readerObj;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/reader/MysqlReader.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.reader;
2 |
3 |
4 | import java.util.Map;
5 |
6 | /**
7 | * mysql reader 构建类
8 | *
9 | * @author zhouhongfa@gz-yibo.com
10 | * @ClassName MysqlReader
11 | * @Version 1.0
12 | * @since 2019/7/30 23:07
13 | */
14 | public class MysqlReader extends BaseReaderPlugin implements DataxReaderInterface {
15 | @Override
16 | public String getName() {
17 | return "mysqlreader";
18 | }
19 |
20 |
21 | @Override
22 | public Map<String, Object> sample() {
23 | return null;
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/reader/OracleReader.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.reader;
2 |
3 | import com.wugui.datax.admin.tool.pojo.DataxMongoDBPojo;
4 |
5 | import java.util.Map;
6 |
7 | /**
8 | * oracle reader 构建类
9 | *
10 | * @author zhouhongfa@gz-yibo.com
11 | * @version 1.0
12 | * @since 2019/8/2
13 | */
14 | public class OracleReader extends BaseReaderPlugin implements DataxReaderInterface {
15 | @Override
16 | public String getName() {
17 | return "oraclereader";
18 | }
19 |
20 | @Override
21 | public Map<String, Object> sample() {
22 | return null;
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/reader/PostgresqlReader.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.reader;
2 |
3 | import java.util.Map;
4 |
5 | /**
6 | * postgresql 构建类
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @version 1.0
10 | * @since 2019/8/2
11 | */
12 | public class PostgresqlReader extends BaseReaderPlugin implements DataxReaderInterface {
13 | @Override
14 | public String getName() {
15 | return "postgresqlreader";
16 | }
17 |
18 | @Override
19 | public Map<String, Object> sample() {
20 | return null;
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/reader/SqlServerReader.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.reader;
2 |
3 | import java.util.Map;
4 |
5 | /**
6 | * sqlserver reader 构建类
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @version 1.0
10 | * @since 2019/8/2
11 | */
12 | public class SqlServerReader extends BaseReaderPlugin implements DataxReaderInterface {
13 | @Override
14 | public String getName() {
15 | return "sqlserverreader";
16 | }
17 |
18 | @Override
19 | public Map<String, Object> sample() {
20 | return null;
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/writer/ClickHouseWriter.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.writer;
2 |
3 | import java.util.Map;
4 |
5 | public class ClickHouseWriter extends BaseWriterPlugin implements DataxWriterInterface {
6 | @Override
7 | public String getName() {
8 | return "clickhousewriter";
9 | }
10 |
11 | @Override
12 | public Map<String, Object> sample() {
13 | return null;
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/writer/DataxWriterInterface.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.writer;
2 |
3 | import com.wugui.datax.admin.tool.datax.DataxPluginInterface;
4 |
5 | /**
6 | * 用于构建writer的接口
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @version 1.0
10 | * @since 2019/7/30
11 | */
12 | public interface DataxWriterInterface extends DataxPluginInterface {
13 |
14 |
15 | }
16 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/writer/HBaseWriter.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.writer;
2 |
3 | import com.alibaba.fastjson.JSON;
4 | import com.google.common.collect.Maps;
5 | import com.wugui.datax.admin.tool.pojo.DataxHbasePojo;
6 | import org.apache.commons.lang3.StringUtils;
7 |
8 | import java.util.Map;
9 |
10 | public class HBaseWriter extends BaseWriterPlugin implements DataxWriterInterface {
11 | @Override
12 | public String getName() {
13 | return "hbase11xwriter";
14 | }
15 |
16 | @Override
17 | public Map<String, Object> sample() {
18 | return null;
19 | }
20 |
21 | public Map<String, Object> buildHbase(DataxHbasePojo plugin) {
22 | //构建
23 | Map<String, Object> writerObj = Maps.newLinkedHashMap();
24 | writerObj.put("name", getName());
25 | Map<String, Object> parameterObj = Maps.newLinkedHashMap();
26 | Map<String, Object> confige = Maps.newLinkedHashMap();
27 | confige.put("hbase.zookeeper.quorum", plugin.getWriterHbaseConfig());
28 | parameterObj.put("hbaseConfig", confige);
29 | parameterObj.put("table", plugin.getWriterTable());
30 | parameterObj.put("mode", plugin.getWriterMode());
31 | parameterObj.put("column", plugin.getColumns());
32 | parameterObj.put("rowkeyColumn", JSON.parseArray(plugin.getWriterRowkeyColumn()));
33 | if (StringUtils.isNotBlank(plugin.getWriterVersionColumn().getValue())) {
34 | parameterObj.put("versionColumn", plugin.getWriterVersionColumn());
35 | }
36 | writerObj.put("parameter", parameterObj);
37 | return writerObj;
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/writer/HiveWriter.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.writer;
2 |
3 | import com.google.common.collect.Maps;
4 | import com.wugui.datax.admin.tool.pojo.DataxHivePojo;
5 |
6 | import java.util.Map;
7 |
8 | /**
9 | * hive writer构建类
10 | *
11 | * @author jingwk
12 | * @version 2.0
13 | * @since 2020/01/05
14 | */
15 | public class HiveWriter extends BaseWriterPlugin implements DataxWriterInterface {
16 | @Override
17 | public String getName() {
18 | return "hdfswriter";
19 | }
20 |
21 |
22 | @Override
23 | public Map<String, Object> sample() {
24 | return null;
25 | }
26 |
27 | @Override
28 | public Map<String, Object> buildHive(DataxHivePojo plugin) {
29 | Map<String, Object> writerObj = Maps.newLinkedHashMap();
30 | writerObj.put("name", getName());
31 |
32 | Map<String, Object> parameterObj = Maps.newLinkedHashMap();
33 | parameterObj.put("defaultFS", plugin.getWriterDefaultFS());
34 | parameterObj.put("fileType", plugin.getWriterFileType());
35 | parameterObj.put("path", plugin.getWriterPath());
36 | parameterObj.put("fileName", plugin.getWriterFileName());
37 | parameterObj.put("writeMode", plugin.getWriteMode());
38 | parameterObj.put("fieldDelimiter", plugin.getWriteFieldDelimiter());
39 | parameterObj.put("column", plugin.getColumns());
40 | writerObj.put("parameter", parameterObj);
41 | return writerObj;
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/writer/MysqlWriter.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.writer;
2 |
3 | import java.util.Map;
4 |
5 | /**
6 | * mysql writer构建类
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @ClassName MysqlWriter
10 | * @Version 1.0
11 | * @since 2019/7/30 23:08
12 | */
13 | public class MysqlWriter extends BaseWriterPlugin implements DataxWriterInterface {
14 | @Override
15 | public String getName() {
16 | return "mysqlwriter";
17 | }
18 |
19 |
20 | @Override
21 | public Map<String, Object> sample() {
22 | return null;
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/writer/OraclelWriter.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.writer;
2 |
3 | import java.util.Map;
4 |
5 | /**
6 | * oracle writer构建类
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @version 1.0
10 | * @since 2019/8/2
11 | */
12 | public class OraclelWriter extends BaseWriterPlugin implements DataxWriterInterface {
13 | @Override
14 | public String getName() {
15 | return "oraclewriter";
16 | }
17 |
18 | @Override
19 | public Map<String, Object> sample() {
20 | return null;
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/writer/PostgresqllWriter.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.writer;
2 |
3 | import java.util.Map;
4 |
5 | /**
6 | * postgresql writer构建类
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @version 1.0
10 | * @since 2019/8/2
11 | */
12 | public class PostgresqllWriter extends BaseWriterPlugin implements DataxWriterInterface {
13 | @Override
14 | public String getName() {
15 | return "postgresqlwriter";
16 | }
17 |
18 |
19 | @Override
20 | public Map<String, Object> sample() {
21 | return null;
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/datax/writer/SqlServerlWriter.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax.writer;
2 |
3 |
4 | import java.util.Map;
5 |
6 | /**
7 | * sql server writer构建类
8 | *
9 | * @author zhouhongfa@gz-yibo.com
10 | * @version 1.0
11 | * @since 2019/8/2
12 | */
13 | public class SqlServerlWriter extends BaseWriterPlugin implements DataxWriterInterface {
14 | @Override
15 | public String getName() {
16 | return "sqlserverwriter";
17 | }
18 |
19 | @Override
20 | public Map<String, Object> sample() {
21 | return null;
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/meta/BaseDatabaseMeta.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.meta;
2 |
3 | /**
4 | * meta信息interface
5 | *
6 | * @author zhouhongfa@gz-yibo.com
7 | * @ClassName BaseDatabaseMeta
8 | * @Version 1.0
9 | * @since 2019/7/17 15:45
10 | */
11 | public abstract class BaseDatabaseMeta implements DatabaseInterface {
12 |
13 | @Override
14 | public String getSQLQueryFields(String tableName) {
15 | return "SELECT * FROM " + tableName + " where 1=0";
16 | }
17 |
18 | @Override
19 | public String getSQLQueryTablesNameComments() {
20 | return "select table_name,table_comment from information_schema.tables where table_schema=?";
21 | }
22 |
23 | @Override
24 | public String getSQLQueryTableNameComment() {
25 | return "select table_name,table_comment from information_schema.tables where table_schema=? and table_name = ?";
26 | }
27 |
28 | @Override
29 | public String getSQLQueryPrimaryKey() {
30 | return null;
31 | }
32 |
33 | @Override
34 | public String getSQLQueryComment(String schemaName, String tableName, String columnName) {
35 | return null;
36 | }
37 |
38 | @Override
39 | public String getSQLQueryColumns(String... args) {
40 | return null;
41 | }
42 |
43 | @Override
44 | public String getMaxId(String tableName, String primaryKey) {
45 | return String.format("select max(%s) from %s",primaryKey,tableName);
46 | }
47 |
48 | @Override
49 | public String getSQLQueryTableSchema(String... args) {
50 | return null;
51 | }
52 |
53 | @Override
54 | public String getSQLQueryTables() {
55 | return null;
56 | }
57 |
58 | @Override
59 | public String getSQLQueryTables(String... tableSchema) {
60 | return null;
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/meta/ClickHouseDataBaseMeta.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.meta;
2 |
3 | public class ClickHouseDataBaseMeta extends BaseDatabaseMeta implements DatabaseInterface {
4 | private volatile static ClickHouseDataBaseMeta single;
5 | public static ClickHouseDataBaseMeta getInstance() {
6 | if (single == null) {
7 | synchronized (ClickHouseDataBaseMeta.class) {
8 | if (single == null) {
9 | single = new ClickHouseDataBaseMeta();
10 | }
11 | }
12 | }
13 | return single;
14 | }
15 | @Override
16 | public String getSQLQueryTables() {
17 | return "show tables";
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/meta/DatabaseInterface.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.meta;
2 |
3 | public interface DatabaseInterface {
4 |
5 | /**
6 | * Returns the minimal SQL to launch in order to determine the layout of the resultset for a given com.com.wugui.datax.admin.tool.database table
7 | *
8 | * @param tableName The name of the table to determine the layout for
9 | * @return The SQL to launch.
10 | */
11 | String getSQLQueryFields(String tableName);
12 |
13 | /**
14 | * 获取主键字段
15 | *
16 | * @return
17 | */
18 | String getSQLQueryPrimaryKey();
19 |
20 | String getSQLQueryTableNameComment();
21 |
22 | String getSQLQueryTablesNameComments();
23 |
24 | /**
25 | * 获取所有表名的sql
26 | *
27 | * @return
28 | */
29 | String getSQLQueryTables(String... tableSchema);
30 |
31 | /**
32 | * 获取所有表名的sql
33 | *
34 | * @return
35 | */
36 | String getSQLQueryTables();
37 |
38 | /**
39 | * 获取 Table schema
40 | *
41 | * @return
42 | */
43 | String getSQLQueryTableSchema(String... args);
44 | /**
45 | * 获取所有的字段的sql
46 | *
47 | * @return
48 | */
49 | String getSQLQueryColumns(String... args);
50 |
51 | /**
52 | * 获取表和字段注释的sql语句
53 | *
54 | * @return The SQL to launch.
55 | */
56 | String getSQLQueryComment(String schemaName, String tableName, String columnName);
57 |
58 |
59 | /**
60 | * 获取当前表maxId
61 | * @param tableName
62 | * @param primaryKey
63 | * @return
64 | */
65 | String getMaxId(String tableName,String primaryKey);
66 | }
67 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/meta/DatabaseMetaFactory.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.meta;
2 |
3 | import com.wugui.datax.admin.util.JdbcConstants;
4 |
5 | /**
6 | * meta信息工厂
7 | *
8 | * @author zhouhongfa@gz-yibo.com
9 | * @ClassName DatabaseMetaFactory
10 | * @Version 1.0
11 | * @since 2019/7/17 15:55
12 | */
13 | public class DatabaseMetaFactory {
14 |
15 | //根据数据库类型返回对应的接口
16 | public static DatabaseInterface getByDbType(String dbType) {
17 | if (JdbcConstants.MYSQL.equals(dbType)) {
18 | return MySQLDatabaseMeta.getInstance();
19 | } else if (JdbcConstants.ORACLE.equals(dbType)) {
20 | return OracleDatabaseMeta.getInstance();
21 | } else if (JdbcConstants.POSTGRESQL.equals(dbType)) {
22 | return PostgresqlDatabaseMeta.getInstance();
23 | } else if (JdbcConstants.SQL_SERVER.equals(dbType)) {
24 | return SqlServerDatabaseMeta.getInstance();
25 | } else if (JdbcConstants.HIVE.equals(dbType)) {
26 | return HiveDatabaseMeta.getInstance();
27 | }else if(JdbcConstants.CLICKHOUSE.equals(dbType)) {
28 | return ClickHouseDataBaseMeta.getInstance();
29 | } else if(JdbcConstants.HBASE20XSQL.equals(dbType)) {
30 | return Hbase20xsqlMeta.getInstance();
31 | } else {
32 | throw new UnsupportedOperationException("暂不支持的类型：".concat(dbType));
33 | }
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/meta/Hbase20xsqlMeta.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.meta;
2 |
3 | /**
4 | * MySQL数据库 meta信息查询
5 | *
6 | * @author zhouhongfa@gz-yibo.com
7 | * @ClassName MySQLDatabaseMeta
8 | * @Version 1.0
9 | * @since 2019/7/17 15:48
10 | */
11 | public class Hbase20xsqlMeta extends BaseDatabaseMeta implements DatabaseInterface {
12 |
13 | private volatile static Hbase20xsqlMeta single;
14 |
15 | public static Hbase20xsqlMeta getInstance() {
16 | if (single == null) {
17 | synchronized (Hbase20xsqlMeta.class) {
18 | if (single == null) {
19 | single = new Hbase20xsqlMeta();
20 | }
21 | }
22 | }
23 | return single;
24 | }
25 |
26 |
27 | @Override
28 | public String getSQLQueryTables(String... tableSchema) {
29 | return null;
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/meta/HiveDatabaseMeta.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.meta;
2 |
3 | /**
4 | * hive元数据信息
5 | *
6 | * @author jingwk
7 | * @ClassName HiveDatabaseMeta
8 | * @Version 2.0
9 | * @since 2020/01/05 15:45
10 | */
11 | public class HiveDatabaseMeta extends BaseDatabaseMeta implements DatabaseInterface {
12 | private volatile static HiveDatabaseMeta single;
13 |
14 | public static HiveDatabaseMeta getInstance() {
15 | if (single == null) {
16 | synchronized (HiveDatabaseMeta.class) {
17 | if (single == null) {
18 | single = new HiveDatabaseMeta();
19 | }
20 | }
21 | }
22 | return single;
23 | }
24 |
25 | @Override
26 | public String getSQLQueryTables() {
27 | return "show tables";
28 | }
29 |
30 |
31 | }
32 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/meta/MySQLDatabaseMeta.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.meta;
2 |
3 | /**
4 | * MySQL数据库 meta信息查询
5 | *
6 | * @author zhouhongfa@gz-yibo.com
7 | * @ClassName MySQLDatabaseMeta
8 | * @Version 1.0
9 | * @since 2019/7/17 15:48
10 | */
11 | public class MySQLDatabaseMeta extends BaseDatabaseMeta implements DatabaseInterface {
12 |
13 | private volatile static MySQLDatabaseMeta single;
14 |
15 | public static MySQLDatabaseMeta getInstance() {
16 | if (single == null) {
17 | synchronized (MySQLDatabaseMeta.class) {
18 | if (single == null) {
19 | single = new MySQLDatabaseMeta();
20 | }
21 | }
22 | }
23 | return single;
24 | }
25 |
26 | @Override
27 | public String getSQLQueryComment(String schemaName, String tableName, String columnName) {
28 | return String.format("SELECT COLUMN_COMMENT FROM information_schema.COLUMNS where TABLE_SCHEMA = '%s' and TABLE_NAME = '%s' and COLUMN_NAME = '%s'", schemaName, tableName, columnName);
29 | }
30 |
31 | @Override
32 | public String getSQLQueryPrimaryKey() {
33 | return "select column_name from information_schema.columns where table_schema=? and table_name=? and column_key = 'PRI'";
34 | }
35 |
36 | @Override
37 | public String getSQLQueryTables() {
38 | return "show tables";
39 | }
40 |
41 | @Override
42 | public String getSQLQueryColumns(String... args) {
43 | return "select column_name from information_schema.columns where table_schema=? and table_name=?";
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/meta/SqlServerDatabaseMeta.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.meta;
2 |
3 | /**
4 | * SqlServer数据库 meta信息查询
5 | *
6 | * @author zhouhongfa@gz-yibo.com
7 | * @ClassName SqlServerDatabaseMeta
8 | * @Version 1.0
9 | * @since 2019/8/2 15:45
10 | */
11 | public class SqlServerDatabaseMeta extends BaseDatabaseMeta implements DatabaseInterface {
12 | private volatile static SqlServerDatabaseMeta single;
13 |
14 | public static SqlServerDatabaseMeta getInstance() {
15 | if (single == null) {
16 | synchronized (SqlServerDatabaseMeta.class) {
17 | if (single == null) {
18 | single = new SqlServerDatabaseMeta();
19 | }
20 | }
21 | }
22 | return single;
23 | }
24 |
25 | @Override
26 | public String getSQLQueryTables() {
27 | return "SELECT Name FROM SysObjects Where XType='U' ORDER BY Name";
28 | }
29 |
30 | @Override
31 | public String getSQLQueryTables(String... tableSchema) {
32 | return "select schema_name(schema_id)+'.'+object_name(object_id) from sys.objects \n" +
33 | "where type ='U' \n" +
34 | "and schema_name(schema_id) ='" + tableSchema[0] + "'";
35 |
36 | }
37 |
38 | @Override
39 | public String getSQLQueryTableSchema(String... args) {
40 | return "select distinct schema_name(schema_id) from sys.objects where type ='U';";
41 | }
42 |
43 | }
44 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/pojo/DataxHbasePojo.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.pojo;
2 |
3 | import com.wugui.datax.admin.dto.Range;
4 | import com.wugui.datax.admin.dto.VersionColumn;
5 | import com.wugui.datax.admin.entity.JobDatasource;
6 | import lombok.Data;
7 |
8 | import java.util.List;
9 | import java.util.Map;
10 |
11 | @Data
12 | public class DataxHbasePojo {
13 |
14 | private List<Map<String,Object>> columns;
15 |
16 | /**
17 | * 数据源信息
18 | */
19 | private JobDatasource jdbcDatasource;
20 |
21 |
22 | private String readerHbaseConfig;
23 |
24 | private String readerTable;
25 |
26 | private String readerMode;
27 |
28 | private String readerMaxVersion;
29 |
30 | private Range readerRange;
31 |
32 | private String writerHbaseConfig;
33 |
34 | private String writerTable;
35 |
36 | private String writerMode;
37 |
38 | private VersionColumn writerVersionColumn;
39 |
40 | private String writerRowkeyColumn;
41 | }
42 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/pojo/DataxHivePojo.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.pojo;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 | import lombok.Data;
5 |
6 | import java.util.List;
7 | import java.util.Map;
8 |
9 | /**
10 | * 用于传参，构建json
11 | *
12 | * @author jingwk
13 | * @ClassName DataxHivePojo
14 | * @Version 2.0
15 | * @since 2020/01/11 17:15
16 | */
17 | @Data
18 | public class DataxHivePojo {
19 |
20 | /**
21 | * hive列名
22 | */
23 | private List<Map<String,Object>> columns;
24 |
25 | /**
26 | * 数据源信息
27 | */
28 | private JobDatasource jdbcDatasource;
29 |
30 | private String readerPath;
31 |
32 | private String readerDefaultFS;
33 |
34 | private String readerFileType;
35 |
36 | private String readerFieldDelimiter;
37 |
38 | private String writerDefaultFS;
39 |
40 | private String writerFileType;
41 |
42 | private String writerPath;
43 |
44 | private String writerFileName;
45 |
46 | private String writeMode;
47 |
48 | private String writeFieldDelimiter;
49 |
50 | private Boolean skipHeader;
51 | }
52 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/pojo/DataxMongoDBPojo.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.pojo;
2 |
3 | import com.wugui.datax.admin.dto.UpsertInfo;
4 | import com.wugui.datax.admin.entity.JobDatasource;
5 | import lombok.Data;
6 |
7 | import java.util.List;
8 | import java.util.Map;
9 |
10 | /**
11 | * 用于传参，构建json
12 | *
13 | * @author jingwk
14 | * @ClassName DataxMongoDBPojo
15 | * @Version 2.0
16 | * @since 2020/03/14 11:15
17 | */
18 | @Data
19 | public class DataxMongoDBPojo {
20 |
21 | /**
22 | * hive列名
23 | */
24 | private List<Map<String, Object>> columns;
25 |
26 | /**
27 | * 数据源信息
28 | */
29 | private JobDatasource jdbcDatasource;
30 |
31 | private String address;
32 |
33 | private String dbName;
34 |
35 | private String readerTable;
36 |
37 | private String writerTable;
38 |
39 | private UpsertInfo upsertInfo;
40 |
41 | }
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/pojo/DataxRdbmsPojo.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.pojo;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 | import lombok.Data;
5 |
6 | import java.util.List;
7 |
8 | /**
9 | * 用于传参，构建json
10 | *
11 | * @author jingwk
12 | * @ClassName DataxRdbmsPojo
13 | * @Version 2.0
14 | * @since 2020/01/11 15:19
15 | */
16 | @Data
17 | public class DataxRdbmsPojo {
18 |
19 | /**
20 | * 表名
21 | */
22 | private List<String> tables;
23 |
24 | /**
25 | * 列名
26 | */
27 | private List<String> rdbmsColumns;
28 |
29 | /**
30 | * 数据源信息
31 | */
32 | private JobDatasource jobDatasource;
33 |
34 | /**
35 | * querySql 属性，如果指定了，则优先于columns参数
36 | */
37 | private String querySql;
38 |
39 | /**
40 | * preSql 属性
41 | */
42 | private String preSql;
43 |
44 | /**
45 | * postSql 属性
46 | */
47 | private String postSql;
48 |
49 | /**
50 | * 切分主键
51 | */
52 | private String splitPk;
53 |
54 | /**
55 | * where
56 | */
57 | private String whereParam;
58 | }
59 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/query/ClickHouseQueryTool.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 |
5 | import java.sql.SQLException;
6 |
7 | /**
8 | * ClickHouse
9 | */
10 |
11 | public class ClickHouseQueryTool extends BaseQueryTool implements QueryToolInterface {
12 | /**
13 | * 构造方法
14 | *
15 | * @param jobJdbcDatasource
16 | */
17 | public ClickHouseQueryTool(JobDatasource jobJdbcDatasource) throws SQLException {
18 | super(jobJdbcDatasource);
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/query/HiveQueryTool.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 |
5 | import java.sql.SQLException;
6 |
7 | /**
8 | * hive
9 | *
10 | * @author wenkaijing
11 | * @version 2.0
12 | * @since 2020/01/05
13 | */
14 | public class HiveQueryTool extends BaseQueryTool implements QueryToolInterface {
15 | public HiveQueryTool(JobDatasource jobDatasource) throws SQLException {
16 | super(jobDatasource);
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/query/MySQLQueryTool.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 |
5 | import java.sql.SQLException;
6 |
7 | /**
8 | * mysql数据库使用的查询工具
9 | *
10 | * @author zhouhongfa@gz-yibo.com
11 | * @ClassName MySQLQueryTool
12 | * @Version 1.0
13 | * @since 2019/7/18 9:31
14 | */
15 | public class MySQLQueryTool extends BaseQueryTool implements QueryToolInterface {
16 |
17 | public MySQLQueryTool(JobDatasource jobDatasource) throws SQLException {
18 | super(jobDatasource);
19 | }
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/query/OracleQueryTool.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 |
5 | import java.sql.SQLException;
6 |
7 | /**
8 | * Oracle数据库使用的查询工具
9 | *
10 | * @author zhouhongfa@gz-yibo.com
11 | * @ClassName MySQLQueryTool
12 | * @Version 1.0
13 | * @since 2019/7/18 9:31
14 | */
15 | public class OracleQueryTool extends BaseQueryTool implements QueryToolInterface {
16 |
17 | public OracleQueryTool(JobDatasource jobDatasource) throws SQLException {
18 | super(jobDatasource);
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/query/PostgresqlQueryTool.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 |
5 | import java.sql.SQLException;
6 |
7 | /**
8 | * TODO
9 | *
10 | * @author zhouhongfa@gz-yibo.com
11 | * @ClassName PostgresqlQueryTool
12 | * @Version 1.0
13 | * @since 2019/8/2 11:28
14 | */
15 | public class PostgresqlQueryTool extends BaseQueryTool implements QueryToolInterface {
16 | public PostgresqlQueryTool(JobDatasource jobDatasource) throws SQLException {
17 | super(jobDatasource);
18 | }
19 |
20 | }
21 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/tool/query/SqlServerQueryTool.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 |
5 | import java.sql.SQLException;
6 |
7 | /**
8 | * sql server
9 | *
10 | * @author zhouhongfa@gz-yibo.com
11 | * @version 1.0
12 | * @since 2019/8/2
13 | */
14 | public class SqlServerQueryTool extends BaseQueryTool implements QueryToolInterface {
15 | public SqlServerQueryTool(JobDatasource jobDatasource) throws SQLException {
16 | super(jobDatasource);
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/util/DateFormatUtils.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.util;
2 |
3 | import java.util.ArrayList;
4 | import java.util.List;
5 |
6 | public class DateFormatUtils {
7 |
8 | public static final String DATE_FORMAT = "yyyy/MM/dd";
9 | public static final String DATETIME_FORMAT = "yyyy/MM/dd HH:mm:ss";
10 | public static final String TIME_FORMAT = "HH:mm:ss";
11 | public static final String TIMESTAMP = "Timestamp";
12 |
13 | public static final List<String> formatList() {
14 | List<String> formatList = new ArrayList<>();
15 | formatList.add(DATE_FORMAT);
16 | formatList.add(TIME_FORMAT);
17 | formatList.add(DATETIME_FORMAT);
18 | formatList.add(TIMESTAMP);
19 | return formatList;
20 | }
21 |
22 | }
23 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/util/ErrorCode.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.util;
2 |
3 | /**
4 | * 尤其注意：最好提供toString()实现。例如：
5 | *
6 | * <pre>
7 | *
8 | * @Override
9 | * public String toString() {
10 | * return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
11 | * }
12 | * </pre>
13 | *
14 | */
15 | public interface ErrorCode {
16 | // 错误码编号
17 | String getCode();
18 |
19 | // 错误码描述
20 | String getDescription();
21 |
22 | /** 必须提供toString的实现
23 | *
24 | * <pre>
25 | * @Override
26 | * public String toString() {
27 | * return String.format("Code:[%s], Description:[%s]. ", this.code, this.describe);
28 | * }
29 | * </pre>
30 | *
31 | */
32 | String toString();
33 | }
34 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/util/PageUtils.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.util;
2 |
3 | import cn.hutool.core.lang.Filter;
4 | import cn.hutool.core.map.MapUtil;
5 | import cn.hutool.core.util.StrUtil;
6 | import com.google.common.collect.ImmutableList;
7 |
8 | import java.util.List;
9 | import java.util.Map;
10 |
11 | /**
12 | * 分页工具类
13 | *
14 | * @author zhouhongfa@gz-yibo.com
15 | * @version 1.0
16 | * @since 2019/6/15
17 | */
18 | public class PageUtils {
19 | /**
20 | * 用于保存分页查询用到的关键字
21 | */
22 | public static final List<String> PAGE_QUERY_KEY_LIST = ImmutableList.of("current", "size", "sortBy", "orderby", "order", "sort", "ifCount", "ascs", "descs");
23 |
24 |
25 | /**
26 | * 过滤pageHelper的参数、空值等
27 | * 返回查询条件
28 | *
29 | * @return
30 | */
31 | public static Map<String, Object> filterColumnQueryParams(Map<String, Object> map) {
32 | return MapUtil.filter(map, (Filter<Map.Entry<String, Object>>) e -> {
33 | if (StrUtil.isBlank(StrUtil.toString(e.getValue()))) {
34 | return false;
35 | }
36 | if (PAGE_QUERY_KEY_LIST.contains(e.getKey())) {
37 | return false;
38 | }
39 | return true;
40 | });
41 | }
42 |
43 | /**
44 | * 返回pageHelper用到的参数
45 | *
46 | * @return
47 | */
48 | public static Map<String, Object> filterPageParams(Map<String, Object> map) {
49 | return MapUtil.filter(map, (Filter<Map.Entry<String, Object>>) e -> {
50 | if (StrUtil.isBlank(StrUtil.toString(e.getValue()))) {
51 | return false;
52 | }
53 | if (PAGE_QUERY_KEY_LIST.contains(e.getKey())) {
54 | return true;
55 | }
56 | return false;
57 | });
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/datax-admin/src/main/java/com/wugui/datax/admin/util/ReflectionUtil.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.util;
2 |
3 | import java.lang.reflect.Field;
4 | import java.lang.reflect.InvocationTargetException;
5 | import java.lang.reflect.Method;
6 |
7 | /**
8 | * @explain JAVA反射工具类
9 | * @author Song
10 | * @date 2019/12/17
11 | */
12 | public class ReflectionUtil {
13 |
14 | /**
15 | * 获取私有成员变量的值
16 | * @param instance 要获取的对象
17 | * @param filedName 获取的变量名称
18 | * @return 返回获取变量的信息（需要强转）
19 | */
20 | public static Object getPrivateField(Object instance, String filedName) throws NoSuchFieldException, IllegalAccessException {
21 | Field field = instance.getClass().getDeclaredField(filedName);
22 | field.setAccessible(true);
23 | return field.get(instance);
24 | }
25 |
26 | /**
27 | * 设置私有成员的值
28 | * @param instance 要获取的对象
29 | * @param fieldName 要获取的变量名
30 | * @param value 设置的值
31 | */
32 | public static void setPrivateField(Object instance, String fieldName, Object value) throws NoSuchFieldException, IllegalAccessException {
33 | Field field = instance.getClass().getDeclaredField(fieldName);
34 | field.setAccessible(true);
35 | field.set(instance, value);
36 | }
37 |
38 | /**
39 | * 访问私有方法
40 | * @param instance 要获取的对象
41 | * @param methodName 私有方法的名称
42 | * @param classes CLASS的返回信息
43 | * @param objects 参数信息
44 | * @return
45 | */
46 | public static Object invokePrivateMethod(Object instance, String methodName, Class[] classes, String objects) throws NoSuchMethodException, IllegalAccessException, InvocationTargetException {
47 | Method method = instance.getClass().getDeclaredMethod(methodName, classes);
48 | method.setAccessible(true);
49 | return method.invoke(instance, objects);
50 | }
51 |
52 | }
53 |
--------------------------------------------------------------------------------
/datax-admin/src/main/lib/HiveJDBC41.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/lib/HiveJDBC41.jar
--------------------------------------------------------------------------------
/datax-admin/src/main/lib/ojdbc6-11.2.0.3.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/lib/ojdbc6-11.2.0.3.jar
--------------------------------------------------------------------------------
/datax-admin/src/main/lib/sqljdbc4-4.0.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/lib/sqljdbc4-4.0.jar
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/bootstrap.properties:
--------------------------------------------------------------------------------
1 | #Database
2 | #DB_HOST=
3 | #DB_PORT=
4 | #DB_USERNAME=
5 | #DB_PASSWORD=
6 | #DB_DATABASE=
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <configuration debug="false" scan="true" scanPeriod="1 seconds">
3 |
4 | <contextName>admin</contextName>
5 | <property name="LOG_PATH"
6 | value="${LOG_PATH:-${LOG_TEMP:-${java.io.tmpdir:-/data/applogs/admin}}}"/>
7 |
8 | <!--控制台日志， 控制台输出 -->
9 | <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
10 | <encoder>
11 | <!--格式化输出：%d表示日期，%thread表示线程名，%-5level：级别从左显示5个字符宽度,%msg：日志消息，%n是换行符-->
12 | <pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{5} - %msg%n</pattern>
13 | </encoder>
14 | </appender>
15 |
16 | <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
17 | <file>${LOG_PATH}/datax-admin.log</file>
18 | <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
19 | <fileNamePattern>${LOG_PATH}.%d{yyyy-MM-dd}.zip</fileNamePattern>
20 | </rollingPolicy>
21 | <encoder>
22 | <pattern>%date %level [%thread] %logger{36} [%file : %line] %msg%n
23 | </pattern>
24 | </encoder>
25 | </appender>
26 |
27 | <!--mybatis log configure-->
28 | <logger name="com.apache.ibatis" level="TRACE"/>
29 | <logger name="java.sql.Connection" level="DEBUG"/>
30 | <logger name="java.sql.Statement" level="DEBUG"/>
31 | <logger name="java.sql.PreparedStatement" level="DEBUG"/>
32 |
33 | <root level="info">
34 | <appender-ref ref="console"/>
35 | <appender-ref ref="file"/>
36 | </root>
37 |
38 | </configuration>
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/mybatis-mapper/JobDatasourceMapper.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
3 | <mapper namespace="com.wugui.datax.admin.mapper.JobDatasourceMapper">
4 |
5 | <update id="update" parameterType="com.wugui.datax.admin.entity.JobDatasource">
6 | UPDATE job_jdbc_datasource
7 | <set>
8 | <if test="datasourceName!=null">datasource_name = #{datasourceName},</if>
9 | <if test="datasource!=null">datasource = #{datasource},</if>
10 | <if test="datasourceGroup!=null">datasource_group = #{datasourceGroup},</if>
11 | <if test="jdbcUsername!=null">jdbc_username = #{jdbcUsername},</if>
12 | <if test="jdbcPassword!=null">jdbc_password = #{jdbcPassword},</if>
13 | <if test="jdbcUrl!=null">jdbc_url = #{jdbcUrl},</if>
14 | <if test="jdbcDriverClass!=null">jdbc_driver_class = #{jdbcDriverClass},</if>
15 | <if test="status!=null">status = #{status},</if>
16 | <if test="comments!=null">comments = #{comments},</if>
17 | </set>
18 | WHERE id = #{id}
19 | </update>
20 |
21 | </mapper>
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/mybatis-mapper/JobProjectMapper.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
3 | <mapper namespace="com.wugui.datax.admin.mapper.JobProjectMapper">
4 |
5 | <select id="getProjectListPaging" resultType="com.wugui.datax.admin.entity.JobProject">
6 | select
7 | p.*,u.username
8 | from job_project p
9 | join job_user u on p.user_id = u.id
10 | where p.flag=1
11 | <if test="searchName!=null and searchName != ''">
12 | and p.name like concat('%', #{searchName}, '%')
13 | </if>
14 | order by p.create_time desc
15 | </select>
16 | </mapper>
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/mybatis-mapper/PermissionDaoMapper.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8" ?>
2 | <!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
3 | <mapper namespace="com.wugui.datax.admin.mapper.PermissionMapper">
4 |
5 | <select id="findAll" resultType="com.wugui.datax.admin.entity.JobPermission">
6 |
7 | SELECT * from job_permission ;
8 | </select>
9 |
10 | <select id="findByAdminUserId" parameterType="int" resultType="com.wugui.datax.admin.entity.JobPermission">
11 | select p.*
12 | from job_User u
13 | LEFT JOIN role_user sru on u.id= sru.job_User_id
14 | LEFT JOIN job_Role r on sru.job_Role_id=r.id
15 | LEFT JOIN job_permission_role spr on spr.role_id=r.id
16 | LEFT JOIN job_permission p on p.id =spr.permission_id
17 | where u.id=#{userId}
18 | </select>
19 | </mapper>
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/avatar.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/resources/static/avatar.jpg
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/resources/static/favicon.ico
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-03e96171.42d5ee4e.css:
--------------------------------------------------------------------------------
1 | .editor-container[data-v-75d10b0a]{height:82vh;overflow:auto}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-35cb90ad.1ff60aed.css:
--------------------------------------------------------------------------------
1 | .json-editor[data-v-fad11014]{height:100%;position:relative}.json-editor[data-v-fad11014] .CodeMirror{height:auto;min-height:300px}.json-editor[data-v-fad11014] .CodeMirror-scroll{min-height:300px}.json-editor[data-v-fad11014] .cm-s-rubyblue span.cm-string{color:#f08047}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-3a8a15ea.3dacfa09.css:
--------------------------------------------------------------------------------
1 | .el-dropdown-link{cursor:pointer;color:#409eff}.el-dropdown+.el-dropdown{margin-left:15px}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-46fcab30.030f9169.css:
--------------------------------------------------------------------------------
1 | .errPage-container[data-v-2c77c6fb]{width:800px;max-width:100%;margin:100px auto}.errPage-container .pan-back-btn[data-v-2c77c6fb]{background:#008489;color:#fff;border:none!important}.errPage-container .pan-gif[data-v-2c77c6fb]{margin:0 auto;display:block}.errPage-container .pan-img[data-v-2c77c6fb]{display:block;margin:0 auto;width:100%}.errPage-container .text-jumbo[data-v-2c77c6fb]{font-size:60px;font-weight:700;color:#484848}.errPage-container .list-unstyled[data-v-2c77c6fb]{font-size:14px}.errPage-container .list-unstyled li[data-v-2c77c6fb]{padding-bottom:5px}.errPage-container .list-unstyled a[data-v-2c77c6fb]{color:#008489;text-decoration:none}.errPage-container .list-unstyled a[data-v-2c77c6fb]:hover{text-decoration:underline}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-4ba76b80.53ac87fa.css:
--------------------------------------------------------------------------------
1 | .waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-4c5a09d8.19618a3b.css:
--------------------------------------------------------------------------------
1 | .waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}.log-container[data-v-59e47e62]{margin-bottom:20px;background:#f5f5f5;width:100%;height:400px;overflow:scroll}.log-container pre[data-v-59e47e62]{display:block;padding:10px;margin:0 0 10.5px;word-break:break-all;word-wrap:break-word;color:#334851;background-color:#f5f5f5;border-radius:1px}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-60797987.53ac87fa.css:
--------------------------------------------------------------------------------
1 | .waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-69a205c2.2b021e6b.css:
--------------------------------------------------------------------------------
1 | .log-container[data-v-55de34aa]{background:#f5f5f5;height:500px;overflow:scroll;margin:20px;border:1px solid #ddd}.log-container pre[data-v-55de34aa]{display:block;padding:10px;margin:0 0 10.5px;word-break:break-all;word-wrap:break-word;color:#334851;background-color:#f5f5f5;border-radius:1px}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-6ace4dde.3c1adc17.css:
--------------------------------------------------------------------------------
1 | .json-editor[data-v-fad11014]{height:100%;position:relative}.json-editor[data-v-fad11014] .CodeMirror{height:auto;min-height:300px}.json-editor[data-v-fad11014] .CodeMirror-scroll{min-height:300px}.json-editor[data-v-fad11014] .cm-s-rubyblue span.cm-string{color:#f08047}.shell-editor[data-v-8a0faab0]{height:100%;position:relative}.shell-editor[data-v-8a0faab0] .CodeMirror{height:auto;min-height:300px}.shell-editor[data-v-8a0faab0] .CodeMirror-scroll{min-height:300px}.shell-editor[data-v-8a0faab0] .cm-s-rubyblue span.cm-string{color:#f08047}.python-editor[data-v-0cf95e77]{height:100%;position:relative}.python-editor[data-v-0cf95e77] .CodeMirror{height:auto;min-height:300px}.python-editor[data-v-0cf95e77] .CodeMirror-scroll{min-height:300px}.python-editor[data-v-0cf95e77] .cm-s-rubyblue span.cm-string{color:#f08047}.powershell-editor[data-v-44735eba]{height:100%;position:relative}.powershell-editor[data-v-44735eba] .CodeMirror{height:auto;min-height:300px}.powershell-editor[data-v-44735eba] .CodeMirror-scroll{min-height:300px}.powershell-editor[data-v-44735eba] .cm-s-rubyblue span.cm-string{color:#f08047}.el-dropdown-link{cursor:pointer;color:#409eff}.el-dropdown+.el-dropdown{margin-left:15px}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-6d2adc59.28c6d689.css:
--------------------------------------------------------------------------------
1 | .waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}.container{overflow:hidden}.container p{font-size:14px;color:#666;padding:10px 0}.container p .fl{float:left}.container p .fr{float:right}.container .loadAverage p{text-align:center}.container .loadAverage .title{font-size:18px;font-weight:700;color:#333;padding:5px 0;margin:0}.container .loadAverage .number{font-size:50px;color:#3d90d0}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-7ff5eb73.3e37c774.css:
--------------------------------------------------------------------------------
1 | .el-checkbox+.el-checkbox{margin-left:10px}.waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}.cron{text-align:left;padding:10px;background:#fff;border:1px solid #dcdfe6;-webkit-box-shadow:0 2px 4px 0 rgba(0,0,0,.12),0 0 6px 0 rgba(0,0,0,.04);box-shadow:0 2px 4px 0 rgba(0,0,0,.12),0 0 6px 0 rgba(0,0,0,.04)}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-b192701e.53ac87fa.css:
--------------------------------------------------------------------------------
1 | .waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-cc2bc43e.53ac87fa.css:
--------------------------------------------------------------------------------
1 | .waves-ripple{position:absolute;border-radius:100%;background-color:rgba(0,0,0,.15);background-clip:padding-box;pointer-events:none;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transform:scale(0);transform:scale(0);opacity:1}.waves-ripple.z-active{opacity:0;-webkit-transform:scale(2);transform:scale(2);-webkit-transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,-webkit-transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out;transition:opacity 1.2s ease-out,transform .6s ease-out,-webkit-transform .6s ease-out}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/css/chunk-commons.c2ac5852.css:
--------------------------------------------------------------------------------
1 | .pagination-container[data-v-f3b72548]{background:#fff;padding:32px 16px}.pagination-container.hidden[data-v-f3b72548]{display:none}
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/fonts/element-icons.535877f5.woff:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/resources/static/static/fonts/element-icons.535877f5.woff
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/fonts/element-icons.732389de.ttf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/resources/static/static/fonts/element-icons.732389de.ttf
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/img/401.089007e7.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/resources/static/static/img/401.089007e7.gif
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/img/404.a57b6f31.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/resources/static/static/img/404.a57b6f31.png
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/img/404_cloud.0f4bc32b.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/resources/static/static/img/404_cloud.0f4bc32b.png
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/img/avatar.1f4ad1c3.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/main/resources/static/static/img/avatar.1f4ad1c3.jpg
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/js/chunk-03e96171.346c650a.js:
--------------------------------------------------------------------------------
1 | (window["webpackJsonp"]=window["webpackJsonp"]||[]).push([["chunk-03e96171"],{"6e10":function(t,e,n){"use strict";var a=n("c030"),i=n.n(a);i.a},ab3b:function(t,e,n){"use strict";n.r(e);var a=function(){var t=this,e=t.$createElement,n=t._self._c||e;return n("div",{staticClass:"app-container"},[n("split-pane",{attrs:{split:"vertical"},on:{resize:t.resize}},[n("template",{slot:"paneL"},[n("el-input",{attrs:{type:"textarea",rows:36,placeholder:"请输入内容",clearable:""},on:{change:t.originChange},model:{value:t.originText,callback:function(e){t.originText=e},expression:"originText"}})],1),t._v(" "),n("template",{slot:"paneR"},[n("div",{staticClass:"editor-container"},[n("JSONEditor",{attrs:{json:t.formatedValue}})],1)])],2)],1)},i=[],o=n("19ab"),r=n.n(o),s=n("33c3"),c={name:"JsonFormat",components:{splitPane:r.a,JSONEditor:s["a"]},data:function(){return{originText:"",formatedValue:""}},watch:{originText:function(t){try{this.formatedValue=JSON.parse(t)}catch(e){}}},methods:{resize:function(){},originChange:function(t){}}},l=c,u=(n("6e10"),n("2877")),p=Object(u["a"])(l,a,i,!1,null,"75d10b0a",null);e["default"]=p.exports},c030:function(t,e,n){}}]);
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/js/chunk-2d230fe7.341b11ea.js:
--------------------------------------------------------------------------------
1 | (window["webpackJsonp"]=window["webpackJsonp"]||[]).push([["chunk-2d230fe7"],{ef3c:function(e,r,n){"use strict";n.r(r);n("a481");var t,u,a={created:function(){var e=this.$route,r=e.params,n=e.query,t=r.path;this.$router.replace({path:"/"+t,query:n})},render:function(e){return e()}},c=a,o=n("2877"),p=Object(o["a"])(c,t,u,!1,null,null,null);r["default"]=p.exports}}]);
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/js/chunk-541ff83b.0585aa55.js:
--------------------------------------------------------------------------------
1 | (window["webpackJsonp"]=window["webpackJsonp"]||[]).push([["chunk-541ff83b"],{"386d":function(n,t,e){"use strict";var r=e("cb7c"),i=e("83a1"),o=e("5f1b");e("214f")("search",1,(function(n,t,e,a){return[function(e){var r=n(this),i=void 0==e?void 0:e[t];return void 0!==i?i.call(e,r):new RegExp(e)[t](String(r))},function(n){var t=a(e,n,this);if(t.done)return t.value;var c=r(n),u=String(this),d=c.lastIndex;i(d,0)||(c.lastIndex=0);var l=o(c,u);return i(c.lastIndex,d)||(c.lastIndex=d),null===l?-1:l.index}]}))},"83a1":function(n,t){n.exports=Object.is||function(n,t){return n===t?0!==n||1/n===1/t:n!=n&&t!=t}},b829:function(n,t,e){"use strict";e.r(t);e("386d");var r,i,o={name:"AuthRedirect",created:function(){var n=window.location.search.slice(1);window.localStorage&&(window.localStorage.setItem("x-admin-oauth-code",n),window.close())},render:function(n){return n()}},a=o,c=e("2877"),u=Object(c["a"])(a,r,i,!1,null,null,null);t["default"]=u.exports}}]);
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/js/chunk-69a205c2.ce29f459.js:
--------------------------------------------------------------------------------
1 | (window["webpackJsonp"]=window["webpackJsonp"]||[]).push([["chunk-69a205c2"],{"32e8":function(t,n,e){"use strict";e.d(n,"b",(function(){return r})),e.d(n,"a",(function(){return i})),e.d(n,"c",(function(){return a})),e.d(n,"d",(function(){return u}));var o=e("b775");function r(t){return Object(o["a"])({url:"api/log/pageList",method:"get",params:t})}function i(t,n,e){return Object(o["a"])({url:"/api/log/clearLog?jobGroup="+t+"&jobId="+n+"&type="+e,method:"post"})}function a(t){return Object(o["a"])({url:"/api/log/killJob",method:"post",data:t})}function u(t,n,e,r){return Object(o["a"])({url:"/api/log/logDetailCat?executorAddress="+t+"&triggerTime="+n+"&logId="+e+"&fromLineNum="+r,method:"get"})}},"50ef":function(t,n,e){},"65e6":function(t,n,e){"use strict";var o=e("50ef"),r=e.n(o);r.a},ce2b:function(t,n,e){"use strict";e.r(n);var o=function(){var t=this,n=t.$createElement,e=t._self._c||n;return e("div",[e("div",{staticStyle:{"background-color":"#304156",padding:"10px 0","text-align":"right"}},[e("el-button",{staticStyle:{"margin-right":"20px"},attrs:{type:"primary"},on:{click:t.loadLog}},[t._v("刷新日志")])],1),t._v(" "),e("div",{staticClass:"log-container"},[e("pre",{attrs:{loading:t.logLoading},domProps:{textContent:t._s(t.logContent)}})])])},r=[],i=e("32e8"),a={data:function(){return{logContent:"",logLoading:!1}},created:function(){this.loadLog()},methods:{loadLog:function(){var t=this;this.logLoading=!0,i["d"](this.$route.query.executorAddress,this.$route.query.triggerTime,this.$route.query.id,this.$route.query.fromLineNum).then((function(n){"\n"===n.content.logContent||(t.logContent=n.content.logContent),t.logLoading=!1}))}}},u=a,c=(e("65e6"),e("2877")),l=Object(c["a"])(u,o,r,!1,null,"55de34aa",null);n["default"]=l.exports}}]);
--------------------------------------------------------------------------------
/datax-admin/src/main/resources/static/static/js/chunk-commons.81a9f846.js:
--------------------------------------------------------------------------------
1 | (window["webpackJsonp"]=window["webpackJsonp"]||[]).push([["chunk-commons"],{"1c64":function(e,t,a){},"1cc6":function(e,t,a){"use strict";var n=a("1c64"),i=a.n(n);i.a},"333d":function(e,t,a){"use strict";var n=function(){var e=this,t=e.$createElement,a=e._self._c||t;return a("div",{staticClass:"pagination-container",class:{hidden:e.hidden}},[a("el-pagination",e._b({attrs:{background:e.background,"current-page":e.currentPage,"page-size":e.pageSize,layout:e.layout,"page-sizes":e.pageSizes,total:e.total},on:{"update:currentPage":function(t){e.currentPage=t},"update:current-page":function(t){e.currentPage=t},"update:pageSize":function(t){e.pageSize=t},"update:page-size":function(t){e.pageSize=t},"size-change":e.handleSizeChange,"current-change":e.handleCurrentChange}},"el-pagination",e.$attrs,!1))],1)},i=[],u=(a("c5f6"),a("09f4")),r={name:"Pagination",props:{total:{required:!0,type:Number},page:{type:Number,default:1},limit:{type:Number,default:20},pageSizes:{type:Array,default:function(){return[10,20,30,50]}},layout:{type:String,default:"total, sizes, prev, pager, next, jumper"},background:{type:Boolean,default:!0},autoScroll:{type:Boolean,default:!0},hidden:{type:Boolean,default:!1}},computed:{currentPage:{get:function(){return this.page},set:function(e){this.$emit("update:page",e)}},pageSize:{get:function(){return this.limit},set:function(e){this.$emit("update:limit",e)}}},methods:{handleSizeChange:function(e){this.$emit("pagination",{page:this.currentPage,limit:e}),this.autoScroll&&Object(u["a"])(0,800)},handleCurrentChange:function(e){this.$emit("pagination",{page:e,limit:this.pageSize}),this.autoScroll&&Object(u["a"])(0,800)}}},o=r,c=(a("1cc6"),a("2877")),p=Object(c["a"])(o,n,i,!1,null,"f3b72548",null);t["a"]=p.exports}}]);
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/admin/controller/AbstractSpringMvcTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.admin.controller;
2 |
3 | import org.junit.Before;
4 | import org.junit.runner.RunWith;
5 | import org.springframework.beans.factory.annotation.Autowired;
6 | import org.springframework.boot.test.context.SpringBootTest;
7 | import org.springframework.test.context.junit4.SpringRunner;
8 | import org.springframework.test.web.servlet.MockMvc;
9 | import org.springframework.test.web.servlet.setup.MockMvcBuilders;
10 | import org.springframework.web.context.WebApplicationContext;
11 |
12 | @RunWith(SpringRunner.class)
13 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
14 | public class AbstractSpringMvcTest {
15 |
16 | @Autowired
17 | private WebApplicationContext applicationContext;
18 | protected MockMvc mockMvc;
19 |
20 | @Before
21 | public void setup() {
22 | this.mockMvc = MockMvcBuilders.webAppContextSetup(this.applicationContext).build();
23 | }
24 |
25 | }
26 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/admin/core/util/JacksonUtilTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.admin.core.util;
2 |
3 | import com.wugui.datax.admin.core.util.JacksonUtil;
4 | import org.hamcrest.core.Is;
5 | import org.junit.Test;
6 |
7 | import java.util.HashMap;
8 | import java.util.Map;
9 |
10 | import static com.wugui.datax.admin.core.util.JacksonUtil.writeValueAsString;
11 | import static org.hamcrest.core.Is.is;
12 | import static org.junit.Assert.assertThat;
13 |
14 | public class JacksonUtilTest {
15 |
16 | @Test
17 | public void shouldWriteValueAsString() {
18 | //given
19 | Map<String, String> map = new HashMap<>();
20 | map.put("aaa", "111");
21 | map.put("bbb", "222");
22 |
23 | //when
24 | String json = writeValueAsString(map);
25 |
26 | //then
27 | assertThat(json, is("{\"aaa\":\"111\",\"bbb\":\"222\"}"));
28 | }
29 |
30 | @Test
31 | public void shouldReadValueAsObject() {
32 | //given
33 | String jsonString = "{\"aaa\":\"111\",\"bbb\":\"222\"}";
34 |
35 | //when
36 | Map result = JacksonUtil.readValue(jsonString, Map.class);
37 |
38 | //then
39 | assertThat(result.get("aaa"), Is.<Object>is("111"));
40 | assertThat(result.get("bbb"), Is.<Object>is("222"));
41 |
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/admin/dao/JobGroupMapperTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.admin.dao;
2 |
3 | import com.wugui.datax.admin.entity.JobGroup;
4 | import com.wugui.datax.admin.mapper.JobGroupMapper;
5 | import org.junit.Test;
6 | import org.junit.runner.RunWith;
7 | import org.springframework.boot.test.context.SpringBootTest;
8 | import org.springframework.test.context.junit4.SpringRunner;
9 |
10 | import javax.annotation.Resource;
11 | import java.util.List;
12 |
13 | @RunWith(SpringRunner.class)
14 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
15 | public class JobGroupMapperTest {
16 |
17 | @Resource
18 | private JobGroupMapper jobGroupMapper;
19 |
20 | @Test
21 | public void test(){
22 | List<JobGroup> list = jobGroupMapper.findAll();
23 |
24 | List<JobGroup> list2 = jobGroupMapper.findByAddressType(0);
25 |
26 | JobGroup group = new JobGroup();
27 | group.setAppName("setAppName");
28 | group.setTitle("setTitle");
29 | group.setOrder(1);
30 | group.setAddressType(0);
31 | group.setAddressList("setAddressList");
32 |
33 | int ret = jobGroupMapper.save(group);
34 |
35 | JobGroup group2 = jobGroupMapper.load(group.getId());
36 | group2.setAppName("setAppName2");
37 | group2.setTitle("setTitle2");
38 | group2.setOrder(2);
39 | group2.setAddressType(2);
40 | group2.setAddressList("setAddressList2");
41 |
42 | int ret2 = jobGroupMapper.update(group2);
43 |
44 | int ret3 = jobGroupMapper.remove(group.getId());
45 | }
46 |
47 | }
48 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/admin/dao/JobLogGlueMapperTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.admin.dao;
2 |
3 | import com.wugui.datax.admin.entity.JobLogGlue;
4 | import com.wugui.datax.admin.mapper.JobLogGlueMapper;
5 | import org.junit.Test;
6 | import org.junit.runner.RunWith;
7 | import org.springframework.boot.test.context.SpringBootTest;
8 | import org.springframework.test.context.junit4.SpringRunner;
9 |
10 | import javax.annotation.Resource;
11 | import java.util.Date;
12 | import java.util.List;
13 |
14 | @RunWith(SpringRunner.class)
15 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
16 | public class JobLogGlueMapperTest {
17 |
18 | @Resource
19 | private JobLogGlueMapper jobLogGlueMapper;
20 |
21 | @Test
22 | public void test(){
23 | JobLogGlue logGlue = new JobLogGlue();
24 | logGlue.setJobId(1);
25 | logGlue.setGlueType("1");
26 | logGlue.setGlueSource("1");
27 | logGlue.setGlueRemark("1");
28 |
29 | logGlue.setAddTime(new Date());
30 | logGlue.setUpdateTime(new Date());
31 | int ret = jobLogGlueMapper.save(logGlue);
32 |
33 | List<JobLogGlue> list = jobLogGlueMapper.findByJobId(1);
34 |
35 | int ret2 = jobLogGlueMapper.removeOld(1, 1);
36 |
37 | int ret3 = jobLogGlueMapper.deleteByJobId(1);
38 | }
39 |
40 | }
41 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/admin/dao/JobRegistryMapperTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.admin.dao;
2 |
3 | import com.wugui.datax.admin.entity.JobRegistry;
4 | import com.wugui.datax.admin.mapper.JobRegistryMapper;
5 | import org.junit.Test;
6 | import org.junit.runner.RunWith;
7 | import org.springframework.boot.test.context.SpringBootTest;
8 | import org.springframework.test.context.junit4.SpringRunner;
9 |
10 | import javax.annotation.Resource;
11 | import java.util.Arrays;
12 | import java.util.Date;
13 | import java.util.List;
14 |
15 | @RunWith(SpringRunner.class)
16 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
17 | public class JobRegistryMapperTest {
18 |
19 | @Resource
20 | private JobRegistryMapper jobRegistryMapper;
21 |
22 | @Test
23 | public void test(){
24 | int ret = jobRegistryMapper.registryUpdate("g1", "k1", "v1",0,0,0, new Date());
25 | if (ret < 1) {
26 | ret = jobRegistryMapper.registrySave("g1", "k1", "v1", 0,0,0,new Date());
27 | }
28 |
29 | List<JobRegistry> list = jobRegistryMapper.findAll(1, new Date());
30 |
31 | int ret2 = jobRegistryMapper.removeDead(Arrays.asList(1));
32 | }
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/admin/util/I18nUtilTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.admin.util;
2 |
3 | import com.wugui.datax.admin.core.util.I18nUtil;
4 | import org.junit.Test;
5 | import org.junit.runner.RunWith;
6 | import org.springframework.boot.test.context.SpringBootTest;
7 | import org.springframework.test.context.junit4.SpringRunner;
8 |
9 | /**
10 | * email util test
11 | *
12 | * @author xuxueli 2017-12-22 17:16:23
13 | */
14 | @RunWith(SpringRunner.class)
15 | @SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
16 | public class I18nUtilTest {
17 |
18 | @Test
19 | public void test(){
20 | System.out.println(I18nUtil.getString("admin_name"));
21 | System.out.println(I18nUtil.getMultString("admin_name", "admin_name_full"));
22 | System.out.println(I18nUtil.getMultString());
23 | }
24 |
25 | }
26 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/admin/util/MinStack.java:
--------------------------------------------------------------------------------
1 | package com.wugui.admin.util;
2 |
3 | import java.util.Stack;
4 |
5 | public class MinStack {
6 | private Stack<Integer> stack;
7 | private Stack<Integer> minStack;
8 |
9 | public MinStack(){
10 | stack= new Stack<>();
11 | minStack= new Stack<>();
12 | }
13 |
14 | public void push(int v){
15 | stack.push(v);
16 | if(minStack.empty()|| minStack.peek()>=v){
17 | minStack.push(v);
18 | }
19 | }
20 | public void pop(){
21 | if(minStack.peek().equals(stack.peek())){ //因为是Integer类型，所以千万不能==判断，要用equals
22 | minStack.pop();
23 | }
24 | stack.pop();
25 | }
26 | public int top(){
27 | return stack.peek();
28 | }
29 | public int getMin(){
30 | return minStack.peek();
31 | }
32 |
33 | public static void main(String[] args) {
34 | MinStack m = new MinStack();
35 | m.push(512);
36 | m.push(-1024);
37 | m.push(-1024);
38 | m.push(512);
39 | m.pop();
40 | System.out.println(m.getMin());
41 | m.pop();
42 | System.out.println(m.getMin());
43 | m.pop();
44 | System.out.println(m.getMin());
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/datax/admin/tool/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/WeiYe-Jing/datax-web/f0aac36b6f3c5c6182b8985bd0bcf1470201e92f/datax-admin/src/test/java/com/wugui/datax/admin/tool/.DS_Store
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/datax/admin/tool/datax/DataxJsonHelperTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 |
5 | public class DataxJsonHelperTest {
6 |
7 | private JobDatasource getReaderDatasource() {
8 | JobDatasource readerDatasource = new JobDatasource();
9 | readerDatasource.setDatasourceName("z01_mysql_3306");
10 | readerDatasource.setJdbcUsername("root");
11 | readerDatasource.setJdbcPassword("root");
12 | readerDatasource.setJdbcUrl("jdbc:mysql://localhost:3306/datax_web?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8");
13 | readerDatasource.setJdbcDriverClass("com.mysql.jdbc.Driver");
14 | return readerDatasource;
15 | }
16 |
17 | private JobDatasource getWriterDatasource() {
18 | JobDatasource writerDatasource = new JobDatasource();
19 | writerDatasource.setDatasourceName("z01_mysql_3306");
20 | writerDatasource.setJdbcUsername("root");
21 | writerDatasource.setJdbcPassword("root");
22 | writerDatasource.setJdbcUrl("jdbc:mysql://localhost:3306/datax_web_demo?serverTimezone=Asia/Shanghai&useLegacyDatetimeCode=false&useSSL=false&nullNamePatternMatchesAll=true&useUnicode=true&characterEncoding=UTF-8");
23 | writerDatasource.setJdbcDriverClass("com.mysql.jdbc.Driver");
24 | return writerDatasource;
25 | }
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/datax/admin/tool/datax/SpringELUtil.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.datax;
2 |
3 | import org.springframework.expression.EvaluationContext;
4 | import org.springframework.expression.ExpressionParser;
5 | import org.springframework.expression.spel.standard.SpelExpressionParser;
6 | import org.springframework.expression.spel.support.StandardEvaluationContext;
7 |
8 | import java.util.Date;
9 |
10 | public class SpringELUtil {
11 | public static void main(String[] args) {
12 | // 定义变量
13 | EvaluationContext context = new StandardEvaluationContext(); // 表达式的上下文,
14 | String date="2019-11-22";
15 | context.setVariable("today", new Date()); // 为了让表达式可以访问该对象, 先把对象放到上下文中
16 | ExpressionParser parser = new SpelExpressionParser();
17 | Date a = parser.parseExpression("#today").getValue(context, Date.class); // Tom , 使用变量
18 | System.out.println(a);
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/datax/admin/tool/query/Hbase20xsqlQueryToolTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 | import com.wugui.datax.admin.util.JdbcConstants;
5 | import lombok.extern.slf4j.Slf4j;
6 | import org.junit.Before;
7 | import org.junit.Test;
8 |
9 | import java.util.List;
10 |
11 | @Slf4j
12 | public class Hbase20xsqlQueryToolTest {
13 |
14 | private BaseQueryTool queryTool;
15 | private JobDatasource jdbcDatasource;
16 |
17 | @Before
18 | public void before() {
19 | genMysqlDemo();
20 | queryTool = QueryToolFactory.getByDbType(jdbcDatasource);
21 | }
22 |
23 | private void genMysqlDemo() {
24 | jdbcDatasource = new JobDatasource();
25 | jdbcDatasource.setDatasource(JdbcConstants.HBASE20XSQL);
26 | jdbcDatasource.setDatasourceName(JdbcConstants.HBASE20XSQL);
27 | jdbcDatasource.setJdbcDriverClass(JdbcConstants.HBASE20XSQL_DRIVER);
28 | jdbcDatasource.setJdbcUrl("jdbc:phoenix:hadoop1,hadoop2,hadoop3:2181");
29 | }
30 |
31 | @Test
32 | public void getTableNames() {
33 | List<String> tableNames = queryTool.getTableNames(null);
34 | for (String table : tableNames) {
35 | System.out.println(table);
36 | }
37 | }
38 |
39 | @Test
40 | public void getColumnNames() {
41 | List<String> columns = queryTool.getColumnNames("STOCK_SYMBOL", null);
42 |
43 | for (String column : columns) {
44 | System.out.println(column);
45 | }
46 | }
47 |
48 |
49 | }
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/datax/admin/tool/query/KerberosLogin.java:
--------------------------------------------------------------------------------
1 | //package com.wugui.datax.admin.tool.query;
2 | //
3 | //public class KerberosLogin {
4 | //
5 | // private Log logger = LogFactory.getLog(KerberosLogin.class);
6 | //
7 | // public void login() {
8 | // String hiveUserName = "xxxxx@BCHKDC";//kerberos认证的用户principal名称
9 | // String hiveKeytab = "F:/NL_MYECLIPSE2014_WORK/usrkrb5/conf/xxxxx.keytab";//用户的keytab认证文件
10 | // String krbconf = "F:/NL_MYECLIPSE2014_WORK/usrkrb5/conf/krb5.conf";//kerberos5的配置文件
11 | //
12 | // System.setProperty("java.security.krb5.conf", krbconf);
13 | // Configuration conf = new Configuration();
14 | // conf.set("hadoop.security.authentication", "Kerberos");
15 | // UserGroupInformation.setConfiguration(conf);
16 | // try {
17 | // UserGroupInformation.loginUserFromKeytab(hiveUserName, hiveKeytab);
18 | // } catch (IOException e) {
19 | // logger.error("Kerberos login fail.", e);
20 | // }
21 | // }
22 | //}
23 |
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/datax/admin/tool/query/OracleQueryToolTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 | import lombok.extern.slf4j.Slf4j;
5 | import org.junit.Before;
6 | import org.junit.Test;
7 |
8 | import java.util.List;
9 |
10 | @Slf4j
11 | public class OracleQueryToolTest {
12 |
13 | private BaseQueryTool queryTool;
14 | private JobDatasource jdbcDatasource;
15 |
16 | @Before
17 | public void before() {
18 | genMysqlDemo();
19 | queryTool = QueryToolFactory.getByDbType(jdbcDatasource);
20 | }
21 |
22 | private void genMysqlDemo() {
23 | jdbcDatasource = new JobDatasource();
24 | jdbcDatasource.setDatasourceName("test");
25 | jdbcDatasource.setJdbcUsername("scott");
26 | jdbcDatasource.setJdbcPassword("tiger");
27 | jdbcDatasource.setJdbcUrl("jdbc:oracle:thin:@localhost:1521/orcl");
28 | jdbcDatasource.setJdbcDriverClass("oracle.jdbc.OracleDriver");
29 | }
30 |
31 | @Test
32 | public void getTableNames() {
33 | List<String> tableNames = queryTool.getTableNames(null);
34 | tableNames.forEach(System.out::println);
35 | }
36 |
37 | @Test
38 | public void getColumnNames() {
39 | List<String> columns = queryTool.getColumnNames("EMP",jdbcDatasource.getJdbcDriverClass());
40 | log.info(columns.toString());
41 | }
42 | }
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/datax/admin/tool/query/PostgresqlQueryToolTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 | import org.junit.Before;
5 | import org.junit.Test;
6 |
7 | import java.util.List;
8 |
9 | public class PostgresqlQueryToolTest {
10 |
11 | private BaseQueryTool queryTool;
12 | private JobDatasource jdbcDatasource;
13 |
14 | @Before
15 | public void before() {
16 | genDs();
17 | queryTool = QueryToolFactory.getByDbType(jdbcDatasource);
18 | }
19 |
20 | private void genDs() {
21 | jdbcDatasource = new JobDatasource();
22 | jdbcDatasource.setDatasourceName("test");
23 | jdbcDatasource.setJdbcUsername("postgres");
24 | jdbcDatasource.setJdbcPassword("postgres");
25 | jdbcDatasource.setJdbcUrl("jdbc:postgresql://localhost:5432/data");
26 | jdbcDatasource.setJdbcDriverClass("org.postgresql.Driver");
27 | }
28 |
29 | @Test
30 | public void getTableNames() {
31 | List<String> tableNames = queryTool.getTableNames(null);
32 | tableNames.forEach(System.out::println);
33 | }
34 |
35 | @Test
36 | public void getTableColumns() {
37 | List<String> tableNames = queryTool.getColumnNames("BD_EMR_TYPE",jdbcDatasource.getJdbcDriverClass());
38 | tableNames.forEach(System.out::println);
39 | }
40 |
41 | }
--------------------------------------------------------------------------------
/datax-admin/src/test/java/com/wugui/datax/admin/tool/query/SqlServerQueryToolTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.admin.tool.query;
2 |
3 | import com.wugui.datax.admin.entity.JobDatasource;
4 | import lombok.extern.slf4j.Slf4j;
5 | import org.junit.Before;
6 | import org.junit.Test;
7 |
8 | import java.util.List;
9 |
10 | @Slf4j
11 | public class SqlServerQueryToolTest {
12 | private BaseQueryTool queryTool;
13 | private JobDatasource jdbcDatasource;
14 |
15 | @Before
16 | public void before() {
17 | genMysqlDemo();
18 | queryTool = QueryToolFactory.getByDbType(jdbcDatasource);
19 | }
20 |
21 | private void genMysqlDemo() {
22 | jdbcDatasource = new JobDatasource();
23 | jdbcDatasource.setDatasourceName("test");
24 | jdbcDatasource.setJdbcUsername("sa");
25 | jdbcDatasource.setJdbcPassword("Sa123");
26 | jdbcDatasource.setJdbcUrl("jdbc:sqlserver://10.20.1.196;databaseName=DataCenter");
27 | jdbcDatasource.setJdbcDriverClass("com.microsoft.sqlserver.jdbc.SQLServerDriver");
28 | }
29 |
30 | @Test
31 | public void getTableNames() {
32 | List<String> tableNames = queryTool.getTableNames(null);
33 | tableNames.forEach(System.out::println);
34 | }
35 |
36 | @Test
37 | public void getColumnNames() {
38 | List<String> columns = queryTool.getColumnNames("BD_EMR_TYPE",jdbcDatasource.getJdbcDriverClass());
39 | log.info(columns.toString());
40 | }
41 | }
--------------------------------------------------------------------------------
/datax-assembly/package.xml:
--------------------------------------------------------------------------------
1 | <assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
2 | xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3 | xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
4 | <id>main</id>
5 | <formats>
6 | <format>tar.gz</format>
7 | </formats>
8 | <includeBaseDirectory>true</includeBaseDirectory>
9 | <fileSets>
10 | <fileSet>
11 | <directory>../packages</directory>
12 | <includes>
13 | <include>datax-*</include>
14 | </includes>
15 | <outputDirectory>packages</outputDirectory>
16 | </fileSet>
17 | <fileSet>
18 | <lineEnding>unix</lineEnding>
19 | <directory>../bin</directory>
20 | <outputDirectory>bin</outputDirectory>
21 | <fileMode>0755</fileMode>
22 | </fileSet>
23 | <!--<fileSet>
24 | <directory>../doc</directory>
25 | <outputDirectory>doc</outputDirectory>
26 | </fileSet>-->
27 | <fileSet>
28 | <directory>../</directory>
29 | <lineEnding>unix</lineEnding>
30 | <includes>
31 | <include>README.md</include>
32 | <include>userGuid.md</include>
33 | </includes>
34 | <outputDirectory>/</outputDirectory>
35 | </fileSet>
36 | </fileSets>
37 | </assembly>
--------------------------------------------------------------------------------
/datax-assembly/pom.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
3 | xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4 | xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5 | <parent>
6 | <artifactId>datax-web</artifactId>
7 | <groupId>com.wugui</groupId>
8 | <version>2.1.2</version>
9 | </parent>
10 | <modelVersion>4.0.0</modelVersion>
11 |
12 | <groupId>com.wugui</groupId>
13 | <artifactId>datax-assembly</artifactId>
14 |
15 | <build>
16 | <plugins>
17 | <plugin>
18 | <groupId>org.apache.maven.plugins</groupId>
19 | <artifactId>maven-assembly-plugin</artifactId>
20 | <version>2.2.1</version>
21 | <executions>
22 | <execution>
23 | <id>assemble</id>
24 | <goals>
25 | <goal>single</goal>
26 | </goals>
27 | <!-- install -->
28 | <phase>install</phase>
29 | </execution>
30 | </executions>
31 | <configuration>
32 | <appendAssemblyId>false</appendAssemblyId>
33 | <attach>false</attach>
34 | <descriptors>
35 | <descriptor>${basedir}/package.xml</descriptor>
36 | </descriptors>
37 | <finalName>${project.parent.artifactId}-${project.parent.version}</finalName>
38 | <outputDirectory>../build</outputDirectory>
39 | </configuration>
40 | </plugin>
41 | </plugins>
42 | </build>
43 |
44 | </project>
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/biz/AdminBiz.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.biz;
2 |
3 | import com.wugui.datatx.core.biz.model.HandleCallbackParam;
4 | import com.wugui.datatx.core.biz.model.HandleProcessCallbackParam;
5 | import com.wugui.datatx.core.biz.model.RegistryParam;
6 | import com.wugui.datatx.core.biz.model.ReturnT;
7 |
8 | import java.util.List;
9 |
10 | /**
11 | * @author xuxueli 2017-07-27 21:52:49
12 | */
13 | public interface AdminBiz {
14 |
15 |
16 | // ---------------------- callback ----------------------
17 |
18 | /**
19 | * callback
20 | *
21 | * @param callbackParamList
22 | * @return
23 | */
24 | ReturnT<String> callback(List<HandleCallbackParam> callbackParamList);
25 |
26 | /**
27 | * processCallback
28 | *
29 | * @param processCallbackParamList
30 | * @return
31 | */
32 | ReturnT<String> processCallback(List<HandleProcessCallbackParam> processCallbackParamList);
33 |
34 | // ---------------------- registry ----------------------
35 |
36 | /**
37 | * registry
38 | *
39 | * @param registryParam
40 | * @return
41 | */
42 | ReturnT<String> registry(RegistryParam registryParam);
43 |
44 | /**
45 | * registry remove
46 | *
47 | * @param registryParam
48 | * @return
49 | */
50 | ReturnT<String> registryRemove(RegistryParam registryParam);
51 |
52 | }
53 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/biz/ExecutorBiz.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.biz;
2 |
3 | import com.wugui.datatx.core.biz.model.LogResult;
4 | import com.wugui.datatx.core.biz.model.ReturnT;
5 | import com.wugui.datatx.core.biz.model.TriggerParam;
6 |
7 | /**
8 | * Created by xuxueli on 17/3/1.
9 | */
10 | public interface ExecutorBiz {
11 |
12 | /**
13 | * beat
14 | *
15 | * @return
16 | */
17 | ReturnT<String> beat();
18 |
19 | /**
20 | * idle beat
21 | *
22 | * @param jobId
23 | * @return
24 | */
25 | ReturnT<String> idleBeat(int jobId);
26 |
27 | /**
28 | * kill
29 | *
30 | * @param jobId
31 | * @return
32 | */
33 | ReturnT<String> kill(int jobId);
34 |
35 | /**
36 | * log
37 | *
38 | * @param logDateTim
39 | * @param logId
40 | * @param fromLineNum
41 | * @return
42 | */
43 | ReturnT<LogResult> log(long logDateTim, long logId, int fromLineNum);
44 |
45 | /**
46 | * run
47 | *
48 | * @param triggerParam
49 | * @return
50 | */
51 | ReturnT<String> run(TriggerParam triggerParam);
52 | }
53 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/biz/model/HandleCallbackParam.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.biz.model;
2 |
3 | import java.io.Serializable;
4 |
5 | /**
6 | * Created by xuxueli on 17/3/2.
7 | */
8 | public class HandleCallbackParam implements Serializable {
9 | private static final long serialVersionUID = 42L;
10 |
11 | private long logId;
12 | private long logDateTim;
13 |
14 | private ReturnT<String> executeResult;
15 |
16 | public HandleCallbackParam(){}
17 | public HandleCallbackParam(long logId, long logDateTim, ReturnT<String> executeResult) {
18 | this.logId = logId;
19 | this.logDateTim = logDateTim;
20 | this.executeResult = executeResult;
21 | }
22 |
23 | public long getLogId() {
24 | return logId;
25 | }
26 |
27 | public void setLogId(long logId) {
28 | this.logId = logId;
29 | }
30 |
31 | public long getLogDateTim() {
32 | return logDateTim;
33 | }
34 |
35 | public void setLogDateTim(long logDateTim) {
36 | this.logDateTim = logDateTim;
37 | }
38 |
39 | public ReturnT<String> getExecuteResult() {
40 | return executeResult;
41 | }
42 |
43 | public void setExecuteResult(ReturnT<String> executeResult) {
44 | this.executeResult = executeResult;
45 | }
46 |
47 | @Override
48 | public String toString() {
49 | return "HandleCallbackParam{" +
50 | "logId=" + logId +
51 | ", logDateTim=" + logDateTim +
52 | ", executeResult=" + executeResult +
53 | '}';
54 | }
55 |
56 | }
57 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/biz/model/HandleProcessCallbackParam.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.biz.model;
2 |
3 | import java.io.Serializable;
4 |
5 | /**
6 | * Created by jingwk on 2019/12/14.
7 | */
8 | public class HandleProcessCallbackParam implements Serializable {
9 | private static final long serialVersionUID = 42L;
10 |
11 | private long logId;
12 | private String processId;
13 | private long logDateTime;
14 |
15 |
16 | public HandleProcessCallbackParam(){}
17 |
18 | public HandleProcessCallbackParam(long logId,long logDateTime, String processId) {
19 | this.logId = logId;
20 | this.processId = processId;
21 | this.logDateTime=logDateTime;
22 | }
23 |
24 | public long getLogId() {
25 | return logId;
26 | }
27 |
28 | public void setLogId(long logId) {
29 | this.logId = logId;
30 | }
31 |
32 | public String getProcessId() {
33 | return processId;
34 | }
35 |
36 | public void setProcessId(String processId) {
37 | this.processId = processId;
38 | }
39 |
40 | public long getLogDateTime() {
41 | return logDateTime;
42 | }
43 |
44 | public void setLogDateTime(long logDateTime) {
45 | this.logDateTime = logDateTime;
46 | }
47 |
48 | @Override
49 | public String toString() {
50 | return "HandleCallbackParam{" +
51 | "logId=" + logId +
52 | ", processId=" + processId +
53 | ", logDateTime=" + logDateTime +
54 | '}';
55 | }
56 |
57 | }
58 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/biz/model/LogResult.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.biz.model;
2 |
3 | import java.io.Serializable;
4 |
5 | /**
6 | * Created by xuxueli on 17/3/23.
7 | */
8 | public class LogResult implements Serializable {
9 | private static final long serialVersionUID = 42L;
10 |
11 | public LogResult(int fromLineNum, int toLineNum, String logContent, boolean isEnd) {
12 | this.fromLineNum = fromLineNum;
13 | this.toLineNum = toLineNum;
14 | this.logContent = logContent;
15 | this.isEnd = isEnd;
16 | }
17 |
18 | private int fromLineNum;
19 | private int toLineNum;
20 | private String logContent;
21 | private boolean isEnd;
22 |
23 | public int getFromLineNum() {
24 | return fromLineNum;
25 | }
26 |
27 | public void setFromLineNum(int fromLineNum) {
28 | this.fromLineNum = fromLineNum;
29 | }
30 |
31 | public int getToLineNum() {
32 | return toLineNum;
33 | }
34 |
35 | public void setToLineNum(int toLineNum) {
36 | this.toLineNum = toLineNum;
37 | }
38 |
39 | public String getLogContent() {
40 | return logContent;
41 | }
42 |
43 | public void setLogContent(String logContent) {
44 | this.logContent = logContent;
45 | }
46 |
47 | public boolean isEnd() {
48 | return isEnd;
49 | }
50 |
51 | public void setEnd(boolean end) {
52 | isEnd = end;
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/biz/model/ReturnT.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.biz.model;
2 |
3 | import java.io.Serializable;
4 |
5 | /**
6 | * common return
7 | * @author xuxueli 2015-12-4 16:32:31
8 | * @param <T>
9 | */
10 | public class ReturnT<T> implements Serializable {
11 | public static final long serialVersionUID = 42L;
12 |
13 | public static final int SUCCESS_CODE = 200;
14 | public static final int FAIL_CODE = 500;
15 |
16 | public static final ReturnT<String> SUCCESS = new ReturnT<>(null);
17 | public static final ReturnT<String> FAIL = new ReturnT<>(FAIL_CODE, null);
18 |
19 | private int code;
20 | private String msg;
21 | private T content;
22 |
23 | public ReturnT(){}
24 | public ReturnT(int code, String msg) {
25 | this.code = code;
26 | this.msg = msg;
27 | }
28 | public ReturnT(T content) {
29 | this.code = SUCCESS_CODE;
30 | this.content = content;
31 | }
32 |
33 | public int getCode() {
34 | return code;
35 | }
36 | public void setCode(int code) {
37 | this.code = code;
38 | }
39 | public String getMsg() {
40 | return msg;
41 | }
42 | public void setMsg(String msg) {
43 | this.msg = msg;
44 | }
45 | public T getContent() {
46 | return content;
47 | }
48 | public void setContent(T content) {
49 | this.content = content;
50 | }
51 |
52 | @Override
53 | public String toString() {
54 | return "ReturnT [code=" + code + ", msg=" + msg + ", content=" + content + "]";
55 | }
56 |
57 | }
58 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/enums/ExecutorBlockStrategyEnum.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.enums;
2 |
3 | /**
4 | * Created by xuxueli on 17/5/9.
5 | */
6 | public enum ExecutorBlockStrategyEnum {
7 |
8 | SERIAL_EXECUTION("Serial execution"),
9 | /*CONCURRENT_EXECUTION("并行"),*/
10 | DISCARD_LATER("Discard Later"),
11 | COVER_EARLY("Cover Early");
12 |
13 | private String title;
14 | ExecutorBlockStrategyEnum(String title) {
15 | this.title = title;
16 | }
17 |
18 | public void setTitle(String title) {
19 | this.title = title;
20 | }
21 | public String getTitle() {
22 | return title;
23 | }
24 |
25 | public static ExecutorBlockStrategyEnum match(String name, ExecutorBlockStrategyEnum defaultItem) {
26 | if (name != null) {
27 | for (ExecutorBlockStrategyEnum item:ExecutorBlockStrategyEnum.values()) {
28 | if (item.name().equals(name)) {
29 | return item;
30 | }
31 | }
32 | }
33 | return defaultItem;
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/enums/IncrementTypeEnum.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.enums;
2 |
3 | /**
4 | * increment type
5 | */
6 | public enum IncrementTypeEnum {
7 | /**
8 | * 2 TIME
9 | * 1 ID
10 | * 3 PARTITION
11 | */
12 | TIME(2, "时间"),
13 | ID(1, "自增主键"),
14 | PARTITION(3, "HIVE分区");
15 |
16 | IncrementTypeEnum(int code, String descp){
17 | this.code = code;
18 | this.descp = descp;
19 | }
20 |
21 | private final int code;
22 | private final String descp;
23 |
24 | public int getCode() {
25 | return code;
26 | }
27 |
28 | public String getDescp() {
29 | return descp;
30 | }
31 | }
32 |
33 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/enums/RegistryConfig.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.enums;
2 |
3 | /**
4 | * Created by xuxueli on 17/5/10.
5 | */
6 | public class RegistryConfig {
7 |
8 | public static final int BEAT_TIMEOUT = 30;
9 | public static final int DEAD_TIMEOUT = BEAT_TIMEOUT * 3;
10 |
11 | public enum RegistType{ EXECUTOR, ADMIN }
12 |
13 | }
14 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/glue/GlueTypeEnum.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.glue;
2 |
3 | /**
4 | * Created by xuxueli on 17/4/26.
5 | */
6 | public enum GlueTypeEnum {
7 |
8 | BEAN("BEAN", false, null, null),
9 | GLUE_GROOVY("GLUE(Java)", false, null, null),
10 | GLUE_SHELL("GLUE(Shell)", true, "bash", ".sh"),
11 | GLUE_PYTHON("GLUE(Python)", true, "python", ".py"),
12 | GLUE_PHP("GLUE(PHP)", true, "php", ".php"),
13 | GLUE_NODEJS("GLUE(Nodejs)", true, "node", ".js"),
14 | GLUE_POWERSHELL("GLUE(PowerShell)", true, "powershell", ".ps1");
15 |
16 | private String desc;
17 | private boolean isScript;
18 | private String cmd;
19 | private String suffix;
20 |
21 | private GlueTypeEnum(String desc, boolean isScript, String cmd, String suffix) {
22 | this.desc = desc;
23 | this.isScript = isScript;
24 | this.cmd = cmd;
25 | this.suffix = suffix;
26 | }
27 |
28 | public String getDesc() {
29 | return desc;
30 | }
31 |
32 | public boolean isScript() {
33 | return isScript;
34 | }
35 |
36 | public String getCmd() {
37 | return cmd;
38 | }
39 |
40 | public String getSuffix() {
41 | return suffix;
42 | }
43 |
44 | public static GlueTypeEnum match(String name){
45 | for (GlueTypeEnum item: GlueTypeEnum.values()) {
46 | if (item.name().equals(name)) {
47 | return item;
48 | }
49 | }
50 | return null;
51 | }
52 |
53 | }
54 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/handler/IJobHandler.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.handler;
2 |
3 | import com.wugui.datatx.core.biz.model.ReturnT;
4 | import com.wugui.datatx.core.biz.model.TriggerParam;
5 |
6 | import java.util.concurrent.ConcurrentHashMap;
7 | import java.util.concurrent.ConcurrentMap;
8 |
9 | /**
10 | * job handler
11 | *
12 | * @author xuxueli 2015-12-19 19:06:38
13 | */
14 | public abstract class IJobHandler {
15 |
16 |
17 | /** success */
18 | public static final ReturnT<String> SUCCESS = new ReturnT<>(200, null);
19 | /** fail */
20 | public static final ReturnT<String> FAIL = new ReturnT<>(500, null);
21 | /** fail timeout */
22 | public static final ReturnT<String> FAIL_TIMEOUT = new ReturnT<>(502, null);
23 |
24 | public static final ConcurrentMap<String, String> jobTmpFiles = new ConcurrentHashMap<>();
25 | /**
26 | * execute handler, invoked when executor receives a scheduling request
27 | *
28 | * @param tgParam
29 | * @return
30 | * @throws Exception
31 | */
32 | public abstract ReturnT<String> execute(TriggerParam tgParam) throws Exception;
33 |
34 | /**
35 | * init handler, invoked when JobThread init
36 | */
37 | public void init() {
38 | // do something
39 | }
40 |
41 |
42 | /**
43 | * destroy handler, invoked when JobThread destroy
44 | */
45 | public void destroy() {
46 | // do something
47 | }
48 |
49 |
50 | }
51 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/handler/annotation/JobHandler.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.handler.annotation;
2 |
3 | import java.lang.annotation.*;
4 |
5 | /**
6 | * annotation for job handler
7 | * @author 2016-5-17 21:06:49
8 | */
9 | @Target({ElementType.TYPE})
10 | @Retention(RetentionPolicy.RUNTIME)
11 | @Inherited
12 | public @interface JobHandler {
13 |
14 | String value() default "";
15 |
16 | }
17 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/handler/impl/GlueJobHandler.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.handler.impl;
2 |
3 | import com.wugui.datatx.core.biz.model.ReturnT;
4 | import com.wugui.datatx.core.biz.model.TriggerParam;
5 | import com.wugui.datatx.core.handler.IJobHandler;
6 | import com.wugui.datatx.core.log.JobLogger;
7 |
8 | /**
9 | * glue job handler
10 | * @author xuxueli 2016-5-19 21:05:45
11 | */
12 | public class GlueJobHandler extends IJobHandler {
13 |
14 | private long glueUpdatetime;
15 | private IJobHandler jobHandler;
16 | public GlueJobHandler(IJobHandler jobHandler, long glueUpdatetime) {
17 | this.jobHandler = jobHandler;
18 | this.glueUpdatetime = glueUpdatetime;
19 | }
20 | public long getGlueUpdatetime() {
21 | return glueUpdatetime;
22 | }
23 |
24 | @Override
25 | public ReturnT<String> execute(TriggerParam tgParam) throws Exception {
26 | JobLogger.log("----------- glue.version:"+ glueUpdatetime +" -----------");
27 | return jobHandler.execute(tgParam);
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/util/Kernel32.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.util;
2 |
3 | import com.sun.jna.Library;
4 | import com.sun.jna.Native;
5 |
6 | /**
7 | * Kernel32
8 | *
9 | * @author jingwk
10 | * @version 1.0
11 | * @since 2019/11/09
12 | */
13 |
14 | public interface Kernel32 extends Library {
15 | Kernel32 INSTANCE = (Kernel32) Native.loadLibrary("kernel32", Kernel32.class);
16 |
17 | long GetProcessId(Long hProcess);
18 | }
19 |
--------------------------------------------------------------------------------
/datax-core/src/main/java/com/wugui/datatx/core/util/ShardingUtil.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datatx.core.util;
2 |
3 | /**
4 | * sharding vo
5 | *
6 | * @author xuxueli 2017-07-25 21:26:38
7 | */
8 | public class ShardingUtil {
9 |
10 | private static InheritableThreadLocal<ShardingVO> contextHolder = new InheritableThreadLocal<>();
11 |
12 | public static class ShardingVO {
13 |
14 | private int index; // sharding index
15 | private int total; // sharding total
16 |
17 | public ShardingVO(int index, int total) {
18 | this.index = index;
19 | this.total = total;
20 | }
21 |
22 | public int getIndex() {
23 | return index;
24 | }
25 |
26 | public void setIndex(int index) {
27 | this.index = index;
28 | }
29 |
30 | public int getTotal() {
31 | return total;
32 | }
33 |
34 | public void setTotal(int total) {
35 | this.total = total;
36 | }
37 | }
38 |
39 | public static void setShardingVo(ShardingVO shardingVo) {
40 | contextHolder.set(shardingVo);
41 | }
42 |
43 | public static ShardingVO getShardingVo() {
44 | return contextHolder.get();
45 | }
46 |
47 | }
48 |
--------------------------------------------------------------------------------
/datax-executor/src/main/assembly/deploy.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
3 | xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
4 | <id>dist</id>
5 | <formats>
6 | <format>tar.gz</format>
7 | </formats>
8 | <includeBaseDirectory>false</includeBaseDirectory>
9 | <dependencySets>
10 | <dependencySet>
11 | <useProjectArtifact>true</useProjectArtifact>
12 | <outputDirectory>datax-executor/lib</outputDirectory>
13 | <scope>runtime</scope>
14 | </dependencySet>
15 | </dependencySets>
16 | <fileSets>
17 | <fileSet>
18 | <lineEnding>unix</lineEnding>
19 | <directory>./src/main/bin</directory>
20 | <outputDirectory>datax-executor/bin</outputDirectory>
21 | <includes>
22 | <include>**/*</include>
23 | </includes>
24 | <fileMode>0755</fileMode>
25 | </fileSet>
26 | <fileSet>
27 | <directory>./src/main/logs</directory>
28 | <outputDirectory>datax-executor/logs</outputDirectory>
29 | </fileSet>
30 | <fileSet>
31 | <directory>./src/main/resources</directory>
32 | <includes>
33 | <include>logback.xml</include>
34 | <include>application.yml</include>
35 | </includes>
36 | <outputDirectory>datax-executor/conf</outputDirectory>
37 | </fileSet>
38 | </fileSets>
39 | </assembly>
--------------------------------------------------------------------------------
/datax-executor/src/main/bin/env.properties:
--------------------------------------------------------------------------------
1 | # environment variables
2 |
3 | #JAVA_HOME=""
4 |
5 | SERVICE_LOG_PATH=${BIN}/../logs
6 | SERVICE_CONF_PATH=${BIN}/../conf
7 | DATA_PATH=${BIN}/../data
8 |
9 |
10 | ## datax json文件存放位置
11 | JSON_PATH=${BIN}/../json
12 |
13 |
14 | ## executor_port
15 | EXECUTOR_PORT=9999
16 |
17 |
18 | ## 保持和datax-admin端口一致
19 | DATAX_ADMIN_PORT=
20 |
21 | ## PYTHON脚本执行位置
22 | #PYTHON_PATH=/home/hadoop/install/datax/bin/datax.py
23 | PYTHON_PATH=
24 |
25 |
26 |
27 | ## dataxweb 服务端口
28 | SERVER_PORT=9504
29 |
30 | #PID_FILE_PATH=${BIN}/service.pid
31 |
32 |
33 | #debug 远程调试端口
34 | #REMOTE_DEBUG_SWITCH=true
35 | #REMOTE_DEBUG_PORT=7004
--------------------------------------------------------------------------------
/datax-executor/src/main/java/com/wugui/datax/executor/DataXExecutorApplication.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.executor;
2 |
3 | import org.springframework.boot.SpringApplication;
4 | import org.springframework.boot.autoconfigure.SpringBootApplication;
5 |
6 | /**
7 | * @author xuxueli 2018-10-28 00:38:13
8 | */
9 | @SpringBootApplication
10 | public class DataXExecutorApplication {
11 |
12 | public static void main(String[] args) {
13 | SpringApplication.run(DataXExecutorApplication.class, args);
14 | }
15 |
16 | }
--------------------------------------------------------------------------------
/datax-executor/src/main/java/com/wugui/datax/executor/service/jobhandler/DataXConstant.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.executor.service.jobhandler;
2 |
3 | /**
4 | * DataX启动参数
5 | *
6 | * @author jingwk 2019-12-15
7 | */
8 | public class DataXConstant {
9 |
10 | public static final String SPLIT_SPACE = " ";
11 |
12 | public static final String TRANSFORM_SPLIT_SPACE = "\" \"";
13 |
14 | public static final String PERCENT ="%";
15 |
16 | public static final String TRANSFORM_QUOTES = "\"";
17 |
18 | public static final String JVM_CM = "-j";
19 |
20 | public static final String PARAMS_CM = "-p";
21 |
22 | public static final String PARAMS_CM_V_PT = "-Dpartition=%s";
23 |
24 | public static final String DEFAULT_JSON = "jsons";
25 |
26 | public static final String DEFAULT_DATAX_PY = "datax.py";
27 |
28 | public static final String TASK_START_TIME_SUFFIX = "任务启动时刻";
29 | public static final String TASK_END_TIME_SUFFIX = "任务结束时刻";
30 |
31 | public static final String TASK_TOTAL_TIME_SUFFIX = "任务总计耗时";
32 | public static final String TASK_AVERAGE_FLOW_SUFFIX = "任务平均流量";
33 | public static final String TASK_RECORD_WRITING_SPEED_SUFFIX = "记录写入速度";
34 | public static final String TASK_RECORD_READER_NUM_SUFFIX = "读出记录总数";
35 | public static final String TASK_RECORD_WRITING_NUM_SUFFIX = "读写失败总数";
36 | }
37 |
--------------------------------------------------------------------------------
/datax-executor/src/main/java/com/wugui/datax/executor/service/jobhandler/KillJobHandler.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.executor.service.jobhandler;
2 |
3 |
4 | import cn.hutool.core.io.FileUtil;
5 | import com.wugui.datatx.core.biz.model.ReturnT;
6 | import com.wugui.datatx.core.biz.model.TriggerParam;
7 | import com.wugui.datatx.core.handler.IJobHandler;
8 | import com.wugui.datatx.core.handler.annotation.JobHandler;
9 | import com.wugui.datatx.core.util.ProcessUtil;
10 | import org.springframework.stereotype.Component;
11 | import org.springframework.util.CollectionUtils;
12 |
13 | import java.io.File;
14 |
15 | /**
16 | * DataX任务终止
17 | *
18 | * @author jingwk 2019-12-16
19 | */
20 |
21 | @JobHandler(value = "killJobHandler")
22 | @Component
23 | public class KillJobHandler extends IJobHandler {
24 |
25 | @Override
26 | public ReturnT<String> execute(TriggerParam tgParam) {
27 | String processId = tgParam.getProcessId();
28 | boolean result = ProcessUtil.killProcessByPid(processId);
29 | // 删除临时文件
30 | if (!CollectionUtils.isEmpty(jobTmpFiles)) {
31 | String pathname = jobTmpFiles.get(processId);
32 | if (pathname != null) {
33 | FileUtil.del(new File(pathname));
34 | jobTmpFiles.remove(processId);
35 | }
36 | }
37 | return result ? IJobHandler.SUCCESS : IJobHandler.FAIL;
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/datax-executor/src/main/java/com/wugui/datax/executor/util/SystemUtils.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.executor.util;
2 |
3 | import org.apache.commons.lang3.StringUtils;
4 | import org.slf4j.Logger;
5 | import org.slf4j.LoggerFactory;
6 |
7 | import java.io.File;
8 |
9 | /**
10 | * @author maokeluo
11 | * @description 多隆镇楼，bug退散🙏🙏🙏
12 | * 系统工具
13 | * @date 2020/1/7
14 | */
15 | public class SystemUtils {
16 |
17 | private static final Logger LOGGER = LoggerFactory.getLogger(SystemUtils.class);
18 |
19 | private static String DATAX_HOME;
20 |
21 | private SystemUtils() {
22 | }
23 |
24 | /**
25 | * 获取环境变量中的Datax路径
26 | *
27 | * @return
28 | */
29 | public static String getDataXHomePath() {
30 | if (StringUtils.isNotEmpty(DATAX_HOME)) return DATAX_HOME;
31 | String dataXHome = System.getenv("DATAX_HOME");
32 | if (StringUtils.isBlank(dataXHome)) {
33 | //LOGGER.warn("DATAX_HOME 环境变量为NULL");
34 | return null;
35 | }
36 | DATAX_HOME = dataXHome.endsWith(File.separator) ? dataXHome : dataXHome.concat(File.separator);
37 | //LOGGER.info("DATAX_HOME:{}", DATAX_HOME);
38 | return DATAX_HOME;
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/datax-executor/src/main/resources/application.yml:
--------------------------------------------------------------------------------
1 | # web port
2 | server:
3 | port: ${server.port}
4 | #port: 8081
5 |
6 | # log config
7 | logging:
8 | config: classpath:logback.xml
9 | path: ${data.path}/applogs/executor/jobhandler
10 | #path: ./data/applogs/executor/jobhandler
11 |
12 | datax:
13 | job:
14 | admin:
15 | ### datax admin address list, such as "http://address" or "http://address01,http://address02"
16 | #addresses: http://127.0.0.1:8080
17 | addresses: http://127.0.0.1:${datax.admin.port}
18 | executor:
19 | appname: datax-executor
20 | ip:
21 | #port: 9999
22 | port: ${executor.port:9999}
23 | ### job log path
24 | #logpath: ./data/applogs/executor/jobhandler
25 | logpath: ${data.path}/applogs/executor/jobhandler
26 | ### job log retention days
27 | logretentiondays: 30
28 | ### job, access token
29 | accessToken:
30 |
31 | executor:
32 | #jsonpath: D:\\temp\\executor\\json\\
33 | jsonpath: ${json.path}
34 |
35 | #pypath: F:\tools\datax\bin\datax.py
36 | pypath: ${python.path}
--------------------------------------------------------------------------------
/datax-executor/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
1 | <?xml version="1.0" encoding="UTF-8"?>
2 | <configuration debug="false" scan="true" scanPeriod="1 seconds">
3 |
4 | <contextName>exe</contextName>
5 | <property name="LOG_PATH"
6 | value="${LOG_PATH:-${LOG_TEMP:-${java.io.tmpdir:-/data/applogs/executor/jobhandler}}}" />
7 |
8 | <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
9 | <encoder>
10 | <pattern>%d{HH:mm:ss.SSS} %contextName [%thread] %-5level %logger{10} - %msg%n</pattern>
11 | </encoder>
12 | </appender>
13 |
14 | <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
15 | <file>${LOG_PATH}/datax-executor.log</file>
16 | <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
17 | <fileNamePattern>${LOG_PATH}.%d{yyyy-MM-dd}.zip</fileNamePattern>
18 | </rollingPolicy>
19 | <encoder>
20 | <pattern>%date %level [%thread] %logger{20} [%file : %line] %msg%n
21 | </pattern>
22 | </encoder>
23 | </appender>
24 |
25 | <root level="info">
26 | <appender-ref ref="console"/>
27 | <appender-ref ref="file"/>
28 | </root>
29 |
30 | </configuration>
--------------------------------------------------------------------------------
/datax-executor/src/test/java/com/wugui/datax/executor/test/DataXExecutorExampleBootApplicationTests.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.executor.test;
2 |
3 | import org.junit.Test;
4 | import org.junit.runner.RunWith;
5 | import org.springframework.boot.test.context.SpringBootTest;
6 | import org.springframework.test.context.junit4.SpringRunner;
7 |
8 | @RunWith(SpringRunner.class)
9 | @SpringBootTest
10 | public class DataXExecutorExampleBootApplicationTests {
11 |
12 | @Test
13 | public void test() {
14 |
15 | }
16 |
17 | }
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/old/registry/impl/test/XxlZkClientTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.old.registry.impl.test;//package com.xxl.rpc.test;
2 | //
3 | //import com.xxl.rpc.util.XxlZkClient;
4 | //
5 | //import java.util.concurrent.TimeUnit;
6 | //
7 | //public class XxlZkClientTest {
8 | //
9 | // public static void main(String[] args) throws InterruptedException {
10 | //
11 | // XxlZkClient client = null;
12 | // try {
13 | // client = new XxlZkClient("127.0.0.1:2181", "/xxl-rpc/test", null, null);
14 | // } catch (Exception e) {
15 | // e.printStackTrace();
16 | // }
17 | //
18 | //
19 | // for (int i = 0; i < 100; i++) {
20 | // System.out.println("------------- " + i);
21 | // try {
22 | // System.out.println(client.getClient());
23 | // } catch (Exception e) {
24 | // e.printStackTrace();
25 | // }
26 | // TimeUnit.SECONDS.sleep(5);
27 | // }
28 | //
29 | // }
30 | //
31 | //}
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/old/remoting/net/NetEnum.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.old.remoting.net;//package com.xxl.rpc.remoting.net;
2 | //
3 | //import com.xxl.rpc.remoting.net.impl.netty.client.NettyClient;
4 | //import com.xxl.rpc.remoting.net.impl.netty.server.NettyServer;
5 | //import com.xxl.rpc.remoting.net.impl.netty_http.client.NettyHttpClient;
6 | //import com.xxl.rpc.remoting.net.impl.netty_http.server.NettyHttpServer;
7 | //
8 | ///**
9 | // * remoting net
10 | // *
11 | // * @author xuxueli 2015-11-24 22:09:57
12 | // */
13 | //public enum NetEnum {
14 | //
15 | //
16 | // /**
17 | // * netty tcp server
18 | // */
19 | // NETTY(NettyServer.class, NettyClient.class),
20 | //
21 | // /**
22 | // * netty http server
23 | // */
24 | // NETTY_HTTP(NettyHttpServer.class, NettyHttpClient.class);
25 | //
26 | //
27 | // public final Class<? extends Server> serverClass;
28 | // public final Class<? extends Client> clientClass;
29 | //
30 | // NetEnum(Class<? extends Server> serverClass, Class<? extends Client> clientClass) {
31 | // this.serverClass = serverClass;
32 | // this.clientClass = clientClass;
33 | // }
34 | //
35 | // public static NetEnum autoMatch(String name, NetEnum defaultEnum) {
36 | // for (NetEnum item : NetEnum.values()) {
37 | // if (item.name().equals(name)) {
38 | // return item;
39 | // }
40 | // }
41 | // return defaultEnum;
42 | // }
43 | //
44 | //}
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/old/remoting/net/impl/mina/client/MinaClient.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.old.remoting.net.impl.mina.client;//package com.xxl.rpc.remoting.net.impl.mina.client;
2 | //
3 | //import com.xxl.rpc.remoting.net.Client;
4 | //import com.xxl.rpc.remoting.net.params.XxlRpcRequest;
5 | //import com.xxl.rpc.remoting.net.common.ConnectClient;
6 | //
7 | ///**
8 | // * mina client
9 | // *
10 | // * @author xuxueli 2015-11-24 22:25:15
11 | // */
12 | //public class MinaClient extends Client {
13 | //
14 | // private Class<? extends ConnectClient> connectClientImpl = MinaConnectClient.class;
15 | //
16 | // @Override
17 | // public void asyncSend(String address, XxlRpcRequest xxlRpcRequest) throws Exception {
18 | // ConnectClient.asyncSend(xxlRpcRequest, address, connectClientImpl, xxlRpcReferenceBean);
19 | // }
20 | //
21 | //}
22 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/old/remoting/net/impl/mina/codec/MinaDecoder.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.old.remoting.net.impl.mina.codec;//package com.xxl.rpc.remoting.net.impl.mina.codec;
2 | //
3 | //import org.apache.mina.core.buffer.IoBuffer;
4 | //import org.apache.mina.core.session.IoSession;
5 | //import org.apache.mina.filter.codec.CumulativeProtocolDecoder;
6 | //import org.apache.mina.filter.codec.ProtocolDecoderOutput;
7 | //
8 | //import com.xxl.rpc.serialize.Serializer;
9 | //
10 | //public class MinaDecoder extends CumulativeProtocolDecoder {
11 | //
12 | // private Class<?> genericClass;
13 | // private Serializer serializer;
14 | //
15 | // public MinaDecoder(Class<?> genericClass, final Serializer serializer) {
16 | // this.genericClass = genericClass;
17 | // this.serializer = serializer;
18 | // }
19 | //
20 | // @Override
21 | // protected boolean doDecode(IoSession session, IoBuffer in, ProtocolDecoderOutput out) throws Exception {
22 | //
23 | // if (in.remaining() < 4) {
24 | // return false;
25 | // }
26 | // in.mark(); // mark 2 reset, reset call rollback to mark place
27 | //
28 | // int dataLength = in.getInt(); // data length
29 | // if (dataLength < in.remaining()) {
30 | // return false;
31 | // }
32 | // if (dataLength > in.remaining()) {
33 | // in.reset();
34 | // }
35 | //
36 | // byte[] datas = new byte[dataLength]; // data
37 | // in.get(datas, 0, dataLength);
38 | //
39 | // Object obj = serializer.deserialize(datas, genericClass);
40 | // out.write(obj);
41 | // return true;
42 | // }
43 | //
44 | //}
45 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/old/remoting/net/impl/mina/codec/MinaEncoder.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.old.remoting.net.impl.mina.codec;//package com.xxl.rpc.remoting.net.impl.mina.codec;
2 | //
3 | //import org.apache.mina.core.buffer.IoBuffer;
4 | //import org.apache.mina.core.session.IoSession;
5 | //import org.apache.mina.filter.codec.ProtocolEncoder;
6 | //import org.apache.mina.filter.codec.ProtocolEncoderOutput;
7 | //
8 | //import com.xxl.rpc.serialize.Serializer;
9 | //
10 | //public class MinaEncoder implements ProtocolEncoder {
11 | //
12 | // private Class<?> genericClass;
13 | // private Serializer serializer;
14 | //
15 | // public MinaEncoder(Class<?> genericClass, final Serializer serializer) {
16 | // this.genericClass = genericClass;
17 | // this.serializer = serializer;
18 | // }
19 | //
20 | // @Override
21 | // public void encode(IoSession session, Object message, ProtocolEncoderOutput out) throws Exception {
22 | // if (genericClass.isInstance(message)) {
23 | // byte[] datas = serializer.serialize(message);
24 | //
25 | // IoBuffer buffer = IoBuffer.allocate(256);
26 | // buffer.setAutoExpand(true);
27 | // buffer.setAutoShrink(true);
28 | //
29 | // buffer.putInt(datas.length);
30 | // buffer.put(datas);
31 | //
32 | // buffer.flip();
33 | // session.write(buffer);
34 | // }
35 | // }
36 | //
37 | // @Override
38 | // public void dispose(IoSession session) throws Exception {
39 | //
40 | // }
41 | //
42 | //}
43 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/registry/ServiceRegistry.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.registry;
2 |
3 |
4 | import java.util.Map;
5 | import java.util.Set;
6 | import java.util.TreeSet;
7 |
8 | /**
9 | * service registry
10 | *
11 | * /xxl-rpc/dev/
12 | * - key01(service01)
13 | * - value01 (ip:port01)
14 | * - value02 (ip:port02)
15 | *
16 | * @author xuxueli 2018-10-17
17 | */
18 | public abstract class ServiceRegistry {
19 |
20 | /**
21 | * start
22 | */
23 | public abstract void start(Map<String, String> param);
24 |
25 | /**
26 | * start
27 | */
28 | public abstract void stop();
29 |
30 |
31 | /**
32 | * registry service, for mult
33 | *
34 | * @param keys service key
35 | * @param value service value/ip:port
36 | * @return
37 | */
38 | public abstract boolean registry(Set<String> keys, String value);
39 |
40 |
41 | /**
42 | * remove service, for mult
43 | *
44 | * @param keys
45 | * @param value
46 | * @return
47 | */
48 | public abstract boolean remove(Set<String> keys, String value);
49 |
50 | /**
51 | * discovery services, for mult
52 | *
53 | * @param keys
54 | * @return
55 | */
56 | public abstract Map<String, TreeSet<String>> discovery(Set<String> keys);
57 |
58 | /**
59 | * discovery service, for one
60 | *
61 | * @param key service key
62 | * @return service value/ip:port
63 | */
64 | public abstract TreeSet<String> discovery(String key);
65 |
66 | }
67 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/invoker/annotation/XxlRpcReference.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.invoker.annotation;
2 |
3 | import com.wugui.datax.rpc.remoting.invoker.call.CallType;
4 | import com.wugui.datax.rpc.remoting.invoker.route.LoadBalance;
5 | import com.wugui.datax.rpc.remoting.net.Client;
6 | import com.wugui.datax.rpc.remoting.net.impl.netty.client.NettyClient;
7 | import com.wugui.datax.rpc.serialize.Serializer;
8 | import com.wugui.datax.rpc.serialize.impl.HessianSerializer;
9 |
10 | import java.lang.annotation.*;
11 |
12 | /**
13 | * rpc service annotation, skeleton of stub ("@Inherited" allow service use "Transactional")
14 | *
15 | * @author 2015-10-29 19:44:33
16 | */
17 | @Target({ElementType.FIELD})
18 | @Retention(RetentionPolicy.RUNTIME)
19 | @Inherited
20 | public @interface XxlRpcReference {
21 |
22 | Class<? extends Client> client() default NettyClient.class;
23 |
24 | Class<? extends Serializer> serializer() default HessianSerializer.class;
25 |
26 | CallType callType() default CallType.SYNC;
27 |
28 | LoadBalance loadBalance() default LoadBalance.ROUND;
29 |
30 | //Class<?> iface;
31 | String version() default "";
32 |
33 | long timeout() default 1000;
34 |
35 | String address() default "";
36 |
37 | String accessToken() default "";
38 |
39 | //XxlRpcInvokeCallback invokeCallback() ;
40 |
41 | }
42 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/invoker/call/CallType.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.invoker.call;
2 |
3 | /**
4 | * rpc call type
5 | *
6 | * @author xuxueli 2018-10-19
7 | */
8 | public enum CallType {
9 |
10 |
11 | SYNC,
12 |
13 | FUTURE,
14 |
15 | CALLBACK,
16 |
17 | ONEWAY;
18 |
19 |
20 | public static CallType match(String name, CallType defaultCallType){
21 | for (CallType item : CallType.values()) {
22 | if (item.name().equals(name)) {
23 | return item;
24 | }
25 | }
26 | return defaultCallType;
27 | }
28 |
29 | }
30 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/invoker/call/XxlRpcInvokeCallback.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.invoker.call;
2 |
3 |
4 | /**
5 | * @author xuxueli 2018-10-23
6 | */
7 | public abstract class XxlRpcInvokeCallback<T> {
8 |
9 | public abstract void onSuccess(T result);
10 |
11 | public abstract void onFailure(Throwable exception);
12 |
13 |
14 | // ---------------------- thread invoke callback ----------------------
15 |
16 | private static ThreadLocal<XxlRpcInvokeCallback> threadInvokerFuture = new ThreadLocal<XxlRpcInvokeCallback>();
17 |
18 | /**
19 | * get callback
20 | *
21 | * @return
22 | */
23 | public static XxlRpcInvokeCallback getCallback() {
24 | XxlRpcInvokeCallback invokeCallback = threadInvokerFuture.get();
25 | threadInvokerFuture.remove();
26 | return invokeCallback;
27 | }
28 |
29 | /**
30 | * set future
31 | *
32 | * @param invokeCallback
33 | */
34 | public static void setCallback(XxlRpcInvokeCallback invokeCallback) {
35 | threadInvokerFuture.set(invokeCallback);
36 | }
37 |
38 | /**
39 | * remove future
40 | */
41 | public static void removeCallback() {
42 | threadInvokerFuture.remove();
43 | }
44 |
45 |
46 | }
47 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/invoker/generic/XxlRpcGenericService.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.invoker.generic;
2 |
3 | /**
4 | * @author xuxueli 2018-12-04
5 | */
6 | public interface XxlRpcGenericService {
7 |
8 | /**
9 | * generic invoke
10 | *
11 | * @param iface iface name
12 | * @param version iface version
13 | * @param method method name
14 | * @param parameterTypes parameter types, limit base type like "int、java.lang.Integer、java.util.List、java.util.Map ..."
15 | * @param args
16 | * @return
17 | */
18 | Object invoke(String iface, String version, String method, String[] parameterTypes, Object[] args);
19 |
20 | }
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/invoker/reference/impl/XxlRpcSpringReferenceBean.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.invoker.reference.impl;
2 |
3 | import com.wugui.datax.rpc.remoting.invoker.reference.XxlRpcReferenceBean;
4 | import org.springframework.beans.factory.FactoryBean;
5 | import org.springframework.beans.factory.InitializingBean;
6 |
7 | /**
8 | * rpc reference bean, use by spring xml and annotation (for spring)
9 | *
10 | * @author xuxueli 2015-10-29 20:18:32
11 | */
12 | public class XxlRpcSpringReferenceBean implements FactoryBean<Object>, InitializingBean {
13 |
14 |
15 | // ---------------------- util ----------------------
16 |
17 | private XxlRpcReferenceBean xxlRpcReferenceBean;
18 |
19 | @Override
20 | public void afterPropertiesSet() {
21 |
22 | // init config
23 | this.xxlRpcReferenceBean = new XxlRpcReferenceBean();
24 | }
25 |
26 |
27 | @Override
28 | public Object getObject() throws Exception {
29 | return xxlRpcReferenceBean.getObject();
30 | }
31 |
32 | @Override
33 | public Class<?> getObjectType() {
34 | return xxlRpcReferenceBean.getIface();
35 | }
36 |
37 | @Override
38 | public boolean isSingleton() {
39 | return false;
40 | }
41 |
42 |
43 | /**
44 | * public static <T> ClientProxy ClientProxy<T> getFuture(Class<T> type) {
45 | * <T> ClientProxy proxy = (<T>) new ClientProxy();
46 | * return proxy;
47 | * }
48 | */
49 |
50 |
51 | }
52 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/invoker/route/XxlRpcLoadBalance.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.invoker.route;
2 |
3 | import java.util.TreeSet;
4 |
5 | /**
6 | * 分组下机器地址相同，不同JOB均匀散列在不同机器上，保证分组下机器分配JOB平均；且每个JOB固定调度其中一台机器；
7 | * a、virtual node：解决不均衡问题
8 | * b、hash method replace hashCode：String的hashCode可能重复，需要进一步扩大hashCode的取值范围
9 | *
10 | * @author xuxueli 2018-12-04
11 | */
12 | public abstract class XxlRpcLoadBalance {
13 |
14 | public abstract String route(String serviceKey, TreeSet<String> addressSet);
15 |
16 | }
17 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/invoker/route/impl/XxlRpcLoadBalanceRandomStrategy.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.invoker.route.impl;
2 |
3 | import com.wugui.datax.rpc.remoting.invoker.route.XxlRpcLoadBalance;
4 |
5 | import java.util.Random;
6 | import java.util.TreeSet;
7 |
8 | /**
9 | * random
10 | *
11 | * @author xuxueli 2018-12-04
12 | */
13 | public class XxlRpcLoadBalanceRandomStrategy extends XxlRpcLoadBalance {
14 |
15 | private Random random = new Random();
16 |
17 | @Override
18 | public String route(String serviceKey, TreeSet<String> addressSet) {
19 | // arr
20 | String[] addressArr = addressSet.toArray(new String[addressSet.size()]);
21 |
22 | // random
23 | String finalAddress = addressArr[random.nextInt(addressSet.size())];
24 | return finalAddress;
25 | }
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/invoker/route/impl/XxlRpcLoadBalanceRoundStrategy.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.invoker.route.impl;
2 |
3 | import com.wugui.datax.rpc.remoting.invoker.route.XxlRpcLoadBalance;
4 |
5 | import java.util.Random;
6 | import java.util.TreeSet;
7 | import java.util.concurrent.ConcurrentHashMap;
8 | import java.util.concurrent.ConcurrentMap;
9 |
10 | /**
11 | * round
12 | *
13 | * @author xuxueli 2018-12-04
14 | */
15 | public class XxlRpcLoadBalanceRoundStrategy extends XxlRpcLoadBalance {
16 |
17 | private ConcurrentMap<String, Integer> routeCountEachJob = new ConcurrentHashMap<String, Integer>();
18 | private long CACHE_VALID_TIME = 0;
19 | private int count(String serviceKey) {
20 | // cache clear
21 | if (System.currentTimeMillis() > CACHE_VALID_TIME) {
22 | routeCountEachJob.clear();
23 | CACHE_VALID_TIME = System.currentTimeMillis() + 24*60*60*1000;
24 | }
25 |
26 | // count++
27 | Integer count = routeCountEachJob.get(serviceKey);
28 | count = (count==null || count>1000000)?(new Random().nextInt(100)):++count; // 初始化时主动Random一次，缓解首次压力
29 | routeCountEachJob.put(serviceKey, count);
30 | return count;
31 | }
32 |
33 | @Override
34 | public String route(String serviceKey, TreeSet<String> addressSet) {
35 | // arr
36 | String[] addressArr = addressSet.toArray(new String[addressSet.size()]);
37 |
38 | // round
39 | String finalAddress = addressArr[count(serviceKey)%addressArr.length];
40 | return finalAddress;
41 | }
42 |
43 | }
44 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/Client.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net;
2 |
3 | import com.wugui.datax.rpc.remoting.invoker.reference.XxlRpcReferenceBean;
4 | import com.wugui.datax.rpc.remoting.net.params.XxlRpcRequest;
5 | import org.slf4j.Logger;
6 | import org.slf4j.LoggerFactory;
7 |
8 | /**
9 | * i client
10 | * @author xuxueli 2015-11-24 22:18:10
11 | */
12 | public abstract class Client {
13 | protected static final Logger logger = LoggerFactory.getLogger(Client.class);
14 |
15 |
16 | // ---------------------- init ----------------------
17 |
18 | protected volatile XxlRpcReferenceBean xxlRpcReferenceBean;
19 |
20 | public void init(XxlRpcReferenceBean xxlRpcReferenceBean) {
21 | this.xxlRpcReferenceBean = xxlRpcReferenceBean;
22 | }
23 |
24 |
25 | // ---------------------- send ----------------------
26 |
27 | /**
28 | * async send, bind requestId and future-response
29 | *
30 | * @param address
31 | * @param xxlRpcRequest
32 | * @return
33 | * @throws Exception
34 | */
35 | public abstract void asyncSend(String address, XxlRpcRequest xxlRpcRequest) throws Exception;
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/Server.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net;
2 |
3 | import com.wugui.datax.rpc.remoting.net.params.BaseCallback;
4 | import com.wugui.datax.rpc.remoting.provider.XxlRpcProviderFactory;
5 | import org.slf4j.Logger;
6 | import org.slf4j.LoggerFactory;
7 |
8 | /**
9 | * server
10 | *
11 | * @author xuxueli 2015-11-24 20:59:49
12 | */
13 | public abstract class Server {
14 | protected static final Logger logger = LoggerFactory.getLogger(Server.class);
15 |
16 |
17 | private BaseCallback startedCallback;
18 | private BaseCallback stopedCallback;
19 |
20 | public void setStartedCallback(BaseCallback startedCallback) {
21 | this.startedCallback = startedCallback;
22 | }
23 |
24 | public void setStopedCallback(BaseCallback stopedCallback) {
25 | this.stopedCallback = stopedCallback;
26 | }
27 |
28 |
29 | /**
30 | * start server
31 | *
32 | * @param xxlRpcProviderFactory
33 | * @throws Exception
34 | */
35 | public abstract void start(final XxlRpcProviderFactory xxlRpcProviderFactory) throws Exception;
36 |
37 | /**
38 | * callback when started
39 | */
40 | public void onStarted() {
41 | if (startedCallback != null) {
42 | try {
43 | startedCallback.run();
44 | } catch (Exception e) {
45 | logger.error(">>>>>>>>>>> xxl-rpc, server startedCallback error.", e);
46 | }
47 | }
48 | }
49 |
50 | /**
51 | * stop server
52 | *
53 | * @throws Exception
54 | */
55 | public abstract void stop() throws Exception;
56 |
57 | /**
58 | * callback when stoped
59 | */
60 | public void onStopped() {
61 | if (stopedCallback != null) {
62 | try {
63 | stopedCallback.run();
64 | } catch (Exception e) {
65 | logger.error(">>>>>>>>>>> xxl-rpc, server stopedCallback error.", e);
66 | }
67 | }
68 | }
69 |
70 | }
71 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/common/NettyConstant.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net.common;
2 |
3 | public class NettyConstant {
4 |
5 | public static int MAX_LENGTH = 20 * 1024 * 1024;
6 | }
7 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/impl/netty/client/NettyClient.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net.impl.netty.client;
2 |
3 | import com.wugui.datax.rpc.remoting.net.Client;
4 | import com.wugui.datax.rpc.remoting.net.common.ConnectClient;
5 | import com.wugui.datax.rpc.remoting.net.params.XxlRpcRequest;
6 |
7 | /**
8 | * netty client
9 | *
10 | * @author xuxueli 2015-11-24 22:25:15
11 | */
12 | public class NettyClient extends Client {
13 |
14 | private Class<? extends ConnectClient> connectClientImpl = NettyConnectClient.class;
15 |
16 | @Override
17 | public void asyncSend(String address, XxlRpcRequest xxlRpcRequest) throws Exception {
18 | ConnectClient.asyncSend(xxlRpcRequest, address, connectClientImpl, xxlRpcReferenceBean);
19 | }
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/impl/netty/codec/NettyDecoder.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net.impl.netty.codec;
2 |
3 | import com.wugui.datax.rpc.serialize.Serializer;
4 | import io.netty.buffer.ByteBuf;
5 | import io.netty.channel.ChannelHandlerContext;
6 | import io.netty.handler.codec.ByteToMessageDecoder;
7 |
8 | import java.util.List;
9 |
10 | /**
11 | * decoder
12 | *
13 | * @author xuxueli 2015-10-29 19:02:36
14 | */
15 | public class NettyDecoder extends ByteToMessageDecoder {
16 |
17 | private Class<?> genericClass;
18 | private Serializer serializer;
19 |
20 | public NettyDecoder(Class<?> genericClass, final Serializer serializer) {
21 | this.genericClass = genericClass;
22 | this.serializer = serializer;
23 | }
24 |
25 | @Override
26 | public final void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) throws Exception {
27 | if (in.readableBytes() < 4) {
28 | return;
29 | }
30 | in.markReaderIndex();
31 | int dataLength = in.readInt();
32 | if (dataLength < 0) {
33 | ctx.close();
34 | }
35 | if (in.readableBytes() < dataLength) {
36 | in.resetReaderIndex();
37 | return; // fix 1024k buffer splice limix
38 | }
39 | byte[] data = new byte[dataLength];
40 | in.readBytes(data);
41 |
42 | Object obj = serializer.deserialize(data, genericClass);
43 | out.add(obj);
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/impl/netty/codec/NettyEncoder.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net.impl.netty.codec;
2 |
3 | import com.wugui.datax.rpc.serialize.Serializer;
4 | import io.netty.buffer.ByteBuf;
5 | import io.netty.channel.ChannelHandlerContext;
6 | import io.netty.handler.codec.MessageToByteEncoder;
7 |
8 | /**
9 | * encoder
10 | *
11 | * @author xuxueli 2015-10-29 19:43:00
12 | */
13 | public class NettyEncoder extends MessageToByteEncoder<Object> {
14 |
15 | private Class<?> genericClass;
16 | private Serializer serializer;
17 |
18 | public NettyEncoder(Class<?> genericClass, final Serializer serializer) {
19 | this.genericClass = genericClass;
20 | this.serializer = serializer;
21 | }
22 |
23 | @Override
24 | public void encode(ChannelHandlerContext ctx, Object in, ByteBuf out) throws Exception {
25 | if (genericClass.isInstance(in)) {
26 | byte[] data = serializer.serialize(in);
27 | out.writeInt(data.length);
28 | out.writeBytes(data);
29 | }
30 | }
31 | }
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/impl/netty_http/client/NettyHttpClient.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net.impl.netty_http.client;
2 |
3 | import com.wugui.datax.rpc.remoting.net.Client;
4 | import com.wugui.datax.rpc.remoting.net.common.ConnectClient;
5 | import com.wugui.datax.rpc.remoting.net.params.XxlRpcRequest;
6 |
7 | /**
8 | * netty_http client
9 | *
10 | * @author xuxueli 2015-11-24 22:25:15
11 | */
12 | public class NettyHttpClient extends Client {
13 |
14 | private Class<? extends ConnectClient> connectClientImpl = NettyHttpConnectClient.class;
15 |
16 | @Override
17 | public void asyncSend(String address, XxlRpcRequest xxlRpcRequest) throws Exception {
18 | ConnectClient.asyncSend(xxlRpcRequest, address, connectClientImpl, xxlRpcReferenceBean);
19 | }
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/params/BaseCallback.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net.params;
2 |
3 | /**
4 | * @author xuxueli 2018-10-19
5 | */
6 | public abstract class BaseCallback {
7 |
8 | public abstract void run() throws Exception;
9 |
10 | }
11 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/params/Beat.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net.params;
2 |
3 | /**
4 | * beat for keep-alive
5 | *
6 | * @author xuxueli 2019-09-27
7 | */
8 | public final class Beat {
9 |
10 | public static final int BEAT_INTERVAL = 30;
11 | public static final String BEAT_ID = "BEAT_PING_PONG";
12 |
13 | public static XxlRpcRequest BEAT_PING;
14 |
15 | static {
16 | BEAT_PING = new XxlRpcRequest(){};
17 | BEAT_PING.setRequestId(BEAT_ID);
18 | }
19 |
20 | }
21 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/net/params/XxlRpcResponse.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.net.params;
2 |
3 | import java.io.Serializable;
4 |
5 | /**
6 | * response
7 | *
8 | * @author xuxueli 2015-10-29 19:39:54
9 | */
10 | public class XxlRpcResponse implements Serializable {
11 | private static final long serialVersionUID = 42L;
12 |
13 |
14 | private String requestId;
15 | private String errorMsg;
16 | private Object result;
17 |
18 |
19 | public String getRequestId() {
20 | return requestId;
21 | }
22 |
23 | public void setRequestId(String requestId) {
24 | this.requestId = requestId;
25 | }
26 |
27 | public String getErrorMsg() {
28 | return errorMsg;
29 | }
30 |
31 | public void setErrorMsg(String errorMsg) {
32 | this.errorMsg = errorMsg;
33 | }
34 |
35 | public Object getResult() {
36 | return result;
37 | }
38 |
39 | public void setResult(Object result) {
40 | this.result = result;
41 | }
42 |
43 | @Override
44 | public String toString() {
45 | return "XxlRpcResponse{" +
46 | "requestId='" + requestId + '\'' +
47 | ", errorMsg='" + errorMsg + '\'' +
48 | ", result=" + result +
49 | '}';
50 | }
51 |
52 | }
53 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/remoting/provider/annotation/XxlRpcService.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.remoting.provider.annotation;
2 |
3 | import java.lang.annotation.*;
4 |
5 | /**
6 | * rpc service annotation, skeleton of stub ("@Inherited" allow service use "Transactional")
7 | *
8 | * @author 2015-10-29 19:44:33
9 | */
10 | @Target({ElementType.TYPE})
11 | @Retention(RetentionPolicy.RUNTIME)
12 | @Inherited
13 | public @interface XxlRpcService {
14 |
15 | /**
16 | * @return
17 | */
18 | String version() default "";
19 |
20 | }
21 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/serialize/Serializer.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.serialize;
2 |
3 | /**
4 | * serializer
5 | *
6 | * Tips：模板方法模式：定义一个操作中算法的骨架（或称为顶级逻辑），将一些步骤（或称为基本方法）的执行延迟到其子类中；
7 | * Tips：基本方法：抽象方法 + 具体方法final + 钩子方法；
8 | * Tips：Enum 时最好的单例方案；枚举单例会初始化全部实现，此处改为托管Class，避免无效的实例化；
9 | *
10 | * @author xuxueli 2015-10-30 21:02:55
11 | */
12 | public abstract class Serializer {
13 |
14 | public abstract <T> byte[] serialize(T obj);
15 | public abstract <T> Object deserialize(byte[] bytes, Class<T> clazz);
16 |
17 | /*public enum SerializeEnum {
18 | HESSIAN(HessianSerializer.class),
19 | HESSIAN1(Hessian1Serializer.class);
20 |
21 | private Class<? extends Serializer> serializerClass;
22 | private SerializeEnum (Class<? extends Serializer> serializerClass) {
23 | this.serializerClass = serializerClass;
24 | }
25 |
26 | public Serializer getSerializer() {
27 | try {
28 | return serializerClass.newInstance();
29 | } catch (Exception e) {
30 | throw new XxlRpcException(e);
31 | }
32 | }
33 |
34 | public static SerializeEnum match(String name, SerializeEnum defaultSerializer){
35 | for (SerializeEnum item : SerializeEnum.values()) {
36 | if (item.name().equals(name)) {
37 | return item;
38 | }
39 | }
40 | return defaultSerializer;
41 | }
42 | }*/
43 |
44 | }
45 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/serialize/impl/HessianSerializer.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.serialize.impl;
2 |
3 | import com.caucho.hessian.io.Hessian2Input;
4 | import com.caucho.hessian.io.Hessian2Output;
5 | import com.wugui.datax.rpc.serialize.Serializer;
6 | import com.wugui.datax.rpc.util.XxlRpcException;
7 |
8 | import java.io.ByteArrayInputStream;
9 | import java.io.ByteArrayOutputStream;
10 | import java.io.IOException;
11 |
12 | /**
13 | * hessian serialize
14 | * @author xuxueli 2015-9-26 02:53:29
15 | */
16 | public class HessianSerializer extends Serializer {
17 |
18 | @Override
19 | public <T> byte[] serialize(T obj){
20 | ByteArrayOutputStream os = new ByteArrayOutputStream();
21 | Hessian2Output ho = new Hessian2Output(os);
22 | try {
23 | ho.writeObject(obj);
24 | ho.flush();
25 | byte[] result = os.toByteArray();
26 | return result;
27 | } catch (IOException e) {
28 | throw new XxlRpcException(e);
29 | } finally {
30 | try {
31 | ho.close();
32 | } catch (IOException e) {
33 | throw new XxlRpcException(e);
34 | }
35 | try {
36 | os.close();
37 | } catch (IOException e) {
38 | throw new XxlRpcException(e);
39 | }
40 | }
41 |
42 | }
43 |
44 | @Override
45 | public <T> Object deserialize(byte[] bytes, Class<T> clazz) {
46 | ByteArrayInputStream is = new ByteArrayInputStream(bytes);
47 | Hessian2Input hi = new Hessian2Input(is);
48 | try {
49 | Object result = hi.readObject();
50 | return result;
51 | } catch (IOException e) {
52 | throw new XxlRpcException(e);
53 | } finally {
54 | try {
55 | hi.close();
56 | } catch (Exception e) {
57 | throw new XxlRpcException(e);
58 | }
59 | try {
60 | is.close();
61 | } catch (IOException e) {
62 | throw new XxlRpcException(e);
63 | }
64 | }
65 | }
66 |
67 | }
68 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/util/ClassUtil.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.util;
2 |
3 | import java.util.HashMap;
4 |
5 | /**
6 | * @author xuxueli 2019-02-19
7 | */
8 | public class ClassUtil {
9 |
10 | private static final HashMap<String, Class<?>> primClasses = new HashMap<>();
11 |
12 | static {
13 | primClasses.put("boolean", boolean.class);
14 | primClasses.put("byte", byte.class);
15 | primClasses.put("char", char.class);
16 | primClasses.put("short", short.class);
17 | primClasses.put("int", int.class);
18 | primClasses.put("long", long.class);
19 | primClasses.put("float", float.class);
20 | primClasses.put("double", double.class);
21 | primClasses.put("void", void.class);
22 | }
23 |
24 | public static Class<?> resolveClass(String className) throws ClassNotFoundException {
25 | try {
26 | return Class.forName(className);
27 | } catch (ClassNotFoundException ex) {
28 | Class<?> cl = primClasses.get(className);
29 | if (cl != null) {
30 | return cl;
31 | } else {
32 | throw ex;
33 | }
34 | }
35 | }
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/util/ThreadPoolUtil.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.util;
2 |
3 | import java.util.concurrent.*;
4 |
5 | /**
6 | * @author xuxueli 2019-02-18
7 | */
8 | public class ThreadPoolUtil {
9 |
10 | /**
11 | * make server thread pool
12 | *
13 | * @param serverType
14 | * @return
15 | */
16 | public static ThreadPoolExecutor makeServerThreadPool(final String serverType, int corePoolSize, int maxPoolSize) {
17 | ThreadPoolExecutor serverHandlerPool = new ThreadPoolExecutor(
18 | corePoolSize,
19 | maxPoolSize,
20 | 60L,
21 | TimeUnit.SECONDS,
22 | new LinkedBlockingQueue<>(1000),
23 | r -> new Thread(r, "xxl-rpc, " + serverType + "-serverHandlerPool-" + r.hashCode()),
24 | (r, executor) -> {
25 | throw new XxlRpcException("xxl-rpc " + serverType + " Thread pool is EXHAUSTED!");
26 | }); // default maxThreads 300, minThreads 60
27 |
28 | return serverHandlerPool;
29 | }
30 |
31 | }
32 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/util/ThrowableUtil.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.util;
2 |
3 | import java.io.PrintWriter;
4 | import java.io.StringWriter;
5 |
6 | /**
7 | * @author xuxueli 2018-10-20 20:07:26
8 | */
9 | public class ThrowableUtil {
10 |
11 | /**
12 | * parse error to string
13 | *
14 | * @param e
15 | * @return
16 | */
17 | public static String toString(Throwable e) {
18 | StringWriter stringWriter = new StringWriter();
19 | e.printStackTrace(new PrintWriter(stringWriter));
20 | String errorMsg = stringWriter.toString();
21 | return errorMsg;
22 | }
23 |
24 | }
25 |
--------------------------------------------------------------------------------
/datax-rpc/src/main/java/com/wugui/datax/rpc/util/XxlRpcException.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.util;
2 |
3 | /**
4 | * @author xuxueli 2018-10-20 23:00:40
5 | */
6 | public class XxlRpcException extends RuntimeException {
7 |
8 | private static final long serialVersionUID = 42L;
9 |
10 | public XxlRpcException(String msg) {
11 | super(msg);
12 | }
13 |
14 | public XxlRpcException(String msg, Throwable cause) {
15 | super(msg, cause);
16 | }
17 |
18 | public XxlRpcException(Throwable cause) {
19 | super(cause);
20 | }
21 |
22 | }
--------------------------------------------------------------------------------
/datax-rpc/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | log4j.rootLogger=info,console
2 |
3 | # console
4 | log4j.appender.console=org.apache.log4j.ConsoleAppender
5 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
6 | log4j.appender.console.layout.ConversionPattern=%d - %p [%c] - <%m>%n
--------------------------------------------------------------------------------
/datax-rpc/src/test/java/com/wugui/datax/rpc/test/IpUtilTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.test;
2 |
3 | import com.wugui.datax.rpc.util.IpUtil;
4 |
5 | import java.net.UnknownHostException;
6 |
7 | /**
8 | * @author xuxueli 2018-10-23
9 | */
10 | public class IpUtilTest {
11 |
12 | public static void main(String[] args) throws UnknownHostException {
13 | System.out.println(IpUtil.getIp());
14 | System.out.println(IpUtil.getIpPort(8080));
15 | }
16 |
17 | }
18 |
--------------------------------------------------------------------------------
/datax-rpc/src/test/java/com/wugui/datax/rpc/test/SerializerTest.java:
--------------------------------------------------------------------------------
1 | package com.wugui.datax.rpc.test;
2 |
3 | import com.wugui.datax.rpc.serialize.Serializer;
4 | import com.wugui.datax.rpc.serialize.impl.HessianSerializer;
5 |
6 | import java.util.HashMap;
7 | import java.util.Map;
8 |
9 | /**
10 | * @author xuxueli 2015-10-30 21:02:55
11 | */
12 | public class SerializerTest {
13 |
14 | public static void main(String[] args) throws IllegalAccessException, InstantiationException {
15 | Serializer serializer = HessianSerializer.class.newInstance();
16 | System.out.println(serializer);
17 | try {
18 | Map<String, String> map = new HashMap<String, String>();
19 | map.put("aaa", "111");
20 | map.put("bbb", "222");
21 | System.out.println(serializer.deserialize(serializer.serialize("ddddddd"), String.class));
22 | } catch (Exception e) {
23 | e.printStackTrace();
24 | }
25 | }
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/doc/datax-web/idea-start-datax.md:
--------------------------------------------------------------------------------
1 | 
2 |
3 | ## vm option
4 | 需要写上你用maven打包后生成的target目录
5 | ```
6 | -Ddatax.home=/Users/huzekang/openSource/DataX/target/datax/datax
7 | ```
8 | ## program args
9 | ```
10 | -mode standalone -jobid -1 -job /Users/huzekang/openSource/DataX/job-sample/oracle2mysql.json
11 | ```
--------------------------------------------------------------------------------