├── .gitignore ├── README.md ├── common ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── baifendian │ │ │ └── swordfish │ │ │ └── common │ │ │ ├── config │ │ │ └── BaseConfig.java │ │ │ ├── consts │ │ │ └── Constants.java │ │ │ ├── enums │ │ │ ├── ExternalJobType.java │ │ │ ├── FileColumnType.java │ │ │ ├── HiveColumnType.java │ │ │ ├── ImpExpType.java │ │ │ ├── MongoWriteMode.java │ │ │ ├── MysqlWriteMode.java │ │ │ ├── ReaderType.java │ │ │ ├── StormType.java │ │ │ ├── WriteHdfsType.java │ │ │ ├── WriteMode.java │ │ │ └── WriterType.java │ │ │ ├── hadoop │ │ │ ├── ConfigurationUtil.java │ │ │ ├── HdfsClient.java │ │ │ ├── HdfsException.java │ │ │ └── YarnRestClient.java │ │ │ ├── hive │ │ │ ├── metastore │ │ │ │ ├── HiveMetaPoolClient.java │ │ │ │ └── HiveMetaStorePoolFactory.java │ │ │ └── service2 │ │ │ │ ├── HiveService2Client.java │ │ │ │ ├── HiveService2ConnectionInfo.java │ │ │ │ └── HiveService2PoolFactory.java │ │ │ ├── job │ │ │ └── struct │ │ │ │ ├── datasource │ │ │ │ ├── Datasource.java │ │ │ │ ├── DatasourceFactory.java │ │ │ │ ├── FtpDatasource.java │ │ │ │ ├── HBaseDatasource.java │ │ │ │ ├── MongoDatasource.java │ │ │ │ ├── MysqlDatasource.java │ │ │ │ ├── OracleDatasource.java │ │ │ │ └── PostgreDatasource.java │ │ │ │ ├── node │ │ │ │ ├── BaseParam.java │ │ │ │ ├── BaseParamFactory.java │ │ │ │ ├── JobType.java │ │ │ │ ├── adhoc │ │ │ │ │ └── AdHocParam.java │ │ │ │ ├── common │ │ │ │ │ ├── ProcessParam.java │ │ │ │ │ └── UdfsInfo.java │ │ │ │ ├── hql │ │ │ │ │ └── HqlParam.java │ │ │ │ ├── impexp │ │ │ │ │ ├── ImpExpBuilder.java │ │ │ │ │ ├── ImpExpParam.java │ │ │ │ │ ├── column │ │ │ │ │ │ ├── FileColumn.java │ │ │ │ │ │ ├── HiveColumn.java │ │ │ │ │ │ └── MongoColumn.java │ │ │ │ │ ├── reader │ │ │ │ │ │ ├── FileReader.java │ │ │ │ │ │ ├── HdfsReader.java │ │ │ │ │ │ ├── HiveReader.java │ │ │ │ │ │ ├── MysqlReader.java │ │ │ │ │ │ ├── PostgreReader.java │ │ │ │ │ │ ├── Reader.java │ │ │ │ │ │ └── ReaderFactory.java │ │ │ │ │ ├── setting │ │ │ │ │ │ ├── ErrorLimit.java │ │ │ │ │ │ ├── Setting.java │ │ │ │ │ │ └── Speed.java │ │ │ │ │ └── writer │ │ │ │ │ │ ├── HdfsWriter.java │ │ │ │ │ │ ├── HiveWriter.java │ │ │ │ │ │ ├── MongoWriter.java │ │ │ │ │ │ ├── MysqlWriter.java │ │ │ │ │ │ ├── PostgreWriter.java │ │ │ │ │ │ ├── Writer.java │ │ │ │ │ │ └── WriterFactory.java │ │ │ │ ├── mr │ │ │ │ │ └── MrParam.java │ │ │ │ ├── shell │ │ │ │ │ └── ShellParam.java │ │ │ │ ├── spark │ │ │ │ │ └── SparkParam.java │ │ │ │ ├── storm │ │ │ │ │ ├── StormBuilder.java │ │ │ │ │ ├── StormParam.java │ │ │ │ │ ├── dto │ │ │ │ │ │ ├── TopologyDto.java │ │ │ │ │ │ ├── TopologyInfoDto.java │ │ │ │ │ │ ├── TopologyOperationDto.java │ │ │ │ │ │ ├── TopologySummaryDto.java │ │ │ │ │ │ └── TopologyWorkerDto.java │ │ │ │ │ └── param │ │ │ │ │ │ ├── IStormParam.java │ │ │ │ │ │ ├── StormJarParam.java │ │ │ │ │ │ ├── StormParamFactory.java │ │ │ │ │ │ ├── StormShellParam.java │ │ │ │ │ │ └── StormSqlParam.java │ │ │ │ └── virtual │ │ │ │ │ └── VirtualParam.java │ │ │ │ └── resource │ │ │ │ ├── ResScope.java │ │ │ │ └── ResourceInfo.java │ │ │ ├── json │ │ │ └── JsonOrdinalSerializer.java │ │ │ ├── mail │ │ │ ├── EmailManager.java │ │ │ └── MailSendUtil.java │ │ │ ├── search │ │ │ └── EsSearch.java │ │ │ ├── storm │ │ │ └── StormRestUtil.java │ │ │ └── utils │ │ │ ├── CommonUtil.java │ │ │ ├── DateUtils.java │ │ │ ├── NanoSampleConverter.java │ │ │ ├── PermUtil.java │ │ │ ├── StructuredArguments.java │ │ │ ├── ThriftUtil.java │ │ │ ├── VerifyUtil.java │ │ │ ├── graph │ │ │ ├── DAGGraph.java │ │ │ ├── Graph.java │ │ │ └── VertexKey.java │ │ │ └── http │ │ │ └── HttpUtil.java │ └── resources │ │ ├── bin │ │ └── swordfish-daemon.sh │ │ └── common │ │ ├── base_config.properties │ │ ├── hadoop │ │ └── hadoop.properties │ │ ├── mail.properties │ │ ├── search.properties │ │ └── storm.properties │ └── test │ └── java │ └── com │ └── baifendian │ └── swordfish │ └── common │ ├── hadoop │ └── HdfsClientTest.java │ ├── job │ └── struct │ │ ├── datasource │ │ ├── FtpDatasourceTest.java │ │ ├── OracleDatasourceTest.java │ │ └── PostgreDatasourceTest.java │ │ └── node │ │ └── BaseParamFactoryTest.java │ ├── mail │ └── MailSendUtilTest.java │ └── utils │ ├── VerifyUtilTest.java │ └── graph │ ├── DAGGraphTest.java │ └── GraphTest.java ├── dao ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── baifendian │ │ │ └── swordfish │ │ │ └── dao │ │ │ ├── AdHocDao.java │ │ │ ├── BaseDao.java │ │ │ ├── DaoFactory.java │ │ │ ├── DatasourceDao.java │ │ │ ├── FlowDao.java │ │ │ ├── MasterDao.java │ │ │ ├── StreamingDao.java │ │ │ ├── datasource │ │ │ ├── ConnectionFactory.java │ │ │ └── DatabaseConfiguration.java │ │ │ ├── enums │ │ │ ├── DbType.java │ │ │ ├── DepPolicyType.java │ │ │ ├── ExecType.java │ │ │ ├── FailurePolicyType.java │ │ │ ├── FlowStatus.java │ │ │ ├── NodeDepType.java │ │ │ ├── NotifyType.java │ │ │ ├── ScheduleStatus.java │ │ │ ├── ScheduleType.java │ │ │ ├── SqlEngineType.java │ │ │ └── UserRoleType.java │ │ │ ├── exception │ │ │ ├── DagException.java │ │ │ ├── DaoSemanticException.java │ │ │ └── SqlException.java │ │ │ ├── mapper │ │ │ ├── AdHocMapper.java │ │ │ ├── AdHocMapperProvider.java │ │ │ ├── AdHocResultMapper.java │ │ │ ├── AdHocResultMapperProvider.java │ │ │ ├── DataSourceMapper.java │ │ │ ├── DataSourceMapperProvider.java │ │ │ ├── ExecutionFlowMapper.java │ │ │ ├── ExecutionFlowMapperProvider.java │ │ │ ├── ExecutionNodeMapper.java │ │ │ ├── ExecutionNodeMapperProvider.java │ │ │ ├── FlowNodeMapper.java │ │ │ ├── FlowNodeMapperProvider.java │ │ │ ├── MasterServerMapper.java │ │ │ ├── MasterServerMapperProvider.java │ │ │ ├── ProjectFlowMapper.java │ │ │ ├── ProjectFlowMapperProvider.java │ │ │ ├── ProjectMapper.java │ │ │ ├── ProjectMapperProvider.java │ │ │ ├── ProjectUserMapper.java │ │ │ ├── ProjectUserMapperProvider.java │ │ │ ├── ResourceMapper.java │ │ │ ├── ResourceMapperProvider.java │ │ │ ├── ScheduleMapper.java │ │ │ ├── ScheduleMapperProvider.java │ │ │ ├── SessionMapper.java │ │ │ ├── SessionMapperProvider.java │ │ │ ├── StreamingJobMapper.java │ │ │ ├── StreamingJobMapperProvider.java │ │ │ ├── StreamingResultMapper.java │ │ │ ├── StreamingResultMapperProvider.java │ │ │ ├── UserMapper.java │ │ │ ├── UserMapperProvider.java │ │ │ └── utils │ │ │ │ ├── EnumFieldUtil.java │ │ │ │ └── EqualUtils.java │ │ │ ├── model │ │ │ ├── AdHoc.java │ │ │ ├── AdHocJsonObject.java │ │ │ ├── AdHocResult.java │ │ │ ├── DataSource.java │ │ │ ├── ExecutionFlow.java │ │ │ ├── ExecutionFlowError.java │ │ │ ├── ExecutionNode.java │ │ │ ├── ExecutionState.java │ │ │ ├── FlowNode.java │ │ │ ├── FlowNodeRelation.java │ │ │ ├── MasterServer.java │ │ │ ├── Project.java │ │ │ ├── ProjectFlow.java │ │ │ ├── ProjectUser.java │ │ │ ├── Resource.java │ │ │ ├── Schedule.java │ │ │ ├── Session.java │ │ │ ├── StreamingJob.java │ │ │ ├── StreamingResult.java │ │ │ ├── User.java │ │ │ └── flow │ │ │ │ ├── DepWorkflow.java │ │ │ │ ├── FlowDag.java │ │ │ │ └── Property.java │ │ │ └── utils │ │ │ ├── DagHelper.java │ │ │ └── json │ │ │ ├── JsonObjectDeserializer.java │ │ │ ├── JsonObjectSerializer.java │ │ │ └── JsonUtil.java │ └── resources │ │ └── dao │ │ └── data_source.properties │ └── test │ └── java │ └── com │ └── baifendian │ └── swordfish │ └── dao │ ├── AdHocDaoTest.java │ ├── FlowDaoTest.java │ ├── datasource │ └── DatabaseConfigurationTest.java │ ├── mapper │ ├── FlowNodeMapperTest.java │ ├── ProjectFlowMapperTest.java │ └── ScheduleMapperTest.java │ ├── model │ ├── FlowNodeTest.java │ └── flow │ │ └── FlowDagTest.java │ └── utils │ └── DagHelperTest.java ├── docs ├── architecture.png ├── dependency_policy.png └── fault_tolerance.png ├── exec-server ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── package.xml │ ├── java │ │ └── com │ │ │ └── baifendian │ │ │ └── swordfish │ │ │ └── execserver │ │ │ ├── ExecThriftServer.java │ │ │ ├── common │ │ │ ├── ExecResult.java │ │ │ ├── FunctionUtil.java │ │ │ └── ResultCallback.java │ │ │ ├── engine │ │ │ ├── SqlUtil.java │ │ │ ├── hive │ │ │ │ ├── HiveConfig.java │ │ │ │ ├── HiveMetaExec.java │ │ │ │ ├── HiveSqlExec.java │ │ │ │ ├── HiveUtil.java │ │ │ │ └── MyHiveFactoryUtil.java │ │ │ └── phoenix │ │ │ │ ├── PhoenixSqlExec.java │ │ │ │ └── PhoenixUtil.java │ │ │ ├── exception │ │ │ ├── ConfigException.java │ │ │ ├── ExecException.java │ │ │ └── ExecTimeoutException.java │ │ │ ├── job │ │ │ ├── AbstractStormProcessJob.java │ │ │ ├── AbstractYarnJob.java │ │ │ ├── AbstractYarnProcessJob.java │ │ │ ├── Job.java │ │ │ ├── JobContext.java │ │ │ ├── JobManager.java │ │ │ ├── JobProps.java │ │ │ ├── ProcessJob.java │ │ │ ├── impexp │ │ │ │ ├── Args │ │ │ │ │ ├── HdfsWriterArg.java │ │ │ │ │ ├── HiveReaderArg.java │ │ │ │ │ ├── HqlColumn.java │ │ │ │ │ ├── ImpExpProps.java │ │ │ │ │ ├── MongoWriterArg.java │ │ │ │ │ ├── MysqlReaderArg.java │ │ │ │ │ ├── MysqlWriterArg.java │ │ │ │ │ ├── PostgreReaderArg.java │ │ │ │ │ ├── PostgreWriterArg.java │ │ │ │ │ ├── ReaderArg.java │ │ │ │ │ └── WriterArg.java │ │ │ │ ├── DataXJob.java │ │ │ │ ├── FileToHiveJob.java │ │ │ │ ├── HiveToMongoJob.java │ │ │ │ ├── HiveToMysqlJob.java │ │ │ │ ├── HiveToPostgreJob.java │ │ │ │ ├── ImpExpJobConst.java │ │ │ │ ├── ImpExpJobManager.java │ │ │ │ ├── ImpExpUtil.java │ │ │ │ ├── MysqlToHdfsJob.java │ │ │ │ ├── MysqlToHiveJob.java │ │ │ │ ├── PostgreToHdfsJob.java │ │ │ │ ├── PostgreToHiveJob.java │ │ │ │ └── WriteHiveJob.java │ │ │ ├── mr │ │ │ │ ├── HadoopJarArgsConst.java │ │ │ │ ├── HadoopJarArgsUtil.java │ │ │ │ └── MrJob.java │ │ │ ├── shell │ │ │ │ └── ShellJob.java │ │ │ ├── spark │ │ │ │ ├── SparkJob.java │ │ │ │ ├── SparkSubmitArgsConst.java │ │ │ │ └── SparkSubmitArgsUtil.java │ │ │ ├── sql │ │ │ │ └── EtlSqlJob.java │ │ │ ├── storm │ │ │ │ ├── StormJob.java │ │ │ │ ├── StormSubmitArgsConst.java │ │ │ │ └── StormSubmitArgsUtil.java │ │ │ └── virtual │ │ │ │ └── VirtualJob.java │ │ │ ├── parameter │ │ │ ├── ParamHelper.java │ │ │ ├── SystemParamManager.java │ │ │ └── placeholder │ │ │ │ ├── CalculateUtil.java │ │ │ │ ├── PlaceholderUtil.java │ │ │ │ └── TimePlaceholderUtil.java │ │ │ ├── runner │ │ │ ├── adhoc │ │ │ │ ├── AdHocRunner.java │ │ │ │ ├── AdHocRunnerManager.java │ │ │ │ └── AdHocSqlJob.java │ │ │ ├── flow │ │ │ │ ├── FlowRunner.java │ │ │ │ ├── FlowRunnerContext.java │ │ │ │ └── FlowRunnerManager.java │ │ │ ├── node │ │ │ │ └── NodeRunner.java │ │ │ └── streaming │ │ │ │ ├── StreamingRunner.java │ │ │ │ └── StreamingRunnerManager.java │ │ │ ├── service │ │ │ └── ExecServiceImpl.java │ │ │ └── utils │ │ │ ├── Constants.java │ │ │ ├── EnvHelper.java │ │ │ ├── JobLogger.java │ │ │ ├── LoggerUtil.java │ │ │ ├── OsUtil.java │ │ │ ├── ProcessUtil.java │ │ │ └── ResultHelper.java │ └── resources │ │ ├── common │ │ ├── hive │ │ │ └── hive.properties │ │ └── phoenix.properties │ │ ├── execserver_logback.xml │ │ └── worker.properties │ └── test │ └── java │ └── com │ └── baifendian │ └── swordfish │ └── execserver │ ├── engine │ └── hive │ │ ├── HiveMetaExecTest.java │ │ └── HiveUtilTest.java │ └── parameter │ └── ParamHelperTest.java ├── install.sh ├── master-server ├── pom.xml └── src │ ├── main │ ├── assembly │ │ └── package.xml │ ├── java │ │ └── com │ │ │ └── baifendian │ │ │ └── swordfish │ │ │ └── masterserver │ │ │ ├── MasterThriftServer.java │ │ │ ├── config │ │ │ └── MasterConfig.java │ │ │ ├── exception │ │ │ ├── ExecException.java │ │ │ ├── MasterException.java │ │ │ └── QuartzException.java │ │ │ ├── exec │ │ │ ├── ExecutorClient.java │ │ │ ├── ExecutorServerInfo.java │ │ │ └── ExecutorServerManager.java │ │ │ ├── master │ │ │ ├── ExecFlowInfo.java │ │ │ ├── ExecutorCheckThread.java │ │ │ ├── FlowExecManager.java │ │ │ ├── MasterServiceImpl.java │ │ │ ├── StreamingCheckThread.java │ │ │ └── Submit2ExecutorServerThread.java │ │ │ ├── quartz │ │ │ ├── FlowScheduleJob.java │ │ │ └── QuartzManager.java │ │ │ └── utils │ │ │ ├── ResultDetailHelper.java │ │ │ ├── ResultHelper.java │ │ │ └── crontab │ │ │ ├── CrontabUtil.java │ │ │ ├── CycleHelper.java │ │ │ ├── CycleHelperFactory.java │ │ │ ├── DayHelper.java │ │ │ ├── HourHelper.java │ │ │ ├── MinHelper.java │ │ │ ├── MonthHelper.java │ │ │ ├── Next.java │ │ │ ├── SecHelper.java │ │ │ └── WeekHelper.java │ └── resources │ │ ├── master.properties │ │ ├── masterserver_logback.xml │ │ └── quartz.properties │ └── test │ └── java │ └── com │ └── baifendian │ └── swordfish │ └── masterserver │ └── utils │ └── crontab │ └── TestCrontabUtil.java ├── package.xml ├── pom.xml ├── python-client ├── __init__.py ├── adhoc │ ├── __init__.py │ ├── __init__.pyc │ ├── adhoc.py │ ├── test_adhoc.py │ └── test_adhoc.pyc ├── login │ ├── __init__.py │ ├── __init__.pyc │ ├── login.py │ ├── login.pyc │ ├── test_login.py │ └── test_login.pyc ├── project │ ├── __init__.py │ ├── __init__.pyc │ ├── project.py │ ├── test_project.py │ └── test_project.pyc ├── settings.py ├── settings.pyc ├── streaming │ ├── __init__.py │ ├── __init__.pyc │ ├── streaming.py │ ├── streaming.pyc │ ├── streaming_exec.py │ ├── test_streaming.py │ ├── test_streaming.pyc │ ├── test_streaming_exec.py │ └── test_streaming_exec.pyc ├── test_modules.py └── user │ ├── __init__.py │ ├── __init__.pyc │ ├── test_user.py │ ├── test_user.pyc │ └── user.py ├── rpc ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── baifendian │ │ └── swordfish │ │ └── rpc │ │ ├── ExecInfo.java │ │ ├── HeartBeatData.java │ │ ├── MasterService.java │ │ ├── RetInfo.java │ │ ├── RetResultInfo.java │ │ ├── ScheduleInfo.java │ │ ├── WorkerService.java │ │ └── client │ │ └── MasterClient.java │ └── resources │ └── scheduler.thrift ├── sql ├── CreateQuartTables.sql └── create-table.sql └── web-server ├── pom.xml └── src └── main ├── assembly └── package.xml ├── java └── com │ └── baifendian │ └── swordfish │ └── webserver │ ├── RestfulApiApplication.java │ ├── StartupRunner.java │ ├── config │ └── ApplicationConfig.java │ ├── controller │ ├── AdhocController.java │ ├── DatasourceController.java │ ├── DatasourceTestController.java │ ├── ExecController.java │ ├── LoginController.java │ ├── ProjectController.java │ ├── ResourceController.java │ ├── ResourceCopyController.java │ ├── ScheduleController.java │ ├── StatController.java │ ├── StreamingController.java │ ├── StreamingExecController.java │ ├── SysInfoController.java │ ├── UserController.java │ └── WorkflowController.java │ ├── dto │ ├── AdHocDto.java │ ├── AdHocLogDto.java │ ├── AdHocResultDto.java │ ├── BaseStatusDto.java │ ├── DatasourceDto.java │ ├── ExecWorkflowsDto.java │ ├── ExecutionFlowData.java │ ├── ExecutionFlowDto.java │ ├── ExecutionNodeDto.java │ ├── ExecutorIdDto.java │ ├── ExecutorIdsDto.java │ ├── IsProhibitDto.java │ ├── LogResult.java │ ├── ProjectDto.java │ ├── ProjectUserDto.java │ ├── ResourceDto.java │ ├── ScheduleDto.java │ ├── ScheduleParam.java │ ├── StatDto.java │ ├── StreamingJobDto.java │ ├── StreamingResultDto.java │ ├── StreamingResultsDto.java │ ├── UserDto.java │ ├── UserSessionDto.java │ ├── WorkflowData.java │ ├── WorkflowDto.java │ └── WorkflowNodeDto.java │ ├── exception │ ├── BadRequestException.java │ ├── ControllerExceptionAdvice.java │ ├── CustomErrorType.java │ ├── NotFoundException.java │ ├── ParameterException.java │ ├── PermissionException.java │ ├── PreFailedException.java │ ├── ServerErrorException.java │ └── UnAuthorizedException.java │ ├── interceptor │ └── LoginInterceptor.java │ ├── service │ ├── AdhocService.java │ ├── DatasourceService.java │ ├── ExecService.java │ ├── LogHelper.java │ ├── ProjectService.java │ ├── ResourceService.java │ ├── ScheduleService.java │ ├── SessionService.java │ ├── StatService.java │ ├── StreamingExecService.java │ ├── StreamingService.java │ ├── UserService.java │ ├── WorkflowService.java │ └── storage │ │ ├── FileSystemStorageService.java │ │ ├── StorageException.java │ │ ├── StorageFileNotFoundException.java │ │ └── StorageService.java │ └── utils │ └── ParamVerify.java └── resources ├── application.properties └── webserver_logback.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .settings/ 2 | target 3 | log 4 | .idea 5 | .classpath 6 | .project 7 | .metadata 8 | RemoteSystemsTempFiles 9 | *.iml 10 | *.log 11 | logs 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # swordfish 编译方法 2 | 3 | ## 1.下载 swordfish 源码 4 | ``` 5 | $ git clone https://github.com/baifendian/swordfish.git 6 | ``` 7 | 8 | ## 2.通过maven打包 9 | ``` 10 | $ cd {swordfish_source_code_home} 11 | $ mvn -U clean package assembly:assembly -Dmaven.test.skip=true 12 | ``` 13 | 14 | 打包成功，日志显示如下： 15 | ``` 16 | [INFO] BUILD SUCCESS 17 | [INFO] ----------------------------------------------------------------- 18 | [INFO] Total time: 4.385 s 19 | [INFO] Finished at: 2017-02-27T16:13:50+08:00 20 | [INFO] Final Memory: 52M/314M 21 | [INFO] ----------------------------------------------------------------- 22 | ``` 23 | 24 | 在 target 目录下看到 swordfish-all-{version}, 这个目录下面为打包好的项目信息 25 | 26 | # 如何部署 27 | [服务部署手册](https://github.com/baifendian/swordfish/wiki/deploy) 28 | 29 | # 任务开发 30 | [任务开发手册](https://github.com/baifendian/swordfish/wiki/job-dev) 31 | 32 | # Restful API 33 | [API 说明](https://github.com/baifendian/swordfish/wiki) 34 | 35 | # Python client 36 | 该模块当前仅用于测试, 详情点击 [python client](https://github.com/baifendian/swordfish/wiki/python-client) 37 | 38 | # 系统设计参考 39 | [系统设计参考](https://github.com/baifendian/swordfish/wiki/develop) 40 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/ExternalJobType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.enums; 17 | 18 | public enum ExternalJobType { 19 | ADHOC, WORKFLOW 20 | } 21 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/FileColumnType.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.enums; 2 | 3 | /** 4 | * 文件字段类型枚举 5 | */ 6 | public enum FileColumnType { 7 | TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, VARCHAR, CHAR, BOOLEAN, DATE, TIMESTAMP, OTHER; 8 | } 9 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/HiveColumnType.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.enums; 2 | 3 | /** 4 | * hive 字段类型 5 | */ 6 | public enum HiveColumnType { 7 | /** 8 | * 对应支持的hive字段类型 9 | */ 10 | TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, VARCHAR, CHAR, BOOLEAN, DATE, TIMESTAMP; 11 | } 12 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/ImpExpType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.enums; 17 | 18 | /** 19 | * 导入导出类型 20 | */ 21 | public enum ImpExpType { 22 | /** 23 | * 0 mysql 到 hive ,1 mysql 到 hdfs 24 | */ 25 | MYSQL_TO_HIVE(ReaderType.MYSQL, WriterType.HIVE), 26 | MYSQL_TO_HDFS(ReaderType.MYSQL, WriterType.HDFS), 27 | HIVE_TO_MYSQL(ReaderType.HIVE, WriterType.MYSQL), 28 | HIVE_TO_MONGODB(ReaderType.HIVE, WriterType.MONGO), 29 | FILE_TO_HIVE(ReaderType.FILE, WriterType.HIVE), 30 | POSTGRES_TO_HIVE(ReaderType.POSTGRES, WriterType.HIVE), 31 | HIVE_TO_POSTGRES(ReaderType.HIVE, WriterType.POSTGRES), 32 | POSTGRES_TO_HDFS(ReaderType.POSTGRES, WriterType.HDFS); 33 | 34 | private ReaderType reader; 35 | private WriterType writer; 36 | 37 | public ReaderType getReader() { 38 | return reader; 39 | } 40 | 41 | public void setReader(ReaderType reader) { 42 | this.reader = reader; 43 | } 44 | 45 | public WriterType getWriter() { 46 | return writer; 47 | } 48 | 49 | public void setWriter(WriterType writer) { 50 | this.writer = writer; 51 | } 52 | 53 | ImpExpType(ReaderType reader, WriterType writer) { 54 | this.reader = reader; 55 | this.writer = writer; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/MongoWriteMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.enums; 17 | 18 | /** 19 | * mongo 写入模式枚举 20 | */ 21 | public enum MongoWriteMode { 22 | OVERWRITE, INSERT, UPDATE, UPSET; 23 | 24 | /** 25 | * 判断是否需要upsetKey 26 | * @return 27 | */ 28 | public boolean hasUpsetKey(){ 29 | return this == UPDATE || this == UPSET; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/MysqlWriteMode.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.enums; 2 | 3 | /** 4 | * mysql 写入类型 5 | */ 6 | public enum MysqlWriteMode { 7 | INSERT, REPLACE, UPDATE; 8 | } 9 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/ReaderType.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.enums; 2 | 3 | public enum ReaderType { 4 | HDFS, HIVE, MYSQL, POSTGRES, FILE 5 | } 6 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/StormType.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.enums; 2 | 3 | /** 4 | * Storm 任务类型枚举 5 | */ 6 | public enum StormType { 7 | JAR, SQL, SHELL; 8 | } 9 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/WriteHdfsType.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.enums; 2 | 3 | /** 4 | * 读hdfs文件类型 5 | */ 6 | public enum WriteHdfsType { 7 | ORC("orcfile"),TEXT("textfile"); 8 | 9 | private String type; 10 | 11 | WriteHdfsType(String type) { 12 | this.type = type; 13 | } 14 | 15 | public String getType() { 16 | return type; 17 | } 18 | 19 | public void setType(String type) { 20 | this.type = type; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/WriteMode.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.enums; 2 | 3 | /** 4 | * 写入模式 5 | */ 6 | public enum WriteMode { 7 | /** 8 | * 0 追加写入，1 覆盖写入 9 | */ 10 | APPEND, OVERWRITE; 11 | 12 | /** 13 | * 获取写入模式中的hivesql 14 | * 15 | * @return 16 | */ 17 | public String gethiveSql() { 18 | switch (this) { 19 | case APPEND: 20 | return "INTO"; 21 | case OVERWRITE: 22 | return "OVERWRITE"; 23 | default: 24 | return null; 25 | } 26 | } 27 | 28 | /** 29 | * 获取hdfs 对应的写入模式 30 | * @return 31 | */ 32 | public String getHdfsType() { 33 | switch (this) { 34 | case APPEND: 35 | return "append"; 36 | case OVERWRITE: 37 | return "nonConflict"; 38 | default: 39 | return null; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/enums/WriterType.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.enums; 2 | 3 | public enum WriterType { 4 | HDFS, HIVE, MONGO, MYSQL, POSTGRES 5 | } 6 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/hadoop/HdfsException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.hadoop; 17 | 18 | /** 19 | * Hdfs 异常

20 | * 21 | * @author : dsfan 22 | * @date : 2016年9月12日 23 | */ 24 | public class HdfsException extends RuntimeException { 25 | 26 | /** 27 | * Serial version UID 28 | */ 29 | private static final long serialVersionUID = -3271763024261592214L; 30 | 31 | /** 32 | * @param msg 33 | */ 34 | public HdfsException(String msg) { 35 | super(msg); 36 | } 37 | 38 | /** 39 | * @param msg 40 | * @param th 41 | */ 42 | public HdfsException(String msg, Throwable th) { 43 | super(msg, th); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/datasource/Datasource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.datasource; 17 | 18 | /** 19 | * 数据源基类 20 | */ 21 | public abstract class Datasource { 22 | 23 | public abstract void isConnectable() throws Exception; 24 | } 25 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/datasource/FtpDatasource.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.datasource; 17 | 18 | import org.apache.commons.net.ftp.FTPClient; 19 | 20 | /** 21 | * Ftp参数配置 22 | */ 23 | public class FtpDatasource extends Datasource { 24 | private String host; 25 | 26 | private int port; 27 | 28 | private String user; 29 | 30 | private String password; 31 | 32 | public String getHost() { 33 | return host; 34 | } 35 | 36 | public void setHost(String host) { 37 | this.host = host; 38 | } 39 | 40 | public int getPort() { 41 | return port; 42 | } 43 | 44 | public void setPort(int port) { 45 | this.port = port; 46 | } 47 | 48 | public String getUser() { 49 | return user; 50 | } 51 | 52 | public void setUser(String user) { 53 | this.user = user; 54 | } 55 | 56 | public String getPassword() { 57 | return password; 58 | } 59 | 60 | public void setPassword(String password) { 61 | this.password = password; 62 | } 63 | 64 | @Override 65 | public void isConnectable() throws Exception { 66 | FTPClient ftpClient = new FTPClient(); 67 | ftpClient.connect(this.host, this.port); 68 | if (!ftpClient.login(this.user, this.password)){ 69 | throw new Exception("wrong user name or password"); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/BaseParam.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node; 17 | 18 | import com.baifendian.swordfish.common.job.struct.resource.ResourceInfo; 19 | import java.util.List; 20 | import java.util.stream.Collectors; 21 | import org.apache.commons.collections.CollectionUtils; 22 | 23 | /** 24 | * 参数基类(需要校验参数和获取资源的子类需要 @Override 对应的方法) 25 | */ 26 | public abstract class BaseParam { 27 | 28 | /** 29 | * 校验参数是否合法 (需要校验的子类 @Override 该方法) 30 | * 31 | * @return 是否合法 32 | */ 33 | public boolean checkValid() { 34 | return true; 35 | } 36 | 37 | /** 38 | * 获取 node 需要的项目级资源文件清单 39 | */ 40 | public List getProjectResourceFiles() { 41 | return null; 42 | } 43 | 44 | /** 45 | * @param resourceInfos 46 | * @param resFiles 47 | */ 48 | public static void addProjectResourceFiles(List resourceInfos, List resFiles) { 49 | if (CollectionUtils.isNotEmpty(resourceInfos)) { 50 | resFiles.addAll(resourceInfos.stream().filter(p -> p.isProjectScope()) 51 | .map(p -> p.getRes()).collect(Collectors.toList())); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/JobType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node; 17 | 18 | public class JobType { 19 | 20 | public static final String HQL = "HQL"; // in fact, "HQL" is not accurate, "SQL" is more accuracy 21 | public static final String MR = "MR"; 22 | public static final String SHELL = "SHELL"; 23 | public static final String SPARK = "SPARK"; 24 | public static final String STORM = "STORM"; 25 | public static final String VIRTUAL = "VIRTUAL"; 26 | public static final String SPARK_STREAMING = "SPARK_STREAMING"; // 长任务类型 27 | public static final String IMPEXP = "IMPEXP"; 28 | 29 | /** 30 | * 判断是否是长任务 31 | */ 32 | public static boolean isLongJob(String job) { 33 | switch (job) { 34 | case SPARK_STREAMING: 35 | return true; 36 | case STORM: 37 | return true; 38 | case HQL: 39 | case MR: 40 | case SHELL: 41 | case SPARK: 42 | case VIRTUAL: 43 | return false; 44 | default: 45 | throw new IllegalArgumentException("job not valid"); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/adhoc/AdHocParam.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.adhoc; 17 | 18 | import com.baifendian.swordfish.common.job.struct.node.common.UdfsInfo; 19 | import java.util.List; 20 | 21 | /** 22 | * 即席查询节点参数 23 | */ 24 | public class AdHocParam { 25 | /** 26 | * 原始 sql 语句（多条，内部可能包含换行等符号，执行时需要处理） 27 | */ 28 | private String stms; 29 | 30 | /** 31 | * udfs 函数 32 | */ 33 | private List udfs; 34 | 35 | /** 36 | * 限制返回结果条目 37 | */ 38 | private Integer limit; 39 | 40 | public String getStms() { 41 | return stms; 42 | } 43 | 44 | public void setStms(String stms) { 45 | this.stms = stms; 46 | } 47 | 48 | public List getUdfs() { 49 | return udfs; 50 | } 51 | 52 | public void setUdfs(List udfs) { 53 | this.udfs = udfs; 54 | } 55 | 56 | public Integer getLimit() { 57 | return limit; 58 | } 59 | 60 | public void setLimit(Integer limit) { 61 | this.limit = limit; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/common/ProcessParam.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.common; 17 | 18 | import com.baifendian.swordfish.common.job.struct.node.BaseParam; 19 | import java.util.List; 20 | import java.util.Map; 21 | 22 | public class ProcessParam extends BaseParam { 23 | 24 | /** 25 | * 具体的脚本 26 | */ 27 | private String script; 28 | 29 | /** 30 | * 脚本参数 31 | */ 32 | private List args; 33 | 34 | /** 35 | * 环境变量 36 | */ 37 | private Map envMap; 38 | 39 | public String getScript() { 40 | return script; 41 | } 42 | 43 | public void setScript(String script) { 44 | this.script = script; 45 | } 46 | 47 | public List getArgs() { 48 | return args; 49 | } 50 | 51 | public void setArgs(List args) { 52 | this.args = args; 53 | } 54 | 55 | public Map getEnvMap() { 56 | return envMap; 57 | } 58 | 59 | public void setEnvMap(Map envMap) { 60 | this.envMap = envMap; 61 | } 62 | 63 | @Override 64 | public List getProjectResourceFiles() { 65 | return null; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/common/UdfsInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.common; 17 | 18 | import com.baifendian.swordfish.common.job.struct.resource.ResourceInfo; 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | public class UdfsInfo { 23 | private String func; 24 | 25 | private String className; 26 | 27 | private List libJars = new ArrayList<>(); 28 | 29 | private String returnType; 30 | 31 | private List argTypes; 32 | 33 | public String getFunc() { 34 | return func; 35 | } 36 | 37 | public void setFunc(String func) { 38 | this.func = func; 39 | } 40 | 41 | public String getClassName() { 42 | return className; 43 | } 44 | 45 | public void setClassName(String className) { 46 | this.className = className; 47 | } 48 | 49 | public List getLibJars() { 50 | return libJars; 51 | } 52 | 53 | public void setLibJars(List libJars) { 54 | this.libJars = libJars; 55 | } 56 | 57 | public String getReturnType() { 58 | return returnType; 59 | } 60 | 61 | public void setReturnType(String returnType) { 62 | this.returnType = returnType; 63 | } 64 | 65 | public List getArgTypes() { 66 | return argTypes; 67 | } 68 | 69 | public void setArgTypes(List argTypes) { 70 | this.argTypes = argTypes; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/column/FileColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.column; 17 | 18 | import com.baifendian.swordfish.common.enums.FileColumnType; 19 | 20 | /** 21 | * 文件列 22 | */ 23 | public class FileColumn { 24 | private String name; 25 | private FileColumnType type; 26 | private String dateFormat = "yyyy-MM-dd"; 27 | 28 | public String getName() { 29 | return name; 30 | } 31 | 32 | public void setName(String name) { 33 | this.name = name; 34 | } 35 | 36 | public FileColumnType getType() { 37 | return type; 38 | } 39 | 40 | public void setType(FileColumnType type) { 41 | this.type = type; 42 | } 43 | 44 | public String getDateFormat() { 45 | return dateFormat; 46 | } 47 | 48 | public void setDateFormat(String dateFormat) { 49 | this.dateFormat = dateFormat; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/column/HiveColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.column; 17 | 18 | import com.baifendian.swordfish.common.enums.HiveColumnType; 19 | 20 | /** 21 | * hive Column信息 22 | */ 23 | public class HiveColumn { 24 | private String name; 25 | private HiveColumnType type; 26 | 27 | public String getName() { 28 | return name; 29 | } 30 | 31 | public void setName(String name) { 32 | this.name = name; 33 | } 34 | 35 | public HiveColumnType getType() { 36 | return type; 37 | } 38 | 39 | public void setType(HiveColumnType type) { 40 | this.type = type; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/column/MongoColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.column; 17 | 18 | /** 19 | * Mongo Column 20 | */ 21 | public class MongoColumn { 22 | private String name; 23 | private String type; 24 | 25 | public String getName() { 26 | return name; 27 | } 28 | 29 | public void setName(String name) { 30 | this.name = name; 31 | } 32 | 33 | public String getType() { 34 | return type; 35 | } 36 | 37 | public void setType(String type) { 38 | this.type = type; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/reader/HdfsReader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.reader; 17 | 18 | /** 19 | * hdfs 读参数 20 | */ 21 | public class HdfsReader implements Reader { 22 | 23 | @Override 24 | public boolean checkValid() { 25 | return false; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/reader/Reader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.reader; 17 | 18 | /** 19 | * 读接口 20 | */ 21 | public interface Reader { 22 | /** 23 | * 校验reader是否合法 24 | * @return 25 | */ 26 | default boolean checkValid(){ 27 | return true; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/reader/ReaderFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.reader; 17 | 18 | import com.baifendian.swordfish.common.enums.ImpExpType; 19 | import com.baifendian.swordfish.common.enums.ReaderType; 20 | import com.baifendian.swordfish.dao.utils.json.JsonUtil; 21 | 22 | /** 23 | * 读配置工厂 24 | */ 25 | public class ReaderFactory { 26 | 27 | public static Reader getReader(ReaderType type, String reader) { 28 | switch (type) { 29 | case MYSQL: 30 | return JsonUtil.parseObject(reader, MysqlReader.class); 31 | case HIVE: 32 | return JsonUtil.parseObject(reader, HiveReader.class); 33 | case FILE: 34 | return JsonUtil.parseObject(reader, FileReader.class); 35 | case POSTGRES: 36 | return JsonUtil.parseObject(reader, PostgreReader.class); 37 | default: 38 | return null; 39 | } 40 | } 41 | 42 | public static Reader getReader(ImpExpType type, String reader) { 43 | return getReader(type.getReader(), reader); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/setting/ErrorLimit.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.setting; 17 | 18 | import com.fasterxml.jackson.annotation.JsonInclude; 19 | 20 | /** 21 | * 异常阈值 22 | */ 23 | @JsonInclude(JsonInclude.Include.NON_NULL) 24 | public class ErrorLimit { 25 | /** 26 | * 数据错误数阈值 27 | */ 28 | private Long record; 29 | /** 30 | * 数据错误百分比阈值 31 | */ 32 | private Double percentage; 33 | 34 | public Long getRecord() { 35 | return record; 36 | } 37 | 38 | public void setRecord(Long record) { 39 | this.record = record; 40 | } 41 | 42 | public Double getPercentage() { 43 | return percentage; 44 | } 45 | 46 | public void setPercentage(Double percentage) { 47 | this.percentage = percentage; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/setting/Setting.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.setting; 17 | 18 | import com.fasterxml.jackson.annotation.JsonInclude; 19 | 20 | /** 21 | * 其他配置设定 22 | */ 23 | @JsonInclude(JsonInclude.Include.NON_NULL) 24 | public class Setting { 25 | 26 | /** 27 | * 速度控制 28 | */ 29 | private Speed speed; 30 | /** 31 | * 脏数据控制 32 | */ 33 | private ErrorLimit errorLimit; 34 | 35 | public Setting(Speed speed, ErrorLimit errorLimit) { 36 | this.speed = speed; 37 | this.errorLimit = errorLimit; 38 | } 39 | 40 | public Setting(Speed speed) { 41 | this.speed = speed; 42 | } 43 | 44 | public Setting() { 45 | } 46 | 47 | public Speed getSpeed() { 48 | return speed; 49 | } 50 | 51 | public void setSpeed(Speed speed) { 52 | this.speed = speed; 53 | } 54 | 55 | public ErrorLimit getErrorLimit() { 56 | return errorLimit; 57 | } 58 | 59 | public void setErrorLimit(ErrorLimit errorLimit) { 60 | this.errorLimit = errorLimit; 61 | } 62 | 63 | public boolean checkValid() { 64 | return speed != null && speed.checkValid(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/setting/Speed.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.setting; 17 | 18 | import com.fasterxml.jackson.annotation.JsonInclude; 19 | import com.fasterxml.jackson.annotation.JsonProperty; 20 | 21 | /** 22 | * DataX速度配置 23 | */ 24 | @JsonInclude(JsonInclude.Include.NON_NULL) 25 | public class Speed { 26 | /** 27 | * 最大并发数 28 | */ 29 | private Integer channel = 1; 30 | /** 31 | * 最大流量 32 | */ 33 | @JsonProperty("byte") 34 | private Long byteInt = 104857600L; 35 | 36 | public Speed(Integer channel, Long byteInt) { 37 | this.channel = channel; 38 | this.byteInt = byteInt; 39 | } 40 | 41 | public Speed() { 42 | } 43 | 44 | public int getChannel() { 45 | return channel; 46 | } 47 | 48 | public void setChannel(int channel) { 49 | this.channel = channel; 50 | } 51 | 52 | public Long getByteInt() { 53 | return byteInt; 54 | } 55 | 56 | public void setByteInt(Long byteInt) { 57 | this.byteInt = byteInt; 58 | } 59 | 60 | public boolean checkValid() { 61 | return channel != null && byteInt != null; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/writer/Writer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.writer; 17 | 18 | /** 19 | * 写接口 20 | */ 21 | public interface Writer { 22 | /** 23 | * 校验writer是否合法 24 | * @return 25 | */ 26 | default boolean checkValid(){ 27 | return true; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/impexp/writer/WriterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.impexp.writer; 17 | 18 | import com.baifendian.swordfish.common.enums.ImpExpType; 19 | import com.baifendian.swordfish.common.enums.WriterType; 20 | import com.baifendian.swordfish.dao.utils.json.JsonUtil; 21 | 22 | /** 23 | * 写配置工厂 24 | */ 25 | public class WriterFactory { 26 | 27 | public static Writer getWriter(WriterType type, String writer) { 28 | switch (type) { 29 | case HDFS: 30 | return JsonUtil.parseObject(writer, HdfsWriter.class); 31 | case HIVE: 32 | return JsonUtil.parseObject(writer, HiveWriter.class); 33 | case MYSQL: 34 | return JsonUtil.parseObject(writer, MysqlWriter.class); 35 | case MONGO: 36 | return JsonUtil.parseObject(writer, MongoWriter.class); 37 | case POSTGRES: 38 | return JsonUtil.parseObject(writer, PostgreWriter.class); 39 | default: 40 | return null; 41 | } 42 | } 43 | 44 | public static Writer getWriter(ImpExpType type, String writer) { 45 | return getWriter(type.getWriter(), writer); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/shell/ShellParam.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.shell; 17 | 18 | import com.baifendian.swordfish.common.job.struct.node.BaseParam; 19 | import com.baifendian.swordfish.common.job.struct.resource.ResourceInfo; 20 | import java.util.List; 21 | import java.util.stream.Collectors; 22 | 23 | public class ShellParam extends BaseParam { 24 | /** 25 | * 原始 shell 语句 26 | */ 27 | private String script; 28 | 29 | /** 30 | * 资料列表 31 | */ 32 | private List resources; 33 | 34 | public String getScript() { 35 | return script; 36 | } 37 | 38 | public void setScript(String script) { 39 | this.script = script; 40 | } 41 | 42 | public List getResources() { 43 | return resources; 44 | } 45 | 46 | public void setResources(List resources) { 47 | this.resources = resources; 48 | } 49 | 50 | @Override 51 | public boolean checkValid() { 52 | return script != null && !script.isEmpty(); 53 | } 54 | 55 | @Override 56 | public List getProjectResourceFiles() { 57 | if (resources != null) { 58 | return resources.stream().filter(p -> p.isProjectScope()) 59 | .map(p -> p.getRes()).collect(Collectors.toList()); 60 | } 61 | 62 | return null; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/storm/dto/TopologyInfoDto.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.job.struct.node.storm.dto; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Created by caojingwei on 2017/7/15. 7 | */ 8 | public class TopologyInfoDto { 9 | private String name; 10 | private String id; 11 | private String status; 12 | List workers; 13 | 14 | public String getName() { 15 | return name; 16 | } 17 | 18 | public void setName(String name) { 19 | this.name = name; 20 | } 21 | 22 | public String getId() { 23 | return id; 24 | } 25 | 26 | public void setId(String id) { 27 | this.id = id; 28 | } 29 | 30 | public String getStatus() { 31 | return status; 32 | } 33 | 34 | public void setStatus(String status) { 35 | this.status = status; 36 | } 37 | 38 | public List getWorkers() { 39 | return workers; 40 | } 41 | 42 | public void setWorkers(List workers) { 43 | this.workers = workers; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/storm/dto/TopologyOperationDto.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.job.struct.node.storm.dto; 2 | 3 | /** 4 | * Created by caojingwei on 2017/7/15. 5 | */ 6 | public class TopologyOperationDto { 7 | private String topologyOperation; 8 | private String topologyId; 9 | private String status; 10 | 11 | public String getTopologyOperation() { 12 | return topologyOperation; 13 | } 14 | 15 | public void setTopologyOperation(String topologyOperation) { 16 | this.topologyOperation = topologyOperation; 17 | } 18 | 19 | public String getTopologyId() { 20 | return topologyId; 21 | } 22 | 23 | public void setTopologyId(String topologyId) { 24 | this.topologyId = topologyId; 25 | } 26 | 27 | public String getStatus() { 28 | return status; 29 | } 30 | 31 | public void setStatus(String status) { 32 | this.status = status; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/storm/dto/TopologySummaryDto.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.job.struct.node.storm.dto; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Created by caojingwei on 2017/7/15. 7 | */ 8 | public class TopologySummaryDto { 9 | private List topologies; 10 | private boolean schedulerDisplayResource; 11 | 12 | public List getTopologies() { 13 | return topologies; 14 | } 15 | 16 | public void setTopologies(List topologies) { 17 | this.topologies = topologies; 18 | } 19 | 20 | public boolean isSchedulerDisplayResource() { 21 | return schedulerDisplayResource; 22 | } 23 | 24 | public void setSchedulerDisplayResource(boolean schedulerDisplayResource) { 25 | this.schedulerDisplayResource = schedulerDisplayResource; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/storm/dto/TopologyWorkerDto.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.job.struct.node.storm.dto; 2 | 3 | /** 4 | * Created by caojingwei on 2017/7/15. 5 | */ 6 | public class TopologyWorkerDto { 7 | private String workerLogLink; 8 | 9 | public String getWorkerLogLink() { 10 | return workerLogLink; 11 | } 12 | 13 | public void setWorkerLogLink(String workerLogLink) { 14 | this.workerLogLink = workerLogLink; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/storm/param/IStormParam.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.storm.param; 17 | 18 | import com.baifendian.swordfish.common.job.struct.resource.ResourceInfo; 19 | import java.util.List; 20 | import java.util.stream.Collectors; 21 | import org.apache.commons.collections.CollectionUtils; 22 | 23 | /** 24 | * storm param 接口 25 | */ 26 | public interface IStormParam { 27 | /** 28 | * 校验Storm param是否合法 29 | * @return 30 | */ 31 | default boolean checkValid(){ 32 | return true; 33 | } 34 | 35 | List getProjectResourceFiles(); 36 | 37 | default void addProjectResourceFiles(List resourceInfos, List resFiles) { 38 | if (CollectionUtils.isNotEmpty(resourceInfos)) { 39 | resFiles.addAll(resourceInfos.stream().filter(p -> p.isProjectScope()) 40 | .map(p -> p.getRes()).collect(Collectors.toList())); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/storm/param/StormParamFactory.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.job.struct.node.storm.param; 2 | 3 | import com.baifendian.swordfish.common.enums.StormType; 4 | import com.baifendian.swordfish.dao.utils.json.JsonUtil; 5 | 6 | /** 7 | * Storm param 工厂 8 | */ 9 | public class StormParamFactory { 10 | public static IStormParam getStormParam(StormType type, String param) { 11 | switch (type) { 12 | case JAR: 13 | return JsonUtil.parseObject(param, StormJarParam.class); 14 | case SQL: 15 | return JsonUtil.parseObject(param, StormSqlParam.class); 16 | case SHELL: 17 | return JsonUtil.parseObject(param, StormShellParam.class); 18 | default: 19 | return null; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/storm/param/StormShellParam.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.storm.param; 17 | 18 | import com.baifendian.swordfish.common.job.struct.resource.ResourceInfo; 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | import org.apache.commons.lang.StringUtils; 22 | 23 | /** 24 | * Storm shell 方式提交任务参数 25 | */ 26 | public class StormShellParam implements IStormParam { 27 | private ResourceInfo resources; 28 | private String command; 29 | 30 | public ResourceInfo getResources() { 31 | return resources; 32 | } 33 | 34 | public void setResources(ResourceInfo resources) { 35 | this.resources = resources; 36 | } 37 | 38 | public String getCommand() { 39 | return command; 40 | } 41 | 42 | public void setCommand(String command) { 43 | this.command = command; 44 | } 45 | 46 | @Override 47 | public boolean checkValid() { 48 | return resources != null && 49 | StringUtils.isNotEmpty(command); 50 | } 51 | 52 | @Override 53 | public List getProjectResourceFiles() { 54 | List resFiles = new ArrayList<>(); 55 | if (resources.isProjectScope()) { 56 | resFiles.add(resources.getRes()); 57 | } 58 | return resFiles; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/node/virtual/VirtualParam.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.node.virtual; 17 | 18 | import com.baifendian.swordfish.common.job.struct.node.BaseParam; 19 | 20 | public class VirtualParam extends BaseParam { 21 | private static VirtualParam virtualParam; 22 | 23 | private VirtualParam() { 24 | 25 | } 26 | 27 | public static VirtualParam getInstance() { 28 | if (virtualParam == null) { 29 | synchronized (VirtualParam.class) { 30 | if (virtualParam == null) { 31 | virtualParam = new VirtualParam(); 32 | } 33 | } 34 | } 35 | 36 | return virtualParam; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/resource/ResScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.resource; 17 | 18 | public enum ResScope { 19 | PROJECT, WORKFLOW 20 | } 21 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/job/struct/resource/ResourceInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.resource; 17 | 18 | import com.fasterxml.jackson.annotation.JsonIgnore; 19 | import org.apache.commons.lang3.StringUtils; 20 | 21 | /** 22 | * 资源信息 23 | */ 24 | public class ResourceInfo { 25 | 26 | private ResScope scope; 27 | 28 | private String res; 29 | 30 | private String alias; 31 | 32 | public ResScope getScope() { 33 | return scope; 34 | } 35 | 36 | public void setScope(ResScope scope) { 37 | this.scope = scope; 38 | } 39 | 40 | public String getRes() { 41 | return res; 42 | } 43 | 44 | public void setRes(String res) { 45 | this.res = res; 46 | } 47 | 48 | public String getAlias() { 49 | return alias; 50 | } 51 | 52 | public void setAlias(String alias) { 53 | this.alias = alias; 54 | } 55 | 56 | /** 57 | * 得到资源的符号连接 58 | * 59 | * @return 60 | */ 61 | @JsonIgnore 62 | public String getSymbolicRes() { 63 | if (StringUtils.isNotEmpty(alias)) { 64 | return String.format("%s#%s", res, alias); 65 | } 66 | 67 | return res; 68 | } 69 | 70 | /** 71 | * scope 没有值时默认为 PROJECT 72 | */ 73 | @JsonIgnore 74 | public boolean isProjectScope() { 75 | switch (scope) { 76 | case WORKFLOW: 77 | return false; 78 | default: 79 | return true; 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/json/JsonOrdinalSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.json; 17 | 18 | import com.fasterxml.jackson.core.JsonGenerator; 19 | import com.fasterxml.jackson.databind.JsonSerializer; 20 | import com.fasterxml.jackson.databind.SerializerProvider; 21 | import java.io.IOException; 22 | 23 | /** 24 | * 直接序列化出枚举序列 25 | */ 26 | public class JsonOrdinalSerializer extends JsonSerializer { 27 | 28 | @Override 29 | public void serialize(E e, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException { 30 | jsonGenerator.writeNumber(e.ordinal()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/utils/NanoSampleConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.utils; 17 | 18 | import ch.qos.logback.classic.pattern.ClassicConverter; 19 | import ch.qos.logback.classic.spi.ILoggingEvent; 20 | 21 | /** 22 | * 用于日志中的 "纳秒" 输出 23 | */ 24 | public class NanoSampleConverter extends ClassicConverter { 25 | @Override 26 | public String convert(ILoggingEvent event) { 27 | long nowInNanos = System.nanoTime(); 28 | return Long.toString(nowInNanos); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/utils/PermUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.utils; 17 | 18 | import com.baifendian.swordfish.common.consts.Constants; 19 | 20 | public class PermUtil { 21 | /** 22 | * 判断是否具备写权限 23 | * 24 | * @param perm 25 | * @return 26 | */ 27 | public static boolean hasWritePerm(int perm) { 28 | return (perm & Constants.PROJECT_USER_PERM_WRITE) != 0; 29 | } 30 | 31 | /** 32 | * 判断是否具备读权限 33 | * 34 | * @param perm 35 | * @return 36 | */ 37 | public static boolean hasReadPerm(int perm) { 38 | return (perm & Constants.PROJECT_USER_PERM_READ) != 0; 39 | } 40 | 41 | /** 42 | * 判断是否具备执行权限 43 | * 44 | * @param perm 45 | * @return 46 | */ 47 | public static boolean hasExecPerm(int perm) { 48 | return (perm & Constants.PROJECT_USER_PERM_EXEC) != 0; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/utils/StructuredArguments.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.utils; 17 | 18 | public class StructuredArguments { 19 | 20 | /** 21 | * 支持 key value 22 | * 23 | * @param key 24 | * @param value 25 | * @return 26 | */ 27 | public static String keyValue(String key, String value) { 28 | return String.format("[%s=%s]", key, value); 29 | } 30 | 31 | /** 32 | * 支持 jobId 33 | * 34 | * @param value 35 | * @return 36 | */ 37 | public static String jobValue(String value) { 38 | return String.format("[jobId=%s]", value); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /common/src/main/java/com/baifendian/swordfish/common/utils/graph/VertexKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.utils.graph; 17 | 18 | public class VertexKey { 19 | private String key; 20 | 21 | public VertexKey(String key) { 22 | this.key = key; 23 | } 24 | 25 | public String getKey() { 26 | return key; 27 | } 28 | 29 | @Override 30 | public boolean equals(Object other) { 31 | if (this == other) { 32 | return true; 33 | } 34 | if (other instanceof VertexKey) { 35 | VertexKey vertexKey = (VertexKey) other; 36 | return key.equals(vertexKey.getKey()); 37 | } 38 | return false; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /common/src/main/resources/common/base_config.properties: -------------------------------------------------------------------------------- 1 | # prohibit users 2 | prohibit.user.list=root,swordfish,bfd_hz 3 | 4 | # local data base dir 5 | local.data.basepath=/tmp/swordfish/filecache 6 | 7 | # local data base dir for download 8 | local.download.basepath=/tmp/swordfish/downloadcache 9 | 10 | # local exec base dir 11 | local.exec.basepath=/tmp/swordfish/exec 12 | 13 | # hdfs data base dir 14 | hdfs.data.basepath=/tmp/swordfish/filecache 15 | 16 | # hdfs file used for exec 17 | hdfs.udfjar.basepath=/tmp/swordfish/hiveJars 18 | 19 | # hdfs import and export tmp dir 20 | hdfs.impexp.basepath=/tmp/swordfish/impexp 21 | 22 | # system env path 23 | sf.env.file=/home/swordfish/.sf_env.sh 24 | 25 | # develop mode 26 | develop.mode=false -------------------------------------------------------------------------------- /common/src/main/resources/common/hadoop/hadoop.properties: -------------------------------------------------------------------------------- 1 | # mapreduce submit application will be cross-platform or not 2 | mapreduce.app-submission.cross-platform=true 3 | 4 | # fs URI 5 | fs.defaultFS=hdfs://172.18.1.22:8020 6 | 7 | # mapreduce framework 8 | mapreduce.framework.name=yarn 9 | 10 | # yarn rm address 11 | yarn.resourcemanager.address=172.18.1.22:8032 12 | 13 | # yarn scheduler address 14 | yarn.resourcemanager.scheduler.address=172.18.1.22:8030 15 | 16 | # mr jobhistory address 17 | mapreduce.jobhistory.address=172.18.1.22:10020 18 | 19 | # yarn rm web app address 20 | yarn.resourcemanager.webapp.address=http://172.18.1.22:8088/cluster/app/%s 21 | 22 | # yarn application status rest api 23 | yarn.application.status.address=http://172.18.1.22:8088/ws/v1/cluster/apps/%s 24 | -------------------------------------------------------------------------------- /common/src/main/resources/common/mail.properties: -------------------------------------------------------------------------------- 1 | # mail server configuration 2 | mail.protocol=SMTP 3 | mail.server.host=mail.baifendian.com 4 | mail.server.port=25 5 | mail.sender=dw_alert@baifendian.com 6 | mail.passwd=4eRU23Smtp -------------------------------------------------------------------------------- /common/src/main/resources/common/search.properties: -------------------------------------------------------------------------------- 1 | # the elastic search address 2 | es.address = 172.18.1.22:9300 3 | es.cluster.name = logging-dev 4 | 5 | # setMaxRetryTimeoutMillis 6 | es.max.retry.timeout.millis = 3000 7 | 8 | # path 9 | swordfish.endpoint = swordfish-logstash-* 10 | -------------------------------------------------------------------------------- /common/src/main/resources/common/storm.properties: -------------------------------------------------------------------------------- 1 | # storm rest api 2 | storm.rest.url=http://bgs-8p95-zhanglifeng.bfdabc.com:8744 3 | 4 | # storm status api 5 | storm.rest.topology=/topology.html?id= 6 | -------------------------------------------------------------------------------- /common/src/test/java/com/baifendian/swordfish/common/job/struct/datasource/FtpDatasourceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.datasource; 17 | 18 | import org.junit.Test; 19 | 20 | public class FtpDatasourceTest { 21 | 22 | @Test 23 | public void testIsConnectable() throws Exception { 24 | // FtpDatasource ftpDatasource = new FtpDatasource(); 25 | // 26 | // ftpDatasource.setHost("10.12.7.10"); 27 | // ftpDatasource.setPort(21); 28 | // ftpDatasource.setUser("bfd_541"); 29 | // ftpDatasource.setPassword("bfd123456"); 30 | // ftpDatasource.isConnectable(); 31 | } 32 | 33 | @Test(expected = Exception.class) 34 | public void testIsConnectable2() throws Exception { 35 | FtpDatasource ftpDatasource = new FtpDatasource(); 36 | 37 | ftpDatasource.setHost("10.12.7.10"); 38 | ftpDatasource.setPort(21); 39 | ftpDatasource.setUser("bfd_541"); 40 | ftpDatasource.setPassword("bfd1234561111"); 41 | ftpDatasource.isConnectable(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /common/src/test/java/com/baifendian/swordfish/common/job/struct/datasource/OracleDatasourceTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.job.struct.datasource; 17 | 18 | import org.junit.Test; 19 | 20 | public class OracleDatasourceTest { 21 | 22 | @Test 23 | public void testIsConnectable() throws Exception { 24 | OracleDatasource oracleDatasource = new OracleDatasource(); 25 | oracleDatasource.setHost("172.18.1.112"); 26 | oracleDatasource.setPort(1521); 27 | oracleDatasource.setService("orcl"); 28 | oracleDatasource.setUser("test"); 29 | oracleDatasource.setPassword("test"); 30 | oracleDatasource.isConnectable(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /common/src/test/java/com/baifendian/swordfish/common/job/struct/datasource/PostgreDatasourceTest.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.job.struct.datasource; 2 | 3 | import org.junit.Test; 4 | 5 | public class PostgreDatasourceTest { 6 | 7 | @Test 8 | public void testIsConnectable() throws Exception { 9 | PostgreDatasource postgreDatasource = new PostgreDatasource(); 10 | postgreDatasource.setAddress("jdbc:postgresql://172.24.8.98:5432"); 11 | postgreDatasource.setUser("postgres"); 12 | postgreDatasource.setDatabase("test01"); 13 | postgreDatasource.setPassword("postgres-2017"); 14 | postgreDatasource.isConnectable(); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /common/src/test/java/com/baifendian/swordfish/common/job/struct/node/BaseParamFactoryTest.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.common.job.struct.node; 2 | 3 | import static junit.framework.TestCase.assertTrue; 4 | 5 | import com.baifendian.swordfish.common.job.struct.node.impexp.ImpExpParam; 6 | import org.junit.Test; 7 | 8 | public class BaseParamFactoryTest { 9 | 10 | @Test 11 | public void testGetBaseParam() { 12 | String parameter = "{\"type\":\"MYSQL_TO_HDFS\",\"reader\":{\"column\":[\"`id`\",\"`name`\",\"`email`\",\"`desc`\",\"`phone`\",\"`password`\",\"`role`\",\"`proxy_users`\",\"`create_time`\",\"`modify_time`\"],\"datasource\":\"test11111\",\"table\":[\"user\"]},\"writer\":{\"path\":\"/test/temp/here\",\"fileName\":\"filetest\",\"writeMode\":\"APPEND\",\"fileType\":\"ORC\",\"column\":[{\"name\":\"`id`\",\"type\":\"VARCHAR\"},{\"name\":\"`name`\",\"type\":\"VARCHAR\"},{\"name\":\"`email`\",\"type\":\"VARCHAR\"},{\"name\":\"`desc`\",\"type\":\"VARCHAR\"},{\"name\":\"`phone`\",\"type\":\"VARCHAR\"},{\"name\":\"`password`\",\"type\":\"VARCHAR\"},{\"name\":\"`role`\",\"type\":\"VARCHAR\"},{\"name\":\"`proxy_users`\",\"type\":\"VARCHAR\"},{\"name\":\"`create_time`\",\"type\":\"VARCHAR\"},{\"name\":\"`modify_time`\",\"type\":\"VARCHAR\"}]},\"setting\":{\"speed\":{\"channel\":1,\"byte\":104857600},\"errorLimit\":{\"record\":3,\"percentage\":0.05}}}"; 13 | String type = "IMPEXP"; 14 | 15 | ImpExpParam impExpParam = (ImpExpParam) BaseParamFactory.getBaseParam(type, parameter); 16 | 17 | assertTrue(impExpParam.checkValid()); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /common/src/test/java/com/baifendian/swordfish/common/mail/MailSendUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.mail; 17 | 18 | import java.util.Arrays; 19 | import org.junit.Test; 20 | 21 | public class MailSendUtilTest { 22 | @Test 23 | public void testSendMails() { 24 | String[] mails = new String[]{"qifeng.dai@baifendian.com"}; 25 | String title = "test from swordfish"; 26 | String content = "test"; 27 | 28 | MailSendUtil.sendMails(Arrays.asList(mails), title, content); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /common/src/test/java/com/baifendian/swordfish/common/utils/VerifyUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.common.utils; 17 | 18 | import static junit.framework.TestCase.assertTrue; 19 | 20 | import org.junit.Test; 21 | 22 | public class VerifyUtilTest { 23 | @Test 24 | public void testMatcheUserName() { 25 | { 26 | String test = "bfd_test"; 27 | assertTrue(VerifyUtil.matcheUserName(test)); 28 | } 29 | } 30 | 31 | @Test 32 | public void testMatcheProjectName() { 33 | { 34 | String test = "bfd_test"; 35 | assertTrue(VerifyUtil.matcheProjectName(test)); 36 | } 37 | } 38 | 39 | @Test 40 | public void testMatcheResName() { 41 | { 42 | assertTrue(VerifyUtil.matcheResName("aa.jar")); 43 | 44 | assertTrue(VerifyUtil.matcheResName("spark-examples-1.0-SNAPSHOT-hadoop2.6.0.jar")); 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/BaseDao.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao; 17 | 18 | public abstract class BaseDao { 19 | 20 | protected abstract void init(); 21 | } 22 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/DaoFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao; 17 | 18 | import java.lang.reflect.InvocationTargetException; 19 | import java.util.Map; 20 | import java.util.concurrent.ConcurrentHashMap; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | public class DaoFactory { 25 | 26 | private static final Logger logger = LoggerFactory.getLogger(DaoFactory.class); 27 | 28 | private static Map daoMap = new ConcurrentHashMap<>(); 29 | 30 | /** 31 | * 获取 Dao 实例 （单例）

32 | * 33 | * @param clazz 34 | * @return Dao实例 35 | */ 36 | @SuppressWarnings("unchecked") 37 | public static T getDaoInstance(Class clazz) { 38 | String className = clazz.getName(); 39 | synchronized (daoMap) { 40 | if (!daoMap.containsKey(className)) { 41 | try { 42 | T t = clazz.getConstructor().newInstance(); 43 | // 实例初始化 44 | t.init(); 45 | daoMap.put(className, t); 46 | } catch (InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { 47 | logger.error(e.getMessage(), e); 48 | } 49 | } 50 | } 51 | 52 | return (T) daoMap.get(className); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/DatasourceDao.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.dao; 2 | 3 | import com.baifendian.swordfish.dao.datasource.ConnectionFactory; 4 | import com.baifendian.swordfish.dao.mapper.DataSourceMapper; 5 | import com.baifendian.swordfish.dao.model.DataSource; 6 | 7 | /** 8 | * 资源 DAO 9 | */ 10 | public class DatasourceDao extends BaseDao { 11 | private DataSourceMapper dataSourceMapper; 12 | 13 | @Override 14 | protected void init() { 15 | dataSourceMapper = ConnectionFactory.getSqlSession().getMapper(DataSourceMapper.class); 16 | } 17 | 18 | /** 19 | * 根据projectId 和 资源name查询一个数据源 20 | * @param projectId 21 | * @param name 22 | * @return 23 | */ 24 | public DataSource queryResource(int projectId, String name) { 25 | return dataSourceMapper.getByName(projectId, name); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/DbType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | public enum DbType { 19 | /** 20 | * 0 mysql, 1 oracle, 2 mongodb, 3 hbase, 4 ftp 5 postgresql 21 | */ 22 | MYSQL, ORACLE, MONGODB, HBASE, FTP, POSTGRES; 23 | } 24 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/DepPolicyType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | public enum DepPolicyType { 19 | 20 | /** 21 | * 0.不依赖上一调度周期
22 | * 1.自动依赖等待上一周期结束才能继续 23 | **/ 24 | NO_DEP_PRE, DEP_PRE; 25 | } 26 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/ExecType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | /** 19 | * workflow 等运行的类型

20 | */ 21 | public enum ExecType { 22 | 23 | /** 24 | * 0(直接运行) 1(调度的任务) 2(补数据的任务) 25 | **/ 26 | DIRECT, SCHEDULER, COMPLEMENT_DATA; 27 | } 28 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/FailurePolicyType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | public enum FailurePolicyType { 19 | 20 | /** 21 | * 0.结束 1.继续 22 | **/ 23 | END, CONTINUE; 24 | 25 | public static FailurePolicyType valueOfType(Integer type) throws IllegalArgumentException { 26 | if (type == null) { 27 | return null; 28 | } 29 | 30 | try { 31 | return FailurePolicyType.values()[type]; 32 | } catch (Exception ex) { 33 | throw new IllegalArgumentException( 34 | "Cannot convert " + type + " to " + FailurePolicyType.class.getSimpleName() + " .", ex); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/FlowStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | /** 19 | * 描述一个 workflow 和 node 的运行状态

20 | */ 21 | public enum FlowStatus { 22 | /** 23 | * 0(初始化) 1(调度依赖任务中) 2(调度依赖资源中) 3(正在运行) 4(运行成功) 5(kill掉) 6(运行失败) 7(依赖失败) 8(暂停) 24 | **/ 25 | INIT, WAITING_DEP, WAITING_RES, RUNNING, SUCCESS, KILL, FAILED, DEP_FAILED, INACTIVE; 26 | 27 | /** 28 | * 判断是否成功状态

29 | * 30 | * @return 是否成功状态 31 | */ 32 | public boolean typeIsSuccess() { 33 | return this == SUCCESS; 34 | } 35 | 36 | /** 37 | * 判断是否失败状态 38 | * 39 | * @return 是否失败状态 40 | */ 41 | public boolean typeIsFailure() { 42 | return this == KILL || this == FAILED || this == DEP_FAILED; 43 | } 44 | 45 | /** 46 | * 判断是否终止状态

47 | * 48 | * @return 是否终止状态 49 | */ 50 | public boolean typeIsFinished() { 51 | return typeIsSuccess() || typeIsFailure(); 52 | } 53 | 54 | /** 55 | * 判断是否非终止状态 56 | * 57 | * @return 58 | */ 59 | public boolean typeIsNotFinished() { 60 | return !typeIsFinished(); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/NodeDepType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | /** 19 | * 执行任务的节点依赖类型

20 | */ 21 | public enum NodeDepType { 22 | /** 23 | * 0 表示仅执行 node ，1 表示执行 node 本身及其依赖，2 表示执行 node 及依赖当前 node 的 node 24 | */ 25 | NODE_ONLY, NODE_POST, NODE_PRE; 26 | 27 | /** 28 | * 通过 type 获取枚举对象

29 | * 30 | * @param type 31 | * @return {@link NodeDepType} 32 | */ 33 | public static NodeDepType valueOfType(Integer type) throws IllegalArgumentException { 34 | if (type == null) { 35 | return null; 36 | } 37 | try { 38 | return NodeDepType.values()[type]; 39 | } catch (Exception ex) { 40 | throw new IllegalArgumentException("Cannot convert " + type + " to " + NodeDepType.class.getSimpleName() + " .", ex); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/NotifyType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.baifendian.swordfish.dao.enums; 18 | 19 | /** 20 | * 报警类型 21 | */ 22 | public enum NotifyType { 23 | /** 24 | * 0-NONE-都不发 1-SUCCESS-成功发，2-FAILURE-失败发，3-ALL-成功或失败都发 25 | */ 26 | NONE, SUCCESS, FAILURE, ALL; 27 | 28 | /** 29 | * 通过 type 获取枚举对象

30 | * 31 | * @param type 32 | * @return {@link NotifyType} 33 | */ 34 | public static NotifyType valueOfType(Integer type) throws IllegalArgumentException { 35 | if (type == null) { 36 | return null; 37 | } 38 | 39 | try { 40 | return NotifyType.values()[type]; 41 | } catch (Exception ex) { 42 | throw new IllegalArgumentException("Cannot convert " + type + " to " + NotifyType.class.getSimpleName() + " .", ex); 43 | } 44 | } 45 | 46 | public boolean typeIsSendFailureMail() { 47 | return this == FAILURE || this == ALL; 48 | } 49 | 50 | public boolean typeIsSendSuccessMail() { 51 | return this == SUCCESS || this == ALL; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/ScheduleStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | /** 19 | * 调度状态

20 | */ 21 | public enum ScheduleStatus { 22 | OFFLINE, ONLINE; 23 | } 24 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/ScheduleType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | /** 19 | * 调度的类型

20 | */ 21 | public enum ScheduleType { 22 | /** 23 | * 0 分钟 ， 1 小时 ，2 天， 3 周， 4 月 5 年 24 | */ 25 | MINUTE, HOUR, DAY, WEEK, MONTH, YEAR; 26 | 27 | } 28 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/SqlEngineType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | /** 19 | * SQL 执行引擎类型 20 | */ 21 | public enum SqlEngineType { 22 | HIVE, SPARK, PHOENIX 23 | } 24 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/enums/UserRoleType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.enums; 17 | 18 | public enum UserRoleType { 19 | ADMIN_USER, 20 | GENERAL_USER; 21 | } 22 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/exception/DagException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.exception; 17 | 18 | public class DagException extends SqlException { 19 | private static final long serialVersionUID = 1L; 20 | 21 | public DagException(String msg) { 22 | super(msg); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/exception/DaoSemanticException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.exception; 17 | 18 | public class DaoSemanticException extends SqlException { 19 | private static final long serialVersionUID = 1L; 20 | 21 | public DaoSemanticException(String msg) { 22 | super(msg); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/exception/SqlException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.exception; 17 | 18 | /** 19 | * sql语句的语法错误等

20 | */ 21 | public class SqlException extends RuntimeException { 22 | 23 | private static final long serialVersionUID = 1L; 24 | 25 | public SqlException(String msg) { 26 | super(msg); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/mapper/utils/EnumFieldUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.mapper.utils; 17 | 18 | import org.apache.ibatis.type.EnumOrdinalTypeHandler; 19 | 20 | public class EnumFieldUtil { 21 | 22 | /** 23 | * 生成 enums 字段的字符串 24 | * 25 | * @param field 26 | * @param enumClass 27 | * @return 28 | */ 29 | public static String genFieldStr(String field, Class enumClass) { 30 | return "#{" + field + ",javaType=" + enumClass.getName() + ",typeHandler=" + EnumOrdinalTypeHandler.class.getName() + "}"; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/mapper/utils/EqualUtils.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.dao.mapper.utils; 2 | 3 | import java.util.Collection; 4 | import org.apache.commons.collections4.CollectionUtils; 5 | 6 | public class EqualUtils { 7 | 8 | /** 9 | * 两个数组是否相等 10 | * 11 | * @param a 12 | * @param b 13 | * @return 14 | */ 15 | public static boolean equalLists(Collection a, Collection b) { 16 | if (a == null && b == null) { 17 | return true; 18 | } 19 | 20 | if ((a == null && b != null) || a != null && b == null) { 21 | return false; 22 | } 23 | 24 | return CollectionUtils.isEqualCollection(a, b); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/model/AdHocJsonObject.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.model; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | public class AdHocJsonObject { 22 | /** 23 | * 返回的表头 24 | */ 25 | private List titles; 26 | 27 | /** 28 | * 返回的数据 29 | */ 30 | private List> values = new ArrayList<>(); 31 | 32 | /** 33 | * getter method 34 | * 35 | * @return the titles 36 | * @see AdHocJsonObject#titles 37 | */ 38 | public List getTitles() { 39 | return titles; 40 | } 41 | 42 | /** 43 | * setter method 44 | * 45 | * @param titles the titles to set 46 | * @see AdHocJsonObject#titles 47 | */ 48 | public void setTitles(List titles) { 49 | this.titles = titles; 50 | } 51 | 52 | /** 53 | * getter method 54 | * 55 | * @return the values 56 | * @see AdHocJsonObject#values 57 | */ 58 | public List> getValues() { 59 | return values; 60 | } 61 | 62 | /** 63 | * setter method 64 | * 65 | * @param values the values to set 66 | * @see AdHocJsonObject#values 67 | */ 68 | public void setValues(List> values) { 69 | this.values = values; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/model/ExecutionFlowError.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.model; 17 | 18 | /** 19 | * 工作流异常次数统计 20 | */ 21 | public class ExecutionFlowError { 22 | private String projectName; 23 | private String workflowName; 24 | private String owner; 25 | private String proxyUser; 26 | private int times; 27 | 28 | public String getProjectName() { 29 | return projectName; 30 | } 31 | 32 | public void setProjectName(String projectName) { 33 | this.projectName = projectName; 34 | } 35 | 36 | public String getWorkflowName() { 37 | return workflowName; 38 | } 39 | 40 | public void setWorkflowName(String workflowName) { 41 | this.workflowName = workflowName; 42 | } 43 | 44 | public String getOwner() { 45 | return owner; 46 | } 47 | 48 | public void setOwner(String owner) { 49 | this.owner = owner; 50 | } 51 | 52 | public int getTimes() { 53 | return times; 54 | } 55 | 56 | public void setTimes(int times) { 57 | this.times = times; 58 | } 59 | 60 | public String getProxyUser() { 61 | return proxyUser; 62 | } 63 | 64 | public void setProxyUser(String proxyUser) { 65 | this.proxyUser = proxyUser; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/model/FlowNodeRelation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.model; 17 | 18 | public class FlowNodeRelation { 19 | 20 | /** 21 | * 起点名称 22 | */ 23 | private String startNode; 24 | 25 | /** 26 | * 终点名称 27 | */ 28 | private String endNode; 29 | 30 | public FlowNodeRelation() { 31 | } 32 | 33 | public FlowNodeRelation(String startNode, String endNode) { 34 | this.startNode = startNode; 35 | this.endNode = endNode; 36 | } 37 | 38 | public String getStartNode() { 39 | return startNode; 40 | } 41 | 42 | public void setStartNode(String startNode) { 43 | this.startNode = startNode; 44 | } 45 | 46 | public String getEndNode() { 47 | return endNode; 48 | } 49 | 50 | public void setEndNode(String endNode) { 51 | this.endNode = endNode; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/model/MasterServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.model; 17 | 18 | import java.util.Date; 19 | 20 | public class MasterServer { 21 | 22 | /** 23 | * 服务 id 24 | */ 25 | private int id; 26 | 27 | /** 28 | * 服务 ip 信息 29 | */ 30 | private String host; 31 | 32 | /** 33 | * 服务端口 34 | */ 35 | private int port; 36 | 37 | /** 38 | * 创建时间 39 | */ 40 | private Date createTime; 41 | 42 | /** 43 | * 修改时间 44 | */ 45 | private Date modifyTime; 46 | 47 | public int getId() { 48 | return id; 49 | } 50 | 51 | public void setId(int id) { 52 | this.id = id; 53 | } 54 | 55 | public String getHost() { 56 | return host; 57 | } 58 | 59 | public void setHost(String host) { 60 | this.host = host; 61 | } 62 | 63 | public int getPort() { 64 | return port; 65 | } 66 | 67 | public void setPort(int port) { 68 | this.port = port; 69 | } 70 | 71 | public Date getCreateTime() { 72 | return createTime; 73 | } 74 | 75 | public void setCreateTime(Date createTime) { 76 | this.createTime = createTime; 77 | } 78 | 79 | public Date getModifyTime() { 80 | return modifyTime; 81 | } 82 | 83 | public void setModifyTime(Date modifyTime) { 84 | this.modifyTime = modifyTime; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/model/Session.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.model; 17 | 18 | import java.util.Date; 19 | 20 | public class Session { 21 | private String id; 22 | 23 | private int userId; 24 | 25 | private String ip; 26 | 27 | private Date lastLoginTime; 28 | 29 | public Session() { 30 | } 31 | 32 | public String getId() { 33 | return id; 34 | } 35 | 36 | public void setId(String id) { 37 | this.id = id; 38 | } 39 | 40 | public int getUserId() { 41 | return userId; 42 | } 43 | 44 | public void setUserId(int userId) { 45 | this.userId = userId; 46 | } 47 | 48 | public String getIp() { 49 | return ip; 50 | } 51 | 52 | public void setIp(String ip) { 53 | this.ip = ip; 54 | } 55 | 56 | public Date getLastLoginTime() { 57 | return lastLoginTime; 58 | } 59 | 60 | public void setLastLoginTime(Date lastLoginTime) { 61 | this.lastLoginTime = lastLoginTime; 62 | } 63 | 64 | @Override 65 | public String toString() { 66 | return "Session{" + 67 | "id='" + id + '\'' + 68 | ", userId=" + userId + 69 | ", ip='" + ip + '\'' + 70 | ", lastLoginTime=" + lastLoginTime + 71 | '}'; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/model/flow/DepWorkflow.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.model.flow; 17 | 18 | public class DepWorkflow { 19 | 20 | private String projectName; 21 | 22 | private String workflowName; 23 | 24 | public String getProjectName() { 25 | return projectName; 26 | } 27 | 28 | public void setProjectName(String projectName) { 29 | this.projectName = projectName; 30 | } 31 | 32 | public String getWorkflowName() { 33 | return workflowName; 34 | } 35 | 36 | public void setWorkflowName(String workflowName) { 37 | this.workflowName = workflowName; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/model/flow/FlowDag.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.model.flow; 17 | 18 | import com.baifendian.swordfish.dao.model.FlowNode; 19 | import com.baifendian.swordfish.dao.model.FlowNodeRelation; 20 | import java.util.List; 21 | 22 | public class FlowDag { 23 | 24 | /** 25 | * DAG 边信息 26 | **/ 27 | private List edges; 28 | 29 | /** 30 | * 表示 node 的内容详情 31 | */ 32 | private List nodes; 33 | 34 | /** 35 | * getter method 36 | * 37 | * @return the edges 38 | * @see FlowDag#edges 39 | */ 40 | public List getEdges() { 41 | return edges; 42 | } 43 | 44 | /** 45 | * setter method 46 | * 47 | * @param edges the edges to set 48 | * @see FlowDag#edges 49 | */ 50 | public void setEdges(List edges) { 51 | this.edges = edges; 52 | } 53 | 54 | /** 55 | * getter method 56 | * 57 | * @return the nodes 58 | * @see FlowDag#nodes 59 | */ 60 | public List getNodes() { 61 | return nodes; 62 | } 63 | 64 | /** 65 | * setter method 66 | * 67 | * @param nodes the nodes to set 68 | * @see FlowDag#nodes 69 | */ 70 | public void setNodes(List nodes) { 71 | this.nodes = nodes; 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /dao/src/main/java/com/baifendian/swordfish/dao/utils/json/JsonObjectSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | /* 18 | * Copyright (C) 2017 Baifendian Corporation 19 | * 20 | * Licensed under the Apache License, Version 2.0 (the "License"); 21 | * you may not use this file except in compliance with the License. 22 | * You may obtain a copy of the License at 23 | * 24 | * http://www.apache.org/licenses/LICENSE-2.0 25 | * 26 | * Unless required by applicable law or agreed to in writing, software 27 | * distributed under the License is distributed on an "AS IS" BASIS, 28 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 29 | * See the License for the specific language governing permissions and 30 | * limitations under the License. 31 | */ 32 | package com.baifendian.swordfish.dao.utils.json; 33 | 34 | import com.fasterxml.jackson.core.JsonGenerator; 35 | import com.fasterxml.jackson.databind.JsonSerializer; 36 | import com.fasterxml.jackson.databind.SerializerProvider; 37 | import java.io.IOException; 38 | 39 | /** 40 | * 字符串节点的自定义序列化

41 | */ 42 | public class JsonObjectSerializer extends JsonSerializer { 43 | 44 | @Override 45 | public void serialize(String value, JsonGenerator jgen, SerializerProvider provider) throws IOException { 46 | String v = value; 47 | jgen.writeRawValue(v); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /dao/src/main/resources/dao/data_source.properties: -------------------------------------------------------------------------------- 1 | # 比较基础的一些配置 2 | spring.datasource.type=com.alibaba.druid.pool.DruidDataSource 3 | spring.datasource.driver-class-name=com.mysql.jdbc.Driver 4 | spring.datasource.url=jdbc:mysql://172.18.1.22:3306/swordfish?useUnicode=true 5 | spring.datasource.username=swordfish 6 | spring.datasource.password=myswordfish 7 | 8 | # 下面为连接池的补充设置，应用到上面所有数据源中 9 | spring.datasource.initialSize=5 10 | spring.datasource.minIdle=5 11 | spring.datasource.maxActive=20 12 | 13 | # 配置获取连接等待超时的时间 14 | spring.datasource.maxWait=60000 15 | 16 | # 配置间隔多久才进行一次检测，检测需要关闭的空闲连接，单位是毫秒 17 | spring.datasource.timeBetweenEvictionRunsMillis=60000 18 | 19 | # 配置一个连接在池中最小生存的时间，单位是毫秒 20 | spring.datasource.minEvictableIdleTimeMillis=300000 21 | spring.datasource.validationQuery=SELECT 1 22 | spring.datasource.validationQueryTimeout=3 23 | spring.datasource.testWhileIdle=true 24 | spring.datasource.testOnBorrow=true 25 | spring.datasource.testOnReturn=false 26 | 27 | # 打开PSCache，并且指定每个连接上PSCache的大小 28 | spring.datasource.poolPreparedStatements=true 29 | spring.datasource.maxPoolPreparedStatementPerConnectionSize=20 -------------------------------------------------------------------------------- /dao/src/test/java/com/baifendian/swordfish/dao/AdHocDaoTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao; 17 | 18 | import org.junit.Test; 19 | 20 | public class AdHocDaoTest { 21 | 22 | @Test 23 | public void testgetAdHoc() { 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /dao/src/test/java/com/baifendian/swordfish/dao/FlowDaoTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao; 17 | 18 | import org.junit.Before; 19 | import org.junit.Test; 20 | 21 | public class FlowDaoTest { 22 | FlowDao flowDao; 23 | 24 | @Before 25 | public void before() { 26 | flowDao = DaoFactory.getDaoInstance(FlowDao.class); 27 | } 28 | 29 | @Test 30 | public void testQueryExecutionNodeLastAttempt() { 31 | } 32 | 33 | @Test 34 | public void testQueryAllExecutionFlow() { 35 | } 36 | 37 | @Test 38 | public void testQueryExecutionFlow() { 39 | } 40 | 41 | @Test 42 | public void testScheduleFlowToExecution() { 43 | } 44 | 45 | @Test 46 | public void testQuerySchedule() { 47 | } 48 | 49 | @Test 50 | public void testProjectFlowfindByName() { 51 | } 52 | 53 | @Test 54 | public void testProjectFlowfindById() { 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /dao/src/test/java/com/baifendian/swordfish/dao/datasource/DatabaseConfigurationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.datasource; 17 | 18 | import static org.junit.Assert.assertEquals; 19 | 20 | import com.baifendian.swordfish.dao.mapper.SessionMapper; 21 | import com.baifendian.swordfish.dao.model.Session; 22 | import java.util.Date; 23 | import java.util.UUID; 24 | import org.apache.ibatis.session.SqlSession; 25 | import org.junit.BeforeClass; 26 | import org.junit.Test; 27 | 28 | public class DatabaseConfigurationTest { 29 | private static SqlSession sqlSession; 30 | 31 | @BeforeClass 32 | public static void runOnceBeforeClass() { 33 | sqlSession = ConnectionFactory.getSqlSession(); 34 | } 35 | 36 | @Test 37 | public void testSession() { 38 | SessionMapper mapper = sqlSession.getMapper(SessionMapper.class); 39 | 40 | Session userSession = new Session(); 41 | 42 | String uuid = UUID.randomUUID().toString(); 43 | 44 | userSession.setId(uuid); 45 | userSession.setIp("localhost"); 46 | userSession.setUserId(0); 47 | userSession.setLastLoginTime(new Date()); 48 | 49 | mapper.insert(userSession); 50 | 51 | Session userSession2 = mapper.queryById(uuid); 52 | 53 | assertEquals(userSession2.getId(), userSession.getId()); 54 | } 55 | } -------------------------------------------------------------------------------- /dao/src/test/java/com/baifendian/swordfish/dao/mapper/FlowNodeMapperTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.mapper; 17 | 18 | import com.baifendian.swordfish.dao.datasource.ConnectionFactory; 19 | import com.fasterxml.jackson.core.JsonProcessingException; 20 | import org.junit.Before; 21 | import org.junit.Test; 22 | 23 | public class FlowNodeMapperTest { 24 | FlowNodeMapper flowNodeMapper; 25 | 26 | @Before 27 | public void before() { 28 | flowNodeMapper = ConnectionFactory.getSqlSession().getMapper(FlowNodeMapper.class); 29 | } 30 | 31 | @Test 32 | public void testSelectByFlowId() { 33 | } 34 | 35 | @Test 36 | public void testSelectByFlowIds() { 37 | } 38 | 39 | @Test 40 | public void testInsert() throws JsonProcessingException { 41 | // FlowNode flowNode = new FlowNode(); 42 | // 43 | // flowNode.setName("shelljob4"); 44 | // flowNode.setFlowId(2); 45 | // flowNode.setDesc("shelljob4"); 46 | // flowNode.setType("SHELL"); 47 | // flowNode.setParameter("{\"script\":\"echo shelljob4\"}"); 48 | // flowNode.setDepList(new ArrayList()); 49 | // 50 | // flowNodeMapper.insert(flowNode); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /dao/src/test/java/com/baifendian/swordfish/dao/mapper/ProjectFlowMapperTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.mapper; 17 | 18 | import static junit.framework.TestCase.assertTrue; 19 | 20 | import com.baifendian.swordfish.dao.datasource.ConnectionFactory; 21 | import com.baifendian.swordfish.dao.model.ProjectFlow; 22 | import org.junit.Before; 23 | import org.junit.Test; 24 | 25 | public class ProjectFlowMapperTest { 26 | ProjectFlowMapper projectFlowMapper; 27 | 28 | @Before 29 | public void before() { 30 | projectFlowMapper = ConnectionFactory.getSqlSession().getMapper(ProjectFlowMapper.class); 31 | } 32 | 33 | @Test 34 | public void testFindByName() { 35 | ProjectFlow projectFlow = projectFlowMapper.findByName(1,"bdi_base"); 36 | assertTrue(projectFlow!=null); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /dao/src/test/java/com/baifendian/swordfish/dao/mapper/ScheduleMapperTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.mapper; 17 | 18 | import com.fasterxml.jackson.core.JsonProcessingException; 19 | import org.junit.Test; 20 | 21 | public class ScheduleMapperTest { 22 | 23 | @Test 24 | public void testInsert() throws JsonProcessingException { 25 | // ScheduleMapper scheduleMapper = ConnectionFactory.getSqlSession().getMapper(ScheduleMapper.class); 26 | // 27 | // Schedule schedule = new Schedule(); 28 | // schedule.setFlowId(2); 29 | // schedule.setStartDate(new Date()); 30 | // schedule.setEndDate(new Date(System.currentTimeMillis() + 24 * 3600 * 1000)); 31 | // schedule.setCrontab("10 * * * * ?"); 32 | // schedule.setCreateTime(new Date()); 33 | // schedule.setNotifyMails(Arrays.asList("aaron.liu@baifendian.com")); 34 | // schedule.setDepPolicy(DepPolicyType.DEP_PRE); 35 | // schedule.setFailurePolicy(FailurePolicyType.CONTINUE); 36 | // schedule.setMaxTryTimes(3); 37 | // schedule.setTimeout(5000); 38 | // schedule.setNotifyType(NotifyType.ALL); 39 | // schedule.setModifyTime(new Date()); 40 | // schedule.setOwner("1"); 41 | // schedule.setScheduleStatus(ScheduleStatus.OFFLINE); 42 | // scheduleMapper.insert(schedule); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /dao/src/test/java/com/baifendian/swordfish/dao/model/flow/FlowDagTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.dao.model.flow; 17 | 18 | import org.junit.Test; 19 | 20 | public class FlowDagTest { 21 | 22 | @Test 23 | public void testFlowDag() { 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /docs/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/docs/architecture.png -------------------------------------------------------------------------------- /docs/dependency_policy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/docs/dependency_policy.png -------------------------------------------------------------------------------- /docs/fault_tolerance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/docs/fault_tolerance.png -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/common/ResultCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.common; 17 | 18 | import java.util.Date; 19 | 20 | /** 21 | * 执行结果回调处理

22 | */ 23 | public interface ResultCallback { 24 | 25 | /** 26 | * 处理执行结果 27 | * 28 | * @param execResult 29 | * @param startTime 30 | * @param endTime 31 | */ 32 | void handleResult(ExecResult execResult, Date startTime, Date endTime); 33 | } 34 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/exception/ConfigException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.exception; 17 | 18 | /** 19 | * 配置信息错误异常

20 | * 21 | */ 22 | public class ConfigException extends RuntimeException { 23 | private static final long serialVersionUID = 1L; 24 | 25 | /** 26 | * @param msg 27 | */ 28 | public ConfigException(String msg) { 29 | super(msg); 30 | } 31 | 32 | /** 33 | * @param msg 34 | * @param cause 35 | */ 36 | public ConfigException(String msg, Throwable cause) { 37 | super(msg, cause); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/exception/ExecException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.exception; 17 | 18 | public class ExecException extends RuntimeException { 19 | /** 20 | * Serial version UID 21 | */ 22 | private static final long serialVersionUID = 1L; 23 | 24 | /** 25 | * @param msg 26 | */ 27 | public ExecException(String msg) { 28 | super(msg); 29 | } 30 | 31 | /** 32 | * @param msg 33 | * @param cause 34 | */ 35 | public ExecException(String msg, Throwable cause) { 36 | super(msg, cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/exception/ExecTimeoutException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.exception; 17 | 18 | /** 19 | * 执行超时异常

20 | * 21 | */ 22 | public class ExecTimeoutException extends RuntimeException { 23 | /** 24 | * Serial version UID 25 | */ 26 | private static final long serialVersionUID = 1L; 27 | 28 | /** 29 | * @param msg 30 | */ 31 | public ExecTimeoutException(String msg) { 32 | super(msg); 33 | } 34 | 35 | /** 36 | * @param msg 37 | * @param cause 38 | */ 39 | public ExecTimeoutException(String msg, Throwable cause) { 40 | super(msg, cause); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/job/impexp/Args/HqlColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.job.impexp.Args; 17 | 18 | import com.baifendian.swordfish.common.job.struct.node.impexp.column.HiveColumn; 19 | import org.apache.commons.lang.StringUtils; 20 | 21 | /** 22 | * Hive sql 中字段参数 23 | */ 24 | public class HqlColumn { 25 | 26 | private String name; 27 | private String type; 28 | 29 | public HqlColumn() { 30 | } 31 | 32 | public HqlColumn(String name, String type) { 33 | this.name = name; 34 | this.type = type; 35 | } 36 | 37 | public String getName() { 38 | return name; 39 | } 40 | 41 | public void setName(String name) { 42 | this.name = name; 43 | } 44 | 45 | public String getType() { 46 | return type; 47 | } 48 | 49 | public void setType(String type) { 50 | this.type = type; 51 | } 52 | 53 | /** 54 | * 判断一个 HqlColumn 和一个 hiveColumn 是否相同 55 | */ 56 | public boolean equals(HiveColumn hiveColumn) { 57 | return StringUtils.equalsIgnoreCase(name, hiveColumn.getName()) && 58 | StringUtils.containsIgnoreCase(type, hiveColumn.getType().name()); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/job/impexp/Args/ReaderArg.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.job.impexp.Args; 17 | 18 | /** 19 | * 读参数接口 20 | */ 21 | public interface ReaderArg { 22 | 23 | String dataxName(); 24 | } 25 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/job/impexp/Args/WriterArg.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.job.impexp.Args; 17 | 18 | /** 19 | * 写参数接口 20 | */ 21 | public interface WriterArg { 22 | 23 | String dataxName(); 24 | } 25 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/job/impexp/ImpExpJobConst.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.job.impexp; 17 | 18 | import com.baifendian.swordfish.common.enums.WriteHdfsType; 19 | 20 | /** 21 | * 导入导出任务常量 22 | */ 23 | public class ImpExpJobConst { 24 | 25 | /** 26 | * 写入 hdfs 临时文件的默认分隔符 27 | */ 28 | public static final String DEFAULT_DELIMITER = ","; 29 | 30 | /** 31 | * 写入 hdfs 临时文件的默认文件类型 32 | */ 33 | public static final WriteHdfsType DEFAULT_FILE_TYPE = WriteHdfsType.ORC; 34 | 35 | /** 36 | * hive 的默认数据库 37 | */ 38 | public static final String DEFAULT_DB = "default"; 39 | 40 | /** 41 | * 默认创建的 dataX 配置文件文件名 42 | */ 43 | public static final String DATAX_FILE_NAME = "dataXJson"; 44 | 45 | /** 46 | * dataX 配置文件结构 47 | */ 48 | public static final String DATAX_JSON = "'{'\"job\":'{'\"content\":['{'\"reader\":'{'\"name\":\"{0}\",\"parameter\":{1}},\"writer\":'{'\"name\":\"{2}\",\"parameter\":{3}}}],\"setting\":{4}}}"; 49 | 50 | /** 51 | * dataX 命令 52 | */ 53 | public static final String COMMAND = "python {0} {1}"; 54 | 55 | /** 56 | * HIVE 驱动 57 | */ 58 | public static final String HIVE_DRIVER = "org.apache.hive.jdbc.HiveDriver"; 59 | } 60 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/job/mr/HadoopJarArgsConst.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.job.mr; 17 | 18 | /** 19 | * Hadoop jar 参数

20 | */ 21 | public class HadoopJarArgsConst { 22 | 23 | /** 24 | * -D 25 | */ 26 | public static final String D = "-D"; 27 | 28 | /** 29 | * -libjars 30 | */ 31 | public static final String JARS = "-libjars"; 32 | 33 | /** 34 | * -files 35 | */ 36 | public static final String FILES = "-files"; 37 | 38 | /** 39 | * -archives 40 | */ 41 | public static final String ARCHIVES = "-archives"; 42 | 43 | /** 44 | * -Dmapreduce.job.queuename={queue} 45 | */ 46 | public static final String QUEUE = "mapreduce.job.queuename"; 47 | } 48 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/job/storm/StormSubmitArgsConst.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.job.storm; 17 | 18 | /** 19 | * storm 任务常量 20 | */ 21 | public class StormSubmitArgsConst { 22 | 23 | public static String STORM_COMMAND = "storm"; 24 | 25 | public static String JAR = "jar"; 26 | 27 | public static String SQL = "sql"; 28 | 29 | public static String SHELL = "shell"; 30 | 31 | public static String JARS = "--jars"; 32 | 33 | public static String ARTIFACTS = "--artifacts"; 34 | 35 | public static String ARTIFACTREPOSITORIES = "--artifactRepositories"; 36 | 37 | } 38 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/job/virtual/VirtualJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.job.virtual; 17 | 18 | import com.baifendian.swordfish.common.job.struct.node.BaseParam; 19 | import com.baifendian.swordfish.execserver.job.Job; 20 | import com.baifendian.swordfish.execserver.job.JobProps; 21 | import org.slf4j.Logger; 22 | 23 | /** 24 | * 空操作的作业

25 | */ 26 | public class VirtualJob extends Job { 27 | 28 | /** 29 | * @param props 作业配置信息,各类作业根据此配置信息生成具体的作业 30 | * @param logger 日志 31 | */ 32 | public VirtualJob(JobProps props, boolean isLongJob, Logger logger) { 33 | super(props, isLongJob, logger); 34 | } 35 | 36 | @Override 37 | public void process() throws Exception { 38 | // 设置相关状态 39 | exitCode = 0; 40 | } 41 | 42 | @Override 43 | public BaseParam getParam() { 44 | return null; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/utils/LoggerUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.utils; 17 | 18 | import com.baifendian.swordfish.common.utils.http.HttpUtil; 19 | import org.apache.commons.lang3.StringUtils; 20 | 21 | /** 22 | * 日志记录工具类

23 | */ 24 | public class LoggerUtil { 25 | 26 | /** 27 | * 生成 jobId 并返回 28 | * 29 | * @param prefix 30 | * @param execId 31 | * @param nodeName 32 | * @return 33 | */ 34 | public static String genJobId(String prefix, long execId, String nodeName) { 35 | if (StringUtils.isEmpty(nodeName)) { 36 | return String.format("%s_%s", prefix, execId); 37 | } 38 | 39 | String postfix = HttpUtil.getMd5(nodeName).substring(0, 8); 40 | 41 | return String.format("%s_%s_%s", prefix, execId, postfix); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /exec-server/src/main/java/com/baifendian/swordfish/execserver/utils/ResultHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.utils; 17 | 18 | import com.baifendian.swordfish.rpc.RetInfo; 19 | 20 | /** 21 | * 返回结果帮助类

22 | * 23 | */ 24 | public class ResultHelper { 25 | /** 26 | * SUCCESS 27 | */ 28 | public static final RetInfo SUCCESS = new RetInfo(0, "success"); 29 | 30 | /** 31 | * 创建一个错误异常的返回包

32 | * 33 | * @return {@link RetInfo} 34 | */ 35 | public static RetInfo createErrorResult(String msg) { 36 | return new RetInfo(1, msg); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /exec-server/src/main/resources/common/hive/hive.properties: -------------------------------------------------------------------------------- 1 | # hive metastore 地址 2 | hive.metastore.uris=thrift://172.18.1.22:9083 3 | 4 | # hive thrift 地址 5 | hive.thrift.uris=jdbc:hive2://172.18.1.22:10000 -------------------------------------------------------------------------------- /exec-server/src/main/resources/common/phoenix.properties: -------------------------------------------------------------------------------- 1 | # phoenix 地址 2 | phoenix.host = 172.24.8.97 3 | 4 | # phoenix port 5 | phoenix.port = 8765 -------------------------------------------------------------------------------- /exec-server/src/main/resources/worker.properties: -------------------------------------------------------------------------------- 1 | # executor 端口 2 | executor.port=12001 3 | 4 | # executor 心跳汇报间隔 5 | executor.heartbeat.interval=8 6 | 7 | ## adhoc 运行线程池大小 8 | executor.adhocrunner.threads=20 9 | 10 | ## flow 运行线程池大小 11 | executor.flowrunner.threads=50 12 | 13 | ## node 运行线程池大小 14 | executor.noderunner.threads=80 15 | 16 | ## streaming 运行线程池大小 17 | executor.streaming.threads=10 18 | 19 | # datax 入口 20 | executor.datax.home=/home/swordfish/DataX/target/datax/datax -------------------------------------------------------------------------------- /exec-server/src/test/java/com/baifendian/swordfish/execserver/engine/hive/HiveMetaExecTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.engine.hive; 17 | 18 | import org.junit.Test; 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | public class HiveMetaExecTest { 23 | 24 | @Test 25 | public void testGetTables() { 26 | try { 27 | Logger logger = LoggerFactory.getLogger(HiveMetaExec.class); 28 | 29 | HiveMetaExec hiveMetaExec = new HiveMetaExec(logger); 30 | 31 | int times = 100; 32 | 33 | long start = System.currentTimeMillis(); 34 | 35 | for (int i = 0; i < times; ++i) { 36 | hiveMetaExec.getTables("dw"); 37 | } 38 | 39 | long end = System.currentTimeMillis(); 40 | 41 | System.out.println("get tables time(ms): " + (end - start) / times); 42 | 43 | start = System.currentTimeMillis(); 44 | 45 | for (int i = 0; i < times; ++i) { 46 | hiveMetaExec.getTableObjectsByName("dw"); 47 | } 48 | 49 | end = System.currentTimeMillis(); 50 | 51 | System.out.println("get tables objects time(ms): " + (end - start) / times); 52 | } catch (Exception e) { 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /exec-server/src/test/java/com/baifendian/swordfish/execserver/engine/hive/HiveUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.engine.hive; 17 | 18 | import static junit.framework.TestCase.assertFalse; 19 | import static junit.framework.TestCase.assertTrue; 20 | 21 | import org.junit.Test; 22 | 23 | public class HiveUtilTest { 24 | 25 | @Test 26 | public void testDDL() { 27 | assertFalse(HiveUtil.isTokDDL("")); 28 | assertTrue(HiveUtil.isTokDDL("create table abc")); 29 | assertTrue(HiveUtil.isTokDDL("alter table abc")); 30 | assertFalse(HiveUtil.isTokDDL("select * from abc")); 31 | assertFalse(HiveUtil.isTokDDL(null)); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /exec-server/src/test/java/com/baifendian/swordfish/execserver/parameter/ParamHelperTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.execserver.parameter; 17 | 18 | import com.baifendian.swordfish.dao.enums.ExecType; 19 | import java.util.Date; 20 | import java.util.Map; 21 | import org.junit.Test; 22 | 23 | public class ParamHelperTest { 24 | 25 | @Test 26 | public void testResolvePlaceholders() { 27 | Map definedParamMap = SystemParamManager 28 | .buildSystemParam(ExecType.DIRECT, new Date(), 200, "job id 001"); 29 | 30 | // 解析参数 31 | String sqls = "${sf.system.bizdate}"; 32 | System.out.println(ParamHelper.resolvePlaceholders(sqls, definedParamMap)); 33 | 34 | sqls = "${sf.system.bizcurdate}"; 35 | System.out.println(ParamHelper.resolvePlaceholders(sqls, definedParamMap)); 36 | 37 | sqls = "${sf.system.cyctime}"; 38 | System.out.println(ParamHelper.resolvePlaceholders(sqls, definedParamMap)); 39 | 40 | sqls = "$[yyyyMMdd]"; 41 | System.out.println(ParamHelper.resolvePlaceholders(sqls, definedParamMap)); 42 | 43 | sqls = "${sf.system.execId}"; 44 | System.out.println(ParamHelper.resolvePlaceholders(sqls, definedParamMap)); 45 | 46 | sqls = "${sf.system.jobId}"; 47 | System.out.println(ParamHelper.resolvePlaceholders(sqls, definedParamMap)); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/exception/ExecException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.exception; 17 | 18 | public class ExecException extends RuntimeException { 19 | /** 20 | * Serial version UID 21 | */ 22 | private static final long serialVersionUID = 1L; 23 | 24 | /** 25 | * @param msg 26 | */ 27 | public ExecException(String msg) { 28 | super(msg); 29 | } 30 | 31 | /** 32 | * @param msg 33 | * @param cause 34 | */ 35 | public ExecException(String msg, Throwable cause) { 36 | super(msg, cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/exception/MasterException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.exception; 17 | 18 | public class MasterException extends RuntimeException { 19 | 20 | public MasterException(String msg) { 21 | super(msg); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/exception/QuartzException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.exception; 17 | 18 | public class QuartzException extends RuntimeException { 19 | /** 20 | * Serial version UID 21 | */ 22 | private static final long serialVersionUID = 1L; 23 | 24 | /** 25 | * @param msg 26 | */ 27 | public QuartzException(String msg) { 28 | super(msg); 29 | } 30 | 31 | /** 32 | * @param msg 33 | * @param cause 34 | */ 35 | public QuartzException(String msg, Throwable cause) { 36 | super(msg, cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/exec/ExecutorServerInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.exec; 17 | 18 | import com.baifendian.swordfish.rpc.HeartBeatData; 19 | 20 | /** 21 | * executor 的心跳信息 22 | */ 23 | public class ExecutorServerInfo { 24 | 25 | private String host; 26 | 27 | private int port; 28 | 29 | private HeartBeatData heartBeatData; 30 | 31 | public String getHost() { 32 | return host; 33 | } 34 | 35 | public void setHost(String host) { 36 | this.host = host; 37 | } 38 | 39 | public int getPort() { 40 | return port; 41 | } 42 | 43 | public void setPort(int port) { 44 | this.port = port; 45 | } 46 | 47 | public HeartBeatData getHeartBeatData() { 48 | return heartBeatData; 49 | } 50 | 51 | public void setHeartBeatData(HeartBeatData heartBeatData) { 52 | this.heartBeatData = heartBeatData; 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return "ExecutorServerInfo{" + 58 | "host='" + host + '\'' + 59 | ", port=" + port + 60 | ", heartBeatData=" + heartBeatData + 61 | '}'; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/master/ExecFlowInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.master; 17 | 18 | public class ExecFlowInfo { 19 | 20 | private String host; 21 | private int port; 22 | private int execId; 23 | 24 | public ExecFlowInfo() { 25 | } 26 | 27 | public ExecFlowInfo(int execId) { 28 | this.execId = execId; 29 | } 30 | 31 | public ExecFlowInfo(String host, int port, int execId) { 32 | this.host = host; 33 | this.port = port; 34 | this.execId = execId; 35 | } 36 | 37 | public String getHost() { 38 | return host; 39 | } 40 | 41 | public void setHost(String host) { 42 | this.host = host; 43 | } 44 | 45 | public int getPort() { 46 | return port; 47 | } 48 | 49 | public void setPort(int port) { 50 | this.port = port; 51 | } 52 | 53 | public int getExecId() { 54 | return execId; 55 | } 56 | 57 | public void setExecId(int execId) { 58 | this.execId = execId; 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return "ExecFlowInfo{" + 64 | "host='" + host + '\'' + 65 | ", port=" + port + 66 | ", execId=" + execId + 67 | '}'; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/ResultDetailHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils; 17 | 18 | import com.baifendian.swordfish.rpc.RetResultInfo; 19 | import java.util.List; 20 | 21 | public class ResultDetailHelper { 22 | 23 | /** 24 | * 创建一个错误异常的返回包

25 | * 26 | * @param msg 27 | * @return {@link RetResultInfo} 28 | */ 29 | public static RetResultInfo createErrorResult(String msg) { 30 | return new RetResultInfo(ResultHelper.createErrorResult(msg), null); 31 | } 32 | 33 | /** 34 | * 创建成功的返回包

35 | * 36 | * @param execIds 37 | * @return {@link RetResultInfo} 38 | */ 39 | public static RetResultInfo createSuccessResult(List execIds) { 40 | return new RetResultInfo(ResultHelper.SUCCESS, execIds); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/ResultHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils; 17 | 18 | import com.baifendian.swordfish.rpc.RetInfo; 19 | 20 | /** 21 | * 返回结果帮助类

22 | */ 23 | public class ResultHelper { 24 | /** 25 | * SUCCESS 26 | */ 27 | public static final RetInfo SUCCESS = new RetInfo(0, "success"); 28 | 29 | /** 30 | * 创建一个错误异常的返回包

31 | * 32 | * @param msg 33 | * @return {@link RetInfo} 34 | */ 35 | public static RetInfo createErrorResult(String msg) { 36 | return new RetInfo(1, msg); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/crontab/CycleHelperFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils.crontab; 17 | 18 | import com.cronutils.model.Cron; 19 | 20 | /** 21 | * crontab周期工具类工厂 22 | */ 23 | public class CycleHelperFactory { 24 | 25 | public static CycleHelper min(Cron cron) { 26 | return new MinHelper(cron); 27 | } 28 | 29 | public static CycleHelper hour(Cron cron) { 30 | return new HourHelper(cron); 31 | } 32 | 33 | public static CycleHelper day(Cron cron) { 34 | return new DayHelper(cron); 35 | } 36 | 37 | public static CycleHelper week(Cron cron) { 38 | return new WeekHelper(cron); 39 | } 40 | 41 | public static CycleHelper month(Cron cron) { 42 | return new MonthHelper(cron); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/crontab/DayHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils.crontab; 17 | 18 | import com.baifendian.swordfish.dao.enums.ScheduleType; 19 | import com.cronutils.model.Cron; 20 | import com.cronutils.model.field.expression.Always; 21 | import com.cronutils.model.field.expression.On; 22 | import com.cronutils.model.field.expression.QuestionMark; 23 | 24 | /** 25 | * 天周期判断类 26 | */ 27 | public class DayHelper extends CycleHelper { 28 | 29 | public DayHelper(Cron cron) { 30 | super(cron); 31 | } 32 | 33 | @Override 34 | protected ScheduleType getCycle() { 35 | 36 | if (minField.getExpression() instanceof On 37 | && hourField.getExpression() instanceof On 38 | && dayOfMonthField.getExpression() instanceof Always 39 | && dayOfWeekField.getExpression() instanceof QuestionMark 40 | && monthField.getExpression() instanceof Always) { 41 | return ScheduleType.DAY; 42 | } 43 | 44 | return null; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/crontab/HourHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils.crontab; 17 | 18 | import com.baifendian.swordfish.dao.enums.ScheduleType; 19 | import com.cronutils.model.Cron; 20 | import com.cronutils.model.field.expression.Always; 21 | import com.cronutils.model.field.expression.Every; 22 | import com.cronutils.model.field.expression.On; 23 | import com.cronutils.model.field.expression.QuestionMark; 24 | 25 | /** 26 | * 小时周期判断工具 27 | */ 28 | public class HourHelper extends CycleHelper { 29 | 30 | public HourHelper(Cron cron) { 31 | super(cron); 32 | } 33 | 34 | @Override 35 | protected ScheduleType getCycle() { 36 | if (minField.getExpression() instanceof On 37 | && hourField.getExpression() instanceof Every 38 | && dayOfMonthField.getExpression() instanceof Always 39 | && dayOfWeekField.getExpression() instanceof QuestionMark 40 | && monthField.getExpression() instanceof Always) { 41 | return ScheduleType.HOUR; 42 | } 43 | 44 | return null; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/crontab/MinHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils.crontab; 17 | 18 | import com.baifendian.swordfish.dao.enums.ScheduleType; 19 | import com.cronutils.model.Cron; 20 | import com.cronutils.model.field.expression.Always; 21 | import com.cronutils.model.field.expression.Between; 22 | import com.cronutils.model.field.expression.Every; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | /** 27 | * 判断是否是分钟周期 28 | */ 29 | public class MinHelper extends CycleHelper { 30 | 31 | 32 | private static final Logger logger = LoggerFactory.getLogger(MinHelper.class); 33 | 34 | public MinHelper(Cron cron) { 35 | super(cron); 36 | } 37 | 38 | @Override 39 | protected ScheduleType getCycle() { 40 | if (minField.getExpression() instanceof Every 41 | && hourField.getExpression() instanceof Between 42 | && dayOfMonthField.getExpression() instanceof Always 43 | && monthField.getExpression() instanceof Always) { 44 | return ScheduleType.MINUTE; 45 | } 46 | 47 | return null; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/crontab/MonthHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils.crontab; 17 | 18 | import com.baifendian.swordfish.dao.enums.ScheduleType; 19 | import com.cronutils.model.Cron; 20 | import com.cronutils.model.field.expression.Always; 21 | import com.cronutils.model.field.expression.And; 22 | import com.cronutils.model.field.expression.On; 23 | import com.cronutils.model.field.expression.QuestionMark; 24 | 25 | /** 26 | * 月周期判断工具类 27 | */ 28 | public class MonthHelper extends CycleHelper { 29 | 30 | public MonthHelper(Cron cron) { 31 | super(cron); 32 | } 33 | 34 | @Override 35 | protected ScheduleType getCycle() { 36 | if (minField.getExpression() instanceof On 37 | && hourField.getExpression() instanceof On 38 | && (dayOfMonthField.getExpression() instanceof On || dayOfMonthField.getExpression() instanceof And) 39 | && dayOfWeekField.getExpression() instanceof QuestionMark 40 | && monthField.getExpression() instanceof Always) { 41 | return ScheduleType.MONTH; 42 | } 43 | 44 | return null; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/crontab/Next.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils.crontab; 17 | 18 | import com.baifendian.swordfish.dao.enums.ScheduleType; 19 | import com.cronutils.model.Cron; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | 23 | /** 24 | * 链接判断工具 25 | */ 26 | public class Next extends CycleHelper { 27 | private final List cycleHelperList = new ArrayList<>(); 28 | 29 | public Next(Cron cron) { 30 | super(cron); 31 | } 32 | 33 | public Next next(CycleHelper cycleHelper) { 34 | cycleHelperList.add(cycleHelper); 35 | return this; 36 | } 37 | 38 | @Override 39 | protected ScheduleType getCycle() { 40 | for (CycleHelper cycleHelper : cycleHelperList) { 41 | ScheduleType cycle = cycleHelper.getCycle(); 42 | if (cycle != null) { 43 | return cycle; 44 | } 45 | } 46 | 47 | return null; 48 | } 49 | } -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/crontab/SecHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils.crontab; 17 | 18 | import com.baifendian.swordfish.dao.enums.ScheduleType; 19 | import com.cronutils.model.Cron; 20 | 21 | /** 22 | * 秒周期工具类 23 | */ 24 | public class SecHelper extends CycleHelper { 25 | 26 | public SecHelper(Cron cron) { 27 | super(cron); 28 | } 29 | 30 | @Override 31 | protected ScheduleType getCycle() { 32 | // 当前还不支持秒周期 33 | return null; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /master-server/src/main/java/com/baifendian/swordfish/masterserver/utils/crontab/WeekHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.masterserver.utils.crontab; 17 | 18 | import com.baifendian.swordfish.dao.enums.ScheduleType; 19 | import com.cronutils.model.Cron; 20 | import com.cronutils.model.field.expression.Always; 21 | import com.cronutils.model.field.expression.And; 22 | import com.cronutils.model.field.expression.On; 23 | import com.cronutils.model.field.expression.QuestionMark; 24 | 25 | /** 26 | * 周周期判断工具类 27 | */ 28 | public class WeekHelper extends CycleHelper { 29 | public WeekHelper(Cron cron) { 30 | super(cron); 31 | } 32 | 33 | @Override 34 | protected ScheduleType getCycle() { 35 | if (minField.getExpression() instanceof On 36 | && hourField.getExpression() instanceof On 37 | && dayOfMonthField.getExpression() instanceof QuestionMark 38 | && (dayOfWeekField.getExpression() instanceof And || dayOfWeekField.getExpression() instanceof On) 39 | && monthField.getExpression() instanceof Always) { 40 | return ScheduleType.WEEK; 41 | } 42 | 43 | return null; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /master-server/src/main/resources/master.properties: -------------------------------------------------------------------------------- 1 | # master request for worker, if failed, try times, 0 means not try 2 | masterToWorker.failRetry.count=2 3 | 4 | # master request for worker, execution queue 5 | masterToWorker.executionFlow.queueSize=10000 6 | 7 | # master heartbeat interval 8 | master.heartbeat.check.interval=8 9 | 10 | # master heartbeat timeout check 11 | master.heartbeat.timeout.interval=28 12 | 13 | # master and executor max allow ticket interval(ms) 14 | master.executor.maxallow.ticket.interval=2000 15 | 16 | # master start port 17 | master.port=12000 18 | 19 | # streaming job check interval 20 | streaming.check.interval=15 21 | 22 | # streaming job timeout threshold, if job submit but not execution long times 23 | streaming.timeout.threshold=1200 -------------------------------------------------------------------------------- /master-server/src/main/resources/quartz.properties: -------------------------------------------------------------------------------- 1 | # threadPool 2 | org.quartz.scheduler.instanceName=QuartzScheduler 3 | org.quartz.threadPool.threadCount=24 4 | org.quartz.threadPool.class=org.quartz.simpl.SimpleThreadPool 5 | 6 | # jobStore 7 | org.quartz.jobStore.class=org.quartz.impl.jdbcjobstore.JobStoreTX 8 | org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate 9 | org.quartz.jobStore.tablePrefix=QRTZ_ 10 | org.quartz.jobStore.dataSource=myDS 11 | org.quartz.jobStore.misfireThreshold=60000 12 | 13 | # datasoure 14 | org.quartz.dataSource.myDS.driver=com.mysql.jdbc.Driver 15 | 16 | # quartz datasource 17 | org.quartz.dataSource.myDS.URL=jdbc:mysql://172.18.1.22:3306/swordfish?autoReconnect=true 18 | 19 | # quartz mysql user name & password 20 | org.quartz.dataSource.myDS.user=swordfish 21 | org.quartz.dataSource.myDS.password=myswordfish 22 | 23 | # connection parameters 24 | org.quartz.dataSource.myDS.maxConnections=5 25 | org.quartz.dataSource.myDS.validationQuery=select 1 26 | org.quartz.dataSource.myDS.idleConnectionValidationSeconds=50 27 | org.quartz.dataSource.myDS.validateOnCheckout=true 28 | 29 | org.quartz.jobStore.txIsolationLevelSerializable=false -------------------------------------------------------------------------------- /package.xml: -------------------------------------------------------------------------------- 1 | 5 | all 6 | 7 | tar.gz 8 | dir 9 | 10 | false 11 | 12 | 13 | web-server/target/swordfish-web-server-${project.version} 14 | 15 | **/*.* 16 | 17 | . 18 | 19 | 20 | master-server/target/swordfish-master-server-${project.version} 21 | 22 | **/*.* 23 | 24 | . 25 | 26 | 27 | exec-server/target/swordfish-exec-server-${project.version} 28 | 29 | **/*.* 30 | 31 | . 32 | 33 | 34 | sql 35 | 36 | **/*.* 37 | 38 | ./sql 39 | 40 | 41 | ./ 42 | 43 | *.sh 44 | 45 | ./script 46 | 47 | 48 | -------------------------------------------------------------------------------- /python-client/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'smile8' 2 | -------------------------------------------------------------------------------- /python-client/adhoc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/adhoc/__init__.py -------------------------------------------------------------------------------- /python-client/adhoc/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/adhoc/__init__.pyc -------------------------------------------------------------------------------- /python-client/adhoc/test_adhoc.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | 6 | 7 | class TestAdhoc(unittest.TestCase): 8 | def setUp(self): 9 | print 'setUp...' 10 | 11 | def tearDown(self): 12 | print 'tearDown...' 13 | 14 | def test1(self): 15 | self.assertEqual(2, 2) -------------------------------------------------------------------------------- /python-client/adhoc/test_adhoc.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/adhoc/test_adhoc.pyc -------------------------------------------------------------------------------- /python-client/login/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'smile8' 2 | -------------------------------------------------------------------------------- /python-client/login/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/login/__init__.pyc -------------------------------------------------------------------------------- /python-client/login/login.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import requests 5 | import settings 6 | 7 | 8 | def get_session(user, password): 9 | '''登陆, 获取 session 信息''' 10 | r = requests.post('%s/login' % (settings.g_url), data={'name': user, 'password': password}) 11 | 12 | return r.json().get('sessionId') -------------------------------------------------------------------------------- /python-client/login/login.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/login/login.pyc -------------------------------------------------------------------------------- /python-client/login/test_login.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | 6 | 7 | class TestLogin(unittest.TestCase): 8 | def setUp(self): 9 | print 'setUp...' 10 | 11 | def tearDown(self): 12 | print 'tearDown...' 13 | 14 | def test1(self): 15 | self.assertEqual(2, 2) -------------------------------------------------------------------------------- /python-client/login/test_login.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/login/test_login.pyc -------------------------------------------------------------------------------- /python-client/project/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'smile8' 2 | -------------------------------------------------------------------------------- /python-client/project/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/project/__init__.pyc -------------------------------------------------------------------------------- /python-client/project/project.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import json 5 | import requests 6 | import settings 7 | from login import login 8 | 9 | 10 | def create_project(sessionId, name, desc): 11 | '''创建项目''' 12 | r = requests.post('%s/projects/%s' % (settings.g_url, name), 13 | headers={'sessionId': sessionId}, 14 | data={'desc': desc}) 15 | 16 | return (r.status_code, r.json()) 17 | 18 | 19 | def modify_project(sessionId, name, desc): 20 | '''修改项目''' 21 | r = requests.patch('%s/projects/%s' % (settings.g_url, name), 22 | headers={'sessionId': sessionId}, 23 | data={'desc': desc}) 24 | 25 | return (r.status_code, r.json()) 26 | 27 | 28 | def delete_project(sessionId, name): 29 | '''删除项目''' 30 | r = requests.delete('%s/projects/%s' % (settings.g_url, name), 31 | headers={'sessionId': sessionId} 32 | ) 33 | 34 | return r.status_code 35 | 36 | 37 | def query_project_list(sessionId): 38 | '''查询项目列表''' 39 | r = requests.get('%s/projects' % (settings.g_url), 40 | headers={'sessionId': sessionId} 41 | ) 42 | 43 | return (r.status_code, r.json()) 44 | 45 | 46 | if __name__ == '__main__': 47 | sessionId = login.get_session(settings.g_user, settings.g_password) 48 | 49 | print sessionId 50 | 51 | status = delete_project(sessionId, settings.g_project) 52 | 53 | print status 54 | 55 | (status, data) = create_project(sessionId, settings.g_project, "project dec") 56 | 57 | print status, json.dumps(data, indent=4) 58 | 59 | (status, data) = query_project_list(sessionId) 60 | 61 | print status, json.dumps(data, indent=4) 62 | -------------------------------------------------------------------------------- /python-client/project/test_project.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | 6 | 7 | class TestProject(unittest.TestCase): 8 | def setUp(self): 9 | print 'setUp...' 10 | 11 | def tearDown(self): 12 | print 'tearDown...' 13 | 14 | def test1(self): 15 | self.assertEqual(2, 2) 16 | -------------------------------------------------------------------------------- /python-client/project/test_project.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/project/test_project.pyc -------------------------------------------------------------------------------- /python-client/settings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # api url 5 | g_url = "http://%s:%s" % ('172.18.1.22', '12345') 6 | 7 | # user name & password for login 8 | g_user = 'unit_test' 9 | g_email = 'unit_test@baifendian.com' 10 | g_password = '123123' 11 | 12 | # admin user & password 13 | g_admin_user = 'admin' 14 | g_admin_password = '123456' 15 | 16 | # project for test 17 | g_project = 'project_01' -------------------------------------------------------------------------------- /python-client/settings.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/settings.pyc -------------------------------------------------------------------------------- /python-client/streaming/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'smile8' 2 | -------------------------------------------------------------------------------- /python-client/streaming/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/streaming/__init__.pyc -------------------------------------------------------------------------------- /python-client/streaming/streaming.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/streaming/streaming.pyc -------------------------------------------------------------------------------- /python-client/streaming/test_streaming.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | 6 | 7 | class TestStreaming(unittest.TestCase): 8 | def setUp(self): 9 | print 'setUp...' 10 | 11 | def tearDown(self): 12 | print 'tearDown...' 13 | 14 | def test1(self): 15 | self.assertEqual(2, 2) 16 | -------------------------------------------------------------------------------- /python-client/streaming/test_streaming.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/streaming/test_streaming.pyc -------------------------------------------------------------------------------- /python-client/streaming/test_streaming_exec.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | 6 | 7 | class TestStreamingExec(unittest.TestCase): 8 | def setUp(self): 9 | print 'setUp...' 10 | 11 | def tearDown(self): 12 | print 'tearDown...' 13 | 14 | def test1(self): 15 | self.assertEqual(2, 2) 16 | -------------------------------------------------------------------------------- /python-client/streaming/test_streaming_exec.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/streaming/test_streaming_exec.pyc -------------------------------------------------------------------------------- /python-client/test_modules.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | from adhoc.test_adhoc import TestAdhoc 6 | from login.test_login import TestLogin 7 | from project.test_project import TestProject 8 | from streaming.test_streaming import TestStreaming 9 | from streaming.test_streaming_exec import TestStreamingExec 10 | from user.test_user import TestUser 11 | 12 | suite = unittest.TestSuite() 13 | 14 | suite.addTest(unittest.makeSuite(TestLogin)) 15 | suite.addTest(unittest.makeSuite(TestUser)) 16 | suite.addTest(unittest.makeSuite(TestProject)) 17 | suite.addTest(unittest.makeSuite(TestAdhoc)) 18 | suite.addTest(unittest.makeSuite(TestStreaming)) 19 | suite.addTest(unittest.makeSuite(TestStreamingExec)) 20 | 21 | unittest.TextTestRunner(verbosity=2).run(suite) 22 | -------------------------------------------------------------------------------- /python-client/user/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'smile8' 2 | -------------------------------------------------------------------------------- /python-client/user/__init__.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/user/__init__.pyc -------------------------------------------------------------------------------- /python-client/user/test_user.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | import unittest 5 | 6 | 7 | class TestUser(unittest.TestCase): 8 | def setUp(self): 9 | print 'setUp...' 10 | 11 | def tearDown(self): 12 | print 'tearDown...' 13 | 14 | def test1(self): 15 | self.assertEqual(2, 2) 16 | -------------------------------------------------------------------------------- /python-client/user/test_user.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/baifendian/swordfish/797ec987d177dd5e6316df9f07b148715c6fbc9e/python-client/user/test_user.pyc -------------------------------------------------------------------------------- /rpc/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4 | swordfish-all 5 | com.baifendian.swordfish 6 | 1.0-SNAPSHOT 7 | 8 | 4.0.0 9 | 10 | swordfish-rpc 11 | swordfish-rpc 12 | 13 | http://maven.apache.org 14 | 15 | 16 | UTF-8 17 | 18 | 19 | 20 | 21 | org.apache.thrift 22 | libthrift 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/RestfulApiApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver; 17 | 18 | import org.springframework.boot.SpringApplication; 19 | import org.springframework.boot.autoconfigure.SpringBootApplication; 20 | import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration; 21 | import org.springframework.context.annotation.ComponentScan; 22 | 23 | @SpringBootApplication(exclude = { MongoAutoConfiguration.class }) 24 | @ComponentScan("com.baifendian.swordfish") 25 | public class RestfulApiApplication { 26 | public static void main(String[] args) { 27 | SpringApplication.run(RestfulApiApplication.class, args); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/StartupRunner.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver; 17 | 18 | import com.baifendian.swordfish.common.hadoop.ConfigurationUtil; 19 | import com.baifendian.swordfish.common.hadoop.HdfsClient; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | import org.springframework.boot.CommandLineRunner; 23 | import org.springframework.stereotype.Component; 24 | 25 | @Component 26 | public class StartupRunner implements CommandLineRunner { 27 | private static Logger logger = LoggerFactory.getLogger(StartupRunner.class.getName()); 28 | 29 | public void run(String... args) { 30 | logger.info("begin startup..."); 31 | 32 | logger.info("init hdfs instance"); 33 | HdfsClient.init(ConfigurationUtil.getConfiguration()); 34 | 35 | logger.info("end startup"); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/controller/SysInfoController.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.webserver.controller; 2 | 3 | import com.baifendian.swordfish.common.config.BaseConfig; 4 | import com.baifendian.swordfish.dao.model.User; 5 | import com.baifendian.swordfish.webserver.dto.ExecutorIdDto; 6 | import com.baifendian.swordfish.webserver.dto.IsProhibitDto; 7 | import java.util.Set; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | import org.springframework.http.HttpStatus; 11 | import org.springframework.web.bind.annotation.GetMapping; 12 | import org.springframework.web.bind.annotation.PostMapping; 13 | import org.springframework.web.bind.annotation.RequestAttribute; 14 | import org.springframework.web.bind.annotation.RequestMapping; 15 | import org.springframework.web.bind.annotation.RequestParam; 16 | import org.springframework.web.bind.annotation.RestController; 17 | 18 | @RestController 19 | @RequestMapping("") 20 | public class SysInfoController { 21 | 22 | private static Logger logger = LoggerFactory.getLogger(SysInfoController.class.getName()); 23 | 24 | @GetMapping(value = "/prohibitProxyUsers") 25 | public Set prohibitProxyUsers(@RequestAttribute(value = "session.user") User operator) { 26 | logger.info( 27 | "Operator user {}, prohibitProxyUsers", 28 | operator.getName()); 29 | 30 | return BaseConfig.getProhibitUserSet(); 31 | } 32 | 33 | @GetMapping(value = "/prohibitProxyUsers/isProhibit") 34 | public IsProhibitDto prohibitProxyUsers(@RequestAttribute(value = "session.user") User operator, 35 | @RequestParam("proxyUser") String proxyUser) { 36 | logger.info( 37 | "Operator user {}, prohibitProxyUsers", 38 | operator.getName()); 39 | 40 | boolean res = BaseConfig.isProhibitUser(proxyUser); 41 | 42 | return new IsProhibitDto(res ? 1 : 0); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/dto/BaseStatusDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.dto; 17 | 18 | /** 19 | * 基础返回结果 20 | */ 21 | public class BaseStatusDto { 22 | /** 23 | * 状态为 0 表示成功, 否则失败 24 | */ 25 | private int status; 26 | 27 | /** 28 | * 错误消息 29 | */ 30 | private String message; 31 | 32 | public BaseStatusDto() { 33 | } 34 | 35 | public BaseStatusDto(int status, String message) { 36 | this.status = status; 37 | this.message = message; 38 | } 39 | 40 | public int getStatus() { 41 | return status; 42 | } 43 | 44 | public void setStatus(int status) { 45 | this.status = status; 46 | } 47 | 48 | public String getMessage() { 49 | return message; 50 | } 51 | 52 | public void setMessage(String message) { 53 | this.message = message; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/dto/ExecWorkflowsDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.dto; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * 工作流执行结果列表response. 22 | */ 23 | public class ExecWorkflowsDto { 24 | private int total; 25 | private int from; 26 | private List executions; 27 | 28 | public ExecWorkflowsDto() { 29 | } 30 | 31 | public ExecWorkflowsDto(int total, int from, List executions) { 32 | this.total = total; 33 | this.from = from; 34 | this.executions = executions; 35 | } 36 | 37 | public int getTotal() { 38 | return total; 39 | } 40 | 41 | public void setTotal(int total) { 42 | this.total = total; 43 | } 44 | 45 | public int getFrom() { 46 | return from; 47 | } 48 | 49 | public void setFrom(int from) { 50 | this.from = from; 51 | } 52 | 53 | public List getExecutions() { 54 | return executions; 55 | } 56 | 57 | public void setExecutions(List executions) { 58 | this.executions = executions; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/dto/ExecutionFlowData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.dto; 17 | 18 | import com.baifendian.swordfish.dao.model.flow.Property; 19 | import java.util.List; 20 | 21 | /** 22 | * 执行flow node节点信息DTO 23 | */ 24 | public class ExecutionFlowData { 25 | private List nodes; 26 | 27 | private List userDefParams; 28 | 29 | public ExecutionFlowData() { 30 | } 31 | 32 | public ExecutionFlowData(List nodes, List userDefParams) { 33 | this.nodes = nodes; 34 | this.userDefParams = userDefParams; 35 | } 36 | 37 | public List getNodes() { 38 | return nodes; 39 | } 40 | 41 | public void setNodes(List nodes) { 42 | this.nodes = nodes; 43 | } 44 | 45 | public List getUserDefParams() { 46 | return userDefParams; 47 | } 48 | 49 | public void setUserDefParams(List userDefParams) { 50 | this.userDefParams = userDefParams; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/dto/ExecutorIdDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.dto; 17 | 18 | /** 19 | * 获取执行的 id 信息 20 | */ 21 | public class ExecutorIdDto { 22 | /** 23 | * 执行 id 24 | */ 25 | private int execId; 26 | 27 | public ExecutorIdDto(int execId) { 28 | this.execId = execId; 29 | } 30 | 31 | public int getExecId() { 32 | return execId; 33 | } 34 | 35 | public void setExecId(int execId) { 36 | this.execId = execId; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/dto/ExecutorIdsDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.dto; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * 执行的 id 列表 22 | */ 23 | public class ExecutorIdsDto { 24 | /** 25 | * 执行的 id 列表 26 | */ 27 | private List execIds; 28 | 29 | public ExecutorIdsDto(List execIds) { 30 | this.execIds = execIds; 31 | } 32 | 33 | public List getExecIds() { 34 | return execIds; 35 | } 36 | 37 | public void setExecIds(List execIds) { 38 | this.execIds = execIds; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/dto/IsProhibitDto.java: -------------------------------------------------------------------------------- 1 | package com.baifendian.swordfish.webserver.dto; 2 | 3 | public class IsProhibitDto { 4 | private int type; 5 | 6 | public int getType() { 7 | return type; 8 | } 9 | 10 | public void setType(int type) { 11 | this.type = type; 12 | } 13 | 14 | public IsProhibitDto(int type) { 15 | this.type = type; 16 | } 17 | 18 | public IsProhibitDto() { 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/dto/ScheduleParam.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.dto; 17 | 18 | import java.util.Date; 19 | 20 | /** 21 | * 调度参数. 22 | */ 23 | public class ScheduleParam { 24 | private Date startDate; 25 | private Date endDate; 26 | private String crontab; 27 | 28 | public ScheduleParam() { 29 | } 30 | 31 | public ScheduleParam(Date startDate, Date endDate, String crontab) { 32 | this.startDate = startDate; 33 | this.endDate = endDate; 34 | this.crontab = crontab; 35 | } 36 | 37 | public Date getStartDate() { 38 | return startDate; 39 | } 40 | 41 | public void setStartDate(Date startDate) { 42 | this.startDate = startDate; 43 | } 44 | 45 | public Date getEndDate() { 46 | return endDate; 47 | } 48 | 49 | public void setEndDate(Date endDate) { 50 | this.endDate = endDate; 51 | } 52 | 53 | public String getCrontab() { 54 | return crontab; 55 | } 56 | 57 | public void setCrontab(String crontab) { 58 | this.crontab = crontab; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/dto/StreamingResultsDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.dto; 17 | 18 | import java.util.List; 19 | 20 | public class StreamingResultsDto { 21 | private int total; 22 | private int from; 23 | private List executions; 24 | 25 | public StreamingResultsDto() { 26 | } 27 | 28 | public StreamingResultsDto(int total, int from, List executions) { 29 | this.total = total; 30 | this.from = from; 31 | this.executions = executions; 32 | } 33 | 34 | public int getTotal() { 35 | return total; 36 | } 37 | 38 | public void setTotal(int total) { 39 | this.total = total; 40 | } 41 | 42 | public int getFrom() { 43 | return from; 44 | } 45 | 46 | public void setFrom(int from) { 47 | this.from = from; 48 | } 49 | 50 | public List getExecutions() { 51 | return executions; 52 | } 53 | 54 | public void setExecutions(List executions) { 55 | this.executions = executions; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/dto/UserSessionDto.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.dto; 17 | 18 | import com.baifendian.swordfish.dao.model.User; 19 | 20 | public class UserSessionDto { 21 | /** 22 | * 用户 session id 23 | */ 24 | private String sessionId; 25 | 26 | /** 27 | * 用户信息 28 | */ 29 | private UserDto user; 30 | 31 | public UserSessionDto() { 32 | } 33 | 34 | public UserSessionDto(String sessionId, User user) { 35 | this.sessionId = sessionId; 36 | this.user = new UserDto(user); 37 | } 38 | 39 | public String getSessionId() { 40 | return sessionId; 41 | } 42 | 43 | public void setSessionId(String sessionId) { 44 | this.sessionId = sessionId; 45 | } 46 | 47 | public UserDto getUser() { 48 | return user; 49 | } 50 | 51 | public void setUser(UserDto user) { 52 | this.user = user; 53 | } 54 | 55 | @Override 56 | public String toString() { 57 | return "UserSessionDto{" + 58 | "sessionId='" + sessionId + '\'' + 59 | ", user=" + user + 60 | '}'; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/exception/BadRequestException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.exception; 17 | 18 | import java.text.MessageFormat; 19 | 20 | /** 21 | * 错误的请求异常 22 | */ 23 | public class BadRequestException extends RuntimeException { 24 | 25 | public BadRequestException(String msg) { 26 | super(msg); 27 | } 28 | 29 | public BadRequestException(String msg, Throwable cause) { 30 | super(msg, cause); 31 | } 32 | 33 | public BadRequestException(String msg, Object... arguments) { 34 | super(MessageFormat.format(msg, arguments)); 35 | } 36 | 37 | public BadRequestException(Throwable cause, String msg, Object... arguments) { 38 | super(MessageFormat.format(msg, arguments), cause); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/exception/CustomErrorType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.exception; 17 | 18 | import org.springframework.http.HttpStatus; 19 | 20 | public class CustomErrorType { 21 | private int status; 22 | private String error; 23 | private String message; 24 | 25 | public CustomErrorType() { 26 | } 27 | 28 | public CustomErrorType(HttpStatus status, String message) { 29 | this.status = status.value(); 30 | this.error = status.getReasonPhrase(); 31 | this.message = message; 32 | } 33 | 34 | public int getStatus() { 35 | return status; 36 | } 37 | 38 | public void setStatus(int status) { 39 | this.status = status; 40 | } 41 | 42 | public String getError() { 43 | return error; 44 | } 45 | 46 | public void setError(String error) { 47 | this.error = error; 48 | } 49 | 50 | public String getMessage() { 51 | return message; 52 | } 53 | 54 | public void setMessage(String message) { 55 | this.message = message; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/exception/NotFoundException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.exception; 17 | 18 | import java.text.MessageFormat; 19 | 20 | /** 21 | * 404 错误码异常 22 | */ 23 | public class NotFoundException extends RuntimeException { 24 | 25 | public NotFoundException() { 26 | } 27 | 28 | public NotFoundException(String msg) { 29 | super(msg); 30 | } 31 | 32 | public NotFoundException(String msg, Object... arguments) { 33 | super(MessageFormat.format(msg, arguments)); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/exception/ParameterException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.exception; 17 | 18 | import java.text.MessageFormat; 19 | 20 | /** 21 | * 请求参数异常 22 | */ 23 | public class ParameterException extends BadRequestException { 24 | 25 | public ParameterException(String msg) { 26 | super(msg); 27 | } 28 | 29 | public ParameterException(String msg, Object... arguments) { 30 | super(MessageFormat.format(msg, arguments)); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/exception/PermissionException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.exception; 17 | 18 | import java.text.MessageFormat; 19 | 20 | 21 | /** 22 | * 权限异常 23 | */ 24 | public class PermissionException extends UnAuthorizedException { 25 | 26 | public PermissionException(String msg) { 27 | super(msg); 28 | } 29 | 30 | public PermissionException(String msg, Object... arguments) { 31 | super(MessageFormat.format(msg, arguments)); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/exception/PreFailedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.exception; 17 | 18 | import java.text.MessageFormat; 19 | 20 | /** 21 | * 未能满足前置条件异常 22 | */ 23 | public class PreFailedException extends RuntimeException { 24 | public PreFailedException(String msg) { 25 | super(msg); 26 | } 27 | 28 | public PreFailedException(String msg, Throwable cause) { 29 | super(msg, cause); 30 | } 31 | 32 | public PreFailedException(String msg, Object... arguments) { 33 | super(MessageFormat.format(msg, arguments)); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/exception/ServerErrorException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.exception; 17 | 18 | import java.text.MessageFormat; 19 | 20 | /** 21 | * 内部服务异常 22 | */ 23 | public class ServerErrorException extends RuntimeException { 24 | public ServerErrorException(String msg) { 25 | super(msg); 26 | } 27 | 28 | public ServerErrorException(String msg, Throwable cause) { 29 | super(msg, cause); 30 | } 31 | 32 | public ServerErrorException(String msg, Object... arguments) { 33 | super(MessageFormat.format(msg, arguments)); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/exception/UnAuthorizedException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.exception; 17 | 18 | import org.springframework.http.HttpStatus; 19 | import org.springframework.web.bind.annotation.ResponseStatus; 20 | 21 | /** 22 | * 无授权异常 23 | */ 24 | @ResponseStatus(code = HttpStatus.UNAUTHORIZED) 25 | public class UnAuthorizedException extends RuntimeException { 26 | public UnAuthorizedException(String msg) { 27 | super(msg); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/service/storage/StorageException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.service.storage; 17 | 18 | public class StorageException extends RuntimeException { 19 | 20 | public StorageException(String message) { 21 | super(message); 22 | } 23 | 24 | public StorageException(String message, Throwable cause) { 25 | super(message, cause); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/service/storage/StorageFileNotFoundException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.service.storage; 17 | 18 | public class StorageFileNotFoundException extends StorageException { 19 | 20 | public StorageFileNotFoundException(String message) { 21 | super(message); 22 | } 23 | 24 | public StorageFileNotFoundException(String message, Throwable cause) { 25 | super(message, cause); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /web-server/src/main/java/com/baifendian/swordfish/webserver/service/storage/StorageService.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017 Baifendian Corporation 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.baifendian.swordfish.webserver.service.storage; 17 | 18 | import java.io.IOException; 19 | import org.springframework.core.io.Resource; 20 | import org.springframework.web.multipart.MultipartFile; 21 | 22 | public interface StorageService { 23 | 24 | /** 25 | * 存储到目的文件 26 | * 27 | * @param file 28 | * @param destFilename 29 | */ 30 | void store(MultipartFile file, String destFilename); 31 | 32 | /** 33 | * 加载文件 34 | * 35 | * @param filename 36 | * @return 37 | */ 38 | Resource loadAsResource(String filename); 39 | 40 | /** 41 | * 删除目录 42 | * 43 | * @param dir 44 | */ 45 | void deleteDir(String dir) throws IOException; 46 | 47 | /** 48 | * 删除文件 49 | * 50 | * @param filename 51 | */ 52 | void deleteFile(String filename) throws IOException; 53 | } 54 | -------------------------------------------------------------------------------- /web-server/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # server port 2 | server.port=12345 3 | 4 | # session config 5 | server.session.timeout=7200 6 | 7 | # file size limit for upload 8 | spring.http.multipart.max-file-size=256MB 9 | spring.http.multipart.max-request-size=256MB 10 | --------------------------------------------------------------------------------