├── .gitignore ├── .gitlab-ci.yml ├── LICENSE ├── README.md ├── aws_s3 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── aws_s3 │ │ ├── AwsS3Client.java │ │ └── AwsS3Util.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── ci └── sonar_notify.sh ├── clickhouse ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── clickhouse │ │ ├── ClickHouseDownloader.java │ │ ├── ClickhouseAdapter.java │ │ ├── ClickhouseClient.java │ │ ├── ClickhouseConnFactory.java │ │ ├── ClickhouseErrorPattern.java │ │ └── ClickhouseTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── common ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── dtstack │ └── dtcenter │ ├── common │ └── loader │ │ └── common │ │ ├── DtClassConsistent.java │ │ ├── DtClassThreadFactory.java │ │ ├── Pool.java │ │ ├── base │ │ └── CallBack.java │ │ ├── convert │ │ └── ColumnTypeConverter.java │ │ ├── enums │ │ ├── ColumnType.java │ │ ├── SftpAuthType.java │ │ └── StoredType.java │ │ ├── exception │ │ ├── AbsErrorPattern.java │ │ ├── ConnErrorCode.java │ │ ├── ErrorCode.java │ │ ├── IErrorCode.java │ │ ├── IErrorPattern.java │ │ └── SqlErrorCode.java │ │ ├── nosql │ │ └── AbsNoSqlClient.java │ │ ├── service │ │ ├── ErrorAdapterImpl.java │ │ └── IErrorAdapter.java │ │ └── utils │ │ ├── AddressUtil.java │ │ ├── CollectionUtil.java │ │ ├── DBUtil.java │ │ ├── DateUtil.java │ │ ├── DelimiterUtil.java │ │ ├── EnvUtil.java │ │ ├── IOUtils.java │ │ ├── JSONUtil.java │ │ ├── MathUtil.java │ │ ├── PathUtils.java │ │ ├── PropertiesUtil.java │ │ ├── PropertyUtil.java │ │ ├── ReflectUtil.java │ │ ├── SchemaUtil.java │ │ ├── SearchUtil.java │ │ ├── SerializeUtil.java │ │ ├── SqlFormatUtil.java │ │ ├── StringUtil.java │ │ ├── TableUtil.java │ │ ├── TelUtil.java │ │ ├── Xml2JsonUtil.java │ │ └── ZipUtil.java │ └── loader │ ├── kerberos │ └── HadoopConfTool.java │ └── source │ └── DataBaseType.java ├── core ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── dtstack │ └── dtcenter │ └── loader │ ├── ClassLoaderCallBack.java │ ├── ClassLoaderCallBackMethod.java │ ├── DtClassLoader.java │ ├── IDownloader.java │ ├── cache │ ├── connection │ │ ├── CacheConnectionHelper.java │ │ ├── DataSourceCacheNode.java │ │ ├── DataSourceConnection.java │ │ └── HashCacheConnectionKey.java │ └── pool │ │ ├── config │ │ └── PoolConfig.java │ │ └── manager │ │ └── Manager.java │ ├── client │ ├── ClientCache.java │ ├── ClientFactory.java │ ├── IClient.java │ ├── IHbase.java │ ├── IHdfsFile.java │ ├── IKafka.java │ ├── IKerberos.java │ ├── IRedis.java │ ├── IRestful.java │ ├── ITable.java │ ├── ITsdb.java │ ├── hbase │ │ ├── HbaseClientFactory.java │ │ └── HbaseProxy.java │ ├── hdfs │ │ ├── HdfsFileClientCache.java │ │ ├── HdfsFileClientFactory.java │ │ └── HdfsFileProxy.java │ ├── kerberos │ │ ├── KerberosClientFactory.java │ │ └── KerberosProxy.java │ ├── mq │ │ ├── KafkaClientCache.java │ │ ├── KafkaClientFactory.java │ │ └── KafkaClientProxy.java │ ├── redis │ │ ├── RedisClientFactory.java │ │ └── RedisProxy.java │ ├── restful │ │ ├── RestfulClientFactory.java │ │ └── RestfulClientProxy.java │ ├── sql │ │ ├── DataSourceClientCache.java │ │ ├── DataSourceClientFactory.java │ │ └── DataSourceClientProxy.java │ ├── table │ │ ├── TableClientFactory.java │ │ └── TableClientProxy.java │ └── tsdb │ │ ├── TsdbClientFactory.java │ │ └── TsdbClientProxy.java │ ├── downloader │ └── DownloaderProxy.java │ ├── dto │ ├── Column.java │ ├── ColumnMetaDTO.java │ ├── Database.java │ ├── FileStatus.java │ ├── HDFSContentSummary.java │ ├── HDFSImportColumn.java │ ├── HbaseQueryDTO.java │ ├── HdfsWriterDTO.java │ ├── KafkaConsumerDTO.java │ ├── KafkaOffsetDTO.java │ ├── KafkaPartitionDTO.java │ ├── KafkaTopicDTO.java │ ├── RedisQueryDTO.java │ ├── SSLConfigDTO.java │ ├── SolrQueryDTO.java │ ├── SqlQueryDTO.java │ ├── Table.java │ ├── TableInfo.java │ ├── UpsertColumnMetaDTO.java │ ├── comparator │ │ ├── BigDecimalComparator.java │ │ ├── BinaryComparator.java │ │ ├── BinaryPrefixComparator.java │ │ ├── BitComparator.java │ │ ├── ByteArrayComparable.java │ │ ├── LongComparator.java │ │ ├── NullComparator.java │ │ ├── RegexStringComparator.java │ │ └── SubstringComparator.java │ ├── contant │ │ └── SolrCommonParams.java │ ├── filter │ │ ├── Filter.java │ │ ├── FilterList.java │ │ ├── PageFilter.java │ │ ├── RowFilter.java │ │ ├── SingleColumnValueFilter.java │ │ └── TimestampFilter.java │ ├── restful │ │ └── Response.java │ ├── source │ │ ├── AdbForPgSourceDTO.java │ │ ├── AwsS3SourceDTO.java │ │ ├── ClickHouseSourceDTO.java │ │ ├── Db2SourceDTO.java │ │ ├── DmForOracleSourceDTO.java │ │ ├── DmSourceDTO.java │ │ ├── DorisRestfulSourceDTO.java │ │ ├── DorisSourceDTO.java │ │ ├── EMQSourceDTO.java │ │ ├── ES7SourceDTO.java │ │ ├── ESSourceDTO.java │ │ ├── FtpSourceDTO.java │ │ ├── GBaseSourceDTO.java │ │ ├── Greenplum6SourceDTO.java │ │ ├── HbaseSourceDTO.java │ │ ├── HdfsSourceDTO.java │ │ ├── Hive1SourceDTO.java │ │ ├── Hive3CDPSourceDTO.java │ │ ├── Hive3SourceDTO.java │ │ ├── HiveSourceDTO.java │ │ ├── HiveSslConfig.java │ │ ├── HuaweiHbaseSourceDTO.java │ │ ├── HuaweiKafkaSourceDTO.java │ │ ├── ISourceDTO.java │ │ ├── ImpalaSourceDTO.java │ │ ├── InceptorSourceDTO.java │ │ ├── InfluxDBSourceDTO.java │ │ ├── KafkaSourceDTO.java │ │ ├── KingbaseSourceDTO.java │ │ ├── KuduSourceDTO.java │ │ ├── KylinRestfulSourceDTO.java │ │ ├── KylinSourceDTO.java │ │ ├── LibraSourceDTO.java │ │ ├── MongoSourceDTO.java │ │ ├── Mysql5SourceDTO.java │ │ ├── Mysql8SourceDTO.java │ │ ├── OceanBaseSourceDTO.java │ │ ├── OdpsSourceDTO.java │ │ ├── OpenTSDBSourceDTO.java │ │ ├── OracleSourceDTO.java │ │ ├── Phoenix5SourceDTO.java │ │ ├── PhoenixSourceDTO.java │ │ ├── PostgresqlSourceDTO.java │ │ ├── PrestoSourceDTO.java │ │ ├── RdbmsSourceDTO.java │ │ ├── RedisSourceDTO.java │ │ ├── RestfulSourceDTO.java │ │ ├── S3SourceDTO.java │ │ ├── SapHana1SourceDTO.java │ │ ├── SapHana2SourceDTO.java │ │ ├── SapHanaSourceDTO.java │ │ ├── SocketSourceDTO.java │ │ ├── SolrSourceDTO.java │ │ ├── SparkSourceDTO.java │ │ ├── Sqlserver2017SourceDTO.java │ │ ├── SqlserverSourceDTO.java │ │ ├── TbdsHbaseSourceDTO.java │ │ ├── TbdsHdfsSourceDTO.java │ │ ├── TbdsKafkaSourceDTO.java │ │ ├── TrinoSourceDTO.java │ │ ├── VerticaSourceDTO.java │ │ └── WebSocketSourceDTO.java │ └── tsdb │ │ ├── Aggregator.java │ │ ├── DeltaOptions.java │ │ ├── Filter.java │ │ ├── FilterType.java │ │ ├── KeyValue.java │ │ ├── QueryResult.java │ │ ├── RateOptions.java │ │ ├── SubQuery.java │ │ ├── Suggest.java │ │ ├── TsdbPoint.java │ │ └── TsdbQuery.java │ ├── enums │ ├── CommandType.java │ ├── Comparator.java │ ├── CompareOp.java │ ├── ConnectionClearStatus.java │ ├── EsCommandType.java │ ├── FileFormat.java │ ├── Granularity.java │ ├── HbaseFilterType.java │ ├── MatchType.java │ ├── RedisCompareOp.java │ ├── RedisDataType.java │ └── RedisMode.java │ ├── exception │ ├── ClientAccessException.java │ └── DtLoaderException.java │ ├── factory │ └── DtThreadFactory.java │ ├── functions │ └── Callback.java │ ├── kerberos │ └── HadoopConfTool.java │ ├── source │ ├── DataBaseType.java │ └── DataSourceType.java │ └── utils │ ├── AssertUtils.java │ └── DBUtil.java ├── csp_s3 └── pom.xml ├── db2 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── db2 │ │ ├── Db2Client.java │ │ ├── Db2ConnFactory.java │ │ ├── Db2Downloader.java │ │ ├── Db2ErrorPattern.java │ │ └── Db2TableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── dm ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── dm │ │ ├── DmClient.java │ │ ├── DmConnFactory.java │ │ └── DmDownloader.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── docs ├── connectionPool.md ├── fs │ ├── aws_s3.md │ ├── ftp.md │ ├── hdfs.md │ └── s3.md ├── kerberos.md ├── mq │ ├── emq.md │ └── kafka.md ├── nosql │ ├── es.md │ ├── hbase.md │ ├── mongodb.md │ ├── redis.md │ └── solr.md ├── nowstart.md ├── other │ ├── influxdb.md │ ├── socket.md │ └── websocket.md └── rdbms │ ├── clickhouse.md │ ├── greenplum6.md │ ├── hive.md │ ├── kudu.md │ ├── libra.md │ ├── mysql5.md │ ├── mysql8.md │ ├── oceanbase.md │ ├── oracle.md │ ├── postgresql.md │ ├── presto.md │ ├── rdbm.md │ └── sqlserver.md ├── doris ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── doris │ │ ├── DorisClient.java │ │ ├── DorisConnFactory.java │ │ ├── DorisErrorPattern.java │ │ └── DorisTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── dorisrestful ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── dorisrestful │ │ ├── DorisClient.java │ │ └── request │ │ ├── DorisRestfulClient.java │ │ ├── DorisRestfulClientFactory.java │ │ └── HttpAPI.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── emq ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── emq │ │ ├── EMQUtils.java │ │ └── EmqClient.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── es ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── es │ │ ├── EsClient.java │ │ ├── EsErrorPattern.java │ │ └── pool │ │ ├── ElasticSearchManager.java │ │ ├── ElasticSearchPool.java │ │ ├── ElasticSearchPoolConfig.java │ │ └── ElasticSearchPoolFactory.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── es5 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── es5 │ │ ├── EsClient.java │ │ ├── consistent │ │ ├── ESEndPoint.java │ │ └── RequestType.java │ │ └── pool │ │ ├── ElasticSearchManager.java │ │ ├── ElasticSearchPool.java │ │ ├── ElasticSearchPoolConfig.java │ │ └── ElasticSearchPoolFactory.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── es7 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── es7 │ │ ├── EsClient.java │ │ ├── EsErrorPattern.java │ │ └── pool │ │ ├── ElasticSearchManager.java │ │ ├── ElasticSearchPool.java │ │ ├── ElasticSearchPoolConfig.java │ │ └── ElasticSearchPoolFactory.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── ftp ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── ftp │ │ ├── FTPConnectMode.java │ │ ├── FtpClient.java │ │ ├── FtpClientFactory.java │ │ ├── FtpUtil.java │ │ ├── ProtocolEnum.java │ │ └── SFTPHandler.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── gbase ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── gbase │ │ ├── GbaseClient.java │ │ ├── GbaseConnFactory.java │ │ ├── GbaseTableClient.java │ │ └── GbaselErrorPattern.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── greenplum6 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── greenplum │ │ ├── GreenplumClient.java │ │ ├── GreenplumDownloader.java │ │ ├── GreenplumErrorPattern.java │ │ ├── GreenplumFactory.java │ │ └── GreenplumTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── hbase ├── pom.xml └── src │ └── main │ ├── java │ ├── com │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── hbase │ │ │ ├── ComparatorType.java │ │ │ ├── FilterType.java │ │ │ ├── HbaseClient.java │ │ │ ├── HbaseClientSpecial.java │ │ │ ├── HbaseConnFactory.java │ │ │ ├── HbaseErrorPattern.java │ │ │ ├── KerberosClient.java │ │ │ └── pool │ │ │ └── HbasePoolManager.java │ └── org │ │ └── apache │ │ └── zookeeper │ │ └── client │ │ └── ZooKeeperSaslClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IHbase │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── hbase2 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── hbase │ │ ├── ComparatorType.java │ │ ├── FilterType.java │ │ ├── HbaseClient.java │ │ ├── HbaseConnFactory.java │ │ ├── KerberosClient.java │ │ └── pool │ │ └── HbasePoolManager.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── hbase_gateway ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── hbase │ │ │ ├── ComparatorType.java │ │ │ ├── FilterType.java │ │ │ ├── HbaseClient.java │ │ │ ├── HbaseConnFactory.java │ │ │ ├── KerberosClient.java │ │ │ └── pool │ │ │ └── HbasePoolManager.java │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.dtstack.dtcenter.loader.client.IClient │ │ └── com.dtstack.dtcenter.loader.client.IKerberos │ └── test │ └── java │ └── HbaseGatewayTest.java ├── hdfs ├── pom.xml └── src │ └── main │ ├── java │ ├── com │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── hdfs │ │ │ ├── GroupTypeIgnoreCase.java │ │ │ ├── HdfsConnFactory.java │ │ │ ├── OrcColumnTypeConverter.java │ │ │ ├── TimeoutExecutor.java │ │ │ ├── YarnConfUtil.java │ │ │ ├── client │ │ │ ├── HdfsClient.java │ │ │ ├── HdfsFileClient.java │ │ │ └── KerberosClient.java │ │ │ ├── downloader │ │ │ ├── HdfsFileDownload.java │ │ │ ├── HdfsORCDownload.java │ │ │ ├── HdfsParquetDownload.java │ │ │ ├── HdfsTextDownload.java │ │ │ ├── YarnLogDownload │ │ │ │ └── YarnTFileDownload.java │ │ │ └── tableDownload │ │ │ │ ├── GroupTypeIgnoreCase.java │ │ │ │ ├── HiveORCDownload.java │ │ │ │ ├── HiveParquetDownload.java │ │ │ │ └── HiveTextDownload.java │ │ │ ├── fileMerge │ │ │ ├── ECompressType.java │ │ │ ├── core │ │ │ │ ├── CombineMergeBuilder.java │ │ │ │ ├── CombineServer.java │ │ │ │ ├── FileCombineFactory.java │ │ │ │ ├── OrcCombineServer.java │ │ │ │ ├── ParquetCombineServer.java │ │ │ │ └── TextCombineServer.java │ │ │ └── meta │ │ │ │ ├── FileMetaData.java │ │ │ │ ├── OrcMetaData.java │ │ │ │ └── ParquetMetaData.java │ │ │ ├── hdfswriter │ │ │ ├── HdfsOrcWriter.java │ │ │ ├── HdfsParquetWriter.java │ │ │ ├── HdfsTextWriter.java │ │ │ └── HdfsWriter.java │ │ │ └── util │ │ │ ├── FileSystemUtils.java │ │ │ └── StringUtil.java │ └── org │ │ └── apache │ │ └── hadoop │ │ └── yarn │ │ └── logaggregation │ │ └── filecontroller │ │ └── ifile │ │ ├── BoundedRangeFileInputStream.java │ │ ├── Compression.java │ │ └── LogAggregationIndexedFileController.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IHdfsFile │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── hive ├── pom.xml └── src │ └── main │ ├── java │ ├── com │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── hive2 │ │ │ ├── GroupTypeIgnoreCase.java │ │ │ ├── HiveConnFactory.java │ │ │ ├── HiveDriverUtil.java │ │ │ ├── HiveErrorPattern.java │ │ │ ├── TimeoutExecutor.java │ │ │ ├── client │ │ │ ├── HiveClient.java │ │ │ └── HiveTableClient.java │ │ │ └── downloader │ │ │ ├── HiveORCDownload.java │ │ │ ├── HiveParquetDownload.java │ │ │ └── HiveTextDownload.java │ └── org │ │ └── apache │ │ ├── hive │ │ └── jdbc │ │ │ └── HiveConnection.java │ │ ├── orc │ │ └── impl │ │ │ └── TreeReaderFactory.java │ │ └── parquet │ │ └── hadoop │ │ └── ParquetFileReader.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKerberos │ └── com.dtstack.dtcenter.loader.client.ITable ├── hive1 ├── pom.xml └── src │ └── main │ ├── java │ ├── com │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── hive1 │ │ │ ├── GroupTypeIgnoreCase.java │ │ │ ├── Hive1ErrorPattern.java │ │ │ ├── HiveConnFactory.java │ │ │ ├── HiveDriverUtil.java │ │ │ ├── TimeoutExecutor.java │ │ │ ├── client │ │ │ ├── Hive1TableClient.java │ │ │ └── HiveClient.java │ │ │ └── downloader │ │ │ ├── HiveORCDownload.java │ │ │ ├── HiveParquetDownload.java │ │ │ └── HiveTextDownload.java │ └── org │ │ └── apache │ │ └── hive │ │ └── jdbc │ │ └── HiveConnection.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKerberos │ └── com.dtstack.dtcenter.loader.client.ITable ├── hive3 ├── pom.xml └── src │ └── main │ ├── java │ ├── com │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── hive3 │ │ │ ├── GroupTypeIgnoreCase.java │ │ │ ├── HiveConnFactory.java │ │ │ ├── HiveDriverUtil.java │ │ │ ├── HiveErrorPattern.java │ │ │ ├── TimeoutExecutor.java │ │ │ ├── client │ │ │ ├── Hive3Client.java │ │ │ └── Hive3TableClient.java │ │ │ └── downloader │ │ │ ├── HiveORCDownload.java │ │ │ ├── HiveParquetDownload.java │ │ │ └── HiveTextDownload.java │ └── org │ │ └── apache │ │ └── hive │ │ └── jdbc │ │ └── HiveConnection.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKerberos │ └── com.dtstack.dtcenter.loader.client.ITable ├── hive3_cdp ├── pom.xml └── src │ └── main │ ├── java │ ├── com │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── hive3 │ │ │ ├── GroupTypeIgnoreCase.java │ │ │ ├── HiveConnFactory.java │ │ │ ├── HiveDriverUtil.java │ │ │ ├── HiveErrorPattern.java │ │ │ ├── TimeoutExecutor.java │ │ │ ├── client │ │ │ ├── Hive3Client.java │ │ │ └── Hive3TableClient.java │ │ │ └── downloader │ │ │ ├── HiveORCDownload.java │ │ │ ├── HiveParquetDownload.java │ │ │ └── HiveTextDownload.java │ └── org │ │ └── apache │ │ ├── hadoop │ │ └── security │ │ │ └── ssl │ │ │ └── SSLFactory.java │ │ ├── hive │ │ └── jdbc │ │ │ └── HiveConnection.java │ │ └── parquet │ │ └── hadoop │ │ └── ParquetFileReader.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKerberos │ └── com.dtstack.dtcenter.loader.client.ITable ├── huawei_hbase ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── hbase │ │ ├── ComparatorType.java │ │ ├── FilterType.java │ │ ├── HbaseClient.java │ │ ├── HbaseClientSpecial.java │ │ ├── HbaseConnFactory.java │ │ ├── HbaseErrorPattern.java │ │ ├── KerberosClient.java │ │ └── pool │ │ └── HbasePoolManager.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IHbase │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── huawei_kafka ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── kafka │ │ ├── Kafka.java │ │ ├── KafkaClient.java │ │ ├── KafkaConsistent.java │ │ ├── KafkaErrorPattern.java │ │ ├── KafkaKerberos.java │ │ ├── enums │ │ └── EConsumeType.java │ │ └── util │ │ ├── FileUtils.java │ │ └── KafkaUtil.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKafka │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── impala ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── impala │ │ │ ├── ImpalaClient.java │ │ │ ├── ImpalaConnFactory.java │ │ │ ├── ImpalaDownload.java │ │ │ ├── ImpalaDriverUtil.java │ │ │ ├── ImpalaErrorPattern.java │ │ │ └── ImpalaTableClient.java │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.dtstack.dtcenter.loader.client.IClient │ │ ├── com.dtstack.dtcenter.loader.client.IKerberos │ │ └── com.dtstack.dtcenter.loader.client.ITable │ └── test │ └── java │ └── ImpalaTest.java ├── inceptor ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── inceptor │ │ ├── GroupTypeIgnoreCase.java │ │ ├── InceptorConnFactory.java │ │ ├── InceptorDriverUtil.java │ │ ├── InceptorErrorPattern.java │ │ ├── client │ │ ├── InceptorClient.java │ │ └── InceptorTableClient.java │ │ └── downloader │ │ ├── InceptorDownload.java │ │ ├── InceptorORCDownload.java │ │ ├── InceptorParquetDownload.java │ │ └── InceptorTextDownload.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKerberos │ └── com.dtstack.dtcenter.loader.client.ITable ├── influxdb ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── influxdb │ │ ├── InfluxDBClient.java │ │ ├── InfluxDBConnFactory.java │ │ ├── InfluxDBErrorPattern.java │ │ └── InfluxDBUtil.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── kafka ├── pom.xml └── src │ └── main │ ├── java │ ├── com │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── kafka │ │ │ ├── Kafka.java │ │ │ ├── KafkaClient.java │ │ │ ├── KafkaConsistent.java │ │ │ ├── KafkaErrorPattern.java │ │ │ ├── KafkaKerberos.java │ │ │ ├── enums │ │ │ └── EConsumeType.java │ │ │ └── util │ │ │ ├── FileUtils.java │ │ │ └── KafkaUtil.java │ └── org │ │ └── apache │ │ └── zookeeper │ │ └── client │ │ └── ZooKeeperSaslClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKafka │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── kerberos ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── dtstack │ └── dtcenter │ └── common │ └── loader │ └── hadoop │ ├── AbsKerberosClient.java │ ├── hdfs │ ├── HDFSErrorPattern.java │ ├── HadoopConfUtil.java │ └── HdfsOperator.java │ └── util │ ├── JaasUtil.java │ ├── KerberosConfigUtil.java │ ├── KerberosLoginUtil.java │ └── UGICacheData.java ├── kingbase8 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── kingbase │ │ ├── KingbaseAdapter.java │ │ ├── KingbaseClient.java │ │ ├── KingbaseConnFactory.java │ │ ├── KingbaseErrorPattern.java │ │ └── KingbaseTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── kudu ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── kudu │ │ ├── DtKuduClient.java │ │ ├── KuduColumnTypeConverter.java │ │ └── KuduErrorPattern.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── kylin ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── kylin │ │ ├── KylinClient.java │ │ ├── KylinConnFactory.java │ │ ├── KylinDownloader.java │ │ ├── KylinErrorPattern.java │ │ └── KylinTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── kylinrestful ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── kylinRestful │ │ ├── KylinRestfulClient.java │ │ ├── http │ │ ├── HttpAPI.java │ │ ├── HttpAddressManager.java │ │ ├── HttpClient.java │ │ ├── HttpClientFactory.java │ │ └── response │ │ │ ├── HttpStatus.java │ │ │ └── ResultResponse.java │ │ └── request │ │ ├── RestfulClient.java │ │ └── RestfulClientFactory.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── libra ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── libra │ │ ├── LibraClient.java │ │ ├── LibraConnFactory.java │ │ ├── LibraDownloader.java │ │ ├── LibraErrorPattern.java │ │ └── LibraTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── license.txt ├── maxcompute ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── odps │ │ ├── OdpsClient.java │ │ ├── OdpsErrorPattern.java │ │ ├── common │ │ └── OdpsFields.java │ │ └── pool │ │ ├── OdpsManager.java │ │ ├── OdpsPool.java │ │ ├── OdpsPoolConfig.java │ │ └── OdpsPoolFactory.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── mongo ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── mongo │ │ ├── MongoDBClient.java │ │ ├── MongoDBUtils.java │ │ ├── MongoErrorPattern.java │ │ ├── MongoExecutor.java │ │ ├── RegExpUtil.java │ │ └── pool │ │ └── MongoManager.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── mysql5 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── mysql5 │ │ ├── MysqlClient.java │ │ ├── MysqlConnFactory.java │ │ ├── MysqlDownloader.java │ │ ├── MysqlErrorPattern.java │ │ └── MysqlTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── mysql8 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── mysql8 │ │ ├── MysqlClient.java │ │ ├── MysqlConnFactory.java │ │ └── MysqlDownloader.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── oceanbase ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── oceanbase │ │ ├── OceanBaseClient.java │ │ ├── OceanBaseConnFactory.java │ │ ├── OceanBaseDownloader.java │ │ ├── OceanBaseErrorPattern.java │ │ └── OceanBaseTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── opentsdb ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── opentsdb │ │ ├── OpenTSDBClient.java │ │ ├── OpenTSDBClientSpecial.java │ │ ├── http │ │ ├── HttpAPI.java │ │ ├── HttpAddressManager.java │ │ ├── HttpClient.java │ │ ├── HttpClientFactory.java │ │ └── response │ │ │ ├── HttpStatus.java │ │ │ └── ResultResponse.java │ │ ├── query │ │ ├── DeleteMetaRequest.java │ │ ├── MetricTimeRange.java │ │ ├── SuggestValue.java │ │ └── Timeline.java │ │ ├── tsdb │ │ ├── OpenTSDBConnFactory.java │ │ ├── TSDB.java │ │ └── TSDBClient.java │ │ └── type │ │ ├── JSONValue.java │ │ └── Result.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITsdb ├── oracle ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── oracle │ │ │ ├── OracleClient.java │ │ │ ├── OracleConnFactory.java │ │ │ ├── OracleDbAdapter.java │ │ │ ├── OracleDownloader.java │ │ │ ├── OracleErrorPattern.java │ │ │ └── OracleTableClient.java │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.dtstack.dtcenter.loader.client.IClient │ │ └── com.dtstack.dtcenter.loader.client.ITable │ └── test │ └── java │ └── com │ └── dtstack │ └── dtcenter │ └── common │ └── loader │ └── oracle │ └── OracleDaAdapterTest.java ├── phoenix ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── phoenix4 │ │ ├── PhoenixClient.java │ │ ├── PhoenixConnFactory.java │ │ ├── PhoenixErrorPattern.java │ │ └── PhoenixTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── phoenix4_8 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── phoenix4 │ │ ├── PhoenixClient.java │ │ ├── PhoenixConnFactory.java │ │ ├── PhoenixErrorPattern.java │ │ └── PhoenixTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── phoenix5 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── phoenix5 │ │ ├── KerberosClient.java │ │ ├── PhoenixClient.java │ │ ├── PhoenixConnFactory.java │ │ ├── PhoenixErrorPattern.java │ │ └── PhoenixTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKerberos │ └── com.dtstack.dtcenter.loader.client.ITable ├── pom.xml ├── postgresql ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── postgresql │ │ ├── PostgreSqlAdapter.java │ │ ├── PostgreSqlErrorPattern.java │ │ ├── PostgresqlClient.java │ │ ├── PostgresqlConnFactory.java │ │ ├── PostgresqlDownloader.java │ │ └── PostgresqlTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── presto ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── presto │ │ ├── PrestoClient.java │ │ ├── PrestoConnFactory.java │ │ ├── PrestoErrorPattern.java │ │ └── PrestoTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── rdbms ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── dtstack │ └── dtcenter │ └── common │ └── loader │ └── rdbms │ ├── AbsRdbmsClient.java │ ├── AbsTableClient.java │ └── ConnFactory.java ├── redis ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── redis │ │ ├── RedisClient.java │ │ ├── RedisClientSpecial.java │ │ ├── RedisErrorPattern.java │ │ └── RedisUtils.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.IRedis ├── restful ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── restful │ │ ├── RestfulClient.java │ │ ├── RestfulSpecialClient.java │ │ └── http │ │ ├── HttpClient.java │ │ ├── HttpClientFactory.java │ │ ├── request │ │ ├── HttpAddressManager.java │ │ ├── HttpDeleteWithEntity.java │ │ ├── HttpGetWithEntity.java │ │ └── HttpPutWithEntity.java │ │ └── ssl │ │ ├── SSLConfig.java │ │ └── SSLManager.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.IRestful ├── s3 ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── s3 │ │ │ ├── S3Client.java │ │ │ └── S3Utils.java │ └── resources │ │ └── META-INF │ │ └── services │ │ └── com.dtstack.dtcenter.loader.client.IClient │ └── test │ └── java │ └── com │ └── dtstack │ └── dtcenter │ └── common │ └── loader │ └── s3 │ └── S3ClientTest.java ├── sap_hana ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── saphana │ │ ├── SapHanaClient.java │ │ ├── SapHanaConnFactory.java │ │ ├── SapHanaDownloader.java │ │ ├── SapHanaErrorPattern.java │ │ └── SapHanaTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── socket ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── socket │ │ └── SocketClient.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── solr ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── solr │ │ ├── DtSolrClient.java │ │ ├── SolrConsistent.java │ │ ├── SolrErrorPattern.java │ │ ├── SolrUtils.java │ │ └── pool │ │ ├── SolrManager.java │ │ ├── SolrPool.java │ │ ├── SolrPoolConfig.java │ │ └── SolrPoolFactory.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── sonar-project.properties ├── spark ├── pom.xml └── src │ └── main │ ├── java │ ├── com │ │ └── dtstack │ │ │ └── dtcenter │ │ │ └── common │ │ │ └── loader │ │ │ └── spark │ │ │ ├── GroupTypeIgnoreCase.java │ │ │ ├── SparkConnFactory.java │ │ │ ├── SparkErrorPattern.java │ │ │ ├── SparkThriftDriverUtil.java │ │ │ ├── TimeoutExecutor.java │ │ │ ├── client │ │ │ ├── SparkClient.java │ │ │ └── SparkTableClient.java │ │ │ └── downloader │ │ │ ├── SparkORCDownload.java │ │ │ ├── SparkParquetDownload.java │ │ │ └── SparkTextDownload.java │ └── org │ │ └── apache │ │ ├── orc │ │ └── impl │ │ │ └── TreeReaderFactory.java │ │ └── parquet │ │ └── hadoop │ │ └── ParquetFileReader.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKerberos │ └── com.dtstack.dtcenter.loader.client.ITable ├── sqlserver ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── sqlserver │ │ ├── SQLServerConnFactory.java │ │ ├── SqlServerClient.java │ │ ├── SqlServerDownloader.java │ │ ├── SqlServerErrorPattern.java │ │ └── SqlServerTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ └── com.dtstack.dtcenter.loader.client.ITable ├── sqlserver2017 ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── sqlserver │ │ ├── SQLServerConnFactory.java │ │ ├── SqlServerClient.java │ │ └── SqlServerDownloader.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient ├── tbds_hbase ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── tbds │ │ └── hbase │ │ ├── ComparatorType.java │ │ ├── FilterType.java │ │ ├── HbaseClient.java │ │ ├── HbaseClientSpecial.java │ │ ├── HbaseConnFactory.java │ │ ├── HbaseErrorPattern.java │ │ ├── KerberosClient.java │ │ └── pool │ │ └── HbasePoolManager.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IHbase │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── tbds_hdfs ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── tdbs │ │ └── hdfs │ │ ├── GroupTypeIgnoreCase.java │ │ ├── HDFSErrorPattern.java │ │ ├── HadoopConfUtil.java │ │ ├── HdfsConnFactory.java │ │ ├── HdfsOperator.java │ │ ├── TimeoutExecutor.java │ │ ├── YarnConfUtil.java │ │ ├── client │ │ ├── HdfsClient.java │ │ └── HdfsFileClient.java │ │ ├── downloader │ │ ├── HdfsFileDownload.java │ │ ├── HdfsORCDownload.java │ │ ├── HdfsParquetDownload.java │ │ ├── HdfsTextDownload.java │ │ └── YarnLogDownload │ │ │ └── YarnTFileDownload.java │ │ ├── fileMerge │ │ ├── ECompressType.java │ │ ├── core │ │ │ ├── CombineMergeBuilder.java │ │ │ ├── CombineServer.java │ │ │ ├── FileCombineFactory.java │ │ │ ├── OrcCombineServer.java │ │ │ ├── ParquetCombineServer.java │ │ │ └── TextCombineServer.java │ │ └── meta │ │ │ ├── FileMetaData.java │ │ │ ├── OrcMetaData.java │ │ │ └── ParquetMetaData.java │ │ ├── hdfswriter │ │ ├── HdfsOrcWriter.java │ │ ├── HdfsParquetWriter.java │ │ ├── HdfsTextWriter.java │ │ └── HdfsWriter.java │ │ └── util │ │ ├── FileSystemUtils.java │ │ ├── SecurityUtils.java │ │ ├── StringUtil.java │ │ └── UGICacheData.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IHdfsFile │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── tbds_kafka ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── tbds │ │ └── kafka │ │ ├── Kafka.java │ │ ├── KafkaClient.java │ │ ├── KafkaConsistent.java │ │ ├── KafkaErrorPattern.java │ │ ├── KafkaKerberos.java │ │ ├── enums │ │ └── EConsumeType.java │ │ └── util │ │ ├── FileUtils.java │ │ └── KafkaUtil.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKafka │ └── com.dtstack.dtcenter.loader.client.IKerberos ├── test ├── pom.xml └── src │ └── test │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── loader │ │ └── client │ │ ├── BaseTest.java │ │ ├── common │ │ ├── AdapterTest.java │ │ ├── AddressUtilTest.java │ │ ├── CollectionUtilTest.java │ │ ├── DTOUtilTest.java │ │ └── DateUtilTest.java │ │ ├── download │ │ ├── Hive1DownloadTest.java │ │ ├── Hive2DownloadTest.java │ │ ├── Hive3DownloadTest.java │ │ └── SparkDownloadTest.java │ │ ├── issue │ │ ├── Bug1477Test.java │ │ ├── Issue33045Test.java │ │ ├── Issue33097Test.java │ │ ├── Issue34269Test.java │ │ ├── Issue34667Test.java │ │ ├── Issue35211Test.java │ │ ├── Issue35498Test.java │ │ ├── Issue35717Test.java │ │ ├── Issue35748Test.java │ │ ├── Issue35783Test.java │ │ ├── Issue35789Test.java │ │ ├── Issue36310Test.java │ │ ├── Issue37213Test.java │ │ ├── Issue37801Test.java │ │ └── issue36443Test.java │ │ ├── sql │ │ ├── AdbForPgTest.java │ │ ├── AwsS3Test.java │ │ ├── ClickHouseTableTest.java │ │ ├── ClickHouseTest.java │ │ ├── Db2TableTest.java │ │ ├── Db2Test.java │ │ ├── DmDbTest.java │ │ ├── DorisRestfulTest.java │ │ ├── DorisTableTest.java │ │ ├── DorisTest.java │ │ ├── DownloadTest.java │ │ ├── EMQTest.java │ │ ├── Es7Test.java │ │ ├── Es7sslTest.java │ │ ├── EsTest.java │ │ ├── FtpTest.java │ │ ├── Gbase8aV9Test.java │ │ ├── GbaseTableTest.java │ │ ├── GbaseTest.java │ │ ├── Greenplum6Test.java │ │ ├── GreenplumTableTest.java │ │ ├── Hbase2Test.java │ │ ├── HbaseClientKerberosTest.java │ │ ├── HbaseClientTest.java │ │ ├── HbaseKerberosTest.java │ │ ├── HbaseTest.java │ │ ├── HdfsFileKerberosTest.java │ │ ├── HdfsFileTest.java │ │ ├── HdfsTest.java │ │ ├── Hive1KerberosTest.java │ │ ├── Hive1Test.java │ │ ├── Hive1xTableTest.java │ │ ├── Hive3KerberosTest.java │ │ ├── Hive3TableTest.java │ │ ├── Hive3Test.java │ │ ├── HiveKerberosTest.java │ │ ├── HiveTableTest.java │ │ ├── HiveTest.java │ │ ├── ImpalaKerberosTableTest.java │ │ ├── ImpalaKerberosTest.java │ │ ├── ImpalaTableTest.java │ │ ├── ImpalaTest.java │ │ ├── InceptorTableTest.java │ │ ├── InceptorTest.java │ │ ├── InfluxDBTest.java │ │ ├── KafkaKerberosTest.java │ │ ├── KafkaKerberosWithoutBrokerUrlsTest.java │ │ ├── KafkaSaslPlainTest.java │ │ ├── KafkaTest.java │ │ ├── KerberosTest.java │ │ ├── KingbaseTableTest.java │ │ ├── KingbaseTest.java │ │ ├── KuduKerberosTest.java │ │ ├── KuduTest.java │ │ ├── KylinRestfulTest.java │ │ ├── KylinTest.java │ │ ├── LibraTableTest.java │ │ ├── LibraTest.java │ │ ├── MaxComputerTest.java │ │ ├── MongoTest.java │ │ ├── Mysql5TableTest.java │ │ ├── Mysql5Test.java │ │ ├── Mysql8TableTest.java │ │ ├── Mysql8Test.java │ │ ├── OceanBaseTableTest.java │ │ ├── OceanBaseTest.java │ │ ├── OpenTSDBSpecialTest.java │ │ ├── OpenTSDBTest.java │ │ ├── Oracle19CTest.java │ │ ├── OracleTableTest.java │ │ ├── OracleTest.java │ │ ├── Phoenix5KerberosTest.java │ │ ├── Phoenix5Test.java │ │ ├── PhoenixTest.java │ │ ├── PostgreSQLTest.java │ │ ├── PostgresqlTableTest.java │ │ ├── PrestoInceptorTest.java │ │ ├── PrestoTest.java │ │ ├── PrestoWithSchemaTest.java │ │ ├── RedisTest.java │ │ ├── RestfulSpecialTest.java │ │ ├── RestfulTest.java │ │ ├── RestfulsslTest.java │ │ ├── S3Test.java │ │ ├── SQLServer2017Test.java │ │ ├── SQLServerTableTest.java │ │ ├── SQLServerTest.java │ │ ├── SapHana5Test.java │ │ ├── SocketTest.java │ │ ├── SolrKerberosTest.java │ │ ├── SparkKerberosTest.java │ │ ├── SparkTableTest.java │ │ ├── SparkTest.java │ │ ├── TbdsHbaseClientTest.java │ │ ├── TbdsHdfsTest.java │ │ ├── TbdsKafkaTest.java │ │ ├── TiDBTest.java │ │ ├── TrinoKerberosTest.java │ │ ├── TrinoNoSchemaTest.java │ │ ├── TrinoTest.java │ │ ├── VerticaTest.java │ │ └── WebSocketTest.java │ │ └── testutil │ │ └── ReflectUtil.java │ └── resources │ ├── bug │ ├── issue_33045 │ │ └── z.parquet │ ├── issue_33097 │ │ └── _SUCCESS │ ├── issue_34667 │ │ ├── kerberos_dir.zip │ │ └── kerberos_nodir.zip │ └── issue_36310 │ │ └── dir_kerberos.zip │ ├── cdp │ ├── hive.keytab │ └── krb5.conf │ ├── download │ ├── orc │ ├── parquet │ └── text │ ├── eng-cdh │ ├── hadoop.keytab │ ├── hbase-master.keytab │ ├── hive-cdh01.keytab │ ├── hive-cdh03.keytab │ ├── hive-krb.zip │ ├── impalad-cdh3.keytab │ ├── kafka-cdh1.keytab │ ├── krb5.conf │ └── kudu-master.keytab │ ├── eng-cdh2 │ ├── kafka_eng2.keytab │ ├── kafka_krb.xml │ └── krb5.conf │ ├── eng-cdh3 │ ├── core-site.xml │ ├── hbase.keytab │ ├── hive-site.xml │ ├── hive.keytab │ ├── impala.keytab │ ├── krb5.conf │ ├── kudu.keytab │ └── phoenix.keytab │ ├── hdfs-file │ ├── orcfile │ ├── parquetfile │ ├── test.txt │ └── textfile │ ├── hive3 │ ├── hive.keytab │ └── krb5.conf │ ├── krbt │ ├── hdfs.keytab │ ├── kafka.keytab │ ├── krb5.conf │ └── krb5_2.conf │ ├── log4j2.xml │ ├── phoenix5_kerberos │ ├── hbase-site.xml │ ├── hbase.keytab │ └── krb5.conf │ ├── restful │ └── loader_test.jar │ ├── solr │ ├── jaas-client.conf │ ├── krb5.conf │ └── solr.keytab │ ├── ssl-crt │ └── ca.crt │ ├── ssl-p12 │ └── elastic-certificates-89.p12 │ ├── ssl │ ├── cm-auto-global_truststore.jks │ └── ssl-client.xml │ └── tdh │ └── inceptor │ ├── inceptor.keytab │ └── krb5.conf ├── trino ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── trino │ │ ├── TrinoClient.java │ │ ├── TrinoConnFactory.java │ │ ├── TrinoErrorPattern.java │ │ └── TrinoTableClient.java │ └── resources │ └── META-INF │ └── services │ ├── com.dtstack.dtcenter.loader.client.IClient │ ├── com.dtstack.dtcenter.loader.client.IKerberos │ └── com.dtstack.dtcenter.loader.client.ITable ├── vertica ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── dtstack │ │ └── dtcenter │ │ └── common │ │ └── loader │ │ └── vertica │ │ ├── VerticaClient.java │ │ └── VerticaConnFactory.java │ └── resources │ └── META-INF │ └── services │ └── com.dtstack.dtcenter.loader.client.IClient └── websocket ├── pom.xml └── src └── main ├── java └── com │ └── dtstack │ └── dtcenter │ └── common │ └── loader │ └── websocket │ ├── SocketClient.java │ └── WebsocketErrorPattern.java └── resources └── META-INF └── services └── com.dtstack.dtcenter.loader.client.IClient /.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | .project 3 | .idea 4 | dependency-reduced-pom.xml 5 | .settings/ 6 | conf/ 7 | target/ 8 | pluginLibs 9 | 10 | lib/* 11 | run/* 12 | bin/* 13 | out/* 14 | *.iml 15 | .DS_Store 16 | logs -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | build: 2 | stage: test 3 | script: 4 | - mvn clean verify -U -Dmaven.test.failure.ignore=true 5 | - sonar-scanner 6 | - sh ci/sonar_notify.sh 7 | only: 8 | - dev 9 | tags: 10 | - dt-insight -------------------------------------------------------------------------------- /aws_s3/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.aws_s3.AwsS3Client -------------------------------------------------------------------------------- /clickhouse/src/main/java/com/dtstack/dtcenter/common/loader/clickhouse/ClickhouseConnFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.clickhouse; 20 | 21 | import com.dtstack.dtcenter.common.loader.rdbms.ConnFactory; 22 | import com.dtstack.dtcenter.loader.source.DataBaseType; 23 | 24 | /** 25 | * @company: www.dtstack.com 26 | * @Author ：Nanqi 27 | * @Date ：Created in 16:58 2020/1/7 28 | * @Description：ClickHouse 连接工厂 29 | */ 30 | public class ClickhouseConnFactory extends ConnFactory { 31 | public ClickhouseConnFactory() { 32 | this.driverName = DataBaseType.Clickhouse.getDriverClassName(); 33 | this.errorPattern = new ClickhouseErrorPattern(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /clickhouse/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.clickhouse.ClickhouseClient -------------------------------------------------------------------------------- /clickhouse/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.clickhouse.ClickhouseTableClient -------------------------------------------------------------------------------- /common/src/main/java/com/dtstack/dtcenter/common/loader/common/base/CallBack.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.common.base; 20 | 21 | /** 22 | * 方法回调 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 11:59 上午 2021/9/1 26 | * company: www.dtstack.com 27 | */ 28 | public interface CallBack { 29 | 30 | /** 31 | * 方法执行 32 | * 33 | * @param event 入参数 34 | * @return 回参 35 | */ 36 | T execute(M event); 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/java/com/dtstack/dtcenter/common/loader/common/convert/ColumnTypeConverter.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.common.convert; 2 | 3 | /** 4 | * 用于字段类型转换 5 | * 6 | * @author ：wangchuan 7 | * date：Created in 上午10:56 2021/10/11 8 | * company: www.dtstack.com 9 | */ 10 | @FunctionalInterface 11 | public interface ColumnTypeConverter { 12 | 13 | /** 14 | * @param type 字段类型，如 "SHORT", "INT", "TIMESTAMP","INT8" 15 | * @return 转换后的字段类型 TINYINT 16 | */ 17 | String convert(String type); 18 | } 19 | -------------------------------------------------------------------------------- /common/src/main/java/com/dtstack/dtcenter/common/loader/common/enums/SftpAuthType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.common.enums; 20 | 21 | /** 22 | * @company: www.dtstack.com 23 | * @Author ：Nanqi 24 | * @Date ：Created in 13:04 2020/8/22 25 | * @Description：TODO 26 | */ 27 | public enum SftpAuthType { 28 | /** 29 | * 密码校验 30 | */ 31 | PASSWORD(1), 32 | /** 33 | * 密钥登录校验 34 | */ 35 | RSA(2); 36 | 37 | private Integer type; 38 | 39 | SftpAuthType(Integer type) { 40 | this.type = type; 41 | } 42 | 43 | public Integer getType() { 44 | return type; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /common/src/main/java/com/dtstack/dtcenter/common/loader/common/exception/IErrorCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.common.exception; 20 | 21 | /** 22 | * 错误码枚举抽象接口 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 上午10:22 2020/11/6 26 | * company: www.dtstack.com 27 | */ 28 | public interface IErrorCode { 29 | 30 | /** 31 | * 获取错误码 32 | * @return 错误码 33 | */ 34 | Integer getCode (); 35 | 36 | /** 37 | * 获取错误描述 38 | * @return 错误描述 39 | */ 40 | String getDesc (); 41 | 42 | } 43 | -------------------------------------------------------------------------------- /common/src/main/java/com/dtstack/dtcenter/common/loader/common/utils/DelimiterUtil.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.common.utils; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | /** 6 | * 分隔符工具类 7 | * 8 | * @author ：wangchuan 9 | * date：Created in 下午4:35 2022/5/20 10 | * company: www.dtstack.com 11 | */ 12 | public class DelimiterUtil { 13 | 14 | /** 15 | * 取首字符分隔符, 忽略转义 16 | * 17 | * @param delimiter 分隔符 18 | * @return 分隔符 19 | */ 20 | public static String charAtIgnoreEscape(String delimiter) { 21 | if (StringUtils.isEmpty(delimiter)) { 22 | return null; 23 | } 24 | if (delimiter.length() > 1 && delimiter.charAt(0) == '\\') { 25 | return delimiter.substring(0, 2); 26 | } 27 | return delimiter.substring(0, 1); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/com/dtstack/dtcenter/common/loader/common/utils/PathUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.common.utils; 20 | 21 | /** 22 | * @company: www.dtstack.com 23 | * @Author ：Nanqi 24 | * @Date ：Created in 17:24 2020/8/28 25 | * @Description：文件路径处理 26 | */ 27 | public class PathUtils { 28 | /** 29 | * 处理路径中存在多个分隔符的情况 30 | * 31 | * @param path 32 | * @return 33 | */ 34 | public static String removeMultiSeparatorChar(String path) { 35 | return path.replaceAll("//*", "/"); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/java/com/dtstack/dtcenter/common/loader/common/utils/PropertyUtil.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.common.utils; 2 | 3 | import java.util.Objects; 4 | import java.util.Properties; 5 | 6 | /** 7 | * properties 工具类 8 | * 9 | * @author ：wangchuan 10 | * date：Created in 上午10:57 2021/9/13 11 | * company: www.dtstack.com 12 | */ 13 | public class PropertyUtil { 14 | 15 | /** 16 | * key 和 value 不为空是插入 17 | * 18 | * @param properties 配置类 19 | * @param key key 20 | * @param value value 21 | */ 22 | public static void putIfNotNull(Properties properties, Object key, Object value) { 23 | if (Objects.nonNull(key) && Objects.nonNull(value)) { 24 | properties.put(key, value); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /common/src/main/java/com/dtstack/dtcenter/common/loader/common/utils/SchemaUtil.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.common.utils; 2 | 3 | import com.dtstack.dtcenter.loader.dto.SqlQueryDTO; 4 | import com.dtstack.dtcenter.loader.dto.source.ISourceDTO; 5 | import com.dtstack.dtcenter.loader.dto.source.RdbmsSourceDTO; 6 | import com.dtstack.dtcenter.loader.utils.AssertUtils; 7 | import org.apache.commons.lang3.StringUtils; 8 | 9 | import java.util.Objects; 10 | 11 | /** 12 | * schema 工具 13 | * 14 | * @author ：wangchuan 15 | * date：Created in 下午3:31 2022/1/3 16 | * company: www.dtstack.com 17 | */ 18 | public class SchemaUtil { 19 | 20 | /** 21 | * 获取 schema，优先从 SqlQueryDTO 中获取 22 | * 23 | * @param sourceDTO 数据源连接信息 24 | * @param sqlQueryDTO 查询条件 25 | * @return schema 信息 26 | */ 27 | public static String getSchema(ISourceDTO sourceDTO, SqlQueryDTO sqlQueryDTO) { 28 | AssertUtils.notNull(sourceDTO, "sourceDTO can't be null."); 29 | if (sourceDTO instanceof RdbmsSourceDTO) { 30 | if (Objects.isNull(sqlQueryDTO)) { 31 | return ((RdbmsSourceDTO) sourceDTO).getSchema(); 32 | } else { 33 | return StringUtils.isNotBlank(sqlQueryDTO.getSchema()) ? 34 | sqlQueryDTO.getSchema() : ((RdbmsSourceDTO) sourceDTO).getSchema(); 35 | } 36 | } 37 | return Objects.isNull(sqlQueryDTO) ? null : sqlQueryDTO.getSchema(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/ClassLoaderCallBack.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader; 20 | 21 | /** 22 | * @company: www.dtstack.com 23 | * @Author ：Nanqi 24 | * @Date ：Created in 15:56 2020/1/6 25 | */ 26 | @FunctionalInterface 27 | public interface ClassLoaderCallBack { 28 | /** 29 | * 抽象执行类 30 | * @return 31 | * @throws Exception 32 | */ 33 | T execute() throws Exception; 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/client/IRedis.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.client; 2 | 3 | import com.dtstack.dtcenter.loader.dto.RedisQueryDTO; 4 | import com.dtstack.dtcenter.loader.dto.source.ISourceDTO; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | /** 10 | *

提供 redis 相关操作方法

11 | * 12 | * @author ：qianyi 13 | * date：Created in 上午10:06 2021/8/16 14 | * company: www.dtstack.com 15 | */ 16 | public interface IRedis { 17 | /** 18 | * redis 自定义查询 19 | * 20 | * @param source 21 | * @param queryDTO 22 | * @return 23 | */ 24 | Map executeQuery(ISourceDTO source, RedisQueryDTO queryDTO); 25 | 26 | List preViewKey(ISourceDTO source, RedisQueryDTO queryDTO); 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/client/hdfs/HdfsFileClientCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.client.hdfs; 20 | 21 | import com.dtstack.dtcenter.loader.client.ClientCache; 22 | 23 | /** 24 | * @company: www.dtstack.com 25 | * @Author ：Nanqi 26 | * @Date ：Created in 14:38 2020/8/10 27 | * @Description： HDFS 文件操作插件客户端 28 | */ 29 | @Deprecated 30 | public class HdfsFileClientCache extends ClientCache { 31 | } 32 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/client/mq/KafkaClientCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.client.mq; 20 | 21 | import com.dtstack.dtcenter.loader.client.ClientCache; 22 | 23 | /** 24 | * @company: www.dtstack.com 25 | * @Author ：Nanqi 26 | * @Date ：Created in 17:37 2020/9/10 27 | * @Description：Kafka 客户端 28 | */ 29 | @Deprecated 30 | public class KafkaClientCache extends ClientCache { 31 | } 32 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/client/redis/RedisClientFactory.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.client.redis; 2 | 3 | import com.dtstack.dtcenter.loader.ClassLoaderCallBack; 4 | import com.dtstack.dtcenter.loader.ClassLoaderCallBackMethod; 5 | import com.dtstack.dtcenter.loader.client.ClientFactory; 6 | import com.dtstack.dtcenter.loader.client.IRedis; 7 | import com.dtstack.dtcenter.loader.exception.DtLoaderException; 8 | 9 | import java.util.Iterator; 10 | import java.util.ServiceLoader; 11 | 12 | /** 13 | *

redis 客户端工厂

14 | * 15 | * @author ：qianyi 16 | * date：Created in 上午10:06 2021/8/16 17 | * company: www.dtstack.com 18 | */ 19 | public class RedisClientFactory { 20 | public static IRedis createPluginClass(String pluginName) throws Exception { 21 | ClassLoader classLoader = ClientFactory.getClassLoader(pluginName); 22 | return ClassLoaderCallBackMethod.callbackAndReset((ClassLoaderCallBack) () -> { 23 | ServiceLoader redis = ServiceLoader.load(IRedis.class); 24 | Iterator iClientIterator = redis.iterator(); 25 | if (!iClientIterator.hasNext()) { 26 | throw new DtLoaderException("This plugin type is not supported: " + pluginName); 27 | } 28 | IRedis iRedis = iClientIterator.next(); 29 | return new RedisProxy(iRedis); 30 | }, classLoader); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/client/redis/RedisProxy.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.client.redis; 2 | 3 | import com.dtstack.dtcenter.loader.ClassLoaderCallBackMethod; 4 | import com.dtstack.dtcenter.loader.client.IRedis; 5 | import com.dtstack.dtcenter.loader.dto.RedisQueryDTO; 6 | import com.dtstack.dtcenter.loader.dto.source.ISourceDTO; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | public class RedisProxy implements IRedis { 12 | 13 | IRedis targetClient = null; 14 | 15 | public RedisProxy(IRedis iRedis) { 16 | this.targetClient = iRedis; 17 | } 18 | 19 | @Override 20 | public Map executeQuery(ISourceDTO source, RedisQueryDTO queryDTO) { 21 | return ClassLoaderCallBackMethod.callbackAndReset(() -> targetClient.executeQuery(source, queryDTO), 22 | targetClient.getClass().getClassLoader()); 23 | } 24 | 25 | @Override 26 | public List preViewKey(ISourceDTO source, RedisQueryDTO queryDTO) { 27 | return ClassLoaderCallBackMethod.callbackAndReset(() -> targetClient.preViewKey(source, queryDTO), 28 | targetClient.getClass().getClassLoader()); 29 | } 30 | 31 | 32 | } 33 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/client/sql/DataSourceClientCache.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.client.sql; 20 | 21 | import com.dtstack.dtcenter.loader.client.ClientCache; 22 | 23 | /** 24 | * @company: www.dtstack.com 25 | * @Author ：Nanqi 26 | * @Date ：Created in 17:38 2020/9/10 27 | * @Description：数据源插件 28 | */ 29 | @Deprecated 30 | public class DataSourceClientCache extends ClientCache { 31 | } 32 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/Database.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto; 2 | 3 | import lombok.Data; 4 | 5 | /** 6 | * 数据库信息 7 | * 8 | * @author ：wangchuan 9 | * date：Created in 下午1:56 2021/9/14 10 | * company: www.dtstack.com 11 | */ 12 | @Data 13 | public class Database { 14 | 15 | /** 16 | * 数据库名称 17 | */ 18 | private String DbName; 19 | 20 | /** 21 | * 数据库注释 22 | */ 23 | private String comment; 24 | 25 | /** 26 | * 数据库存储位置 27 | */ 28 | private String location; 29 | 30 | /** 31 | * 所有者名称 32 | */ 33 | private String ownerName; 34 | 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/HDFSImportColumn.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto; 20 | 21 | import lombok.Data; 22 | 23 | import java.text.SimpleDateFormat; 24 | 25 | /** 26 | * hdfs导入类 27 | * 28 | * @author ：wangchuan 29 | * date：Created in 上午10:33 2020/8/11 30 | * company: www.dtstack.com 31 | */ 32 | @Data 33 | public class HDFSImportColumn { 34 | 35 | private String key; 36 | 37 | private String format; 38 | 39 | private SimpleDateFormat dateFormat; 40 | 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/KafkaOffsetDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto; 20 | 21 | import lombok.Data; 22 | 23 | /** 24 | * @company: www.dtstack.com 25 | * @Author ：Nanqi 26 | * @Date ：Created in 17:22 2020/2/26 27 | * @Description：Kafka 偏移量 28 | */ 29 | @Data 30 | public class KafkaOffsetDTO { 31 | /** 32 | * 分区编号 33 | */ 34 | private Integer partition; 35 | 36 | /** 37 | * 最新的偏移量 38 | */ 39 | private Long lastOffset; 40 | 41 | /** 42 | * 最早的偏移量 43 | */ 44 | private Long firstOffset; 45 | } 46 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/KafkaTopicDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto; 20 | 21 | import lombok.Builder; 22 | import lombok.Data; 23 | 24 | /** 25 | * @company: www.dtstack.com 26 | * @Author ：Nanqi 27 | * @Date ：Created in 16:45 2020/2/26 28 | * @Description：Kafka Topic 传输对象 29 | */ 30 | @Data 31 | @Builder 32 | public class KafkaTopicDTO { 33 | /** 34 | * 名称 35 | */ 36 | private String topicName; 37 | 38 | /** 39 | * 分区数 40 | */ 41 | @Builder.Default 42 | private Integer partitions = 1; 43 | 44 | /** 45 | * 复制因子 46 | */ 47 | @Builder.Default 48 | private Short replicationFactor = 1; 49 | } 50 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/RedisQueryDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto; 2 | 3 | import com.dtstack.dtcenter.loader.enums.RedisCompareOp; 4 | import com.dtstack.dtcenter.loader.enums.RedisDataType; 5 | import lombok.AllArgsConstructor; 6 | import lombok.Builder; 7 | import lombok.Data; 8 | import lombok.NoArgsConstructor; 9 | import lombok.NonNull; 10 | 11 | import java.util.List; 12 | 13 | /** 14 | * redis queryDTO 15 | * 16 | * @author ：qianyi 17 | * date：Created in 上午15:02 2021/9/8 18 | * company: www.dtstack.com 19 | */ 20 | @Data 21 | @Builder 22 | @NoArgsConstructor 23 | @AllArgsConstructor 24 | public class RedisQueryDTO { 25 | 26 | /** 27 | * redis，模糊查询限制key的条数 28 | */ 29 | private Integer keyLimit; 30 | 31 | /** 32 | * redis，查询结果条数限制， 只针对list,zset 33 | */ 34 | private Integer ResultLimit; 35 | 36 | /** 37 | * redis 数据类型 38 | */ 39 | @NonNull 40 | private RedisDataType redisDataType; 41 | 42 | /** 43 | * 操作符,= or like 44 | */ 45 | private RedisCompareOp redisCompareOp; 46 | 47 | /** 48 | * keys， 49 | */ 50 | private List keys; 51 | 52 | 53 | /** 54 | * keys 模糊查询 55 | */ 56 | private String keyPattern; 57 | 58 | } 59 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/SSLConfigDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto; 2 | 3 | import lombok.Builder; 4 | import lombok.Data; 5 | 6 | /** 7 | * SSL 配置相关 8 | * 9 | * @author ：wangchuan 10 | * date：Created in 上午10:47 2021/9/13 11 | * company: www.dtstack.com 12 | */ 13 | @Data 14 | @Builder 15 | public class SSLConfigDTO { 16 | 17 | /** 18 | * TLS 验证的方法。共有三种模式：（FULL 默认）CA和NONE 19 | * 对于FULL，执行正常的 TLS 验证 20 | * 对于CA，仅验证 CA，但允许主机名不匹配 21 | * 对于NONE，没有验证 22 | */ 23 | private String SSLVerification; 24 | 25 | /** 26 | * 连接到启用了证书身份验证的 Trino 集群时使用。指定PEM或JKS文件的路径 27 | */ 28 | private String SSLKeyStorePath; 29 | 30 | /** 31 | * KeyStore 的密码(如果有) 32 | */ 33 | private String SSLKeyStorePassword; 34 | 35 | /** 36 | * 密钥库的类型。默认类型由 Java keystore.type安全属性提供 37 | */ 38 | private String SSLKeyStoreType; 39 | 40 | /** 41 | * 要使用的 Java TrustStore 文件的位置。验证 HTTPS 服务器证书 42 | */ 43 | private String SSLTrustStorePath; 44 | 45 | /** 46 | * TrustStore 的密码 47 | */ 48 | private String SSLTrustStorePassword; 49 | 50 | /** 51 | * TrustStore 的类型。默认类型由 Java keystore.type安全属性提供 52 | */ 53 | private String SSLTrustStoreType; 54 | } 55 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/TableInfo.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Builder; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | /** 9 | * table Info 10 | * 11 | * @author ：wangchuan 12 | * date：Created in 下午12:25 2022/3/21 13 | * company: www.dtstack.com 14 | */ 15 | @Data 16 | @Builder 17 | @NoArgsConstructor 18 | @AllArgsConstructor 19 | public class TableInfo { 20 | 21 | /** 22 | * db 信息 23 | */ 24 | private String database; 25 | 26 | /** 27 | * schema 信息 28 | */ 29 | private String schema; 30 | 31 | /** 32 | * 表名 33 | */ 34 | private String tableName; 35 | } 36 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/comparator/ByteArrayComparable.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.comparator; 20 | 21 | /** 22 | * - 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 下午5:50 2020/8/24 26 | * company: www.dtstack.com 27 | */ 28 | public abstract class ByteArrayComparable { 29 | 30 | byte[] value; 31 | 32 | public ByteArrayComparable(byte[] value) { 33 | this.value = value; 34 | } 35 | 36 | public byte[] getValue() { 37 | return value; 38 | } 39 | 40 | public abstract Integer getComparatorType(); 41 | 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/comparator/NullComparator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.comparator; 20 | 21 | import com.dtstack.dtcenter.loader.enums.Comparator; 22 | 23 | /** 24 | * - 25 | * 26 | * @author ：wangchuan 27 | * date：Created in 下午5:49 2020/8/24 28 | * company: www.dtstack.com 29 | */ 30 | public class NullComparator extends ByteArrayComparable{ 31 | 32 | public NullComparator() { 33 | super(new byte[0]); 34 | } 35 | 36 | @Override 37 | public Integer getComparatorType() { 38 | return Comparator.NULL.getVal(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/filter/Filter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.filter; 20 | 21 | /** 22 | * @author ：wangchuan 23 | * date：Created in 上午10:01 2020/8/25 24 | * company: www.dtstack.com 25 | */ 26 | public abstract class Filter { 27 | 28 | public abstract Integer getFilterType(); 29 | 30 | } 31 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/AdbForPgSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import lombok.Data; 22 | import lombok.ToString; 23 | import lombok.experimental.SuperBuilder; 24 | 25 | /** 26 | * ADB_PostgreSQL sourceDTO 27 | * 28 | * @author ：wangchuan 29 | * date：Created in 下午5:04 2021/5/28 30 | * company: www.dtstack.com 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class AdbForPgSourceDTO extends PostgresqlSourceDTO { 36 | 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/ClickHouseSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：Nanqi 29 | * @Date ：Created in 16:20 2020/5/22 30 | * @Description：Clickhouse 数据杨信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class ClickHouseSourceDTO extends RdbmsSourceDTO { 36 | 37 | @Override 38 | public Integer getSourceType() { 39 | return DataSourceType.Clickhouse.getVal(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/Db2SourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：Nanqi 29 | * @Date ：Created in 17:21 2020/5/22 30 | * @Description：DB2 数据源信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class Db2SourceDTO extends RdbmsSourceDTO { 36 | 37 | 38 | @Override 39 | public Integer getSourceType() { 40 | return DataSourceType.DB2.getVal(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/DmForOracleSourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.Data; 5 | import lombok.ToString; 6 | import lombok.experimental.SuperBuilder; 7 | 8 | /** 9 | * @company: www.dtstack.com 10 | * @Author ：Nanqi 11 | * @Date ：Created in 17:23 2020/5/22 12 | * @Description：达梦数据源信息 13 | */ 14 | @Data 15 | @ToString 16 | @SuperBuilder 17 | public class DmForOracleSourceDTO extends DmSourceDTO { 18 | 19 | @Override 20 | public Integer getSourceType() { 21 | return DataSourceType.DMDB_For_Oracle.getVal(); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/DmSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：Nanqi 29 | * @Date ：Created in 17:23 2020/5/22 30 | * @Description：达梦数据源信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class DmSourceDTO extends RdbmsSourceDTO { 36 | 37 | @Override 38 | public Integer getSourceType() { 39 | return DataSourceType.DMDB.getVal(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/DorisRestfulSourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.Data; 5 | import lombok.ToString; 6 | import lombok.experimental.SuperBuilder; 7 | 8 | @Data 9 | @ToString 10 | @SuperBuilder 11 | public class DorisRestfulSourceDTO extends RestfulSourceDTO { 12 | 13 | /** 14 | * 集群名称 15 | */ 16 | private String cluster; 17 | 18 | /** 19 | * 库 20 | */ 21 | 22 | private String schema; 23 | 24 | private String userName; 25 | 26 | private String password; 27 | 28 | @Override 29 | public String getUsername() { 30 | return userName; 31 | } 32 | 33 | @Override 34 | public String getPassword() { 35 | return password; 36 | } 37 | 38 | @Override 39 | public Integer getSourceType() { 40 | return DataSourceType.DorisRestful.getVal(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/DorisSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | @Data 27 | @ToString 28 | @SuperBuilder 29 | public class DorisSourceDTO extends Mysql5SourceDTO { 30 | @Override 31 | public Integer getSourceType() { 32 | return DataSourceType.DORIS.getVal(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/ES7SourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.NoArgsConstructor; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | @ToString 27 | @SuperBuilder 28 | @NoArgsConstructor 29 | public class ES7SourceDTO extends ESSourceDTO { 30 | @Override 31 | public Integer getSourceType() { 32 | return DataSourceType.ES7.getVal(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/GBaseSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：Nanqi 29 | * @Date ：Created in 17:38 2020/5/22 30 | * @Description：Gbase 数据源 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class GBaseSourceDTO extends RdbmsSourceDTO { 36 | 37 | @Override 38 | public Integer getSourceType() { 39 | return DataSourceType.GBase_8a.getVal(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/Hive3CDPSourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Builder; 6 | import lombok.Data; 7 | import lombok.NoArgsConstructor; 8 | import lombok.ToString; 9 | import lombok.experimental.SuperBuilder; 10 | 11 | /** 12 | * hive3-cdp sourceDTO 13 | * 14 | * @author ：wangchuan 15 | * date：Created in 下午8:29 2021/12/13 16 | * company: www.dtstack.com 17 | */ 18 | @Data 19 | @ToString 20 | @SuperBuilder 21 | @NoArgsConstructor 22 | @AllArgsConstructor 23 | public class Hive3CDPSourceDTO extends RdbmsSourceDTO { 24 | /** 25 | * Hadoop defaultFS 26 | */ 27 | @Builder.Default 28 | private String defaultFS = ""; 29 | 30 | /** 31 | * metastore 32 | */ 33 | private String metastore; 34 | 35 | /** 36 | * Hadoop/ Hbase 配置信息 37 | */ 38 | private String config; 39 | 40 | /** 41 | * hive ssl 42 | */ 43 | private HiveSslConfig hiveSslConfig; 44 | 45 | 46 | @Override 47 | public Integer getSourceType() { 48 | return DataSourceType.HIVE3_CDP.getVal(); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/HiveSslConfig.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | 6 | /** 7 | * hive 开启ssl认证参数 8 | * 9 | * @author luming 10 | * @date 2022/1/4 11 | */ 12 | @AllArgsConstructor 13 | @Data 14 | public class HiveSslConfig { 15 | /** 16 | * 是否开启ssl 17 | */ 18 | private Boolean useSsl; 19 | 20 | /** 21 | * 密钥库路径 22 | */ 23 | private String keystorePath; 24 | 25 | /** 26 | * 密钥库密码 27 | */ 28 | private String keystorePassword; 29 | } 30 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/HuaweiHbaseSourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.Data; 5 | import lombok.ToString; 6 | import lombok.experimental.SuperBuilder; 7 | 8 | 9 | @Data 10 | @ToString 11 | @SuperBuilder 12 | public class HuaweiHbaseSourceDTO extends HbaseSourceDTO { 13 | 14 | @Override 15 | public Integer getSourceType() { 16 | return DataSourceType.HUAWEI_HBASE.getVal(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/HuaweiKafkaSourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.Data; 5 | import lombok.ToString; 6 | import lombok.experimental.SuperBuilder; 7 | 8 | 9 | @Data 10 | @ToString 11 | @SuperBuilder 12 | public class HuaweiKafkaSourceDTO extends KafkaSourceDTO { 13 | 14 | @Override 15 | public Integer getSourceType() { 16 | return DataSourceType.HUAWEI_KAFKA.getVal(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/KuduSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：Nanqi 29 | * @Date ：Created in 19:31 2020/5/22 30 | * @Description：Kudu 数据源信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class KuduSourceDTO extends RdbmsSourceDTO { 36 | 37 | @Override 38 | public Integer getSourceType() { 39 | return DataSourceType.Kudu.getVal(); 40 | } 41 | 42 | 43 | } 44 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/KylinSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：Wangchuan 29 | * @Date ：Created in 19:24 2020/5/22 30 | * @Description：Mongo 数据源信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class KylinSourceDTO extends RdbmsSourceDTO { 36 | 37 | @Override 38 | public Integer getSourceType() { 39 | return DataSourceType.Kylin.getVal(); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/LibraSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：WangChuan 29 | * @Date ：Created in 19:21 2020/5/22 30 | * @Description：Libra数据源信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class LibraSourceDTO extends RdbmsSourceDTO { 36 | 37 | 38 | @Override 39 | public Integer getSourceType() { 40 | return DataSourceType.LIBRA.getVal(); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/Mysql5SourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：WangChuan 29 | * @Date ：Created in 17:45 2020/5/22 30 | * @Description：Mysql5数据源信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class Mysql5SourceDTO extends RdbmsSourceDTO { 36 | 37 | @Override 38 | public Integer getSourceType() { 39 | return DataSourceType.MySQL.getVal(); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/Phoenix5SourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：Nanqi 29 | * @Date ：Created in 16:56 2020/7/16 30 | * @Description：Phoenix5 数据源信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class Phoenix5SourceDTO extends RdbmsSourceDTO { 36 | 37 | @Override 38 | public Integer getSourceType() { 39 | return DataSourceType.PHOENIX5.getVal(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/PhoenixSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：Nanqi 29 | * @Date ：Created in 18:24 2020/5/22 30 | * @Description：Phoenix 数据源信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class PhoenixSourceDTO extends RdbmsSourceDTO { 36 | 37 | @Override 38 | public Integer getSourceType() { 39 | return DataSourceType.Phoenix.getVal(); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/SapHana1SourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.Data; 5 | import lombok.ToString; 6 | import lombok.experimental.SuperBuilder; 7 | 8 | @Data 9 | @ToString 10 | @SuperBuilder 11 | public class SapHana1SourceDTO extends SapHanaSourceDTO { 12 | 13 | @Override 14 | public Integer getSourceType() { 15 | return DataSourceType.SAP_HANA1.getVal(); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/SapHana2SourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.Data; 5 | import lombok.ToString; 6 | import lombok.experimental.SuperBuilder; 7 | 8 | @Data 9 | @ToString 10 | @SuperBuilder 11 | public class SapHana2SourceDTO extends SapHanaSourceDTO { 12 | 13 | @Override 14 | public Integer getSourceType() { 15 | return DataSourceType.SAP_HANA2.getVal(); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/SapHanaSourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.Data; 5 | import lombok.ToString; 6 | import lombok.experimental.SuperBuilder; 7 | 8 | @Data 9 | @ToString 10 | @SuperBuilder 11 | public class SapHanaSourceDTO extends RdbmsSourceDTO { 12 | 13 | @Override 14 | public Integer getSourceType() { 15 | return DataSourceType.SAP_HANA1.getVal(); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/TbdsHbaseSourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.Data; 5 | import lombok.ToString; 6 | import lombok.experimental.SuperBuilder; 7 | 8 | 9 | @Data 10 | @ToString 11 | @SuperBuilder 12 | public class TbdsHbaseSourceDTO extends HbaseSourceDTO { 13 | 14 | private String tbdsUsername; 15 | 16 | private String tbdsSecureId; 17 | 18 | private String tbdsSecureKey; 19 | 20 | @Override 21 | public Integer getSourceType() { 22 | return DataSourceType.TBDS_HBASE.getVal(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/TbdsKafkaSourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.source.DataSourceType; 4 | import lombok.Data; 5 | import lombok.ToString; 6 | import lombok.experimental.SuperBuilder; 7 | 8 | 9 | @Data 10 | @ToString 11 | @SuperBuilder 12 | public class TbdsKafkaSourceDTO extends KafkaSourceDTO { 13 | 14 | private String secureId; 15 | 16 | private String secureKey; 17 | 18 | 19 | @Override 20 | public Integer getSourceType() { 21 | return DataSourceType.TBDS_KAFKA.getVal(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/TrinoSourceDTO.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.dto.source; 2 | 3 | import com.dtstack.dtcenter.loader.dto.SSLConfigDTO; 4 | import com.dtstack.dtcenter.loader.source.DataSourceType; 5 | import lombok.Data; 6 | import lombok.ToString; 7 | import lombok.experimental.SuperBuilder; 8 | 9 | /** 10 | * trino sourceDTO 11 | * 12 | * @author ：wangchuan 13 | * date：Created in 下午2:23 2021/9/7 14 | * company: www.dtstack.com 15 | */ 16 | @Data 17 | @ToString 18 | @SuperBuilder 19 | public class TrinoSourceDTO extends RdbmsSourceDTO { 20 | 21 | /** 22 | * 判断是否开启 ssl 认证，不为 null 表示开启 ssl 认证 23 | */ 24 | private SSLConfigDTO sslConfigDTO; 25 | 26 | @Override 27 | public Integer getSourceType() { 28 | return DataSourceType.TRINO.getVal(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/source/VerticaSourceDTO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.source; 20 | 21 | import com.dtstack.dtcenter.loader.source.DataSourceType; 22 | import lombok.Data; 23 | import lombok.ToString; 24 | import lombok.experimental.SuperBuilder; 25 | 26 | /** 27 | * @company: www.dtstack.com 28 | * @Author ：Nanqi 29 | * @Date ：Created in 14:35 2020/12/8 30 | * @Description：Vertica 数据源信息 31 | */ 32 | @Data 33 | @ToString 34 | @SuperBuilder 35 | public class VerticaSourceDTO extends RdbmsSourceDTO { 36 | 37 | 38 | @Override 39 | public Integer getSourceType() { 40 | return DataSourceType.VERTICA.getVal(); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/tsdb/DeltaOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.tsdb; 20 | 21 | import lombok.Builder; 22 | import lombok.Data; 23 | 24 | /** 25 | * 聚合器类型 26 | * 27 | * @author ：wangchuan 28 | * date：Created in 上午10:20 2021/6/24 29 | * company: www.dtstack.com 30 | */ 31 | @Data 32 | @Builder 33 | public class DeltaOptions { 34 | 35 | private Boolean counter; 36 | 37 | private Boolean dropResets; 38 | 39 | private Long counterMax; 40 | } 41 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/tsdb/Filter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.tsdb; 20 | 21 | import lombok.Builder; 22 | import lombok.Data; 23 | 24 | /** 25 | * 过滤器 26 | * 27 | * @author ：wangchuan 28 | * date：Created in 上午10:20 2021/6/24 29 | * company: www.dtstack.com 30 | */ 31 | @Data 32 | @Builder 33 | public class Filter { 34 | 35 | private FilterType type; 36 | 37 | private String tagk; 38 | 39 | private String filter; 40 | 41 | @Builder.Default 42 | private Boolean groupBy = false; 43 | } 44 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/tsdb/FilterType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.tsdb; 20 | 21 | /** 22 | * 过滤器类型 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 上午10:20 2021/6/24 26 | * company: www.dtstack.com 27 | */ 28 | public enum FilterType { 29 | LiteralOr("literal_or"), 30 | 31 | NotLiteralOr("not_literal_or"), 32 | 33 | Wildcard("wildcard"), 34 | 35 | Regexp("regexp"), 36 | 37 | GeoIntersects("intersects"); 38 | 39 | private final String name; 40 | 41 | FilterType(String name) { 42 | this.name = name; 43 | } 44 | 45 | @Override 46 | public String toString() { 47 | return name; 48 | } 49 | 50 | } 51 | 52 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/tsdb/RateOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.tsdb; 20 | 21 | import lombok.Builder; 22 | import lombok.Data; 23 | 24 | /** 25 | * @author ：wangchuan 26 | * date：Created in 上午10:20 2021/6/24 27 | * company: www.dtstack.com 28 | */ 29 | @Data 30 | @Builder 31 | public class RateOptions { 32 | 33 | private Boolean counter; 34 | 35 | private Boolean dropResets; 36 | 37 | private Long counterMax; 38 | 39 | private Long resetValue; 40 | } 41 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/dto/tsdb/Suggest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.dto.tsdb; 20 | 21 | /** 22 | * suggest 类型 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 上午10:20 2021/6/24 26 | * company: www.dtstack.com 27 | */ 28 | public enum Suggest { 29 | 30 | Metrics("metrics"), 31 | 32 | Field("field"), 33 | 34 | Tagk("tagk"), 35 | 36 | Tagv("tagv"); 37 | 38 | private final String name; 39 | 40 | Suggest(String name) { 41 | this.name = name; 42 | } 43 | 44 | public String getName() { 45 | return name; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/enums/CommandType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.enums; 20 | 21 | /** 22 | * @Description: 命令操作 23 | * @Author: qianyi 24 | * @Date: 2021/6/10 25 | */ 26 | public enum CommandType { 27 | /** 28 | * insert 操作 29 | */ 30 | INSERT(1), 31 | /** 32 | * update 操作 33 | */ 34 | UPDATE(2), 35 | /** 36 | * delete操作 37 | */ 38 | DELETE(3); 39 | 40 | private Integer type; 41 | 42 | public Integer getType() { 43 | return type; 44 | } 45 | 46 | CommandType(Integer type) { 47 | this.type = type; 48 | } 49 | } -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/enums/HbaseFilterType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.enums; 20 | 21 | /** 22 | * - 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 上午10:52 2020/8/25 26 | * company: www.dtstack.com 27 | */ 28 | public enum HbaseFilterType { 29 | 30 | PAGE_FILTER(1), 31 | 32 | SINGLE_COLUMN_VALUE_FILTER(2), 33 | 34 | ROW_FILTER(3), 35 | 36 | TIMESTAMP_FILTER(4), 37 | 38 | FILTER_LIST(5); 39 | 40 | private final Integer val; 41 | 42 | HbaseFilterType(Integer val) { 43 | this.val = val; 44 | } 45 | 46 | public Integer getVal() { 47 | return val; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/enums/MatchType.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.enums; 2 | 3 | /** 4 | * 匹配类型 5 | * 6 | * @author ：wangchuan 7 | * date：Created in 下午9:07 2021/12/6 8 | * company: www.dtstack.com 9 | */ 10 | public enum MatchType { 11 | 12 | /** 13 | * 前缀匹配 14 | */ 15 | PREFIX(), 16 | 17 | /** 18 | * 后缀匹配 19 | */ 20 | SUFFIX(), 21 | 22 | /** 23 | * 全匹配 24 | */ 25 | ALL(); 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/enums/RedisCompareOp.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.enums; 2 | 3 | /** 4 | * 类作用：redis 操作符 5 | */ 6 | public enum RedisCompareOp { 7 | 8 | /** 9 | * equals = 10 | */ 11 | EQUAL, 12 | 13 | /** 14 | * like 15 | */ 16 | LIKE, 17 | } 18 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/enums/RedisDataType.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.enums; 2 | 3 | /** 4 | * @Author ：qianyi 5 | * @Date ：Created in 10:15 2021/8/16 6 | * @Description：redis 数据类型 7 | */ 8 | public enum RedisDataType { 9 | 10 | /** 11 | * Redis Sorted Sets are, similarly to Redis Sets, non repeating collections of Strings. 12 | */ 13 | STRING, 14 | 15 | /** 16 | * Redis Hashes are maps between string fields and string values. 17 | */ 18 | HASH, 19 | 20 | /** 21 | * Redis Lists are simply lists of strings, sorted by insertion order. 22 | */ 23 | LIST, 24 | 25 | /** 26 | * Redis Sets are an unordered collection of Strings. 27 | */ 28 | SET, 29 | 30 | /** 31 | * Strings are the most basic kind of Redis value. 32 | */ 33 | ZSET 34 | } 35 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/exception/ClientAccessException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.exception; 20 | 21 | /** 22 | * @company: www.dtstack.com 23 | * @Author ：Nanqi 24 | * @Date ：Created in 15:55 2020/1/3 25 | * @Description：插件化异常 26 | */ 27 | public class ClientAccessException extends DtLoaderException { 28 | private static final String CLIENT_INIT_EXCEPTION = "Client access exception. "; 29 | 30 | public ClientAccessException(Throwable cause) { 31 | super(CLIENT_INIT_EXCEPTION, cause); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /core/src/main/java/com/dtstack/dtcenter/loader/functions/Callback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.functions; 20 | 21 | /** 22 | * @company: www.dtstack.com 23 | * @Author ：Nanqi 24 | * @Date ：Created in 14:48 2020/8/21 25 | * @Description：TODO 26 | */ 27 | @FunctionalInterface 28 | public interface Callback { 29 | 30 | /** 31 | * 执行 32 | * 33 | * @param event 34 | * @return 35 | */ 36 | Object submit(E event); 37 | } 38 | -------------------------------------------------------------------------------- /csp_s3/pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/csp_s3/pom.xml -------------------------------------------------------------------------------- /db2/src/main/java/com/dtstack/dtcenter/common/loader/db2/Db2ConnFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.db2; 20 | 21 | import com.dtstack.dtcenter.common.loader.rdbms.ConnFactory; 22 | import com.dtstack.dtcenter.loader.source.DataBaseType; 23 | 24 | /** 25 | * @company: www.dtstack.com 26 | * @Author ：Nanqi 27 | * @Date ：Created in 16:09 2020/1/7 28 | * @Description：Db2 工厂类 29 | */ 30 | public class Db2ConnFactory extends ConnFactory { 31 | public Db2ConnFactory() { 32 | this.driverName = DataBaseType.DB2.getDriverClassName(); 33 | this.errorPattern = new Db2ErrorPattern(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /db2/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.db2.Db2Client -------------------------------------------------------------------------------- /db2/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.db2.Db2TableClient -------------------------------------------------------------------------------- /dm/src/main/java/com/dtstack/dtcenter/common/loader/dm/DmConnFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.dm; 20 | 21 | import com.dtstack.dtcenter.common.loader.rdbms.ConnFactory; 22 | import com.dtstack.dtcenter.loader.source.DataBaseType; 23 | 24 | /** 25 | * @company: www.dtstack.com 26 | * @Author ：Nanqi 27 | * @Date ：Created in 11:23 2020/4/17 28 | * @Description：达梦连接工厂 29 | */ 30 | public class DmConnFactory extends ConnFactory { 31 | public DmConnFactory() { 32 | driverName = DataBaseType.DMDB.getDriverClassName(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /dm/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.dm.DmClient -------------------------------------------------------------------------------- /docs/fs/aws_s3.md: -------------------------------------------------------------------------------- 1 | ## s3 client 2 | 3 | ### 一、插件包名称 4 | 名称：**aws_s3** 5 | 6 | ### 二、对应数据源sourceDTO及参数说明 7 | 8 | [AwsS3SourceDTO](/core/src/main/java/com/dtstack/dtcenter/loader/dto/source/AwsS3SourceDTO.java) 9 | 10 | 参数说明： 11 | 12 | 13 | - **accessKey** 14 | - 描述：aws s3 文件访问密钥 15 | - 必选：是 16 | - 默认值：无 17 | 18 | 19 | 20 | - **secretKey** 21 | - 描述：aws s3 密钥 22 | - 必选：是 23 | - 默认值：无 24 | 25 | 26 | 27 | - **region** 28 | - 描述：桶所在区 29 | - 必选：否 30 | - 默认值：cn-north-1 31 | 32 | #### 三、支持的方发及使用demo 33 | 34 | ##### IClient客户端使用 35 | 36 | 构造sourceDTO 37 | 38 | ```$java 39 | AwsS3SourceDTO sourceDTO = AwsS3SourceDTO.builder() 40 | .accessKey("******") 41 | .secretKey("******") 42 | .region("cn-north-1") 43 | .build(); 44 | ``` 45 | 46 | ###### 1. 校验数据源连通性 47 | 入参类型： 48 | - AwsS3SourceDTO：数据源连接信息 49 | 50 | 出参类型： 51 | - Boolean：是否连通 52 | 53 | 使用： 54 | ```$java 55 | IClient client = ClientCache.getClient(DataSourceType.AWS_S3.getVal()); 56 | Boolean isConnected = client.testCon(sourceDTO); 57 | ``` 58 | 59 | ###### 2. 获取所有的桶列表 60 | 入参类型： 61 | - AwsS3SourceDTO：数据源连接信息 62 | - SqlQueryDTO ：查询信息 63 | 64 | 出参类型： 65 | - List：该region下的bucket(桶)列表 66 | 67 | 使用： 68 | ```$java 69 | IClient client = ClientCache.getClient(DataSourceType.AWS_S3.getVal()); 70 | List buckets = CLIENT.getTableList(SOURCE_DTO, SqlQueryDTO.builder().build()); 71 | // buckets即为桶集合 72 | ``` 73 | -------------------------------------------------------------------------------- /docs/fs/s3.md: -------------------------------------------------------------------------------- 1 | ## s3 client 2 | 3 | ### 一、插件包名称 4 | 名称：**s3** 5 | 6 | ### 二、对应数据源sourceDTO及参数说明 7 | 8 | [S3SourceDTO](/core/src/main/java/com/dtstack/dtcenter/loader/dto/source/S3SourceDTO.java) 9 | 10 | 参数说明： 11 | 12 | 13 | - **username** 14 | - 描述：数据源的用户名 15 | - 必选：否 16 | - 默认值：无 17 | 18 | 19 | 20 | - **password** 21 | - 描述：数据源指定用户名的密码 22 | - 必选：否 23 | - 默认值：无 24 | 25 | 26 | 27 | - **hostname** 28 | - 描述：域名信息 29 | - 必选：否 30 | - 默认值：无 31 | 32 | #### 三、支持的方发及使用demo 33 | 34 | ##### IClient客户端使用 35 | 36 | 构造sourceDTO 37 | 38 | ```$java 39 | S3SourceDTO source = S3SourceDTO.builder() 40 | .username("xxx") 41 | .password("xxxx") 42 | .hostname("xxxx") 43 | .build(); 44 | ``` 45 | 46 | ###### 1. 校验数据源连通性 47 | 入参类型： 48 | - S3SourceDTO：数据源连接信息 49 | 50 | 出参类型： 51 | - Boolean：是否连通 52 | 53 | 使用： 54 | ```$java 55 | IClient client = ClientCache.getClient(DataSourceType.S3.getVal()); 56 | Boolean isConnected = client.testCon(sourceDTO); 57 | ``` 58 | -------------------------------------------------------------------------------- /docs/mq/emq.md: -------------------------------------------------------------------------------- 1 | ## emq client 2 | 3 | ### 一、插件包名称 4 | 名称：**emq** 5 | 6 | ### 二、对应数据源sourceDTO及参数说明 7 | 8 | [EMQSourceDTO](/core/src/main/java/com/dtstack/dtcenter/loader/dto/source/EMQSourceDTO.java) 9 | 10 | 参数说明： 11 | 12 | 13 | - **username** 14 | - 描述：数据源的用户名 15 | - 必选：否 16 | - 默认值：无 17 | 18 | 19 | 20 | - **password** 21 | - 描述：数据源指定用户名的密码 22 | - 必选：否 23 | - 默认值：无 24 | 25 | 26 | 27 | - **url** 28 | - 描述：连接地址 29 | - 必选：否 30 | - 默认值：无 31 | 32 | 33 | #### 三、支持的方发及使用demo 34 | 35 | ##### IClient客户端使用 36 | 37 | 构造sourceDTO 38 | 39 | ```$java 40 | EMQSourceDTO source = EMQSourceDTO.builder() 41 | .username("xxx") 42 | .password("xxx") 43 | .url("tcp://xxx:1883") 44 | .build(); 45 | ``` 46 | 47 | ###### 1. 校验数据源连通性 48 | 入参类型： 49 | - EMQSourceDTO：数据源连接信息 50 | 51 | 出参类型： 52 | - Boolean：数据源连接 53 | 54 | 使用： 55 | ```$java 56 | IClient client = ClientCache.getClient(DataSourceType.EMQ.getVal()); 57 | Boolean isConnected = client.testCon(sourceDTO); 58 | ``` 59 | -------------------------------------------------------------------------------- /docs/other/socket.md: -------------------------------------------------------------------------------- 1 | ## socket client 2 | 3 | ### 一、插件包名称 4 | 名称：**socket** 5 | 6 | ### 二、对应数据源sourceDTO及参数说明 7 | 8 | [SocketSourceDTO](/core/src/main/java/com/dtstack/dtcenter/loader/dto/source/SocketSourceDTO.java) 9 | 10 | 参数说明： 11 | 12 | - **hostPort** 13 | - 描述：host 和 端口号 14 | - 必选：是 15 | - 默认值：无 16 | 17 | 18 | #### 三、支持的方发及使用demo 19 | 20 | ##### IClient客户端使用 21 | 22 | 构造sourceDTO 23 | 24 | ```$java 25 | SocketSourceDTO source = SocketSourceDTO.builder() 26 | .hostPort("xxxx:xxxx") 27 | .build(); 28 | ``` 29 | 30 | ###### 1. 校验数据源连通性 31 | 入参类型： 32 | - SocketSourceDTO：数据源连接信息 33 | 34 | 出参类型： 35 | - Boolean：是否连通 36 | 37 | 使用： 38 | ```$java 39 | IClient client = ClientCache.getClient(DataSourceType.SOCKET.getVal()); 40 | Boolean isConnected = client.testCon(sourceDTO); 41 | ``` 42 | -------------------------------------------------------------------------------- /docs/other/websocket.md: -------------------------------------------------------------------------------- 1 | ## websocket client 2 | 3 | ### 一、插件包名称 4 | 名称：**websocket** 5 | 6 | ### 二、对应数据源sourceDTO及参数说明 7 | 8 | [WebSocketSourceDTO](/core/src/main/java/com/dtstack/dtcenter/loader/dto/source/WebSocketSourceDTO.java) 9 | 10 | 参数说明： 11 | 12 | - **url** 13 | - 描述：websocket url地址 14 | - 必选：是 15 | - 默认值：无 16 | 17 | 18 | 19 | - **authParams** 20 | - 描述：鉴权参数 21 | - 必选：是 22 | - 默认值：无 23 | 24 | #### 三、支持的方发及使用demo 25 | 26 | ##### IClient客户端使用 27 | 28 | 构造sourceDTO 29 | 30 | ```$java 31 | HashMap authMap = new HashMap<>(); 32 | map.put("xxx", "xxx"); 33 | WebSocketSourceDTO sourceDTO = WebSocketSourceDTO.builder().url("ws://xxxx:xxxx/xxxx").authParams(authMap).build(); 34 | ``` 35 | 36 | ###### 1. 校验数据源连通性 37 | 入参类型： 38 | - WebSocketSourceDTO：数据源连接信息 39 | 40 | 出参类型： 41 | - Boolean：是否连通 42 | 43 | 使用： 44 | ```$java 45 | IClient client = ClientCache.getClient(DataSourceType.WEB_SOCKET.getVal()); 46 | Boolean isConnected = client.testCon(sourceDTO); 47 | ``` 48 | -------------------------------------------------------------------------------- /doris/src/main/java/com/dtstack/dtcenter/common/loader/doris/DorisConnFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.doris; 20 | 21 | import com.dtstack.dtcenter.common.loader.mysql5.MysqlConnFactory; 22 | import com.dtstack.dtcenter.loader.source.DataBaseType; 23 | 24 | /** 25 | * @author ：qianyi 26 | * date：Created in 下午1:46 2021/07/09 27 | * company: www.dtstack.com 28 | */ 29 | public class DorisConnFactory extends MysqlConnFactory { 30 | 31 | public DorisConnFactory() { 32 | driverName = DataBaseType.Doris.getDriverClassName(); 33 | this.errorPattern = new DorisErrorPattern(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /doris/src/main/java/com/dtstack/dtcenter/common/loader/doris/DorisErrorPattern.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.doris; 20 | 21 | import com.dtstack.dtcenter.common.loader.mysql5.MysqlErrorPattern; 22 | 23 | /** 24 | * @author ：qianyi 25 | * date：Created in 下午1:46 2021/07/09 26 | * company: www.dtstack.com 27 | */ 28 | public class DorisErrorPattern extends MysqlErrorPattern { 29 | 30 | } 31 | -------------------------------------------------------------------------------- /doris/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.doris.DorisClient -------------------------------------------------------------------------------- /doris/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.doris.DorisTableClient -------------------------------------------------------------------------------- /dorisrestful/src/main/java/com/dtstack/dtcenter/common/loader/dorisrestful/request/DorisRestfulClientFactory.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.dorisrestful.request; 2 | 3 | public class DorisRestfulClientFactory { 4 | public static DorisRestfulClient getRestfulClient() { 5 | return new DorisRestfulClient(); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /dorisrestful/src/main/java/com/dtstack/dtcenter/common/loader/dorisrestful/request/HttpAPI.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.dorisrestful.request; 2 | 3 | public interface HttpAPI { 4 | 5 | /** 6 | * 获取所有的库 cluster 7 | */ 8 | String ALL_DATABASE = "/api/meta/namespaces/%s/databases"; 9 | 10 | /** 11 | * 获取库下的表 cluster schema tableName 12 | */ 13 | String ALL_TABLES = "/api/meta/namespaces/%s/databases/%s:%s/tables"; 14 | 15 | /** 16 | * 获取元数据 cluster cluster schema 17 | */ 18 | String COLUMN_METADATA = "/api/meta/namespaces/%s/databases/%s/tables/%s/schema"; 19 | 20 | /** 21 | * 数据预览 cluster schema 22 | */ 23 | String QUERY_DATA = "/api/query/%s/%s"; 24 | } -------------------------------------------------------------------------------- /dorisrestful/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.dorisrestful.DorisClient -------------------------------------------------------------------------------- /emq/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.emq.EmqClient -------------------------------------------------------------------------------- /es/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.es.EsClient -------------------------------------------------------------------------------- /es5/src/main/java/com/dtstack/dtcenter/common/loader/es5/consistent/ESEndPoint.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.es5.consistent; 2 | 3 | /** 4 | * es endPoint 5 | * 6 | * @author ：wangchuan 7 | * date：Created in 下午10:18 2021/12/8 8 | * company: www.dtstack.com 9 | */ 10 | public interface ESEndPoint { 11 | 12 | /** 13 | * 健康检查 14 | */ 15 | String ENDPOINT_HEALTH_CHECK = "/_cluster/health"; 16 | 17 | /** 18 | * 索引下数据查看 19 | */ 20 | String ENDPOINT_SEARCH_FORMAT = "/%s/_search"; 21 | 22 | /** 23 | * 指定索引、type数据查看 24 | */ 25 | String ENDPOINT_SEARCH_TYPE_FORMAT = "/%s/%s/_search"; 26 | 27 | /** 28 | * 获取所有的索引 29 | */ 30 | String ENDPOINT_INDEX_GET = "/_cat/indices?format=json"; 31 | 32 | /** 33 | * 获取指定索引的 mapping 34 | */ 35 | String ENDPOINT_MAPPING_FORMAT = "/%s/_mapping"; 36 | 37 | /** 38 | * 获取指定索引、type的 mapping 39 | */ 40 | String ENDPOINT_MAPPING_TYPE_FORMAT = "/%s/%s/_mapping"; 41 | 42 | } 43 | -------------------------------------------------------------------------------- /es5/src/main/java/com/dtstack/dtcenter/common/loader/es5/consistent/RequestType.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.es5.consistent; 2 | 3 | /** 4 | * 请求类型 5 | * 6 | * @author ：wangchuan 7 | * date：Created in 下午10:17 2021/12/8 8 | * company: www.dtstack.com 9 | */ 10 | public interface RequestType { 11 | 12 | String GET = "GET"; 13 | 14 | String PUT = "PUT"; 15 | } 16 | -------------------------------------------------------------------------------- /es5/src/main/java/com/dtstack/dtcenter/common/loader/es5/pool/ElasticSearchPool.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.es5.pool; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.Pool; 4 | import org.elasticsearch.client.RestClient; 5 | import org.elasticsearch.client.transport.TransportClient; 6 | 7 | /** 8 | * es5 连接池 9 | * 10 | * @author ：wangchuan 11 | * date：Created in 下午3:04 2021/12/8 12 | * company: www.dtstack.com 13 | */ 14 | public class ElasticSearchPool extends Pool { 15 | 16 | private ElasticSearchPoolConfig config; 17 | 18 | public ElasticSearchPool(ElasticSearchPoolConfig config){ 19 | super(config, new ElasticSearchPoolFactory(config)); 20 | this.config = config; 21 | } 22 | 23 | public ElasticSearchPoolConfig getConfig() { 24 | return config; 25 | } 26 | 27 | public void setConfig(ElasticSearchPoolConfig config) { 28 | this.config = config; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /es5/src/main/java/com/dtstack/dtcenter/common/loader/es5/pool/ElasticSearchPoolConfig.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.es5.pool; 2 | 3 | import com.google.common.collect.Sets; 4 | import lombok.Data; 5 | import lombok.EqualsAndHashCode; 6 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 7 | 8 | import java.util.Set; 9 | 10 | /** 11 | * 连接吃配置类 12 | * 13 | * @author ：wangchuan 14 | * date：Created in 下午3:04 2021/12/8 15 | * company: www.dtstack.com 16 | */ 17 | @Data 18 | @EqualsAndHashCode(callSuper = true) 19 | public class ElasticSearchPoolConfig extends GenericObjectPoolConfig { 20 | 21 | private String clusterName; 22 | 23 | Set nodes = Sets.newHashSet(); 24 | 25 | private String username; 26 | 27 | private String password; 28 | 29 | } 30 | -------------------------------------------------------------------------------- /es5/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.es5.EsClient -------------------------------------------------------------------------------- /es7/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.es7.EsClient -------------------------------------------------------------------------------- /ftp/src/main/java/com/dtstack/dtcenter/common/loader/ftp/FTPConnectMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.ftp; 20 | 21 | /** 22 | * FTP 连接模式 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 下午4:18 2021/6/21 26 | * company: www.dtstack.com 27 | */ 28 | public enum FTPConnectMode { 29 | 30 | /** 31 | * 主动模式 32 | */ 33 | PORT, 34 | 35 | /** 36 | * 被动模式 37 | */ 38 | PASV 39 | } 40 | -------------------------------------------------------------------------------- /ftp/src/main/java/com/dtstack/dtcenter/common/loader/ftp/ProtocolEnum.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.ftp; 20 | 21 | import org.apache.commons.net.ftp.FTP; 22 | 23 | /** 24 | * ftp/sftp 协议枚举类 25 | * 26 | * @author ：wangchuan 27 | * date：Created in 下午4:43 2021/6/21 28 | * company: www.dtstack.com 29 | */ 30 | public enum ProtocolEnum { 31 | 32 | /** 33 | * 文件传输协议 34 | */ 35 | FTP, 36 | 37 | /** 38 | * 安全文件传输协议 39 | */ 40 | SFTP 41 | } 42 | -------------------------------------------------------------------------------- /ftp/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.ftp.FtpClient -------------------------------------------------------------------------------- /gbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.gbase.GbaseClient -------------------------------------------------------------------------------- /gbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.gbase.GbaseTableClient -------------------------------------------------------------------------------- /greenplum6/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.greenplum.GreenplumClient -------------------------------------------------------------------------------- /greenplum6/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.greenplum.GreenplumTableClient -------------------------------------------------------------------------------- /hbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.HbaseClient -------------------------------------------------------------------------------- /hbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IHbase: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.HbaseClientSpecial -------------------------------------------------------------------------------- /hbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.KerberosClient -------------------------------------------------------------------------------- /hbase2/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.HbaseClient -------------------------------------------------------------------------------- /hbase2/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.KerberosClient -------------------------------------------------------------------------------- /hbase_gateway/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.HbaseClient -------------------------------------------------------------------------------- /hbase_gateway/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.KerberosClient -------------------------------------------------------------------------------- /hdfs/src/main/java/com/dtstack/dtcenter/common/loader/hdfs/OrcColumnTypeConverter.java: -------------------------------------------------------------------------------- 1 | 2 | 3 | package com.dtstack.dtcenter.common.loader.hdfs; 4 | 5 | import java.util.Locale; 6 | 7 | /** 8 | * Orc 字段类型转换器 9 | * 10 | * @author ：wangchuan 11 | * date：Created in 下午8:39 2021/11/22 12 | * company: www.dtstack.com 13 | */ 14 | public class OrcColumnTypeConverter { 15 | 16 | /** 17 | * @param type 原始数据类型 18 | * @return 转化后的数据类型 19 | */ 20 | public static String apply(String type) { 21 | switch (type.toUpperCase(Locale.ENGLISH)) { 22 | case "INT": 23 | case "INTEGER": 24 | case "SMALLINT": 25 | case "TINYINT": 26 | return "int"; 27 | case "BINARY": 28 | return "binary"; 29 | case "BIGINT": 30 | return "bigint"; 31 | case "BOOLEAN": 32 | return "boolean"; 33 | case "FLOAT": 34 | return "float"; 35 | case "DOUBLE": 36 | return "double"; 37 | case "DATE": 38 | return "date"; 39 | case "TIMESTAMP": 40 | return "timestamp"; 41 | default: 42 | if (type.contains("DECIMAL")) { 43 | return type.toLowerCase(); 44 | } 45 | return "string"; 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /hdfs/src/main/java/com/dtstack/dtcenter/common/loader/hdfs/downloader/tableDownload/GroupTypeIgnoreCase.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.hdfs.downloader.tableDownload; 2 | 3 | import org.apache.parquet.io.InvalidRecordException; 4 | import org.apache.parquet.schema.GroupType; 5 | import org.apache.parquet.schema.Type; 6 | 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | /** 12 | * @company: www.dtstack.com 13 | * @Author ：Nanqi 14 | * @Date ：Created in 20:33 2020/9/1 15 | * @Description：Parquet 组信息 16 | */ 17 | public class GroupTypeIgnoreCase { 18 | private List fields; 19 | private Map indexByName; 20 | 21 | public GroupTypeIgnoreCase(GroupType groupType) { 22 | this.fields = groupType.getFields(); 23 | this.indexByName = new HashMap(); 24 | 25 | for(int i = 0; i < this.fields.size(); ++i) { 26 | this.indexByName.put((this.fields.get(i)).getName().toLowerCase(), i); 27 | } 28 | 29 | } 30 | 31 | public boolean containsField(String name) { 32 | return this.indexByName.containsKey(name.toLowerCase()); 33 | } 34 | 35 | public int getFieldIndex(String name) { 36 | name = name.toLowerCase(); 37 | if (!this.indexByName.containsKey(name)) { 38 | throw new InvalidRecordException(name + " not found in " + this); 39 | } else { 40 | return (Integer)this.indexByName.get(name); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /hdfs/src/main/java/com/dtstack/dtcenter/common/loader/hdfs/fileMerge/meta/OrcMetaData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.hdfs.fileMerge.meta; 20 | 21 | import org.apache.orc.TypeDescription; 22 | 23 | /** 24 | * orc文件的元信息 25 | */ 26 | public class OrcMetaData extends FileMetaData { 27 | //新的orc文件的schema 28 | private TypeDescription schema; 29 | 30 | public TypeDescription getSchema() { 31 | return schema; 32 | } 33 | 34 | public void setSchema(TypeDescription schema) { 35 | this.schema = schema; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /hdfs/src/main/java/com/dtstack/dtcenter/common/loader/hdfs/fileMerge/meta/ParquetMetaData.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.hdfs.fileMerge.meta; 20 | 21 | import org.apache.parquet.schema.MessageType; 22 | 23 | /** 24 | * parquet文件元信息 25 | */ 26 | public class ParquetMetaData extends FileMetaData { 27 | //新的parquet文件的schema 28 | private MessageType schema; 29 | 30 | public MessageType getSchema() { 31 | return schema; 32 | } 33 | 34 | public void setSchema(MessageType schema) { 35 | this.schema = schema; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /hdfs/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hdfs.client.HdfsClient -------------------------------------------------------------------------------- /hdfs/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IHdfsFile: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hdfs.client.HdfsFileClient -------------------------------------------------------------------------------- /hdfs/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hdfs.client.KerberosClient -------------------------------------------------------------------------------- /hive/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hive2.client.HiveClient -------------------------------------------------------------------------------- /hive/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /hive/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hive2.client.HiveTableClient -------------------------------------------------------------------------------- /hive1/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hive1.client.HiveClient -------------------------------------------------------------------------------- /hive1/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /hive1/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hive1.client.Hive1TableClient -------------------------------------------------------------------------------- /hive3/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hive3.client.Hive3Client -------------------------------------------------------------------------------- /hive3/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /hive3/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hive3.client.Hive3TableClient -------------------------------------------------------------------------------- /hive3_cdp/src/main/java/com/dtstack/dtcenter/common/loader/hive3/GroupTypeIgnoreCase.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.hive3; 2 | 3 | import org.apache.parquet.io.InvalidRecordException; 4 | import org.apache.parquet.schema.GroupType; 5 | import org.apache.parquet.schema.Type; 6 | 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | /** 12 | * @company: www.dtstack.com 13 | * @Author ：qianyi 14 | * @Date ：Created in 14:03 2021/05/13 15 | * @Description：Parquet 组信息 16 | */ 17 | public class GroupTypeIgnoreCase { 18 | private List fields; 19 | private Map indexByName; 20 | 21 | public GroupTypeIgnoreCase(GroupType groupType) { 22 | this.fields = groupType.getFields(); 23 | this.indexByName = new HashMap(); 24 | 25 | for(int i = 0; i < this.fields.size(); ++i) { 26 | this.indexByName.put((this.fields.get(i)).getName().toLowerCase(), i); 27 | } 28 | 29 | } 30 | 31 | public boolean containsField(String name) { 32 | return this.indexByName.containsKey(name.toLowerCase()); 33 | } 34 | 35 | public int getFieldIndex(String name) { 36 | name = name.toLowerCase(); 37 | if (!this.indexByName.containsKey(name)) { 38 | throw new InvalidRecordException(name + " not found in " + this); 39 | } else { 40 | return (Integer)this.indexByName.get(name); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /hive3_cdp/src/main/java/com/dtstack/dtcenter/common/loader/hive3/HiveErrorPattern.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.hive3; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 4 | import com.dtstack.dtcenter.common.loader.common.exception.ConnErrorCode; 5 | 6 | import java.util.regex.Pattern; 7 | 8 | /** 9 | * @company: www.dtstack.com 10 | * @Author ：qianyi 11 | * @Date ：Created in 14:03 2021/05/13 12 | */ 13 | public class HiveErrorPattern extends AbsErrorPattern { 14 | 15 | private static final Pattern DB_NOT_EXISTS = Pattern.compile("(?i)Database\\s+'(?(.*))'\\s+not\\s+found"); 16 | 17 | private static final Pattern DB_PERMISSION_ERROR = Pattern.compile("(?i)Permission\\s*denied"); 18 | 19 | static { 20 | PATTERN_MAP.put(ConnErrorCode.DB_NOT_EXISTS.getCode(), DB_NOT_EXISTS); 21 | PATTERN_MAP.put(ConnErrorCode.DB_PERMISSION_ERROR.getCode(), DB_PERMISSION_ERROR); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /hive3_cdp/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hive3.client.Hive3Client -------------------------------------------------------------------------------- /hive3_cdp/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /hive3_cdp/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hive3.client.Hive3TableClient -------------------------------------------------------------------------------- /huawei_hbase/src/main/java/com/dtstack/dtcenter/common/loader/hbase/HbaseErrorPattern.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.hbase; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 4 | import com.dtstack.dtcenter.common.loader.common.exception.ConnErrorCode; 5 | 6 | import java.util.regex.Pattern; 7 | 8 | /** 9 | * 10 | * @author ：wangchuan 11 | * date：Created in 下午1:46 2020/11/6 12 | * company: www.dtstack.com 13 | */ 14 | public class HbaseErrorPattern extends AbsErrorPattern { 15 | 16 | private static final Pattern CANNOT_ACQUIRE_CONNECT = Pattern.compile("(?i)Connection\\s*refused"); 17 | private static final Pattern ZK_NODE_NOT_EXISTS = Pattern.compile("(?i)The\\s*node.*is\\s*not\\s*in\\s*ZooKeeper"); 18 | private static final Pattern ZK_IS_NOT_CONNECT = Pattern.compile("(?i)Can't\\s*get\\s*connection\\s*to\\s*ZooKeeper"); 19 | static { 20 | PATTERN_MAP.put(ConnErrorCode.CANNOT_ACQUIRE_CONNECT.getCode(), CANNOT_ACQUIRE_CONNECT); 21 | PATTERN_MAP.put(ConnErrorCode.ZK_NODE_NOT_EXISTS.getCode(), ZK_NODE_NOT_EXISTS); 22 | PATTERN_MAP.put(ConnErrorCode.ZK_IS_NOT_CONNECT.getCode(), ZK_IS_NOT_CONNECT); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /huawei_hbase/src/main/java/com/dtstack/dtcenter/common/loader/hbase/KerberosClient.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.hbase; 2 | 3 | import com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient; 4 | import com.dtstack.dtcenter.loader.kerberos.HadoopConfTool; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | import java.util.Map; 8 | 9 | /** 10 | * @company: www.dtstack.com 11 | * @Author ：Nanqi 12 | * @Date ：Created in 10:28 2020/9/9 13 | * @Description：Hbase Kerberos 服务 14 | */ 15 | @Slf4j 16 | public class KerberosClient extends AbsKerberosClient { 17 | @Override 18 | public Boolean prepareKerberosForConnect(Map conf, String localKerberosPath) { 19 | super.prepareKerberosForConnect(conf, localKerberosPath); 20 | // 设置principal账号 21 | if (!conf.containsKey(HadoopConfTool.PRINCIPAL)) { 22 | String principal = getPrincipals(conf).get(0); 23 | log.info("setting principal 为 {}", principal); 24 | conf.put(HadoopConfTool.PRINCIPAL, principal); 25 | } 26 | return true; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /huawei_hbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.HbaseClient -------------------------------------------------------------------------------- /huawei_hbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IHbase: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.HbaseClientSpecial -------------------------------------------------------------------------------- /huawei_hbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hbase.KerberosClient -------------------------------------------------------------------------------- /huawei_kafka/src/main/java/com/dtstack/dtcenter/common/loader/kafka/KafkaClient.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.kafka; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.nosql.AbsNoSqlClient; 4 | import com.dtstack.dtcenter.common.loader.kafka.util.KafkaUtil; 5 | import com.dtstack.dtcenter.loader.dto.source.ISourceDTO; 6 | import com.dtstack.dtcenter.loader.dto.source.KafkaSourceDTO; 7 | 8 | /** 9 | * @company: www.dtstack.com 10 | * @Author ：wangchuan 11 | * @Date ：Created in 下午4:35 2020/6/2 12 | * @Description：Kafka 客户端 支持 Kafka 0.9、0.10、0.11、1.x版本 13 | */ 14 | public class KafkaClient extends AbsNoSqlClient { 15 | @Override 16 | public Boolean testCon(ISourceDTO sourceDTO) { 17 | KafkaSourceDTO kafkaSourceDTO = (KafkaSourceDTO) sourceDTO; 18 | return KafkaUtil.checkConnection(kafkaSourceDTO); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /huawei_kafka/src/main/java/com/dtstack/dtcenter/common/loader/kafka/KafkaConsistent.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.kafka; 2 | 3 | /** 4 | * @company: www.dtstack.com 5 | * @Author ：Nanqi 6 | * @Date ：Created in 22:56 2020/2/26 7 | * @Description：TODO 8 | */ 9 | public class KafkaConsistent { 10 | /** 11 | * Kafka 会话超时时间 12 | */ 13 | public static final int SESSION_TIME_OUT = 30000; 14 | 15 | /** 16 | * Kafka 连接超时时间 17 | */ 18 | public static final int CONNECTION_TIME_OUT = 5000; 19 | 20 | /** 21 | * Kafka 默认创建的 TOPIC 名称 22 | */ 23 | public static final String KAFKA_DEFAULT_CREATE_TOPIC = "__consumer_offsets"; 24 | 25 | /** 26 | * Kafka 默认组名称 27 | */ 28 | public static final String KAFKA_GROUP = "STREAM_APP_KAFKA"; 29 | 30 | /** 31 | * Kafka JAAS 内容 32 | */ 33 | public static final String KAFKA_JAAS_CONTENT = "KafkaClient {\n" + 34 | " com.sun.security.auth.module.Krb5LoginModule required\n" + 35 | " useKeyTab=true\n" + 36 | " storeKey=true\n" + 37 | " keyTab=\"%s\"\n" + 38 | " principal=\"%s\";\n" + 39 | "};"; 40 | 41 | /** 42 | * kafka SASL/PLAIN 认证 43 | */ 44 | public static final String KAFKA_SASL_PLAIN_CONTENT = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";"; 45 | } 46 | -------------------------------------------------------------------------------- /huawei_kafka/src/main/java/com/dtstack/dtcenter/common/loader/kafka/KafkaErrorPattern.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.kafka; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 4 | 5 | /** 6 | * 7 | * @author ：wangchuan 8 | * date：Created in 下午1:46 2020/11/6 9 | * company: www.dtstack.com 10 | */ 11 | public class KafkaErrorPattern extends AbsErrorPattern { 12 | 13 | } 14 | -------------------------------------------------------------------------------- /huawei_kafka/src/main/java/com/dtstack/dtcenter/common/loader/kafka/enums/EConsumeType.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.kafka.enums; 2 | 3 | /** 4 | * kafka 消费方式 5 | * 6 | * @author ：wangchuan 7 | * date：Created in 上午10:21 2021/3/16 8 | * company: www.dtstack.com 9 | */ 10 | public enum EConsumeType { 11 | 12 | /** 13 | * 从当前topic最早位置开始消费 14 | */ 15 | EARLIEST, 16 | 17 | /** 18 | * 从当前位置开始消费 19 | */ 20 | LATEST, 21 | 22 | /** 23 | * 从指定时间开始消费 24 | */ 25 | TIMESTAMP 26 | } 27 | -------------------------------------------------------------------------------- /huawei_kafka/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kafka.KafkaClient -------------------------------------------------------------------------------- /huawei_kafka/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKafka: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kafka.Kafka -------------------------------------------------------------------------------- /huawei_kafka/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kafka.KafkaKerberos -------------------------------------------------------------------------------- /impala/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.impala.ImpalaClient -------------------------------------------------------------------------------- /impala/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /impala/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.impala.ImpalaTableClient -------------------------------------------------------------------------------- /inceptor/src/main/java/com/dtstack/dtcenter/common/loader/inceptor/GroupTypeIgnoreCase.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.inceptor; 2 | 3 | import org.apache.parquet.io.InvalidRecordException; 4 | import org.apache.parquet.schema.GroupType; 5 | import org.apache.parquet.schema.Type; 6 | 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | 12 | public class GroupTypeIgnoreCase { 13 | private List fields; 14 | private Map indexByName; 15 | 16 | public GroupTypeIgnoreCase(GroupType groupType) { 17 | this.fields = groupType.getFields(); 18 | this.indexByName = new HashMap(); 19 | 20 | for(int i = 0; i < this.fields.size(); ++i) { 21 | this.indexByName.put((this.fields.get(i)).getName().toLowerCase(), i); 22 | } 23 | 24 | } 25 | 26 | public boolean containsField(String name) { 27 | return this.indexByName.containsKey(name.toLowerCase()); 28 | } 29 | 30 | public int getFieldIndex(String name) { 31 | name = name.toLowerCase(); 32 | if (!this.indexByName.containsKey(name)) { 33 | throw new InvalidRecordException(name + " not found in " + this); 34 | } else { 35 | return (Integer)this.indexByName.get(name); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /inceptor/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.inceptor.client.InceptorClient -------------------------------------------------------------------------------- /inceptor/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /inceptor/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.inceptor.client.InceptorTableClient -------------------------------------------------------------------------------- /influxdb/src/main/java/com/dtstack/dtcenter/common/loader/influxdb/InfluxDBErrorPattern.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.influxdb; 20 | 21 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 22 | 23 | /** 24 | * influxDB 报错匹配类 25 | * 26 | * @author ：wangchuan 27 | * date：Created in 上午10:33 2021/6/7 28 | * company: www.dtstack.com 29 | */ 30 | public class InfluxDBErrorPattern extends AbsErrorPattern { 31 | 32 | } 33 | -------------------------------------------------------------------------------- /influxdb/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.influxdb.InfluxDBClient -------------------------------------------------------------------------------- /kafka/src/main/java/com/dtstack/dtcenter/common/loader/kafka/KafkaErrorPattern.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.kafka; 20 | 21 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 22 | 23 | /** 24 | * 25 | * @author ：wangchuan 26 | * date：Created in 下午1:46 2020/11/6 27 | * company: www.dtstack.com 28 | */ 29 | public class KafkaErrorPattern extends AbsErrorPattern { 30 | 31 | } 32 | -------------------------------------------------------------------------------- /kafka/src/main/java/com/dtstack/dtcenter/common/loader/kafka/enums/EConsumeType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.kafka.enums; 20 | 21 | /** 22 | * kafka 消费方式 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 上午10:21 2021/3/16 26 | * company: www.dtstack.com 27 | */ 28 | public enum EConsumeType { 29 | 30 | /** 31 | * 从当前topic最早位置开始消费 32 | */ 33 | EARLIEST, 34 | 35 | /** 36 | * 从当前位置开始消费 37 | */ 38 | LATEST, 39 | 40 | /** 41 | * 从指定时间开始消费 42 | */ 43 | TIMESTAMP 44 | } 45 | -------------------------------------------------------------------------------- /kafka/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kafka.KafkaClient -------------------------------------------------------------------------------- /kafka/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKafka: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kafka.Kafka -------------------------------------------------------------------------------- /kafka/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kafka.KafkaKerberos -------------------------------------------------------------------------------- /kingbase8/src/main/java/com/dtstack/dtcenter/common/loader/kingbase/KingbaseConnFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.kingbase; 20 | 21 | import com.dtstack.dtcenter.common.loader.rdbms.ConnFactory; 22 | import com.dtstack.dtcenter.loader.source.DataBaseType; 23 | 24 | /** 25 | * company: www.dtstack.com 26 | * @author ：Nanqi 27 | * Date ：Created in 17:11 2020/09/01 28 | * Description：kingbase 连接工厂 29 | */ 30 | public class KingbaseConnFactory extends ConnFactory { 31 | public KingbaseConnFactory() { 32 | driverName = DataBaseType.KINGBASE8.getDriverClassName(); 33 | errorPattern = new KingbaseErrorPattern(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /kingbase8/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kingbase.KingbaseClient -------------------------------------------------------------------------------- /kingbase8/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kingbase.KingbaseTableClient -------------------------------------------------------------------------------- /kudu/src/main/java/com/dtstack/dtcenter/common/loader/kudu/KuduErrorPattern.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.kudu; 20 | 21 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 22 | 23 | /** 24 | * 25 | * @author ：wangchuan 26 | * date：Created in 下午1:46 2020/11/6 27 | * company: www.dtstack.com 28 | */ 29 | public class KuduErrorPattern extends AbsErrorPattern { 30 | // 后续补充其他报错 31 | } 32 | -------------------------------------------------------------------------------- /kudu/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kudu.DtKuduClient -------------------------------------------------------------------------------- /kudu/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /kylin/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kylin.KylinClient -------------------------------------------------------------------------------- /kylin/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kylin.KylinTableClient -------------------------------------------------------------------------------- /kylinrestful/src/main/java/com/dtstack/dtcenter/common/loader/kylinRestful/http/HttpClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.kylinRestful.http; 20 | 21 | import com.dtstack.dtcenter.loader.dto.source.KylinRestfulSourceDTO; 22 | import lombok.extern.slf4j.Slf4j; 23 | 24 | @Slf4j 25 | public class HttpClientFactory { 26 | public static HttpClient createHttpClient(KylinRestfulSourceDTO sourceDTO) { 27 | return new HttpClient(sourceDTO); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /kylinrestful/src/main/java/com/dtstack/dtcenter/common/loader/kylinRestful/http/response/HttpStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.kylinRestful.http.response; 20 | 21 | /** 22 | * HTTP 响应类型 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 上午10:20 2021/6/24 26 | * company: www.dtstack.com 27 | */ 28 | public enum HttpStatus { 29 | 30 | ServerSuccess, 31 | 32 | ServerSuccessNoContent, 33 | 34 | ServerNotSupport, 35 | 36 | ServerError, 37 | 38 | ServerUnauthorized, 39 | 40 | UnKnow 41 | } 42 | -------------------------------------------------------------------------------- /kylinrestful/src/main/java/com/dtstack/dtcenter/common/loader/kylinRestful/request/RestfulClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.kylinRestful.request; 20 | 21 | public class RestfulClientFactory { 22 | public static RestfulClient getRestfulClient() { 23 | return new RestfulClient(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /kylinrestful/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.kylinRestful.KylinRestfulClient -------------------------------------------------------------------------------- /libra/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.libra.LibraClient -------------------------------------------------------------------------------- /libra/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.libra.LibraTableClient -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | Copyright ${license.git.copyrightYears} the original author or authors. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /maxcompute/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.odps.OdpsClient -------------------------------------------------------------------------------- /mongo/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.mongo.MongoDBClient -------------------------------------------------------------------------------- /mysql5/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.mysql5.MysqlClient -------------------------------------------------------------------------------- /mysql5/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.mysql5.MysqlTableClient -------------------------------------------------------------------------------- /mysql8/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.mysql8.MysqlClient -------------------------------------------------------------------------------- /oceanbase/src/main/java/com/dtstack/dtcenter/common/loader/oceanbase/OceanBaseErrorPattern.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.oceanbase; 20 | 21 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 22 | 23 | /** 24 | * @company: www.dtstack.com 25 | * @Author ：qianyi 26 | * @Date ：Created in 14:18 2021/4/21 27 | */ 28 | public class OceanBaseErrorPattern extends AbsErrorPattern { 29 | } 30 | -------------------------------------------------------------------------------- /oceanbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.oceanbase.OceanBaseClient -------------------------------------------------------------------------------- /oceanbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.oceanbase.OceanBaseTableClient -------------------------------------------------------------------------------- /opentsdb/src/main/java/com/dtstack/dtcenter/common/loader/opentsdb/http/HttpAPI.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.opentsdb.http; 20 | 21 | /** 22 | * OpenTSDB Http API 接口 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 上午10:33 2021/6/17 26 | * company: www.dtstack.com 27 | */ 28 | public interface HttpAPI { 29 | 30 | String PUT = "/api/put"; 31 | 32 | String QUERY = "/api/query"; 33 | 34 | String DELETE_DATA = "/api/delete_data"; 35 | 36 | String DELETE_META = "/api/delete_meta"; 37 | 38 | String SUGGEST = "/api/suggest"; 39 | 40 | String VERSION = "/api/version"; 41 | 42 | String VIP_HEALTH = "/api/vip_health"; 43 | } -------------------------------------------------------------------------------- /opentsdb/src/main/java/com/dtstack/dtcenter/common/loader/opentsdb/http/response/HttpStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.opentsdb.http.response; 20 | 21 | /** 22 | * HTTP 响应类型 23 | * 24 | * @author ：wangchuan 25 | * date：Created in 上午10:20 2021/6/24 26 | * company: www.dtstack.com 27 | */ 28 | public enum HttpStatus { 29 | 30 | ServerSuccess, 31 | 32 | ServerSuccessNoContent, 33 | 34 | ServerNotSupport, 35 | 36 | ServerError, 37 | 38 | ServerUnauthorized, 39 | 40 | UnKnow 41 | } 42 | -------------------------------------------------------------------------------- /opentsdb/src/main/java/com/dtstack/dtcenter/common/loader/opentsdb/type/Result.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.opentsdb.type; 20 | 21 | public class Result extends JSONValue { 22 | } -------------------------------------------------------------------------------- /opentsdb/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.opentsdb.OpenTSDBClient -------------------------------------------------------------------------------- /opentsdb/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITsdb: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.opentsdb.OpenTSDBClientSpecial -------------------------------------------------------------------------------- /oracle/src/main/java/com/dtstack/dtcenter/common/loader/oracle/OracleConnFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.oracle; 20 | 21 | import com.dtstack.dtcenter.common.loader.rdbms.ConnFactory; 22 | import com.dtstack.dtcenter.loader.source.DataBaseType; 23 | 24 | /** 25 | * @company: www.dtstack.com 26 | * @Author ：Nanqi 27 | * @Date ：Created in 12:01 2020/1/6 28 | * @Description：Oracle 连接工厂 29 | */ 30 | public class OracleConnFactory extends ConnFactory { 31 | public OracleConnFactory() { 32 | driverName = DataBaseType.Oracle.getDriverClassName(); 33 | errorPattern = new OracleErrorPattern(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /oracle/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.oracle.OracleClient -------------------------------------------------------------------------------- /oracle/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.oracle.OracleTableClient -------------------------------------------------------------------------------- /phoenix/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.phoenix4.PhoenixClient -------------------------------------------------------------------------------- /phoenix/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.phoenix4.PhoenixTableClient -------------------------------------------------------------------------------- /phoenix4_8/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.phoenix4.PhoenixClient -------------------------------------------------------------------------------- /phoenix4_8/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.phoenix4.PhoenixTableClient -------------------------------------------------------------------------------- /phoenix5/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.phoenix5.PhoenixClient -------------------------------------------------------------------------------- /phoenix5/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.phoenix5.KerberosClient -------------------------------------------------------------------------------- /phoenix5/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.phoenix5.PhoenixTableClient -------------------------------------------------------------------------------- /postgresql/src/main/java/com/dtstack/dtcenter/common/loader/postgresql/PostgresqlConnFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.postgresql; 20 | 21 | import com.dtstack.dtcenter.common.loader.rdbms.ConnFactory; 22 | import com.dtstack.dtcenter.loader.source.DataBaseType; 23 | 24 | /** 25 | * @company: www.dtstack.com 26 | * @Author ：Nanqi 27 | * @Date ：Created in 15:53 2020/1/7 28 | * @Description： 29 | */ 30 | public class PostgresqlConnFactory extends ConnFactory { 31 | public PostgresqlConnFactory() { 32 | this.driverName = DataBaseType.PostgreSQL.getDriverClassName(); 33 | this.errorPattern = new PostgreSqlErrorPattern(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /postgresql/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.postgresql.PostgresqlClient -------------------------------------------------------------------------------- /postgresql/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.postgresql.PostgresqlTableClient -------------------------------------------------------------------------------- /presto/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.presto.PrestoClient -------------------------------------------------------------------------------- /presto/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.presto.PrestoTableClient -------------------------------------------------------------------------------- /redis/src/main/java/com/dtstack/dtcenter/common/loader/redis/RedisClientSpecial.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.redis; 2 | 3 | import com.dtstack.dtcenter.loader.client.IRedis; 4 | import com.dtstack.dtcenter.loader.dto.RedisQueryDTO; 5 | import com.dtstack.dtcenter.loader.dto.source.ISourceDTO; 6 | 7 | import java.util.List; 8 | import java.util.Map; 9 | 10 | public class RedisClientSpecial implements IRedis { 11 | @Override 12 | public Map executeQuery(ISourceDTO source, RedisQueryDTO queryDTO) { 13 | return RedisUtils.executeQuery(source, queryDTO); 14 | } 15 | 16 | @Override 17 | public List preViewKey(ISourceDTO source, RedisQueryDTO queryDTO) { 18 | return RedisUtils.getRedisKeys(source, queryDTO); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /redis/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.redis.RedisClient -------------------------------------------------------------------------------- /redis/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IRedis: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.redis.RedisClientSpecial -------------------------------------------------------------------------------- /restful/src/main/java/com/dtstack/dtcenter/common/loader/restful/http/request/HttpAddressManager.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.restful.http.request; 2 | 3 | import com.dtstack.dtcenter.loader.dto.source.RestfulSourceDTO; 4 | 5 | public class HttpAddressManager { 6 | 7 | private final String address; 8 | 9 | public static final String HTTP_PREFIX = "http://"; 10 | 11 | private static final String HTTPS_PREFIX = "https://"; 12 | 13 | private HttpAddressManager(RestfulSourceDTO sourceDTO) { 14 | // 默认 http 协议 15 | address = (sourceDTO.getUrl().startsWith(HTTP_PREFIX) || sourceDTO.getUrl().startsWith(HTTPS_PREFIX)) ? 16 | sourceDTO.getUrl() : HTTP_PREFIX + sourceDTO.getUrl(); 17 | } 18 | 19 | public static HttpAddressManager createHttpAddressManager(RestfulSourceDTO sourceDTO) { 20 | return new HttpAddressManager(sourceDTO); 21 | } 22 | 23 | public String getAddress() { 24 | return this.address; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /restful/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.restful.RestfulClient -------------------------------------------------------------------------------- /restful/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IRestful: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.restful.RestfulSpecialClient -------------------------------------------------------------------------------- /s3/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.s3.S3Client -------------------------------------------------------------------------------- /s3/src/test/java/com/dtstack/dtcenter/common/loader/s3/S3ClientTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.s3; 20 | 21 | 22 | import org.junit.Ignore; 23 | 24 | @Ignore 25 | public class S3ClientTest { 26 | } -------------------------------------------------------------------------------- /sap_hana/src/main/java/com/dtstack/dtcenter/common/loader/saphana/SapHanaConnFactory.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.saphana; 2 | 3 | import com.dtstack.dtcenter.common.loader.rdbms.ConnFactory; 4 | import com.dtstack.dtcenter.loader.source.DataBaseType; 5 | 6 | /** 7 | * sap hana 连接工厂 8 | * 9 | * @author ：wangchuan 10 | * date：Created in 上午10:13 2021/12/30 11 | * company: www.dtstack.com 12 | */ 13 | public class SapHanaConnFactory extends ConnFactory { 14 | public SapHanaConnFactory() { 15 | driverName = DataBaseType.sapHana1.getDriverClassName(); 16 | this.errorPattern = new SapHanaErrorPattern(); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /sap_hana/src/main/java/com/dtstack/dtcenter/common/loader/saphana/SapHanaErrorPattern.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.saphana; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 4 | import com.dtstack.dtcenter.common.loader.common.exception.ConnErrorCode; 5 | 6 | import java.util.regex.Pattern; 7 | 8 | /** 9 | * sap hana 错误适配器 10 | * 11 | * @author ：wangchuan 12 | * date：Created in 上午10:13 2021/12/30 13 | * company: www.dtstack.com 14 | */ 15 | public class SapHanaErrorPattern extends AbsErrorPattern { 16 | 17 | private static final Pattern USERNAME_PASSWORD_ERROR = Pattern.compile("(?i)Access\\s*denied\\s*for\\s*user.*using\\s*password"); 18 | private static final Pattern DB_NOT_EXISTS = Pattern.compile("(?i)Access\\s*denied\\s*for\\s*user.*to\\s*database"); 19 | private static final Pattern IP_PORT_FORMAT_ERROR = Pattern.compile("(?i)not\\s*received\\s*any\\s*packets"); 20 | static { 21 | PATTERN_MAP.put(ConnErrorCode.USERNAME_PASSWORD_ERROR.getCode(), USERNAME_PASSWORD_ERROR); 22 | PATTERN_MAP.put(ConnErrorCode.DB_NOT_EXISTS.getCode(), DB_NOT_EXISTS); 23 | PATTERN_MAP.put(ConnErrorCode.IP_PORT_FORMAT_ERROR.getCode(), IP_PORT_FORMAT_ERROR); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /sap_hana/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.saphana.SapHanaClient -------------------------------------------------------------------------------- /sap_hana/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.saphana.SapHanaTableClient -------------------------------------------------------------------------------- /socket/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.socket.SocketClient -------------------------------------------------------------------------------- /solr/src/main/java/com/dtstack/dtcenter/common/loader/solr/SolrConsistent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.solr; 20 | 21 | public class SolrConsistent { 22 | /** 23 | * solr JAAS 内容 24 | */ 25 | public static String SOLR_JAAS_CONTENT = "SolrJClient {\n" + 26 | " com.sun.security.auth.module.Krb5LoginModule required\n" + 27 | " useKeyTab=true\n" + 28 | " keyTab=\"%s\"\n" + 29 | " storeKey=true\n" + 30 | " useTicketCache=true\n" + 31 | " debug=true\n" + 32 | " principal=\"%s\";\n" + 33 | "};"; 34 | } -------------------------------------------------------------------------------- /solr/src/main/java/com/dtstack/dtcenter/common/loader/solr/SolrErrorPattern.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.solr; 20 | 21 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 22 | 23 | /** 24 | * @author ：qianyi 25 | * date：Created in 下午10:51 2021/04/27 26 | * company: www.dtstack.com 27 | */ 28 | public class SolrErrorPattern extends AbsErrorPattern { 29 | } 30 | -------------------------------------------------------------------------------- /solr/src/main/java/com/dtstack/dtcenter/common/loader/solr/pool/SolrPoolConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.solr.pool; 20 | 21 | import lombok.Data; 22 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 23 | 24 | /** 25 | * @company: www.dtstack.com 26 | * @Author ：qianyi 27 | * @Date ：Created in 下午5:14 2021/5/7 28 | * @Description： 29 | */ 30 | @Data 31 | public class SolrPoolConfig extends GenericObjectPoolConfig { 32 | 33 | private String zkHosts; 34 | 35 | private String chroot; 36 | 37 | } 38 | -------------------------------------------------------------------------------- /solr/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.solr.DtSolrClient -------------------------------------------------------------------------------- /solr/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /sonar-project.properties: -------------------------------------------------------------------------------- 1 | sonar.projectKey=dt_center_common_loader 2 | sonar.projectVersion=4.2.0 3 | #sonar.tests=src/test 4 | # \u6E90\u7801\u6240\u5728\u4F4D\u7F6E 5 | sonar.sources=. 6 | # \u6392\u9664test\u6A21\u5757\u3001mysql8\u6A21\u5757\u3001hbase2\u6A21\u5757\u3001hbase_gateway\u6A21\u5757\u3001s3\u6A21\u5757\uFF0Chive\u3001hdfs\u7B49\u6E90\u7801\u6587\u4EF6 7 | sonar.exclusions=test/**,mysql8/**,hive3/**,sqlserver2017/**,hbase2/**,hbase_gateway/**,dm/**,emq/**,vertica/**,websocket/**,socket/**,s3/**,**/src/main/java/org/** 8 | # \u5B57\u8282\u7801\u6587\u4EF6\u6240\u5728\u4F4D\u7F6E 9 | sonar.java.binaries=. 10 | sonar.language=java 11 | sonar.sourceEncoding=UTF-8 12 | 13 | # \u6307\u5B9A\u4EE3\u7801\u8986\u76D6\u7387\u5DE5\u5177 14 | sonar.core.codeCoveragePlugin=jacoco 15 | sonar.xml.enabled=false 16 | sonar.tests=test 17 | sonar.test.inclusions=. 18 | # \u53BB\u9664\u4E0D\u9700\u8981\u5355\u6D4B\u8986\u76D6\u7684\u7C7B 19 | sonar.test.exclusions=test/**,mysql8/**,hive3/**,sqlserver2017/**,hbase2/**,hbase_gateway/**,dm/**,emq/**,vertica/**,websocket/**,socket/**,s3/**,**/src/main/java/org/** 20 | 21 | 22 | # sonar \u5355\u6D4B\u6267\u884C\u60C5\u51B5\u5730\u5740 23 | sonar.surefire.reportsPath=test/target/surefire-reports 24 | sonar.dynamicAnalysis=reuseReports 25 | # xml \u5730\u5740 26 | sonar.coverage.jacoco.xmlReportPaths=test/target/site/jacoco-aggregate/jacoco.xml 27 | -------------------------------------------------------------------------------- /spark/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.spark.client.SparkClient -------------------------------------------------------------------------------- /spark/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /spark/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.spark.client.SparkTableClient -------------------------------------------------------------------------------- /sqlserver/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.sqlserver.SqlServerClient -------------------------------------------------------------------------------- /sqlserver/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.sqlserver.SqlServerTableClient -------------------------------------------------------------------------------- /sqlserver2017/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.sqlserver.SqlServerClient -------------------------------------------------------------------------------- /tbds_hbase/src/main/java/com/dtstack/dtcenter/common/loader/tbds/hbase/HbaseErrorPattern.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tbds.hbase; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 4 | import com.dtstack.dtcenter.common.loader.common.exception.ConnErrorCode; 5 | 6 | import java.util.regex.Pattern; 7 | 8 | /** 9 | * 10 | * @author ：wangchuan 11 | * date：Created in 下午1:46 2020/11/6 12 | * company: www.dtstack.com 13 | */ 14 | public class HbaseErrorPattern extends AbsErrorPattern { 15 | 16 | private static final Pattern CANNOT_ACQUIRE_CONNECT = Pattern.compile("(?i)Connection\\s*refused"); 17 | private static final Pattern ZK_NODE_NOT_EXISTS = Pattern.compile("(?i)The\\s*node.*is\\s*not\\s*in\\s*ZooKeeper"); 18 | private static final Pattern ZK_IS_NOT_CONNECT = Pattern.compile("(?i)Can't\\s*get\\s*connection\\s*to\\s*ZooKeeper"); 19 | static { 20 | PATTERN_MAP.put(ConnErrorCode.CANNOT_ACQUIRE_CONNECT.getCode(), CANNOT_ACQUIRE_CONNECT); 21 | PATTERN_MAP.put(ConnErrorCode.ZK_NODE_NOT_EXISTS.getCode(), ZK_NODE_NOT_EXISTS); 22 | PATTERN_MAP.put(ConnErrorCode.ZK_IS_NOT_CONNECT.getCode(), ZK_IS_NOT_CONNECT); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /tbds_hbase/src/main/java/com/dtstack/dtcenter/common/loader/tbds/hbase/KerberosClient.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tbds.hbase; 2 | 3 | import com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient; 4 | import com.dtstack.dtcenter.loader.kerberos.HadoopConfTool; 5 | import lombok.extern.slf4j.Slf4j; 6 | 7 | import java.util.Map; 8 | 9 | /** 10 | * @company: www.dtstack.com 11 | * @Author ：Nanqi 12 | * @Date ：Created in 10:28 2020/9/9 13 | * @Description：Hbase Kerberos 服务 14 | */ 15 | @Slf4j 16 | public class KerberosClient extends AbsKerberosClient { 17 | @Override 18 | public Boolean prepareKerberosForConnect(Map conf, String localKerberosPath) { 19 | super.prepareKerberosForConnect(conf, localKerberosPath); 20 | // 设置principal账号 21 | if (!conf.containsKey(HadoopConfTool.PRINCIPAL)) { 22 | String principal = getPrincipals(conf).get(0); 23 | log.info("setting principal 为 {}", principal); 24 | conf.put(HadoopConfTool.PRINCIPAL, principal); 25 | } 26 | return true; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /tbds_hbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.tbds.hbase.HbaseClient -------------------------------------------------------------------------------- /tbds_hbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IHbase: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.tbds.hbase.HbaseClientSpecial -------------------------------------------------------------------------------- /tbds_hbase/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.tbds.hbase.KerberosClient -------------------------------------------------------------------------------- /tbds_hdfs/src/main/java/com/dtstack/dtcenter/common/loader/tdbs/hdfs/GroupTypeIgnoreCase.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tdbs.hdfs; 2 | 3 | import org.apache.parquet.io.InvalidRecordException; 4 | import org.apache.parquet.schema.GroupType; 5 | import org.apache.parquet.schema.Type; 6 | 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | /** 12 | * @company: www.dtstack.com 13 | * @Author ：Nanqi 14 | * @Date ：Created in 20:33 2020/9/1 15 | * @Description：Parquet 组信息 16 | */ 17 | public class GroupTypeIgnoreCase { 18 | private List fields; 19 | private Map indexByName; 20 | 21 | public GroupTypeIgnoreCase(GroupType groupType) { 22 | this.fields = groupType.getFields(); 23 | this.indexByName = new HashMap(); 24 | 25 | for(int i = 0; i < this.fields.size(); ++i) { 26 | this.indexByName.put((this.fields.get(i)).getName().toLowerCase(), i); 27 | } 28 | 29 | } 30 | 31 | public boolean containsField(String name) { 32 | return this.indexByName.containsKey(name.toLowerCase()); 33 | } 34 | 35 | public int getFieldIndex(String name) { 36 | name = name.toLowerCase(); 37 | if (!this.indexByName.containsKey(name)) { 38 | throw new InvalidRecordException(name + " not found in " + this); 39 | } else { 40 | return (Integer)this.indexByName.get(name); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /tbds_hdfs/src/main/java/com/dtstack/dtcenter/common/loader/tdbs/hdfs/HDFSErrorPattern.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tdbs.hdfs; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 4 | import com.dtstack.dtcenter.common.loader.common.exception.ConnErrorCode; 5 | 6 | import java.util.regex.Pattern; 7 | 8 | /** 9 | * 10 | * @author ：wangchuan 11 | * date：Created in 下午1:46 2020/11/6 12 | * company: www.dtstack.com 13 | */ 14 | public class HDFSErrorPattern extends AbsErrorPattern { 15 | 16 | private static final Pattern DB_PERMISSION_ERROR = Pattern.compile("(?i)Permission\\s*denied"); 17 | 18 | static { 19 | PATTERN_MAP.put(ConnErrorCode.DB_PERMISSION_ERROR.getCode(), DB_PERMISSION_ERROR); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /tbds_hdfs/src/main/java/com/dtstack/dtcenter/common/loader/tdbs/hdfs/HdfsConnFactory.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tdbs.hdfs; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.DtClassConsistent; 4 | import com.dtstack.dtcenter.loader.dto.source.ISourceDTO; 5 | import com.dtstack.dtcenter.loader.dto.source.TbdsHdfsSourceDTO; 6 | import com.dtstack.dtcenter.loader.exception.DtLoaderException; 7 | import org.apache.commons.lang3.StringUtils; 8 | 9 | /** 10 | * @company: www.dtstack.com 11 | * @Author ：Nanqi 12 | * @Date ：Created in 16:53 2020/2/27 13 | * @Description：HDFS 连接工厂 14 | */ 15 | public class HdfsConnFactory { 16 | 17 | public Boolean testConn(ISourceDTO iSource) { 18 | TbdsHdfsSourceDTO hdfsSourceDTO = (TbdsHdfsSourceDTO) iSource; 19 | if (StringUtils.isBlank(hdfsSourceDTO.getDefaultFS())) { 20 | throw new DtLoaderException("defaultFS incorrect format"); 21 | } 22 | 23 | return HdfsOperator.checkConnection(hdfsSourceDTO.getDefaultFS(), hdfsSourceDTO.getConfig(), hdfsSourceDTO.getKerberosConfig()); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tbds_hdfs/src/main/java/com/dtstack/dtcenter/common/loader/tdbs/hdfs/client/HdfsClient.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tdbs.hdfs.client; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.nosql.AbsNoSqlClient; 4 | import com.dtstack.dtcenter.common.loader.tdbs.hdfs.HdfsConnFactory; 5 | import com.dtstack.dtcenter.loader.dto.source.ISourceDTO; 6 | 7 | /** 8 | * @company: www.dtstack.com 9 | * @Author ：Nanqi 10 | * @Date ：Created in 16:53 2020/2/27 11 | * @Description：hdfs 客户端 12 | */ 13 | public class HdfsClient extends AbsNoSqlClient { 14 | private HdfsConnFactory hdfsConnFactory = new HdfsConnFactory(); 15 | 16 | @Override 17 | public Boolean testCon(ISourceDTO source) { 18 | return hdfsConnFactory.testConn(source); 19 | } 20 | } -------------------------------------------------------------------------------- /tbds_hdfs/src/main/java/com/dtstack/dtcenter/common/loader/tdbs/hdfs/fileMerge/meta/FileMetaData.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tdbs.hdfs.fileMerge.meta; 2 | 3 | 4 | import com.dtstack.dtcenter.common.loader.tdbs.hdfs.fileMerge.ECompressType; 5 | 6 | /** 7 | * 文件的元信息 8 | */ 9 | public class FileMetaData { 10 | 11 | //是否压缩 12 | private boolean isCompressed; 13 | //压缩的格式 14 | private ECompressType eCompressType; 15 | //新文件的写入上限 16 | private long limitSize; 17 | 18 | public boolean isCompressed() { 19 | return isCompressed; 20 | } 21 | 22 | public void setCompressed(boolean compressed) { 23 | isCompressed = compressed; 24 | } 25 | 26 | public ECompressType geteCompressType() { 27 | return eCompressType; 28 | } 29 | 30 | public void seteCompressType(ECompressType eCompressType) { 31 | this.eCompressType = eCompressType; 32 | } 33 | 34 | public long getLimitSize() { 35 | return limitSize; 36 | } 37 | 38 | public void setLimitSize(long limitSize) { 39 | this.limitSize = limitSize; 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return "FileMetaData{" + 45 | "isCompressed=" + isCompressed + 46 | ", eCompressType=" + eCompressType + 47 | ", limitSize=" + limitSize + 48 | '}'; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /tbds_hdfs/src/main/java/com/dtstack/dtcenter/common/loader/tdbs/hdfs/fileMerge/meta/OrcMetaData.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tdbs.hdfs.fileMerge.meta; 2 | 3 | import org.apache.orc.TypeDescription; 4 | 5 | /** 6 | * orc文件的元信息 7 | */ 8 | public class OrcMetaData extends FileMetaData { 9 | //新的orc文件的schema 10 | private TypeDescription schema; 11 | 12 | public TypeDescription getSchema() { 13 | return schema; 14 | } 15 | 16 | public void setSchema(TypeDescription schema) { 17 | this.schema = schema; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /tbds_hdfs/src/main/java/com/dtstack/dtcenter/common/loader/tdbs/hdfs/fileMerge/meta/ParquetMetaData.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tdbs.hdfs.fileMerge.meta; 2 | 3 | import org.apache.parquet.schema.MessageType; 4 | 5 | /** 6 | * parquet文件元信息 7 | */ 8 | public class ParquetMetaData extends FileMetaData { 9 | //新的parquet文件的schema 10 | private MessageType schema; 11 | 12 | public MessageType getSchema() { 13 | return schema; 14 | } 15 | 16 | public void setSchema(MessageType schema) { 17 | this.schema = schema; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /tbds_hdfs/src/main/java/com/dtstack/dtcenter/common/loader/tdbs/hdfs/util/UGICacheData.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tdbs.hdfs.util; 2 | 3 | import lombok.Data; 4 | import org.apache.hadoop.security.UserGroupInformation; 5 | 6 | /** 7 | * @company: www.dtstack.com 8 | * @Author ：Nanqi 9 | * @Date ：Created in 19:05 2020/10/29 10 | * @Description：UGI 缓存信息 11 | */ 12 | @Data 13 | public class UGICacheData { 14 | /** 15 | * 过期时间戳 只有头结点才存在 16 | */ 17 | private Long timeoutStamp; 18 | 19 | /** 20 | * UGI 信息 21 | */ 22 | private UserGroupInformation ugi; 23 | 24 | public UGICacheData(UserGroupInformation ugi) { 25 | this.ugi = ugi; 26 | timeoutStamp = System.currentTimeMillis() + 10 * 1000; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /tbds_hdfs/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.tdbs.hdfs.client.HdfsClient -------------------------------------------------------------------------------- /tbds_hdfs/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IHdfsFile: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.tdbs.hdfs.client.HdfsFileClient -------------------------------------------------------------------------------- /tbds_hdfs/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.tdbs.hdfs.client.KerberosClient -------------------------------------------------------------------------------- /tbds_kafka/src/main/java/com/dtstack/dtcenter/common/loader/tbds/kafka/KafkaClient.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tbds.kafka; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.nosql.AbsNoSqlClient; 4 | import com.dtstack.dtcenter.common.loader.tbds.kafka.util.KafkaUtil; 5 | import com.dtstack.dtcenter.loader.dto.source.ISourceDTO; 6 | import com.dtstack.dtcenter.loader.dto.source.TbdsKafkaSourceDTO; 7 | 8 | /** 9 | * @company: www.dtstack.com 10 | * @Author ：wangchuan 11 | * @Date ：Created in 下午4:35 2020/6/2 12 | * @Description：Kafka 客户端 支持 Kafka 0.9、0.10、0.11、1.x版本 13 | */ 14 | public class KafkaClient extends AbsNoSqlClient { 15 | @Override 16 | public Boolean testCon(ISourceDTO sourceDTO) { 17 | TbdsKafkaSourceDTO kafkaSourceDTO = (TbdsKafkaSourceDTO) sourceDTO; 18 | return KafkaUtil.checkConnection(kafkaSourceDTO); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tbds_kafka/src/main/java/com/dtstack/dtcenter/common/loader/tbds/kafka/KafkaConsistent.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tbds.kafka; 2 | 3 | /** 4 | * @company: www.dtstack.com 5 | * @Author ：Nanqi 6 | * @Date ：Created in 22:56 2020/2/26 7 | * @Description：TODO 8 | */ 9 | public class KafkaConsistent { 10 | /** 11 | * Kafka 会话超时时间 12 | */ 13 | public static final int SESSION_TIME_OUT = 30000; 14 | 15 | /** 16 | * Kafka 连接超时时间 17 | */ 18 | public static final int CONNECTION_TIME_OUT = 5000; 19 | 20 | /** 21 | * Kafka 默认创建的 TOPIC 名称 22 | */ 23 | public static final String KAFKA_DEFAULT_CREATE_TOPIC = "__consumer_offsets"; 24 | 25 | /** 26 | * Kafka 默认组名称 27 | */ 28 | public static final String KAFKA_GROUP = "STREAM_APP_KAFKA"; 29 | 30 | /** 31 | * Kafka JAAS 内容 32 | */ 33 | public static final String KAFKA_JAAS_CONTENT = "KafkaClient {\n" + 34 | " com.sun.security.auth.module.Krb5LoginModule required\n" + 35 | " useKeyTab=true\n" + 36 | " storeKey=true\n" + 37 | " keyTab=\"%s\"\n" + 38 | " principal=\"%s\";\n" + 39 | "};"; 40 | 41 | /** 42 | * kafka SASL/PLAIN 认证 43 | */ 44 | public static final String KAFKA_SASL_PLAIN_CONTENT = "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"%s\" password=\"%s\";"; 45 | } 46 | -------------------------------------------------------------------------------- /tbds_kafka/src/main/java/com/dtstack/dtcenter/common/loader/tbds/kafka/KafkaErrorPattern.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tbds.kafka; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 4 | 5 | /** 6 | * 7 | * @author ：wangchuan 8 | * date：Created in 下午1:46 2020/11/6 9 | * company: www.dtstack.com 10 | */ 11 | public class KafkaErrorPattern extends AbsErrorPattern { 12 | 13 | } 14 | -------------------------------------------------------------------------------- /tbds_kafka/src/main/java/com/dtstack/dtcenter/common/loader/tbds/kafka/enums/EConsumeType.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.tbds.kafka.enums; 2 | 3 | /** 4 | * kafka 消费方式 5 | * 6 | * @author ：wangchuan 7 | * date：Created in 上午10:21 2021/3/16 8 | * company: www.dtstack.com 9 | */ 10 | public enum EConsumeType { 11 | 12 | /** 13 | * 从当前topic最早位置开始消费 14 | */ 15 | EARLIEST, 16 | 17 | /** 18 | * 从当前位置开始消费 19 | */ 20 | LATEST, 21 | 22 | /** 23 | * 从指定时间开始消费 24 | */ 25 | TIMESTAMP 26 | } 27 | -------------------------------------------------------------------------------- /tbds_kafka/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.tbds.kafka.KafkaClient -------------------------------------------------------------------------------- /tbds_kafka/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKafka: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.tbds.kafka.Kafka -------------------------------------------------------------------------------- /tbds_kafka/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.tbds.kafka.KafkaKerberos -------------------------------------------------------------------------------- /test/src/test/java/com/dtstack/dtcenter/loader/client/BaseTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.loader.client; 20 | 21 | import org.junit.BeforeClass; 22 | 23 | /** 24 | * 设置基本参数 25 | * 26 | * @author ：wangchuan 27 | * date：Created in 下午4:54 2021/5/18 28 | * company: www.dtstack.com 29 | */ 30 | public class BaseTest { 31 | 32 | @BeforeClass 33 | public static void beforeTest() { 34 | String userDir = ClientCache.getUserDir(); 35 | String coreDir = userDir.replace("/test/", "/core/"); 36 | ClientCache.setUserDir(coreDir); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /test/src/test/java/com/dtstack/dtcenter/loader/client/sql/TbdsKafkaTest.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.loader.client.sql; 2 | 3 | import com.dtstack.dtcenter.loader.client.BaseTest; 4 | import com.dtstack.dtcenter.loader.client.ClientCache; 5 | import com.dtstack.dtcenter.loader.client.IKafka; 6 | import com.dtstack.dtcenter.loader.dto.source.TbdsKafkaSourceDTO; 7 | import com.dtstack.dtcenter.loader.exception.DtLoaderException; 8 | import com.dtstack.dtcenter.loader.source.DataSourceType; 9 | import org.junit.BeforeClass; 10 | import org.junit.Ignore; 11 | import org.junit.Test; 12 | @Ignore 13 | public class TbdsKafkaTest extends BaseTest { 14 | 15 | // 构建kafka数据源信息 16 | private static final TbdsKafkaSourceDTO source = TbdsKafkaSourceDTO.builder() 17 | .url("172.16.101.236:2181,172.16.101.17:2181,172.16.100.109:2181/kafka") 18 | .secureId("secureId") 19 | .secureKey("secureKey") 20 | .build(); 21 | 22 | @BeforeClass 23 | public static void setUp() {} 24 | 25 | @Test 26 | public void testConForKafka() { 27 | IKafka client = ClientCache.getKafka(DataSourceType.TBDS_KAFKA.getVal()); 28 | Boolean isConnected = client.testCon(source); 29 | if (Boolean.FALSE.equals(isConnected)) { 30 | throw new DtLoaderException("connection exception"); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /test/src/test/resources/bug/issue_33045/z.parquet: -------------------------------------------------------------------------------- 1 | PAR1, ,,H spark_schema%id&%idZ^&<Z)org.apache.spark.sql.parquet.row.metadataZ{"type":"struct","fields":[{"name":"id","type":"integer","nullable":false,"metadata":{}}]}Iparquet-mr version 1.8.1 (build 4aba4dae7bb0d4edbcf7923ae1339f28fd3f7fcf))PAR1 -------------------------------------------------------------------------------- /test/src/test/resources/bug/issue_33097/_SUCCESS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/bug/issue_33097/_SUCCESS -------------------------------------------------------------------------------- /test/src/test/resources/bug/issue_34667/kerberos_dir.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/bug/issue_34667/kerberos_dir.zip -------------------------------------------------------------------------------- /test/src/test/resources/bug/issue_34667/kerberos_nodir.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/bug/issue_34667/kerberos_nodir.zip -------------------------------------------------------------------------------- /test/src/test/resources/bug/issue_36310/dir_kerberos.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/bug/issue_36310/dir_kerberos.zip -------------------------------------------------------------------------------- /test/src/test/resources/cdp/hive.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/cdp/hive.keytab -------------------------------------------------------------------------------- /test/src/test/resources/cdp/krb5.conf: -------------------------------------------------------------------------------- 1 | # Configuration snippets may be placed in this directory as well 2 | includedir /etc/krb5.conf.d/ 3 | 4 | [logging] 5 | default = FILE:/var/log/krb5libs.log 6 | kdc = FILE:/var/log/krb5kdc.log 7 | admin_server = FILE:/var/log/kadmind.log 8 | 9 | [libdefaults] 10 | dns_lookup_realm = false 11 | ticket_lifetime = 24h 12 | renew_lifetime = 7d 13 | forwardable = true 14 | rdns = false 15 | pkinit_anchors = FILE:/etc/pki/tls/certs/ca-bundle.crt 16 | default_realm = DTSTACK.COM 17 | #default_ccache_name = KEYRING:persistent:%{uid} 18 | 19 | [realms] 20 | DTSTACK.COM = { 21 | kdc = ladp001.dtstack.com 22 | admin_server = ladp001.dtstack.com 23 | } 24 | 25 | [domain_realm] 26 | .dtstack.com = DTSTACK.COM 27 | dtstack.com = DTSTACK.COM 28 | -------------------------------------------------------------------------------- /test/src/test/resources/download/orc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/download/orc -------------------------------------------------------------------------------- /test/src/test/resources/download/parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/download/parquet -------------------------------------------------------------------------------- /test/src/test/resources/download/text: -------------------------------------------------------------------------------- 1 | 1,loader_test_01 -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh/hadoop.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh/hadoop.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh/hbase-master.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh/hbase-master.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh/hive-cdh01.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh/hive-cdh01.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh/hive-cdh03.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh/hive-cdh03.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh/hive-krb.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh/hive-krb.zip -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh/impalad-cdh3.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh/impalad-cdh3.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh/kafka-cdh1.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh/kafka-cdh1.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh/krb5.conf: -------------------------------------------------------------------------------- 1 | [libdefaults] 2 | default_realm = DTSTACK.COM 3 | dns_lookup_kdc = false 4 | dns_lookup_realm = false 5 | ticket_lifetime = 600 6 | renew_lifetime = 3600 7 | forwardable = true 8 | default_tgs_enctypes = rc4-hmac aes256-cts 9 | default_tkt_enctypes = rc4-hmac aes256-cts 10 | permitted_enctypes = rc4-hmac aes256-cts 11 | udp_preference_limit = 1 12 | kdc_timeout = 3000 13 | [realms] 14 | DTSTACK.COM = { 15 | kdc = eng-cdh1 16 | admin_server = eng-cdh1 17 | } 18 | [domain_realm] 19 | -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh/kudu-master.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh/kudu-master.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh2/kafka_eng2.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh2/kafka_eng2.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh2/kafka_krb.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | sasl.kerberos.service.name 4 | kafka 5 | 填写kafka配置文件中相同的参数 6 | 7 | 8 | -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh2/krb5.conf: -------------------------------------------------------------------------------- 1 | [libdefaults] 2 | default_realm = DTSTACK.COM 3 | dns_lookup_kdc = false 4 | dns_lookup_realm = false 5 | ticket_lifetime = 86400 6 | renew_lifetime = 604800 7 | forwardable = true 8 | default_tgs_enctypes = rc4-hmac aes256-cts 9 | default_tkt_enctypes = rc4-hmac aes256-cts 10 | permitted_enctypes = rc4-hmac aes256-cts 11 | udp_preference_limit = 1 12 | kdc_timeout = 3000 13 | [realms] 14 | DTSTACK.COM = { 15 | kdc = eng-cdh1 16 | admin_server = eng-cdh1 17 | } 18 | [domain_realm] 19 | -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh3/hbase.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh3/hbase.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh3/hive.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh3/hive.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh3/impala.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh3/impala.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh3/krb5.conf: -------------------------------------------------------------------------------- 1 | [libdefaults] 2 | default_realm = DTSTACK.COM 3 | dns_lookup_kdc = false 4 | dns_lookup_realm = false 5 | ticket_lifetime = 24h 6 | renew_lifetime = 10d 7 | forwardable = true 8 | udp_preference_limit = 1 9 | kdc_timeout = 3000 10 | [realms] 11 | DTSTACK.COM = { 12 | kdc = 172.16.100.208 13 | admin_server = 172.16.100.208 14 | } 15 | [domain_realm] 16 | .dtstack.com = DTSTACK.COM 17 | dtstack.com = DTSTACK.COM 18 | 19 | -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh3/kudu.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh3/kudu.keytab -------------------------------------------------------------------------------- /test/src/test/resources/eng-cdh3/phoenix.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/eng-cdh3/phoenix.keytab -------------------------------------------------------------------------------- /test/src/test/resources/hdfs-file/orcfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/hdfs-file/orcfile -------------------------------------------------------------------------------- /test/src/test/resources/hdfs-file/parquetfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/hdfs-file/parquetfile -------------------------------------------------------------------------------- /test/src/test/resources/hdfs-file/test.txt: -------------------------------------------------------------------------------- 1 | hdfs上传下载测试使用 -------------------------------------------------------------------------------- /test/src/test/resources/hdfs-file/textfile: -------------------------------------------------------------------------------- 1 | id,name 2 | 1,wangchuan 3 | 1,wangchuan 4 | 1,wangchuan 5 | 1,wangchuan 6 | 1,wangchuan 7 | 1,wangchuan 8 | 1,wangchuan -------------------------------------------------------------------------------- /test/src/test/resources/hive3/hive.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/hive3/hive.keytab -------------------------------------------------------------------------------- /test/src/test/resources/hive3/krb5.conf: -------------------------------------------------------------------------------- 1 | [libdefaults] 2 | default_realm = DTSTACK.COM 3 | dns_lookup_kdc = false 4 | dns_lookup_realm = false 5 | ticket_lifetime = 24h 6 | forwardable = true 7 | udp_preference_limit = 1 8 | kdc_timeout = 3000 9 | [realms] 10 | DTSTACK.COM = { 11 | kdc = 172.16.101.174 12 | admin_server = 172.16.101.174 13 | } 14 | [domain_realm] 15 | .dtstack.com = DTSTACK.COM 16 | dtstack.com = DTSTACK.COM 17 | 18 | -------------------------------------------------------------------------------- /test/src/test/resources/krbt/hdfs.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/krbt/hdfs.keytab -------------------------------------------------------------------------------- /test/src/test/resources/krbt/kafka.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/krbt/kafka.keytab -------------------------------------------------------------------------------- /test/src/test/resources/krbt/krb5.conf: -------------------------------------------------------------------------------- 1 | # Configuration snippets may be placed in this directory as well 2 | includedir /etc/krb5.conf.d/ 3 | 4 | [logging] 5 | default = FILE:/var/log/krb5libs.log 6 | kdc = FILE:/var/log/krb5kdc.log 7 | admin_server = FILE:/var/log/kadmind.log 8 | 9 | [libdefaults] 10 | dns_lookup_realm = false 11 | #访问凭证的有效时间，默认是24小时 12 | ticket_lifetime = 10000d 13 | #renew_lifetime = 10000d 14 | forwardable = true 15 | rdns = false 16 | pkinit_anchors = FILE:/etc/pki/tls/certs/ca-bundle.crt 17 | default_realm = DTSTACK.COM 18 | # default_ccache_name = KEYRING:persistent:%{uid} 19 | 20 | [realms] 21 | DTSTACK.COM = { 22 | kdc = krbe2:88 23 | admin_server = krbe2:749 24 | } 25 | 26 | [domain_realm] 27 | .dtstack.com = DTSTACK.COM 28 | dtstack.com = DTSTACK.COM -------------------------------------------------------------------------------- /test/src/test/resources/krbt/krb5_2.conf: -------------------------------------------------------------------------------- 1 | # Configuration snippets may be placed in this directory as well 2 | includedir /etc/krb5.conf.d/ 3 | 4 | [logging] 5 | default = FILE:/var/log/krb5libs.log 6 | kdc = FILE:/var/log/krb5kdc.log 7 | admin_server = FILE:/var/log/kadmind.log 8 | 9 | [libdefaults] 10 | dns_lookup_realm = false 11 | #访问凭证的有效时间，默认是24小时 12 | ticket_lifetime = 24h 13 | renew_lifetime = 7d 14 | forwardable = true 15 | rdns = false 16 | pkinit_anchors = FILE:/etc/pki/tls/certs/ca-bundle.crt 17 | default_realm = DTSTACK.COM 18 | # default_ccache_name = KEYRING:persistent:%{uid} 19 | 20 | [realms] 21 | DTSTACK.COM = { 22 | kdc = krbe2:88 23 | admin_server = krbe2:749 24 | } 25 | 26 | [domain_realm] 27 | .dtstack.com = DTSTACK.COM 28 | dtstack.com = DTSTACK.COM -------------------------------------------------------------------------------- /test/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /test/src/test/resources/phoenix5_kerberos/hbase.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/phoenix5_kerberos/hbase.keytab -------------------------------------------------------------------------------- /test/src/test/resources/phoenix5_kerberos/krb5.conf: -------------------------------------------------------------------------------- 1 | [libdefaults] 2 | default_realm = DTSTACK.COM 3 | dns_lookup_kdc = false 4 | dns_lookup_realm = false 5 | ticket_lifetime = 86400 6 | forwardable = true 7 | default_tgs_enctypes = rc4-hmac 8 | default_tkt_enctypes = rc4-hmac 9 | permitted_enctypes = rc4-hmac 10 | udp_preference_limit = 1 11 | kdc_timeout = 3000 12 | [realms] 13 | DTSTACK.COM = { 14 | kdc = 172.16.100.122 15 | admin_server = 172.16.100.122 16 | } 17 | [domain_realm] 18 | -------------------------------------------------------------------------------- /test/src/test/resources/restful/loader_test.jar: -------------------------------------------------------------------------------- 1 | loader_test -------------------------------------------------------------------------------- /test/src/test/resources/solr/jaas-client.conf: -------------------------------------------------------------------------------- 1 | SolrJClient { 2 | com.sun.security.auth.module.Krb5LoginModule required 3 | useKeyTab=true 4 | keyTab="/Users/edz/tmp/solr.keytab" 5 | storeKey=true 6 | useTicketCache=true 7 | debug=true 8 | principal="solr/worker@DTSTACK.COM"; 9 | }; 10 | -------------------------------------------------------------------------------- /test/src/test/resources/solr/krb5.conf: -------------------------------------------------------------------------------- 1 | [libdefaults] 2 | default_realm = DTSTACK.COM 3 | dns_lookup_kdc = false 4 | dns_lookup_realm = false 5 | ticket_lifetime = 24h 6 | renew_lifetime = 10d 7 | forwardable = true 8 | udp_preference_limit = 1 9 | kdc_timeout = 3000 10 | [realms] 11 | DTSTACK.COM = { 12 | kdc = 172.16.100.208 13 | admin_server = 172.16.100.208 14 | } 15 | [domain_realm] 16 | .dtstack.com = DTSTACK.COM 17 | dtstack.com = DTSTACK.COM 18 | 19 | -------------------------------------------------------------------------------- /test/src/test/resources/solr/solr.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/solr/solr.keytab -------------------------------------------------------------------------------- /test/src/test/resources/ssl-crt/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDSTCCAjGgAwIBAgIUfypDVJVL8WYVntPqjlWB+tIlS64wDQYJKoZIhvcNAQEL 3 | BQAwNDEyMDAGA1UEAxMpRWxhc3RpYyBDZXJ0aWZpY2F0ZSBUb29sIEF1dG9nZW5l 4 | cmF0ZWQgQ0EwHhcNMjEwNzE0MDc1MTQ3WhcNMjQwNzEzMDc1MTQ3WjA0MTIwMAYD 5 | VQQDEylFbGFzdGljIENlcnRpZmljYXRlIFRvb2wgQXV0b2dlbmVyYXRlZCBDQTCC 6 | ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALBR16dnOK9RHJLzzYNdUxC4 7 | 3ZpH9FQf9vUCvX08+PS21p1jPw4UrMVlufKm1FUdH8Se1T56nHZjIsLVUfVmD3Nu 8 | LZ0BUXA5TP5IU6RPdomPU3CXZmlzcY+GfvYRTI+/imbnUNztX6PP9axivTc6QDuv 9 | 8Pntn93IvMFCQuo4buyDtbyjnpIfS/kUzYbAQSWykCtU6z1u3xB7DvIAIss0tyWa 10 | VRzzYUt4a+tlN1DT4nnpMw/rIIOuRtSz4YdWJ4igRo4AobRGlBNSbZjZnk720BTp 11 | hkfgSAJCdJtz6Qtwz0HgZYQA73TFeOU2dzcy3A7vGKhNT9GCcUVqdTrlKfqKsr8C 12 | AwEAAaNTMFEwHQYDVR0OBBYEFIlId2UiZ5DiPAi5Z15FR8yrFOm1MB8GA1UdIwQY 13 | MBaAFIlId2UiZ5DiPAi5Z15FR8yrFOm1MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI 14 | hvcNAQELBQADggEBAKd+mKUgVHwdtZONQU1l8jzprz8+w6baSYea1Kag9DaCRCLc 15 | hgtukqpmFP85II0E6/lhrQKyui0Nb8perenzBoTE3m2W2aivT4IdT2VnYnqi49vS 16 | hQgzjog5khchT5LBugHWH4oYIrj2zeilwHEc3pkQV2QiUdTKJv7SChFcJymso9V5 17 | kypqZgT0kycrasCKDscNW9n7bCDYnS8syKtRKUKNUf79vEp9LXnFiZ5FxIOeS1Eq 18 | D3wZA0oE5GZrOEa5xGNO0y3l8a4IlRAIyy8Gz/zpqdts0+hGUwAz8R4O6WxWhFQX 19 | jZTc2cM5AXjQP4YeaPxAJq0uyE4qlJaHL8p877U= 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /test/src/test/resources/ssl-p12/elastic-certificates-89.p12: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/ssl-p12/elastic-certificates-89.p12 -------------------------------------------------------------------------------- /test/src/test/resources/ssl/cm-auto-global_truststore.jks: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/ssl/cm-auto-global_truststore.jks -------------------------------------------------------------------------------- /test/src/test/resources/ssl/ssl-client.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ssl.client.truststore.type 7 | jks 8 | 9 | 10 | ssl.client.truststore.location 11 | /Users/edz/projects/dt-center-common-loader/test/src/test/resources/ssl/cm-auto-global_truststore.jks 12 | 13 | 14 | ssl.client.truststore.password 15 | XfscZOW22LMAgrrZxo8MIhf2L3MTodv6kXpvBt2eg7X 16 | 17 | 18 | ssl.client.truststore.reload.interval 19 | 10000 20 | 21 | 22 | -------------------------------------------------------------------------------- /test/src/test/resources/tdh/inceptor/inceptor.keytab: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DTStack/DatasourceX/0884d29d25ffaa06963b88a7acdd7e36467952b4/test/src/test/resources/tdh/inceptor/inceptor.keytab -------------------------------------------------------------------------------- /test/src/test/resources/tdh/inceptor/krb5.conf: -------------------------------------------------------------------------------- 1 | [logging] 2 | default = FILE:/var/log/krb5libs.log 3 | kdc = FILE:/var/log/krb5kdc.log 4 | admin_server = FILE:/var/log/kadmind.log 5 | 6 | [libdefaults] 7 | default_realm = TDH 8 | dns_lookup_realm = false 9 | dns_lookup_kdc = false 10 | ticket_lifetime = 24h 11 | renew_lifetime = 7d 12 | forwardable = true 13 | allow_weak_crypto = true 14 | udp_preference_limit = 1000000 15 | default_ccache_name = FILE:/tmp/krb5cc_%{uid} 16 | 17 | [realms] 18 | TDH = { 19 | kdc = tdh6-node3:1088 20 | kdc = tdh6-node1:1088 21 | } 22 | -------------------------------------------------------------------------------- /trino/src/main/java/com/dtstack/dtcenter/common/loader/trino/TrinoErrorPattern.java: -------------------------------------------------------------------------------- 1 | package com.dtstack.dtcenter.common.loader.trino; 2 | 3 | import com.dtstack.dtcenter.common.loader.common.exception.AbsErrorPattern; 4 | 5 | /** 6 | * 7 | * 用于 trino 数据源获取连接失败原因匹配 8 | * 9 | * @author ：wangchuan 10 | * date：Created in 下午2:21 2021/9/9 11 | * company: www.dtstack.com 12 | */ 13 | public class TrinoErrorPattern extends AbsErrorPattern { 14 | 15 | } 16 | -------------------------------------------------------------------------------- /trino/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.trino.TrinoClient -------------------------------------------------------------------------------- /trino/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IKerberos: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.hadoop.AbsKerberosClient -------------------------------------------------------------------------------- /trino/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.ITable: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.trino.TrinoTableClient -------------------------------------------------------------------------------- /vertica/src/main/java/com/dtstack/dtcenter/common/loader/vertica/VerticaConnFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.dtstack.dtcenter.common.loader.vertica; 20 | 21 | import com.dtstack.dtcenter.common.loader.rdbms.ConnFactory; 22 | 23 | /** 24 | * @company: www.dtstack.com 25 | * @Author ：Nanqi 26 | * @Date ：Created in 12:03 2020/12/8 27 | * @Description：Vertica 工厂 28 | */ 29 | public class VerticaConnFactory extends ConnFactory { 30 | public VerticaConnFactory() { 31 | driverName = "com.vertica.jdbc.Driver"; 32 | testSql = "select 1"; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /vertica/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.vertica.VerticaClient -------------------------------------------------------------------------------- /websocket/src/main/resources/META-INF/services/com.dtstack.dtcenter.loader.client.IClient: -------------------------------------------------------------------------------- 1 | com.dtstack.dtcenter.common.loader.websocket.SocketClient --------------------------------------------------------------------------------