├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.yml │ ├── feature-request.yml │ └── improve-request.yml ├── PULL_REQUEST_TEMPLATE.md └── workflows │ ├── cicd.yml │ └── deploy-website.yml ├── .gitignore ├── .licenserc.yaml ├── CODE_OF_CONDUCT.md ├── LICENSE ├── NOTICE ├── README.md ├── README_zh.md ├── bitsail-base ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── base │ │ │ ├── catalog │ │ │ ├── TableCatalogFactory.java │ │ │ └── TableCatalogFactoryHelper.java │ │ │ ├── component │ │ │ ├── ComponentBuilder.java │ │ │ └── DefaultComponentBuilderLoader.java │ │ │ ├── connector │ │ │ ├── BuilderGroup.java │ │ │ ├── reader │ │ │ │ ├── DataReaderDAGBuilder.java │ │ │ │ └── v1 │ │ │ │ │ ├── Boundedness.java │ │ │ │ │ ├── Source.java │ │ │ │ │ ├── SourceEvent.java │ │ │ │ │ ├── SourcePipeline.java │ │ │ │ │ ├── SourceReader.java │ │ │ │ │ ├── SourceSplit.java │ │ │ │ │ └── SourceSplitCoordinator.java │ │ │ ├── transform │ │ │ │ ├── DataTransformDAGBuilder.java │ │ │ │ └── v1 │ │ │ │ │ ├── MapTransformer.java │ │ │ │ │ ├── PartitionTransformer.java │ │ │ │ │ └── Transformer.java │ │ │ └── writer │ │ │ │ ├── DataWriterDAGBuilder.java │ │ │ │ └── v1 │ │ │ │ ├── Sink.java │ │ │ │ ├── Writer.java │ │ │ │ ├── WriterCommitter.java │ │ │ │ ├── comittable │ │ │ │ ├── CommittableMessage.java │ │ │ │ └── CommittableState.java │ │ │ │ └── state │ │ │ │ └── EmptyState.java │ │ │ ├── constants │ │ │ ├── BaseMetricsNames.java │ │ │ └── ClientMetricName.java │ │ │ ├── decoder │ │ │ ├── MessageDecodeCompositor.java │ │ │ └── MessageDecoder.java │ │ │ ├── dirty │ │ │ ├── AbstractDirtyCollector.java │ │ │ ├── DirtyCollectorBuilder.java │ │ │ ├── DirtyCollectorFactory.java │ │ │ ├── DirtyRecord.java │ │ │ └── impl │ │ │ │ ├── NoOpDirtyCollector.java │ │ │ │ └── NoopDirtyCollectorBuilder.java │ │ │ ├── enumerate │ │ │ └── ContentType.java │ │ │ ├── execution │ │ │ ├── BaseExecutionEnviron.java │ │ │ ├── ExecutionEnviron.java │ │ │ ├── Mode.java │ │ │ └── ProcessResult.java │ │ │ ├── extension │ │ │ ├── Component.java │ │ │ ├── GlobalCommittable.java │ │ │ ├── ParallelismComputable.java │ │ │ ├── SecurityModule.java │ │ │ ├── SupportProducedType.java │ │ │ └── TypeInfoConverterFactory.java │ │ │ ├── format │ │ │ ├── DeserializationFormat.java │ │ │ ├── DeserializationFormatFactory.java │ │ │ ├── DeserializationSchema.java │ │ │ ├── SerializationFormat.java │ │ │ ├── SerializationFormatFactory.java │ │ │ └── SerializationSchema.java │ │ │ ├── messenger │ │ │ ├── BaseStatisticsMessenger.java │ │ │ ├── Messenger.java │ │ │ ├── MessengerBuilder.java │ │ │ ├── MessengerFactory.java │ │ │ ├── checker │ │ │ │ ├── DirtyRecordChecker.java │ │ │ │ └── LowVolumeTestChecker.java │ │ │ ├── common │ │ │ │ ├── MessageType.java │ │ │ │ ├── MessengerCounterType.java │ │ │ │ └── MessengerGroup.java │ │ │ └── context │ │ │ │ ├── MessengerContext.java │ │ │ │ └── SimpleMessengerContext.java │ │ │ ├── metrics │ │ │ ├── MetricManager.java │ │ │ ├── MetricReporter.java │ │ │ ├── Scheduled.java │ │ │ ├── ScheduledMetricReporterWrap.java │ │ │ ├── manager │ │ │ │ ├── BitSailMetricManager.java │ │ │ │ ├── CallTracer.java │ │ │ │ └── LockFreeSlidingWindowReservoir.java │ │ │ └── reporter │ │ │ │ ├── AbstractReporter.java │ │ │ │ ├── MetricReporterBuilder.java │ │ │ │ ├── MetricReporterFactory.java │ │ │ │ └── impl │ │ │ │ ├── LogMetricReporter.java │ │ │ │ ├── LogMetricReporterBuilder.java │ │ │ │ ├── NopMetricReporter.java │ │ │ │ └── NopMetricReporterBuilder.java │ │ │ ├── packages │ │ │ ├── LocalFSPluginFinder.java │ │ │ ├── Plugin.java │ │ │ ├── PluginErrorCode.java │ │ │ ├── PluginFinder.java │ │ │ ├── PluginFinderFactory.java │ │ │ └── PluginStore.java │ │ │ ├── parallelism │ │ │ └── ParallelismAdvice.java │ │ │ ├── progress │ │ │ ├── JobProgress.java │ │ │ └── NoOpJobProgress.java │ │ │ ├── ratelimit │ │ │ ├── Channel.java │ │ │ ├── Communication.java │ │ │ └── CommunicationTool.java │ │ │ ├── runtime │ │ │ ├── RuntimePluggable.java │ │ │ ├── metrics │ │ │ │ └── BitSailClientMetricsPlugin.java │ │ │ └── progress │ │ │ │ └── JobProgressPlugin.java │ │ │ ├── serializer │ │ │ ├── BinarySerializer.java │ │ │ ├── Converter.java │ │ │ ├── RowConverter.java │ │ │ ├── SimpleVersionedBinarySerializer.java │ │ │ └── VersionedSerializer.java │ │ │ ├── statistics │ │ │ └── VMInfo.java │ │ │ └── version │ │ │ ├── VersionHolder.java │ │ │ └── VersionInfoAnnotation.java │ └── resources │ │ └── META-INF │ │ └── services │ │ ├── com.bytedance.bitsail.base.dirty.DirtyCollectorBuilder │ │ ├── com.bytedance.bitsail.base.metrics.reporter.MetricReporterBuilder │ │ ├── com.bytedance.bitsail.base.packages.PluginFinder │ │ └── com.bytedance.bitsail.base.runtime.RuntimePluggable │ └── test │ ├── java │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── base │ │ ├── connector │ │ └── writer │ │ │ └── v1 │ │ │ └── committable │ │ │ └── CommittableStateTest.java │ │ ├── decoder │ │ └── MessageDecodeCompositorTest.java │ │ ├── dirty │ │ └── DirtyCollectorTest.java │ │ ├── execution │ │ └── ProcessResultTest.java │ │ ├── messenger │ │ └── checker │ │ │ ├── DirtyRecordCheckerTest.java │ │ │ └── LowVolumeTestCheckerTest.java │ │ ├── metrics │ │ ├── ScheduledMetricReporterWrapTest.java │ │ ├── manager │ │ │ ├── BitSailMetricManagerTest.java │ │ │ ├── CallTracerTest.java │ │ │ └── LockFreeSlidingWindowReservoirTest.java │ │ └── reporter │ │ │ ├── MetricReporterFactoryTest.java │ │ │ └── MetricReporterTest.java │ │ ├── packages │ │ ├── LocalFSPluginFinderTest.java │ │ └── PluginStoreTest.java │ │ ├── ratelimit │ │ └── CommunicationToolTest.java │ │ ├── runtime │ │ └── BitSailClientMetricsPluginTest.java │ │ ├── serializer │ │ └── SerializerTest.java │ │ └── version │ │ └── PackageInfoTest.java │ └── resources │ ├── .dataleap.bitsail.version.properties │ ├── classloader │ ├── plugin │ │ ├── test1 │ │ ├── test2 │ │ ├── test3 │ │ └── test4-1.0-SNAPSHOT │ ├── plugin_conf │ │ ├── dts_io.json │ │ ├── test.json │ │ ├── test2.json │ │ └── test3.json │ └── static_lib.json │ └── logback-test.xml ├── bitsail-clients ├── bitsail-client-api │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── client │ │ │ └── api │ │ │ ├── command │ │ │ ├── BaseCommandArgs.java │ │ │ ├── CommandAction.java │ │ │ ├── CommandArgs.java │ │ │ └── CommandArgsParser.java │ │ │ ├── engine │ │ │ └── EngineRunner.java │ │ │ └── utils │ │ │ └── PackageResolver.java │ │ └── test │ │ └── java │ │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── client │ │ └── api │ │ ├── command │ │ ├── BaseCommandArgsParserTest.java │ │ ├── CommandArgsParserTest.java │ │ └── KerberosCommandArgsParserTest.java │ │ └── utils │ │ └── PackageResolverTest.java ├── bitsail-client-entry │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── client │ │ │ └── entry │ │ │ ├── Entry.java │ │ │ ├── constants │ │ │ └── EntryConstants.java │ │ │ ├── option │ │ │ └── ClientCommonOption.java │ │ │ ├── plugins │ │ │ └── ClientPluginFinder.java │ │ │ └── security │ │ │ └── SecurityContextFactory.java │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── client │ │ │ └── entry │ │ │ ├── EntryTest.java │ │ │ ├── FakeEngine.java │ │ │ └── security │ │ │ └── SecurityContextFactoryTest.java │ │ └── resources │ │ ├── META-INF │ │ └── services │ │ │ └── com.bytedance.bitsail.client.api.engine.EngineRunner │ │ └── test_job_conf.json ├── bitsail-client-flink │ ├── bitsail-client-flink-1.11 │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── resources │ │ │ ├── META-INF │ │ │ └── services │ │ │ │ └── com.bytedance.bitsail.client.api.engine.EngineRunner │ │ │ └── bitsail-client-flink-1.11.json │ ├── bitsail-client-flink-base │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── entry │ │ │ │ │ └── flink │ │ │ │ │ ├── command │ │ │ │ │ └── FlinkCommandArgs.java │ │ │ │ │ ├── configuration │ │ │ │ │ └── FlinkRunnerConfigOptions.java │ │ │ │ │ ├── deployment │ │ │ │ │ ├── DeploymentSupplier.java │ │ │ │ │ ├── DeploymentSupplierFactory.java │ │ │ │ │ ├── kubernetes │ │ │ │ │ │ └── KubernetesDeploymentSupplier.java │ │ │ │ │ ├── local │ │ │ │ │ │ └── LocalDeploymentSupplier.java │ │ │ │ │ ├── remote │ │ │ │ │ │ └── RemoteDeploymentSupplier.java │ │ │ │ │ └── yarn │ │ │ │ │ │ └── YarnDeploymentSupplier.java │ │ │ │ │ ├── engine │ │ │ │ │ └── FlinkEngineRunner.java │ │ │ │ │ ├── handlers │ │ │ │ │ └── CustomFlinkPackageHandler.java │ │ │ │ │ ├── savepoint │ │ │ │ │ └── FlinkRunnerSavepointLoader.java │ │ │ │ │ └── utils │ │ │ │ │ └── FlinkPackageResolver.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.bytedance.bitsail.entry.flink.deployment.DeploymentSupplier │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── entry │ │ │ │ └── flink │ │ │ │ ├── command │ │ │ │ └── FlinkRunCommandArgsTest.java │ │ │ │ ├── deployment │ │ │ │ ├── kubernetes │ │ │ │ │ └── KubernetesDeploymentSupplierTest.java │ │ │ │ └── yarn │ │ │ │ │ └── YarnDeploymentSupplierTest.java │ │ │ │ ├── engine │ │ │ │ └── FlinkEngineRunnerTest.java │ │ │ │ ├── handlers │ │ │ │ └── CustomFlinkPackageHandlerTest.java │ │ │ │ ├── savepoint │ │ │ │ └── FlinkRunnerSavepointLoaderTest.java │ │ │ │ └── utils │ │ │ │ └── FlinkPackageResolverTest.java │ │ │ └── resources │ │ │ ├── bitsail.conf │ │ │ ├── bitsail │ │ │ └── hdfs_to_print.json │ │ │ ├── conf │ │ │ ├── flink-conf.yaml │ │ │ └── log4j.properties │ │ │ ├── examples │ │ │ └── Fake_Print_Example.json │ │ │ ├── hadoop │ │ │ └── bin │ │ │ │ ├── hadoop.dll │ │ │ │ └── winutils.exe │ │ │ ├── lib │ │ │ ├── file_a │ │ │ └── flink-dist-a │ │ │ ├── test_dir │ │ │ ├── conf │ │ │ │ └── flink-conf.yaml │ │ │ └── lib │ │ │ │ └── file_a │ │ │ └── test_dir_no_conf_files │ │ │ └── conf │ │ │ └── file_a │ └── pom.xml └── pom.xml ├── bitsail-common ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── common │ │ │ ├── BitSailException.java │ │ │ ├── annotation │ │ │ └── Essential.java │ │ │ ├── catalog │ │ │ ├── TableCatalogErrorCode.java │ │ │ ├── TableCatalogManager.java │ │ │ ├── TableCatalogOptions.java │ │ │ ├── TableCatalogStrategy.java │ │ │ └── table │ │ │ │ ├── CatalogTable.java │ │ │ │ ├── CatalogTableAlterDefinition.java │ │ │ │ ├── CatalogTableColumn.java │ │ │ │ ├── CatalogTableDefinition.java │ │ │ │ ├── CatalogTableSchema.java │ │ │ │ ├── TableCatalog.java │ │ │ │ └── TableOperation.java │ │ │ ├── column │ │ │ ├── ArrayMapColumn.java │ │ │ ├── BooleanColumn.java │ │ │ ├── BytesColumn.java │ │ │ ├── Column.java │ │ │ ├── ColumnCast.java │ │ │ ├── DateColumn.java │ │ │ ├── DoubleColumn.java │ │ │ ├── ListColumn.java │ │ │ ├── LongColumn.java │ │ │ ├── MapColumn.java │ │ │ └── StringColumn.java │ │ │ ├── configuration │ │ │ ├── BitSailConfiguration.java │ │ │ ├── BitSailSystemConfiguration.java │ │ │ └── ConfigParser.java │ │ │ ├── constants │ │ │ └── Constants.java │ │ │ ├── exception │ │ │ ├── CommonErrorCode.java │ │ │ ├── ErrorCode.java │ │ │ └── FrameworkErrorCode.java │ │ │ ├── model │ │ │ └── ColumnInfo.java │ │ │ ├── option │ │ │ ├── AdapterOptions.java │ │ │ ├── CommonOptions.java │ │ │ ├── ConfigOption.java │ │ │ ├── ConfigOptions.java │ │ │ ├── ReaderOptions.java │ │ │ ├── TransformOptions.java │ │ │ └── WriterOptions.java │ │ │ ├── row │ │ │ ├── BinlogRow.java │ │ │ ├── Row.java │ │ │ └── RowKind.java │ │ │ ├── type │ │ │ ├── BitSailTypeInfoConverter.java │ │ │ ├── BitSailTypeParser.java │ │ │ ├── TypeInfoConverter.java │ │ │ └── filemapping │ │ │ │ ├── FileMappingTypeInfoConverter.java │ │ │ │ ├── FileMappingTypeInfoReader.java │ │ │ │ └── HiveTypeInfoConverter.java │ │ │ ├── typeinfo │ │ │ ├── BasicArrayTypeInfo.java │ │ │ ├── BasicTypeInfo.java │ │ │ ├── GenericTypeInfo.java │ │ │ ├── ListTypeInfo.java │ │ │ ├── MapTypeInfo.java │ │ │ ├── RowTypeInfo.java │ │ │ ├── TypeInfo.java │ │ │ ├── TypeInfoBridge.java │ │ │ ├── TypeInfoCompatibles.java │ │ │ ├── TypeInfoUtils.java │ │ │ ├── TypeInfoValueConverter.java │ │ │ ├── TypeInfos.java │ │ │ ├── TypeProperty.java │ │ │ └── Types.java │ │ │ └── util │ │ │ ├── DateTimeFormatterUtils.java │ │ │ ├── DateUtil.java │ │ │ ├── FastJsonUtil.java │ │ │ ├── FieldPathUtils.java │ │ │ ├── HttpManager.java │ │ │ ├── JsonSerializer.java │ │ │ ├── JsonVisitor.java │ │ │ ├── LogUtils.java │ │ │ ├── OverFlowUtil.java │ │ │ ├── Pair.java │ │ │ ├── Preconditions.java │ │ │ ├── ProtoUtils.java │ │ │ ├── ProtoVisitor.java │ │ │ ├── ProtobufUtil.java │ │ │ ├── StrUtil.java │ │ │ ├── TimeUtils.java │ │ │ ├── TypeConvertUtil.java │ │ │ └── timelimit │ │ │ ├── FixedAttemptTimeLimit.java │ │ │ ├── SimpleTimeLimiter.java │ │ │ └── TimeLimiter.java │ └── resources │ │ └── bitsail-type-converter.yaml │ └── test │ ├── java │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── common │ │ ├── BaseTest.java │ │ ├── catalog │ │ ├── TableCatalogManagerTest.java │ │ └── fake │ │ │ └── FakeTableCatalog.java │ │ ├── column │ │ ├── ArrayMapColumnTest.java │ │ ├── DateColumnTest.java │ │ ├── ListColumnTest.java │ │ ├── LongColumnTest.java │ │ ├── MapColumnTest.java │ │ └── StringColumnTest.java │ │ ├── configuration │ │ └── sys │ │ │ └── BitSailSystemConfigurationTest.java │ │ ├── type │ │ ├── BitSailTypeInfoConverterTest.java │ │ ├── BitSailTypeParserTest.java │ │ └── SimpleTypeInfoConverterTest.java │ │ ├── typeinfo │ │ ├── TypeInfoCompatiblesColumnCompareTest.java │ │ ├── TypeInfoCompatiblesTest.java │ │ ├── TypeInfoUtilsTest.java │ │ ├── TypeInfoValueConverterTest.java │ │ └── TypeInfosTest.java │ │ └── util │ │ ├── DateUtilTest.java │ │ ├── FastJsonUtilTest.java │ │ ├── FieldPathUtilsTest.java │ │ ├── JsonVisitorTest.java │ │ └── LogUtilsTest.java │ └── resources │ ├── CheckColumnsCorrectnessTest.json │ ├── fake-type-converter.yaml │ ├── logback-test.xml │ └── multi_conf.json ├── bitsail-components ├── bitsail-component-clients │ └── pom.xml ├── bitsail-component-formats-flink │ ├── bitsail-component-format-flink-api │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── component │ │ │ └── format │ │ │ └── api │ │ │ └── RowBuilder.java │ ├── bitsail-component-format-flink-hive │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── component │ │ │ │ └── format │ │ │ │ └── hive │ │ │ │ └── HiveGeneralRowBuilder.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── component │ │ │ └── format │ │ │ └── hive │ │ │ └── HiveGeneralRowBuilderTest.java │ ├── bitsail-component-format-flink-json │ │ └── pom.xml │ └── pom.xml ├── bitsail-component-formats │ ├── bitsail-component-format-csv │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── component │ │ │ │ └── format │ │ │ │ └── csv │ │ │ │ ├── CsvDeserializationSchema.java │ │ │ │ ├── error │ │ │ │ └── CsvFormatErrorCode.java │ │ │ │ └── option │ │ │ │ └── CsvReaderOptions.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── component │ │ │ └── format │ │ │ └── csv │ │ │ └── CsvDeserializationSchemaTest.java │ ├── bitsail-component-format-debezium │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── component │ │ │ │ └── format │ │ │ │ └── debezium │ │ │ │ ├── deserialization │ │ │ │ ├── DebeziumDeserializationSchema.java │ │ │ │ └── DebeziumJsonDeserializationSchema.java │ │ │ │ ├── option │ │ │ │ └── DebeziumWriterOptions.java │ │ │ │ └── serialization │ │ │ │ └── DebeziumJsonSerializationSchema.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── component │ │ │ └── format │ │ │ └── debezium │ │ │ └── deserialization │ │ │ └── DebeziumJsonDeserializationSchemaTest.java │ ├── bitsail-component-format-json │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── component │ │ │ │ └── format │ │ │ │ └── json │ │ │ │ ├── JsonRowDeserializationSchema.java │ │ │ │ ├── JsonRowSerializationSchema.java │ │ │ │ ├── RowToJsonConverter.java │ │ │ │ ├── error │ │ │ │ └── JsonFormatErrorCode.java │ │ │ │ └── option │ │ │ │ └── JsonReaderOptions.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── component │ │ │ └── format │ │ │ └── json │ │ │ ├── JsonRowDeserializationSchemaTest.java │ │ │ ├── JsonRowSerializationSchemaTest.java │ │ │ └── RowToJsonConverterTest.java │ └── pom.xml ├── bitsail-component-metrics │ ├── bitsail-component-metrics-prometheus │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── component │ │ │ │ │ └── metrics │ │ │ │ │ └── prometheus │ │ │ │ │ ├── AbstractPrometheusReporter.java │ │ │ │ │ ├── error │ │ │ │ │ ├── PrometheusErrorCode.java │ │ │ │ │ └── PrometheusPushGatewayErrorCode.java │ │ │ │ │ ├── impl │ │ │ │ │ ├── PrometheusMetricReporter.java │ │ │ │ │ ├── PrometheusMetricReporterBuilder.java │ │ │ │ │ ├── PrometheusPushGatewayMetricReporterBuilder.java │ │ │ │ │ └── PrometheusPushGatewayReporter.java │ │ │ │ │ └── option │ │ │ │ │ ├── PrometheusOptions.java │ │ │ │ │ └── PrometheusPushGatewayOptions.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ └── com.bytedance.bitsail.base.metrics.reporter.MetricReporterBuilder │ │ │ └── test │ │ │ └── java │ │ │ ├── MetricReporterFactoryTest.java │ │ │ ├── MetricReporterTest.java │ │ │ └── testcontainer │ │ │ └── PushGatewayContainer.java │ └── pom.xml ├── bitsail-component-progress │ └── pom.xml ├── bitsail-component-security │ ├── bitsail-component-security-kerberos │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── component │ │ │ │ └── format │ │ │ │ └── security │ │ │ │ └── kerberos │ │ │ │ ├── common │ │ │ │ ├── KerberosConstants.java │ │ │ │ └── KerberosErrorCode.java │ │ │ │ ├── option │ │ │ │ └── KerberosOptions.java │ │ │ │ └── security │ │ │ │ ├── HadoopSecurityModule.java │ │ │ │ └── KerberosAbstractSecurityModule.java │ │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── com.bytedance.bitsail.base.extension.SecurityModule │ └── pom.xml ├── bitsail-conversion-flink │ ├── bitsail-conversion-flink-hive │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── conversion │ │ │ │ │ └── hive │ │ │ │ │ ├── BitSailColumnConversion.java │ │ │ │ │ ├── ConvertToHiveObjectOptions.java │ │ │ │ │ ├── HiveInspectors.java │ │ │ │ │ ├── HiveObjectConversion.java │ │ │ │ │ ├── WritableHiveObjectConversion.java │ │ │ │ │ ├── extractor │ │ │ │ │ ├── GeneralWritableExtractor.java │ │ │ │ │ ├── HiveWritableExtractor.java │ │ │ │ │ ├── ParquetWritableExtractor.java │ │ │ │ │ ├── StringText.java │ │ │ │ │ └── SupportHiveDataType.java │ │ │ │ │ └── option │ │ │ │ │ └── HiveConversionOptions.java │ │ │ └── resources │ │ │ │ └── hive-type-converter.yaml │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── conversion │ │ │ └── hive │ │ │ └── extractor │ │ │ ├── GeneralWritableExtractorTest.java │ │ │ ├── HiveWritableExtractorTest.java │ │ │ ├── ParquetWritableExtractorTest.java │ │ │ └── StringTextTest.java │ └── pom.xml ├── bitsail-flink-row-parser │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ ├── batch │ │ │ ├── file │ │ │ │ └── parser │ │ │ │ │ ├── BytesParseFactory.java │ │ │ │ │ ├── CsvBytesParser.java │ │ │ │ │ ├── JsonBytesParser.java │ │ │ │ │ └── PbBytesParser.java │ │ │ └── parser │ │ │ │ └── row │ │ │ │ ├── HiveRowBuilder.java │ │ │ │ ├── JsonRowBuilder.java │ │ │ │ ├── KVRowBuilder.java │ │ │ │ └── TextRowBuilder.java │ │ │ ├── dump │ │ │ └── datasink │ │ │ │ └── file │ │ │ │ └── parser │ │ │ │ ├── BytesParser.java │ │ │ │ ├── JsonBytesParser.java │ │ │ │ ├── PbBytesParser.java │ │ │ │ └── RowBytesParser.java │ │ │ └── parser │ │ │ ├── error │ │ │ └── ParserErrorCode.java │ │ │ └── option │ │ │ └── RowParserOptions.java │ │ └── test │ │ └── java │ │ └── com │ │ └── bytedance │ │ └── bitsail │ │ ├── batch │ │ └── file │ │ │ └── parser │ │ │ ├── BytesParserTest.java │ │ │ ├── CsvBytesParserTest.java │ │ │ ├── JsonBytesParserTest.java │ │ │ ├── PbBytesParserTest.java │ │ │ └── RowBytesParserTest.java │ │ └── dump │ │ └── datasink │ │ └── file │ │ └── parser │ │ ├── JsonBytesParserTest.java │ │ ├── PbBytesParserTest.java │ │ ├── PbParseTest.java │ │ ├── StringToDateTest.java │ │ └── TestMap.java └── pom.xml ├── bitsail-connectors ├── bitsail-connectors-legacy │ ├── bitsail-connector-fake │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── legacy │ │ │ │ │ └── fake │ │ │ │ │ ├── option │ │ │ │ │ └── FakeReaderOptions.java │ │ │ │ │ └── source │ │ │ │ │ └── FakeSource.java │ │ │ └── resources │ │ │ │ └── bitsail-connector-legacy-fake.json │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── legacy │ │ │ └── fake │ │ │ └── source │ │ │ └── FakeSourceTest.java │ ├── bitsail-connector-ftp │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── legacy │ │ │ │ └── ftp │ │ │ │ ├── client │ │ │ │ ├── FtpHandler.java │ │ │ │ ├── FtpHandlerFactory.java │ │ │ │ ├── IFtpHandler.java │ │ │ │ └── SftpHandler.java │ │ │ │ ├── common │ │ │ │ └── FtpConfig.java │ │ │ │ ├── error │ │ │ │ └── FtpInputFormatErrorCode.java │ │ │ │ ├── option │ │ │ │ └── FtpReaderOptions.java │ │ │ │ └── source │ │ │ │ ├── FtpInputFormat.java │ │ │ │ ├── FtpInputSplit.java │ │ │ │ └── FtpSeqBufferedReader.java │ │ │ └── resources │ │ │ ├── bitsail-connector-legacy-ftp.json │ │ │ └── ftp-type-converter.yaml │ ├── bitsail-connector-hadoop │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── hadoop │ │ │ │ │ ├── common │ │ │ │ │ ├── SourceEngine.java │ │ │ │ │ └── TextInputFormatErrorCode.java │ │ │ │ │ ├── option │ │ │ │ │ └── HadoopReaderOptions.java │ │ │ │ │ ├── sink │ │ │ │ │ └── FileOutputFormatPlugin.java │ │ │ │ │ ├── source │ │ │ │ │ ├── HadoopInputFormat.java │ │ │ │ │ ├── HadoopInputFormatBasePlugin.java │ │ │ │ │ ├── HadoopInputFormatCommonBasePlugin.java │ │ │ │ │ └── format │ │ │ │ │ │ ├── DelimitedFileInputFormatPlugin.java │ │ │ │ │ │ ├── FileInputFormatPlugin.java │ │ │ │ │ │ └── TextInputFormat.java │ │ │ │ │ ├── split │ │ │ │ │ └── OptimizedHadoopInputSplit.java │ │ │ │ │ └── util │ │ │ │ │ └── HdfsUtils.java │ │ │ └── resources │ │ │ │ └── bitsail-connector-legacy-hadoop.json │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── hadoop │ │ │ └── util │ │ │ └── HdfsUtilsTest.java │ ├── bitsail-connector-hbase │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── hbase │ │ │ │ │ ├── HBaseHelper.java │ │ │ │ │ ├── auth │ │ │ │ │ ├── KerberosAuthenticator.java │ │ │ │ │ └── KerberosFileHandler.java │ │ │ │ │ ├── constant │ │ │ │ │ ├── HBaseConstants.java │ │ │ │ │ └── NullMode.java │ │ │ │ │ ├── error │ │ │ │ │ └── HBasePluginErrorCode.java │ │ │ │ │ ├── format │ │ │ │ │ └── HBaseDeserializationFormat.java │ │ │ │ │ ├── option │ │ │ │ │ ├── HBaseReaderOptions.java │ │ │ │ │ └── HBaseWriterOptions.java │ │ │ │ │ ├── sink │ │ │ │ │ ├── HBaseOutputFormat.java │ │ │ │ │ └── function │ │ │ │ │ │ ├── ConstantFunction.java │ │ │ │ │ │ ├── FunctionFactory.java │ │ │ │ │ │ ├── FunctionParser.java │ │ │ │ │ │ ├── FunctionTree.java │ │ │ │ │ │ ├── IFunction.java │ │ │ │ │ │ ├── Md5Function.java │ │ │ │ │ │ └── StringFunction.java │ │ │ │ │ └── source │ │ │ │ │ ├── HBaseInputFormat.java │ │ │ │ │ └── split │ │ │ │ │ └── RegionSplit.java │ │ │ └── resources │ │ │ │ ├── bitsail-connector-legacy-hbase.json │ │ │ │ └── hbase-type-converter.yaml │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── hbase │ │ │ └── sink │ │ │ └── function │ │ │ └── RowKeyFunctionTest.java │ ├── bitsail-connector-hive │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── legacy │ │ │ │ │ └── hive │ │ │ │ │ ├── common │ │ │ │ │ ├── HiveParqueFormatErrorCode.java │ │ │ │ │ ├── HiveTableCatalog.java │ │ │ │ │ └── HiveTableCatalogFactory.java │ │ │ │ │ ├── option │ │ │ │ │ ├── HiveReaderOptions.java │ │ │ │ │ └── HiveWriterOptions.java │ │ │ │ │ ├── sink │ │ │ │ │ └── HiveOutputFormat.java │ │ │ │ │ ├── source │ │ │ │ │ └── HiveInputFormat.java │ │ │ │ │ └── util │ │ │ │ │ └── HiveConfUtils.java │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── com.bytedance.bitsail.base.catalog.TableCatalogFactory │ │ │ │ └── bitsail-connector-legacy-hive.json │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── legacy │ │ │ │ └── hive │ │ │ │ ├── runtime │ │ │ │ └── MockStreamingRuntimeContextForTest.java │ │ │ │ └── sink │ │ │ │ ├── HiveOutputFormatTypeTest.java │ │ │ │ └── HiveParquetOutputFailoverTest.java │ │ │ └── resources │ │ │ ├── hadoop │ │ │ └── bin │ │ │ │ └── winutils.exe │ │ │ └── hive │ │ │ ├── hive_writer.json │ │ │ └── hive_writer_without_columns.json │ ├── bitsail-connector-hudi │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── legacy │ │ │ │ │ └── hudi │ │ │ │ │ ├── common │ │ │ │ │ └── HudiWriteOptions.java │ │ │ │ │ ├── compact │ │ │ │ │ ├── CompactFunction.java │ │ │ │ │ ├── CompactionCommitEvent.java │ │ │ │ │ ├── CompactionCommitSink.java │ │ │ │ │ ├── CompactionPlanEvent.java │ │ │ │ │ ├── CompactionPlanOperator.java │ │ │ │ │ └── CompactionPlanSourceFunction.java │ │ │ │ │ ├── configuration │ │ │ │ │ ├── FlinkOptions.java │ │ │ │ │ ├── HadoopConfigurations.java │ │ │ │ │ └── OptionsResolver.java │ │ │ │ │ ├── dag │ │ │ │ │ ├── HudiCompactSinkDAGBuilder.java │ │ │ │ │ ├── HudiCompactSourceDAGBuilder.java │ │ │ │ │ ├── HudiSinkFunctionDAGBuilder.java │ │ │ │ │ └── HudiSourceFunctionDAGBuilder.java │ │ │ │ │ ├── format │ │ │ │ │ ├── ParserFormatFactory.java │ │ │ │ │ ├── RowDataDeserializationFunction.java │ │ │ │ │ └── RowDataToRowFunction.java │ │ │ │ │ ├── schema │ │ │ │ │ ├── FilebasedSchemaProvider.java │ │ │ │ │ ├── SchemaProvider.java │ │ │ │ │ └── SchemaRegistryProvider.java │ │ │ │ │ ├── sink │ │ │ │ │ ├── CleanFunction.java │ │ │ │ │ ├── StreamWriteFunction.java │ │ │ │ │ ├── StreamWriteOperator.java │ │ │ │ │ ├── StreamWriteOperatorCoordinator.java │ │ │ │ │ ├── append │ │ │ │ │ │ ├── AppendWriteFunction.java │ │ │ │ │ │ └── AppendWriteOperator.java │ │ │ │ │ ├── bootstrap │ │ │ │ │ │ ├── BootstrapOperator.java │ │ │ │ │ │ ├── IndexRecord.java │ │ │ │ │ │ ├── aggregate │ │ │ │ │ │ │ ├── BootstrapAccumulator.java │ │ │ │ │ │ │ └── BootstrapAggFunction.java │ │ │ │ │ │ └── batch │ │ │ │ │ │ │ └── BatchBootstrapOperator.java │ │ │ │ │ ├── bucket │ │ │ │ │ │ ├── BucketBulkInsertWriterHelper.java │ │ │ │ │ │ ├── BucketStreamWriteFunction.java │ │ │ │ │ │ └── BucketStreamWriteOperator.java │ │ │ │ │ ├── bulk │ │ │ │ │ │ ├── BulkInsertWriteFunction.java │ │ │ │ │ │ ├── BulkInsertWriteOperator.java │ │ │ │ │ │ ├── BulkInsertWriterHelper.java │ │ │ │ │ │ ├── RowDataKeyGen.java │ │ │ │ │ │ └── WriterHelpers.java │ │ │ │ │ ├── common │ │ │ │ │ │ ├── AbstractStreamWriteFunction.java │ │ │ │ │ │ ├── AbstractWriteFunction.java │ │ │ │ │ │ ├── AbstractWriteOperator.java │ │ │ │ │ │ └── WriteOperatorFactory.java │ │ │ │ │ ├── event │ │ │ │ │ │ ├── CommitAckEvent.java │ │ │ │ │ │ └── WriteMetadataEvent.java │ │ │ │ │ ├── meta │ │ │ │ │ │ ├── CkpMessage.java │ │ │ │ │ │ └── CkpMetadata.java │ │ │ │ │ ├── partitioner │ │ │ │ │ │ ├── BucketAssignFunction.java │ │ │ │ │ │ ├── BucketAssigner.java │ │ │ │ │ │ ├── BucketAssigners.java │ │ │ │ │ │ ├── BucketIndexPartitioner.java │ │ │ │ │ │ └── profile │ │ │ │ │ │ │ ├── DeltaWriteProfile.java │ │ │ │ │ │ │ ├── EmptyWriteProfile.java │ │ │ │ │ │ │ ├── WriteProfile.java │ │ │ │ │ │ │ └── WriteProfiles.java │ │ │ │ │ ├── transform │ │ │ │ │ │ ├── ChainedTransformer.java │ │ │ │ │ │ ├── RowDataToHoodieFunction.java │ │ │ │ │ │ ├── RowDataToHoodieFunctions.java │ │ │ │ │ │ ├── RowToHoodieFunction.java │ │ │ │ │ │ └── Transformer.java │ │ │ │ │ └── utils │ │ │ │ │ │ ├── HiveSyncContext.java │ │ │ │ │ │ ├── NonThrownExecutor.java │ │ │ │ │ │ ├── PayloadCreation.java │ │ │ │ │ │ ├── Pipelines.java │ │ │ │ │ │ └── TimeWait.java │ │ │ │ │ ├── source │ │ │ │ │ ├── FileIndex.java │ │ │ │ │ ├── IncrementalInputSplits.java │ │ │ │ │ └── format │ │ │ │ │ │ ├── FilePathUtils.java │ │ │ │ │ │ ├── FormatUtils.java │ │ │ │ │ │ ├── cow │ │ │ │ │ │ ├── CopyOnWriteInputFormat.java │ │ │ │ │ │ ├── ParquetSplitReaderUtil.java │ │ │ │ │ │ └── vector │ │ │ │ │ │ │ ├── ColumnarMapData.java │ │ │ │ │ │ │ ├── HeapArrayVector.java │ │ │ │ │ │ │ ├── HeapMapColumnVector.java │ │ │ │ │ │ │ ├── HeapRowColumnVector.java │ │ │ │ │ │ │ ├── ParquetDecimalVector.java │ │ │ │ │ │ │ └── reader │ │ │ │ │ │ │ ├── AbstractColumnReader.java │ │ │ │ │ │ │ ├── ArrayColumnReader.java │ │ │ │ │ │ │ ├── BaseVectorizedColumnReader.java │ │ │ │ │ │ │ ├── FixedLenBytesColumnReader.java │ │ │ │ │ │ │ ├── Int64TimestampColumnReader.java │ │ │ │ │ │ │ ├── MapColumnReader.java │ │ │ │ │ │ │ ├── ParquetColumnarRowSplitReader.java │ │ │ │ │ │ │ ├── ParquetDataColumnReader.java │ │ │ │ │ │ │ ├── ParquetDataColumnReaderFactory.java │ │ │ │ │ │ │ ├── RowColumnReader.java │ │ │ │ │ │ │ └── RunLengthDecoder.java │ │ │ │ │ │ └── mor │ │ │ │ │ │ ├── MergeOnReadInputFormat.java │ │ │ │ │ │ ├── MergeOnReadInputSplit.java │ │ │ │ │ │ └── MergeOnReadTableState.java │ │ │ │ │ └── util │ │ │ │ │ ├── AvroSchemaConverter.java │ │ │ │ │ ├── AvroToRowDataConverters.java │ │ │ │ │ ├── ChangelogModes.java │ │ │ │ │ ├── CompactionUtil.java │ │ │ │ │ ├── DataTypeUtils.java │ │ │ │ │ ├── FlinkTables.java │ │ │ │ │ ├── InputFormats.java │ │ │ │ │ ├── RowDataProjection.java │ │ │ │ │ ├── RowDataToAvroConverters.java │ │ │ │ │ ├── RowToAvroConverters.java │ │ │ │ │ ├── SchemaUtils.java │ │ │ │ │ ├── StreamerUtil.java │ │ │ │ │ ├── StringToRowDataConverter.java │ │ │ │ │ └── ViewStorageProperties.java │ │ │ └── resources │ │ │ │ └── bitsail-connector-legacy-hudi.json │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── legacy │ │ │ │ └── hudi │ │ │ │ ├── dag │ │ │ │ ├── HudiCompactSourceDAGBuilderTest.java │ │ │ │ └── HudiSinkFunctionDAGBuilderTest.java │ │ │ │ ├── sink │ │ │ │ ├── functional │ │ │ │ │ ├── ITTestDataStreamWrite.java │ │ │ │ │ ├── TestStreamWriteOperatorCoordinator.java │ │ │ │ │ ├── TestWriteCopyOnWrite.java │ │ │ │ │ ├── TestWriteMergeOnRead.java │ │ │ │ │ └── TestWriteMergeOnReadWithCompact.java │ │ │ │ └── utils │ │ │ │ │ ├── CollectorOutput.java │ │ │ │ │ ├── CompactFunctionWrapper.java │ │ │ │ │ ├── InsertFunctionWrapper.java │ │ │ │ │ ├── MockCoordinatorExecutor.java │ │ │ │ │ ├── MockMapState.java │ │ │ │ │ ├── MockOperatorStateStore.java │ │ │ │ │ ├── MockStateInitializationContext.java │ │ │ │ │ ├── MockStreamingRuntimeContext.java │ │ │ │ │ ├── MockValueState.java │ │ │ │ │ ├── StreamWriteFunctionWrapper.java │ │ │ │ │ ├── TestFunctionWrapper.java │ │ │ │ │ └── TestWriteBase.java │ │ │ │ ├── source │ │ │ │ └── TestFileIndex.java │ │ │ │ └── utils │ │ │ │ ├── TestCompactionUtil.java │ │ │ │ ├── TestConfigurations.java │ │ │ │ ├── TestData.java │ │ │ │ ├── TestHoodieRowData.java │ │ │ │ ├── TestSQL.java │ │ │ │ ├── TestStreamerUtil.java │ │ │ │ ├── TestStringToRowDataConverter.java │ │ │ │ ├── TestUtils.java │ │ │ │ ├── TestViewStorageProperties.java │ │ │ │ └── source │ │ │ │ └── ContinuousFileSource.java │ │ │ └── resources │ │ │ ├── debezium_json.data │ │ │ ├── log4j-surefire-quiet.properties │ │ │ ├── log4j-surefire.properties │ │ │ ├── test_read_schema.avsc │ │ │ ├── test_source.data │ │ │ ├── test_source_2.data │ │ │ ├── test_source_3.data │ │ │ ├── test_source_4.data │ │ │ └── test_source_5.data │ ├── bitsail-connector-jdbc │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── legacy │ │ │ │ │ └── jdbc │ │ │ │ │ ├── catalog │ │ │ │ │ ├── MySQLTableCatalog.java │ │ │ │ │ └── MySQLTableCatalogFactory.java │ │ │ │ │ ├── constants │ │ │ │ │ ├── Key.java │ │ │ │ │ └── WriteModeProxy.java │ │ │ │ │ ├── converter │ │ │ │ │ ├── JdbcTypeInfoConverter.java │ │ │ │ │ ├── JdbcValueConverter.java │ │ │ │ │ ├── OracleValueConverter.java │ │ │ │ │ ├── PostgresValueConverter.java │ │ │ │ │ └── SqlServerValueConverter.java │ │ │ │ │ ├── exception │ │ │ │ │ ├── DBUtilErrorCode.java │ │ │ │ │ └── JDBCPluginErrorCode.java │ │ │ │ │ ├── extension │ │ │ │ │ └── DatabaseInterface.java │ │ │ │ │ ├── model │ │ │ │ │ ├── ClusterInfo.java │ │ │ │ │ ├── ConnectionInfo.java │ │ │ │ │ ├── DbClusterInfo.java │ │ │ │ │ ├── DbShardInfo.java │ │ │ │ │ ├── Duration.java │ │ │ │ │ ├── IndexInfo.java │ │ │ │ │ ├── InstanceInfo.java │ │ │ │ │ ├── SqlType.java │ │ │ │ │ ├── TableInfo.java │ │ │ │ │ └── oracle │ │ │ │ │ │ └── ServiceNameType.java │ │ │ │ │ ├── options │ │ │ │ │ ├── JdbcReaderOptions.java │ │ │ │ │ ├── JdbcWriterOptions.java │ │ │ │ │ ├── OracleReaderOptions.java │ │ │ │ │ ├── OracleWriterOptions.java │ │ │ │ │ ├── PostgresWriterOptions.java │ │ │ │ │ └── SqlServerWriterOptions.java │ │ │ │ │ ├── sink │ │ │ │ │ ├── JDBCOutputExtraPartitions.java │ │ │ │ │ ├── JDBCOutputFormat.java │ │ │ │ │ ├── OracleOutputFormat.java │ │ │ │ │ ├── PostgresqlOutputFormat.java │ │ │ │ │ ├── PrepareStatementSqlFunction.java │ │ │ │ │ └── SqlServerOutputFormat.java │ │ │ │ │ ├── source │ │ │ │ │ ├── JDBCInputFormat.java │ │ │ │ │ ├── OracleInputFormat.java │ │ │ │ │ ├── PostgresqlInputFormat.java │ │ │ │ │ └── SqlServerInputFormat.java │ │ │ │ │ ├── split │ │ │ │ │ ├── DbShardWithConn.java │ │ │ │ │ ├── SplitParameterInfo.java │ │ │ │ │ ├── SplitRangeInfo.java │ │ │ │ │ ├── TableRangeInfo.java │ │ │ │ │ ├── cache │ │ │ │ │ │ ├── SplitInfoCache.java │ │ │ │ │ │ └── SplitInfoNoOpCache.java │ │ │ │ │ └── strategy │ │ │ │ │ │ ├── AccurateSplitOneShardCallable.java │ │ │ │ │ │ ├── FixedLenParametersProvider.java │ │ │ │ │ │ ├── NoSplitParametersProvider.java │ │ │ │ │ │ ├── ParallelismSplitterCallable.java │ │ │ │ │ │ ├── ParameterValuesProvider.java │ │ │ │ │ │ ├── QuickSplitOneShardCallable.java │ │ │ │ │ │ └── SplitOneShardCallable.java │ │ │ │ │ └── utils │ │ │ │ │ ├── AbstractJdbcUtil.java │ │ │ │ │ ├── Db2Util.java │ │ │ │ │ ├── JDBCConnHolder.java │ │ │ │ │ ├── JdbcQueryHelper.java │ │ │ │ │ ├── MysqlUtil.java │ │ │ │ │ ├── OracleUtil.java │ │ │ │ │ ├── PostgresqlUtil.java │ │ │ │ │ ├── SqlServerUtil.java │ │ │ │ │ ├── ignore │ │ │ │ │ ├── JDBCInsertIgnoreUtil.java │ │ │ │ │ └── MysqlInsertIgnoreUtil.java │ │ │ │ │ └── upsert │ │ │ │ │ ├── JDBCUpsertUtil.java │ │ │ │ │ ├── MysqlUpsertUtil.java │ │ │ │ │ ├── OracleUpsertUtil.java │ │ │ │ │ ├── PostgreSqlUpsertUtil.java │ │ │ │ │ └── SqlServerUpsertUtil.java │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── com.bytedance.bitsail.base.catalog.TableCatalogFactory │ │ │ │ ├── bitsail-connector-legacy-jdbc.json │ │ │ │ ├── db2-type-converter.yaml │ │ │ │ ├── mysql-type-converter.yaml │ │ │ │ ├── oracle-type-converter.yaml │ │ │ │ ├── postgresql-type-converter.yaml │ │ │ │ └── sqlserver-type-converter.yaml │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── legacy │ │ │ └── jdbc │ │ │ ├── converter │ │ │ ├── BitSailJdbcTypeInfoConverterTest.java │ │ │ └── OracleValueConverterTest.java │ │ │ ├── ignore │ │ │ └── MysqInsertIgnoreUtilTest.java │ │ │ ├── model │ │ │ └── DurationTest.java │ │ │ ├── sink │ │ │ └── OracleOutputFormatTest.java │ │ │ ├── source │ │ │ ├── OracleInputFormatTest.java │ │ │ └── OracleSourceEngineConnectorTest.java │ │ │ └── utils │ │ │ └── upsert │ │ │ └── OracleUpsertUtilTest.java │ ├── bitsail-connector-kafka │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── legacy │ │ │ │ └── kafka │ │ │ │ ├── common │ │ │ │ └── KafkaFormatErrorCode.java │ │ │ │ ├── constants │ │ │ │ └── KafkaConstants.java │ │ │ │ ├── deserialization │ │ │ │ ├── CountKafkaDeserializationSchema.java │ │ │ │ ├── CountKafkaDeserializationSchemaWrapper.java │ │ │ │ ├── DeserializationSchemaFactory.java │ │ │ │ └── RowKafkaDeserializationSchema.java │ │ │ │ ├── option │ │ │ │ └── KafkaWriterOptions.java │ │ │ │ ├── sink │ │ │ │ ├── KafkaOutputFormat.java │ │ │ │ ├── KafkaProducer.java │ │ │ │ └── KafkaRecord.java │ │ │ │ └── source │ │ │ │ └── KafkaSourceFunctionDAGBuilder.java │ │ │ └── resources │ │ │ └── bitsail-connector-legacy-kafka.json │ ├── bitsail-connector-larksheet │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── legacy │ │ │ │ │ └── larksheet │ │ │ │ │ ├── api │ │ │ │ │ ├── SheetConfig.java │ │ │ │ │ ├── TokenHolder.java │ │ │ │ │ └── response │ │ │ │ │ │ ├── AppAccessTokenResponse.java │ │ │ │ │ │ ├── OpenApiBaseResponse.java │ │ │ │ │ │ ├── SheetMetaInfoResponse.java │ │ │ │ │ │ └── SheetRangeResponse.java │ │ │ │ │ ├── error │ │ │ │ │ └── LarkSheetFormatErrorCode.java │ │ │ │ │ ├── meta │ │ │ │ │ ├── SheetHeader.java │ │ │ │ │ ├── SheetInfo.java │ │ │ │ │ ├── SheetMeta.java │ │ │ │ │ └── ValueRange.java │ │ │ │ │ ├── option │ │ │ │ │ └── LarkSheetReaderOptions.java │ │ │ │ │ ├── source │ │ │ │ │ ├── LarkSheetInputFormat.java │ │ │ │ │ └── LarkSheetInputSplit.java │ │ │ │ │ └── util │ │ │ │ │ └── LarkSheetUtil.java │ │ │ └── resources │ │ │ │ └── bitsail-connector-legacy-larksheet.json │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── legacy │ │ │ └── larksheet │ │ │ └── source │ │ │ └── LarkSheetInputFormatTest.java │ ├── bitsail-connector-messagequeue │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── legacy │ │ │ └── messagequeue │ │ │ ├── serialization │ │ │ └── CountDeserializationSchema.java │ │ │ └── source │ │ │ ├── AbstractMessageQueueSourceFunctionDAGBuilder.java │ │ │ └── option │ │ │ └── BaseMessageQueueReaderOptions.java │ ├── bitsail-connector-mongodb │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── legacy │ │ │ │ │ └── mongodb │ │ │ │ │ ├── common │ │ │ │ │ ├── MongoConnConfig.java │ │ │ │ │ └── MongoConnOptions.java │ │ │ │ │ ├── constant │ │ │ │ │ └── MongoDBConstants.java │ │ │ │ │ ├── converter │ │ │ │ │ └── MongoTypeInfoConverter.java │ │ │ │ │ ├── error │ │ │ │ │ └── MongoDBPluginsErrorCode.java │ │ │ │ │ ├── option │ │ │ │ │ ├── MongoDBReaderOptions.java │ │ │ │ │ └── MongoDBWriterOptions.java │ │ │ │ │ ├── sink │ │ │ │ │ ├── MongoDBOutputFormat.java │ │ │ │ │ └── MongoValueConverter.java │ │ │ │ │ ├── source │ │ │ │ │ ├── MongoDBInputFormat.java │ │ │ │ │ └── split │ │ │ │ │ │ ├── MongoPaginatingSplitter.java │ │ │ │ │ │ ├── MongoParallelismSplitter.java │ │ │ │ │ │ ├── MongoSplitter.java │ │ │ │ │ │ ├── MongoSplitterFactory.java │ │ │ │ │ │ ├── Range.java │ │ │ │ │ │ └── TaskGroupInfo.java │ │ │ │ │ └── util │ │ │ │ │ ├── MongoDBUtil.java │ │ │ │ │ └── MongoDBWriterUtil.java │ │ │ └── resources │ │ │ │ ├── bitsail-connector-legacy-mongodb.json │ │ │ │ └── mongodb-type-converter.yaml │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── legacy │ │ │ └── mongodb │ │ │ └── source │ │ │ └── MongoTypeInfoConverterTest.java │ ├── bitsail-connector-print │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── legacy │ │ │ │ └── print │ │ │ │ ├── option │ │ │ │ └── PrintWriterOptions.java │ │ │ │ └── sink │ │ │ │ └── PrintSink.java │ │ │ └── resources │ │ │ └── bitsail-connector-legacy-print.json │ ├── bitsail-connector-redis │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── legacy │ │ │ │ │ └── redis │ │ │ │ │ ├── RedisPipelineProcessor.java │ │ │ │ │ ├── constant │ │ │ │ │ └── RedisConstants.java │ │ │ │ │ ├── core │ │ │ │ │ ├── AbstractPipelineProcessor.java │ │ │ │ │ ├── Command.java │ │ │ │ │ ├── TtlType.java │ │ │ │ │ ├── api │ │ │ │ │ │ ├── FailureHandler.java │ │ │ │ │ │ ├── PipelineProcessor.java │ │ │ │ │ │ └── SplitPolicy.java │ │ │ │ │ └── jedis │ │ │ │ │ │ ├── JedisCommand.java │ │ │ │ │ │ ├── JedisCommandDescription.java │ │ │ │ │ │ ├── JedisDataType.java │ │ │ │ │ │ └── JedisPluginErrorCode.java │ │ │ │ │ ├── error │ │ │ │ │ ├── RedisPluginErrorCode.java │ │ │ │ │ └── UnexpectedException.java │ │ │ │ │ ├── option │ │ │ │ │ └── RedisWriterOptions.java │ │ │ │ │ └── sink │ │ │ │ │ ├── JedisOutputFormat.java │ │ │ │ │ └── RedisOutputFormat.java │ │ │ └── resources │ │ │ │ └── bitsail-connector-legacy-redis.json │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── legacy │ │ │ └── redis │ │ │ └── sink │ │ │ ├── JedisOutputFormatTest.java │ │ │ └── RedisOutputFormatTest.java │ ├── bitsail-connector-rocketmq │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── legacy │ │ │ └── rocketmq │ │ │ ├── config │ │ │ └── RocketMQSinkConfig.java │ │ │ ├── constant │ │ │ └── RocketMQConstants.java │ │ │ ├── error │ │ │ └── RocketMQPluginErrorCode.java │ │ │ ├── option │ │ │ └── RocketMQWriterOptions.java │ │ │ └── sink │ │ │ ├── HashQueueSelector.java │ │ │ ├── RocketMQOutputFormat.java │ │ │ ├── RocketMQProducer.java │ │ │ └── format │ │ │ ├── RocketMQSerializationFactory.java │ │ │ ├── RocketMQSerializationSchema.java │ │ │ ├── RocketMQSinkFormat.java │ │ │ └── json │ │ │ └── JsonSerializationSchema.java │ ├── bitsail-connector-streamingfile │ │ ├── bitsail-connector-streamingfile-common │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ ├── main │ │ │ │ └── java │ │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── legacy │ │ │ │ │ └── streamingfile │ │ │ │ │ └── common │ │ │ │ │ ├── checkpoint │ │ │ │ │ ├── AbstractSnapshotWrapper.java │ │ │ │ │ └── StreamingFileSinkSnapshotWrapper.java │ │ │ │ │ ├── constants │ │ │ │ │ └── StreamingFileSystemMetricsNames.java │ │ │ │ │ ├── datasink │ │ │ │ │ ├── common │ │ │ │ │ │ ├── PartitionInfo.java │ │ │ │ │ │ ├── PartitionMapping.java │ │ │ │ │ │ ├── PartitionType.java │ │ │ │ │ │ └── TimePartitionGroup.java │ │ │ │ │ └── file │ │ │ │ │ │ ├── AbstractPartitionCommitter.java │ │ │ │ │ │ └── AbstractPartitionComputer.java │ │ │ │ │ ├── decoder │ │ │ │ │ └── SkipMessageDecoder.java │ │ │ │ │ ├── extractor │ │ │ │ │ ├── AbstractEventTimeExtractor.java │ │ │ │ │ ├── CustomEventTimeExtractor.java │ │ │ │ │ ├── EventTimeExtractor.java │ │ │ │ │ ├── EventTimeExtractorFactory.java │ │ │ │ │ ├── TextEventTimeExtractor.java │ │ │ │ │ └── binlog │ │ │ │ │ │ ├── DebeziumEventTimeExtractor.java │ │ │ │ │ │ └── MysqlBinlogEventTimeExtractor.java │ │ │ │ │ ├── filestate │ │ │ │ │ ├── DummyFileStateCollector.java │ │ │ │ │ ├── FileState.java │ │ │ │ │ ├── FileStateCollector.java │ │ │ │ │ └── FileStateFactory.java │ │ │ │ │ ├── filesystem │ │ │ │ │ ├── DynamicPartitionWriter.java │ │ │ │ │ ├── FileSystemFactory.java │ │ │ │ │ ├── OutputFormatFactory.java │ │ │ │ │ ├── PartitionComputer.java │ │ │ │ │ ├── PartitionPathUtils.java │ │ │ │ │ ├── PartitionTempFileManager.java │ │ │ │ │ ├── PartitionWriter.java │ │ │ │ │ ├── PartitionWriterFactory.java │ │ │ │ │ ├── RowPartitionComputer.java │ │ │ │ │ ├── SingleDirectoryWriter.java │ │ │ │ │ ├── StreamingJobCommitStatus.java │ │ │ │ │ ├── TableMetaStoreFactory.java │ │ │ │ │ ├── aggregate │ │ │ │ │ │ ├── GlobalAggregateResult.java │ │ │ │ │ │ ├── PartitionCommitFunction.java │ │ │ │ │ │ ├── StreamingJobAccumulator.java │ │ │ │ │ │ └── TaskSnapshotMeta.java │ │ │ │ │ ├── directory │ │ │ │ │ │ ├── DefaultTimeManager.java │ │ │ │ │ │ └── PartitionDirectoryManager.java │ │ │ │ │ ├── partitionstrategy │ │ │ │ │ │ ├── AbstractPartitionCommitFunction.java │ │ │ │ │ │ ├── AbstractPartitionStateProcessFunction.java │ │ │ │ │ │ ├── PartitionStrategyFactory.java │ │ │ │ │ │ ├── partitionfirst │ │ │ │ │ │ │ ├── PartitionCache.java │ │ │ │ │ │ │ ├── PartitionFirstCommitFunction.java │ │ │ │ │ │ │ └── PartitionFirstStateProcessFunction.java │ │ │ │ │ │ └── partitionlast │ │ │ │ │ │ │ ├── PartitionLastCommitFunction.java │ │ │ │ │ │ │ └── PartitionLastStateProcessFunction.java │ │ │ │ │ ├── rollingpolicies │ │ │ │ │ │ ├── DefaultRollingPolicy.java │ │ │ │ │ │ ├── PartFileInfo.java │ │ │ │ │ │ └── RollingPolicy.java │ │ │ │ │ ├── schema │ │ │ │ │ │ ├── FileSystemMeta.java │ │ │ │ │ │ └── FileSystemMetaManager.java │ │ │ │ │ └── syncer │ │ │ │ │ │ └── AbstractMetaSyncer.java │ │ │ │ │ ├── option │ │ │ │ │ ├── FileSystemCommonOptions.java │ │ │ │ │ ├── FileSystemParseOptions.java │ │ │ │ │ ├── FileSystemSchemaOptions.java │ │ │ │ │ └── FileSystemSinkOptions.java │ │ │ │ │ ├── parser │ │ │ │ │ ├── AbstractRowBuilder.java │ │ │ │ │ ├── BytesParseFactory.java │ │ │ │ │ └── FileSystemRowBuilder.java │ │ │ │ │ ├── tools │ │ │ │ │ ├── FileNameUtils.java │ │ │ │ │ ├── MetricsFactory.java │ │ │ │ │ ├── PartitionUtils.java │ │ │ │ │ └── RetryUtils.java │ │ │ │ │ └── validator │ │ │ │ │ └── StreamingFileSystemValidator.java │ │ │ │ └── test │ │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── legacy │ │ │ │ └── streamingfile │ │ │ │ └── common │ │ │ │ └── decoder │ │ │ │ └── MessageDecodeCompositorTest.java │ │ ├── bitsail-connector-streamingfile-core │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── main │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── legacy │ │ │ │ │ └── streamingfile │ │ │ │ │ ├── core │ │ │ │ │ ├── dirty │ │ │ │ │ │ ├── DirtyHdfsTextOutputFormat.java │ │ │ │ │ │ ├── MultiFileHdfsDirtyCollector.java │ │ │ │ │ │ └── SingleFileHdfsDirtyCollector.java │ │ │ │ │ └── sink │ │ │ │ │ │ ├── CommitFileStatus.java │ │ │ │ │ │ ├── DefaultWatermarkEmitter.java │ │ │ │ │ │ ├── FileSystemCommitter.java │ │ │ │ │ │ ├── PartitionLoader.java │ │ │ │ │ │ ├── StreamingFileSystemSink.java │ │ │ │ │ │ ├── WatermarkEmitter.java │ │ │ │ │ │ ├── filestate │ │ │ │ │ │ ├── AbstractFileState.java │ │ │ │ │ │ ├── FileStateBasedOnName.java │ │ │ │ │ │ ├── FileStateBasedOnTimestamp.java │ │ │ │ │ │ ├── FileStateFactory.java │ │ │ │ │ │ ├── FileStateHelper.java │ │ │ │ │ │ └── FileStateSerializer.java │ │ │ │ │ │ ├── format │ │ │ │ │ │ ├── AbstractFileSystemFactory.java │ │ │ │ │ │ ├── hdfs │ │ │ │ │ │ │ ├── AbstractHdfsOutputFormat.java │ │ │ │ │ │ │ ├── Crc32C.java │ │ │ │ │ │ │ ├── HdfsBinlogOutputFormat.java │ │ │ │ │ │ │ ├── HdfsFileSystemFactory.java │ │ │ │ │ │ │ ├── HdfsMetaStoreFactory.java │ │ │ │ │ │ │ ├── HdfsPBOutputFormat.java │ │ │ │ │ │ │ ├── HdfsPartitionCommitter.java │ │ │ │ │ │ │ ├── HdfsPartitionComputer.java │ │ │ │ │ │ │ ├── HdfsSequenceOutputFormat.java │ │ │ │ │ │ │ ├── HdfsTFRecordOutputFormat.java │ │ │ │ │ │ │ └── HdfsTextOutputFormat.java │ │ │ │ │ │ └── hive │ │ │ │ │ │ │ ├── HiveFileSystemFactory.java │ │ │ │ │ │ │ ├── HivePartitionCommitter.java │ │ │ │ │ │ │ ├── HivePartitionComputer.java │ │ │ │ │ │ │ ├── HiveTableMetaStoreFactory.java │ │ │ │ │ │ │ └── HiveWritableOutputFormat.java │ │ │ │ │ │ ├── schema │ │ │ │ │ │ ├── FileSystemSchemaFactory.java │ │ │ │ │ │ ├── HiveFileSystemMetaManager.java │ │ │ │ │ │ └── HiveMeta.java │ │ │ │ │ │ └── syncer │ │ │ │ │ │ └── HiveMetaInfoSyncer.java │ │ │ │ │ └── sink │ │ │ │ │ └── FileSystemSinkFunctionDAGBuilder.java │ │ │ │ └── resources │ │ │ │ └── bitsail-connector-legacy-streamingfile.json │ │ └── pom.xml │ └── pom.xml ├── connector-assert │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── assertion │ │ │ │ └── sink │ │ │ │ ├── AssertSink.java │ │ │ │ ├── AssertWriter.java │ │ │ │ ├── AssertWriterCommitter.java │ │ │ │ ├── constants │ │ │ │ ├── AssertConstants.java │ │ │ │ └── AssertRuleType.java │ │ │ │ ├── error │ │ │ │ └── AssertErrorCode.java │ │ │ │ ├── executor │ │ │ │ └── AssertRuleExecutor.java │ │ │ │ ├── option │ │ │ │ └── AssertWriterOptions.java │ │ │ │ └── rule │ │ │ │ ├── AssertColumnRule.java │ │ │ │ ├── AssertRuleParser.java │ │ │ │ └── AssertSimpleRule.java │ │ └── resources │ │ │ └── bitsail-connector-unified-assert.json │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── assertion │ │ │ └── sink │ │ │ ├── executor │ │ │ └── AssertRuleExecutorTest.java │ │ │ └── rule │ │ │ └── AssertRuleParserTest.java │ │ └── resources │ │ └── assert_sink.json ├── connector-base │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── connector │ │ └── base │ │ └── source │ │ ├── SimpleSourceBase.java │ │ ├── SimpleSourceReaderBase.java │ │ ├── SimpleSourceSplit.java │ │ ├── SimpleSourceSplitCoordinator.java │ │ ├── SimpleSourceState.java │ │ └── split │ │ └── SplitAssigner.java ├── connector-cdc │ ├── connector-cdc-base │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── cdc │ │ │ │ ├── constant │ │ │ │ └── BinlogReaderConstants.java │ │ │ │ ├── error │ │ │ │ └── BinlogReaderErrorCode.java │ │ │ │ ├── model │ │ │ │ ├── ClusterInfo.java │ │ │ │ └── ConnectionInfo.java │ │ │ │ ├── option │ │ │ │ └── BinlogReaderOptions.java │ │ │ │ ├── schema │ │ │ │ └── DebeziumDeserializationFactory.java │ │ │ │ └── source │ │ │ │ ├── BaseCDCSource.java │ │ │ │ ├── coordinator │ │ │ │ ├── CDCSourceSplitCoordinator.java │ │ │ │ └── state │ │ │ │ │ ├── AssignmentStateSerializer.java │ │ │ │ │ ├── BaseAssignmentState.java │ │ │ │ │ ├── BinlogAssignmentState.java │ │ │ │ │ └── SnapshotAssignmentState.java │ │ │ │ ├── event │ │ │ │ ├── BinlogCompleteAckEvent.java │ │ │ │ ├── BinlogCompleteEvent.java │ │ │ │ └── BinlogStopReadEvent.java │ │ │ │ ├── offset │ │ │ │ ├── BinlogOffset.java │ │ │ │ └── BinlogOffsetType.java │ │ │ │ ├── reader │ │ │ │ ├── BaseCDCSourceReader.java │ │ │ │ └── BinlogSplitReader.java │ │ │ │ └── split │ │ │ │ ├── BaseCDCSplit.java │ │ │ │ ├── BaseSplitSerializer.java │ │ │ │ ├── BinlogSplit.java │ │ │ │ ├── SnapshotSplit.java │ │ │ │ └── SplitType.java │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── cdc │ │ │ └── source │ │ │ ├── coordinator │ │ │ ├── CDCSourceSplitCoordinatorTests.java │ │ │ └── state │ │ │ │ └── AssignmentStateSerializerTest.java │ │ │ └── offset │ │ │ └── BinlogOffsetTests.java │ ├── connector-cdc-mysql │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── cdc │ │ │ │ │ └── mysql │ │ │ │ │ └── source │ │ │ │ │ ├── MysqlCDCSource.java │ │ │ │ │ ├── config │ │ │ │ │ └── MysqlConfig.java │ │ │ │ │ ├── constant │ │ │ │ │ └── MysqlConstant.java │ │ │ │ │ ├── debezium │ │ │ │ │ ├── DebeziumHelper.java │ │ │ │ │ ├── InMemoryDatabaseHistory.java │ │ │ │ │ ├── MySqlEventMetadataProvider.java │ │ │ │ │ └── MysqlBinlogSplitReader.java │ │ │ │ │ ├── reader │ │ │ │ │ └── MysqlCDCSourceReader.java │ │ │ │ │ ├── schema │ │ │ │ │ ├── MySqlSchemaFetcher.java │ │ │ │ │ ├── SchemaUtils.java │ │ │ │ │ └── TableChangeConverter.java │ │ │ │ │ ├── split │ │ │ │ │ └── MysqlSplitSerializer.java │ │ │ │ │ └── sql │ │ │ │ │ ├── MysqlFieldDescriber.java │ │ │ │ │ └── MysqlTableDescriber.java │ │ │ └── resources │ │ │ │ └── bitsail-connector-unified-mysqlcdc.json │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── cdc │ │ │ │ └── mysql │ │ │ │ └── source │ │ │ │ ├── MockConnectionsTest.java │ │ │ │ ├── config │ │ │ │ └── MysqlConfigTest.java │ │ │ │ ├── container │ │ │ │ └── MySQLContainerMariadbAdapter.java │ │ │ │ ├── debezium │ │ │ │ ├── DebeziumHelperTest.java │ │ │ │ └── InMemoryDatabaseHistoryTest.java │ │ │ │ └── sql │ │ │ │ ├── MysqlFieldDescriberTest.java │ │ │ │ └── MysqlTableDescriberTest.java │ │ │ └── resources │ │ │ ├── logback-test.xml │ │ │ └── scripts │ │ │ └── insert_test.sql │ ├── connector-cdc-sqlserver │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── connector │ │ │ │ │ └── cdc │ │ │ │ │ └── sqlserver │ │ │ │ │ └── source │ │ │ │ │ ├── SqlServerCDCSource.java │ │ │ │ │ ├── config │ │ │ │ │ └── SqlServerConfig.java │ │ │ │ │ ├── reader │ │ │ │ │ ├── SqlServerBinlogSplitReader.java │ │ │ │ │ ├── SqlServerCDCSourceReader.java │ │ │ │ │ └── SqlServerEventMetadataProvider.java │ │ │ │ │ ├── split │ │ │ │ │ └── SqlServerSplitSerializer.java │ │ │ │ │ └── util │ │ │ │ │ └── DebeziumUtils.java │ │ │ └── resources │ │ │ │ └── bitsail-connector-unified-sqlserver.json │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── cdc │ │ │ │ └── sqlserver │ │ │ │ └── source │ │ │ │ └── reader │ │ │ │ └── SqlServerBinlogSplitReaderTest.java │ │ │ └── resources │ │ │ ├── container-license-acceptance.txt │ │ │ └── scripts │ │ │ └── test_insert_full_type.sql │ └── pom.xml ├── connector-clickhouse │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── clickhouse │ │ │ ├── constant │ │ │ └── ClickhouseConstants.java │ │ │ ├── error │ │ │ └── ClickhouseErrorCode.java │ │ │ ├── option │ │ │ └── ClickhouseReaderOptions.java │ │ │ ├── source │ │ │ ├── ClickhouseSource.java │ │ │ ├── reader │ │ │ │ ├── ClickhouseRowDeserializer.java │ │ │ │ └── ClickhouseSourceReader.java │ │ │ └── split │ │ │ │ ├── ClickhouseSourceSplit.java │ │ │ │ ├── coordinator │ │ │ │ └── ClickhouseSourceSplitCoordinator.java │ │ │ │ └── strategy │ │ │ │ └── SimpleDivideSplitConstructor.java │ │ │ └── util │ │ │ ├── ClickhouseConnectionHolder.java │ │ │ └── ClickhouseJdbcUtils.java │ │ └── resources │ │ ├── bitsail-connector-unified-clickhouse.json │ │ └── clickhouse-type-converter.yaml ├── connector-doris │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── doris │ │ │ │ ├── DorisConnectionHolder.java │ │ │ │ ├── committer │ │ │ │ ├── DorisCommittable.java │ │ │ │ ├── DorisCommittableSerializer.java │ │ │ │ └── DorisCommitter.java │ │ │ │ ├── config │ │ │ │ ├── DorisExecutionOptions.java │ │ │ │ └── DorisOptions.java │ │ │ │ ├── constant │ │ │ │ └── DorisConstants.java │ │ │ │ ├── converter │ │ │ │ └── DorisRowConverter.java │ │ │ │ ├── error │ │ │ │ └── DorisErrorCode.java │ │ │ │ ├── http │ │ │ │ ├── HttpPutBuilder.java │ │ │ │ ├── HttpUtil.java │ │ │ │ ├── ResponseUtil.java │ │ │ │ └── model │ │ │ │ │ └── RespContent.java │ │ │ │ ├── option │ │ │ │ └── DorisWriterOptions.java │ │ │ │ ├── partition │ │ │ │ ├── DorisPartition.java │ │ │ │ ├── DorisPartitionHelper.java │ │ │ │ └── DorisPartitionManager.java │ │ │ │ ├── rest │ │ │ │ ├── RestService.java │ │ │ │ └── model │ │ │ │ │ └── Backend.java │ │ │ │ ├── serialize │ │ │ │ └── DorisRowSerializer.java │ │ │ │ ├── sink │ │ │ │ ├── DorisSink.java │ │ │ │ ├── DorisWriter.java │ │ │ │ ├── DorisWriterState.java │ │ │ │ ├── ddl │ │ │ │ │ └── DorisSchemaManagerGenerator.java │ │ │ │ ├── label │ │ │ │ │ └── LabelGenerator.java │ │ │ │ ├── proxy │ │ │ │ │ ├── AbstractDorisWriteModeProxy.java │ │ │ │ │ ├── DorisReplaceProxy.java │ │ │ │ │ └── DorisUpsertProxy.java │ │ │ │ ├── record │ │ │ │ │ ├── RecordBuffer.java │ │ │ │ │ └── RecordStream.java │ │ │ │ └── streamload │ │ │ │ │ ├── DorisStreamLoad.java │ │ │ │ │ └── LoadStatus.java │ │ │ │ └── typeinfo │ │ │ │ └── DorisDataType.java │ │ └── resources │ │ │ ├── bitsail-connector-unified-doris.json │ │ │ └── doris-type-converter.yaml │ │ └── test │ │ └── java │ │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── connector │ │ └── doris │ │ ├── converter │ │ └── DorisRowConverterTest.java │ │ └── serialize │ │ └── DorisRowSerializerTest.java ├── connector-druid │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── druid │ │ │ │ ├── error │ │ │ │ └── DruidErrorCode.java │ │ │ │ ├── option │ │ │ │ └── DruidWriterOptions.java │ │ │ │ └── sink │ │ │ │ ├── DruidSink.java │ │ │ │ └── DruidWriter.java │ │ └── resources │ │ │ ├── bitsail-connector-unified-druid.json │ │ │ └── druid-type-converter.yaml │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── druid │ │ │ └── sink │ │ │ ├── DruidSinkTest.java │ │ │ └── DruidWriterTest.java │ │ └── resources │ │ ├── druid_writer.json │ │ └── expectedTask.json ├── connector-elasticsearch │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── elasticsearch │ │ │ │ ├── constants │ │ │ │ └── Elasticsearchs.java │ │ │ │ ├── exception │ │ │ │ └── ElasticsearchErrorCode.java │ │ │ │ ├── format │ │ │ │ ├── DefaultRowSerializationSchema.java │ │ │ │ ├── ElasticsearchRowSerializationSchema.java │ │ │ │ └── extractor │ │ │ │ │ ├── DefaultFieldExtractor.java │ │ │ │ │ └── DefaultValueExtractor.java │ │ │ │ ├── option │ │ │ │ └── ElasticsearchOptions.java │ │ │ │ ├── sink │ │ │ │ ├── ElasticsearchSink.java │ │ │ │ ├── ElasticsearchWriter.java │ │ │ │ ├── listener │ │ │ │ │ └── DefaultBulkListener.java │ │ │ │ └── sender │ │ │ │ │ └── ElasticsearchSender.java │ │ │ │ └── utils │ │ │ │ └── ElasticsearchUtils.java │ │ └── resources │ │ │ ├── bitsail-connector-unified-elasticsearch.json │ │ │ └── elasticsearch-type-converter.yaml │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── elasticsearch │ │ │ └── ElasticsearchTypeConverterTest.java │ │ └── resources │ │ ├── es_doc_parameter_test.json │ │ └── es_rest_parameter_test.json ├── connector-fake │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── fake │ │ │ ├── option │ │ │ └── FakeReaderOptions.java │ │ │ └── source │ │ │ ├── FakeRowGenerator.java │ │ │ ├── FakeSource.java │ │ │ └── FakeSourceReader.java │ │ └── resources │ │ └── bitsail-connector-unified-fake.json ├── connector-ftp │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── ftp │ │ │ ├── core │ │ │ ├── FtpConstants.java │ │ │ ├── client │ │ │ │ ├── FtpHandler.java │ │ │ │ ├── FtpHandlerFactory.java │ │ │ │ ├── IFtpHandler.java │ │ │ │ └── SftpHandler.java │ │ │ └── config │ │ │ │ └── FtpConfig.java │ │ │ ├── error │ │ │ └── FtpErrorCode.java │ │ │ ├── option │ │ │ └── FtpReaderOptions.java │ │ │ └── source │ │ │ ├── FtpSource.java │ │ │ ├── reader │ │ │ ├── DeserializationSchemaFactory.java │ │ │ └── FtpSourceReader.java │ │ │ └── split │ │ │ ├── FtpSourceSplit.java │ │ │ └── coordinator │ │ │ └── FtpSourceSplitCoordinator.java │ │ └── resources │ │ ├── bitsail-connector-unified-ftp.json │ │ └── ftp-type-converter.yaml ├── connector-hbase │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── hbase │ │ │ │ ├── HBaseHelper.java │ │ │ │ ├── auth │ │ │ │ ├── KerberosAuthenticator.java │ │ │ │ └── KerberosFileHandler.java │ │ │ │ ├── constant │ │ │ │ ├── HBaseConstants.java │ │ │ │ └── NullMode.java │ │ │ │ ├── error │ │ │ │ └── HBasePluginErrorCode.java │ │ │ │ ├── format │ │ │ │ └── HBaseDeserializationFormat.java │ │ │ │ ├── option │ │ │ │ ├── HBaseReaderOptions.java │ │ │ │ └── HBaseWriterOptions.java │ │ │ │ ├── sink │ │ │ │ ├── HBaseSink.java │ │ │ │ ├── HBaseWriter.java │ │ │ │ └── function │ │ │ │ │ ├── ConstantFunction.java │ │ │ │ │ ├── FunctionFactory.java │ │ │ │ │ ├── FunctionParser.java │ │ │ │ │ ├── FunctionTree.java │ │ │ │ │ ├── IFunction.java │ │ │ │ │ ├── Md5Function.java │ │ │ │ │ └── StringFunction.java │ │ │ │ └── source │ │ │ │ ├── HBaseSource.java │ │ │ │ ├── reader │ │ │ │ └── HBaseSourceReader.java │ │ │ │ └── split │ │ │ │ ├── HBaseSourceSplit.java │ │ │ │ ├── coordinator │ │ │ │ └── HBaseSourceSplitCoordinator.java │ │ │ │ └── strategy │ │ │ │ └── SimpleDivideSplitConstructor.java │ │ └── resources │ │ │ ├── bitsail-connector-unified-hbase.json │ │ │ └── hbase-type-converter.yaml │ │ └── test │ │ └── java │ │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── connector │ │ └── hbase │ │ └── sink │ │ └── function │ │ └── RowKeyFunctionTest.java ├── connector-kafka │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── kafka │ │ │ ├── common │ │ │ └── KafkaErrorCode.java │ │ │ ├── constants │ │ │ ├── FormatType.java │ │ │ └── KafkaConstants.java │ │ │ ├── discoverer │ │ │ ├── DefaultPartitionDiscoverer.java │ │ │ ├── FixedTopicPartitionDiscoverer.java │ │ │ └── PartitionDiscoverer.java │ │ │ ├── format │ │ │ ├── DefaultRowSerializationSchema.java │ │ │ ├── ProducerRecordRowSerializationSchema.java │ │ │ └── ProducerRecordSerializationSchemaFactory.java │ │ │ ├── option │ │ │ └── KafkaOptions.java │ │ │ └── sink │ │ │ ├── KafkaSender.java │ │ │ ├── KafkaSink.java │ │ │ ├── KafkaWriter.java │ │ │ └── callback │ │ │ └── CallbackWrapper.java │ │ └── resources │ │ └── bitsail-connector-unified-kafka.json ├── connector-kudu │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── kudu │ │ │ │ ├── core │ │ │ │ ├── KuduConstants.java │ │ │ │ ├── KuduFactory.java │ │ │ │ └── config │ │ │ │ │ ├── KuduClientConfig.java │ │ │ │ │ └── KuduSessionConfig.java │ │ │ │ ├── error │ │ │ │ └── KuduErrorCode.java │ │ │ │ ├── option │ │ │ │ ├── KuduReaderOptions.java │ │ │ │ └── KuduWriterOptions.java │ │ │ │ ├── sink │ │ │ │ ├── KuduRowBuilder.java │ │ │ │ ├── KuduSink.java │ │ │ │ └── KuduWriter.java │ │ │ │ ├── source │ │ │ │ ├── KuduSource.java │ │ │ │ ├── reader │ │ │ │ │ ├── KuduRowDeserializer.java │ │ │ │ │ └── KuduSourceReader.java │ │ │ │ └── split │ │ │ │ │ ├── KuduSourceSplit.java │ │ │ │ │ ├── coordinator │ │ │ │ │ └── KuduSourceSplitCoordinator.java │ │ │ │ │ └── strategy │ │ │ │ │ └── SimpleDivideSplitConstructor.java │ │ │ │ ├── type │ │ │ │ └── KuduTypeConverter.java │ │ │ │ └── util │ │ │ │ ├── KuduPredicateUtil.java │ │ │ │ └── KuduSchemaUtils.java │ │ └── resources │ │ │ ├── bitsail-connector-unified-kudu.json │ │ │ └── kudu-type-converter.yaml │ │ └── test │ │ └── java │ │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── connector │ │ └── kudu │ │ ├── sink │ │ └── KuduRowBuilderTest.java │ │ ├── source │ │ └── split │ │ │ └── KuduSourceSplitTest.java │ │ └── util │ │ ├── KuduPredicateUtilTest.java │ │ └── KuduSchemaUtilsTest.java ├── connector-larksheet │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── larksheet │ │ │ │ ├── api │ │ │ │ ├── SheetConfig.java │ │ │ │ ├── TokenHolder.java │ │ │ │ └── response │ │ │ │ │ ├── AppAccessTokenResponse.java │ │ │ │ │ ├── OpenApiBaseResponse.java │ │ │ │ │ ├── SheetMetaInfoResponse.java │ │ │ │ │ └── SheetRangeResponse.java │ │ │ │ ├── constant │ │ │ │ └── LarkSheetConstant.java │ │ │ │ ├── error │ │ │ │ └── LarkSheetFormatErrorCode.java │ │ │ │ ├── meta │ │ │ │ ├── SheetHeader.java │ │ │ │ ├── SheetInfo.java │ │ │ │ ├── SheetMeta.java │ │ │ │ └── ValueRange.java │ │ │ │ ├── option │ │ │ │ └── LarkSheetReaderOptions.java │ │ │ │ ├── source │ │ │ │ ├── LarkSheetSource.java │ │ │ │ ├── coordinate │ │ │ │ │ └── LarkSheetSourceSplitCoordinator.java │ │ │ │ ├── reader │ │ │ │ │ └── LarkSheetReader.java │ │ │ │ └── split │ │ │ │ │ ├── LarkSheetSplit.java │ │ │ │ │ └── strategy │ │ │ │ │ └── SimpleDivideSplitConstructor.java │ │ │ │ └── util │ │ │ │ ├── Feature.java │ │ │ │ └── LarkSheetUtil.java │ │ └── resources │ │ │ └── bitsail-connector-unified-larksheet.json │ │ └── test │ │ └── java │ │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── connector │ │ └── larksheet │ │ └── util │ │ └── LarkSheetUtilTest.java ├── connector-localfilesystem │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── localfilesystem │ │ │ │ ├── core │ │ │ │ └── config │ │ │ │ │ └── LocalFileSystemConfig.java │ │ │ │ ├── error │ │ │ │ └── LocalFileSystemErrorCode.java │ │ │ │ ├── option │ │ │ │ └── LocalFileSystemReaderOptions.java │ │ │ │ ├── reader │ │ │ │ ├── DeserializationSchemaFactory.java │ │ │ │ └── LocalFileSystemSourceReader.java │ │ │ │ └── source │ │ │ │ └── LocalFileSystemSource.java │ │ └── resources │ │ │ └── bitsail-connector-unified-localfilesystem.json │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── localfilesystem │ │ │ └── reader │ │ │ ├── LocalFileSystemSourceReaderTest.java │ │ │ └── SimpleSourcePipeLine.java │ │ └── resources │ │ ├── data │ │ ├── csv │ │ │ └── test.csv │ │ └── json │ │ │ └── test.json │ │ └── scripts │ │ ├── local-csv-to-print.json │ │ └── local-json-to-print.json ├── connector-mongodb │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── mongodb │ │ │ ├── config │ │ │ ├── MongoDBConnConfig.java │ │ │ └── MongoDBConnOptions.java │ │ │ ├── constant │ │ │ ├── MongoDBConstants.java │ │ │ └── MongoDBSplitMode.java │ │ │ ├── converter │ │ │ └── MongoDBTypeInfoConverter.java │ │ │ ├── error │ │ │ └── MongoDBErrorCode.java │ │ │ ├── option │ │ │ └── MongoDBReaderOptions.java │ │ │ ├── source │ │ │ ├── MongoDBSource.java │ │ │ ├── reader │ │ │ │ ├── MongoDBRowDeserializer.java │ │ │ │ └── MongoDBSourceReader.java │ │ │ └── split │ │ │ │ ├── MongoDBSourceSplit.java │ │ │ │ ├── MongoDBSourceSplitCoordinator.java │ │ │ │ ├── MongoDBSplitter.java │ │ │ │ ├── MongoDBSplitter4Paginating.java │ │ │ │ ├── MongoDBSplitter4Parallelism.java │ │ │ │ └── MongoDBSplitterFactory.java │ │ │ └── util │ │ │ └── MongoDBUtils.java │ │ └── resources │ │ ├── bitsail-connector-unified-mongodb.json │ │ └── mongodb-type-converter.yaml ├── connector-print │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── print │ │ │ └── sink │ │ │ ├── PrintErrorCode.java │ │ │ ├── PrintSink.java │ │ │ ├── PrintWriter.java │ │ │ ├── PrintWriterCommitter.java │ │ │ └── option │ │ │ └── PrintWriterOptions.java │ │ └── resources │ │ └── bitsail-connector-unified-print.json ├── connector-redis │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── redis │ │ │ ├── constant │ │ │ └── RedisConstants.java │ │ │ ├── core │ │ │ ├── AbstractPipelineProcessor.java │ │ │ ├── Command.java │ │ │ ├── TtlType.java │ │ │ ├── api │ │ │ │ ├── FailureHandler.java │ │ │ │ ├── PipelineProcessor.java │ │ │ │ └── SplitPolicy.java │ │ │ └── jedis │ │ │ │ ├── JedisCommand.java │ │ │ │ └── JedisDataType.java │ │ │ ├── error │ │ │ ├── JedisPluginErrorCode.java │ │ │ ├── RedisPluginErrorCode.java │ │ │ └── RedisUnexpectedException.java │ │ │ ├── option │ │ │ └── RedisWriterOptions.java │ │ │ └── sink │ │ │ ├── RedisPipelineProcessor.java │ │ │ ├── RedisSink.java │ │ │ └── RedisWriter.java │ │ └── resources │ │ └── bitsail-connector-unified-redis.json ├── connector-rocketmq │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── connector │ │ │ │ └── rocketmq │ │ │ │ ├── constants │ │ │ │ ├── OptionalProducerConfig.java │ │ │ │ └── RocketMQConstants.java │ │ │ │ ├── error │ │ │ │ └── RocketMQErrorCode.java │ │ │ │ ├── format │ │ │ │ ├── JsonRocketMQSerializationSchema.java │ │ │ │ ├── RocketMQDeserializationSchema.java │ │ │ │ ├── RocketMQSerializationFactory.java │ │ │ │ └── RocketMQSerializationSchema.java │ │ │ │ ├── option │ │ │ │ ├── RocketMQSourceOptions.java │ │ │ │ └── RocketMQWriterOptions.java │ │ │ │ ├── sink │ │ │ │ ├── HashQueueSelector.java │ │ │ │ ├── RocketMQProducer.java │ │ │ │ ├── RocketMQSink.java │ │ │ │ ├── RocketMQWriter.java │ │ │ │ ├── config │ │ │ │ │ └── RocketMQSinkConfig.java │ │ │ │ └── format │ │ │ │ │ └── RocketMQSinkFormat.java │ │ │ │ ├── source │ │ │ │ ├── RocketMQSource.java │ │ │ │ ├── coordinator │ │ │ │ │ ├── FairRocketMQSplitAssigner.java │ │ │ │ │ └── RocketMQSourceSplitCoordinator.java │ │ │ │ ├── reader │ │ │ │ │ └── RocketMQSourceReader.java │ │ │ │ └── split │ │ │ │ │ ├── RocketMQSplit.java │ │ │ │ │ └── RocketMQState.java │ │ │ │ └── utils │ │ │ │ └── RocketMQUtils.java │ │ └── resources │ │ │ └── bitsail-connector-unified-rocketmq.json │ │ └── test │ │ └── java │ │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── connector │ │ └── rocketmq │ │ └── sink │ │ └── HashQueueSelectorTest.java ├── connector-selectdb │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── connector │ │ │ └── selectdb │ │ │ ├── committer │ │ │ ├── SelectdbCommittable.java │ │ │ ├── SelectdbCommittableSerializer.java │ │ │ └── SelectdbCommitter.java │ │ │ ├── config │ │ │ ├── SelectdbExecutionOptions.java │ │ │ ├── SelectdbOptions.java │ │ │ └── SelectdbWriterOptions.java │ │ │ ├── constant │ │ │ └── SelectdbConstants.java │ │ │ ├── copyinto │ │ │ └── CopySQLBuilder.java │ │ │ ├── error │ │ │ └── SelectdbErrorCode.java │ │ │ ├── http │ │ │ ├── HttpPostBuilder.java │ │ │ ├── HttpPutBuilder.java │ │ │ ├── HttpUtil.java │ │ │ ├── ResponseUtil.java │ │ │ └── model │ │ │ │ ├── BaseResponse.java │ │ │ │ └── CopyIntoResp.java │ │ │ ├── serialize │ │ │ ├── SelectdbRowSerializer.java │ │ │ └── converter │ │ │ │ └── SelectdbRowConverter.java │ │ │ ├── sink │ │ │ ├── SelectdbSink.java │ │ │ ├── SelectdbWriter.java │ │ │ ├── SelectdbWriterState.java │ │ │ ├── label │ │ │ │ └── LabelGenerator.java │ │ │ ├── proxy │ │ │ │ ├── AbstractSelectdbWriteModeProxy.java │ │ │ │ └── SelectdbUpsertProxy.java │ │ │ ├── record │ │ │ │ ├── RecordBuffer.java │ │ │ │ └── RecordStream.java │ │ │ └── uploadload │ │ │ │ ├── LoadStatus.java │ │ │ │ └── SelectdbUploadLoad.java │ │ │ └── typeinfo │ │ │ └── SelectdbDataType.java │ │ └── resources │ │ ├── bitsail-connector-unified-selectdb.json │ │ └── selectdb-type-converter.yaml └── pom.xml ├── bitsail-cores ├── bitsail-core-api │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── core │ │ │ │ └── api │ │ │ │ ├── command │ │ │ │ └── CoreCommandArgs.java │ │ │ │ ├── interceptor │ │ │ │ ├── ConfigInterceptor.java │ │ │ │ └── ConfigInterceptorHelper.java │ │ │ │ ├── parser │ │ │ │ ├── ConfigurationHelper.java │ │ │ │ ├── ConfigurationParser.java │ │ │ │ └── DefaultConfigurationParser.java │ │ │ │ └── program │ │ │ │ ├── Program.java │ │ │ │ ├── UnifiedProgram.java │ │ │ │ └── factory │ │ │ │ └── ProgramDAGBuilderFactory.java │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── com.bytedance.bitsail.core.api.parser.ConfigurationParser │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── core │ │ │ └── api │ │ │ ├── interceptor │ │ │ ├── ConfigInterceptorHelperTest.java │ │ │ └── FakeConfigInterceptor.java │ │ │ └── program │ │ │ └── UnifiedProgramTest.java │ │ └── resources │ │ └── META-INF │ │ └── services │ │ └── com.bytedance.bitsail.core.api.interceptor.ConfigInterceptor ├── bitsail-core-entry │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── core │ │ │ ├── Engine.java │ │ │ ├── program │ │ │ └── ProgramFactory.java │ │ │ └── util │ │ │ └── ExceptionTracker.java │ │ └── test │ │ ├── java │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── core │ │ │ ├── EngineTest.java │ │ │ ├── FakePluginFinder.java │ │ │ ├── program │ │ │ ├── FakeProgram.java │ │ │ └── ProgramFactoryTest.java │ │ │ └── util │ │ │ └── ExceptionTrackerTest.java │ │ └── resources │ │ ├── META-INF │ │ └── services │ │ │ ├── com.bytedance.bitsail.base.packages.PluginFinder │ │ │ └── com.bytedance.bitsail.core.api.program.Program │ │ ├── conf.json │ │ └── logback-test.xml ├── bitsail-core-flink │ ├── bitsail-core-flink-1.11-bridge │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── core │ │ │ │ │ └── flink │ │ │ │ │ └── bridge │ │ │ │ │ ├── program │ │ │ │ │ ├── FlinkDAGBuilderFactory.java │ │ │ │ │ └── FlinkProgram.java │ │ │ │ │ ├── reader │ │ │ │ │ ├── builder │ │ │ │ │ │ └── FlinkSourceDAGBuilder.java │ │ │ │ │ └── delegate │ │ │ │ │ │ ├── DelegateFlinkSource.java │ │ │ │ │ │ ├── DelegateFlinkSourceReader.java │ │ │ │ │ │ ├── DelegateFlinkSourceSplit.java │ │ │ │ │ │ ├── DelegateFlinkSourceSplitEnumerator.java │ │ │ │ │ │ ├── DelegateSourceEvent.java │ │ │ │ │ │ ├── DelegateSourcePipeline.java │ │ │ │ │ │ └── operator │ │ │ │ │ │ ├── DelegateSourceOperator.java │ │ │ │ │ │ ├── DelegateSourceOperatorFactory.java │ │ │ │ │ │ └── DelegateSourceReaderContext.java │ │ │ │ │ ├── serializer │ │ │ │ │ ├── CommittableStateSerializer.java │ │ │ │ │ ├── DelegateCheckpointVersionedSerializer.java │ │ │ │ │ ├── DelegateFlinkSourceSplitSerializer.java │ │ │ │ │ └── DelegateSimpleVersionedSerializer.java │ │ │ │ │ └── writer │ │ │ │ │ ├── bounded │ │ │ │ │ ├── BoundedDataStreamSink.java │ │ │ │ │ ├── BoundedStreamSinkOperator.java │ │ │ │ │ └── BridgeBoundedDataStreamSinkBuilder.java │ │ │ │ │ ├── builder │ │ │ │ │ └── FlinkWriterBuilder.java │ │ │ │ │ └── delegate │ │ │ │ │ ├── DelegateFlinkCommitter.java │ │ │ │ │ └── DelegateFlinkWriter.java │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ ├── com.bytedance.bitsail.core.api.program.Program │ │ │ │ │ └── com.bytedance.bitsail.flink.core.writer.bounded.BoundedDataStreamSinkBuilder │ │ │ │ ├── bitsail-core-flink-1.11-bridge.json │ │ │ │ └── logback-test.xml │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── core │ │ │ │ └── flink │ │ │ │ └── bridge │ │ │ │ ├── program │ │ │ │ ├── FlinkDAGBuilderFactoryTest.java │ │ │ │ ├── FlinkProgramTest.java │ │ │ │ ├── MockDataReaderDAGBuilder.java │ │ │ │ ├── MockDataWriterDAGBuilder.java │ │ │ │ ├── MockInputFormatPlugin.java │ │ │ │ └── MockOutputFormatPlugin.java │ │ │ │ ├── reader │ │ │ │ ├── builder │ │ │ │ │ ├── FlinkSourceDAGBuilderTest.java │ │ │ │ │ └── MockSource.java │ │ │ │ └── delegate │ │ │ │ │ └── DelegateFlinkSourceSplitTest.java │ │ │ │ ├── serializer │ │ │ │ ├── CommittableStateSerializerTest.java │ │ │ │ └── DelegateCheckpointVersionedSerializerTest.java │ │ │ │ └── transform │ │ │ │ ├── builder │ │ │ │ └── FlinkBaseTransformerBuilderTest.java │ │ │ │ └── impl │ │ │ │ └── MockMapTransformer.java │ │ │ └── resources │ │ │ ├── META-INF │ │ │ └── services │ │ │ │ └── com.bytedance.bitsail.base.connector.transform.v1.Transformer │ │ │ ├── bitsail_connector_unified_conf.json │ │ │ └── bitsail_transform_conf.json │ ├── bitsail-core-flink-1.16-bridge │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── core │ │ │ │ │ └── flink116 │ │ │ │ │ └── bridge │ │ │ │ │ ├── program │ │ │ │ │ ├── Flink116Program.java │ │ │ │ │ └── FlinkDAGBuilderFactory.java │ │ │ │ │ ├── reader │ │ │ │ │ ├── builder │ │ │ │ │ │ └── FlinkSourceDAGBuilder.java │ │ │ │ │ └── delegate │ │ │ │ │ │ ├── DelegateFlinkSource.java │ │ │ │ │ │ ├── DelegateFlinkSourceReader.java │ │ │ │ │ │ ├── DelegateFlinkSourceSplit.java │ │ │ │ │ │ ├── DelegateFlinkSourceSplitEnumerator.java │ │ │ │ │ │ ├── DelegateSourceEvent.java │ │ │ │ │ │ └── DelegateSourcePipeline.java │ │ │ │ │ ├── serializer │ │ │ │ │ ├── CommittableStateSerializer.java │ │ │ │ │ ├── DelegateFlinkSourceSplitSerializer.java │ │ │ │ │ └── DelegateSimpleVersionedSerializer.java │ │ │ │ │ └── writer │ │ │ │ │ ├── bounded │ │ │ │ │ ├── BoundedDataStreamSink.java │ │ │ │ │ ├── BoundedStreamSinkOperator.java │ │ │ │ │ └── BridgeBoundedDataStreamSinkBuilder.java │ │ │ │ │ ├── builder │ │ │ │ │ └── FlinkWriterBuilder.java │ │ │ │ │ └── delegate │ │ │ │ │ ├── DelegateFlinkCommitter.java │ │ │ │ │ └── DelegateFlinkWriter.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ ├── com.bytedance.bitsail.core.api.program.Program │ │ │ │ └── com.bytedance.bitsail.flink.core.writer.bounded.BoundedDataStreamSinkBuilder │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── core │ │ │ │ └── flink116 │ │ │ │ └── bridge │ │ │ │ └── program │ │ │ │ └── Flink116ProgramTest.java │ │ │ └── resources │ │ │ ├── bitsail_connector_unified_conf.json │ │ │ └── logback-test.xml │ ├── bitsail-core-flink-base │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── flink │ │ │ │ │ └── core │ │ │ │ │ ├── constants │ │ │ │ │ └── TypeSystem.java │ │ │ │ │ ├── delagate │ │ │ │ │ ├── converter │ │ │ │ │ │ └── FlinkRowConverter.java │ │ │ │ │ └── translation │ │ │ │ │ │ └── BoundednessTranslation.java │ │ │ │ │ ├── dirty │ │ │ │ │ ├── FlinkBatchDirtyCollector.java │ │ │ │ │ └── FlinkBatchDirtyCollectorBuilder.java │ │ │ │ │ ├── execution │ │ │ │ │ ├── DefaultExecutionModeAdapter.java │ │ │ │ │ ├── ExecutionModeAdapter.java │ │ │ │ │ ├── FlinkExecutionEnviron.java │ │ │ │ │ ├── configurer │ │ │ │ │ │ └── FlinkDAGBuilderInterceptor.java │ │ │ │ │ └── utils │ │ │ │ │ │ └── ExecutionUtils.java │ │ │ │ │ ├── legacy │ │ │ │ │ └── connector │ │ │ │ │ │ ├── InputFormatPlugin.java │ │ │ │ │ │ ├── InputFormatSourceFunction.java │ │ │ │ │ │ ├── OutputFormatPlugin.java │ │ │ │ │ │ └── Pluggable.java │ │ │ │ │ ├── option │ │ │ │ │ └── FlinkCommonOptions.java │ │ │ │ │ ├── parallelism │ │ │ │ │ ├── FlinkParallelismAdvisor.java │ │ │ │ │ ├── UnionParallelismStrategy.java │ │ │ │ │ └── batch │ │ │ │ │ │ └── FlinkBatchReaderParallelismComputer.java │ │ │ │ │ ├── parser │ │ │ │ │ ├── BytesParser.java │ │ │ │ │ └── RowBytesParser.java │ │ │ │ │ ├── plugins │ │ │ │ │ ├── AdapterPlugin.java │ │ │ │ │ ├── InputAdapter.java │ │ │ │ │ └── OutputAdapter.java │ │ │ │ │ ├── reader │ │ │ │ │ ├── FlinkDataReaderDAGBuilder.java │ │ │ │ │ └── PluginableInputFormatDAGBuilder.java │ │ │ │ │ ├── runtime │ │ │ │ │ ├── RuntimeContextInjectable.java │ │ │ │ │ ├── messenger │ │ │ │ │ │ ├── DropwizardCounterWrapper.java │ │ │ │ │ │ ├── FlinkAccumulatorMessengerBuilder.java │ │ │ │ │ │ └── impl │ │ │ │ │ │ │ └── FlinkAccumulatorStatisticsMessenger.java │ │ │ │ │ └── restart │ │ │ │ │ │ ├── FailureRateRestartStrategyBuilder.java │ │ │ │ │ │ ├── FixedDelayRestartStrategyBuilder.java │ │ │ │ │ │ ├── FlinkRestartStrategyBuilder.java │ │ │ │ │ │ └── RestartStrategy.java │ │ │ │ │ ├── serialization │ │ │ │ │ └── AbstractDeserializationSchema.java │ │ │ │ │ ├── transform │ │ │ │ │ ├── FlinkDataTransformDAGBuilder.java │ │ │ │ │ ├── builder │ │ │ │ │ │ ├── FlinkBaseTransformerBuilder.java │ │ │ │ │ │ ├── FlinkMapTransformerBuilder.java │ │ │ │ │ │ └── FlinkPartitionTransformerBuilder.java │ │ │ │ │ └── delegate │ │ │ │ │ │ ├── DelegateFlinkKeySelector.java │ │ │ │ │ │ ├── DelegateFlinkMapFunction.java │ │ │ │ │ │ └── DelegateFlinkPartitioner.java │ │ │ │ │ ├── typeinfo │ │ │ │ │ ├── ListColumnTypeInfo.java │ │ │ │ │ ├── MapColumnTypeInfo.java │ │ │ │ │ └── PrimitiveColumnTypeInfo.java │ │ │ │ │ ├── typeutils │ │ │ │ │ ├── AutoDetectFlinkTypeInfoUtil.java │ │ │ │ │ ├── ColumnFlinkTypeInfoUtil.java │ │ │ │ │ ├── NativeFlinkTypeInfoUtil.java │ │ │ │ │ ├── TypeInfoColumnBridge.java │ │ │ │ │ ├── TypeInfoNativeBridge.java │ │ │ │ │ └── base │ │ │ │ │ │ ├── BoolColumnComparator.java │ │ │ │ │ │ ├── BoolColumnSerializer.java │ │ │ │ │ │ ├── BytesColumnComparator.java │ │ │ │ │ │ ├── BytesColumnSerializer.java │ │ │ │ │ │ ├── ColumnTypeComparator.java │ │ │ │ │ │ ├── DateColumnComparator.java │ │ │ │ │ │ ├── DateColumnSerializer.java │ │ │ │ │ │ ├── DoubleColumnComparator.java │ │ │ │ │ │ ├── DoubleColumnSerializer.java │ │ │ │ │ │ ├── ListColumnSerializer.java │ │ │ │ │ │ ├── ListColumnSerializerSnapshot.java │ │ │ │ │ │ ├── LongColumnComparator.java │ │ │ │ │ │ ├── LongColumnSerializer.java │ │ │ │ │ │ ├── MapColumnSerializer.java │ │ │ │ │ │ ├── MapColumnSerializerSnapshot.java │ │ │ │ │ │ ├── StringColumnComparator.java │ │ │ │ │ │ └── StringColumnSerializer.java │ │ │ │ │ ├── util │ │ │ │ │ ├── AccumulatorRestorer.java │ │ │ │ │ └── RowUtil.java │ │ │ │ │ └── writer │ │ │ │ │ ├── FlinkDataWriterDAGBuilder.java │ │ │ │ │ ├── PluginableOutputFormatDAGBuilder.java │ │ │ │ │ └── bounded │ │ │ │ │ ├── BoundedDataStreamSinkBuilder.java │ │ │ │ │ └── BoundedDataStreamSinkFactory.java │ │ │ └── resources │ │ │ │ └── META-INF │ │ │ │ └── services │ │ │ │ ├── com.bytedance.bitsail.base.dirty.DirtyCollectorBuilder │ │ │ │ └── com.bytedance.bitsail.base.messenger.MessengerBuilder │ │ │ └── test │ │ │ └── java │ │ │ └── com │ │ │ └── bytedance │ │ │ └── bitsail │ │ │ └── flink │ │ │ └── core │ │ │ ├── delagate │ │ │ └── converter │ │ │ │ └── FlinkRowConvertSerializerTest.java │ │ │ ├── dirty │ │ │ └── FlinkBatchDirtyConnectorTest.java │ │ │ ├── execution │ │ │ ├── FlinkExecutionEnvironTest.java │ │ │ └── configurer │ │ │ │ └── FlinkDAGBuilderInterceptorTest.java │ │ │ ├── legacy │ │ │ └── connector │ │ │ │ └── InputFormatPluginTest.java │ │ │ ├── parallelism │ │ │ ├── BaseParallelismTest.java │ │ │ ├── BatchParallelismAdvisorTest.java │ │ │ └── StreamParallelismAdvisorTest.java │ │ │ ├── parser │ │ │ ├── BytesParserTest.java │ │ │ └── RowBytesParserTest.java │ │ │ ├── typeutils │ │ │ ├── ColumnFlinkTypeInfoUtilTest.java │ │ │ ├── NativeFlinkTypeInfoUtilTest.java │ │ │ └── base │ │ │ │ ├── ColumnComparatorTestBase.java │ │ │ │ ├── DateColumnComparatorTest.java │ │ │ │ └── view │ │ │ │ ├── TestInputView.java │ │ │ │ └── TestOutputView.java │ │ │ └── util │ │ │ └── RowUtilTest.java │ ├── bitsail-core-flink-bridge-provider │ │ └── pom.xml │ └── pom.xml └── pom.xml ├── bitsail-dist ├── pom.xml └── src │ └── main │ ├── assemblies │ └── assembly-bin.xml │ └── resources │ ├── Dockerfile │ ├── bin │ └── bitsail │ ├── conf │ ├── bitsail-e2e.conf │ ├── bitsail.conf │ └── logback.xml │ └── examples │ ├── Fake_Doris_Example.json │ ├── Fake_HBase_Example.json │ ├── Fake_Hive_Example.json │ ├── Fake_Hudi_Example.json │ ├── Fake_Kafka_Example.json │ ├── Fake_MongoDB_Example.json │ ├── Fake_Mysql_Example.json │ ├── Fake_Postgresql_Example.json │ ├── Fake_Print_Example.json │ ├── Fake_Redis_Example.json │ ├── Fake_RocketMQ_Example.json │ ├── Fake_SelectDB_Example.json │ ├── Fake_SqlServer_Example.json │ ├── File_Print_Example.json │ ├── File_Print_Example_Csv.json │ ├── HBase_Print_Example.json │ ├── Hive_Print_Example.json │ ├── Hudi_Compaction_Example.json │ ├── Kafka_Hudi_Example.json │ ├── LarkSheet_Print_Example.json │ ├── MongoDB_Print_Example.json │ ├── Mysql_Print_Example.json │ ├── Postgresql_Print_Example.json │ ├── SqlServer_Print_Example.json │ └── example-datas │ ├── Example_File.csv │ └── Example_File.json ├── bitsail-shade ├── bitsail-shaded-hadoop │ └── pom.xml ├── bitsail-shaded-hive │ ├── pom.xml │ └── src │ │ └── main │ │ └── java │ │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── shaded │ │ └── hive │ │ ├── client │ │ └── HiveMetaClientUtil.java │ │ └── shim │ │ ├── HiveShim.java │ │ ├── HiveShimLoader.java │ │ ├── HiveShimV120.java │ │ ├── HiveShimV121.java │ │ ├── HiveShimV122.java │ │ ├── HiveShimV200.java │ │ ├── HiveShimV210.java │ │ ├── HiveShimV211.java │ │ ├── HiveShimV230.java │ │ ├── HiveShimV239.java │ │ ├── HiveShimV310.java │ │ └── HiveShimV312.java └── pom.xml ├── bitsail-test ├── bitsail-test-end-to-end │ ├── bitsail-test-e2e-base │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── e2e │ │ │ │ │ ├── AbstractE2ETest.java │ │ │ │ │ ├── TestJob.java │ │ │ │ │ ├── annotation │ │ │ │ │ ├── ExecutorPatterns.java │ │ │ │ │ └── ReuseContainers.java │ │ │ │ │ ├── base │ │ │ │ │ ├── AbstractContainer.java │ │ │ │ │ └── transfer │ │ │ │ │ │ ├── FileMappingTransfer.java │ │ │ │ │ │ └── TransferableFile.java │ │ │ │ │ ├── datasource │ │ │ │ │ ├── AbstractDataSource.java │ │ │ │ │ ├── DataSourceFactory.java │ │ │ │ │ ├── EmptyDataSource.java │ │ │ │ │ └── util │ │ │ │ │ │ └── RowGenerator.java │ │ │ │ │ ├── error │ │ │ │ │ └── E2ETestErrorCode.java │ │ │ │ │ ├── executor │ │ │ │ │ ├── AbstractExecutor.java │ │ │ │ │ ├── ExecutorLoader.java │ │ │ │ │ ├── flink │ │ │ │ │ │ ├── AbstractFlinkExecutor.java │ │ │ │ │ │ └── Flink11Executor.java │ │ │ │ │ └── generic │ │ │ │ │ │ ├── GenericExecutor.java │ │ │ │ │ │ └── GenericExecutorSetting.java │ │ │ │ │ ├── mapping │ │ │ │ │ └── ConnectorMapping.java │ │ │ │ │ ├── option │ │ │ │ │ └── EndToEndOptions.java │ │ │ │ │ └── util │ │ │ │ │ └── Modules.java │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ ├── com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource │ │ │ │ │ └── com.bytedance.bitsail.test.e2e.executor.AbstractExecutor │ │ │ │ ├── docker │ │ │ │ └── flink │ │ │ │ │ ├── README.md │ │ │ │ │ ├── flink_1.11.6_hadoop_2.7.5.dockerfile │ │ │ │ │ └── flink_1.11.6_hadoop_3.1.1.dockerfile │ │ │ │ └── logback-test.xml │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── e2e │ │ │ │ ├── base │ │ │ │ └── transfer │ │ │ │ │ └── FileMappingTransferTest.java │ │ │ │ └── executor │ │ │ │ └── generic │ │ │ │ ├── GenericExecutorSettingTest.java │ │ │ │ └── GenericExecutorTest.java │ │ │ └── resources │ │ │ └── executor │ │ │ └── generic │ │ │ └── TestGenericExecutorSetting.json │ ├── bitsail-test-e2e-connector-v1 │ │ ├── bitsail-test-e2e-connector-v1-clickhouse │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── e2e │ │ │ │ │ ├── ClickhouseToPrintE2ETest.java │ │ │ │ │ └── datasource │ │ │ │ │ └── ClickhouseDataSource.java │ │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource │ │ │ │ └── clickhouse_to_print.json │ │ ├── bitsail-test-e2e-connector-v1-elasticsearch │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── e2e │ │ │ │ │ ├── FakeToElasticsearchE2ETest.java │ │ │ │ │ └── datasource │ │ │ │ │ └── ElasticsearchDataSource.java │ │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource │ │ │ │ └── fake_to_elasticsearch.json │ │ ├── bitsail-test-e2e-connector-v1-ftp │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── e2e │ │ │ │ │ ├── SftpToPrintE2ETest.java │ │ │ │ │ └── datasource │ │ │ │ │ └── SftpDataSource.java │ │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource │ │ │ │ ├── data │ │ │ │ └── csv │ │ │ │ │ ├── _SUCCESS │ │ │ │ │ ├── upload1 │ │ │ │ │ ├── test1.csv │ │ │ │ │ └── test2.csv │ │ │ │ │ └── upload2 │ │ │ │ │ ├── test1.csv │ │ │ │ │ └── test2.csv │ │ │ │ └── sftp_to_print.json │ │ ├── bitsail-test-e2e-connector-v1-print │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── e2e │ │ │ │ │ └── FakeToPrintE2ETest.java │ │ │ │ └── resources │ │ │ │ └── fake_to_print.json │ │ ├── bitsail-test-e2e-connector-v1-redis │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── e2e │ │ │ │ │ ├── FakeToRedisE2ETest.java │ │ │ │ │ └── datasource │ │ │ │ │ └── RedisDataSource.java │ │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource │ │ │ │ └── fake_to_redis.json │ │ ├── bitsail-test-e2e-connector-v1-rocketmq │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── e2e │ │ │ │ │ ├── RocketmqToPrintE2ETest.java │ │ │ │ │ └── datasource │ │ │ │ │ └── RocketMQDataSource.java │ │ │ │ └── resources │ │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource │ │ │ │ ├── broker.conf │ │ │ │ └── rocketmq_to_print.json │ │ └── pom.xml │ ├── bitsail-test-e2e-generic-executor-templates │ │ ├── pom.xml │ │ └── src │ │ │ └── main │ │ │ └── settings │ │ │ └── flink_1.11.6_executor.json │ └── pom.xml ├── bitsail-test-integration │ ├── bitsail-test-integration-assert │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── assertion │ │ │ │ └── FakeToAssertITCase.java │ │ │ └── resources │ │ │ ├── assert_sink.json │ │ │ ├── assert_sink_col_check.json │ │ │ └── assert_sink_row_check.json │ ├── bitsail-test-integration-base │ │ ├── pom.xml │ │ └── src │ │ │ ├── main │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ ├── AbstractIntegrationTest.java │ │ │ │ │ ├── engine │ │ │ │ │ ├── EngineType.java │ │ │ │ │ ├── IntegrationEngine.java │ │ │ │ │ └── flink │ │ │ │ │ │ ├── Flink111Engine.java │ │ │ │ │ │ └── Flink116Engine.java │ │ │ │ │ ├── error │ │ │ │ │ └── IntegrationTestErrorCode.java │ │ │ │ │ └── utils │ │ │ │ │ └── JobConfUtils.java │ │ │ └── resources │ │ │ │ └── logback-test.xml │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── FakeToPrintITCase.java │ │ │ └── resources │ │ │ └── fake_to_print.json │ ├── bitsail-test-integration-cdc │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ ├── cdc │ │ │ │ ├── mysql │ │ │ │ │ ├── MysqlCDCSourceITCase.java │ │ │ │ │ └── container │ │ │ │ │ │ ├── MySQLContainerMariadbAdapter.java │ │ │ │ │ │ └── util │ │ │ │ │ │ └── TestDatabase.java │ │ │ │ └── sqlserver │ │ │ │ │ └── SqlServerBinlogSplitReaderITCase.java │ │ │ │ └── kafka │ │ │ │ └── container │ │ │ │ ├── KafkaCluster.java │ │ │ │ └── KafkaContainers.java │ │ │ └── resources │ │ │ ├── bitsail_mysql_cdc_kafka.json │ │ │ ├── bitsail_mysql_cdc_print.json │ │ │ ├── bitsail_sqlserver_cdc_print.json │ │ │ └── scripts │ │ │ ├── jdbc_to_print.sql │ │ │ └── test_insert_full_type.sql │ ├── bitsail-test-integration-clickhouse │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── clickhouse │ │ │ │ ├── ClickhouseReaderITCase.java │ │ │ │ └── container │ │ │ │ └── ClickhouseContainerHolder.java │ │ │ └── resources │ │ │ ├── clickhouse_to_print.json │ │ │ └── example_data.csv │ ├── bitsail-test-integration-connector-legacy │ │ ├── bitsail-test-integration-ftp-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ └── ftp │ │ │ │ │ ├── FtpSourceITCase.java │ │ │ │ │ ├── SftpSourceITCase.java │ │ │ │ │ ├── client │ │ │ │ │ ├── FtpHandlerITCase.java │ │ │ │ │ └── SftpHandlerITCase.java │ │ │ │ │ └── container │ │ │ │ │ ├── FtpConfigUtils.java │ │ │ │ │ ├── FtpDataSource.java │ │ │ │ │ ├── SftpDataSource.java │ │ │ │ │ └── constant │ │ │ │ │ └── FtpTestConstants.java │ │ │ │ └── resources │ │ │ │ ├── scripts │ │ │ │ └── ftp_to_print.json │ │ │ │ └── upload │ │ │ │ ├── _SUCCESS │ │ │ │ ├── test1.csv │ │ │ │ └── test2.csv │ │ ├── bitsail-test-integration-hadoop-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ └── hadoop │ │ │ │ │ └── HadoopInputFormatITCase.java │ │ │ │ └── resources │ │ │ │ ├── hadoop_to_print.json │ │ │ │ └── source │ │ │ │ └── test.json │ │ ├── bitsail-test-integration-hbase-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ └── hbase │ │ │ │ │ ├── HBaseITCase.java │ │ │ │ │ └── container │ │ │ │ │ ├── HbaseCluster.java │ │ │ │ │ └── HbaseContainer.java │ │ │ │ └── resources │ │ │ │ ├── fake_to_hbase.json │ │ │ │ ├── hbase_to_print.json │ │ │ │ └── logback-test.xml │ │ ├── bitsail-test-integration-hudi-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ ├── kafka │ │ │ │ │ └── container │ │ │ │ │ │ ├── KafkaCluster.java │ │ │ │ │ │ └── KafkaContainers.java │ │ │ │ │ └── legacy │ │ │ │ │ └── hudi │ │ │ │ │ ├── FakeToHudiITCase.java │ │ │ │ │ ├── HudiSource2PrintITCase.java │ │ │ │ │ ├── Kafka2HudiITCase.java │ │ │ │ │ └── compaction │ │ │ │ │ └── HudiCompactionITCase.java │ │ │ │ └── resources │ │ │ │ └── test_read_schema.avsc │ │ ├── bitsail-test-integration-jdbc-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ ├── mysql │ │ │ │ │ ├── MysqlConnectorITCase.java │ │ │ │ │ └── container │ │ │ │ │ │ └── MySQLContainerMariadbAdapter.java │ │ │ │ │ └── oracle │ │ │ │ │ ├── OracleConnectorITCase.java │ │ │ │ │ └── OracleITCaseAppleChipWorkaround.md │ │ │ │ └── resources │ │ │ │ ├── mysql │ │ │ │ ├── fake_to_mysql_sink.json │ │ │ │ ├── init_database.sql │ │ │ │ ├── mysql_to_print.json │ │ │ │ └── mysql_to_print_without_columns.json │ │ │ │ └── oracle │ │ │ │ ├── fake_to_oracle_sink.json │ │ │ │ ├── init_database.sql │ │ │ │ └── oracle_source_to_print.json │ │ ├── bitsail-test-integration-kafka-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ ├── KafkaSinkITCase.java │ │ │ │ │ ├── KafkaSourceITCase.java │ │ │ │ │ └── container │ │ │ │ │ ├── KafkaCluster.java │ │ │ │ │ └── KafkaContainers.java │ │ │ │ └── resources │ │ │ │ ├── fake_to_kafka.json │ │ │ │ └── kafka_to_print.json │ │ ├── bitsail-test-integration-larksheet-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ └── larksheet │ │ │ │ │ └── LarkSheetSourceITCase.java │ │ │ │ └── resources │ │ │ │ └── larksheet_to_print.json │ │ ├── bitsail-test-integration-mongodb-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ └── mongodb │ │ │ │ │ ├── MongoDBConnectorITCase.java │ │ │ │ │ └── container │ │ │ │ │ └── TestMongoDBContainer.java │ │ │ │ └── resources │ │ │ │ ├── fake_to_mongodb.json │ │ │ │ └── mongodb_to_print.json │ │ ├── bitsail-test-integration-redis-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ └── redis │ │ │ │ │ ├── RedisOutputFormatITCase.java │ │ │ │ │ └── container │ │ │ │ │ └── RedisContainer.java │ │ │ │ └── resources │ │ │ │ └── fake_to_redis.json │ │ ├── bitsail-test-integration-rocketmq-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ └── rocketmq │ │ │ │ │ ├── RocketMQOutputFormatITCase.java │ │ │ │ │ └── container │ │ │ │ │ └── RocketMQDataSource.java │ │ │ │ └── resources │ │ │ │ └── fake_to_rocketmq.json │ │ ├── bitsail-test-integration-streamingfile-legacy │ │ │ ├── pom.xml │ │ │ └── src │ │ │ │ └── test │ │ │ │ ├── java │ │ │ │ └── com │ │ │ │ │ └── bytedance │ │ │ │ │ └── bitsail │ │ │ │ │ └── test │ │ │ │ │ └── integration │ │ │ │ │ └── legacy │ │ │ │ │ ├── kafka │ │ │ │ │ └── container │ │ │ │ │ │ ├── KafkaCluster.java │ │ │ │ │ │ └── KafkaContainers.java │ │ │ │ │ └── streamingfile │ │ │ │ │ ├── StreamingFileSystemSinkHdfsITCase.java │ │ │ │ │ └── StreamingFileSystemSinkHiveITCase.java │ │ │ │ └── resources │ │ │ │ └── kafka_to_hive.json │ │ └── pom.xml │ ├── bitsail-test-integration-doris │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── doris │ │ │ │ └── DorisSinkITCase.java │ │ │ └── resources │ │ │ └── fake_to_doris.json │ ├── bitsail-test-integration-druid │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── druid │ │ │ │ └── DruidSinkITCase.java │ │ │ └── resources │ │ │ ├── docker-compose.yml │ │ │ ├── environment │ │ │ └── fake_to_druid.json │ ├── bitsail-test-integration-elasticsearch │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── elasticsearch │ │ │ │ ├── ElasticsearchSinkITCase.java │ │ │ │ ├── ElasticsearchWriterITCase.java │ │ │ │ └── container │ │ │ │ └── ElasticsearchCluster.java │ │ │ └── resources │ │ │ ├── es_sink_test.json │ │ │ ├── es_writer_parameter_test.json │ │ │ └── logback.xml │ ├── bitsail-test-integration-ftp │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── ftp │ │ │ │ ├── FtpSourceITCase.java │ │ │ │ ├── SftpSourceITCase.java │ │ │ │ ├── client │ │ │ │ ├── FtpHandlerITCase.java │ │ │ │ └── SftpHandlerITCase.java │ │ │ │ ├── container │ │ │ │ ├── FtpConfigUtils.java │ │ │ │ ├── FtpDataSource.java │ │ │ │ ├── SftpDataSource.java │ │ │ │ └── constant │ │ │ │ │ └── FtpTestConstants.java │ │ │ │ └── coordinator │ │ │ │ └── FtpSourceSplitCoordinatorITCase.java │ │ │ └── resources │ │ │ ├── data │ │ │ ├── csv │ │ │ │ ├── _SUCCESS │ │ │ │ ├── upload1 │ │ │ │ │ ├── test1.csv │ │ │ │ │ └── test2.csv │ │ │ │ └── upload2 │ │ │ │ │ ├── test1.csv │ │ │ │ │ └── test2.csv │ │ │ ├── files │ │ │ │ ├── p1 │ │ │ │ │ ├── f1 │ │ │ │ │ └── p11 │ │ │ │ │ │ └── f11 │ │ │ │ └── p2 │ │ │ │ │ └── f2 │ │ │ └── json │ │ │ │ ├── _SUCCESS │ │ │ │ ├── upload1 │ │ │ │ ├── test1.json │ │ │ │ └── test2.json │ │ │ │ └── upload2 │ │ │ │ ├── test1.json │ │ │ │ └── test2.json │ │ │ └── scripts │ │ │ ├── csv_ftp_to_print.json │ │ │ └── json_ftp_to_print.json │ ├── bitsail-test-integration-hbase │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── hbase │ │ │ │ ├── HBaseSinkITCase.java │ │ │ │ ├── HBaseSourceITCase.java │ │ │ │ └── container │ │ │ │ ├── HbaseCluster.java │ │ │ │ └── HbaseContainer.java │ │ │ └── resources │ │ │ ├── fake_to_hbase.json │ │ │ └── hbase_to_print.json │ ├── bitsail-test-integration-kafka │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── kafka │ │ │ │ ├── FakeToKafkaITCase.java │ │ │ │ └── container │ │ │ │ ├── KafkaCluster.java │ │ │ │ └── KafkaContainers.java │ │ │ └── resources │ │ │ └── fake_to_kafka.json │ ├── bitsail-test-integration-kudu │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── kudu │ │ │ │ ├── KuduSinkITCase.java │ │ │ │ ├── KuduSourceITCase.java │ │ │ │ └── container │ │ │ │ └── KuduDataSource.java │ │ │ └── resources │ │ │ ├── fake_to_kudu.json │ │ │ └── kudu_to_print.json │ ├── bitsail-test-integration-larksheet │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── larksheet │ │ │ │ └── LarkSheetSourceITCase.java │ │ │ └── resources │ │ │ └── larksheet_to_print.json │ ├── bitsail-test-integration-localfilesystem │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── localfilesystem │ │ │ │ └── LocalFileSystemSourceITCase.java │ │ │ └── resources │ │ │ ├── data │ │ │ ├── csv │ │ │ │ └── test.csv │ │ │ └── json │ │ │ │ └── test.json │ │ │ └── scripts │ │ │ ├── local_file_to_print_csv.json │ │ │ └── local_file_to_print_json.json │ ├── bitsail-test-integration-mongodb │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── mongodb │ │ │ │ ├── MongoDBReaderITCase.java │ │ │ │ └── container │ │ │ │ └── TestMongoDBContainer.java │ │ │ └── resources │ │ │ ├── logback-test.xml │ │ │ └── mongodb_to_print.json │ ├── bitsail-test-integration-print │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── print │ │ │ │ └── FakeToPrintITCase.java │ │ │ └── resources │ │ │ └── transform_test.json │ ├── bitsail-test-integration-redis │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── redis │ │ │ │ ├── RedisSinkITCase.java │ │ │ │ └── container │ │ │ │ └── RedisContainer.java │ │ │ └── resources │ │ │ ├── fake_to_redis_hash.json │ │ │ ├── fake_to_redis_mhash.json │ │ │ └── fake_to_redis_string.json │ ├── bitsail-test-integration-rocketmq │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── rocketmq │ │ │ │ ├── RocketMQSinkITCase.java │ │ │ │ ├── RocketMQSourceTest.java │ │ │ │ └── container │ │ │ │ └── RocketMQDataSource.java │ │ │ └── resources │ │ │ ├── bitsail_rocketmq_print.json │ │ │ ├── broker.conf │ │ │ └── fake_to_rocketmq.json │ ├── bitsail-test-integration-selectdb │ │ ├── pom.xml │ │ └── src │ │ │ └── test │ │ │ ├── java │ │ │ └── com │ │ │ │ └── bytedance │ │ │ │ └── bitsail │ │ │ │ └── test │ │ │ │ └── integration │ │ │ │ └── selectdb │ │ │ │ └── SelectDBSinkITCase.java │ │ │ └── resources │ │ │ └── fake_to_selectdb.json │ └── pom.xml └── pom.xml ├── bitsail-transforms ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── bytedance │ │ └── bitsail │ │ └── transforms │ │ ├── map │ │ └── encrypt │ │ │ ├── EncryptFieldMapTransformer.java │ │ │ ├── EncryptOptions.java │ │ │ └── Encrypts.java │ │ └── partition │ │ └── field │ │ ├── FieldBaseOptions.java │ │ └── FieldBasePartitionTransformer.java │ └── resources │ └── META-INF │ └── services │ └── com.bytedance.bitsail.base.connector.transform.v1.Transformer ├── build.sh ├── package.json ├── pom.xml ├── test-e2e.sh ├── tools ├── maven │ └── checkstyle.xml ├── release │ ├── create_binary_tar.sh │ ├── create_source_release.sh │ └── validate_source_release.sh └── test │ ├── list-test-modules.py │ └── run-test.sh ├── website ├── .vuepress │ ├── config.ts │ ├── footer.ts │ ├── navbar │ │ ├── en.ts │ │ ├── index.ts │ │ └── zh.ts │ ├── public │ │ ├── bitsail_arch.png │ │ ├── bitsail_logo.png │ │ ├── logo.jpg │ │ └── wechat_QR.png │ ├── sidebar │ │ ├── en.ts │ │ ├── index.ts │ │ └── zh.ts │ ├── styles │ │ ├── config.scss │ │ ├── index.scss │ │ └── palette.scss │ └── theme.ts ├── README.md ├── en │ ├── README.md │ ├── blog │ │ └── blog.md │ ├── community │ │ ├── community.md │ │ ├── connector_quick_start.md │ │ ├── contribute.md │ │ ├── mailing.md │ │ ├── pr_guide.md │ │ ├── release_guide.md │ │ ├── sink_connector_detail.md │ │ ├── source_connector_detail.md │ │ └── team.md │ ├── documents │ │ ├── components │ │ │ ├── README.md │ │ │ ├── clients │ │ │ │ └── introduction.md │ │ │ ├── conversion │ │ │ │ ├── hive-convert.md │ │ │ │ └── introduction.md │ │ │ ├── format │ │ │ │ ├── hive-format.md │ │ │ │ ├── introduction.md │ │ │ │ └── json-format.md │ │ │ └── parser │ │ │ │ └── introduction.md │ │ ├── connectors │ │ │ ├── README.md │ │ │ ├── assert │ │ │ │ ├── assert-example.md │ │ │ │ └── assert.md │ │ │ ├── clickhouse │ │ │ │ ├── clickhouse-example.md │ │ │ │ └── clickhouse.md │ │ │ ├── doris │ │ │ │ ├── doris-example.md │ │ │ │ └── doris.md │ │ │ ├── druid │ │ │ │ ├── druid-example.md │ │ │ │ └── druid.md │ │ │ ├── elasticsearch │ │ │ │ ├── elasticsearch-example.md │ │ │ │ └── elasticsearch.md │ │ │ ├── fake │ │ │ │ ├── fake-example.md │ │ │ │ └── fake.md │ │ │ ├── ftp │ │ │ │ ├── ftp-example.md │ │ │ │ ├── ftp.md │ │ │ │ └── v1 │ │ │ │ │ ├── ftp-v1-example.md │ │ │ │ │ └── ftp-v1.md │ │ │ ├── hadoop │ │ │ │ ├── hadoop-example.md │ │ │ │ └── hadoop.md │ │ │ ├── hbase │ │ │ │ ├── hbase-example.md │ │ │ │ └── hbase.md │ │ │ ├── hive │ │ │ │ ├── hive-example.md │ │ │ │ └── hive.md │ │ │ ├── hudi │ │ │ │ ├── hudi-example.md │ │ │ │ └── hudi.md │ │ │ ├── jdbc │ │ │ │ ├── jdbc.md │ │ │ │ └── jdbc_example.md │ │ │ ├── kafka │ │ │ │ ├── kafka-example.md │ │ │ │ └── kafka.md │ │ │ ├── kudu │ │ │ │ ├── kudu-example.md │ │ │ │ └── kudu.md │ │ │ ├── larksheet │ │ │ │ ├── larksheet-example.md │ │ │ │ ├── larksheet.md │ │ │ │ └── v1 │ │ │ │ │ ├── larksheet-example.md │ │ │ │ │ └── larksheet-v1.md │ │ │ ├── localfilesystem │ │ │ │ ├── localfilesystem-example.md │ │ │ │ └── localfilesystem.md │ │ │ ├── mongodb │ │ │ │ ├── mongodb-example.md │ │ │ │ └── mongodb.md │ │ │ ├── print │ │ │ │ ├── print-example.md │ │ │ │ ├── print.md │ │ │ │ └── v1 │ │ │ │ │ ├── print-example-v1.md │ │ │ │ │ └── print-v1.md │ │ │ ├── redis │ │ │ │ ├── redis-example.md │ │ │ │ ├── redis.md │ │ │ │ └── v1 │ │ │ │ │ ├── redis-v1-example.md │ │ │ │ │ └── redis-v1.md │ │ │ ├── rocketmq │ │ │ │ ├── rocketmq-example.md │ │ │ │ └── rocketmq.md │ │ │ ├── selectdb │ │ │ │ ├── selectdb-example.md │ │ │ │ └── selectdb.md │ │ │ └── streamingfile │ │ │ │ ├── streamingfile.md │ │ │ │ └── streamingfile_example.md │ │ ├── faq │ │ │ ├── README.md │ │ │ ├── e2e.md │ │ │ └── parallelism.md │ │ ├── introduce.md │ │ └── start │ │ │ ├── README.md │ │ │ ├── config.md │ │ │ ├── deployment.md │ │ │ ├── env_setup.md │ │ │ └── quick_guide.md │ ├── release │ │ └── release.md │ └── usercases │ │ ├── case.md │ │ └── cases │ │ └── bytedance.md ├── images │ ├── bitsail.png │ ├── bitsail_arch.png │ ├── bitsail_logo.png │ ├── change-hive-version.png │ ├── checkstyle_plugin.png │ ├── community │ │ ├── connector_quick_start │ │ │ ├── bitsail_model.png │ │ │ ├── code_structure_en.png │ │ │ ├── code_structure_zh.png │ │ │ ├── connector_pom.png │ │ │ ├── dist_pom.png │ │ │ ├── sink_connector.png │ │ │ ├── source_connector.png │ │ │ └── test_container.png │ │ ├── pr_guide │ │ │ ├── after_git_reset.png │ │ │ ├── commit_info.png │ │ │ ├── create_pr.png │ │ │ ├── git_clone_example.png │ │ │ ├── git_history.png │ │ │ ├── git_rebase_example.png │ │ │ ├── github_pr.png │ │ │ ├── github_status.png │ │ │ ├── repository_fork.png │ │ │ └── repository_structure.png │ │ ├── release_guide │ │ │ └── release_procedure.png │ │ ├── sink_connector │ │ │ ├── sink_diagram.png │ │ │ └── writer_diagram.png │ │ └── source_connector │ │ │ ├── bitsail_converter.png │ │ │ ├── deserialization_schema_diagram.png │ │ │ ├── file_mapping_converter.png │ │ │ ├── source_diagram.png │ │ │ ├── source_reader_diagram.png │ │ │ ├── source_split_coordinator_diagram.png │ │ │ └── source_split_diagram.png │ ├── documents │ │ └── start │ │ │ └── quick_guide │ │ │ ├── compile_product_structure.png │ │ │ ├── product_structure.png │ │ │ └── source_code_structure.png │ ├── license_header.png │ ├── set_checkstyle.png │ └── wechat_QR.png └── zh │ ├── README.md │ ├── blog │ └── blog.md │ ├── community │ ├── community.md │ ├── connector_quick_start.md │ ├── contribute.md │ ├── mailing.md │ ├── pr_guide.md │ ├── release_guide.md │ ├── sink_connector_detail.md │ ├── source_connector_detail.md │ └── team.md │ ├── documents │ ├── components │ │ ├── README.md │ │ ├── clients │ │ │ └── introduction.md │ │ ├── conversion │ │ │ ├── hive-convert.md │ │ │ └── introduction.md │ │ ├── format │ │ │ ├── hive-format.md │ │ │ ├── introduction.md │ │ │ └── json-format.md │ │ └── parser │ │ │ └── introduction.md │ ├── connectors │ │ ├── README.md │ │ ├── assert │ │ │ ├── assert-example.md │ │ │ └── assert.md │ │ ├── clickhouse │ │ │ ├── clickhouse-example.md │ │ │ └── clickhouse.md │ │ ├── doris │ │ │ ├── doris-example.md │ │ │ └── doris.md │ │ ├── druid │ │ │ ├── druid-example.md │ │ │ └── druid.md │ │ ├── elasticsearch │ │ │ ├── elasticsearch-example.md │ │ │ └── elasticsearch.md │ │ ├── fake │ │ │ ├── fake-example.md │ │ │ └── fake.md │ │ ├── ftp │ │ │ ├── ftp-example.md │ │ │ ├── ftp.md │ │ │ └── v1 │ │ │ │ ├── ftp-v1-example.md │ │ │ │ └── ftp-v1.md │ │ ├── hadoop │ │ │ ├── hadoop-example.md │ │ │ └── hadoop.md │ │ ├── hbase │ │ │ ├── hbase-example.md │ │ │ └── hbase.md │ │ ├── hive │ │ │ ├── hive-example.md │ │ │ └── hive.md │ │ ├── hudi │ │ │ ├── hudi-example.md │ │ │ └── hudi.md │ │ ├── jdbc │ │ │ ├── jdbc-example.md │ │ │ └── jdbc.md │ │ ├── kafka │ │ │ ├── kafka-example.md │ │ │ └── kafka.md │ │ ├── kudu │ │ │ ├── kudu-example.md │ │ │ └── kudu.md │ │ ├── larksheet │ │ │ ├── larksheet-example.md │ │ │ ├── larksheet.md │ │ │ └── v1 │ │ │ │ ├── larksheet-example.md │ │ │ │ └── larksheet-v1.md │ │ ├── localfilesystem │ │ │ ├── localfilesystem-example.md │ │ │ └── localfilesystem.md │ │ ├── mongodb │ │ │ ├── mongodb-example.md │ │ │ └── mongodb.md │ │ ├── print │ │ │ ├── print-example.md │ │ │ ├── print.md │ │ │ └── v1 │ │ │ │ ├── print-v1-example.md │ │ │ │ └── print-v1.md │ │ ├── redis │ │ │ ├── redis-example.md │ │ │ ├── redis.md │ │ │ └── v1 │ │ │ │ ├── redis-v1-example.md │ │ │ │ └── redis-v1.md │ │ ├── rocketmq │ │ │ ├── rocketmq-example.md │ │ │ └── rocketmq.md │ │ ├── selectdb │ │ │ ├── selectdb-example.md │ │ │ └── selectdb.md │ │ └── streamingfile │ │ │ ├── streamingfile.md │ │ │ └── streamingfile_example.md │ ├── faq │ │ ├── README.md │ │ ├── e2e.md │ │ └── parallelism.md │ ├── introduce.md │ └── start │ │ ├── README.md │ │ ├── config.md │ │ ├── deployment.md │ │ ├── env_setup.md │ │ └── quick_guide.md │ ├── release │ └── release.md │ ├── team │ └── team.md │ └── usercases │ ├── case.md │ └── cases │ └── bytedance.md └── yarn.lock /.github/workflows/deploy-website.yml: -------------------------------------------------------------------------------- 1 | 2 | name: 部署文档 3 | 4 | on: 5 | push: 6 | branches: 7 | - master 8 | paths: 9 | - 'website/**' 10 | - 'package.json' 11 | - 'yarn.lock' 12 | 13 | jobs: 14 | deploy-gh-pages: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v3 19 | with: 20 | fetch-depth: 0 21 | 22 | - name: 设置 Node.js 23 | uses: actions/setup-node@v3 24 | with: 25 | node-version: 18 26 | cache: yarn 27 | 28 | - name: 安装依赖 29 | run: yarn install --frozen-lockfile 30 | 31 | - name: 构建文档 32 | env: 33 | NODE_OPTIONS: --max_old_space_size=8192 34 | run: |- 35 | yarn run docs:build 36 | > website/.vuepress/dist/.nojekyll 37 | 38 | - name: 部署文档 39 | uses: JamesIves/github-pages-deploy-action@v4 40 | with: 41 | branch: gh-pages 42 | folder: website/.vuepress/dist 43 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | .idea 3 | */target 4 | target 5 | */*.iml 6 | *.iml 7 | output 8 | */.classpath 9 | */.factorypath 10 | */.project 11 | */.settings 12 | *dependency-reduced-pom.xml 13 | .settings 14 | .project 15 | *pom.xml.versionsBackup 16 | .DS_Store 17 | */.DS_Store 18 | .flattened-pom.xml 19 | */*.flattened-pom.xml 20 | node_modules/ 21 | **/.vuepress/.cache/ 22 | **/.vuepress/.temp/ 23 | **/.vuepress/dist/ 24 | yarn-error.log 25 | yarn.lock 26 | package.json -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- 2 | 3 | This repository also contains code from the following projects under Apache-2.0 license: 4 | alibaba/DataX, 5 | alink_core_flink, 6 | Apache Flink, 7 | Apache HBase, 8 | apache/hudi, 9 | guava-retrying, 10 | Guava: Google Core Libraries for Java 11 | Files from these projects have notices at the top of each file. 12 | 13 | -------------------------------------------------------------------------------- 14 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/component/ComponentBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.component; 18 | 19 | import com.bytedance.bitsail.base.extension.Component; 20 | 21 | import java.io.Serializable; 22 | 23 | /** 24 | * Created 2022/8/31 25 | */ 26 | public interface ComponentBuilder extends Serializable, Component { 27 | 28 | 29 | } 30 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/connector/BuilderGroup.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.connector; 18 | 19 | public enum BuilderGroup { 20 | 21 | READER, 22 | WRITER, 23 | TRANSFORMER; 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/connector/reader/v1/Boundedness.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.connector.reader.v1; 18 | 19 | import java.io.Serializable; 20 | 21 | public enum Boundedness implements Serializable { 22 | 23 | BOUNDEDNESS, 24 | 25 | UNBOUNDEDNESS 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/connector/reader/v1/SourceEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.connector.reader.v1; 18 | 19 | import java.io.Serializable; 20 | 21 | public interface SourceEvent extends Serializable { 22 | 23 | } 24 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/connector/reader/v1/SourcePipeline.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.connector.reader.v1; 18 | 19 | import java.io.IOException; 20 | 21 | public interface SourcePipeline { 22 | 23 | void output(T record) throws IOException; 24 | 25 | void output(T record, long timestamp) throws IOException; 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/connector/reader/v1/SourceSplit.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.connector.reader.v1; 18 | 19 | import java.io.Serializable; 20 | 21 | public interface SourceSplit extends Serializable { 22 | 23 | String uniqSplitId(); 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/connector/transform/v1/MapTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.connector.transform.v1; 18 | 19 | import com.bytedance.bitsail.common.row.Row; 20 | 21 | public interface MapTransformer extends Transformer { 22 | 23 | T map(T element); 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/connector/transform/v1/PartitionTransformer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.connector.transform.v1; 18 | 19 | import com.bytedance.bitsail.common.row.Row; 20 | 21 | public interface PartitionTransformer extends Transformer { 22 | 23 | K selectKey(I element); 24 | 25 | int partition(K key, int totalPartitions); 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/connector/writer/v1/state/EmptyState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.connector.writer.v1.state; 18 | 19 | import java.io.Serializable; 20 | 21 | public class EmptyState implements Serializable { 22 | 23 | public static EmptyState fromBytes() { 24 | return new EmptyState(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/decoder/MessageDecoder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.decoder; 18 | 19 | import java.io.Serializable; 20 | 21 | /** 22 | * Created 2020/10/20. 23 | */ 24 | public interface MessageDecoder extends Serializable { 25 | 26 | /** 27 | * decode input byte array. 28 | */ 29 | byte[] decode(byte[] input); 30 | } 31 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/enumerate/ContentType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.enumerate; 18 | 19 | /** 20 | * Created at 2018/10/29. 21 | */ 22 | public enum ContentType { 23 | JSON, 24 | PROTOBUF, 25 | CSV, 26 | PLAIN, 27 | 28 | /** 29 | * ProtoBuf sequence file 30 | */ 31 | BINARY 32 | } 33 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/extension/Component.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.extension; 18 | 19 | import java.io.Serializable; 20 | 21 | /** 22 | * Created 2022/8/31 23 | */ 24 | public interface Component extends Serializable { 25 | 26 | String getComponentName(); 27 | } 28 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/extension/SupportProducedType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.extension; 18 | 19 | import com.bytedance.bitsail.common.typeinfo.RowTypeInfo; 20 | 21 | import java.io.Serializable; 22 | 23 | public interface SupportProducedType extends Serializable { 24 | 25 | RowTypeInfo getProducedType(); 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/extension/TypeInfoConverterFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.extension; 18 | 19 | import com.bytedance.bitsail.common.type.TypeInfoConverter; 20 | 21 | import java.io.Serializable; 22 | 23 | public interface TypeInfoConverterFactory extends Serializable { 24 | 25 | TypeInfoConverter createTypeInfoConverter(); 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/format/DeserializationFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.format; 18 | 19 | import com.bytedance.bitsail.common.typeinfo.TypeInfo; 20 | 21 | import java.io.Serializable; 22 | 23 | public interface DeserializationFormat extends Serializable { 24 | 25 | DeserializationSchema createRuntimeDeserializationSchema(TypeInfo[] typeInfos); 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/format/DeserializationFormatFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.format; 18 | 19 | import com.bytedance.bitsail.common.configuration.BitSailConfiguration; 20 | 21 | public interface DeserializationFormatFactory { 22 | 23 | DeserializationFormat createDeserializationFormat(BitSailConfiguration configuration); 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/format/DeserializationSchema.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.format; 18 | 19 | import java.io.Serializable; 20 | 21 | public interface DeserializationSchema extends Serializable { 22 | 23 | O deserialize(I message); 24 | 25 | boolean isEndOfStream(O nextElement); 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/format/SerializationFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.format; 18 | 19 | import com.bytedance.bitsail.common.typeinfo.TypeInfo; 20 | 21 | import java.io.Serializable; 22 | 23 | public interface SerializationFormat extends Serializable { 24 | 25 | T createRuntimeSerializationSchema(TypeInfo[] typeInfos); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/format/SerializationFormatFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.format; 18 | 19 | import com.bytedance.bitsail.common.configuration.BitSailConfiguration; 20 | 21 | public interface SerializationFormatFactory { 22 | 23 | SerializationFormat createSerializationFormat(BitSailConfiguration configuration); 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/format/SerializationSchema.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.format; 18 | 19 | import java.io.Serializable; 20 | 21 | public interface SerializationSchema extends Serializable { 22 | 23 | byte[] serialize(T element); 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/messenger/common/MessageType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.messenger.common; 18 | 19 | /** 20 | * Created 2022/7/14 21 | */ 22 | public enum MessageType { 23 | 24 | SUCCESS, 25 | 26 | FAILED 27 | } 28 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/serializer/BinarySerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.serializer; 18 | 19 | /** 20 | * Created 2022/6/21 21 | */ 22 | public interface BinarySerializer extends VersionedSerializer { 23 | 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-base/src/main/java/com/bytedance/bitsail/base/serializer/RowConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.base.serializer; 18 | 19 | import com.bytedance.bitsail.common.row.Row; 20 | 21 | /** 22 | * Created 2022/6/21 23 | */ 24 | public interface RowConverter extends Converter { 25 | 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-base/src/main/resources/META-INF/services/com.bytedance.bitsail.base.dirty.DirtyCollectorBuilder: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.base.dirty.impl.NoopDirtyCollectorBuilder -------------------------------------------------------------------------------- /bitsail-base/src/main/resources/META-INF/services/com.bytedance.bitsail.base.metrics.reporter.MetricReporterBuilder: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.base.metrics.reporter.impl.LogMetricReporterBuilder 18 | com.bytedance.bitsail.base.metrics.reporter.impl.NopMetricReporterBuilder -------------------------------------------------------------------------------- /bitsail-base/src/main/resources/META-INF/services/com.bytedance.bitsail.base.packages.PluginFinder: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.base.packages.LocalFSPluginFinder -------------------------------------------------------------------------------- /bitsail-base/src/main/resources/META-INF/services/com.bytedance.bitsail.base.runtime.RuntimePluggable: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.base.runtime.metrics.BitSailClientMetricsPlugin 18 | com.bytedance.bitsail.base.runtime.progress.JobProgressPlugin -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/.dataleap.bitsail.version.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | git.build.version=1.0-SNAPSHOT 18 | git.commit.id=123 -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/classloader/plugin/test1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/bitsail-base/src/test/resources/classloader/plugin/test1 -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/classloader/plugin/test2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/bitsail-base/src/test/resources/classloader/plugin/test2 -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/classloader/plugin/test3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/bitsail-base/src/test/resources/classloader/plugin/test3 -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/classloader/plugin/test4-1.0-SNAPSHOT: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/bitsail-base/src/test/resources/classloader/plugin/test4-1.0-SNAPSHOT -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/classloader/plugin_conf/dts_io.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail_io", 3 | "class": "", 4 | "libs": [ 5 | "test3" 6 | ] 7 | } -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/classloader/plugin_conf/test.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test1", 3 | "class": "com.bytedance.bitsail.batch.test1", 4 | "libs": [ 5 | "test1", 6 | "test2", 7 | "test4-${version}" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/classloader/plugin_conf/test2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test2", 3 | "class": "com.bytedance.bitsail.batch.test2", 4 | "libs": [ 5 | "test2", 6 | "test1", 7 | "test3" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/classloader/plugin_conf/test3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test3", 3 | "libs": [ 4 | "test1" 5 | ] 6 | } -------------------------------------------------------------------------------- /bitsail-base/src/test/resources/classloader/static_lib.json: -------------------------------------------------------------------------------- 1 | { 2 | "all_static_libs": [ 3 | "test1", 4 | "test2", 5 | "test3" 6 | ] 7 | } -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-api/src/main/java/com/bytedance/bitsail/client/api/command/CommandAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.client.api.command; 18 | 19 | /** 20 | * Created 2022/8/5 21 | */ 22 | public interface CommandAction { 23 | 24 | String RUN_COMMAND = "run"; 25 | String STOP_COMMAND = "stop"; 26 | String HELP_COMMAND = "help"; 27 | } 28 | -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-api/src/main/java/com/bytedance/bitsail/client/api/command/CommandArgs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.client.api.command; 18 | 19 | import java.io.Serializable; 20 | 21 | /** 22 | * Created 2022/8/5 23 | */ 24 | public interface CommandArgs extends Serializable { 25 | } 26 | -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-entry/src/test/resources/META-INF/services/com.bytedance.bitsail.client.api.engine.EngineRunner: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.client.entry.FakeEngine -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-entry/src/test/resources/test_job_conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": 312, 5 | "instance_id": 3124, 6 | "job_name": "BitSail_fake_to_print_test", 7 | "user_name": "test" 8 | }, 9 | "writer": { 10 | "class": "com.bytedance.bitsail.connector.print.sink.PrintSink", 11 | "columns": [ 12 | { 13 | "name": "name", 14 | "type": "string" 15 | }, 16 | { 17 | "name": "age", 18 | "type": "int" 19 | } 20 | ] 21 | }, 22 | "reader": { 23 | "class": "com.bytedance.bitsail.connector.fake.source.FakeSource", 24 | "total_count": 30, 25 | "rate": 10, 26 | "null_percentage": 20, 27 | "columns": [ 28 | { 29 | "name": "name", 30 | "type": "string" 31 | }, 32 | { 33 | "name": "age", 34 | "type": "int" 35 | } 36 | ] 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-1.11/src/main/resources/META-INF/services/com.bytedance.bitsail.client.api.engine.EngineRunner: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.entry.flink.engine.FlinkEngineRunner -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-1.11/src/main/resources/bitsail-client-flink-1.11.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flink", 3 | "libs": [ 4 | "bitsail-client-flink-1.11-${version}.jar" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/conf/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/examples/Fake_Print_Example.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": 312, 5 | "instance_id": 3124, 6 | "job_name": "bitsail_fake_to_print_test", 7 | "user_name": "test" 8 | }, 9 | "writer": { 10 | "class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink" 11 | }, 12 | "reader": { 13 | "class": "com.bytedance.bitsail.connector.legacy.fake.source.FakeSource", 14 | "total_count": 300, 15 | "rate": 1, 16 | "random_null_rate": 0.1, 17 | "columns": [ 18 | { 19 | "name": "name", 20 | "type": "string" 21 | }, 22 | { 23 | "name": "age", 24 | "type": "int" 25 | } 26 | ] 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/hadoop/bin/hadoop.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/hadoop/bin/hadoop.dll -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/hadoop/bin/winutils.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/hadoop/bin/winutils.exe -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/lib/file_a: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ################################################################################ 18 | -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/lib/flink-dist-a: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ################################################################################ 18 | -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/test_dir/lib/file_a: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ################################################################################ 18 | -------------------------------------------------------------------------------- /bitsail-clients/bitsail-client-flink/bitsail-client-flink-base/src/test/resources/test_dir_no_conf_files/conf/file_a: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # 3 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | ################################################################################ 18 | -------------------------------------------------------------------------------- /bitsail-common/src/main/java/com/bytedance/bitsail/common/constants/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.common.constants; 18 | 19 | /** 20 | * 21 | */ 22 | public interface Constants { 23 | int RETRY_TIMES = 5; 24 | int RETRY_DELAY = 100; 25 | 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-common/src/main/java/com/bytedance/bitsail/common/exception/ErrorCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.common.exception; 18 | 19 | public interface ErrorCode { 20 | String getCode(); 21 | 22 | String getDescription(); 23 | 24 | @Override 25 | String toString(); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /bitsail-common/src/main/java/com/bytedance/bitsail/common/util/TypeConvertUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.common.util; 18 | 19 | /** 20 | * @desc: 21 | */ 22 | public class TypeConvertUtil { 23 | 24 | public enum StorageEngine { 25 | mysql, 26 | oracle, 27 | db2, 28 | sqlserver, 29 | hive, 30 | postgresql 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /bitsail-common/src/main/resources/bitsail-type-converter.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/bitsail-common/src/main/resources/bitsail-type-converter.yaml -------------------------------------------------------------------------------- /bitsail-common/src/test/resources/fake-type-converter.yaml: -------------------------------------------------------------------------------- 1 | engine.type.to.bitsail.type.converter: 2 | - source.type: boolean 3 | target.type: boolean 4 | 5 | - source.type: bytes 6 | target.type: bytes 7 | 8 | - source.type: date 9 | target.type: date.date 10 | 11 | - source.type: long 12 | target.type: long 13 | 14 | - source.type: string 15 | target.type: string 16 | 17 | - source.type: double 18 | target.type: double 19 | 20 | - source.type: int 21 | target.type: long 22 | 23 | - source.type: bigint 24 | target.type: long 25 | 26 | 27 | bitsail.type.to.engine.type.converter: 28 | - source.type: long 29 | target.type: long 30 | 31 | - source.type: double 32 | target.type: double 33 | 34 | - source.type: string 35 | target.type: string 36 | 37 | - source.type: boolean 38 | target.type: boolean 39 | 40 | - source.type: date.date 41 | target.type: string 42 | 43 | - source.type: bytes 44 | target.type: bytes 45 | -------------------------------------------------------------------------------- /bitsail-components/bitsail-component-metrics/bitsail-component-metrics-prometheus/src/main/resources/META-INF/services/com.bytedance.bitsail.base.metrics.reporter.MetricReporterBuilder: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.component.metrics.prometheus.impl.PrometheusMetricReporterBuilder 18 | com.bytedance.bitsail.component.metrics.prometheus.impl.PrometheusPushGatewayMetricReporterBuilder -------------------------------------------------------------------------------- /bitsail-components/bitsail-component-security/bitsail-component-security-kerberos/src/main/resources/META-INF/services/com.bytedance.bitsail.base.extension.SecurityModule: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.component.format.security.kerberos.security.HadoopSecurityModule -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-fake/src/main/resources/bitsail-connector-legacy-fake.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-fake", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.fake.source.FakeSource", 5 | "com.bytedance.bitsail.connector.legacy.fake.source.FakeFlinkSource" 6 | ], 7 | "libs": [ 8 | "bitsail-connector-fake-${version}.jar" 9 | ] 10 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-ftp/src/main/resources/bitsail-connector-legacy-ftp.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-ftp", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.ftp.source.FtpInputFormat" 5 | ], 6 | "libs": [ 7 | "bitsail-connector-ftp-${version}.jar" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hadoop/src/main/resources/bitsail-connector-legacy-hadoop.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-hadoop", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.hadoop.source.HadoopInputFormat" 5 | ], 6 | "libs": [ 7 | "bitsail-connector-hadoop-${version}.jar" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hbase/src/main/java/com/bytedance/bitsail/connector/hbase/constant/NullMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.hbase.constant; 18 | 19 | public enum NullMode { 20 | /** 21 | * Skip the null value. 22 | */ 23 | SKIP, 24 | /** 25 | * Directly use null or replace the value with "" (empty string). 26 | */ 27 | EMPTY; 28 | } 29 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hbase/src/main/java/com/bytedance/bitsail/connector/hbase/sink/function/IFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.hbase.sink.function; 18 | 19 | public interface IFunction { 20 | String evaluate(Object val) throws Exception; 21 | } 22 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hbase/src/main/java/com/bytedance/bitsail/connector/hbase/sink/function/StringFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.hbase.sink.function; 18 | 19 | public class StringFunction implements IFunction { 20 | 21 | @Override 22 | public String evaluate(Object str) { 23 | return str.toString(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hbase/src/main/resources/bitsail-connector-legacy-hbase.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-hbase", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.hbase.source.HBaseInputFormat", 5 | "com.bytedance.bitsail.connector.hbase.sink.HBaseOutputFormat" 6 | ], 7 | "libs": [ 8 | "bitsail-connector-hbase-${version}.jar", 9 | "bitsail-connector-hadoop-${version}.jar" 10 | ] 11 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.TableCatalogFactory: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.connector.legacy.hive.common.HiveTableCatalogFactory -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/main/resources/bitsail-connector-legacy-hive.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-hive", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.hive.source.HiveInputFormat", 5 | "com.bytedance.bitsail.connector.legacy.hive.sink.HiveOutputFormat" 6 | ], 7 | "libs": [ 8 | "bitsail-connector-hive-${version}.jar" 9 | ] 10 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/test/resources/hadoop/bin/winutils.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/test/resources/hadoop/bin/winutils.exe -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hive/src/test/resources/hive/hive_writer_without_columns.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "writer": { 4 | "write_mode": "overwrite", 5 | "partition": "date=20210916", 6 | "db_name": "test", 7 | "metastore_properties": "{\"metastore_uri\":\"thrift://:9083\"}", 8 | "table_name": "bitsail_unit_test_hive_output" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hudi/src/main/resources/bitsail-connector-legacy-hudi.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-hudi", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.hudi.dag.HudiSinkFunctionDAGBuilder", 5 | "com.bytedance.bitsail.connector.legacy.hudi.dag.HudiSourceFunctionDAGBuilder", 6 | "com.bytedance.bitsail.connector.legacy.hudi.dag.HudiCompactSinkDAGBuilder", 7 | "com.bytedance.bitsail.connector.legacy.hudi.dag.HudiCompactSourceDAGBuilder" 8 | ], 9 | "libs": ["bitsail-connector-hudi-${version}.jar"] 10 | } 11 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hudi/src/test/resources/test_source.data: -------------------------------------------------------------------------------- 1 | {"uuid": "id1", "name": "Danny", "age": 23, "ts": "1970-01-01T00:00:01", "partition": "par1"} 2 | {"uuid": "id2", "name": "Stephen", "age": 33, "ts": "1970-01-01T00:00:02", "partition": "par1"} 3 | {"uuid": "id3", "name": "Julian", "age": 53, "ts": "1970-01-01T00:00:03", "partition": "par2"} 4 | {"uuid": "id4", "name": "Fabian", "age": 31, "ts": "1970-01-01T00:00:04", "partition": "par2"} 5 | {"uuid": "id5", "name": "Sophia", "age": 18, "ts": "1970-01-01T00:00:05", "partition": "par3"} 6 | {"uuid": "id6", "name": "Emma", "age": 20, "ts": "1970-01-01T00:00:06", "partition": "par3"} 7 | {"uuid": "id7", "name": "Bob", "age": 44, "ts": "1970-01-01T00:00:07", "partition": "par4"} 8 | {"uuid": "id8", "name": "Han", "age": 56, "ts": "1970-01-01T00:00:08", "partition": "par4"} 9 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hudi/src/test/resources/test_source_2.data: -------------------------------------------------------------------------------- 1 | {"uuid": "id1", "name": "Danny", "age": 24, "ts": "1970-01-01T00:00:01", "partition": "par1"} 2 | {"uuid": "id2", "name": "Stephen", "age": 34, "ts": "1970-01-01T00:00:02", "partition": "par1"} 3 | {"uuid": "id3", "name": "Julian", "age": 54, "ts": "1970-01-01T00:00:03", "partition": "par2"} 4 | {"uuid": "id4", "name": "Fabian", "age": 32, "ts": "1970-01-01T00:00:04", "partition": "par2"} 5 | {"uuid": "id5", "name": "Sophia", "age": 18, "ts": "1970-01-01T00:00:05", "partition": "par3"} 6 | {"uuid": "id9", "name": "Jane", "age": 19, "ts": "1970-01-01T00:00:06", "partition": "par3"} 7 | {"uuid": "id10", "name": "Ella", "age": 38, "ts": "1970-01-01T00:00:07", "partition": "par4"} 8 | {"uuid": "id11", "name": "Phoebe", "age": 52, "ts": "1970-01-01T00:00:08", "partition": "par4"} 9 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hudi/src/test/resources/test_source_3.data: -------------------------------------------------------------------------------- 1 | {"uuid": "id1", "name": "Danny", "age": 24, "ts": "1970-01-01T00:00:01", "partition": "par1"} 2 | {"uuid": "id2", "name": "Stephen", "age": 34, "ts": "1970-01-01T00:00:02", "partition": "par1"} 3 | {"uuid": "id1", "name": "Danny", "age": 24, "ts": "1970-01-01T00:00:01", "partition": "par1"} 4 | {"uuid": "id2", "name": "Stephen", "age": 34, "ts": "1970-01-01T00:00:02", "partition": "par1"} 5 | {"uuid": "id1", "name": "Danny", "age": 24, "ts": "1970-01-01T00:00:01", "partition": "par1"} 6 | {"uuid": "id2", "name": "Stephen", "age": 34, "ts": "1970-01-01T00:00:02", "partition": "par1"} 7 | {"uuid": "id1", "name": "Danny", "age": 24, "ts": "1970-01-01T00:00:01", "partition": "par1"} 8 | {"uuid": "id2", "name": "Stephen", "age": 34, "ts": "1970-01-01T00:00:02", "partition": "par1"} 9 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hudi/src/test/resources/test_source_4.data: -------------------------------------------------------------------------------- 1 | {"uuid": "id1", "name": "Danny", "age": 24, "ts": "1970-01-01T00:00:01", "partition": "par1"} 2 | {"uuid": "id1", "name": "Stephen", "age": 34, "ts": "1970-01-01T00:00:02", "partition": "par1"} 3 | {"uuid": "id1", "name": "Julian", "age": 54, "ts": "1970-01-01T00:00:03", "partition": "par2"} 4 | {"uuid": "id1", "name": "Fabian", "age": 32, "ts": "1970-01-01T00:00:04", "partition": "par2"} 5 | {"uuid": "id1", "name": "Sophia", "age": 18, "ts": "1970-01-01T00:00:05", "partition": "par3"} 6 | {"uuid": "id1", "name": "Jane", "age": 19, "ts": "1970-01-01T00:00:06", "partition": "par3"} 7 | {"uuid": "id1", "name": "Ella", "age": 38, "ts": "1970-01-01T00:00:07", "partition": "par4"} 8 | {"uuid": "id1", "name": "Phoebe", "age": 52, "ts": "1970-01-01T00:00:08", "partition": "par4"} 9 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-hudi/src/test/resources/test_source_5.data: -------------------------------------------------------------------------------- 1 | id1,Danny,23,1970-01-01 00:00:01,par1 2 | id2,Stephen,33,1970-01-01 00:00:02,par1 3 | id3,Julian,53,1970-01-01 00:00:03,par2 4 | id4,Fabian,31,1970-01-01 00:00:04,par2 5 | id5,Sophia,18,1970-01-01 00:00:05,par3 6 | id6,Emma,20,1970-01-01 00:00:06,par3 7 | id7,Bob,44,1970-01-01 00:00:07,par4 8 | id8,Han,56,1970-01-01 00:00:08,par4 9 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/model/oracle/ServiceNameType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.legacy.jdbc.model.oracle; 18 | 19 | /** 20 | * Created 2022/8/15 21 | */ 22 | 23 | public enum ServiceNameType { 24 | SERVICE_NAME, 25 | SID 26 | } 27 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/java/com/bytedance/bitsail/connector/legacy/jdbc/sink/PrepareStatementSqlFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.legacy.jdbc.sink; 18 | 19 | import java.io.IOException; 20 | 21 | @FunctionalInterface 22 | public interface PrepareStatementSqlFunction { 23 | R apply(T t) throws IOException; 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/resources/META-INF/services/com.bytedance.bitsail.base.catalog.TableCatalogFactory: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.connector.legacy.jdbc.catalog.MySQLTableCatalogFactory -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-jdbc/src/main/resources/bitsail-connector-legacy-jdbc.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-jdbc", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.jdbc.sink.JDBCOutputFormat", 5 | "com.bytedance.bitsail.connector.legacy.jdbc.source.JDBCInputFormat", 6 | "com.bytedance.bitsail.connector.legacy.jdbc.sink.PostgresqlOutputFormat", 7 | "com.bytedance.bitsail.connector.legacy.jdbc.source.PostgresqlInputFormat", 8 | "com.bytedance.bitsail.connector.legacy.jdbc.sink.SqlServerOutputFormat", 9 | "com.bytedance.bitsail.connector.legacy.jdbc.source.SqlServerInputFormat", 10 | "com.bytedance.bitsail.connector.legacy.jdbc.sink.OracleOutputFormat", 11 | "com.bytedance.bitsail.connector.legacy.jdbc.source.OracleInputFormat" 12 | ], 13 | "libs": [ 14 | "bitsail-connector-jdbc-${version}.jar" 15 | ] 16 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-kafka/src/main/resources/bitsail-connector-legacy-kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-kafka", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.kafka.source.KafkaSourceFunctionDAGBuilder", 5 | "com.bytedance.bitsail.connector.legacy.kafka.sink.KafkaOutputFormat" 6 | ], 7 | "libs": [ 8 | "bitsail-connector-kafka-${version}.jar" 9 | ] 10 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-larksheet/src/main/resources/bitsail-connector-legacy-larksheet.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-larksheet", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.larksheet.source.LarkSheetInputFormat" 5 | ], 6 | "libs": [ 7 | "bitsail-connector-larksheet-${version}.jar" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-mongodb/src/main/java/com/bytedance/bitsail/connector/legacy/mongodb/constant/MongoDBConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.legacy.mongodb.constant; 18 | 19 | public class MongoDBConstants { 20 | 21 | public static final String CONNECTOR_TYPE_NAME = "mongodb"; 22 | public static final int MAX_PARALLELISM_OUTPUT_MONGODB = 5; 23 | } 24 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-mongodb/src/main/resources/bitsail-connector-legacy-mongodb.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-mongodb", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.mongodb.source.MongoDBInputFormat", 5 | "com.bytedance.bitsail.connector.legacy.mongodb.sink.MongoDBOutputFormat" 6 | ], 7 | "libs": [ 8 | "bitsail-connector-mongodb-${version}.jar" 9 | ] 10 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-print/src/main/resources/bitsail-connector-legacy-print.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-print", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink" 5 | ], 6 | "libs": [ 7 | "bitsail-connector-print-${version}.jar" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-redis/src/main/resources/bitsail-connector-legacy-redis.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-redis", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.redis.source.RedisOutputFormat" 5 | ], 6 | "libs": [ 7 | "bitsail-connector-redis-${version}.jar" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-rocketmq/src/main/java/com/bytedance/bitsail/connector/legacy/rocketmq/constant/RocketMQConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.legacy.rocketmq.constant; 18 | 19 | public class RocketMQConstants { 20 | 21 | public static final String CONNECTOR_NAME = "rocketmq"; 22 | 23 | public static final int MAX_PARALLELISM_OUTPUT_ROCKETMQ = 5; 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-rocketmq/src/main/java/com/bytedance/bitsail/connector/legacy/rocketmq/sink/format/RocketMQSinkFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.legacy.rocketmq.sink.format; 18 | 19 | public enum RocketMQSinkFormat { 20 | JSON 21 | } 22 | -------------------------------------------------------------------------------- /bitsail-connectors/bitsail-connectors-legacy/bitsail-connector-streamingfile/bitsail-connector-streamingfile-core/src/main/resources/bitsail-connector-legacy-streamingfile.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-legacy-streamingfile", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.legacy.streamingfile.sink.FileSystemSinkFunctionDAGBuilder" 5 | ], 6 | "libs": [ 7 | "bitsail-connector-streamingfile-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-assert/src/main/java/com/bytedance/bitsail/connector/assertion/sink/constants/AssertConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.assertion.sink.constants; 18 | 19 | public class AssertConstants { 20 | 21 | public static final int DEFAULT_PARALLELISM = 1; 22 | } 23 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-assert/src/main/java/com/bytedance/bitsail/connector/assertion/sink/constants/AssertRuleType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.assertion.sink.constants; 18 | 19 | public enum AssertRuleType { 20 | 21 | NOT_NULL, 22 | MIN, 23 | MAX, 24 | MIN_LEN, 25 | MAX_LEN, 26 | MIN_ROW, 27 | MAX_ROW; 28 | } 29 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-assert/src/main/resources/bitsail-connector-unified-assert.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-assert", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.assertion.sink.AssertSink" 5 | ], 6 | "libs": [ 7 | "connector-assert-${version}.jar" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-connectors/connector-base/src/main/java/com/bytedance/bitsail/connector/base/source/SimpleSourceSplit.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.base.source; 18 | 19 | import com.bytedance.bitsail.base.connector.reader.v1.SourceSplit; 20 | 21 | public class SimpleSourceSplit implements SourceSplit { 22 | 23 | @Override 24 | public String uniqSplitId() { 25 | return ""; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-base/src/main/java/com/bytedance/bitsail/connector/base/source/SimpleSourceState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.base.source; 18 | 19 | import java.io.Serializable; 20 | 21 | public class SimpleSourceState implements Serializable { 22 | } 23 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/constant/BinlogReaderConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.cdc.constant; 18 | 19 | public class BinlogReaderConstants { 20 | public static String CDC_CONNECTOR_NAME = "cdc"; 21 | } 22 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-cdc/connector-cdc-base/src/main/java/com/bytedance/bitsail/connector/cdc/source/split/SplitType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.cdc.source.split; 18 | 19 | public class SplitType { 20 | public static final int BINLOG_SPLIT_TYPE = 1; 21 | public static final int SNAPSHOT_SPLIT_TYPE = 2; 22 | } 23 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/java/com/bytedance/bitsail/connector/cdc/mysql/source/constant/MysqlConstant.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.cdc.mysql.source.constant; 18 | 19 | public class MysqlConstant { 20 | public static String BINLOG_PROPS_FILENAME = "binlog_filename"; 21 | public static String BINLOG_PROPS_OFFSET = "binlog_offset"; 22 | 23 | } 24 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-cdc/connector-cdc-mysql/src/main/resources/bitsail-connector-unified-mysqlcdc.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-mysqlcdc", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.cdc.mysql.source.MysqlCDCSource" 5 | ], 6 | "libs": [ 7 | "connector-cdc-mysql-${version}.jar" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-connectors/connector-cdc/connector-cdc-sqlserver/src/main/resources/bitsail-connector-unified-sqlserver.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-sqlserver", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.cdc.sqlserver.source.SqlServerCDCSource" 5 | ], 6 | "libs": [ 7 | "connector-cdc-sqlserver-${version}.jar" 8 | ] 9 | } -------------------------------------------------------------------------------- /bitsail-connectors/connector-cdc/connector-cdc-sqlserver/src/test/resources/container-license-acceptance.txt: -------------------------------------------------------------------------------- 1 | mcr.microsoft.com/mssql/server:2019-latest -------------------------------------------------------------------------------- /bitsail-connectors/connector-clickhouse/src/main/java/com/bytedance/bitsail/connector/clickhouse/constant/ClickhouseConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.clickhouse.constant; 18 | 19 | public class ClickhouseConstants { 20 | public static String CLICKHOUSE_CONNECTOR_NAME = "clickhouse"; 21 | } 22 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-clickhouse/src/main/resources/bitsail-connector-unified-clickhouse.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-clickhouse", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.clickhouse.source.ClickhouseSource" 5 | ], 6 | "libs": [ 7 | "connector-clickhouse-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-doris/src/main/java/com/bytedance/bitsail/connector/doris/constant/DorisConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.doris.constant; 18 | 19 | public class DorisConstants { 20 | public static final String NULL_VALUE = "\\N"; 21 | public static final String DORIS_DELETE_SIGN = "__DORIS_DELETE_SIGN__"; 22 | } 23 | 24 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-doris/src/main/resources/bitsail-connector-unified-doris.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-doris", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.doris.sink.DorisSink" 5 | ], 6 | "libs": [ 7 | "bitsail-connector-doris-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-druid/src/main/resources/bitsail-connector-unified-druid.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-druid", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.druid.sink.DruidSink" 5 | ], 6 | "libs": [ 7 | "bitsail-connector-druid-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-druid/src/main/resources/druid-type-converter.yaml: -------------------------------------------------------------------------------- 1 | engine.type.to.bitsail.type.converter: 2 | - source.type: char 3 | target.type: string 4 | 5 | - source.type: varchar 6 | target.type: string 7 | 8 | - source.type: string 9 | target.type: string 10 | 11 | - source.type: decimal 12 | target.type: double 13 | 14 | - source.type: float 15 | target.type: float 16 | 17 | - source.type: double 18 | target.type: double 19 | 20 | - source.type: real 21 | target.type: double 22 | 23 | - source.type: boolean 24 | target.type: long 25 | 26 | - source.type: tinyint 27 | target.type: long 28 | 29 | - source.type: smallint 30 | target.type: long 31 | 32 | - source.type: int 33 | target.type: long 34 | 35 | - source.type: long 36 | target.type: long 37 | 38 | - source.type: integer 39 | target.type: long 40 | 41 | - source.type: bigint 42 | target.type: long 43 | 44 | - source.type: timestamp 45 | target.type: long 46 | 47 | - source.type: date 48 | target.type: long -------------------------------------------------------------------------------- /bitsail-connectors/connector-druid/src/test/resources/druid_writer.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "writer": { 4 | "class": "com.bytedance.bitsail.connector.druid.sink.DruidSink", 5 | "datasource": "testDataSource", 6 | "coordinator_url": "localhost", 7 | "columns": [ 8 | { 9 | "index": 0, 10 | "name": "string_type", 11 | "type": "string" 12 | }, 13 | { 14 | "index": 1, 15 | "name": "int_type", 16 | "type": "int" 17 | }, 18 | { 19 | "index": 2, 20 | "name": "long_type", 21 | "type": "long" 22 | }, 23 | { 24 | "index": 3, 25 | "name": "float_type", 26 | "type": "float" 27 | }, 28 | { 29 | "index": 4, 30 | "name": "double_type", 31 | "type": "double" 32 | } 33 | ] 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-elasticsearch/src/main/java/com/bytedance/bitsail/connector/elasticsearch/constants/Elasticsearchs.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.elasticsearch.constants; 18 | 19 | public class Elasticsearchs { 20 | 21 | public static final String COMMA = ","; 22 | } 23 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-elasticsearch/src/main/resources/bitsail-connector-unified-elasticsearch.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-elasticsearch", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.elasticsearch.sink.ElasticsearchSink" 5 | ], 6 | "libs": [ 7 | "connector-elasticsearch-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-fake/src/main/resources/bitsail-connector-unified-fake.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-fake", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.fake.source.FakeSource" 5 | ], 6 | "libs": [ 7 | "connector-fake-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-ftp/src/main/java/com/bytedance/bitsail/connector/ftp/core/FtpConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.ftp.core; 18 | 19 | public class FtpConstants { 20 | 21 | public static String FTP_CONNECTOR_NAME = "ftp"; 22 | } 23 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-ftp/src/main/resources/bitsail-connector-unified-ftp.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-ftp", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.ftp.source.FtpSource" 5 | ], 6 | "libs": [ 7 | "connector-ftp-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-hbase/src/main/java/com/bytedance/bitsail/connector/hbase/constant/NullMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.hbase.constant; 18 | 19 | public enum NullMode { 20 | /** 21 | * Skip the null value. 22 | */ 23 | SKIP, 24 | /** 25 | * Directly use null or replace the value with "" (empty string). 26 | */ 27 | EMPTY; 28 | } 29 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-hbase/src/main/java/com/bytedance/bitsail/connector/hbase/sink/function/IFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.hbase.sink.function; 18 | 19 | public interface IFunction { 20 | String evaluate(Object val) throws Exception; 21 | } 22 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-hbase/src/main/java/com/bytedance/bitsail/connector/hbase/sink/function/StringFunction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.hbase.sink.function; 18 | 19 | public class StringFunction implements IFunction { 20 | 21 | @Override 22 | public String evaluate(Object str) { 23 | return str.toString(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-hbase/src/main/resources/bitsail-connector-unified-hbase.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-hbase", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.hbase.source.HBaseSource", 5 | "com.bytedance.bitsail.connector.hbase.sink.HBaseSink" 6 | ], 7 | "libs": [ 8 | "connector-hbase-${version}.jar" 9 | ] 10 | } -------------------------------------------------------------------------------- /bitsail-connectors/connector-kafka/src/main/java/com/bytedance/bitsail/connector/kafka/constants/FormatType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.kafka.constants; 18 | 19 | public enum FormatType { 20 | 21 | JSON, 22 | 23 | DEBEZIUM_JSON 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-kafka/src/main/java/com/bytedance/bitsail/connector/kafka/constants/KafkaConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.kafka.constants; 18 | 19 | public class KafkaConstants { 20 | public static final String CONNECTOR_TYPE_VALUE_KAFKA = "kafka"; 21 | } 22 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-kafka/src/main/resources/bitsail-connector-unified-kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-kafka", 3 | "classes": ["com.bytedance.bitsail.connector.kafka.sink.KafkaSink"], 4 | "libs": ["connector-kafka-${version}.jar"] 5 | } -------------------------------------------------------------------------------- /bitsail-connectors/connector-kudu/src/main/java/com/bytedance/bitsail/connector/kudu/core/KuduConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.kudu.core; 18 | 19 | public class KuduConstants { 20 | 21 | public static String KUDU_CONNECTOR_NAME = "kudu"; 22 | } 23 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-kudu/src/main/java/com/bytedance/bitsail/connector/kudu/type/KuduTypeConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.kudu.type; 18 | 19 | import com.bytedance.bitsail.common.type.filemapping.FileMappingTypeInfoConverter; 20 | 21 | public class KuduTypeConverter extends FileMappingTypeInfoConverter { 22 | public KuduTypeConverter() { 23 | super("kudu"); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-kudu/src/main/resources/bitsail-connector-unified-kudu.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-kudu", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.kudu.sink.KuduSink", 5 | "com.bytedance.bitsail.connector.kudu.source.KuduSource" 6 | ], 7 | "libs": [ 8 | "connector-kudu-${version}.jar" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-larksheet/src/main/resources/bitsail-connector-unified-larksheet.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-larksheet", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.larksheet.source.LarkSheetSource" 5 | ], 6 | "libs": [ 7 | "connector-larksheet-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-localfilesystem/src/main/resources/bitsail-connector-unified-localfilesystem.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-localfilesystem", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.localfilesystem.source.LocalFileSystemSource" 5 | ], 6 | "libs": [ 7 | "connector-localfilesystem-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-localfilesystem/src/test/resources/data/csv/test.csv: -------------------------------------------------------------------------------- 1 | 0,2050-11-26,931451976403,Roberto,false 2 | 1,1992-08-15,101947222173,Reggie,true 3 | 2,2074-04-22,1344242983149,Thomas,true 4 | 3,2065-11-13,2337343299527,Corey,false 5 | 4,2025-04-08,565336730103,Lee,true 6 | 5,2037-02-15,3008830815937,Ann,false 7 | 6,2045-07-16,1173180419599,Virgen,true 8 | 7,1975-06-26,1522037304441,Scott,false 9 | 8,2068-07-26,1211639748316,Crysta,false 10 | 9,2077-04-13,2856898427345,Lorette,false 11 | 10,1989-11-01,578487185312,Luis,true 12 | 11,2011-04-01,419227136697,Deandrea,true -------------------------------------------------------------------------------- /bitsail-connectors/connector-mongodb/src/main/resources/bitsail-connector-unified-mongodb.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-mongodb", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.mongodb.source.MongoDBSource" 5 | ], 6 | "libs": [ 7 | "connector-mongodb-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-print/src/main/resources/bitsail-connector-unified-print.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-print", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.print.sink.PrintSink" 5 | ], 6 | "libs": [ 7 | "connector-print-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-redis/src/main/java/com/bytedance/bitsail/connector/redis/constant/RedisConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.redis.constant; 18 | 19 | public class RedisConstants { 20 | /** 21 | * connector name 22 | */ 23 | public static final String REDIS_CONNECTOR_NAME = "redis"; 24 | } 25 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-redis/src/main/resources/bitsail-connector-unified-redis.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-redis", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.redis.sink.RedisSink" 5 | ], 6 | "libs": [ 7 | "connector-redis-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-rocketmq/src/main/java/com/bytedance/bitsail/connector/rocketmq/constants/RocketMQConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.rocketmq.constants; 18 | 19 | public class RocketMQConstants { 20 | 21 | public static final String CONNECTOR_NAME = "rocketmq"; 22 | 23 | } 24 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-rocketmq/src/main/java/com/bytedance/bitsail/connector/rocketmq/sink/format/RocketMQSinkFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.rocketmq.sink.format; 18 | 19 | public enum RocketMQSinkFormat { 20 | JSON 21 | } 22 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-rocketmq/src/main/resources/bitsail-connector-unified-rocketmq.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-rocketmq", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.rocketmq.source.RocketMQSource" 5 | ], 6 | "libs": [ 7 | "connector-rocketmq-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-selectdb/src/main/java/com/bytedance/bitsail/connector/selectdb/constant/SelectdbConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.selectdb.constant; 18 | 19 | public class SelectdbConstants { 20 | public static final String NULL_VALUE = "\\N"; 21 | public static final String SELECTDB_DELETE_SIGN = "__DELETE_SIGN__"; 22 | } 23 | 24 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-selectdb/src/main/java/com/bytedance/bitsail/connector/selectdb/sink/uploadload/LoadStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.connector.selectdb.sink.uploadload; 18 | 19 | public class LoadStatus { 20 | public static final int SUCCESS = 0; 21 | public static final String FAIL = "1"; 22 | } 23 | -------------------------------------------------------------------------------- /bitsail-connectors/connector-selectdb/src/main/resources/bitsail-connector-unified-selectdb.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-connector-unified-selectdb", 3 | "classes": [ 4 | "com.bytedance.bitsail.connector.selectdb.sink.SelectdbSink" 5 | ], 6 | "libs": [ 7 | "connector-selectdb-${version}.jar" 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-api/src/main/resources/META-INF/services/com.bytedance.bitsail.core.api.parser.ConfigurationParser: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.core.api.parser.DefaultConfigurationParser -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-api/src/test/resources/META-INF/services/com.bytedance.bitsail.core.api.interceptor.ConfigInterceptor: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.core.api.interceptor.FakeConfigInterceptor -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-entry/src/test/resources/META-INF/services/com.bytedance.bitsail.base.packages.PluginFinder: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.core.FakePluginFinder -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-entry/src/test/resources/META-INF/services/com.bytedance.bitsail.core.api.program.Program: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.core.program.FakeProgram -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-entry/src/test/resources/conf.json: -------------------------------------------------------------------------------- 1 | { 2 | "key": "value" 3 | } -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/resources/META-INF/services/com.bytedance.bitsail.core.api.program.Program: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.core.flink.bridge.program.FlinkProgram -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/resources/META-INF/services/com.bytedance.bitsail.flink.core.writer.bounded.BoundedDataStreamSinkBuilder: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.core.flink.bridge.writer.bounded.BridgeBoundedDataStreamSinkBuilder -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/main/resources/bitsail-core-flink-1.11-bridge.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flink", 3 | "libs": [ 4 | "bitsail-core-flink-1.11-bridge-${version}.jar" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/test/java/com/bytedance/bitsail/core/flink/bridge/transform/builder/FlinkBaseTransformerBuilderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.core.flink.bridge.transform.builder; 18 | 19 | public class FlinkBaseTransformerBuilderTest { 20 | 21 | } -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge/src/test/resources/META-INF/services/com.bytedance.bitsail.base.connector.transform.v1.Transformer: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.core.flink.bridge.transform.impl.MockMapTransformer -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.16-bridge/src/main/resources/META-INF/services/com.bytedance.bitsail.core.api.program.Program: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.core.flink116.bridge.program.Flink116Program -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.16-bridge/src/main/resources/META-INF/services/com.bytedance.bitsail.flink.core.writer.bounded.BoundedDataStreamSinkBuilder: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.core.flink116.bridge.writer.bounded.BridgeBoundedDataStreamSinkBuilder -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-base/src/main/java/com/bytedance/bitsail/flink/core/runtime/restart/RestartStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.flink.core.runtime.restart; 18 | 19 | public enum RestartStrategy { 20 | 21 | FAILURE_RATE_RESTART, 22 | 23 | FIXED_DELAY_RESTART; 24 | 25 | } -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-base/src/main/resources/META-INF/services/com.bytedance.bitsail.base.dirty.DirtyCollectorBuilder: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.flink.core.dirty.FlinkBatchDirtyCollectorBuilder -------------------------------------------------------------------------------- /bitsail-cores/bitsail-core-flink/bitsail-core-flink-base/src/main/resources/META-INF/services/com.bytedance.bitsail.base.messenger.MessengerBuilder: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.flink.core.runtime.messenger.FlinkAccumulatorMessengerBuilder -------------------------------------------------------------------------------- /bitsail-dist/src/main/resources/examples/Fake_Print_Example.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": 312, 5 | "instance_id": 3124, 6 | "job_name": "BitSail_fake_to_print_test", 7 | "user_name": "test" 8 | }, 9 | "writer": { 10 | "class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink" 11 | }, 12 | "reader": { 13 | "class": "com.bytedance.bitsail.connector.legacy.fake.source.FakeSource", 14 | "total_count": 300, 15 | "rate": 1, 16 | "random_null_rate": 0.1, 17 | "columns": [ 18 | { 19 | "name": "name", 20 | "type": "string" 21 | }, 22 | { 23 | "name": "age", 24 | "type": "int" 25 | } 26 | ] 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /bitsail-dist/src/main/resources/examples/Fake_Redis_Example.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": -2413, 5 | "job_name": "bitsail_fake_to_redis_test", 6 | "instance_id": -20413, 7 | "user_name": "user" 8 | }, 9 | "reader": { 10 | "class": "com.bytedance.bitsail.connector.legacy.fake.source.FakeSource", 11 | "total_count": 300, 12 | "rate": 100000, 13 | "random_null_rate": 0, 14 | "unique_fields": "fake_key", 15 | "columns": [ 16 | { 17 | "index": 0, 18 | "name": "fake_key", 19 | "type": "string" 20 | }, 21 | { 22 | "index": 1, 23 | "name": "fake_value", 24 | "type": "string" 25 | } 26 | ] 27 | }, 28 | "writer": { 29 | "class": "com.bytedance.bitsail.connector.legacy.redis.sink.RedisOutputFormat", 30 | "redis_data_type": "string", 31 | "redis_host": "localhost", 32 | "redis_port": 6379 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /bitsail-dist/src/main/resources/examples/File_Print_Example.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": 312, 5 | "instance_id": 3124, 6 | "job_name": "BitSail_fake_to_print_test", 7 | "user_name": "test" 8 | }, 9 | "writer": { 10 | "class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink", 11 | "writer_parallelism_num": 2 12 | }, 13 | "reader": { 14 | "columns": [ 15 | { 16 | "index": 0, 17 | "name": "id", 18 | "type": "int" 19 | } 20 | ], 21 | "source_engine": "text", 22 | "content_type": "json", 23 | "path_list": "hdfs://opensource/example_data/Example_File.json", 24 | "reader_parallelism_num": 1, 25 | "class": "com.bytedance.bitsail.connector.hadoop.source.HadoopInputFormat" 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /bitsail-dist/src/main/resources/examples/File_Print_Example_Csv.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": 312, 5 | "instance_id": 3124, 6 | "job_name": "BitSail_fake_to_print_test", 7 | "user_name": "test" 8 | }, 9 | "writer": { 10 | "class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink", 11 | "writer_parallelism_num": 2 12 | }, 13 | "reader": { 14 | "columns": [ 15 | { 16 | "name": "id", 17 | "type": "int" 18 | }, 19 | { 20 | "name": "name", 21 | "type": "string" 22 | } 23 | ], 24 | "source_engine": "text", 25 | "content_type": "csv", 26 | "path_list": "hdfs://opensource/example_data/Example_File.csv", 27 | "reader_parallelism_num": 1, 28 | "class": "com.bytedance.bitsail.connector.hadoop.source.HadoopInputFormat" 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /bitsail-dist/src/main/resources/examples/Hive_Print_Example.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": 312, 5 | "instance_id": 3124, 6 | "job_name": "BitSail_fake_to_print_test", 7 | "user_name": "test" 8 | }, 9 | "writer": { 10 | "class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink" 11 | }, 12 | "reader": { 13 | "class": "com.bytedance.bitsail.connector.legacy.hive.source.HiveInputFormat", 14 | "columns": [ 15 | { 16 | "index": 0, 17 | "name": "id", 18 | "type": "int" 19 | } 20 | ], 21 | "db_name": "bitsail", 22 | "table_name": "test_opensource", 23 | "metastore_properties": "{\"hive.metastore.uris\":\"\"}", 24 | "partition": "date=20180101", 25 | "reader_parallelism_num": 2, 26 | "source_engine": "hive" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /bitsail-dist/src/main/resources/examples/Hudi_Compaction_Example.json: -------------------------------------------------------------------------------- 1 | { 2 | "job":{ 3 | "common":{ 4 | "job_type":"BATCH", 5 | "job_plugin_lib_dir":"plugin", 6 | "enable_dynamic_loader":false, 7 | "job_plugin_conf_dir":"plugin_conf", 8 | "job_id":-1, 9 | "sync_ddl":false, 10 | "dry_run":true 11 | }, 12 | "reader":{ 13 | "path":"/path/to/table", 14 | "class":"com.bytedance.bitsail.connector.legacy.hudi.dag.HudiCompactSourceDAGBuilder" 15 | }, 16 | "writer":{ 17 | "path":"/path/to/table", 18 | "class":"com.bytedance.bitsail.connector.legacy.hudi.dag.HudiCompactSinkDAGBuilder" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /bitsail-dist/src/main/resources/examples/example-datas/Example_File.csv: -------------------------------------------------------------------------------- 1 | 1,andy 2 | 2,cindy 3 | 3,null -------------------------------------------------------------------------------- /bitsail-dist/src/main/resources/examples/example-datas/Example_File.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 1 3 | } 4 | { 5 | "id": 2 6 | } -------------------------------------------------------------------------------- /bitsail-shade/bitsail-shaded-hive/src/main/java/com/bytedance/bitsail/shaded/hive/shim/HiveShimV121.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.shaded.hive.shim; 18 | 19 | public class HiveShimV121 extends HiveShimV120 { 20 | 21 | @Override 22 | public String getVersion() { 23 | return "1.2.1"; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /bitsail-shade/bitsail-shaded-hive/src/main/java/com/bytedance/bitsail/shaded/hive/shim/HiveShimV210.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.shaded.hive.shim; 18 | 19 | public class HiveShimV210 extends HiveShimV200 { 20 | 21 | @Override 22 | public String getVersion() { 23 | return "2.1.0"; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /bitsail-shade/bitsail-shaded-hive/src/main/java/com/bytedance/bitsail/shaded/hive/shim/HiveShimV211.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.shaded.hive.shim; 18 | 19 | public class HiveShimV211 extends HiveShimV210 { 20 | 21 | @Override 22 | public String getVersion() { 23 | return "2.1.1"; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /bitsail-shade/bitsail-shaded-hive/src/main/java/com/bytedance/bitsail/shaded/hive/shim/HiveShimV239.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.shaded.hive.shim; 18 | 19 | public class HiveShimV239 extends HiveShimV230 { 20 | 21 | @Override 22 | public String getVersion() { 23 | return "2.3.9"; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /bitsail-shade/bitsail-shaded-hive/src/main/java/com/bytedance/bitsail/shaded/hive/shim/HiveShimV312.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.bytedance.bitsail.shaded.hive.shim; 18 | 19 | /** 20 | * Shim for Hive version 3.1.2. 21 | */ 22 | public class HiveShimV312 extends HiveShimV310 { 23 | 24 | @Override 25 | public String getVersion() { 26 | return "3.1.2"; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/resources/META-INF/services/com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.test.e2e.datasource.EmptyDataSource -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/main/resources/META-INF/services/com.bytedance.bitsail.test.e2e.executor.AbstractExecutor: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.test.e2e.executor.flink.Flink11Executor -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-base/src/test/resources/executor/generic/TestGenericExecutorSetting.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test-executor", 3 | "executor-image": "test-image", 4 | "exec-commands": [ 5 | "pwd", "&&", "sleep 5000" 6 | ], 7 | "failure-handle-commands": [ 8 | "pwd", "&&", "sleep 1000" 9 | ], 10 | "global-job-config": "{\"job\":{\"reader\":{\"reader_parallelism_num\":1},\"writer\":{\"writer_parallelism_num\":2}}}", 11 | "additional-files": [ 12 | { 13 | "host-path": "/local/1.jar", 14 | "executor-path": "/executor/1.jar" 15 | }, 16 | { 17 | "host-path": "/local/2.jar", 18 | "executor-path": "/executor/2.jar" 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-clickhouse/src/test/resources/META-INF/services/com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.test.e2e.datasource.EmptyDataSource 18 | com.bytedance.bitsail.test.e2e.datasource.ClickhouseDataSource -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-elasticsearch/src/test/resources/META-INF/services/com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.test.e2e.datasource.EmptyDataSource 18 | com.bytedance.bitsail.test.e2e.datasource.ElasticsearchDataSource -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-ftp/src/test/resources/META-INF/services/com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.test.e2e.datasource.EmptyDataSource 18 | com.bytedance.bitsail.test.e2e.datasource.SftpDataSource -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-ftp/src/test/resources/data/csv/_SUCCESS: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-ftp/src/test/resources/data/csv/upload1/test1.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4,c5,c6,c7,c8 2 | 111,aaa,1.1,1.12345,true,2022-11-01,999999999,1669705101,123456.789 3 | 222,bbb,1.1,1.45678,true,2022-10-01,999999999,1669705101,345678.123 4 | 333,ccc,0.1,1.12345,false,2022-01-01,88888888,1669715106,123456.789 5 | 444,ccc,0.1,1.12345,false,2022-01-01,88888888,1669715106,123456.789 6 | 555,ccc,0.1,1.12345,false,2022-01-01,11111111,1669715106,343.345 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-ftp/src/test/resources/data/csv/upload1/test2.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4,c5,c6,c7 2 | -111,aaa,-1.1,-1.12345,false,2022-11-01,1111111111,1669605101,766355.001 3 | -999,eee,-1.1,-1.12345,false,2022-11-01,1111111111,1669605101,766355.001 4 | -222,fff,-1.1,-1.12345,false,2022-11-01,1111111111,1669605202,766355.001 5 | -555,bbd,100.1,-1.12345,false,2022-11-01,3253636344,1669605101,766355.001 6 | -666,ewe,9.1,-1.12345,false,2022-11-01,1111111111,1669604113,353436.123 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-ftp/src/test/resources/data/csv/upload2/test1.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4,c5,c6,c7,c8 2 | 222,bbb,-9.1,1.12345,true,2022-11-01,999999999,1669705101,123456.789 3 | 111,bbb,-9.1,1.12345,true,2022-11-01,999999999,1669705101,123456.789 4 | 999,bbb,78.1,1.12345,true,2022-05-01,999999999,1669705101,43536.5 5 | 666,bbb,32.3,3.456,true,2022-11-01,35636346,1669403121,3463.3 6 | 444,bbb,-9.1,1.12345,true,2022-03-31,4364646,1669705101,757.51 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-ftp/src/test/resources/data/csv/upload2/test2.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4,c5,c6,c7 2 | -222,ccc,1.1,-1.12345,false,2022-11-01,1111111111,1669605101,2313.001 3 | 789,ccc,1.1,-1.12345,false,2022-08-27,1111111111,1669605101,43536.001 4 | 456,ccc,5.6,-1.12345,false,2022-11-01,1111111111,1669605101,76969.657 5 | 4636,ccc,8.8,-1.5345,false,2022-06-21,1111111111,1669343101,766355.001 6 | -398,ccc,1.1,-1.12345,false,2022-11-01,1111111111,1669605101,47547.345 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-redis/src/test/resources/META-INF/services/com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.test.e2e.datasource.EmptyDataSource 18 | com.bytedance.bitsail.test.e2e.datasource.RedisDataSource -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-rocketmq/src/test/resources/META-INF/services/com.bytedance.bitsail.test.e2e.datasource.AbstractDataSource: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.test.e2e.datasource.EmptyDataSource 18 | com.bytedance.bitsail.test.e2e.datasource.RocketMQDataSource -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-connector-v1/bitsail-test-e2e-connector-v1-rocketmq/src/test/resources/broker.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | brokerClusterName = DefaultCluster 17 | brokerName = broker-a 18 | brokerId = 0 19 | deleteWhen = 04 20 | fileReservedTime = 48 21 | brokerRole = ASYNC_MASTER 22 | flushDiskType = ASYNC_FLUSH 23 | brokerIP1 = broker -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-end-to-end/bitsail-test-e2e-generic-executor-templates/src/main/settings/flink_1.11.6_executor.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "example-flink-1.11.6", 3 | "executor-image": "flink:1.11.6", 4 | "client-module": "bitsail-clients/bitsail-client-flink/bitsail-client-flink-1.11", 5 | "core-module": "bitsail-cores/bitsail-core-flink/bitsail-core-flink-1.11-bridge", 6 | "exec-commands": [ 7 | "bin/bitsail run", 8 | "--engine flink", 9 | "--execution-mode run", 10 | "--deployment-mode local", 11 | "--conf /opt/bitsail/jobConf.json" 12 | ], 13 | "failure-handle-commands": [ 14 | "cat /opt/flink/log/flink-*-client-*.log" 15 | ], 16 | "global-job-config": "{\"job\":{\"reader\":{\"reader_parallelism_num\":1},\"writer\":{\"writer_parallelism_num\":1}}}" 17 | } -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/resources/bitsail_mysql_cdc_kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_type": "BATCH", 5 | "job_plugin_lib_dir": "plugin", 6 | "job_plugin_conf_dir": "plugin_conf", 7 | "enable_dynamic_loader": true, 8 | "instance_id": "1", 9 | "internal_instance_id": "1", 10 | "extra_properties": { 11 | "update-mode": "append" 12 | }, 13 | "job_name": "bitsail_test_integration_cdc_to_kafka" 14 | }, 15 | "reader": { 16 | "class": "com.bytedance.bitsail.connector.cdc.mysql.source.MysqlCDCSource", 17 | "db_name": "test", 18 | "format_type": "debezium_json", 19 | "table_name": "jdbc_source_test", 20 | "connections": [ 21 | ], 22 | "user_name": "root", 23 | "password": "pw" 24 | }, 25 | "writer": { 26 | "class": "com.bytedance.bitsail.connector.kafka.sink.KafkaSink", 27 | "content_type": "debezium_json" 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-cdc/src/test/resources/bitsail_mysql_cdc_print.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_type": "BATCH", 5 | "job_plugin_lib_dir": "plugin", 6 | "job_plugin_conf_dir": "plugin_conf", 7 | "enable_dynamic_loader": true, 8 | "instance_id": "1", 9 | "internal_instance_id": "1", 10 | "extra_properties": { 11 | "update-mode": "append" 12 | }, 13 | "job_name": "bitsail_test_integration_cdc_to_print" 14 | }, 15 | "reader": { 16 | "class": "com.bytedance.bitsail.connector.cdc.mysql.source.MysqlCDCSource", 17 | "db_name": "test", 18 | "table_name": "jdbc_source_test", 19 | "format_type": "debezium_json", 20 | "connections": [ 21 | ], 22 | "user_name": "root", 23 | "password": "pw" 24 | }, 25 | "writer": { 26 | "class": "com.bytedance.bitsail.connector.print.sink.PrintSink" 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-ftp-legacy/src/test/resources/upload/_SUCCESS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-ftp-legacy/src/test/resources/upload/_SUCCESS -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-ftp-legacy/src/test/resources/upload/test1.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4 2 | 111,aaa,1.1,1.12345,true,2022-11-01 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-ftp-legacy/src/test/resources/upload/test2.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4 2 | -111,aaa,-1.1,-1.12345,false,2022-11-01 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-hadoop-legacy/src/test/resources/source/test.json: -------------------------------------------------------------------------------- 1 | {"id":0,"string_type":"test_string","map_string_string":{"k1":"v1","k2":"v2","k3":"v3"},"array_string":["a1","a2","a3","a4"]} -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-jdbc-legacy/src/test/resources/mysql/mysql_to_print_without_columns.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": -23, 5 | "job_name": "bitsail_test_integration_legacy_mysql_to_print_without_column_info", 6 | "instance_id": -203, 7 | "user_name": "test" 8 | }, 9 | "reader": { 10 | "class": "com.bytedance.bitsail.connector.legacy.jdbc.source.JDBCInputFormat", 11 | "db_name": "test", 12 | "table_name": "jdbc_source_test", 13 | "split_pk": "id", 14 | "connections": [ 15 | ], 16 | "user_name": "test", 17 | "password": "test" 18 | }, 19 | "writer": { 20 | "class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink" 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-mongodb-legacy/src/test/resources/mongodb_to_print.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": -2501346, 5 | "job_name": "bitsail_test_integration_legacy_mongodb_to_print", 6 | "instance_id": -20501346, 7 | "user_name": "user" 8 | }, 9 | "reader": { 10 | "class": "com.bytedance.bitsail.connector.legacy.mongodb.source.MongoDBInputFormat", 11 | "columns": [ 12 | { 13 | "name": "_id", 14 | "type": "objectid" 15 | }, 16 | { 17 | "name": "string_field", 18 | "type": "string" 19 | }, 20 | { 21 | "name": "int_field", 22 | "type": "long" 23 | } 24 | ], 25 | "reader_parallelism_num":1 26 | }, 27 | "writer": { 28 | "class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink" 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-connector-legacy/bitsail-test-integration-redis-legacy/src/test/resources/fake_to_redis.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_id": -2413, 5 | "job_name": "bitsail_test_integration_legacy_fake_to_redis", 6 | "instance_id": -20413, 7 | "user_name": "user" 8 | }, 9 | "reader": { 10 | "class": "com.bytedance.bitsail.connector.legacy.fake.source.FakeSource", 11 | "total_count": 300, 12 | "rate": 100000, 13 | "random_null_rate": 0, 14 | "unique_fields": "fake_key", 15 | "columns": [ 16 | { 17 | "name": "fake_key", 18 | "type": "string" 19 | }, 20 | { 21 | "name": "fake_value", 22 | "type": "string" 23 | } 24 | ] 25 | }, 26 | "writer": { 27 | "class": "com.bytedance.bitsail.connector.legacy.redis.sink.RedisOutputFormat", 28 | "redis_data_type": "string", 29 | "redis_host": "localhost", 30 | "redis_port": 6379 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-elasticsearch/src/test/resources/es_writer_parameter_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "writer": { 4 | "es_id_fields": "id", 5 | "columns": [ 6 | { 7 | "name": "id", 8 | "type": "string" 9 | }, 10 | { 11 | "name": "varchar_type", 12 | "type": "string" 13 | }, 14 | { 15 | "name": "text_type", 16 | "type": "string" 17 | }, 18 | { 19 | "name": "bigint_type", 20 | "type": "bigint" 21 | }, 22 | { 23 | "name": "date", 24 | "type": "string" 25 | } 26 | ] 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/csv/_SUCCESS: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/csv/upload1/test1.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4,c5,c6,c7,c8 2 | 111,aaa,1.1,1.12345,true,2022-11-01,999999999,1669705101,123456.789 3 | 222,bbb,1.1,1.45678,true,2022-10-01,999999999,1669705101,345678.123 4 | 333,ccc,0.1,1.12345,false,2022-01-01,88888888,1669715106,123456.789 5 | 444,ccc,0.1,1.12345,false,2022-01-01,88888888,1669715106,123456.789 6 | 555,ccc,0.1,1.12345,false,2022-01-01,11111111,1669715106,343.345 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/csv/upload1/test2.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4,c5,c6,c7 2 | -111,aaa,-1.1,-1.12345,false,2022-11-01,1111111111,1669605101,766355.001 3 | -999,eee,-1.1,-1.12345,false,2022-11-01,1111111111,1669605101,766355.001 4 | -222,fff,-1.1,-1.12345,false,2022-11-01,1111111111,1669605202,766355.001 5 | -555,bbd,100.1,-1.12345,false,2022-11-01,3253636344,1669605101,766355.001 6 | -666,ewe,9.1,-1.12345,false,2022-11-01,1111111111,1669604113,353436.123 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/csv/upload2/test1.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4,c5,c6,c7,c8 2 | 222,bbb,-9.1,1.12345,true,2022-11-01,999999999,1669705101,123456.789 3 | 111,bbb,-9.1,1.12345,true,2022-11-01,999999999,1669705101,123456.789 4 | 999,bbb,78.1,1.12345,true,2022-05-01,999999999,1669705101,43536.5 5 | 666,bbb,32.3,3.456,true,2022-11-01,35636346,1669403121,3463.3 6 | 444,bbb,-9.1,1.12345,true,2022-03-31,4364646,1669705101,757.51 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/csv/upload2/test2.csv: -------------------------------------------------------------------------------- 1 | c0,c1,c2,c3,c4,c5,c6,c7 2 | -222,ccc,1.1,-1.12345,false,2022-11-01,1111111111,1669605101,2313.001 3 | 789,ccc,1.1,-1.12345,false,2022-08-27,1111111111,1669605101,43536.001 4 | 456,ccc,5.6,-1.12345,false,2022-11-01,1111111111,1669605101,76969.657 5 | 4636,ccc,8.8,-1.5345,false,2022-06-21,1111111111,1669343101,766355.001 6 | -398,ccc,1.1,-1.12345,false,2022-11-01,1111111111,1669605101,47547.345 -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/files/p1/f1: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/files/p1/p11/f11: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/files/p2/f2: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/json/_SUCCESS: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/json/upload1/test1.json: -------------------------------------------------------------------------------- 1 | {"c0":222,"c1":"bbb","c2":-9.1,"c3":1.12345,"c4":true,"c5":"2022-11-01","c6":999999999,"c7":1669705101,"c8":123456.789} 2 | {"c0":222,"c1":"bbb","c2":-9.1,"c3":1.12345,"c4":true,"c5":"2022-11-01","c6":999999999,"c7":1669705101,"c8":123456.789} 3 | {"c0":222,"c1":"bbb","c2":-9.1,"c3":1.12345,"c4":true,"c5":"2022-11-01","c6":999999999,"c7":1669705101,"c8":123456.789} -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/json/upload1/test2.json: -------------------------------------------------------------------------------- 1 | {"c0":111,"c1":"bbb","c2":-9.1,"c3":1.12345,"c4":true,"c5":"2022-11-01","c6":999999999,"c7":1669705101,"c8":123456.789} -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/json/upload2/test1.json: -------------------------------------------------------------------------------- 1 | {"c0":-111,"c1":"mmm","c2":-9.1,"c3":1.12345,"c4":true,"c5":"2022-11-01","c6":999999999,"c7":1669705101,"c8":123456.789} -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-ftp/src/test/resources/data/json/upload2/test2.json: -------------------------------------------------------------------------------- 1 | {"c0":-222,"c1":"ccc","c2":-9.1,"c3":1.12345,"c4":true,"c5":"2022-11-01","c6":1111111111,"c7":1669605101,"c8":766355.001} -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-kafka/src/test/resources/fake_to_kafka.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "cid": 0, 5 | "domain": "test", 6 | "job_id": -24, 7 | "job_name": "bitsail_test_integration_fake_to_kafka", 8 | "instance_id": -204, 9 | "user_name": "root" 10 | }, 11 | "reader": { 12 | "class": "com.bytedance.bitsail.connector.fake.source.FakeSource", 13 | "total_count": 300, 14 | "rate": 100, 15 | "columns": [ 16 | { 17 | "name": "varchar_type", 18 | "type": "string" 19 | }, 20 | { 21 | "name": "text_type", 22 | "type": "string" 23 | }, 24 | { 25 | "name": "bigint_type", 26 | "type": "long" 27 | } 28 | ] 29 | }, 30 | "writer": { 31 | "class": "com.bytedance.bitsail.connector.kafka.sink.KafkaSink", 32 | "content_type": "json", 33 | "partition_field": "text_type" 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-localfilesystem/src/test/resources/data/csv/test.csv: -------------------------------------------------------------------------------- 1 | 0,2050-11-26,931451976403,Roberto,false 2 | 1,1992-08-15,101947222173,Reggie,true 3 | 2,2074-04-22,1344242983149,Thomas,true 4 | 3,2065-11-13,2337343299527,Corey,false 5 | 4,2025-04-08,565336730103,Lee,true 6 | 5,2037-02-15,3008830815937,Ann,false 7 | 6,2045-07-16,1173180419599,Virgen,true 8 | 7,1975-06-26,1522037304441,Scott,false 9 | 8,2068-07-26,1211639748316,Crysta,false 10 | 9,2077-04-13,2856898427345,Lorette,false 11 | 10,1989-11-01,578487185312,Luis,true 12 | 11,2011-04-01,419227136697,Deandrea,true -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-rocketmq/src/test/resources/bitsail_rocketmq_print.json: -------------------------------------------------------------------------------- 1 | { 2 | "job": { 3 | "common": { 4 | "job_type": "STREAMING", 5 | "job_name": "bitsail_test_integration_rocketmq_to_print", 6 | "instance_id": "1", 7 | "internal_instance_id": "1" 8 | }, 9 | "reader": { 10 | "cluster": "localhost:9876", 11 | "topic": "TBW102", 12 | "consumer_group": "sss", 13 | "columns": [ 14 | { 15 | "name": "id", 16 | "type": "long" 17 | } 18 | ], 19 | "format_type": "json", 20 | "class": "com.bytedance.bitsail.connector.rocketmq.source.RocketMQSource" 21 | }, 22 | "writer": { 23 | "class": "com.bytedance.bitsail.connector.print.sink.PrintSink", 24 | "sample_write": true, 25 | "sample_limit": 10, 26 | "columns": [ 27 | { 28 | "name": "id", 29 | "type": "long" 30 | } 31 | ] 32 | } 33 | } 34 | } -------------------------------------------------------------------------------- /bitsail-test/bitsail-test-integration/bitsail-test-integration-rocketmq/src/test/resources/broker.conf: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | brokerClusterName = DefaultCluster 17 | brokerName = broker-a 18 | brokerId = 0 19 | deleteWhen = 04 20 | fileReservedTime = 48 21 | brokerRole = ASYNC_MASTER 22 | flushDiskType = ASYNC_FLUSH 23 | brokerIP1 = localhost -------------------------------------------------------------------------------- /bitsail-transforms/src/main/resources/META-INF/services/com.bytedance.bitsail.base.connector.transform.v1.Transformer: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | 17 | com.bytedance.bitsail.transforms.partition.field.FieldBasePartitionTransformer 18 | com.bytedance.bitsail.transforms.map.encrypt.EncryptFieldMapTransformer -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bitsail-docs", 3 | "version": "1.0.0", 4 | "description": "docs of bitsail", 5 | "license": "MIT", 6 | "type": "module", 7 | "scripts": { 8 | "docs:build": "vuepress build website", 9 | "docs:clean-dev": "vuepress dev website --clean-cache", 10 | "docs:dev": "vuepress dev website" 11 | }, 12 | "devDependencies": { 13 | "@vuepress/client": "2.0.0-beta.53", 14 | "vue": "^3.2.45", 15 | "vuepress": "2.0.0-beta.53", 16 | "vuepress-theme-hope": "2.0.0-beta.131" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /test-e2e.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | test_module=$1 19 | 20 | mvn clean verify \ 21 | -DskipUT=true \ 22 | -DskipITCase=true \ 23 | -DskipE2E=false \ 24 | -D"checkstyle.skip"=true \ 25 | -D"license.skipAddThirdParty"=true \ 26 | --no-snapshot-updates \ 27 | -P _maven.oracle.com_ \ 28 | -am \ 29 | -pl :${test_module} -------------------------------------------------------------------------------- /tools/release/create_binary_tar.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | CURR_DIR=`pwd` 19 | if [[ `basename $CURR_DIR` != "bitsail" ]] ; then 20 | echo "You have to call the script from the bitsail/ dir" 21 | exit 1 22 | fi 23 | 24 | RELEASE_VERSION=`grep -A 1 "" pom.xml | grep '' | sed -e 's///' -e 's/<\/revision>//' -e 's/ //g'` 25 | 26 | bash build.sh 27 | tar czf bitsail-dist-${RELEASE_VERSION}.tgz output -------------------------------------------------------------------------------- /website/.vuepress/navbar/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./en.js"; 18 | export * from "./zh.js"; 19 | -------------------------------------------------------------------------------- /website/.vuepress/public/bitsail_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/.vuepress/public/bitsail_arch.png -------------------------------------------------------------------------------- /website/.vuepress/public/bitsail_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/.vuepress/public/bitsail_logo.png -------------------------------------------------------------------------------- /website/.vuepress/public/logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/.vuepress/public/logo.jpg -------------------------------------------------------------------------------- /website/.vuepress/public/wechat_QR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/.vuepress/public/wechat_QR.png -------------------------------------------------------------------------------- /website/.vuepress/sidebar/en.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { sidebar } from "vuepress-theme-hope"; 18 | 19 | export const enSidebar = sidebar({ 20 | "/en/documents": "structure", 21 | "/en/start": "structure", 22 | "/en/community": "structure", 23 | "/en/blog": "structure" 24 | }); 25 | -------------------------------------------------------------------------------- /website/.vuepress/sidebar/index.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | export * from "./en.js"; 18 | export * from "./zh.js"; 19 | -------------------------------------------------------------------------------- /website/.vuepress/sidebar/zh.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | import { sidebar } from "vuepress-theme-hope"; 18 | 19 | export const zhSidebar = sidebar({ 20 | "/zh/documents": "structure", 21 | "/zh/start": "structure", 22 | "/zh/community": "structure", 23 | "/zh/blog": "structure" 24 | }); 25 | -------------------------------------------------------------------------------- /website/.vuepress/styles/config.scss: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // you can change config here 18 | $colors: #c0392b, #d35400, #f39c12, #27ae60, #16a085, #2980b9, #8e44ad, #2c3e50, 19 | #7f8c8d !default; 20 | -------------------------------------------------------------------------------- /website/.vuepress/styles/index.scss: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // place your custom styles here 18 | .navbar .nav-links { 19 | font-size: 1.1rem; 20 | } 21 | 22 | .footer-wrapper { 23 | background-color: #1b404e; 24 | color: white; 25 | 26 | .footer { 27 | width: 100%; 28 | } 29 | } -------------------------------------------------------------------------------- /website/.vuepress/styles/palette.scss: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright 2022-2023 Bytedance Ltd. and/or its affiliates. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // you can change colors here 18 | $theme-color: #2980b9; 19 | -------------------------------------------------------------------------------- /website/en/community/community.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 1 3 | --- 4 | 5 | # Community 6 | 7 | English | [简体中文](../../zh/community/community.md) 8 | 9 | ----- 10 | 11 | - [Contribute Guide](contribute.md) 12 | - [How to get involved](contribute.md#how-to-get-involved) 13 | - [Pull Request Guide](pr_guide.md) 14 | - [Open a github issue](contribute.md#open-a-github-issue) 15 | - [Developing tips](contribute.md#developing-tips) 16 | - [BitSail Release Guide](release_guide.md) 17 | - [Connector Quick Start](connector_quick_start.md) 18 | - [Source Connector Details](source_connector_detail.md) 19 | - [Sink Connector Details](sink_connector_detail.md) 20 | - [Subscribe Mailing list](mailing.md) 21 | - [Team introduction](team.md) -------------------------------------------------------------------------------- /website/en/community/mailing.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 5 3 | --- 4 | 5 | # Subscribe Mailing Lists 6 | 7 | English | [简体中文](../../zh/community/mailing.md) 8 | 9 | ----- 10 | 11 | Currently, BitSail community use Google Group as the mailing list provider. 12 | You need to subscribe to the mailing list before starting a conversation 13 | 14 | ## Subscribe 15 | 16 | Email to this address `bitsail+subscribe@googlegroups.com` 17 | 18 | ## Start a conversation 19 | 20 | Email to this address `bitsail@googlegroups.com` 21 | 22 | ## Unsubscribe 23 | 24 | Email to this address `bitsail+unsubscribe@googlegroups.com` -------------------------------------------------------------------------------- /website/en/community/team.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 6 3 | --- 4 | 5 | # Team 6 | 7 | English | [简体中文](../../zh/community/team.md) 8 | 9 | ----- 10 | 11 | ## Contributors 12 | 13 | 14 | 15 | 16 | 17 | **Thanks all contributors！**
-------------------------------------------------------------------------------- /website/en/documents/components/conversion/introduction.md: -------------------------------------------------------------------------------- 1 | # bitsail-conversion-flink 2 | 3 | ----- 4 | 5 | Parent document: [bitsail-components](../README.md) 6 | 7 | ## Content 8 | 9 | When **BitSail** transmits data to a specified data source, it needs to convert the intermediate format (`bitsail rows`) used in the transmission process into a data type acceptable to the data source. 10 | This module provides convenient tools for converting. 11 | 12 | - In this context, `bitsail rows` means `com.bytedance.bitsail.common.column.Column` data wrapped by `org.apache.flink.types.Row`。 13 | 14 | Specific supported data types are as follows: 15 | 16 | 17 | | Name | Function | Link | 18 | |---------------------------------|----------------------------------------------------|-------------------------| 19 | | `bitsail-conversion-flink-hive` | Provides a way to convert `Row` to hive `Writable` | [link](hive-convert.md) | 20 | 21 | 22 | -------------------------------------------------------------------------------- /website/en/documents/components/format/json-format.md: -------------------------------------------------------------------------------- 1 | # bitsail-component-format-flink-json 2 | 3 | ----- 4 | 5 | Parent document: [bitsail-component-format-flink](./introduction.md) 6 | 7 | To be continued ... -------------------------------------------------------------------------------- /website/en/documents/connectors/fake/fake-example.md: -------------------------------------------------------------------------------- 1 | # Fake connector examples 2 | 3 | Parent document: [fake-connector](./fake.md) 4 | 5 | ----- 6 | 7 | 8 | 9 | ## Fake reader example 10 | 11 | Suppose you want to generate a data set of 300 records, and specify that each record has two fields named name and age, you can use the following configuration to read. 12 | 13 | ```json 14 | { 15 | "job": { 16 | "reader": { 17 | "class": "com.bytedance.bitsail.connector.legacy.fake.source.FakeSource", 18 | "total_count": 300, 19 | "rate": 100, 20 | "random_null_rate": 0.1, 21 | "columns": [ 22 | { 23 | "name": "name", 24 | "type": "string" 25 | }, 26 | { 27 | "name": "age", 28 | "type": "int" 29 | } 30 | ] 31 | } 32 | } 33 | } 34 | ``` 35 | -------------------------------------------------------------------------------- /website/en/documents/connectors/ftp/ftp-example.md: -------------------------------------------------------------------------------- 1 | # FTP/SFTP connector example 2 | 3 | Parent document: [FTP connector](./ftp.md) 4 | 5 | The following configuration shows how to organize parameter configuration to read the following CSV format file. 6 | 7 | - Example CSV data 8 | 9 | ```csv 10 | c1,c2 11 | aaa,bbb 12 | ``` 13 | 14 | - Configuration file used to read the above file: 15 | 16 | ```json 17 | { 18 | "job": { 19 | "reader": { 20 | "class": "com.bytedance.bitsail.connector.legacy.ftp.source.FtpInputFormat", 21 | "protocol":"FTP", 22 | "host": "localhost", 23 | "port": 21, 24 | "user": "user", 25 | "password": "password", 26 | "path_list": "/upload/", 27 | "success_file_path": "/upload/_SUCCESS", 28 | "content_type": "csv", 29 | "columns": [ 30 | { 31 | "name": "c1", 32 | "type": "string" 33 | }, 34 | { 35 | "name": "c2", 36 | "type": "string" 37 | } 38 | ] 39 | } 40 | } 41 | } 42 | ``` 43 | -------------------------------------------------------------------------------- /website/en/documents/connectors/ftp/v1/ftp-v1-example.md: -------------------------------------------------------------------------------- 1 | # FTP/SFTP-v1 connector example 2 | 3 | Parent document: [FTP-v1 connector](./ftp-v1.md) 4 | 5 | The following configuration shows how to organize parameter configuration to read the following CSV format file. 6 | 7 | - Example CSV data 8 | 9 | ```csv 10 | c1,c2 11 | aaa,bbb 12 | ``` 13 | 14 | - Configuration file used to read the above file: 15 | 16 | ```json 17 | { 18 | "job": { 19 | "reader": { 20 | "class": "com.bytedance.bitsail.connector.ftp.source.FtpSource", 21 | "protocol":"FTP", 22 | "host": "localhost", 23 | "port": 21, 24 | "user": "user", 25 | "password": "password", 26 | "path_list": "/upload/", 27 | "success_file_path": "/upload/_SUCCESS", 28 | "content_type": "csv", 29 | "columns": [ 30 | { 31 | "name": "c1", 32 | "type": "string" 33 | }, 34 | { 35 | "name": "c2", 36 | "type": "string" 37 | } 38 | ] 39 | } 40 | } 41 | } 42 | ``` 43 | -------------------------------------------------------------------------------- /website/en/documents/connectors/print/print-example.md: -------------------------------------------------------------------------------- 1 | # Print connector examples 2 | 3 | Parent documents: [print-connector](./print.md) 4 | 5 | ----- 6 | 7 | ## Print write connector 8 | 9 | the data can be printed out with the following configuration. 10 | 11 | ```json 12 | { 13 | "job": { 14 | "writer": { 15 | "class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink", 16 | "batch_size": "10" 17 | } 18 | } 19 | } 20 | ``` -------------------------------------------------------------------------------- /website/en/documents/connectors/print/v1/print-example-v1.md: -------------------------------------------------------------------------------- 1 | # Print connector V1 examples 2 | 3 | Parent documents: [print-connector](./print-v1.md) 4 | 5 | ----- 6 | 7 | ## Print write connector 8 | 9 | the data can be printed out with the following configuration. 10 | 11 | 12 | ```json 13 | { 14 | "job": { 15 | "writer": { 16 | "class": "com.bytedance.bitsail.connector.print.sink.PrintSink", 17 | "sample_write": true, 18 | "sample_limit": 10 19 | } 20 | } 21 | } 22 | ``` -------------------------------------------------------------------------------- /website/en/documents/connectors/redis/redis-example.md: -------------------------------------------------------------------------------- 1 | # Redis connector example 2 | 3 | Parent document: [Redis connector](./redis.md) 4 | 5 | ## Redis Writer example 6 | 7 | Suppose we start a local Redis with port 6379. 8 | 9 | Configuration for writing the Redis cluster is: 10 | 11 | ```json 12 | { 13 | "job": { 14 | "writer": { 15 | "class": "com.bytedance.bitsail.connector.legacy.redis.sink.RedisOutputFormat", 16 | "redis_data_type": "string", 17 | "redis_host": "localhost", 18 | "redis_port": 6379 19 | } 20 | } 21 | } 22 | ``` -------------------------------------------------------------------------------- /website/en/documents/faq/parallelism.md: -------------------------------------------------------------------------------- 1 | # Parallelism Computing 2 | 3 | Todo. -------------------------------------------------------------------------------- /website/en/documents/start/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | index: false 3 | dir: 4 | order: 2 5 | --- 6 | 7 | 8 | # Quick Start 9 | 10 | - [Develop Environment Setup](env_setup.md) 11 | - [Deployment Guide](deployment.md) 12 | - [Job Configuration Guide](config.md) 13 | - [BitSail Guide Video](quick_guide.md) -------------------------------------------------------------------------------- /website/en/release/release.md: -------------------------------------------------------------------------------- 1 | # Releases 2 | 3 | - [0.1.0](https://github.com/bytedance/bitsail) -------------------------------------------------------------------------------- /website/en/usercases/case.md: -------------------------------------------------------------------------------- 1 | # User Cases 2 | 3 | - [ByteDance use cases](cases/bytedance.md) -------------------------------------------------------------------------------- /website/en/usercases/cases/bytedance.md: -------------------------------------------------------------------------------- 1 | # BitSail cases in ByteDance 2 | 3 | Todo. -------------------------------------------------------------------------------- /website/images/bitsail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/bitsail.png -------------------------------------------------------------------------------- /website/images/bitsail_arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/bitsail_arch.png -------------------------------------------------------------------------------- /website/images/bitsail_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/bitsail_logo.png -------------------------------------------------------------------------------- /website/images/change-hive-version.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/change-hive-version.png -------------------------------------------------------------------------------- /website/images/checkstyle_plugin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/checkstyle_plugin.png -------------------------------------------------------------------------------- /website/images/community/connector_quick_start/bitsail_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/connector_quick_start/bitsail_model.png -------------------------------------------------------------------------------- /website/images/community/connector_quick_start/code_structure_en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/connector_quick_start/code_structure_en.png -------------------------------------------------------------------------------- /website/images/community/connector_quick_start/code_structure_zh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/connector_quick_start/code_structure_zh.png -------------------------------------------------------------------------------- /website/images/community/connector_quick_start/connector_pom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/connector_quick_start/connector_pom.png -------------------------------------------------------------------------------- /website/images/community/connector_quick_start/dist_pom.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/connector_quick_start/dist_pom.png -------------------------------------------------------------------------------- /website/images/community/connector_quick_start/sink_connector.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/connector_quick_start/sink_connector.png -------------------------------------------------------------------------------- /website/images/community/connector_quick_start/source_connector.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/connector_quick_start/source_connector.png -------------------------------------------------------------------------------- /website/images/community/connector_quick_start/test_container.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/connector_quick_start/test_container.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/after_git_reset.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/after_git_reset.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/commit_info.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/commit_info.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/create_pr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/create_pr.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/git_clone_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/git_clone_example.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/git_history.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/git_history.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/git_rebase_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/git_rebase_example.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/github_pr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/github_pr.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/github_status.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/github_status.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/repository_fork.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/repository_fork.png -------------------------------------------------------------------------------- /website/images/community/pr_guide/repository_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/pr_guide/repository_structure.png -------------------------------------------------------------------------------- /website/images/community/release_guide/release_procedure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/release_guide/release_procedure.png -------------------------------------------------------------------------------- /website/images/community/sink_connector/sink_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/sink_connector/sink_diagram.png -------------------------------------------------------------------------------- /website/images/community/sink_connector/writer_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/sink_connector/writer_diagram.png -------------------------------------------------------------------------------- /website/images/community/source_connector/bitsail_converter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/source_connector/bitsail_converter.png -------------------------------------------------------------------------------- /website/images/community/source_connector/deserialization_schema_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/source_connector/deserialization_schema_diagram.png -------------------------------------------------------------------------------- /website/images/community/source_connector/file_mapping_converter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/source_connector/file_mapping_converter.png -------------------------------------------------------------------------------- /website/images/community/source_connector/source_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/source_connector/source_diagram.png -------------------------------------------------------------------------------- /website/images/community/source_connector/source_reader_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/source_connector/source_reader_diagram.png -------------------------------------------------------------------------------- /website/images/community/source_connector/source_split_coordinator_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/source_connector/source_split_coordinator_diagram.png -------------------------------------------------------------------------------- /website/images/community/source_connector/source_split_diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/community/source_connector/source_split_diagram.png -------------------------------------------------------------------------------- /website/images/documents/start/quick_guide/compile_product_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/documents/start/quick_guide/compile_product_structure.png -------------------------------------------------------------------------------- /website/images/documents/start/quick_guide/product_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/documents/start/quick_guide/product_structure.png -------------------------------------------------------------------------------- /website/images/documents/start/quick_guide/source_code_structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/documents/start/quick_guide/source_code_structure.png -------------------------------------------------------------------------------- /website/images/license_header.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/license_header.png -------------------------------------------------------------------------------- /website/images/set_checkstyle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/set_checkstyle.png -------------------------------------------------------------------------------- /website/images/wechat_QR.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bytedance/bitsail/2ef2e157aef228beeb11fee143640fac37b5d57b/website/images/wechat_QR.png -------------------------------------------------------------------------------- /website/zh/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | home: true 3 | title: 首页 4 | heroImage: /bitsail_logo.png 5 | heroText: false 6 | tagline: 一款基于分布式架构的高性能数据集成引擎，支持多种异构数据源间的数据同步，目前服务于抖音、今日头条等业务线，每日同步百万亿级数据。 7 | actions: 8 | - text: 快速开始 9 | link: /zh/documents/start 10 | type: primary 11 | - text: 开源仓库 12 | link: https://github.com/bytedance/bitsail 13 | type: primary 14 | 15 | features: 16 | - title: 异构数据源同步 17 | details: 支持多种异构数据源间高性能、高可靠的海量数据同步，并支持不同数据源类型之间的转换 18 | 19 | - title: 湖仓一体 20 | details: 支持流批一体、湖仓一体架构，用一套框架覆盖几乎所有数据同步场景 21 | 22 | - title: 分布式架构 23 | details: 基于Hadoop生态，支持分布式水平扩展，通过一套架构解决批式、流式、增量场景下的数据同步 24 | 25 | - title: DDL自动同步 26 | details: 支持上下游数据源之间灵活的表结构同步，包括表创建和列级别的增、删、改 27 | 28 | - title: 脏数据收集 29 | details: 支持脏数据收集展示，并支持扩展脏数据存储方式 30 | 31 | - title: 高效开发 32 | details: 独立于引擎的读写接口，开发成本低 33 | 34 | --- 35 | -------------------------------------------------------------------------------- /website/zh/community/community.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 1 3 | --- 4 | 5 | # 社区 6 | 7 | [English](../../en/community/community.md) | 简体中文 8 | 9 | ----- 10 | 11 | - [贡献指南](contribute.md) 12 | - [如何参与](contribute.md#如何参与) 13 | - [如何发起一个issue](contribute.md#打开一个GitHub-Issue) 14 | - [PR发布指南](pr_guide.md) 15 | - [开发Tips](contribute.md#开发小技巧) 16 | - [BitSail 发版指南](release_guide.md) 17 | - [Connector开发指南](connector_quick_start.md) 18 | - [Source Connector 详解](source_connector_detail.md) 19 | - [Sink Connector 详解](sink_connector_detail.md) 20 | - [邮件列表](mailing.md) 21 | - [团队介绍](team.md) -------------------------------------------------------------------------------- /website/zh/community/mailing.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 5 3 | --- 4 | 5 | # 邮件列表 6 | 7 | [English](../../en/community/mailing.md) | 简体中文 8 | 9 | ----- 10 | 11 | 当前，BitSail社区通过谷歌群组作为邮件列表的提供者，邮件列表可以在绝大部分地区正常收发邮件。 12 | 在订阅BitSail小组的邮件列表后可以通过发送邮件发言 13 | 14 | ## 订阅 15 | 16 | 发送Email到此地址 `bitsail+subscribe@googlegroups.com`， 17 | 你会收到一封回信询问你是否希望加入BitSail群组，`Join This Group`按钮可能因网络原因无法使用，直接回复此封邮件便可确认加入。 18 | 19 | ## 开启一个话题 20 | 21 | 发送Email到此地址 `bitsail@googlegroups.com` 22 | 23 | ## 取消订阅 24 | 25 | 发送Email到此地址 `bitsail+unsubscribe@googlegroups.com` -------------------------------------------------------------------------------- /website/zh/community/team.md: -------------------------------------------------------------------------------- 1 | --- 2 | order: 6 3 | --- 4 | 5 | # Team 6 | 7 | [English](../../en/community/team.md) | 简体中文 8 | 9 | ----- 10 | 11 | ## 贡献者 12 | 13 | 14 | 15 | 16 | 17 | **感谢所有贡献者！**
18 | -------------------------------------------------------------------------------- /website/zh/documents/components/conversion/introduction.md: -------------------------------------------------------------------------------- 1 | # bitsail-conversion-flink 2 | 3 | ----- 4 | 5 | 上级文档: [bitsail-components](../README.md) 6 | 7 | ## 内容 8 | 9 | **BitSail** 在将数据传输到指定数据源时，需要将传输过程中使用的中间格式 `bitsail rows` 转化为数据源可接受的数据类型。 10 | 本模块提供了用于转化`bitsail rows`为其他数据类型的便捷工具。 11 | 12 | - 文中`bitsail rows`指由`org.apache.flink.types.Row`包住的`com.bytedance.bitsail.common.column.Column`数据。 13 | 14 | 具体支持的数据类型如下： 15 | 16 | | 子模块 | 支持的功能 | 链接 | 17 | |---------------------------------|------------------------------|----------------------------| 18 | | `bitsail-conversion-flink-hive` | 提供转化`Row`为hive `Writable`的方法 | [link](hive-convert.md) | 19 | 20 | 21 | -------------------------------------------------------------------------------- /website/zh/documents/components/format/json-format.md: -------------------------------------------------------------------------------- 1 | # bitsail-component-format-flink-json 2 | 3 | ----- 4 | 5 | Parent document: [bitsail-component-format-flink](./introduction.md) 6 | 7 | To be continued ... -------------------------------------------------------------------------------- /website/zh/documents/connectors/assert/assert-example.md: -------------------------------------------------------------------------------- 1 | # Assert 连接器示例 2 | 3 | 上级文档：[Assert 连接器](./assert.md) 4 | 5 | 下面配置的具体含义： 6 | - 数据的总条数范围为 `[10, 20]` 7 | - `name` 列非空，字符串长度范围为 `[1, 1000]` 8 | - `price` 列非空，取值范围是 `[2, 180]` 9 | 10 | ```json 11 | { 12 | "job": { 13 | "writer": { 14 | "class": "com.bytedance.bitsail.connector.assertion.sink.AssertSink", 15 | "columns": [ 16 | { 17 | "name": "name", 18 | "type": "string" 19 | }, 20 | { 21 | "name": "price", 22 | "type": "double" 23 | } 24 | ], 25 | "row_rules": { 26 | "min_row": 10, 27 | "max_row": 20 28 | }, 29 | "column_rules": { 30 | "name": { 31 | "not_null": true, 32 | "min_len": 1, 33 | "max_len": 1000 34 | }, 35 | "price": { 36 | "not_null": true, 37 | "min": 2, 38 | "max": 180 39 | } 40 | } 41 | } 42 | } 43 | } 44 | ``` 45 | -------------------------------------------------------------------------------- /website/zh/documents/connectors/elasticsearch/elasticsearch-example.md: -------------------------------------------------------------------------------- 1 | # Elasticsearch 连接器示例 2 | 3 | 上级文档：[Elasticsearch 连接器](./elasticsearch.md) 4 | 5 | 如下展示了如何使用用户参数配置写入指定的elasticsearch索引。 6 | 7 | ```json 8 | { 9 | "job": { 10 | "writer": { 11 | "class": "com.bytedance.bitsail.connector.elasticsearch.sink.ElasticsearchSink", 12 | "es_id_fields": "id", 13 | "es_index": "es_index_test", 14 | "es_hosts": ["http://localhost:1234"], 15 | "json_serializer_features": "QuoteFieldNames,UseSingleQuotes", 16 | "bulk_backoff_max_retry_count": 10, 17 | "columns": [ 18 | { 19 | "name": "id", 20 | "type": "int" 21 | }, 22 | { 23 | "name": "double_type", 24 | "type": "double" 25 | }, 26 | { 27 | "name": "text_type", 28 | "type": "text" 29 | }, 30 | { 31 | "name": "bigint_type", 32 | "type": "bigint" 33 | } 34 | ] 35 | } 36 | } 37 | } 38 | ``` 39 | -------------------------------------------------------------------------------- /website/zh/documents/connectors/fake/fake-example.md: -------------------------------------------------------------------------------- 1 | # Fake连接器配置示例 2 | 3 | 父目录: [fake-connector](./fake.md) 4 | 5 | ----- 6 | 7 | 8 | 9 | ## Fake读连接器 10 | 11 | 假设你想要生成一个300条的数据集，并且指定每个record有2个字段分别为name、age，则可用如下配置读取。 12 | 13 | ```json 14 | { 15 | "job": { 16 | "reader": { 17 | "class": "com.bytedance.bitsail.connector.legacy.fake.source.FakeSource", 18 | "total_count": 300, 19 | "rate": 100, 20 | "random_null_rate": 0.1, 21 | "columns": [ 22 | { 23 | "name": "name", 24 | "type": "string" 25 | }, 26 | { 27 | "name": "age", 28 | "type": "int" 29 | } 30 | ] 31 | } 32 | } 33 | } 34 | ``` 35 | -------------------------------------------------------------------------------- /website/zh/documents/connectors/ftp/ftp-example.md: -------------------------------------------------------------------------------- 1 | # FTP/SFTP 连接器示例 2 | 3 | 上级文档：[FTP/SFTP 连接器](./ftp.md) 4 | 5 | 下面展示了如何使用用户参数配置读取如下 CSV 格式文件。 6 | 7 | - 示例 CSV 数据 8 | 9 | ```csv 10 | c1,c2 11 | aaa,bbb 12 | ``` 13 | 14 | - 用于读取上述格式文件的配置 15 | 16 | ```json 17 | { 18 | "job": { 19 | "reader": { 20 | "class": "com.bytedance.bitsail.connector.legacy.ftp.source.FtpInputFormat", 21 | "protocol":"FTP", 22 | "host": "localhost", 23 | "port": 21, 24 | "user": "user", 25 | "password": "password", 26 | "path_list": "/upload/", 27 | "success_file_path": "/upload/_SUCCESS", 28 | "content_type": "csv", 29 | "columns": [ 30 | { 31 | "name": "c1", 32 | "type": "string" 33 | }, 34 | { 35 | "name": "c2", 36 | "type": "string" 37 | } 38 | ] 39 | } 40 | } 41 | } 42 | ``` 43 | -------------------------------------------------------------------------------- /website/zh/documents/connectors/ftp/v1/ftp-v1-example.md: -------------------------------------------------------------------------------- 1 | # FTP/SFTP-v1 连接器示例 2 | 3 | 上级文档：[FTP/SFTP-v1 连接器](./ftp-v1.md) 4 | 5 | 下面展示了如何使用用户参数配置读取如下 CSV 格式文件。 6 | 7 | - 示例 CSV 数据 8 | 9 | ```csv 10 | c1,c2 11 | aaa,bbb 12 | ``` 13 | 14 | - 用于读取上述格式文件的配置 15 | 16 | ```json 17 | { 18 | "job": { 19 | "reader": { 20 | "class": "com.bytedance.bitsail.connector.ftp.source.FtpSource", 21 | "protocol":"FTP", 22 | "host": "localhost", 23 | "port": 21, 24 | "user": "user", 25 | "password": "password", 26 | "path_list": "/upload/", 27 | "success_file_path": "/upload/_SUCCESS", 28 | "content_type": "csv", 29 | "columns": [ 30 | { 31 | "name": "c1", 32 | "type": "string" 33 | }, 34 | { 35 | "name": "c2", 36 | "type": "string" 37 | } 38 | ] 39 | } 40 | } 41 | } 42 | ``` 43 | -------------------------------------------------------------------------------- /website/zh/documents/connectors/hadoop/hadoop-example.md: -------------------------------------------------------------------------------- 1 | # Hadoop 连接器示例 2 | 3 | 上级文档：[Hadoop 连接器](./hadoop.md) 4 | 5 | 下面展示了如何使用用户参数配置读取如下json格式hdfs文件。 6 | 7 | - 示例json数据 8 | ```json 9 | {"id":0,"string_type":"test_string","map_string_string":{"k1":"v1","k2":"v2","k3":"v3"},"array_string":["a1","a2","a3","a4"]} 10 | ``` 11 | 12 | 13 | - 用于读取上述格式hdfs文件的配置 14 | 15 | ```json 16 | { 17 | "job": { 18 | "reader": { 19 | "class": "com.bytedance.bitsail.connector.hadoop.source.HadoopInputFormat", 20 | "path_list": "hdfs://test_namespace/test.json", 21 | "content_type":"json", 22 | "reader_parallelism_num": 1, 23 | "columns": [ 24 | { 25 | "name":"id", 26 | "type": "int" 27 | }, 28 | { 29 | "name": "string_type", 30 | "type": "string" 31 | }, 32 | { 33 | "name": "map_string_string", 34 | "type": "map" 35 | }, 36 | { 37 | "name": "array_string", 38 | "type": "list" 39 | } 40 | ] 41 | } 42 | } 43 | } 44 | ``` -------------------------------------------------------------------------------- /website/zh/documents/connectors/print/print-example.md: -------------------------------------------------------------------------------- 1 | # Print连接器配置示例 2 | 3 | 父目录: [print-connector](./print.md) 4 | 5 | ----- 6 | 7 | ## Print写连接器 8 | 9 | 不管上游来的数据有多少个字段、是什么数据类型，都可以用如下配置将数据打印出来。 10 | 11 | ```json 12 | { 13 | "job": { 14 | "writer": { 15 | "class": "com.bytedance.bitsail.connector.legacy.print.sink.PrintSink", 16 | "batch_size": "10" 17 | } 18 | } 19 | } 20 | ``` -------------------------------------------------------------------------------- /website/zh/documents/connectors/print/v1/print-v1-example.md: -------------------------------------------------------------------------------- 1 | # Print-V1 连接器配置示例 2 | 3 | 父目录: [print-connector](./print-v1.md) 4 | 5 | ----- 6 | 7 | ## Print写连接器 8 | 9 | 不管上游来的数据有多少个字段、是什么数据类型，都可以用如下配置将数据打印出来。 10 | 11 | ```json 12 | { 13 | "job": { 14 | "writer": { 15 | "class": "com.bytedance.bitsail.connector.print.sink.PrintSink", 16 | "sample_write": true, 17 | "sample_limit": 10 18 | } 19 | } 20 | } 21 | ``` -------------------------------------------------------------------------------- /website/zh/documents/connectors/redis/redis-example.md: -------------------------------------------------------------------------------- 1 | # Redis 连接器示例 2 | 3 | 父目录：[Redis 连接器](./redis.md) 4 | 5 | ## Redis 写连接器 6 | 7 | 假设在本地起了一个端口为6379的Redis。用于写入该Redis的配置如下: 8 | 9 | ```json 10 | { 11 | "job": { 12 | "writer": { 13 | "class": "com.bytedance.bitsail.connector.legacy.redis.sink.RedisOutputFormat", 14 | "redis_data_type": "string", 15 | "redis_host": "localhost", 16 | "redis_port": 6379 17 | } 18 | } 19 | } 20 | ``` -------------------------------------------------------------------------------- /website/zh/documents/faq/parallelism.md: -------------------------------------------------------------------------------- 1 | # 并发度计算 2 | 3 | Todo. -------------------------------------------------------------------------------- /website/zh/documents/start/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | index: false 3 | dir: 4 | order: 1 5 | --- 6 | 7 | # 快速上手 8 | 9 | - [开发环境配置](env_setup.md) 10 | - [部署指南](deployment.md) 11 | - [任务配置指南](config.md) 12 | - [BitSail 实机演示](quick_guide.md) -------------------------------------------------------------------------------- /website/zh/release/release.md: -------------------------------------------------------------------------------- 1 | # 发版记录 2 | 3 | - [0.1.0](https://github.com/bytedance/bitsail) -------------------------------------------------------------------------------- /website/zh/team/team.md: -------------------------------------------------------------------------------- 1 | # BitSail开源团队 2 | 3 | ## 成员 -------------------------------------------------------------------------------- /website/zh/usercases/case.md: -------------------------------------------------------------------------------- 1 | # 用户案例 2 | 3 | - [Bytedance 内部使用案例](cases/bytedance.md) -------------------------------------------------------------------------------- /website/zh/usercases/cases/bytedance.md: -------------------------------------------------------------------------------- 1 | # BitSail在字节跳动中的应用案例 --------------------------------------------------------------------------------