├── connectors ├── rocketmq-connect-debezium │ ├── README.md │ ├── rocketmq-connect-debezium-mongodb │ │ └── src │ │ │ └── main │ │ │ └── resources │ │ │ └── debezium-mongodb-source-config.yaml │ ├── rocketmq-connect-debezium-oracle │ │ └── src │ │ │ └── main │ │ │ └── resources │ │ │ └── debezium-oracle-source-config.yaml │ ├── rocketmq-connect-debezium-postgresql │ │ └── src │ │ │ └── main │ │ │ └── resources │ │ │ └── debezium-postgres-source-config.yaml │ ├── rocketmq-connect-debezium-sqlserver │ │ └── src │ │ │ └── main │ │ │ └── resources │ │ │ └── debezium-sqlserver-source-config.yaml │ ├── rocketmq-connect-debezium-core │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── debezium │ │ │ └── DebeziumConnector.java │ └── kafka-connect-adaptor │ │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── kafka │ │ └── connect │ │ └── adaptor │ │ └── context │ │ └── RocketMQKafkaErrantRecordReporter.java ├── rocketmq-connect-connectors-archetype │ ├── src │ │ ├── test │ │ │ └── resources │ │ │ │ └── projects │ │ │ │ └── basic │ │ │ │ ├── goal.txt │ │ │ │ └── archetype.properties │ │ └── main │ │ │ └── resources │ │ │ └── archetype-resources │ │ │ └── src │ │ │ ├── test │ │ │ └── java │ │ │ │ └── source │ │ │ │ └── __dbNameToCamel__SourceTaskTest.java │ │ │ └── main │ │ │ └── java │ │ │ └── helper │ │ │ └── __dbNameToCamel__Record.java │ ├── README.md │ └── pom.xml ├── rocketmq-connect-cassandra │ ├── scripts │ │ └── requirements.txt │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── cassandra │ │ │ ├── strategy │ │ │ ├── DivideStrategyEnum.java │ │ │ └── TaskDivideStrategy.java │ │ │ ├── common │ │ │ ├── DataType.java │ │ │ └── ConstDefine.java │ │ │ └── schema │ │ │ └── column │ │ │ ├── BooleanColumnParser.java │ │ │ ├── DefaultColumnParser.java │ │ │ └── TimeColumnParser.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── cassandra │ │ └── common │ │ ├── TestClone.java │ │ └── CloneUtilsTest.java ├── rocketmq-connect-sftp │ └── doc │ │ ├── 使用场景.png │ │ └── RocketMQ Connect SFTP 分享.md ├── rocketmq-replicator │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── replicator │ │ │ ├── schema │ │ │ ├── SchemaEnum.java │ │ │ └── FieldName.java │ │ │ ├── strategy │ │ │ ├── DivideStrategyEnum.java │ │ │ └── TaskDivideStrategy.java │ │ │ ├── config │ │ │ ├── DataType.java │ │ │ ├── FailoverStrategy.java │ │ │ └── ConsumeFromWhere.java │ │ │ ├── common │ │ │ ├── LoggerName.java │ │ │ └── ConstDefine.java │ │ │ └── exception │ │ │ ├── ParamInvalidException.java │ │ │ ├── StartTaskException.java │ │ │ ├── GetMetaDataException.java │ │ │ └── InitMQClientException.java │ └── .gitignore ├── rocketmq-connect-jdbc │ └── src │ │ ├── main │ │ ├── resources │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ └── org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectFactory │ │ │ └── examples │ │ │ │ ├── openmldb │ │ │ │ └── openmldb-jdbc-sink.conf │ │ │ │ └── mysql │ │ │ │ ├── mysql-jdbc-source.conf │ │ │ │ └── mysql-jdbc-sink.conf │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── jdbc │ │ │ ├── source │ │ │ └── common │ │ │ │ └── QueryMode.java │ │ │ ├── common │ │ │ └── HeaderField.java │ │ │ ├── dialect │ │ │ └── DatabaseDialectFactory.java │ │ │ ├── binder │ │ │ └── JdbcRecordBinder.java │ │ │ ├── util │ │ │ └── BytesUtil.java │ │ │ └── exception │ │ │ └── TableAlterOrCreateException.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── jdbc │ │ └── connector │ │ └── source │ │ └── JdbcSourceTaskTest.java ├── rocketmq-connect-hudi │ └── src │ │ └── test │ │ └── java │ │ └── user.avsc ├── aliyun │ ├── rocketmq-connect-rocketmq │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── rocketmq │ │ │ ├── utils │ │ │ └── OnsUtils.java │ │ │ └── common │ │ │ └── RocketMQConstant.java │ ├── rocketmq-connect-mns │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── mns │ │ │ └── source │ │ │ ├── constant │ │ │ └── MNSConstant.java │ │ │ └── utils │ │ │ └── AliyunMnsUtil.java │ ├── rocketmq-connect-dingtalk │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── dingtalk │ │ │ └── sink │ │ │ └── constant │ │ │ └── DingTalkConstant.java │ ├── rocketmq-connect-oss │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── oss │ │ │ └── sink │ │ │ └── constant │ │ │ └── OssConstant.java │ ├── rocketmq-connect-fc │ │ └── src │ │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── fc │ │ │ └── sink │ │ │ └── constant │ │ │ └── FcConstant.java │ └── rocketmq-connect-eventbridge │ │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── eventbridge │ │ └── sink │ │ ├── utils │ │ ├── DateUtils.java │ │ └── CheckUtils.java │ │ └── constant │ │ └── EventBridgeConstant.java ├── rocketmq-connect-redis │ ├── src │ │ ├── test │ │ │ └── java │ │ │ │ └── org │ │ │ │ └── apache │ │ │ │ └── rocketmq │ │ │ │ └── redis │ │ │ │ └── test │ │ │ │ ├── pojo │ │ │ │ └── GeoTest.java │ │ │ │ ├── common │ │ │ │ └── OptionsTest.java │ │ │ │ └── connector │ │ │ │ └── RedisSourceConnectorTest.java │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── redis │ │ │ ├── parser │ │ │ ├── Parser.java │ │ │ ├── ExecParser.java │ │ │ ├── ScriptParser.java │ │ │ ├── DecrParser.java │ │ │ ├── IncrParser.java │ │ │ ├── LPopParser.java │ │ │ ├── MultiParser.java │ │ │ ├── RPopParser.java │ │ │ ├── PersistParser.java │ │ │ ├── FlushAllParser.java │ │ │ └── FlushDbParser.java │ │ │ ├── processor │ │ │ └── RedisEventProcessorCallback.java │ │ │ └── converter │ │ │ └── KVEntryConverter.java │ └── README-CN.md ├── rocketmq-connect-doris │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── doris │ │ ├── writer │ │ ├── load │ │ │ ├── GroupCommitMode.java │ │ │ └── DataLoad.java │ │ └── LoadStatus.java │ │ ├── model │ │ └── LoadOperation.java │ │ ├── converter │ │ └── type │ │ │ ├── connect │ │ │ └── AbstractConnectSchemaType.java │ │ │ ├── AbstractGeometryType.java │ │ │ ├── doris │ │ │ └── DorisTypeProperties.java │ │ │ ├── debezium │ │ │ └── GeographyType.java │ │ │ └── AbstractDateType.java │ │ ├── service │ │ └── DorisSinkServiceFactory.java │ │ ├── connection │ │ └── ConnectionProvider.java │ │ ├── exception │ │ ├── DorisException.java │ │ ├── UploadException.java │ │ ├── CopyLoadException.java │ │ ├── ArgumentsException.java │ │ └── DataFormatException.java │ │ └── utils │ │ └── HttpGetWithEntity.java ├── rocketmq-connect-kafka-connector-adapter │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── kafka │ │ ├── util │ │ └── KafkaPluginsUtil.java │ │ └── connector │ │ ├── RocketmqKafkaConnectorContext.java │ │ ├── KafkaRocketmqSinkConnector.java │ │ └── KafkaRocketmqSourceConnector.java ├── rocketmq-connect-mqtt │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── mqtt │ │ └── util │ │ ├── MqttConnectionUtil.java │ │ └── Utils.java ├── rocketmq-connect-http │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── http │ │ ├── auth │ │ └── Auth.java │ │ ├── HttpConfig.java │ │ └── constant │ │ ├── MethodEnum.java │ │ └── HttpHeaderConstant.java ├── rocketmq-connect-kafka │ ├── README.md │ └── src │ │ └── main │ │ └── resources │ │ └── connect-kafka-source.properties ├── rocketmq-connect-jms │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── jms │ │ └── ErrorCode.java ├── rocketmq-connect-activemq │ ├── src │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── activemq │ │ │ └── ErrorCode.java │ └── README-CN.md ├── rocketmq-connect-rabbitmq │ └── src │ │ ├── main │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── rabbitmq │ │ │ └── ErrorCode.java │ │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── jms │ │ └── RabbitmqConfigTest.java ├── rocketmq-connect-iotdb │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── iotdb │ │ ├── exception │ │ └── IotdbRuntimeException.java │ │ └── config │ │ └── IotdbConstant.java ├── rocketmq-connect-deltalake │ └── src │ │ └── main │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── deltalake │ │ ├── exception │ │ └── WriteParquetException.java │ │ └── writer │ │ └── DeltalakeWriter.java └── rocketmq-connect-hive │ └── README.md ├── .gitignore ├── rocketmq-connect-runtime ├── src │ ├── main │ │ ├── resources │ │ │ ├── META-INF │ │ │ │ └── services │ │ │ │ │ ├── org.apache.rocketmq.connect.runtime.service.ClusterManagementService │ │ │ │ │ ├── org.apache.rocketmq.connect.runtime.service.ConfigManagementService │ │ │ │ │ ├── org.apache.rocketmq.connect.runtime.service.StateManagementService │ │ │ │ │ └── org.apache.rocketmq.connect.runtime.service.PositionManagementService │ │ │ ├── user.avsc │ │ │ └── metrics.conf │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── connect │ │ │ └── runtime │ │ │ ├── service │ │ │ ├── StagingMode.java │ │ │ └── ICommonConfiguration.java │ │ │ ├── connectorwrapper │ │ │ ├── WorkerState.java │ │ │ └── WorkerTaskState.java │ │ │ ├── serialization │ │ │ ├── ByteArraySerializer.java │ │ │ ├── ByteArrayDeserializer.java │ │ │ ├── ListSerde.java │ │ │ ├── JsonSerde.java │ │ │ ├── ByteBufferDeserializer.java │ │ │ ├── ShortSerializer.java │ │ │ ├── store │ │ │ │ ├── RecordOffsetSerde.java │ │ │ │ └── RecordPartitionSerde.java │ │ │ └── IntegerSerializer.java │ │ │ ├── controller │ │ │ ├── standalone │ │ │ │ └── StandaloneConfig.java │ │ │ ├── distributed │ │ │ │ └── DistributedConfig.java │ │ │ └── ConnectController.java │ │ │ ├── errors │ │ │ ├── Operation.java │ │ │ └── ToleranceType.java │ │ │ ├── utils │ │ │ ├── Callback.java │ │ │ └── datasync │ │ │ │ └── DataSynchronizerCallback.java │ │ │ └── config │ │ │ └── SourceConnectorConfig.java │ └── test │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── runtime │ │ ├── converter │ │ └── record │ │ │ ├── AbstractRecordConverterTest.java │ │ │ ├── LongConverterTest.java │ │ │ ├── ShortConverterTest.java │ │ │ ├── IntegerConverterTest.java │ │ │ ├── DoubleConverterTest.java │ │ │ └── StringConverterTest.java │ │ └── common │ │ └── FastJsonTest.java ├── connect-standalone.sh └── connect-distributed.sh ├── schema-converter ├── schema-converter-common │ ├── src │ │ ├── test │ │ │ └── java │ │ │ │ └── com │ │ │ │ └── apache │ │ │ │ └── rocketmq │ │ │ │ └── schema │ │ │ │ └── common │ │ │ │ └── BaseRegistrySchema.java │ │ └── main │ │ │ └── java │ │ │ └── org │ │ │ └── apache │ │ │ └── rocketmq │ │ │ └── schema │ │ │ └── common │ │ │ ├── TopicNameStrategy.java │ │ │ └── Serializer.java │ └── pom.xml ├── avro-schema-converter │ └── src │ │ └── main │ │ └── resources │ │ ├── avro-sink.template │ │ └── avro-source.template └── json-schema-converter │ └── src │ └── main │ ├── resources │ ├── json-sink.template │ └── json-source.template │ └── java │ └── org │ └── apache │ └── rocketmq │ └── schema │ └── json │ └── DecimalFormat.java ├── .github └── workflows │ └── maven.yml ├── rocketmq-connect-cli ├── src │ └── main │ │ ├── resources │ │ └── connect.conf │ │ └── java │ │ └── org │ │ └── apache │ │ └── rocketmq │ │ └── connect │ │ └── cli │ │ ├── commom │ │ └── CLIConfigDefine.java │ │ └── command │ │ └── SubCommand.java └── connectAdmin ├── distribution ├── conf │ ├── connect-file-sink.properties │ ├── connect-file-source.properties │ ├── metrics.conf │ └── connect-standalone.conf ├── bin │ └── connectshutdown.sh └── NOTICE-BIN ├── metric-exporter └── src │ └── main │ └── java │ └── org │ └── apache │ └── rocketmq │ └── connect │ └── metrics │ ├── stats │ ├── Measure.java │ └── AbstractHistogram.java │ ├── IReporter.java │ └── AutoConfiguration.java ├── docker └── connect │ ├── bin │ ├── connectAdmin │ └── connectshutdown.sh │ └── README.md ├── rocketmq-connect-sample └── src │ └── main │ └── java │ └── org │ └── apache │ └── rocketmq │ └── connect │ └── file │ ├── LoggerName.java │ └── FileConstants.java └── .asf.yaml /connectors/rocketmq-connect-debezium/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-connectors-archetype/src/test/resources/projects/basic/goal.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/scripts/requirements.txt: -------------------------------------------------------------------------------- 1 | SQLAlchemy==1.3.16 2 | PyMySQL==0.9.3 -------------------------------------------------------------------------------- /connectors/rocketmq-connect-sftp/doc/使用场景.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apache/rocketmq-connect/HEAD/connectors/rocketmq-connect-sftp/doc/使用场景.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | *.class 3 | *.jar 4 | *dependency-reduced-pom.xml 5 | .classpath 6 | .project 7 | .settings/ 8 | target/ 9 | devenv 10 | *.log* 11 | .idea/ 12 | *.versionsBackup 13 | .DS_Store 14 | .vscode -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/schema/SchemaEnum.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.replicator.schema; 2 | 3 | public enum SchemaEnum { 4 | MESSAGE, 5 | OFFSET, 6 | } -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.jar 3 | *dependency-reduced-pom.xml 4 | .classpath 5 | .project 6 | .settings/ 7 | target/ 8 | devenv 9 | *.log* 10 | *.iml 11 | .idea/ 12 | *.versionsBackup 13 | .DS_Store -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/resources/META-INF/services/org.apache.rocketmq.connect.runtime.service.ClusterManagementService: -------------------------------------------------------------------------------- 1 | org.apache.rocketmq.connect.runtime.service.memory.MemoryClusterManagementServiceImpl 2 | org.apache.rocketmq.connect.runtime.service.ClusterManagementServiceImpl -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/resources/META-INF/services/org.apache.rocketmq.connect.jdbc.dialect.DatabaseDialectFactory: -------------------------------------------------------------------------------- 1 | org.apache.rocketmq.connect.jdbc.dialect.mysql.MysqlDatabaseDialectFactory 2 | org.apache.rocketmq.connect.jdbc.dialect.openmldb.OpenMLDBDatabaseDialectFactory 3 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/resources/user.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "example.avro", 2 | "type": "record", 3 | "name": "example.avro.User", 4 | "fields": [ 5 | {"name": "name", "type": "string"}, 6 | {"name": "favorite_number", "type": ["int", "null"]}, 7 | {"name": "favorite_color", "type": ["string", "null"]} 8 | ] 9 | } -------------------------------------------------------------------------------- /connectors/rocketmq-connect-hudi/src/test/java/user.avsc: -------------------------------------------------------------------------------- 1 | {"namespace": "example.avro", 2 | "type": "record", 3 | "name": "User", 4 | "fields": [ 5 | {"name": "name", "type": "string"}, 6 | {"name": "favorite_number", "type": ["int", "null"]}, 7 | {"name": "favorite_color", "type": ["string", "null"]} 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/resources/META-INF/services/org.apache.rocketmq.connect.runtime.service.ConfigManagementService: -------------------------------------------------------------------------------- 1 | org.apache.rocketmq.connect.runtime.service.memory.MemoryConfigManagementServiceImpl 2 | org.apache.rocketmq.connect.runtime.service.local.LocalConfigManagementServiceImpl 3 | org.apache.rocketmq.connect.runtime.service.rocketmq.RocketMqConfigManagementServiceImpl -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/resources/META-INF/services/org.apache.rocketmq.connect.runtime.service.StateManagementService: -------------------------------------------------------------------------------- 1 | org.apache.rocketmq.connect.runtime.service.memory.MemoryStateManagementServiceImpl 2 | org.apache.rocketmq.connect.runtime.service.local.LocalStateManagementServiceImpl 3 | org.apache.rocketmq.connect.runtime.service.rocketmq.RocketMqStateManagementServiceImpl -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/resources/META-INF/services/org.apache.rocketmq.connect.runtime.service.PositionManagementService: -------------------------------------------------------------------------------- 1 | org.apache.rocketmq.connect.runtime.service.memory.FilePositionManagementServiceImpl 2 | org.apache.rocketmq.connect.runtime.service.local.LocalPositionManagementServiceImpl 3 | org.apache.rocketmq.connect.runtime.service.rocketmq.RocketMqPositionManagementServiceImpl -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-rocketmq/src/main/java/org/apache/rocketmq/connect/rocketmq/utils/OnsUtils.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.rocketmq.utils; 2 | 3 | public class OnsUtils { 4 | 5 | public static String parseEndpoint(String namesrvAddr) { 6 | final String[] split = namesrvAddr.split("\\."); 7 | return "ons." + split[1] + ".aliyuncs.com"; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-connectors-archetype/src/test/resources/projects/basic/archetype.properties: -------------------------------------------------------------------------------- 1 | #Mon Aug 14 22:31:14 CST 2023 2 | package=it.pkg 3 | version=0.1-SNAPSHOT 4 | groupId=archetype.it 5 | artifactId=basic 6 | databaseName=test 7 | dbNameToUpperCase=TEST 8 | dbNameToCamel=Test 9 | dbNameToLowerCase=test 10 | archetype.filteredExtensions=java,sql,yml,xml,properties,factories,ftl,md 11 | 12 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/converter/record/AbstractRecordConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.runtime.converter.record; 2 | 3 | import io.openmessaging.connector.api.data.RecordConverter; 4 | 5 | /** 6 | * record converter test 7 | */ 8 | public class AbstractRecordConverterTest { 9 | protected final String TEST_TOPIC = "test_topic"; 10 | protected RecordConverter recordConverter; 11 | 12 | } 13 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/test/java/org/apache/rocketmq/redis/test/pojo/GeoTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.redis.test.pojo; 2 | 3 | import org.apache.rocketmq.connect.redis.pojo.Geo; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | public class GeoTest { 8 | @Test 9 | public void test(){ 10 | Geo geo = new Geo(); 11 | geo.setLatitude(1L); 12 | geo.setLongitude(2L); 13 | geo.setMember("hello"); 14 | 15 | Assert.assertTrue(1L == geo.getLatitude()); 16 | Assert.assertTrue(2L == geo.getLongitude()); 17 | Assert.assertEquals("hello", geo.getMember()); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /schema-converter/schema-converter-common/src/test/java/com/apache/rocketmq/schema/common/BaseRegistrySchema.java: -------------------------------------------------------------------------------- 1 | package com.apache.rocketmq.schema.common; 2 | 3 | import org.apache.rocketmq.schema.registry.client.SchemaRegistryClient; 4 | import org.apache.rocketmq.schema.registry.client.SchemaRegistryClientFactory; 5 | 6 | public abstract class BaseRegistrySchema { 7 | protected final SchemaRegistryClient schemaRegistryClient; 8 | protected final String schemaRegistryUrl = "http://localhost:8080"; 9 | public BaseRegistrySchema(){ 10 | this.schemaRegistryClient = SchemaRegistryClientFactory.newClient(schemaRegistryUrl,null); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/resources/examples/openmldb/openmldb-jdbc-sink.conf: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.sink.JdbcSinkConnector", 3 | "max.tasks":"2", 4 | "connect.topicnames":"employee_test", 5 | "connection.url":"jdbc:openmldb:///rocketmq_test?zk=127.0.0.1:2181&zkPath=/openmldb_cluster", 6 | "insert.mode":"INSERT", 7 | "db.timezone":"UTC", 8 | "table.types":"TABLE", 9 | "auto.create":"true", 10 | "key.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter", 11 | "value.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter" 12 | } 13 | -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-mns/src/main/java/org/apache/rocketmq/connect/mns/source/constant/MNSConstant.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.mns.source.constant; 2 | 3 | public class MNSConstant { 4 | 5 | public static final String ACCESS_KEY_ID = "accessKeyId"; 6 | public static final String ACCESS_KEY_SECRET = "accessKeySecret"; 7 | public static final String ACCOUNT_ENDPOINT = "accountEndpoint"; 8 | public static final String QUEUE_NAME = "queueName"; 9 | public static final String ACCOUNT_ID = "accountId"; 10 | public static final String BATCH_SIZE = "batchSize"; 11 | public static final String IS_BASE64_DECODE = "isBase64Decode"; 12 | } 13 | -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-dingtalk/src/main/java/org/apache/rocketmq/connect/dingtalk/sink/constant/DingTalkConstant.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.dingtalk.sink.constant; 2 | 3 | public class DingTalkConstant { 4 | 5 | public static final String CONTENT_TYPE = "Content-Type"; 6 | public static final String APPLICATION_JSON_UTF_8_TYPE = "application/json"; 7 | public static final String WEB_HOOK = "webHook"; 8 | public static final String SECRET_KEY = "secretKey"; 9 | public static final String TIMESTAMP_CONSTANT = "timestamp"; 10 | public static final String SIGN_CONSTANT = "sign"; 11 | public static final String HMAC_SHA256_CONSTANT = "HmacSHA256"; 12 | } 13 | -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-mns/src/main/java/org/apache/rocketmq/connect/mns/source/utils/AliyunMnsUtil.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.mns.source.utils; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | public class AliyunMnsUtil { 6 | 7 | private static final int ARRAY_LENGTH = 3; 8 | 9 | public static String parseRegionIdFromEndpoint(String endpoint) { 10 | String regionId = null; 11 | if (StringUtils.isBlank(endpoint)) { 12 | return regionId; 13 | } 14 | String[] split = endpoint.split("\\."); 15 | if (split.length >= ARRAY_LENGTH) { 16 | regionId = split[2]; 17 | } 18 | return regionId; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/resources/examples/mysql/mysql-jdbc-source.conf: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.source.JdbcSourceConnector", 3 | "max.tasks":"1", 4 | "connect.topicname":"employee-test-topic-json-00002", 5 | "connection.url":"jdbc:mysql://localhost:3306", 6 | "connection.user":"*****", 7 | "connection.password":"*****", 8 | "table.whitelist":"test_database.employee_copy_test03", 9 | "mode": "incrementing", 10 | "incrementing.column.name":"id", 11 | "key.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter", 12 | "value.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter" 13 | } 14 | 15 | 16 | -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-rocketmq/src/main/java/org/apache/rocketmq/connect/rocketmq/common/RocketMQConstant.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.rocketmq.common; 2 | 3 | public class RocketMQConstant { 4 | 5 | public static final String ACCESS_KEY_ID = "accessKeyId"; 6 | 7 | public static final String ACCESS_KEY_SECRET = "accessKeySecret"; 8 | 9 | public static final String NAMESRV_ADDR = "namesrvAddr"; 10 | 11 | public static final String TOPIC = "topic"; 12 | 13 | public static final String INSTANCE_ID = "instanceId"; 14 | 15 | public static final String CONSUMER_GROUP = "consumerGroup"; 16 | 17 | public static final String KEY = "KEYS"; 18 | 19 | public static final String TAG = "TAGS"; 20 | 21 | } 22 | -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-oss/src/main/java/org/apache/rocketmq/connect/oss/sink/constant/OssConstant.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.oss.sink.constant; 2 | 3 | public class OssConstant { 4 | 5 | public static final String ACCESS_KEY_ID = "accessKeyId"; 6 | public static final String ACCESS_KEY_SECRET = "accessKeySecret"; 7 | public static final String ACCOUNT_ENDPOINT = "accountEndpoint"; 8 | public static final String BUCKET_NAME = "bucketName"; 9 | public static final String FILE_URL_PREFIX = "fileUrlPrefix"; 10 | public static final String OBJECT_NAME = "objectName"; 11 | public static final String REGION = "region"; 12 | public static final String PARTITION_METHOD = "partitionMethod"; 13 | public static final String COMPRESS_TYPE = "compressType"; 14 | } 15 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/writer/load/GroupCommitMode.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.doris.writer.load; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | public enum GroupCommitMode { 7 | OFF_MODE("off_mode"), 8 | SYNC_MODE("sync_mode"), 9 | ASYNC_MODE("async_mode"); 10 | 11 | private final String name; 12 | 13 | GroupCommitMode(String name) { 14 | this.name = name; 15 | } 16 | 17 | public String getName() { 18 | return name; 19 | } 20 | 21 | public static LoadModel of(String name) { 22 | return LoadModel.valueOf(name.toUpperCase()); 23 | } 24 | 25 | public static List instances() { 26 | return Arrays.asList(OFF_MODE.name, SYNC_MODE.name, ASYNC_MODE.name); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven 3 | 4 | name: Java CI with Maven 5 | 6 | on: 7 | push: 8 | branches: [ "master" ] 9 | pull_request: 10 | types: [ opened, reopened, synchronize ] 11 | branches: [ "master" ] 12 | 13 | jobs: 14 | build: 15 | 16 | runs-on: ubuntu-latest 17 | 18 | steps: 19 | - uses: actions/checkout@v3 20 | - name: Set up JDK 8 21 | uses: actions/setup-java@v3 22 | with: 23 | java-version: '8' 24 | distribution: 'temurin' 25 | cache: maven 26 | - name: Build with Maven 27 | run: mvn -B clean install -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/test/java/org/apache/rocketmq/connect/jdbc/connector/source/JdbcSourceTaskTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.jdbc.connector.source; 2 | 3 | import io.openmessaging.KeyValue; 4 | import io.openmessaging.internal.DefaultKeyValue; 5 | import org.apache.rocketmq.connect.jdbc.source.JdbcSourceTaskConfig; 6 | import org.junit.Test; 7 | 8 | /** 9 | * jdbc source task 10 | */ 11 | public class JdbcSourceTaskTest { 12 | // jdbc source task 13 | KeyValue conf = new DefaultKeyValue(); 14 | 15 | public void init() { 16 | conf.put(JdbcSourceTaskConfig.CONNECTION_URL_CONFIG, ""); 17 | conf.put(JdbcSourceTaskConfig.CONNECTION_USER_CONFIG, ""); 18 | conf.put(JdbcSourceTaskConfig.CONNECTION_PASSWORD_CONFIG, ""); 19 | } 20 | 21 | @Test 22 | public void pollTest() { 23 | 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-fc/src/main/java/org/apache/rocketmq/connect/fc/sink/constant/FcConstant.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.fc.sink.constant; 2 | 3 | public class FcConstant { 4 | 5 | public static final String REGION_ID_CONSTANT = "regionId"; 6 | public static final String ACCESS_KEY_ID_CONSTANT = "accessKeyId"; 7 | public static final String ACCESS__KEY_SECRET_CONSTANT = "accessKeySecret"; 8 | public static final String ACCOUNT_ID_CONSTANT = "accountId"; 9 | public static final String SERVICE_NAME_CONSTANT = "serviceName"; 10 | public static final String FUNCTION_NAME_CONSTANT = "functionName"; 11 | public static final String INVOCATION_TYPE_CONSTANT = "invocationType"; 12 | public static final String QUALIFIER_CONSTANT = "qualifier"; 13 | public static final String DEFAULT_QUALIFIER_CONSTANT = "LATEST"; 14 | } 15 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/resources/examples/mysql/mysql-jdbc-sink.conf: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.sink.JdbcSinkConnector", 3 | "max.tasks":"2", 4 | "connect.topicnames":"employee-test-topic-json-00002", 5 | "connection.url":"jdbc:mysql://localhost:3306/test_database_001", 6 | "connection.user":"*****", 7 | "connection.password":"*****", 8 | "pk.fields":"id", 9 | "pk.mode":"record_value", 10 | "insert.mode":"UPSERT", 11 | "db.timezone":"UTC", 12 | "table.types":"TABLE", 13 | "errors.deadletterqueue.topic.name":"dlq-topic", 14 | "errors.log.enable":"true", 15 | "errors.tolerance":"ALL", 16 | "key.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter", 17 | "value.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter" 18 | } -------------------------------------------------------------------------------- /connectors/rocketmq-connect-kafka-connector-adapter/src/main/java/org/apache/rocketmq/connect/kafka/util/KafkaPluginsUtil.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.kafka.util; 2 | 3 | import org.apache.kafka.connect.runtime.isolation.Plugins; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | 8 | public class KafkaPluginsUtil { 9 | 10 | 11 | public static final String PLUGIN_PATH = "plugin.path"; 12 | private static final Map CACHE = new HashMap<>(); 13 | 14 | public static Plugins getPlugins(Map props){ 15 | String path = props.get(PLUGIN_PATH); 16 | synchronized (CACHE){ 17 | Plugins plugins = CACHE.get(path); 18 | if(plugins == null){ 19 | plugins = new Plugins(props); 20 | CACHE.put(path, plugins); 21 | } 22 | return plugins; 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-eventbridge/src/main/java/org/apache/rocketmq/connect/eventbridge/sink/utils/DateUtils.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.eventbridge.sink.utils; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.text.SimpleDateFormat; 7 | import java.util.Date; 8 | 9 | public class DateUtils { 10 | private static final Logger log = LoggerFactory.getLogger(DateUtils.class); 11 | public static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss"; 12 | 13 | public static synchronized Date getDate(String date, String dateFormat) { 14 | try { 15 | SimpleDateFormat simpleDateFormat = new SimpleDateFormat(dateFormat); 16 | return simpleDateFormat.parse(date); 17 | } catch (Exception e) { 18 | log.error("DateUtils | getDate | error => ", e); 19 | } 20 | return null; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-kafka-connector-adapter/src/main/java/org/apache/rocketmq/connect/kafka/connector/RocketmqKafkaConnectorContext.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.kafka.connector; 2 | 3 | import io.openmessaging.connector.api.component.connector.ConnectorContext; 4 | 5 | public class RocketmqKafkaConnectorContext implements org.apache.kafka.connect.connector.ConnectorContext{ 6 | protected ConnectorContext rocketMqConnectorContext; 7 | 8 | public RocketmqKafkaConnectorContext(ConnectorContext rocketMqConnectorContext) { 9 | this.rocketMqConnectorContext = rocketMqConnectorContext; 10 | } 11 | 12 | @Override 13 | public void requestTaskReconfiguration() { 14 | this.rocketMqConnectorContext.requestTaskReconfiguration(); 15 | } 16 | 17 | @Override 18 | public void raiseError(Exception e) { 19 | this.rocketMqConnectorContext.raiseError(e); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-eventbridge/src/main/java/org/apache/rocketmq/connect/eventbridge/sink/utils/CheckUtils.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.eventbridge.sink.utils; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | public class CheckUtils { 6 | private static final String NULL_CONSTANT = "null"; 7 | 8 | public static Boolean checkNull(String check) { 9 | if (StringUtils.isBlank(check)) { 10 | return Boolean.TRUE; 11 | } 12 | if (StringUtils.isNotBlank(check) && NULL_CONSTANT.equalsIgnoreCase(check)) { 13 | return Boolean.TRUE; 14 | } 15 | return Boolean.FALSE; 16 | } 17 | 18 | public static Boolean checkNotNull(String check) { 19 | if (StringUtils.isNotBlank(check) && !NULL_CONSTANT.equalsIgnoreCase(check)) { 20 | return Boolean.TRUE; 21 | } 22 | return Boolean.FALSE; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-mongodb/src/main/resources/debezium-mongodb-source-config.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/converter/record/LongConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.runtime.converter.record; 2 | 3 | import io.openmessaging.connector.api.data.SchemaAndValue; 4 | import io.openmessaging.connector.api.data.SchemaBuilder; 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | public class LongConverterTest extends AbstractRecordConverterTest { 10 | 11 | 12 | @Before 13 | public void before() { 14 | this.recordConverter = new LongConverter(); 15 | } 16 | 17 | 18 | @Test 19 | public void testIntegerConverter() { 20 | Long int64 = 64L; 21 | byte[] bytes = recordConverter.fromConnectData(TEST_TOPIC, SchemaBuilder.int64().build(), int64); 22 | SchemaAndValue schemaAndValue = recordConverter.toConnectData(TEST_TOPIC, bytes); 23 | Assert.assertEquals(schemaAndValue.value(), int64); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/converter/record/ShortConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.runtime.converter.record; 2 | 3 | import io.openmessaging.connector.api.data.SchemaAndValue; 4 | import io.openmessaging.connector.api.data.SchemaBuilder; 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | public class ShortConverterTest extends AbstractRecordConverterTest { 10 | 11 | 12 | @Before 13 | public void before() { 14 | this.recordConverter = new ShortConverter(); 15 | } 16 | 17 | 18 | @Test 19 | public void testShortConverter() { 20 | Short int16 = 16; 21 | byte[] bytes = recordConverter.fromConnectData(TEST_TOPIC, SchemaBuilder.int16().build(), int16); 22 | SchemaAndValue schemaAndValue = recordConverter.toConnectData(TEST_TOPIC, bytes); 23 | Assert.assertEquals(schemaAndValue.value(), int16); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-oracle/src/main/resources/debezium-oracle-source-config.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-postgresql/src/main/resources/debezium-postgres-source-config.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | -------------------------------------------------------------------------------- /schema-converter/avro-schema-converter/src/main/resources/avro-sink.template: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.sink.MysqlJdbcSinkConnector", 3 | "max.tasks":"2", 4 | "connect.topicnames":"employee-test-topic-avro-0003", 5 | "connection.url":"jdbc:mysql://localhost:3306/test_database_02", 6 | "connection.user":"XXXXXX", 7 | "connection.password":"XXXXXX", 8 | "pk.fields":"id", 9 | "pk.mode":"record_value", 10 | "insert.mode":"UPSERT", 11 | "db.timezone":"UTC", 12 | "table.types":"TABLE", 13 | "errors.deadletterqueue.topic.name":"dlq-topic", 14 | "errors.log.enable":"true", 15 | "errors.tolerance":"ALL", 16 | "key.converter.schema.registry.url":"http://localhost:8080", 17 | "value.converter.schema.registry.url":"http://localhost:8080", 18 | "key.converter":"org.apache.rocketmq.schema.avro.AvroConverter", 19 | "value.converter":"org.apache.rocketmq.schema.avro.AvroConverter" 20 | } 21 | 22 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-sqlserver/src/main/resources/debezium-sqlserver-source-config.yaml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # speercific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/converter/record/IntegerConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.runtime.converter.record; 2 | 3 | import io.openmessaging.connector.api.data.SchemaAndValue; 4 | import io.openmessaging.connector.api.data.SchemaBuilder; 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | public class IntegerConverterTest extends AbstractRecordConverterTest { 10 | 11 | 12 | @Before 13 | public void before() { 14 | this.recordConverter = new IntegerConverter(); 15 | } 16 | 17 | 18 | @Test 19 | public void testIntegerConverter() { 20 | Integer int32 = 32; 21 | byte[] bytes = recordConverter.fromConnectData(TEST_TOPIC, SchemaBuilder.int32().build(), int32); 22 | SchemaAndValue schemaAndValue = recordConverter.toConnectData(TEST_TOPIC, bytes); 23 | Assert.assertEquals(schemaAndValue.value(), int32); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/converter/record/DoubleConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.runtime.converter.record; 2 | 3 | import io.openmessaging.connector.api.data.SchemaAndValue; 4 | import io.openmessaging.connector.api.data.SchemaBuilder; 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | public class DoubleConverterTest extends AbstractRecordConverterTest { 10 | 11 | 12 | @Before 13 | public void before() { 14 | this.recordConverter = new DoubleConverter(); 15 | } 16 | 17 | 18 | @Test 19 | public void testDoubleConverter() { 20 | Double float64 = 32.00; 21 | byte[] bytes = recordConverter.fromConnectData(TEST_TOPIC, SchemaBuilder.float64().build(), float64); 22 | SchemaAndValue schemaAndValue = recordConverter.toConnectData(TEST_TOPIC, bytes); 23 | Assert.assertEquals(schemaAndValue.value(), float64); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/converter/record/StringConverterTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.runtime.converter.record; 2 | 3 | import io.openmessaging.connector.api.data.SchemaAndValue; 4 | import io.openmessaging.connector.api.data.SchemaBuilder; 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | 9 | public class StringConverterTest extends AbstractRecordConverterTest { 10 | 11 | 12 | @Before 13 | public void before() { 14 | this.recordConverter = new StringConverter(); 15 | } 16 | 17 | 18 | @Test 19 | public void testStringConverter() { 20 | String str = "test_converter"; 21 | byte[] bytes = recordConverter.fromConnectData(TEST_TOPIC, SchemaBuilder.string().build(), str); 22 | SchemaAndValue schemaAndValue = recordConverter.toConnectData(TEST_TOPIC, bytes); 23 | Assert.assertEquals(schemaAndValue.value(), str); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /schema-converter/json-schema-converter/src/main/resources/json-sink.template: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.sink.MysqlJdbcSinkConnector", 3 | "max.tasks":"2", 4 | "connect.topicnames":"employee-test-topic-avro-0003", 5 | "connection.url":"jdbc:mysql://localhost:3306/test_database_02", 6 | "connection.user":"XXXXXX", 7 | "connection.password":"XXXXXX", 8 | "pk.fields":"id", 9 | "pk.mode":"record_value", 10 | "insert.mode":"UPSERT", 11 | "db.timezone":"UTC", 12 | "table.types":"TABLE", 13 | "errors.deadletterqueue.topic.name":"dlq-topic", 14 | "errors.log.enable":"true", 15 | "errors.tolerance":"ALL", 16 | "key.converter.schema.registry.url":"http://localhost:8080", 17 | "value.converter.schema.registry.url":"http://localhost:8080", 18 | "key.converter":"org.apache.rocketmq.schema.json.JsonSchemaConverter", 19 | "value.converter":"org.apache.rocketmq.schema.json.JsonSchemaConverter" 20 | } 21 | 22 | -------------------------------------------------------------------------------- /rocketmq-connect-cli/src/main/resources/connect.conf: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | ## Http port for user to access REST API 17 | httpPort=8081 18 | 19 | # RocketMQ Connect Runtime Addr 20 | httpAddr=localhost -------------------------------------------------------------------------------- /rocketmq-connect-cli/connectAdmin: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | cd target/distribution/ && java -cp .:./conf/:./lib/* org.apache.rocketmq.connect.cli.ConnectAdminStartup -s conf/connect.conf $@ 19 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-connectors-archetype/README.md: -------------------------------------------------------------------------------- 1 | ## How to Use Connnector-Archetype 2 | 3 | 1. 进入脚手架文件夹 4 | 5 | ```shell 6 | cd rocketmq-connect-connectors-archetype/ 7 | ``` 8 | 9 | 2. 将脚手架安装到本地 10 | 11 | ```shell 12 | mvn -e clean install 13 | ``` 14 | 15 | 3. 创建connector模版工程 16 | 17 | ```shell 18 | cd connectors/ 19 | mvn archetype:generate \ 20 | -DarchetypeGroupId=org.apache.rocketmq \ 21 | -DarchetypeArtifactId=rocketmq-connect-connectors-archetype \ 22 | -DarchetypeVersion=1.0-SNAPSHOT \ 23 | -DdatabaseName= 24 | ``` 25 | 26 | 例：创建Clickhouse-Connector 27 | 28 | ```shell 29 | mvn archetype:generate \ 30 | -DarchetypeGroupId=org.apache.rocketmq \ 31 | -DarchetypeArtifactId=rocketmq-connect-connectors-archetype \ 32 | -DarchetypeVersion=1.0-SNAPSHOT \ 33 | -DdatabaseName=clickhouse 34 | ``` 35 | 36 | 4. 如上指令将创建一个connector的框架，开发者主要关心`helper/xxxHelperClient`以及`xxxxSourceTask`,`xxxSinkTask`的实现即可，剩余配置可以按需修改。 37 | -------------------------------------------------------------------------------- /distribution/conf/connect-file-sink.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | name=FileSinkConnector 17 | connector.class=org.apache.rocketmq.connect.file.FileSinkConnector 18 | file=test-sink.txt 19 | connect.topicnames=TopicTest -------------------------------------------------------------------------------- /distribution/conf/connect-file-source.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | name=FileSourceConnector 17 | connector.class=org.apache.rocketmq.connect.file.FileSourceConnector 18 | file=test-source.txt 19 | connect.topicname=TopicTest -------------------------------------------------------------------------------- /connectors/aliyun/rocketmq-connect-eventbridge/src/main/java/org/apache/rocketmq/connect/eventbridge/sink/constant/EventBridgeConstant.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.eventbridge.sink.constant; 2 | 3 | public class EventBridgeConstant { 4 | 5 | public static final String ACCESS_KEY_ID = "accessKeyId"; 6 | public static final String ACCESS_KEY_SECRET = "accessKeySecret"; 7 | public static final String ACCOUNT_ENDPOINT = "accountEndpoint"; 8 | public static final String ROLE_ARN = "roleArn"; 9 | public static final String ROLE_SESSION_NAME = "roleSessionName"; 10 | public static final String EVENT_ID = "id"; 11 | public static final String EVENT_SOURCE = "eventSource"; 12 | public static final String EVENT_TIME = "eventTime"; 13 | public static final String EVENT_TYPE = "eventType"; 14 | public static final String EVENT_SUBJECT = "eventSubject"; 15 | public static final String ALIYUN_EVENT_BUS_NAME = "aliyuneventbusname"; 16 | public static final String STS_ENDPOINT = "stsEndpoint"; 17 | } 18 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/connect-standalone.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | echo "run runtime worker" 17 | cd target/distribution/ && java -cp .:./conf/:./lib/* org.apache.rocketmq.connect.runtime.StandaloneConnectStartup -c conf/connect-standalone.conf -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/test/java/org/apache/rocketmq/redis/test/common/OptionsTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.redis.test.common; 2 | 3 | import org.apache.rocketmq.connect.redis.common.Options; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | 7 | 8 | public class OptionsTest { 9 | 10 | @Test 11 | public void test(){ 12 | Options options1 = Options.valueOf("TEST"); 13 | Assert.assertNull(options1); 14 | 15 | Options options2 = Options.REDIS_PARTITION; 16 | Assert.assertEquals(options2.name(), "DEFAULT_PARTITION"); 17 | 18 | Assert.assertNotNull(options2.toString()); 19 | 20 | Options options3 = Options.valueOf("DEFAULT_PARTITION"); 21 | Assert.assertTrue(options3.equals(options2)); 22 | } 23 | 24 | @Test 25 | public void testNull(){ 26 | Exception ex = null; 27 | try { 28 | Options.valueOf(""); 29 | }catch (Exception e){ 30 | ex = e; 31 | } 32 | Assert.assertNotNull(ex); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/connect-distributed.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | echo "run runtime worker" 17 | cd target/distribution/ && java -cp .:./conf/:./lib/* org.apache.rocketmq.connect.runtime.DistributedConnectStartup -c conf/connect-distributed.conf -------------------------------------------------------------------------------- /connectors/rocketmq-connect-connectors-archetype/src/main/resources/archetype-resources/src/test/java/source/__dbNameToCamel__SourceTaskTest.java: -------------------------------------------------------------------------------- 1 | #set( $symbol_pound = '#' ) 2 | #set( $symbol_dollar = '$' ) 3 | #set( $symbol_escape = '\' ) 4 | package ${package}.source; 5 | 6 | import io.openmessaging.KeyValue; 7 | import io.openmessaging.connector.api.data.ConnectRecord; 8 | import io.openmessaging.internal.DefaultKeyValue; 9 | import java.util.List; 10 | import junit.framework.TestCase; 11 | import ${package}.config.${dbNameToCamel}Constants; 12 | 13 | import static java.lang.Thread.sleep; 14 | 15 | public class ${dbNameToCamel}SourceTaskTest { 16 | 17 | // private static final String host = "127.0.0.1"; 18 | // private static final String port = "8123"; 19 | // private static final String db = "default"; 20 | // private static final String username = "default"; 21 | // private static final String password = "123456"; 22 | // 23 | // public void testPoll() { 24 | // } 25 | // 26 | // public void testStart() throws InterruptedException { 27 | // } 28 | } -------------------------------------------------------------------------------- /metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/stats/Measure.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.metrics.stats; 18 | 19 | /** 20 | * gauge 21 | */ 22 | public interface Measure { 23 | double value(); 24 | } -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/source/common/QueryMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.jdbc.source.common; 19 | 20 | public enum QueryMode { 21 | TABLE, 22 | QUERY 23 | } -------------------------------------------------------------------------------- /connectors/rocketmq-connect-mqtt/src/main/java/org/apache/rocketmq/connect/mqtt/util/MqttConnectionUtil.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.mqtt.util; 2 | 3 | import java.security.InvalidKeyException; 4 | import java.security.NoSuchAlgorithmException; 5 | import org.apache.rocketmq.connect.mqtt.config.ConnectorConfig; 6 | import org.eclipse.paho.client.mqttv3.MqttConnectOptions; 7 | 8 | public class MqttConnectionUtil { 9 | public static MqttConnectOptions buildMqttConnectOptions( 10 | String clientId, ConnectorConfig connectorConfig) throws NoSuchAlgorithmException, InvalidKeyException { 11 | MqttConnectOptions connOpts = new MqttConnectOptions(); 12 | connOpts.setCleanSession(true); 13 | connOpts.setKeepAliveInterval(60); 14 | connOpts.setAutomaticReconnect(true); 15 | connOpts.setMaxInflight(10000); 16 | connOpts.setUserName(connectorConfig.getMqttAccessKey()); 17 | connOpts.setPassword(HmacSHA1Util.macSignature(clientId, connectorConfig.getMqttSecretKey()).toCharArray()); 18 | return connOpts; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/test/java/org/apache/rocketmq/connect/runtime/common/FastJsonTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.runtime.common; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.alibaba.fastjson.JSONObject; 5 | import com.alibaba.fastjson.util.TypeUtils; 6 | 7 | import java.nio.charset.StandardCharsets; 8 | 9 | public class FastJsonTest { 10 | public static void main(String[] args) { 11 | TestByte testByte = new TestByte(); 12 | byte[] originalJson = JSON.toJSONBytes(testByte); 13 | JSONObject str = (JSONObject) JSON.parse(originalJson); 14 | byte[] byte1 = TypeUtils.castToBytes(str.get("bytes").toString()); 15 | System.out.println(new java.lang.String(byte1, StandardCharsets.UTF_8)); 16 | } 17 | 18 | static class TestByte { 19 | private Object bytes = "123".getBytes(StandardCharsets.UTF_8); 20 | 21 | public Object getBytes() { 22 | return bytes; 23 | } 24 | 25 | public void setBytes(Object bytes) { 26 | this.bytes = bytes; 27 | } 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-http/src/main/java/org/apache/rocketmq/connect/http/auth/Auth.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.http.auth; 18 | 19 | import java.util.Map; 20 | 21 | public interface Auth { 22 | Map auth(); 23 | } 24 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/DivideStrategyEnum.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.strategy; 18 | 19 | public enum DivideStrategyEnum { 20 | 21 | BY_TOPIC, 22 | BY_QUEUE 23 | } 24 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/StagingMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.service; 19 | 20 | public enum StagingMode { 21 | STANDALONE, 22 | DISTRIBUTED, 23 | UNIVERSAL 24 | } 25 | -------------------------------------------------------------------------------- /schema-converter/avro-schema-converter/src/main/resources/avro-source.template: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.source.MysqlJdbcSourceConnector", 3 | "max.tasks":"1", 4 | "connect.topicname":"employee-test-topic-avro-0003", 5 | "connection.url":"jdbc:mysql://localhost:3306", 6 | "connection.user":"XXXXXX", 7 | "connection.password":"XXXXXX", 8 | "table.whitelist":"test_database.employee_copy_test03", 9 | "mode": "incrementing", 10 | "incrementing.column.name":"id", 11 | "transforms": "Replace", 12 | "transforms.Replace.delete.handling.mode": "none", 13 | "transforms.Replace.field.pattern": "company", 14 | "transforms.Replace.field.replacement": "company02", 15 | "transforms.Replace.class": "org.apache.rocketmq.connect.transforms.PatternRename$Value", 16 | "key.converter.schema.registry.url":"http://localhost:8080", 17 | "value.converter.schema.registry.url":"http://localhost:8080", 18 | "key.converter":"org.apache.rocketmq.schema.avro.AvroConverter", 19 | "value.converter":"org.apache.rocketmq.schema.avro.AvroConverter" 20 | } -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | package org.apache.rocketmq.connect.runtime.connectorwrapper; 20 | 21 | public enum WorkerState { 22 | STARTED, 23 | TERMINATED, 24 | } 25 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/service/ICommonConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.service; 19 | 20 | public interface ICommonConfiguration { 21 | boolean enabledCompactTopic(); 22 | } 23 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/strategy/DivideStrategyEnum.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.cassandra.strategy; 18 | 19 | public enum DivideStrategyEnum { 20 | 21 | BY_TOPIC, 22 | BY_QUEUE 23 | } 24 | -------------------------------------------------------------------------------- /schema-converter/json-schema-converter/src/main/java/org/apache/rocketmq/schema/json/DecimalFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.schema.json; 18 | 19 | 20 | /** 21 | * decimal format 22 | */ 23 | public enum DecimalFormat { 24 | BASE64, 25 | NUMERIC 26 | } 27 | -------------------------------------------------------------------------------- /schema-converter/json-schema-converter/src/main/resources/json-source.template: -------------------------------------------------------------------------------- 1 | { 2 | "connector.class":"org.apache.rocketmq.connect.jdbc.mysql.source.MysqlJdbcSourceConnector", 3 | "max.tasks":"1", 4 | "connect.topicname":"employee-test-topic-avro-0003", 5 | "connection.url":"jdbc:mysql://localhost:3306", 6 | "connection.user":"XXXXXX", 7 | "connection.password":"XXXXXX", 8 | "table.whitelist":"test_database.employee_copy_test03", 9 | "mode": "incrementing", 10 | "incrementing.column.name":"id", 11 | "transforms": "Replace", 12 | "transforms.Replace.delete.handling.mode": "none", 13 | "transforms.Replace.field.pattern": "company", 14 | "transforms.Replace.field.replacement": "company02", 15 | "transforms.Replace.class": "org.apache.rocketmq.connect.transforms.PatternRename$Value", 16 | "key.converter.schema.registry.url":"http://localhost:8080", 17 | "value.converter.schema.registry.url":"http://localhost:8080", 18 | "key.converter":"org.apache.rocketmq.schema.json.JsonSchemaConverter", 19 | "value.converter":"org.apache.rocketmq.schema.json.JsonSchemaConverter" 20 | } -------------------------------------------------------------------------------- /connectors/rocketmq-connect-kafka/README.md: -------------------------------------------------------------------------------- 1 | **rocketmq-connect-kafka** 2 | 3 | 在启动runtime之后，通过发送http消息到runtime，携带connector和task的参数，启动connector 4 | 5 | **参数说明** 6 | 7 | - **connector-class**: connector的类名 8 | - **oms-driver-url**: rocketmq地址 9 | - **source-record-converter**: source record converter 10 | - **tasks.num**: 启动的task数目 11 | - **kafka.topics**: topic列表，多个topic通过逗号“,”隔开 12 | - **kafka.group.id**: 消费组名，多个connector中，需要保证topic和groupid的一致性 13 | - **kafka.bootstrap.server**: kafka地址 14 | 15 | 16 | **启动Connector** 17 | 18 | http://127.0.0.1:8081/connectors/connector-name?config={"connector-class":"org.apache.rocketmq.connect.kafka.connector.KafkaSourceConnector","tasks.num":"1","kafka.topics":"test1,test2","kafka.group.id":"group0","kafka.bootstrap.server":"127.0.0.1:9092","source-record-converter":"org.apache.rocketmq.connect.runtime.converter.JsonConverter"} 19 | 20 | **查看Connector运行状态** 21 | 22 | http://127.0.0.1:8081/connectors/connector-name/status 23 | 24 | **查看Connector配置** 25 | 26 | http://127.0.0.1:8081/connectors/connector-name/config 27 | 28 | **关闭Connector** 29 | 30 | http://127.0.0.1:8081/connectors/connector-name/stop 31 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/DataType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.config; 18 | 19 | public enum DataType { 20 | 21 | COMMON_MESSAGE, 22 | TOPIC_CONFIG, 23 | BROKER_CONFIG, 24 | SUB_CONFIG, 25 | OFFSET 26 | } 27 | -------------------------------------------------------------------------------- /schema-converter/schema-converter-common/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | schema-converter 7 | org.apache.rocketmq 8 | 0.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | schema-converter-common 12 | 13 | 14 | 8 15 | 8 16 | UTF-8 17 | 18 | 19 | 20 | 21 | org.apache.rocketmq 22 | schema-registry-client 23 | ${schema-registry.version} 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/Parser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import com.moilioncircle.redis.replicator.cmd.Command; 21 | 22 | public interface Parser { 23 | T parse(Command command); 24 | } 25 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/FailoverStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.config; 18 | 19 | /** 20 | * @author osgoo 21 | * @date 2022/6/20 22 | */ 23 | public enum FailoverStrategy { 24 | DISMISS, 25 | CUSTOMER, 26 | } 27 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jms/src/main/java/org/apache/rocketmq/connect/jms/ErrorCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.jms; 19 | 20 | public class ErrorCode { 21 | 22 | public static final int START_ERROR_CODE = 10001; 23 | 24 | public static final int STOP_ERROR_CODE = 10002; 25 | } 26 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/processor/RedisEventProcessorCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.processor; 19 | 20 | public interface RedisEventProcessorCallback { 21 | 22 | void onStop(RedisEventProcessor eventProcessor); 23 | } 24 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/common/DataType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.cassandra.common; 18 | 19 | public enum DataType { 20 | 21 | COMMON_MESSAGE, 22 | TOPIC_CONFIG, 23 | BROKER_CONFIG, 24 | SUB_CONFIG, 25 | OFFSET 26 | } 27 | -------------------------------------------------------------------------------- /metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/IReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.metrics; 18 | 19 | /** 20 | * astrict reporter 21 | */ 22 | public interface IReporter extends AutoCloseable { 23 | 24 | /** 25 | * start exporter 26 | */ 27 | void start(); 28 | } 29 | -------------------------------------------------------------------------------- /docker/connect/bin/connectAdmin: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | # 19 | 20 | 21 | #cd target/distribution/ && java -cp .:./conf/:./lib/* org.apache.rocketmq.connect.cli.ConnectAdminStartup -s conf/connect.conf $@ 22 | cd /home/admin/mq-connect/cli && java -cp .:./conf/:./lib/* org.apache.rocketmq.connect.cli.ConnectAdminStartup -s conf/connect.conf $@ 23 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-activemq/src/main/java/org/apache/rocketmq/connect/activemq/ErrorCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.activemq; 19 | 20 | public class ErrorCode { 21 | 22 | public static final int START_ERROR_CODE = 10001; 23 | 24 | public static final int STOP_ERROR_CODE = 10002; 25 | } 26 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-rabbitmq/src/main/java/org/apache/rocketmq/connect/rabbitmq/ErrorCode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.rabbitmq; 19 | 20 | public class ErrorCode { 21 | 22 | public static final int START_ERROR_CODE = 10001; 23 | 24 | public static final int STOP_ERROR_CODE = 10002; 25 | } 26 | -------------------------------------------------------------------------------- /rocketmq-connect-sample/src/main/java/org/apache/rocketmq/connect/file/LoggerName.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.file; 19 | 20 | /** 21 | * Define all the logger name of the file connector. 22 | */ 23 | public class LoggerName { 24 | public static final String FILE_CONNECTOR = "FileConnector"; 25 | } 26 | -------------------------------------------------------------------------------- /distribution/bin/connectshutdown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | pid=$(ps ax | grep -i 'ConnectStartup' | grep java | grep -v grep | awk '{print $1}') 18 | if [ -z "$pid" ]; then 19 | echo "No connect running." 20 | exit -1 21 | fi 22 | 23 | echo "The connect(${pid}) is running..." 24 | 25 | kill ${pid} 26 | 27 | echo "Send shutdown request to connect(${pid}) OK" 28 | -------------------------------------------------------------------------------- /docker/connect/bin/connectshutdown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Licensed to the Apache Software Foundation (ASF) under one or more 4 | # contributor license agreements. See the NOTICE file distributed with 5 | # this work for additional information regarding copyright ownership. 6 | # The ASF licenses this file to You under the Apache License, Version 2.0 7 | # (the "License"); you may not use this file except in compliance with 8 | # the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | pid=$(ps ax | grep -i 'ConnectStartup' | grep java | grep -v grep | awk '{print $1}') 18 | if [ -z "$pid" ]; then 19 | echo "No connect running." 20 | exit -1 21 | fi 22 | 23 | echo "The connect(${pid}) is running..." 24 | 25 | kill ${pid} 26 | 27 | echo "Send shutdown request to connect(${pid}) OK" 28 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-rabbitmq/src/test/java/org/apache/rocketmq/connect/jms/RabbitmqConfigTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.jms; 19 | 20 | import org.apache.rocketmq.connect.rabbitmq.RabbitmqConfig; 21 | 22 | public class RabbitmqConfigTest { 23 | 24 | RabbitmqConfig config; 25 | 26 | 27 | 28 | } 29 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/common/HeaderField.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.jdbc.common; 19 | 20 | /** 21 | * header field 22 | */ 23 | public interface HeaderField { 24 | String SOURCE_TABLE_KEY = "__source_table"; 25 | String SOURCE_DB_KEY = "__source_db"; 26 | } 27 | -------------------------------------------------------------------------------- /rocketmq-connect-cli/src/main/java/org/apache/rocketmq/connect/cli/commom/CLIConfigDefine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.cli.commom; 19 | 20 | /** 21 | * Define all the logger name of the runtime. 22 | */ 23 | public class CLIConfigDefine { 24 | public static final String PROTOCOL = "http"; 25 | } 26 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/connectorwrapper/WorkerTaskState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.runtime.connectorwrapper; 18 | 19 | public enum WorkerTaskState { 20 | NEW, 21 | PENDING, 22 | RUNNING, 23 | ERROR, 24 | STOPPING, 25 | STOPPED, 26 | TERMINATED, 27 | } 28 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-connectors-archetype/src/main/resources/archetype-resources/src/main/java/helper/__dbNameToCamel__Record.java: -------------------------------------------------------------------------------- 1 | #set( $symbol_pound = '#' ) 2 | #set( $symbol_dollar = '$' ) 3 | #set( $symbol_escape = '\' ) 4 | /* 5 | * Licensed to the Apache Software Foundation (ASF) under one or more 6 | * contributor license agreements. See the NOTICE file distributed with 7 | * this work for additional information regarding copyright ownership. 8 | * The ASF licenses this file to You under the Apache License, Version 2.0 9 | * (the "License"); you may not use this file except in compliance with 10 | * the License. You may obtain a copy of the License at 11 | * 12 | * http://www.apache.org/licenses/LICENSE-2.0 13 | * 14 | * Unless required by applicable law or agreed to in writing, software 15 | * distributed under the License is distributed on an "AS IS" BASIS, 16 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | * See the License for the specific language governing permissions and 18 | * limitations under the License. 19 | */ 20 | 21 | package ${package}.helper; 22 | 23 | public class ${dbNameToCamel}Record { 24 | // FIXME: Write your code here 25 | } 26 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/config/ConsumeFromWhere.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.config; 18 | 19 | /** 20 | * @author osgoo 21 | * @date 2022/6/21 22 | */ 23 | public enum ConsumeFromWhere { 24 | CONSUME_FROM_FIRST_OFFSET, 25 | CONSUME_FROM_LAST_OFFSET, 26 | CONSUME_FROM_TIMESTAMP 27 | } 28 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ByteArraySerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.runtime.serialization; 18 | 19 | public class ByteArraySerializer implements Serializer { 20 | @Override 21 | public byte[] serialize(String topic, byte[] data) { 22 | return data; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/test/java/org/apache/rocketmq/redis/test/connector/RedisSourceConnectorTest.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.redis.test.connector; 2 | 3 | import java.util.List; 4 | 5 | import io.openmessaging.KeyValue; 6 | import io.openmessaging.internal.DefaultKeyValue; 7 | import org.apache.rocketmq.connect.redis.connector.RedisSourceConnector; 8 | import org.junit.Assert; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | 12 | public class RedisSourceConnectorTest { 13 | private KeyValue keyValue; 14 | 15 | @Before 16 | public void initKeyValue(){ 17 | this.keyValue = new DefaultKeyValue(); 18 | this.keyValue.put("redisAddr", "127.0.0.1"); 19 | this.keyValue.put("redisPort", "6379"); 20 | this.keyValue.put("redisPassword", ""); 21 | } 22 | @Test 23 | public void testConnector(){ 24 | RedisSourceConnector connector = new RedisSourceConnector(); 25 | Class cl = connector.taskClass(); 26 | Assert.assertNotNull(cl); 27 | List keyValues = connector.taskConfigs(2); 28 | Assert.assertNotNull(keyValues); 29 | connector.start(keyValue); 30 | connector.stop(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-activemq/README-CN.md: -------------------------------------------------------------------------------- 1 | ##### ActiveConnector完全限定名 2 | org.apache.rocketmq.connect.activemq.connector.ActivemqConnector 3 | 4 | **activemq-source-connector** 启动 5 | 6 | ``` 7 | POST http://${runtime-ip}:${runtime-port}/connectors/activeSourceConnector 8 | { 9 | "connector.class":"org.apache.rocketmq.connect.activemq.connector.ActivemqSourceConnector", 10 | "max-task":"2", 11 | "activemqUrl":"tcp://localhost:61616", 12 | "destinationType":"queue", 13 | "destinationName":"testQueue", 14 | "connect.topicname":"targetTopic", 15 | "value-converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter", 16 | "key-converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter" 17 | } 18 | ``` 19 | 20 | ##### 配置参数 21 | 22 | 参数 | 作用 | 是否必填 | 默认值 23 | ---|--- |--- | --- 24 | activemqUrl | activemq ip与端口号 | 是 | 无 25 | activemqUsername | 用户名 | 否 | 无 26 | activemqPassword| 密码 | 否 | 无 27 | destinationName | 读取的队列或者主题名 | 是 | 无 28 | destinationType | 读取的类型：queue(队列)或者topic(主题) | 是 | 无 29 | messageSelector | 过滤器 | 否 |无 30 | sessionAcknowledgeMode | 消息确认 | 否 | Session.AUTO_ACKNOWLEDGE 31 | sessionTransacted | 是否是事务会话 | 否 | false 32 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/model/LoadOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.model; 21 | 22 | public enum LoadOperation { 23 | SEND_REQUEST_TO_DORIS, 24 | 25 | COMMIT_TRANSACTION, 26 | 27 | GET_UPLOAD_ADDRESS, 28 | 29 | UPLOAD_FILE, 30 | 31 | EXECUTE_COPY; 32 | } 33 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ByteArrayDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.runtime.serialization; 18 | 19 | public class ByteArrayDeserializer implements Deserializer { 20 | 21 | @Override 22 | public byte[] deserialize(String topic, byte[] data) { 23 | return data; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/standalone/StandaloneConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.runtime.controller.standalone; 18 | 19 | import org.apache.rocketmq.connect.runtime.config.WorkerConfig; 20 | 21 | /** 22 | * worker standalone config 23 | */ 24 | public class StandaloneConfig extends WorkerConfig { 25 | } 26 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/connect/AbstractConnectSchemaType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.converter.type.connect; 21 | 22 | import org.apache.rocketmq.connect.doris.converter.type.AbstractType; 23 | 24 | public abstract class AbstractConnectSchemaType extends AbstractType { 25 | } 26 | -------------------------------------------------------------------------------- /distribution/conf/metrics.conf: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # metrics reporter class 18 | metrics.reporter=org.apache.rocketmq.connect.metrics.reporter.RocketMQScheduledReporter 19 | 20 | # metrics topic 21 | metrics.topic=metrics-topic 22 | group.id=metrics-gid 23 | # Rocketmq namesrvAddr 24 | name.srv.addr=localhost:9876 25 | 26 | # RocketMQ acl 27 | acl.enable=false 28 | access.key=rocketmq 29 | secret.key=12345678 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/AutoConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.metrics; 18 | 19 | import java.util.Map; 20 | 21 | /** 22 | * auto configuration 23 | */ 24 | public interface AutoConfiguration { 25 | /** 26 | * configs 27 | * 28 | * @param configs 29 | */ 30 | void config(Map configs); 31 | } 32 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/distributed/DistributedConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.controller.distributed; 19 | 20 | import org.apache.rocketmq.connect.runtime.config.WorkerConfig; 21 | 22 | /** 23 | * distributed worker config 24 | */ 25 | public class DistributedConfig extends WorkerConfig { 26 | } 27 | -------------------------------------------------------------------------------- /docker/connect/README.md: -------------------------------------------------------------------------------- 1 | #connect docker start 2 | ##准备 3 | 4 | ###环境 5 | 1. Linux/Unix/Mac 6 | 2. 64bit JDK 1.8+; 7 | 3. Maven 3.2.x或以上版本; 8 | 4. 安装docker 9 | 10 | ###启动RocketMQ 11 | 12 | RocketMQ镜像下载 13 | 14 | https://hub.docker.com/r/apache/rocketmq/tags 15 | 16 | RocketMQ镜像启动 17 | 18 | RocketMQ镜像 quick start 19 | 20 | ###构建Connector镜像 21 | ``` 22 | sh build_image.sh 23 | ``` 24 | 25 | ##启动Connector 26 | 27 | 使用默认connect-distributed.conf启动 28 | 29 | ``` 30 | docker run --name rmqconnect --link rmqnamesrv:namesrv -e "NAMESRV_ADDR=namesrv:9876" -p 8082:8082 apache/rocketmqconnect:0.0.1-SNAPSHOT sh /home/connect/mq-connect/bin/connect-distributed.sh -c /home/connect/mq-connect/conf/connect-distributed.conf 31 | ``` 32 | 33 | 使用自定义配置启动 34 | 35 | 替换自定义connect-distributed.conf配置 36 | 37 | 可以将 ~/rocketmq-connect/distribution/conf/connect-distributed.conf 替换为自定义的配置 38 | 39 | ``` 40 | docker run -d -v ~/rocketmq-connect/distribution/conf/connect-distributed.conf:/home/connect/mq-connect/conf/connect-distributed.conf --name rmqconnect --link rmqnamesrv:namesrv -e "NAMESRV_ADDR=namesrv:9876" -p 8082:8082 apache/rocketmqconnect:0.0.1-SNAPSHOT sh /home/connect/mq-connect/bin/connect-distributed.sh -c /home/connect/mq-connect/conf/connect-distributed.conf 41 | ``` -------------------------------------------------------------------------------- /connectors/rocketmq-connect-iotdb/src/main/java/org/apache/rocketmq/connect/iotdb/exception/IotdbRuntimeException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.iotdb.exception; 19 | 20 | import io.openmessaging.connector.api.errors.ConnectException; 21 | 22 | public class IotdbRuntimeException extends ConnectException { 23 | 24 | public IotdbRuntimeException(String msg) { 25 | super(msg); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/dialect/DatabaseDialectFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.jdbc.dialect; 18 | 19 | import java.util.Set; 20 | import org.apache.rocketmq.connect.jdbc.config.AbstractConfig; 21 | 22 | public interface DatabaseDialectFactory { 23 | 24 | Set subProtocols(); 25 | 26 | DatabaseDialect create(AbstractConfig config); 27 | } -------------------------------------------------------------------------------- /connectors/rocketmq-connect-kafka/src/main/resources/connect-kafka-source.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | name=rocketmq-connect-kafka 17 | connector-class=org.apache.rocketmq.connect.kafka.connector.KafkaSourceConnector 18 | source-record-converter=io.openmessaging.connect.runtime.converter.JsonConverter 19 | task.num=2 20 | kafka.bootstrap.server=47.112.213.204:9092;47.112.213.204:9092 21 | kafka.topic=jonnxu 22 | kafka.group.id=connect-kafka-source-consumer-group 23 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/errors/Operation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.runtime.errors; 18 | 19 | import java.util.concurrent.Callable; 20 | 21 | /** 22 | * A recoverable operation evaluated in the connector pipeline. 23 | * 24 | * @param return type of the result of the operation. 25 | */ 26 | public interface Operation extends Callable { 27 | 28 | } 29 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ListSerde.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.serialization; 19 | 20 | import java.util.List; 21 | 22 | /** 23 | * list serde 24 | */ 25 | public class ListSerde extends WrapperSerde { 26 | public ListSerde(Class clazz) { 27 | super(new ListSerializer(clazz), new ListDeserializer(clazz)); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/resources/metrics.conf: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | # metrics reporter class 18 | metrics.reporter=org.apache.rocketmq.connect.metrics.reporter.RocketMQScheduledReporter 19 | 20 | # metrics topic 21 | metrics.topic=metrics-topic 22 | # Rocketmq namesrvAddr 23 | name.srv.addr=localhost:9876 24 | 25 | group.id=metrics-topic-group 26 | 27 | # RocketMQ acl 28 | acl.enable=false 29 | access.key=rocketmq 30 | secret.key=12345678 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-deltalake/src/main/java/org/apache/rocketmq/connect/deltalake/exception/WriteParquetException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | package org.apache.rocketmq.connect.deltalake.exception; 20 | 21 | /** 22 | * @author osgoo 23 | * @date 2022/8/24 24 | */ 25 | public class WriteParquetException extends Exception { 26 | public WriteParquetException(String msg, Exception e) { 27 | super(msg, e); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/schema/FieldName.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.schema; 18 | 19 | public enum FieldName { 20 | COMMON_MESSAGE("MessageExt"), 21 | OFFSET("Offset"); 22 | 23 | private String key; 24 | 25 | FieldName(String key) { 26 | this.key = key; 27 | } 28 | 29 | public String getKey() { 30 | return key; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/controller/ConnectController.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.controller; 19 | 20 | /** 21 | * connect controller 22 | */ 23 | public interface ConnectController { 24 | 25 | /** 26 | * start controller 27 | */ 28 | void start(); 29 | 30 | /** 31 | * shutdown controller 32 | */ 33 | void shutdown(); 34 | 35 | } 36 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-debezium/rocketmq-connect-debezium-core/src/main/java/org/apache/rocketmq/connect/debezium/DebeziumConnector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.debezium; 19 | 20 | import org.apache.rocketmq.connect.kafka.connect.adaptor.connector.KafkaSourceAdaptorConnector; 21 | 22 | 23 | /** 24 | * debezium connector 25 | */ 26 | public abstract class DebeziumConnector extends KafkaSourceAdaptorConnector { 27 | 28 | } 29 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/converter/KVEntryConverter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.converter; 19 | 20 | import io.openmessaging.connector.api.data.ConnectRecord; 21 | import java.util.List; 22 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 23 | 24 | public interface KVEntryConverter { 25 | 26 | List kVEntryToConnectRecord(KVEntry kvEntry); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-hive/README.md: -------------------------------------------------------------------------------- 1 | ##### HiveSourceConnector fully-qualified name 2 | org.apache.rocketmq.connect.hive.connector.HiveSourceConnector 3 | 4 | **hive-source-connector** start 5 | 6 | ``` 7 | POST http://${runtime-ip}:${runtime-port}/connectors/hiveSourceConnector 8 | { 9 | "connector.class":"org.apache.rocketmq.connect.hive.connector.HiveSourceConnector", 10 | "host":"localhost", 11 | "port":10000, 12 | "database":"default", 13 | "tables":{ 14 | "invites": { 15 | "foo":1 16 | } 17 | }, 18 | "max.tasks":2, 19 | "connect.topicname":"hiveTopic", 20 | "value.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter", 21 | "key.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter" 22 | } 23 | ``` 24 | 25 | ##### parameter configuration 26 | 27 | parameter | effect | required |default 28 | ---|-----------------------------|----------| --- 29 | host | The Host of the hive server | yes | null 30 | port | The Port of the hive server | yes | null 31 | tables| The info of the table | yes | null 32 | database | The info of the database | yes | null 33 | username | The info of the hive server | no | null 34 | password | The info of the hive server | no | null -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/JsonSerde.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.serialization; 19 | 20 | public class JsonSerde extends WrapperSerde { 21 | public JsonSerde() { 22 | super(new JsonSerializer(), new JsonDeserializer()); 23 | } 24 | 25 | public JsonSerde(Class aClass) { 26 | super(new JsonSerializer(), new JsonDeserializer(aClass)); 27 | } 28 | 29 | } -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/common/LoggerName.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.replicator.common; 19 | 20 | /** 21 | * Define all the logger name of the runtime. 22 | */ 23 | public class LoggerName { 24 | public static final String CONNECT_BUG = "ConnectBug"; 25 | public static final String REPLICATRO_RUNTIME = "ReplicatorRuntime"; 26 | public static final String WORKER_ERROR_MSG_ID = "WorkerErrorMsgId"; 27 | } 28 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/exception/ParamInvalidException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.exception; 18 | 19 | /** 20 | * @author osgoo 21 | * @date 2022/7/25 22 | */ 23 | public class ParamInvalidException extends RuntimeException { 24 | public ParamInvalidException(String msg) { 25 | super(msg); 26 | } 27 | public ParamInvalidException(String msg, Exception e) { 28 | super(msg, e); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-debezium/kafka-connect-adaptor/src/main/java/org/apache/rocketmq/connect/kafka/connect/adaptor/context/RocketMQKafkaErrantRecordReporter.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.kafka.connect.adaptor.context; 2 | 3 | import io.openmessaging.connector.api.component.task.sink.ErrorRecordReporter; 4 | import io.openmessaging.connector.api.component.task.sink.SinkTaskContext; 5 | import io.openmessaging.connector.api.data.ConnectRecord; 6 | import org.apache.kafka.connect.sink.ErrantRecordReporter; 7 | import org.apache.kafka.connect.sink.SinkRecord; 8 | import org.apache.rocketmq.connect.kafka.connect.adaptor.schema.Converters; 9 | 10 | import java.util.concurrent.Future; 11 | 12 | /** 13 | * rocketmq kafka error record reporter 14 | */ 15 | public class RocketMQKafkaErrantRecordReporter implements ErrantRecordReporter { 16 | final ErrorRecordReporter errorRecordReporter; 17 | 18 | RocketMQKafkaErrantRecordReporter(SinkTaskContext context) { 19 | errorRecordReporter = context.errorRecordReporter(); 20 | } 21 | 22 | @Override 23 | public Future report(SinkRecord sinkRecord, Throwable throwable) { 24 | // 数据转换 25 | ConnectRecord record = Converters.fromSinkRecord(sinkRecord); 26 | errorRecordReporter.report(record, throwable); 27 | return null; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/binder/JdbcRecordBinder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.jdbc.binder; 19 | 20 | import io.openmessaging.connector.api.data.ConnectRecord; 21 | import java.sql.SQLException; 22 | 23 | public interface JdbcRecordBinder { 24 | /** 25 | * bind record 26 | * 27 | * @param record 28 | * @throws SQLException 29 | */ 30 | void bindRecord(ConnectRecord record) throws SQLException; 31 | } -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/README-CN.md: -------------------------------------------------------------------------------- 1 | # RocketMQ-connect-redis 2 | ##### RedisSourceConnector 完全限定名 3 | org.apache.rocketmq.connect.redis.connector.RedisSourceConnector 4 | 5 | * **redis-source-connector** 启动 6 | 7 | ``` 8 | POST http://${runtime-ip}:${runtime-port}/connectors/redisSourceConnector 9 | { 10 | "connector.class":"org.apache.rocketmq.connect.redis.connector.RedisSourceConnector", 11 | "max.tasks":"3", 12 | "redisAddr":"localhost", 13 | "redisPort":"6379", 14 | "connect.topicname":"targetTopic", 15 | "value.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter", 16 | "key.converter":"org.apache.rocketmq.connect.runtime.converter.record.json.JsonConverter" 17 | } 18 | ``` 19 | 20 | ##### 配置参数 21 | 22 | 参数 | 作用 | 是否必填 | 默认值 23 | ---|--- |--- | --- 24 | redisAddr | Redis服务IP地址。 | 是 | null 25 | redisPort | Redis服务端口。 | 是 | null 26 | redisPassword | Redis auth 密码。 | 是 | null 27 | timeout | 连接Redis超时时间，单位毫秒。 | 否 | 3000 28 | syncMod | 数据同步模式。 | 否 | CUSTOM_OFFSET 29 | offset | Redis位点信息。 | 否 | -1 30 | replId | Redis master_replId，可用redis-cli通过info命令获取。 | 否 | null 31 | commands | 需要监听的Redis操作命令。 | 否 | * 32 | eventCommitRetryTimes | 收到Redis event后提交到RedisEventProcessor的失败重试次数。 | 否 | 5 33 | eventCommitRetryInterval | 收到Redis event后提交到RedisEventProcessor的失败重试的时间间隔，单位毫秒。 | 否 | 100 34 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-connectors-archetype/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | org.apache.rocketmq 6 | rocketmq-connect-connectors-archetype 7 | 1.0-SNAPSHOT 8 | maven-archetype 9 | 10 | rocketmq-connect-connectors-archetype 11 | 12 | 13 | 14 | 15 | org.apache.maven.archetype 16 | archetype-packaging 17 | 3.2.1 18 | 19 | 20 | 21 | 22 | 23 | 24 | maven-archetype-plugin 25 | 3.2.1 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | The Apache Software License, Version 2.0 34 | http://www.apache.org/licenses/LICENSE-2.0.txt 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /.asf.yaml: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | github: 17 | description: "A tool for scalable and reliably streaming data between Apache RocketMQ and other systems." 18 | features: 19 | # Enable issue management 20 | issues: true 21 | # Enable wiki 22 | wiki: true 23 | 24 | protected_branches: 25 | 26 | notifications: 27 | commits: commits@rocketmq.apache.org 28 | issues: commits@rocketmq.apache.org 29 | pullrequests: commits@rocketmq.apache.org 30 | jobs: commits@rocketmq.apache.org 31 | discussions: dev@rocketmq.apache.org 32 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/writer/LoadStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.writer; 21 | 22 | /** 23 | * enum of LoadStatus. 24 | */ 25 | public class LoadStatus { 26 | public static final String SUCCESS = "Success"; 27 | public static final String PUBLISH_TIMEOUT = "Publish Timeout"; 28 | public static final String LABEL_ALREADY_EXIST = "Label Already Exists"; 29 | public static final String FAIL = "Fail"; 30 | } 31 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-http/src/main/java/org/apache/rocketmq/connect/http/HttpConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.http; 18 | 19 | import org.apache.rocketmq.connect.http.constant.HttpConstant; 20 | 21 | import java.util.HashSet; 22 | import java.util.Set; 23 | 24 | public class HttpConfig { 25 | public static final Set REQUEST_CONFIG = new HashSet() { 26 | { 27 | add(HttpConstant.URL); 28 | add(HttpConstant.METHOD); 29 | } 30 | }; 31 | } 32 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ByteBufferDeserializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.runtime.serialization; 18 | 19 | import java.nio.ByteBuffer; 20 | 21 | public class ByteBufferDeserializer implements Deserializer { 22 | @Override 23 | public ByteBuffer deserialize(String topic, byte[] data) { 24 | if (data == null) { 25 | return null; 26 | } 27 | 28 | return ByteBuffer.wrap(data); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/service/DorisSinkServiceFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.service; 21 | 22 | import io.openmessaging.KeyValue; 23 | 24 | /** 25 | * A factory to create {@link DorisSinkService} 26 | */ 27 | public class DorisSinkServiceFactory { 28 | 29 | public static DorisSinkService getDorisSinkService(KeyValue connectorConfig) { 30 | return new DorisDefaultSinkService(connectorConfig); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/ShortSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.runtime.serialization; 18 | 19 | public class ShortSerializer implements Serializer { 20 | @Override 21 | public byte[] serialize(String topic, Short data) { 22 | if (data == null) { 23 | return null; 24 | } 25 | 26 | return new byte[] { 27 | (byte) (data >>> 8), 28 | data.byteValue() 29 | }; 30 | } 31 | } -------------------------------------------------------------------------------- /schema-converter/schema-converter-common/src/main/java/org/apache/rocketmq/schema/common/TopicNameStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.schema.common; 18 | 19 | /** 20 | * subject 21 | */ 22 | public class TopicNameStrategy { 23 | 24 | /** 25 | * generate subject name 26 | * 27 | * @param topic 28 | * @param isKey 29 | * @return 30 | */ 31 | public static String subjectName(String topic, boolean isKey) { 32 | return isKey ? topic + "-key" : topic + "-value"; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-mqtt/src/main/java/org/apache/rocketmq/connect/mqtt/util/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.mqtt.util; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | public class Utils { 24 | private static final Logger log = LoggerFactory.getLogger(Utils.class); 25 | 26 | public static String createClientId(String prefix) { 27 | return new StringBuilder().append(prefix).append("-").append(System.currentTimeMillis()).toString(); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/common/ConstDefine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.common; 18 | 19 | public class ConstDefine { 20 | 21 | public static String TASK_GROUP_NAME_PREFIX = "REPLICATOR-TASK"; 22 | 23 | public static String REPLICATOR_ADMIN_PREFIX = "REPLICATOR-ADMIN"; 24 | 25 | public static String REPLICATOR_ADMIN_GROUP = "REPLICATOR-ADMIN-GROUP"; 26 | 27 | public static String REPLICATOR_TASK_ADMIN_GROUP = "REPLICATOR-TASK-ADMIN-GROUP"; 28 | 29 | } 30 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/src/test/java/org/apache/rocketmq/connect/cassandra/common/TestClone.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.cassandra.common; 19 | 20 | import java.io.Serializable; 21 | 22 | public class TestClone implements Serializable { 23 | 24 | private static final long serialVersionUID = -5349483632695152550L; 25 | 26 | private long id; 27 | 28 | public long getId() { 29 | return id; 30 | } 31 | 32 | public void setId(long id) { 33 | this.id = id; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractGeometryType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.converter.type; 21 | 22 | import io.openmessaging.connector.api.data.Schema; 23 | import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType; 24 | 25 | public abstract class AbstractGeometryType extends AbstractType { 26 | @Override 27 | public String getTypeName(Schema schema) { 28 | return DorisType.STRING; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-kafka-connector-adapter/src/main/java/org/apache/rocketmq/connect/kafka/connector/KafkaRocketmqSinkConnector.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.kafka.connector; 2 | 3 | 4 | import io.openmessaging.KeyValue; 5 | import io.openmessaging.connector.api.component.task.Task; 6 | import io.openmessaging.connector.api.component.task.sink.SinkConnector; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.util.List; 11 | 12 | public class KafkaRocketmqSinkConnector extends SinkConnector { 13 | 14 | private static final Logger log = LoggerFactory.getLogger(KafkaRocketmqSinkConnector.class); 15 | 16 | private KafkaRocketmqConnector parentConnector = new KafkaRocketmqConnector(this); 17 | 18 | @Override 19 | public List taskConfigs(int maxTasks) { 20 | return parentConnector.taskConfigs(maxTasks); 21 | } 22 | 23 | @Override 24 | public Class taskClass() { 25 | return parentConnector.taskClass(); 26 | } 27 | 28 | @Override 29 | public void start(KeyValue config) { 30 | parentConnector.start(config); 31 | } 32 | 33 | @Override 34 | public void stop() { 35 | parentConnector.stop(); 36 | } 37 | 38 | @Override 39 | public void validate(KeyValue config) { 40 | parentConnector.validate(config); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/schema/column/BooleanColumnParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | 19 | package org.apache.rocketmq.connect.cassandra.schema.column; 20 | 21 | public class BooleanColumnParser extends ColumnParser { 22 | @Override 23 | public Object getValue(Object value) { 24 | if (value == null) { 25 | return null; 26 | } 27 | 28 | if (value instanceof Boolean) { 29 | return value; 30 | } 31 | 32 | return value; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/src/test/java/org/apache/rocketmq/connect/cassandra/common/CloneUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.cassandra.common; 19 | 20 | import org.junit.Assert; 21 | import org.junit.Test; 22 | 23 | public class CloneUtilsTest { 24 | 25 | @Test 26 | public void cloneTest() { 27 | TestClone testClone = new TestClone(); 28 | testClone.setId(1); 29 | final TestClone clone = CloneUtils.clone(testClone); 30 | Assert.assertEquals(1, clone.getId()); 31 | 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /metric-exporter/src/main/java/org/apache/rocketmq/connect/metrics/stats/AbstractHistogram.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.metrics.stats; 19 | 20 | import org.apache.rocketmq.connect.metrics.MetricName; 21 | 22 | /** 23 | * Abstract Histogram 24 | */ 25 | public abstract class AbstractHistogram implements Stat { 26 | public AbstractHistogram(MetricName name) { 27 | name.setType(type()); 28 | } 29 | 30 | public AbstractHistogram(MetricName name, HistogramType type) { 31 | name.setType(type.name()); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/common/ConstDefine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.cassandra.common; 18 | 19 | public class ConstDefine { 20 | 21 | public static String CASSANDRA_CONNECTOR_ADMIN_PREFIX = "CASSANDRA-CONNECTOR-ADMIN"; 22 | 23 | public static final String PREFIX = "cassandra"; 24 | 25 | public static final String DATABASE_NAME = "database"; 26 | 27 | public static final String TABLE = "table"; 28 | 29 | public static final String INCREASE = "increase"; 30 | } 31 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-kafka-connector-adapter/src/main/java/org/apache/rocketmq/connect/kafka/connector/KafkaRocketmqSourceConnector.java: -------------------------------------------------------------------------------- 1 | package org.apache.rocketmq.connect.kafka.connector; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import io.openmessaging.KeyValue; 6 | import io.openmessaging.connector.api.component.task.Task; 7 | import io.openmessaging.connector.api.component.task.source.SourceConnector; 8 | 9 | import java.util.List; 10 | 11 | 12 | public class KafkaRocketmqSourceConnector extends SourceConnector { 13 | 14 | private static final Logger log = LoggerFactory.getLogger(KafkaRocketmqSourceConnector.class); 15 | 16 | private KafkaRocketmqConnector parentConnector = new KafkaRocketmqConnector(this); 17 | 18 | @Override 19 | public List taskConfigs(int maxTasks) { 20 | return parentConnector.taskConfigs(maxTasks); 21 | } 22 | 23 | @Override 24 | public Class taskClass() { 25 | return parentConnector.taskClass(); 26 | } 27 | 28 | @Override 29 | public void start(KeyValue config) { 30 | parentConnector.start(config); 31 | } 32 | 33 | @Override 34 | public void stop() { 35 | parentConnector.stop(); 36 | } 37 | 38 | @Override 39 | public void validate(KeyValue config) { 40 | parentConnector.validate(config); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/exception/StartTaskException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.exception; 18 | 19 | /** 20 | * @author osgoo 21 | * @date 2022/7/25 22 | */ 23 | public class StartTaskException extends RuntimeException { 24 | public StartTaskException(String msg) { 25 | super(msg); 26 | } 27 | public StartTaskException(Exception e) { 28 | super(e); 29 | } 30 | public StartTaskException(String msg, Exception e) { 31 | super(msg, e); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/connection/ConnectionProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.connection; 21 | 22 | import java.sql.Connection; 23 | 24 | public interface ConnectionProvider { 25 | 26 | /** 27 | * Get existing connection or establish an new one if there is none. 28 | */ 29 | Connection getOrEstablishConnection() throws Exception; 30 | 31 | /** 32 | * Close possible existing connection. 33 | */ 34 | void closeConnection(); 35 | } 36 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/writer/load/DataLoad.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.writer.load; 21 | 22 | import com.fasterxml.jackson.databind.ObjectMapper; 23 | import java.io.Serializable; 24 | 25 | public abstract class DataLoad implements Serializable { 26 | protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); 27 | protected String database; 28 | protected String table; 29 | protected String user; 30 | protected String password; 31 | } 32 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/errors/ToleranceType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.errors; 19 | 20 | import java.util.Locale; 21 | 22 | /** 23 | * The different levels of error tolerance. 24 | */ 25 | public enum ToleranceType { 26 | 27 | /** 28 | * Tolerate no errors. 29 | */ 30 | NONE, 31 | 32 | /** 33 | * Tolerate all errors. 34 | */ 35 | ALL; 36 | 37 | public String value() { 38 | return name().toLowerCase(Locale.ROOT); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/Callback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.runtime.utils; 18 | 19 | /** 20 | * Generic interface for callbacks 21 | */ 22 | public interface Callback { 23 | /** 24 | * Invoked upon completion of the operation. 25 | * 26 | * @param error the error that caused the operation to fail, or null if no error occurred 27 | * @param result the return value, or null if the operation failed 28 | */ 29 | void onCompletion(Throwable error, V result); 30 | } 31 | -------------------------------------------------------------------------------- /rocketmq-connect-sample/src/main/java/org/apache/rocketmq/connect/file/FileConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.file; 18 | 19 | public class FileConstants { 20 | 21 | public static final String FILENAME_FIELD = "filename"; 22 | 23 | public static final String NEXT_POSITION = "nextPosition"; 24 | 25 | public static final String FILE_LINE_CONTENT = "fileLineContent"; 26 | 27 | public static final String LINE = "_line"; 28 | 29 | public static String getPartition(String filename) { 30 | return filename; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/exception/GetMetaDataException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.exception; 18 | 19 | /** 20 | * @author osgoo 21 | * @date 2022/7/25 22 | */ 23 | public class GetMetaDataException extends RuntimeException { 24 | public GetMetaDataException(String msg) { 25 | super(msg); 26 | } 27 | public GetMetaDataException(Exception e) { 28 | super(e); 29 | } 30 | public GetMetaDataException(String msg, Exception e) { 31 | super(msg, e); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/utils/datasync/DataSynchronizerCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.utils.datasync; 19 | 20 | /** 21 | * Callback for {@link DataSynchronizer} 22 | * 23 | * @param 24 | * @param 25 | */ 26 | public interface DataSynchronizerCallback { 27 | 28 | /** 29 | * Receive a message. 30 | * 31 | * @param error 32 | * @param key 33 | * @param result 34 | */ 35 | void onCompletion(Throwable error, K key, V result); 36 | } 37 | -------------------------------------------------------------------------------- /schema-converter/schema-converter-common/src/main/java/org/apache/rocketmq/schema/common/Serializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.schema.common; 18 | 19 | import java.util.Map; 20 | 21 | public interface Serializer { 22 | 23 | default void configure(Map props) { 24 | } 25 | 26 | /** 27 | * serialize data 28 | * 29 | * @param topic 30 | * @param isKey 31 | * @param schema 32 | * @param value 33 | * @return 34 | */ 35 | byte[] serialize(String topic, boolean isKey, T schema, Object value); 36 | } 37 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-deltalake/src/main/java/org/apache/rocketmq/connect/deltalake/writer/DeltalakeWriter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.deltalake.writer; 19 | 20 | import io.openmessaging.connector.api.data.ConnectRecord; 21 | import org.apache.rocketmq.connect.deltalake.exception.WriteParquetException; 22 | 23 | import java.util.Collection; 24 | 25 | /** 26 | * @author osgoo 27 | * @date 2022/8/23 28 | */ 29 | public interface DeltalakeWriter { 30 | void writeEntries(Collection entries) throws WriteParquetException; 31 | } 32 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/util/BytesUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.jdbc.util; 18 | 19 | public class BytesUtil { 20 | private static final char[] HEX_CODE = "0123456789ABCDEF".toCharArray(); 21 | public static String toHex(byte[] data) { 22 | StringBuilder r = new StringBuilder(data.length * 2); 23 | for (byte b : data) { 24 | r.append(HEX_CODE[(b >> 4) & 0xF]); 25 | r.append(HEX_CODE[b & 0xF]); 26 | } 27 | return r.toString(); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/ExecParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * exec 26 | */ 27 | public class ExecParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.BOOLEAN); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/ScriptParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * 26 | */ 27 | public class ScriptParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.STRING); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/exception/InitMQClientException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.exception; 18 | 19 | /** 20 | * @author osgoo 21 | * @date 2022/7/25 22 | */ 23 | public class InitMQClientException extends RuntimeException { 24 | public InitMQClientException(String msg) { 25 | super(msg); 26 | } 27 | public InitMQClientException(Exception e) { 28 | super(e); 29 | } 30 | public InitMQClientException(String msg, Exception e) { 31 | super(msg, e); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/DecrParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * DECR key 26 | */ 27 | public class DecrParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.STRING); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/IncrParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * INCR key 26 | */ 27 | public class IncrParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.STRING); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/LPopParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * LPOP key 26 | */ 27 | public class LPopParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.STRING); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/MultiParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * Multi 26 | */ 27 | public class MultiParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.STRING); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/RPopParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * RPOP key 26 | */ 27 | public class RPopParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.STRING); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-sftp/doc/RocketMQ Connect SFTP 分享.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | - [connect **介绍**](#connect-介绍) 4 | - [SFTP 介绍](#sftp-介绍) 5 | - [connect-sftp 使用场景](#connect-sftp-使用场景) 6 | - [使用 sftp-source-connector 同步对账文件](#使用-sftp-source-connector-同步对账文件) 7 | - [使用 sftp-sink-connector 生成对账文件](#使用-sftp-sink-connector-生成对账文件) 8 | - [使用 sftp-sink-connector 和 sftp-source-connector 同步 sftp 文件](#使用-sftp-sink-connector-和-sftp-source-connector-同步-sftp-文件) 9 | - [connect-sftp 演示](#connect-sftp-演示) 10 | - [参考](#参考) 11 | 12 | 13 | 14 | ## connect **介绍** 15 | 16 | * 基于 openMessage connect 标准之上。实现了对 Connector 的管理（Rest API、Shell Command） 17 | * 实现了对 task 的任务调度 18 | * 通过插件机制和继承的扩展方式，使得扩展 connect 支持各种不同的应用协议变得比较轻量和容易。 19 | 20 | ## SFTP 介绍 21 | 22 | 全称 SSH File Transfer Protocol，运行在 SSH 协议之上（与 FTP 没有任何共同点），用于文件的传输。 23 | 24 | ## connect-sftp 使用场景 25 | 26 | ### 使用 sftp-source-connector 同步对账文件 27 | 28 | 使用 sftp-source-connector 同步对账文件到 MQ，业务系统对接MQ，消费相应主题的消息进行业务处理。 29 | ![sftp connect 使用场景](使用场景.png) 30 | 31 | 优势： 32 | 33 | * 借助MQ，业务系统作为消费者可以轻松实现负载均衡 34 | * 业务系统不必关心数据的读取和转换过程。该过程的可靠性由 connect 保证。 35 | 36 | 劣势： 37 | 38 | * 需要独立部署一个 connect 服务，并做相应的维护和配置。 39 | 40 | ### 使用 sftp-sink-connector 生成对账文件 41 | 42 | ### 使用 sftp-sink-connector 和 sftp-source-connector 同步 sftp 文件 43 | 44 | ## connect-sftp 演示 45 | 46 | [RocketMQ Connect SFTP 实战.md](RocketMQ Connect SFTP 实战.md) 47 | 48 | ## 参考 49 | 50 | [sftp.net](https://www.sftp.net/) -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/doris/DorisTypeProperties.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.converter.type.doris; 21 | 22 | public class DorisTypeProperties { 23 | 24 | /* Max precision of datetime type of Doris. */ 25 | public static final int MAX_SUPPORTED_DATE_TIME_PRECISION = 6; 26 | 27 | public static final int TIMESTAMP_TYPE_MAX_PRECISION = 9; 28 | 29 | public static final int MAX_VARCHAR_SIZE = 65533; 30 | 31 | public static final int MAX_CHAR_SIZE = 255; 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-iotdb/src/main/java/org/apache/rocketmq/connect/iotdb/config/IotdbConstant.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.iotdb.config; 19 | 20 | public class IotdbConstant { 21 | 22 | private IotdbConstant() {} 23 | 24 | public static final String IOTDB_HOST = "iotdbHost"; 25 | 26 | public static final String IOTDB_PORT = "iotdbPort"; 27 | 28 | public static final String IOTDB_PATHS = "iotdbPaths"; 29 | 30 | public static final String IOTDB_PATH = "path"; 31 | 32 | public static final String IOTDB_PARTITION = "IOTDB_PARTITION"; 33 | } 34 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/PersistParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * PERSIST key 26 | */ 27 | public class PersistParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.STRING); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /distribution/conf/connect-standalone.conf: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | workerId=standalone-worker 17 | storePathRootDir=/tmp/standalone/storeRoot 18 | 19 | ## Http port for user to access REST API 20 | httpPort=8082 21 | 22 | # Rocketmq namesrvAddr 23 | namesrvAddr=localhost:9876 24 | 25 | # RocketMQ acl 26 | aclEnable=false 27 | accessKey=rocketmq 28 | secretKey=12345678 29 | 30 | autoCreateGroupEnable=false 31 | clusterName="DefaultCluster" 32 | 33 | # Source or sink connector jar file dir,The default value is rocketmq-connect-sample 34 | pluginPaths=rocketmq-connect-sample/target/rocketmq-connect-sample-0.0.1-SNAPSHOT.jar 35 | 36 | 37 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/strategy/TaskDivideStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.cassandra.strategy; 19 | 20 | import io.openmessaging.KeyValue; 21 | import org.apache.rocketmq.connect.cassandra.config.DbConnectorConfig; 22 | import org.apache.rocketmq.connect.cassandra.config.TaskDivideConfig; 23 | 24 | import java.util.List; 25 | 26 | public abstract class TaskDivideStrategy { 27 | public abstract List divide(DbConnectorConfig dbConnectorConfig, TaskDivideConfig tdc, KeyValue keyValue); 28 | } 29 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/FlushAllParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * FLUSHALL [ASYNC] 26 | */ 27 | public class FlushAllParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.STRING); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-redis/src/main/java/org/apache/rocketmq/connect/redis/parser/FlushDbParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.redis.parser; 19 | 20 | import io.openmessaging.connector.api.data.FieldType; 21 | import org.apache.rocketmq.connect.redis.pojo.KVEntry; 22 | import org.apache.rocketmq.connect.redis.pojo.RedisEntry; 23 | 24 | /** 25 | * FLUSHDB [async] 26 | */ 27 | public class FlushDbParser extends AbstractCommandParser { 28 | @Override 29 | public KVEntry createBuilder() { 30 | return RedisEntry.newEntry(FieldType.STRING); 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /connectors/rocketmq-replicator/src/main/java/org/apache/rocketmq/replicator/strategy/TaskDivideStrategy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.replicator.strategy; 18 | 19 | import io.openmessaging.KeyValue; 20 | import java.util.Set; 21 | import org.apache.rocketmq.replicator.config.TaskDivideConfig; 22 | import java.util.List; 23 | import java.util.Map; 24 | import org.apache.rocketmq.replicator.config.TaskTopicInfo; 25 | 26 | public abstract class TaskDivideStrategy { 27 | 28 | public abstract List divide(Map> topicMap, TaskDivideConfig tdc, int maxTasks); 29 | } 30 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordOffsetSerde.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.serialization.store; 19 | 20 | import io.openmessaging.connector.api.data.RecordOffset; 21 | import org.apache.rocketmq.connect.runtime.serialization.WrapperSerde; 22 | 23 | /** 24 | * ByteBuffer converter. 25 | */ 26 | public class RecordOffsetSerde extends WrapperSerde { 27 | public RecordOffsetSerde() { 28 | super(new RecordOffsetSerializer(), new RecordOffsetDeserializer()); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DorisException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.exception; 21 | 22 | public class DorisException extends RuntimeException { 23 | 24 | public DorisException() { 25 | } 26 | 27 | public DorisException(String message) { 28 | super(message); 29 | } 30 | 31 | public DorisException(String message, Throwable cause) { 32 | super(message, cause); 33 | } 34 | 35 | public DorisException(Throwable cause) { 36 | super(cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/IntegerSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.runtime.serialization; 18 | 19 | public class IntegerSerializer implements Serializer { 20 | @Override 21 | public byte[] serialize(String topic, Integer data) { 22 | if (data == null) { 23 | return null; 24 | } 25 | 26 | return new byte[] { 27 | (byte) (data >>> 24), 28 | (byte) (data >>> 16), 29 | (byte) (data >>> 8), 30 | data.byteValue() 31 | }; 32 | } 33 | } -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/UploadException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.exception; 21 | 22 | public class UploadException extends DorisException { 23 | 24 | public UploadException() { 25 | } 26 | 27 | public UploadException(String message) { 28 | super(message); 29 | } 30 | 31 | public UploadException(String message, Throwable cause) { 32 | super(message, cause); 33 | } 34 | 35 | public UploadException(Throwable cause) { 36 | super(cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /rocketmq-connect-cli/src/main/java/org/apache/rocketmq/connect/cli/command/SubCommand.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.cli.command; 19 | 20 | import org.apache.commons.cli.CommandLine; 21 | import org.apache.commons.cli.Options; 22 | import org.apache.rocketmq.tools.command.SubCommandException; 23 | 24 | public interface SubCommand { 25 | String commandName(); 26 | 27 | String commandDesc(); 28 | 29 | Options buildCommandlineOptions(Options options); 30 | 31 | void execute(CommandLine commandLine, Options options) throws SubCommandException; 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/CopyLoadException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.exception; 21 | 22 | public class CopyLoadException extends DorisException { 23 | 24 | public CopyLoadException() { 25 | } 26 | 27 | public CopyLoadException(String message) { 28 | super(message); 29 | } 30 | 31 | public CopyLoadException(String message, Throwable cause) { 32 | super(message, cause); 33 | } 34 | 35 | public CopyLoadException(Throwable cause) { 36 | super(cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/ArgumentsException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.exception; 21 | 22 | public class ArgumentsException extends DorisException { 23 | 24 | public ArgumentsException() { 25 | } 26 | 27 | public ArgumentsException(String message) { 28 | super(message); 29 | } 30 | 31 | public ArgumentsException(String message, Throwable cause) { 32 | super(message, cause); 33 | } 34 | 35 | public ArgumentsException(Throwable cause) { 36 | super(cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-jdbc/src/main/java/org/apache/rocketmq/connect/jdbc/exception/TableAlterOrCreateException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.jdbc.exception; 18 | 19 | import io.openmessaging.connector.api.errors.ConnectException; 20 | 21 | /** 22 | * table alter or create exception 23 | */ 24 | public class TableAlterOrCreateException extends ConnectException { 25 | 26 | public TableAlterOrCreateException(String reason) { 27 | super(reason); 28 | } 29 | 30 | public TableAlterOrCreateException(String reason, Throwable throwable) { 31 | super(reason, throwable); 32 | } 33 | } 34 | 35 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/debezium/GeographyType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.converter.type.debezium; 21 | 22 | import io.debezium.data.geometry.Geography; 23 | import org.apache.rocketmq.connect.doris.converter.type.AbstractGeometryType; 24 | 25 | public class GeographyType extends AbstractGeometryType { 26 | public static final GeographyType INSTANCE = new GeographyType(); 27 | 28 | @Override 29 | public String[] getRegistrationKeys() { 30 | return new String[] {Geography.LOGICAL_NAME}; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/exception/DataFormatException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.exception; 21 | 22 | public class DataFormatException extends DorisException { 23 | 24 | public DataFormatException() { 25 | } 26 | 27 | public DataFormatException(String message) { 28 | super(message); 29 | } 30 | 31 | public DataFormatException(String message, Throwable cause) { 32 | super(message, cause); 33 | } 34 | 35 | public DataFormatException(Throwable cause) { 36 | super(cause); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-http/src/main/java/org/apache/rocketmq/connect/http/constant/MethodEnum.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.http.constant; 18 | 19 | public enum MethodEnum { 20 | 21 | /** 22 | * GET 23 | */ 24 | GET("GET"), 25 | POST("POST"), 26 | DELETE("DELETE"), 27 | PUT("PUT"), 28 | HEAD("HEAD"), 29 | TRACE("TRACE"), 30 | PATCH("PATCH"), 31 | OPTIONS("OPTIONS"); 32 | private final String method; 33 | 34 | MethodEnum(String method) { 35 | this.method = method; 36 | } 37 | 38 | public String getMethod() { 39 | return method; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/serialization/store/RecordPartitionSerde.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.runtime.serialization.store; 19 | 20 | import org.apache.rocketmq.connect.runtime.serialization.WrapperSerde; 21 | import org.apache.rocketmq.connect.runtime.store.ExtendRecordPartition; 22 | 23 | /** 24 | * ByteBuffer converter. 25 | */ 26 | public class RecordPartitionSerde extends WrapperSerde { 27 | 28 | public RecordPartitionSerde() { 29 | super(new RecordPartitionSerializer(), new RecordPartitionDeserializer()); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/converter/type/AbstractDateType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.converter.type; 21 | 22 | import io.openmessaging.connector.api.data.Schema; 23 | import org.apache.rocketmq.connect.doris.converter.type.doris.DorisType; 24 | 25 | /** 26 | * An abstract base class for all temporal date implementations of {@link Type}. 27 | */ 28 | public abstract class AbstractDateType extends AbstractTemporalType { 29 | 30 | @Override 31 | public String getTypeName(Schema schema) { 32 | return DorisType.DATE; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /distribution/NOTICE-BIN: -------------------------------------------------------------------------------- 1 | Apache RocketMQ 2 | Copyright 2016-2022 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | ------ 8 | This product has a bundle netty: 9 | The Netty Project 10 | ================= 11 | 12 | Please visit the Netty web site for more information: 13 | 14 | * http://netty.io/ 15 | 16 | Copyright 2014 The Netty Project 17 | 18 | The Netty Project licenses this file to you under the Apache License, 19 | version 2.0 (the "License"); you may not use this file except in compliance 20 | with the License. You may obtain a copy of the License at: 21 | 22 | http://www.apache.org/licenses/LICENSE-2.0 23 | 24 | Unless required by applicable law or agreed to in writing, software 25 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 26 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 27 | License for the specific language governing permissions and limitations 28 | under the License. 29 | 30 | Also, please refer to each LICENSE..txt file, which is located in 31 | the 'license' directory of the distribution file, for the license terms of the 32 | components that this product depends on. 33 | 34 | ------ 35 | This product has a bundle commons-lang, which includes software from the Spring Framework, 36 | under the Apache License 2.0 (see: StringUtils.containsWhitespace()) 37 | -------------------------------------------------------------------------------- /rocketmq-connect-runtime/src/main/java/org/apache/rocketmq/connect/runtime/config/SourceConnectorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | 19 | package org.apache.rocketmq.connect.runtime.config; 20 | 21 | import org.apache.rocketmq.connect.runtime.common.ConnectKeyValue; 22 | 23 | /** 24 | * source connector config 25 | */ 26 | public class SourceConnectorConfig extends ConnectorConfig { 27 | 28 | public static final String CONNECT_TOPICNAME = "connect.topicname"; 29 | public static final String ORDERING_MSG_ENABLE = "ordering.msg.enable"; 30 | 31 | public SourceConnectorConfig(ConnectKeyValue config) { 32 | super(config); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/schema/column/DefaultColumnParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.cassandra.schema.column; 19 | 20 | import org.apache.commons.codec.binary.Base64; 21 | 22 | public class DefaultColumnParser extends ColumnParser { 23 | 24 | @Override 25 | public Object getValue(Object value) { 26 | 27 | if (value == null) { 28 | return null; 29 | } 30 | 31 | if (value instanceof byte[]) { 32 | return Base64.encodeBase64String((byte[]) value); 33 | } 34 | 35 | return value; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-cassandra/src/main/java/org/apache/rocketmq/connect/cassandra/schema/column/TimeColumnParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | package org.apache.rocketmq.connect.cassandra.schema.column; 19 | 20 | import java.sql.Time; 21 | import java.sql.Timestamp; 22 | 23 | public class TimeColumnParser extends ColumnParser { 24 | 25 | @Override 26 | public Object getValue(Object value) { 27 | 28 | if (value == null) { 29 | return null; 30 | } 31 | 32 | if (value instanceof Timestamp) { 33 | 34 | return new Time(((Timestamp) value).getTime()); 35 | } 36 | 37 | return value; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-doris/src/main/java/org/apache/rocketmq/connect/doris/utils/HttpGetWithEntity.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | package org.apache.rocketmq.connect.doris.utils; 21 | 22 | import java.net.URI; 23 | import org.apache.http.client.methods.HttpEntityEnclosingRequestBase; 24 | 25 | public class HttpGetWithEntity extends HttpEntityEnclosingRequestBase { 26 | private static final String METHOD_NAME = "GET"; 27 | 28 | @Override 29 | public String getMethod() { 30 | return METHOD_NAME; 31 | } 32 | 33 | public HttpGetWithEntity(final String uri) { 34 | super(); 35 | setURI(URI.create(uri)); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /connectors/rocketmq-connect-http/src/main/java/org/apache/rocketmq/connect/http/constant/HttpHeaderConstant.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | package org.apache.rocketmq.connect.http.constant; 18 | 19 | public class HttpHeaderConstant { 20 | public static final String CONTENT_TYPE = "Content-Type"; 21 | public static final String CONTENT_LENGTH = "Content-Length"; 22 | public static final String CONTENT_ENCODING = "Content-Encoding"; 23 | public static final String AUTHORIZATION = "Authorization"; 24 | public static final String POST_METHOD = "POST"; 25 | public static final String PUT_METHOD = "PUT"; 26 | public static final String PATCH_METHOD = "PATCH"; 27 | } 28 | --------------------------------------------------------------------------------