├── .github ├── CODEOWNERS ├── release-drafter.yml ├── workflows │ ├── release-note.yml │ ├── integration-test.yml │ ├── release.yml │ └── unit-test.yml └── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── docs └── rop-architecture.png ├── rocketmq-impl ├── README.md ├── src │ ├── main │ │ ├── java │ │ │ └── org │ │ │ │ └── streamnative │ │ │ │ └── pulsar │ │ │ │ └── handlers │ │ │ │ └── rocketmq │ │ │ │ ├── package-info.java │ │ │ │ ├── inner │ │ │ │ ├── package-info.java │ │ │ │ ├── bean │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── RopOffsetWrapper.java │ │ │ │ │ └── RopConsumeStats.java │ │ │ │ ├── proxy │ │ │ │ │ └── package-info.java │ │ │ │ ├── timer │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── Time.java │ │ │ │ │ ├── SystemTime.java │ │ │ │ │ ├── Timer.java │ │ │ │ │ └── TimerTask.java │ │ │ │ ├── trace │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── TraceContext.java │ │ │ │ │ └── TraceStatsReportService.java │ │ │ │ ├── format │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── RopMessageFilter.java │ │ │ │ │ └── EntryFormatter.java │ │ │ │ ├── namesvr │ │ │ │ │ └── package-info.java │ │ │ │ ├── pulsar │ │ │ │ │ ├── package-info.java │ │ │ │ │ └── PulsarMessageStore.java │ │ │ │ ├── consumer │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── metadata │ │ │ │ │ │ ├── package-info.java │ │ │ │ │ │ ├── Deserializer.java │ │ │ │ │ │ └── GroupOffsetConstant.java │ │ │ │ │ ├── CommitLogOffset.java │ │ │ │ │ └── RopGetMessageResult.java │ │ │ │ ├── exception │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── RopSendException.java │ │ │ │ │ ├── RopDecodeException.java │ │ │ │ │ ├── RopEncodeException.java │ │ │ │ │ ├── RopPullMessageException.java │ │ │ │ │ ├── RopPersistentTopicException.java │ │ │ │ │ ├── RopServerException.java │ │ │ │ │ ├── RopRuntimeException.java │ │ │ │ │ └── RopTopicNotExistsException.java │ │ │ │ ├── listener │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── NotifyMessageArrivingListener.java │ │ │ │ │ └── DefaultConsumerIdsChangeListener.java │ │ │ │ ├── processor │ │ │ │ │ └── package-info.java │ │ │ │ ├── producer │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── ClientGroupAndTopicName.java │ │ │ │ │ ├── ClientGroupName.java │ │ │ │ │ └── ClientTopicName.java │ │ │ │ ├── zookeeper │ │ │ │ │ ├── package-info.java │ │ │ │ │ ├── RopCoordinatorContent.java │ │ │ │ │ ├── RopGroupContent.java │ │ │ │ │ ├── RopTopicContent.java │ │ │ │ │ ├── RopZkUtils.java │ │ │ │ │ ├── RopZkClient.java │ │ │ │ │ └── RopClusterContent.java │ │ │ │ ├── coordinator │ │ │ │ │ ├── package-info.java │ │ │ │ │ └── RopBroker.java │ │ │ │ ├── PutMessageCallback.java │ │ │ │ ├── PutMessageResult.java │ │ │ │ ├── RopMessage.java │ │ │ │ ├── RopPutMessageResult.java │ │ │ │ ├── InternalProducer.java │ │ │ │ ├── RopClientChannelCnx.java │ │ │ │ ├── TransactionalMessageCheckService.java │ │ │ │ └── InternalServerCnx.java │ │ │ │ ├── utils │ │ │ │ ├── package-info.java │ │ │ │ ├── OffsetSearchPredicate.java │ │ │ │ ├── CommitLogOffsetUtils.java │ │ │ │ ├── FileRegionEncoder.java │ │ │ │ └── CoreUtils.java │ │ │ │ ├── metrics │ │ │ │ ├── package-info.java │ │ │ │ └── FilteringJmxReporter.java │ │ │ │ └── RocketMQChannelInitializer.java │ │ └── resources │ │ │ └── META-INF │ │ │ └── services │ │ │ └── pulsar-protocol-handler.yml │ ├── test │ │ ├── java │ │ │ └── org │ │ │ │ └── streamnative │ │ │ │ └── pulsar │ │ │ │ └── handlers │ │ │ │ └── rocketmq │ │ │ │ ├── inner │ │ │ │ ├── consumer │ │ │ │ │ └── metadata │ │ │ │ │ │ ├── GroupOffsetKeyTest.java │ │ │ │ │ │ └── GroupOffsetValueTest.java │ │ │ │ ├── producer │ │ │ │ │ ├── ClientGroupNameTest.java │ │ │ │ │ └── ClientTopicNameTest.java │ │ │ │ └── timer │ │ │ │ │ └── MockTimer.java │ │ │ │ └── utils │ │ │ │ └── TopicNameUtilsTest.java │ │ └── resources │ │ │ └── log4j2.xml │ └── assemble │ │ └── bin.xml └── conf │ └── rop_env.sh ├── conf └── ipTable.properties ├── resources ├── license.template ├── findbugsExclude.xml └── suppression.xml ├── scripts ├── dev │ ├── __init__.py │ └── get-project-version.py ├── set-project-version.sh └── set-pulsar-version.sh ├── .gitignore ├── examples ├── src │ └── main │ │ ├── resources │ │ └── application.properties │ │ └── java │ │ └── org │ │ └── streamnative │ │ └── rocketmq │ │ └── example │ │ ├── benchmark │ │ └── AclClient.java │ │ ├── springboot │ │ ├── SpringBootProduceDemo.java │ │ └── SpringBootPushConsumeDemo.java │ │ ├── namespace │ │ ├── ProducerWithNamespace.java │ │ └── PushConsumerWithNamespace.java │ │ ├── batch │ │ └── SimpleBatchProducer.java │ │ ├── simple │ │ ├── LitePullConsumerSubscribe.java │ │ ├── SenderWithTags.java │ │ ├── RoundRobinProducer.java │ │ ├── QueueSelectorProducer.java │ │ └── AsyncProducer.java │ │ ├── delaymessage │ │ └── DelayProduceDemo.java │ │ ├── broadcast │ │ └── PushConsumer.java │ │ ├── ordermessage │ │ ├── OrderConsumer.java │ │ └── OrderProducer.java │ │ └── quickstart │ │ └── SimpleProducer.java ├── golang │ ├── simple │ │ ├── consumer.go │ │ └── producer.go │ ├── batch │ │ └── producer.go │ ├── tag │ │ ├── producer.go │ │ └── consumer.go │ ├── namespace │ │ ├── producer.go │ │ └── consumer.go │ ├── broadcast │ │ └── consumer.go │ ├── acl │ │ ├── producer.go │ │ └── consumer.go │ ├── orderly │ │ └── consumer.go │ └── async │ │ └── producer.go └── pom.xml └── tests ├── src └── test │ ├── java │ └── org │ │ └── streamnative │ │ └── pulsar │ │ └── handlers │ │ └── rocketmq │ │ └── PortManager.java │ └── resources │ └── log4j2.xml └── pom.xml /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @streamnative/platform 2 | -------------------------------------------------------------------------------- /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | template: | 2 | ## What's changed 3 | $CHANGES 4 | -------------------------------------------------------------------------------- /docs/rop-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/streamnative/rop/HEAD/docs/rop-architecture.png -------------------------------------------------------------------------------- /rocketmq-impl/README.md: -------------------------------------------------------------------------------- 1 | # RoP 2 | 3 | RoP stands for Rocketmq on Pulsar. The RoP broker supports Rocketmq protocols, and is backed by Pulsar. 4 | 5 | For details, see [README](https://github.com/streamnative/rop/blob/master/README.md) for detail. 6 | -------------------------------------------------------------------------------- /conf/ipTable.properties: -------------------------------------------------------------------------------- 1 | ## broker 127.0.0.1 2 | internal_ins@127.0.0.1=127.0.0.1:6650 3 | qcloud_ins@127.0.0.1=127.0.0.1:5010 4 | public_ins@127.0.0.1=127.0.0.1:5010 5 | internal_http_ins@127.0.0.1=127.0.0.1:8080 6 | qcloud_http_ins@127.0.0.1=127.0.0.1:5021 7 | public_http_ins@127.0.0.1=127.0.0.1:5020 8 | internal_rop_ins@127.0.0.1=127.0.0.1:9876 9 | qcloud_rop_ins@127.0.0.1=127.0.0.1:5032 10 | public_rop_ins@9127.0.0.1=127.0.0.1:5030 11 | -------------------------------------------------------------------------------- /.github/workflows/release-note.yml: -------------------------------------------------------------------------------- 1 | name: RoP Release Notes 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | path-ignores: 8 | - 'docs/**' 9 | - 'README.md' 10 | - 'CONTRIBUTING.md' 11 | jobs: 12 | build: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - name: release note 18 | uses: toolmantim/release-drafter@v5.2.0 19 | env: 20 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 21 | -------------------------------------------------------------------------------- /resources/license.template: -------------------------------------------------------------------------------- 1 | Licensed under the Apache License, Version 2.0 (the "License"); 2 | you may not use this file except in compliance with the License. 3 | You may obtain a copy of the License at 4 | 5 | http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | -------------------------------------------------------------------------------- /scripts/dev/__init__.py: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report about: Create a report to help us improve title: "[BUG]" 3 | labels: type/bug assignees: '' 4 | 5 | --- 6 | 7 | **Describe the bug** 8 | A clear and concise description of what the bug is. 9 | 10 | **To Reproduce** 11 | Steps to reproduce the behavior: 12 | 13 | 1. Go to '...' 14 | 2. Click on '....' 15 | 3. Scroll down to '....' 16 | 4. See error 17 | 18 | **Expected behavior** 19 | A clear and concise description of what you expected to happen. 20 | 21 | **Screenshots** 22 | If applicable, add screenshots to help explain your problem. 23 | 24 | **Additional context** 25 | Add any other context about the problem here. 26 | -------------------------------------------------------------------------------- /resources/findbugsExclude.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request about: Suggest an idea for this project title: "[FEATURE]" 3 | labels: type/feature assignees: '' 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .metadata 2 | dependency-reduced-pom.xml 3 | logs 4 | /data 5 | pulsar-rop-*-bin.tar.gz 6 | *.log 7 | *.nar 8 | 9 | *.versionsBackup 10 | */bin 11 | 12 | # Eclipse 13 | .classpath 14 | .project 15 | .settings/ 16 | .recommenders/ 17 | .factorypath 18 | 19 | # Intellij 20 | .idea/ 21 | *.iml 22 | *.iws 23 | 24 | # Mac 25 | **/.DS_Store 26 | 27 | # VisualStudioCode artifacts 28 | .vscode/ 29 | 30 | # Maven 31 | log/ 32 | target/ 33 | 34 | # Python 35 | *.pyc 36 | 37 | # Perf tools 38 | *.hgrm 39 | 40 | # tmp files 41 | *.pid 42 | 43 | # CI generated files 44 | .repository 45 | docker.debug-info 46 | 47 | # Yarn 48 | 49 | **/yarn.lock 50 | 51 | # docusaurus 52 | 53 | **/website/i18n/* 54 | **/website/translated_docs* 55 | 56 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/utils/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.utils; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/metrics/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.metrics; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/bean/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.bean; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/proxy/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.proxy; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/timer/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.timer; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/trace/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.trace; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/format/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.format; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/namesvr/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.namesvr; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/pulsar/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.pulsar; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/consumer/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.consumer; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/exception/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.exception; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/listener/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.listener; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/processor/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.processor; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/producer/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.producer; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/zookeeper/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.zookeeper; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/coordinator/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.coordinator; 16 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/consumer/metadata/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.consumer.metadata; 16 | -------------------------------------------------------------------------------- /scripts/set-project-version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | set -e 17 | 18 | if [ $# -eq 0 ]; then 19 | echo "Required argument with new project version" 20 | exit 1 21 | fi 22 | 23 | NEW_VERSION=$1 24 | 25 | # Go to top level project directory 26 | pushd $(dirname "$0")/.. 27 | 28 | mvn versions:set -DnewVersion=$NEW_VERSION 29 | 30 | popd 31 | -------------------------------------------------------------------------------- /scripts/set-pulsar-version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | set -e 17 | 18 | if [ $# -eq 0 ]; then 19 | echo "Required argument with new pulsar version" 20 | exit 1 21 | fi 22 | 23 | NEW_VERSION=$1 24 | 25 | # Go to top level project directory 26 | pushd $(dirname "$0")/.. 27 | 28 | mvn versions:set-property -Dproperty=pulsar.version -DnewVersion=$NEW_VERSION 29 | 30 | popd 31 | -------------------------------------------------------------------------------- /examples/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed under the Apache License, Version 2.0 (the "License"); 3 | # you may not use this file except in compliance with the License. 4 | # You may obtain a copy of the License at 5 | # 6 | # http://www.apache.org/licenses/LICENSE-2.0 7 | # 8 | # Unless required by applicable law or agreed to in writing, software 9 | # distributed under the License is distributed on an "AS IS" BASIS, 10 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | # See the License for the specific language governing permissions and 12 | # limitations under the License. 13 | # 14 | 15 | rocketmq.name-server=rocketmq-og7vojz5e33k.rocketmq.ap-bj.public.tencenttdmq.com:9876 16 | rocketmq.producer.group=test-producer-group 17 | rocketmq.producer.accessKey=eyJrZXlJZCI6InJvY2tldG1xLW9nN3Zvano1ZTMzayIsImFsZyI6IkhTMjU2In0.eyJzdWIiOiJyb2NrZXRtcS1vZzd2b2p6NWUzM2tfdGVzdC1yb2xlIn0.YPYh-7dB8lzH178b6_tP_mzdNiy6dbnpRR9RFijIT3U 18 | rocketmq.producer.secretKey=ROP -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/PutMessageCallback.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner; 16 | 17 | import org.apache.rocketmq.store.PutMessageResult; 18 | 19 | /** 20 | * Put message callback. 21 | */ 22 | public interface PutMessageCallback { 23 | 24 | default void callback(PutMessageResult putMessageResult) { 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/timer/Time.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.timer; 16 | 17 | /** 18 | * Time util class. 19 | */ 20 | public interface Time { 21 | 22 | Time SYSTEM = new SystemTime(); 23 | 24 | long milliseconds(); 25 | 26 | long hiResClockMs(); 27 | 28 | long nanoseconds(); 29 | 30 | void sleep(long var1); 31 | } 32 | -------------------------------------------------------------------------------- /scripts/dev/get-project-version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | import xml.etree.ElementTree as ET 17 | from os.path import dirname, realpath, join 18 | 19 | # Derive the POM path from the current script location 20 | TOP_LEVEL_PATH = dirname(dirname(dirname(realpath(__file__)))) 21 | POM_PATH = join(TOP_LEVEL_PATH, 'pom.xml') 22 | 23 | root = ET.XML(open(POM_PATH).read()) 24 | print(root.find('{http://maven.apache.org/POM/4.0.0}version').text) 25 | -------------------------------------------------------------------------------- /.github/workflows/integration-test.yml: -------------------------------------------------------------------------------- 1 | name: rop mvn build check integration test 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | push: 8 | branches: 9 | - master 10 | - branch-* 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v1 19 | - name: Set up JDK 1.8 20 | uses: actions/setup-java@v1 21 | with: 22 | java-version: 1.8 23 | 24 | # - name: clean disk 25 | # if: steps.docs.outputs.changed_only == 'no' 26 | # run: | 27 | # df -h 28 | # sudo swapoff /swapfile 29 | # sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc 30 | # sudo apt clean 31 | # docker rmi $(docker images -q) -f 32 | # df -h 33 | 34 | - name: Build with Maven skipTests 35 | run: mvn clean install -DskipTests 36 | 37 | - name: rocketmq-impl test after build 38 | run: mvn test -DfailIfNoTests=false -pl tests 39 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: RoP Release 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | upload: 9 | name: Upload Release files 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v1 13 | - name: Set up JDK 1.8 14 | uses: actions/setup-java@v1 15 | with: 16 | java-version: 1.8 17 | 18 | - name: build 19 | run: | 20 | version=`./scripts/dev/get-project-version.py` 21 | mvn clean install -DskipTests 22 | rm -rf pulsar-* 23 | cp rocketmq-impl/target/pulsar-protocol-handler-rocketmq-${version}.nar ./ 24 | cp rocketmq-impl/target/pulsar-rop-${version}-bin.tar.gz ./ 25 | cp README.md ./pulsar-protocol-handler-rocketmq-readme.md 26 | echo "ls pulsar-*" 27 | ls pulsar-* 28 | 29 | - name: publish 30 | uses: skx/github-action-publish-binaries@master 31 | env: 32 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 33 | with: 34 | args: 'pulsar-*' 35 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/resources/META-INF/services/pulsar-protocol-handler.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | # 19 | 20 | name: rocketmq 21 | description: rocketmq Protocol Handler 22 | handlerClass: org.streamnative.pulsar.handlers.rocketmq.RocketMQProtocolHandler 23 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/zookeeper/RopCoordinatorContent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.zookeeper; 16 | 17 | import lombok.Data; 18 | import lombok.EqualsAndHashCode; 19 | import lombok.extern.slf4j.Slf4j; 20 | 21 | /** 22 | * Rop coordinator broker. 23 | */ 24 | @Slf4j 25 | @Data 26 | @EqualsAndHashCode 27 | public class RopCoordinatorContent { 28 | public final String serviceUrl; 29 | } 30 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/zookeeper/RopGroupContent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.zookeeper; 16 | 17 | import lombok.AllArgsConstructor; 18 | import lombok.Data; 19 | import lombok.NoArgsConstructor; 20 | import org.apache.rocketmq.common.subscription.SubscriptionGroupConfig; 21 | 22 | /** 23 | * Rop group content. 24 | */ 25 | @Data 26 | @AllArgsConstructor 27 | @NoArgsConstructor 28 | public class RopGroupContent { 29 | private SubscriptionGroupConfig config; 30 | } 31 | -------------------------------------------------------------------------------- /.github/workflows/unit-test.yml: -------------------------------------------------------------------------------- 1 | name: rop mvn build check and rocketmq-impl test 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - master 7 | push: 8 | branches: 9 | - master 10 | - branch-* 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v1 19 | - name: Set up JDK 1.8 20 | uses: actions/setup-java@v1 21 | with: 22 | java-version: 1.8 23 | 24 | - name: clean disk 25 | if: steps.docs.outputs.changed_only == 'no' 26 | run: | 27 | df -h 28 | sudo swapoff /swapfile 29 | sudo rm -rf /swapfile /usr/share/dotnet /usr/local/lib/android /opt/ghc 30 | sudo apt clean 31 | docker rmi $(docker images -q) -f 32 | df -h 33 | 34 | - name: License check 35 | run: mvn license:check 36 | 37 | - name: Build with Maven skipTests 38 | run: mvn clean install -DskipTests 39 | 40 | - name: Style check 41 | run: mvn checkstyle:check 42 | 43 | - name: Spotbugs check 44 | run: mvn spotbugs:check 45 | 46 | - name: rocketmq-impl test after build 47 | run: mvn test -DfailIfNoTests=false -pl rocketmq-impl 48 | 49 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/exception/RopSendException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.exception; 16 | 17 | /** 18 | * Rop send exception. 19 | */ 20 | public class RopSendException extends Exception { 21 | 22 | public RopSendException() { 23 | super(); 24 | } 25 | 26 | public RopSendException(String message) { 27 | super(message); 28 | } 29 | 30 | public RopSendException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public RopSendException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/consumer/metadata/Deserializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.consumer.metadata; 16 | 17 | import java.nio.ByteBuffer; 18 | import org.streamnative.pulsar.handlers.rocketmq.inner.exception.RopDecodeException; 19 | import org.streamnative.pulsar.handlers.rocketmq.inner.exception.RopEncodeException; 20 | 21 | /** 22 | * Deserializer method. 23 | * @param 24 | */ 25 | public interface Deserializer { 26 | 27 | ByteBuffer encode() throws RopEncodeException; 28 | 29 | T decode(ByteBuffer buffer) throws RopDecodeException; 30 | 31 | int estimateSize(); 32 | } 33 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/exception/RopDecodeException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.exception; 16 | 17 | /** 18 | * Rop encode exception. 19 | */ 20 | public class RopDecodeException extends Exception { 21 | 22 | public RopDecodeException() { 23 | super(); 24 | } 25 | 26 | public RopDecodeException(String message) { 27 | super(message); 28 | } 29 | 30 | public RopDecodeException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public RopDecodeException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/exception/RopEncodeException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.exception; 16 | 17 | /** 18 | * Rop encode exception. 19 | */ 20 | public class RopEncodeException extends Exception { 21 | 22 | public RopEncodeException() { 23 | super(); 24 | } 25 | 26 | public RopEncodeException(String message) { 27 | super(message); 28 | } 29 | 30 | public RopEncodeException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public RopEncodeException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/exception/RopPullMessageException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.exception; 16 | 17 | /** 18 | * Rop pull message exception. 19 | */ 20 | public class RopPullMessageException extends Exception { 21 | 22 | public RopPullMessageException() { 23 | super(); 24 | } 25 | 26 | public RopPullMessageException(String message) { 27 | super(message); 28 | } 29 | 30 | public RopPullMessageException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public RopPullMessageException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/exception/RopPersistentTopicException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.exception; 16 | 17 | /** 18 | * Rop find persistent topic exception. 19 | */ 20 | public class RopPersistentTopicException extends Exception { 21 | 22 | public RopPersistentTopicException() { 23 | super(); 24 | } 25 | 26 | public RopPersistentTopicException(String message) { 27 | super(message); 28 | } 29 | 30 | public RopPersistentTopicException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public RopPersistentTopicException(Throwable cause) { 35 | super(cause); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/benchmark/AclClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.benchmark; 16 | 17 | import org.apache.rocketmq.acl.common.AclClientRPCHook; 18 | import org.apache.rocketmq.acl.common.SessionCredentials; 19 | import org.apache.rocketmq.remoting.RPCHook; 20 | 21 | /** 22 | * Acl client. 23 | */ 24 | public class AclClient { 25 | 26 | private static final String ACL_ACCESS_KEY = "eyJrZXlJZCI6InB1bHNhci04xxxxxxxxxxUzI1NiJ9." 27 | + "eyJzdWIiOiJwdWxxxxxx3QtMTExMTExIn0.cIsxxGtnuXxxxxxES-0WccDZjPEGUFzT-th-f6I"; 28 | private static final String ACL_SECRET_KEY = "rop"; 29 | 30 | static RPCHook getAclRPCHook() { 31 | return new AclClientRPCHook(new SessionCredentials(ACL_ACCESS_KEY, ACL_SECRET_KEY)); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/zookeeper/RopTopicContent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.zookeeper; 16 | 17 | import com.google.common.collect.Maps; 18 | import java.util.List; 19 | import java.util.Map; 20 | import lombok.AllArgsConstructor; 21 | import lombok.Data; 22 | import lombok.NoArgsConstructor; 23 | import org.apache.rocketmq.common.TopicConfig; 24 | 25 | /** 26 | * Rop topic content. 27 | */ 28 | @Data 29 | @AllArgsConstructor 30 | @NoArgsConstructor 31 | public class RopTopicContent { 32 | 33 | private TopicConfig config; 34 | // key: brokerTag, value: partition list 35 | private Map> routeMap; 36 | 37 | public Map> getRouteMap() { 38 | return routeMap == null ? Maps.newHashMap() : routeMap; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /rocketmq-impl/conf/rop_env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | 16 | 17 | # Set JAVA_HOME here to override the environment setting 18 | # JAVA_HOME= 19 | 20 | # Log4j configuration file 21 | # ROP_LOG_CONF= 22 | 23 | # Configuration file of settings used in Rop 24 | # ROP_CONF= 25 | 26 | # Extra options to be passed to the jvm 27 | ROP_MEM=${ROP_MEM:-"-Xms2g -Xmx2g -XX:MaxDirectMemorySize=4g"} 28 | 29 | # Garbage collection options 30 | ROP_GC=" -XX:+UseG1GC -XX:MaxGCPauseMillis=10 -XX:+ParallelRefProcEnabled -XX:+UnlockExperimentalVMOptions -XX:+AggressiveOpts -XX:+DoEscapeAnalysis -XX:ParallelGCThreads=32 -XX:ConcGCThreads=32 -XX:G1NewSizePercent=50 -XX:+DisableExplicitGC -XX:-ResizePLAB" 31 | 32 | # Extra options to be passed to the jvm 33 | ROP_EXTRA_OPTS="${ROP_EXTRA_OPTS} ${ROP_MEM} ${ROP_GC} -Dio.netty.leakDetectionLevel=disabled -Dio.netty.recycler.maxCapacity.default=1000 -Dio.netty.recycler.linkCapacity=1024" 34 | 35 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/timer/SystemTime.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.timer; 16 | 17 | import java.util.concurrent.TimeUnit; 18 | 19 | /** 20 | * System time util class. 21 | */ 22 | public class SystemTime implements Time { 23 | 24 | public SystemTime() { 25 | } 26 | 27 | public long milliseconds() { 28 | return System.currentTimeMillis(); 29 | } 30 | 31 | public long hiResClockMs() { 32 | return TimeUnit.NANOSECONDS.toMillis(this.nanoseconds()); 33 | } 34 | 35 | public long nanoseconds() { 36 | return System.nanoTime(); 37 | } 38 | 39 | public void sleep(long ms) { 40 | try { 41 | Thread.sleep(ms); 42 | } catch (InterruptedException ex) { 43 | Thread.currentThread().interrupt(); 44 | } 45 | 46 | } 47 | } -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/exception/RopServerException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.exception; 16 | 17 | /** 18 | * Rop Server Exception. 19 | */ 20 | public class RopServerException extends Exception { 21 | 22 | public RopServerException() { 23 | super(); 24 | } 25 | 26 | public RopServerException(String message) { 27 | super(message); 28 | } 29 | 30 | public RopServerException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public RopServerException(Throwable cause) { 35 | super(cause); 36 | } 37 | 38 | protected RopServerException(String message, Throwable cause, boolean enableSuppression, 39 | boolean writableStackTrace) { 40 | super(message, cause, enableSuppression, writableStackTrace); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/exception/RopRuntimeException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.exception; 16 | 17 | /** 18 | * Rop Runtime exception. 19 | */ 20 | public class RopRuntimeException extends RuntimeException { 21 | 22 | public RopRuntimeException() { 23 | super(); 24 | } 25 | 26 | public RopRuntimeException(String message) { 27 | super(message); 28 | } 29 | 30 | public RopRuntimeException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public RopRuntimeException(Throwable cause) { 35 | super(cause); 36 | } 37 | 38 | protected RopRuntimeException(String message, Throwable cause, boolean enableSuppression, 39 | boolean writableStackTrace) { 40 | super(message, cause, enableSuppression, writableStackTrace); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/PutMessageResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner; 16 | 17 | import lombok.AllArgsConstructor; 18 | import lombok.Data; 19 | import lombok.NoArgsConstructor; 20 | 21 | /** 22 | * Put message result. 23 | */ 24 | @Data 25 | @AllArgsConstructor 26 | @NoArgsConstructor 27 | public class PutMessageResult { 28 | 29 | private String msgId; 30 | private String offsetMsgId; 31 | private String pulsarMsgId; 32 | private Integer partition; 33 | private Long offset; 34 | 35 | private String msgKey; 36 | private String msgTag; 37 | 38 | public PutMessageResult(String magId, String pulsarMsgId, long offset, String msgKey, String msgTag) { 39 | this.msgId = magId; 40 | this.pulsarMsgId = pulsarMsgId; 41 | this.offset = offset; 42 | this.msgKey = msgKey; 43 | this.msgTag = msgTag; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/exception/RopTopicNotExistsException.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.exception; 16 | 17 | /** 18 | * Rop Topic Not Exists Exception. 19 | */ 20 | public class RopTopicNotExistsException extends Exception { 21 | 22 | public RopTopicNotExistsException() { 23 | super(); 24 | } 25 | 26 | public RopTopicNotExistsException(String message) { 27 | super(message); 28 | } 29 | 30 | public RopTopicNotExistsException(String message, Throwable cause) { 31 | super(message, cause); 32 | } 33 | 34 | public RopTopicNotExistsException(Throwable cause) { 35 | super(cause); 36 | } 37 | 38 | protected RopTopicNotExistsException(String message, Throwable cause, boolean enableSuppression, 39 | boolean writableStackTrace) { 40 | super(message, cause, enableSuppression, writableStackTrace); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /resources/suppression.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/producer/ClientGroupAndTopicName.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.producer; 16 | 17 | import java.io.Serializable; 18 | import lombok.Data; 19 | import lombok.EqualsAndHashCode; 20 | import lombok.ToString; 21 | 22 | /** 23 | * Client group and client topic name. 24 | */ 25 | @Data 26 | @EqualsAndHashCode 27 | @ToString 28 | public class ClientGroupAndTopicName implements Serializable { 29 | 30 | private final ClientGroupName clientGroupName; 31 | private final ClientTopicName clientTopicName; 32 | 33 | public ClientGroupAndTopicName(String rmqGroupName, String rmqTopicName) { 34 | this.clientGroupName = new ClientGroupName(rmqGroupName); 35 | this.clientTopicName = new ClientTopicName(rmqTopicName); 36 | } 37 | 38 | public ClientGroupAndTopicName(ClientGroupName clientGroupName, 39 | ClientTopicName clientTopicName) { 40 | this.clientGroupName = clientGroupName; 41 | this.clientTopicName = clientTopicName; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /examples/golang/simple/consumer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "time" 22 | 23 | "github.com/apache/rocketmq-client-go/v2" 24 | "github.com/apache/rocketmq-client-go/v2/consumer" 25 | "github.com/apache/rocketmq-client-go/v2/primitive" 26 | ) 27 | 28 | func main() { 29 | c, _ := rocketmq.NewPushConsumer( 30 | consumer.WithGroupName("testGroup"), 31 | consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 32 | ) 33 | err := c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context, 34 | msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) { 35 | for i := range msgs { 36 | fmt.Printf("subscribe callback: %v \n", msgs[i]) 37 | } 38 | 39 | return consumer.ConsumeSuccess, nil 40 | }) 41 | if err != nil { 42 | fmt.Println(err.Error()) 43 | } 44 | // Note: start after subscribe 45 | err = c.Start() 46 | if err != nil { 47 | fmt.Println(err.Error()) 48 | os.Exit(-1) 49 | } 50 | time.Sleep(time.Hour) 51 | err = c.Shutdown() 52 | if err != nil { 53 | fmt.Printf("shutdown Consumer error: %s", err.Error()) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/timer/Timer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.timer; 16 | 17 | /** 18 | * The timer interface to execute delayed operations. 19 | */ 20 | public interface Timer { 21 | 22 | /** 23 | * Add a new task to this executor. It will be executed after the task's delay 24 | * (beginning from the time of submission) 25 | * 26 | * @param timerTask the task to add 27 | */ 28 | void add(TimerTask timerTask); 29 | 30 | /** 31 | * Advance the internal clock, executing any tasks whose expiration has been 32 | * reached within the duration of the passed timeout. 33 | * 34 | * @param timeoutMs 35 | * @return whether or not any tasks were executed 36 | */ 37 | boolean advanceClock(long timeoutMs); 38 | 39 | /** 40 | * Get the number of tasks pending execution. 41 | * 42 | * @return the number of tasks 43 | */ 44 | int size(); 45 | 46 | /** 47 | * Shutdown the timer service, leaving pending tasks unexecuted. 48 | */ 49 | void shutdown(); 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/springboot/SpringBootProduceDemo.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.springboot; 16 | 17 | import javax.annotation.Resource; 18 | import lombok.extern.slf4j.Slf4j; 19 | import org.apache.rocketmq.spring.core.RocketMQTemplate; 20 | import org.springframework.boot.CommandLineRunner; 21 | import org.springframework.boot.SpringApplication; 22 | import org.springframework.boot.autoconfigure.SpringBootApplication; 23 | 24 | /** 25 | * SpringBoot produce demo. 26 | */ 27 | @Slf4j 28 | @SpringBootApplication 29 | public class SpringBootProduceDemo implements CommandLineRunner { 30 | 31 | @Resource 32 | private RocketMQTemplate rocketMQTemplate; 33 | 34 | public static void main(String[] args) { 35 | SpringApplication.run(SpringBootProduceDemo.class, args); 36 | } 37 | 38 | public void run(String... args) { 39 | log.info("Send message: [Hello world!]"); 40 | 41 | // 使用主题全名 [namespace]%[topic], 例如：rocketmq-og7vojz5e33k|test-ns%test-topic 42 | rocketMQTemplate.convertAndSend("rocketmq-og7vojz5e33k|test-ns%test-topic", "Hello, World!"); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/RopMessage.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner; 16 | 17 | import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; 18 | import io.netty.buffer.ByteBuf; 19 | import lombok.Data; 20 | 21 | /** 22 | * RoP message. 23 | */ 24 | @Data 25 | @SuppressFBWarnings 26 | public class RopMessage { 27 | 28 | private String msgId; 29 | private int partitionId; 30 | private long offset; 31 | private ByteBuf payload; 32 | 33 | private String msgKey; 34 | private String msgTag; 35 | private byte[] msgBody; 36 | 37 | public RopMessage() { 38 | } 39 | 40 | public RopMessage(String msgId, int partitionId, long offset, ByteBuf payload) { 41 | this.msgId = msgId; 42 | this.partitionId = partitionId; 43 | this.offset = offset; 44 | this.payload = payload; 45 | } 46 | 47 | public RopMessage(String msgId, String msgKey, String msgTag, byte[] msgBody) { 48 | this.msgId = msgId; 49 | this.msgKey = msgKey; 50 | this.msgTag = msgTag; 51 | this.msgBody = msgBody; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/RopPutMessageResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import org.apache.rocketmq.store.AppendMessageResult; 20 | import org.apache.rocketmq.store.PutMessageStatus; 21 | 22 | /** 23 | * Rop put message result. 24 | */ 25 | public class RopPutMessageResult extends org.apache.rocketmq.store.PutMessageResult { 26 | 27 | private final List putMessageResults = new ArrayList<>(); 28 | 29 | public RopPutMessageResult(PutMessageStatus putMessageStatus, AppendMessageResult appendMessageResult) { 30 | super(putMessageStatus, appendMessageResult); 31 | } 32 | 33 | public void addPutMessageId(PutMessageResult putMessageResult) { 34 | putMessageResults.add(putMessageResult); 35 | } 36 | 37 | public void addPutMessageIds(List putMessageResults) { 38 | this.putMessageResults.addAll(putMessageResults); 39 | } 40 | 41 | public List getPutMessageResults() { 42 | return putMessageResults; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/namespace/ProducerWithNamespace.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.namespace; 16 | 17 | import org.apache.rocketmq.client.producer.DefaultMQProducer; 18 | import org.apache.rocketmq.client.producer.SendResult; 19 | import org.apache.rocketmq.common.message.Message; 20 | 21 | /** 22 | * Producer with namespace. 23 | */ 24 | public class ProducerWithNamespace { 25 | 26 | public static void main(String[] args) throws Exception { 27 | 28 | DefaultMQProducer producer = new DefaultMQProducer("test1|InstanceTest", "pidTest"); 29 | producer.setNamesrvAddr("127.0.0.1:9876"); 30 | producer.start(); 31 | for (int i = 0; i < 100; i++) { 32 | Message message = new Message("topicTest3", "TagA", "Hello world".getBytes("GBK")); 33 | try { 34 | SendResult result = producer.send(message); 35 | System.out.printf("Topic:%s send success, misId is:%s%n", message.getTopic(), result.getMsgId()); 36 | } catch (Exception e) { 37 | e.printStackTrace(); 38 | } 39 | } 40 | 41 | producer.shutdown(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /examples/golang/batch/producer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "strconv" 22 | 23 | "github.com/apache/rocketmq-client-go/v2" 24 | "github.com/apache/rocketmq-client-go/v2/primitive" 25 | "github.com/apache/rocketmq-client-go/v2/producer" 26 | ) 27 | 28 | func main() { 29 | p, _ := rocketmq.NewProducer( 30 | producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 31 | producer.WithRetry(2), 32 | producer.WithNamespace("test1|InstanceTest"), 33 | ) 34 | err := p.Start() 35 | if err != nil { 36 | fmt.Printf("start producer error: %s", err.Error()) 37 | os.Exit(1) 38 | } 39 | var msgs []*primitive.Message 40 | for i := 0; i < 10; i++ { 41 | msgs = append(msgs, primitive.NewMessage("test_batch", 42 | []byte("Hello RocketMQ Go Client! num: "+strconv.Itoa(i)))) 43 | } 44 | 45 | res, err := p.SendSync(context.Background(), msgs...) 46 | 47 | if err != nil { 48 | fmt.Printf("send message error: %s\n", err) 49 | } else { 50 | fmt.Printf("send message success: result=%s\n", res.String()) 51 | } 52 | err = p.Shutdown() 53 | if err != nil { 54 | fmt.Printf("shutdown producer error: %s", err.Error()) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /examples/golang/tag/producer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | 22 | "github.com/apache/rocketmq-client-go/v2" 23 | "github.com/apache/rocketmq-client-go/v2/primitive" 24 | "github.com/apache/rocketmq-client-go/v2/producer" 25 | ) 26 | 27 | func main() { 28 | p, _ := rocketmq.NewProducer( 29 | producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 30 | producer.WithRetry(2), 31 | producer.WithNamespace("test1|InstanceTest"), 32 | ) 33 | err := p.Start() 34 | if err != nil { 35 | fmt.Printf("start producer error: %s", err.Error()) 36 | os.Exit(1) 37 | } 38 | tags := []string{"TagA", "TagB", "TagC"} 39 | for i := 0; i < 3; i++ { 40 | tag := tags[i%3] 41 | msg := primitive.NewMessage("test_tag", 42 | []byte("Hello RocketMQ Go Client!")) 43 | msg.WithTag(tag) 44 | 45 | res, err := p.SendSync(context.Background(), msg) 46 | if err != nil { 47 | fmt.Printf("send message error: %s\n", err) 48 | } else { 49 | fmt.Printf("send message success: result=%s\n", res.String()) 50 | } 51 | } 52 | err = p.Shutdown() 53 | if err != nil { 54 | fmt.Printf("shutdown producer error: %s", err.Error()) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /examples/golang/tag/consumer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "time" 22 | 23 | "github.com/apache/rocketmq-client-go/v2" 24 | "github.com/apache/rocketmq-client-go/v2/consumer" 25 | "github.com/apache/rocketmq-client-go/v2/primitive" 26 | ) 27 | 28 | func main() { 29 | c, _ := rocketmq.NewPushConsumer( 30 | consumer.WithGroupName("testGroup"), 31 | consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 32 | consumer.WithNamespace("test1|InstanceTest"), 33 | ) 34 | selector := consumer.MessageSelector{ 35 | Type: consumer.TAG, 36 | Expression: "TagA || TagC", 37 | } 38 | err := c.Subscribe("test_tag", selector, func(ctx context.Context, 39 | msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) { 40 | fmt.Printf("subscribe callback: %v \n", msgs) 41 | return consumer.ConsumeSuccess, nil 42 | }) 43 | if err != nil { 44 | fmt.Println(err.Error()) 45 | } 46 | err = c.Start() 47 | if err != nil { 48 | fmt.Println(err.Error()) 49 | os.Exit(-1) 50 | } 51 | time.Sleep(time.Hour) 52 | err = c.Shutdown() 53 | if err != nil { 54 | fmt.Printf("shutdown Consumer error: %s", err.Error()) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /examples/golang/namespace/producer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | 22 | "github.com/apache/rocketmq-client-go/v2" 23 | "github.com/apache/rocketmq-client-go/v2/primitive" 24 | "github.com/apache/rocketmq-client-go/v2/producer" 25 | ) 26 | 27 | func main() { 28 | p, err := rocketmq.NewProducer( 29 | producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 30 | producer.WithRetry(2), 31 | producer.WithNamespace("test1|InstanceTest"), 32 | ) 33 | 34 | if err != nil { 35 | fmt.Println("init producer error: " + err.Error()) 36 | os.Exit(0) 37 | } 38 | 39 | err = p.Start() 40 | if err != nil { 41 | fmt.Printf("start producer error: %s", err.Error()) 42 | os.Exit(1) 43 | } 44 | for i := 0; i < 100; i++ { 45 | res, err := p.SendSync(context.Background(), primitive.NewMessage("test", 46 | []byte("Hello RocketMQ Go Client!"))) 47 | 48 | if err != nil { 49 | fmt.Printf("send message error: %s\n", err) 50 | } else { 51 | fmt.Printf("send message success: result=%s\n", res.String()) 52 | } 53 | } 54 | err = p.Shutdown() 55 | if err != nil { 56 | fmt.Printf("shutdown producer error: %s", err.Error()) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /examples/golang/simple/producer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "strconv" 22 | 23 | "github.com/apache/rocketmq-client-go/v2" 24 | "github.com/apache/rocketmq-client-go/v2/primitive" 25 | "github.com/apache/rocketmq-client-go/v2/producer" 26 | ) 27 | 28 | // Package main implements a simple producer to send message. 29 | func main() { 30 | p, _ := rocketmq.NewProducer( 31 | producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 32 | producer.WithRetry(2), 33 | ) 34 | err := p.Start() 35 | if err != nil { 36 | fmt.Printf("start producer error: %s", err.Error()) 37 | os.Exit(1) 38 | } 39 | topic := "test" 40 | 41 | for i := 0; i < 10; i++ { 42 | msg := &primitive.Message{ 43 | Topic: topic, 44 | Body: []byte("Hello RocketMQ Go Client! " + strconv.Itoa(i)), 45 | } 46 | res, err := p.SendSync(context.Background(), msg) 47 | 48 | if err != nil { 49 | fmt.Printf("send message error: %s\n", err) 50 | } else { 51 | fmt.Printf("send message success: result=%s\n", res.String()) 52 | } 53 | } 54 | err = p.Shutdown() 55 | if err != nil { 56 | fmt.Printf("shutdown producer error: %s", err.Error()) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /examples/golang/broadcast/consumer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "time" 22 | 23 | "github.com/apache/rocketmq-client-go/v2" 24 | "github.com/apache/rocketmq-client-go/v2/consumer" 25 | "github.com/apache/rocketmq-client-go/v2/primitive" 26 | ) 27 | 28 | func main() { 29 | c, _ := rocketmq.NewPushConsumer( 30 | consumer.WithGroupName("testGroup"), 31 | consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 32 | consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset), 33 | consumer.WithConsumerModel(consumer.BroadCasting), 34 | ) 35 | err := c.Subscribe("min", consumer.MessageSelector{}, func(ctx context.Context, 36 | msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) { 37 | fmt.Printf("subscribe callback: %v \n", msgs) 38 | return consumer.ConsumeSuccess, nil 39 | }) 40 | if err != nil { 41 | fmt.Println(err.Error()) 42 | } 43 | // Note: start after subscribe 44 | err = c.Start() 45 | if err != nil { 46 | fmt.Println(err.Error()) 47 | os.Exit(-1) 48 | } 49 | time.Sleep(time.Hour) 50 | err = c.Shutdown() 51 | if err != nil { 52 | fmt.Printf("Shutdown Consumer error: %s", err.Error()) 53 | } 54 | } 55 | 56 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/listener/NotifyMessageArrivingListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.listener; 16 | 17 | import java.util.Map; 18 | import org.apache.rocketmq.store.MessageArrivingListener; 19 | import org.streamnative.pulsar.handlers.rocketmq.inner.PullRequestHoldService; 20 | 21 | /** 22 | * Notify message arriving listener. 23 | */ 24 | public class NotifyMessageArrivingListener implements MessageArrivingListener { 25 | 26 | private final PullRequestHoldService pullRequestHoldService; 27 | 28 | public NotifyMessageArrivingListener(final PullRequestHoldService pullRequestHoldService) { 29 | this.pullRequestHoldService = pullRequestHoldService; 30 | } 31 | 32 | @Override 33 | public void arriving(String topic, int partitionId, long logicOffset, long tagsCode, 34 | long msgStoreTime, byte[] filterBitMap, Map properties) { 35 | this.pullRequestHoldService.updateMessageOffset(topic, partitionId, logicOffset); 36 | this.pullRequestHoldService.notifyMessageArriving(topic, partitionId, logicOffset, tagsCode, 37 | msgStoreTime, filterBitMap, properties); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/bean/RopOffsetWrapper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package org.streamnative.pulsar.handlers.rocketmq.inner.bean; 15 | 16 | /** 17 | * Rop offset wrapper. 18 | */ 19 | public class RopOffsetWrapper { 20 | private long brokerOffset; 21 | private long consumerOffset; 22 | 23 | private long lastTimestamp; 24 | 25 | private long msgBacklog; 26 | 27 | public long getBrokerOffset() { 28 | return brokerOffset; 29 | } 30 | 31 | public void setBrokerOffset(long brokerOffset) { 32 | this.brokerOffset = brokerOffset; 33 | } 34 | 35 | public long getConsumerOffset() { 36 | return consumerOffset; 37 | } 38 | 39 | public void setConsumerOffset(long consumerOffset) { 40 | this.consumerOffset = consumerOffset; 41 | } 42 | 43 | public long getLastTimestamp() { 44 | return lastTimestamp; 45 | } 46 | 47 | public void setLastTimestamp(long lastTimestamp) { 48 | this.lastTimestamp = lastTimestamp; 49 | } 50 | 51 | public long getMsgBacklog() { 52 | return msgBacklog; 53 | } 54 | 55 | public void setMsgBacklog(long msgBacklog) { 56 | this.msgBacklog = msgBacklog; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /examples/golang/namespace/consumer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "time" 22 | 23 | "github.com/apache/rocketmq-client-go/v2" 24 | "github.com/apache/rocketmq-client-go/v2/consumer" 25 | "github.com/apache/rocketmq-client-go/v2/primitive" 26 | ) 27 | 28 | func main() { 29 | c, err := rocketmq.NewPushConsumer( 30 | consumer.WithGroupName("testGroup"), 31 | consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 32 | consumer.WithNamespace("test1|InstanceTest"), 33 | ) 34 | if err != nil { 35 | fmt.Println("init consumer error: " + err.Error()) 36 | os.Exit(0) 37 | } 38 | 39 | err = c.Subscribe("test", consumer.MessageSelector{}, func(ctx context.Context, 40 | msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) { 41 | fmt.Printf("subscribe callback: %v \n", msgs) 42 | return consumer.ConsumeSuccess, nil 43 | }) 44 | if err != nil { 45 | fmt.Println(err.Error()) 46 | } 47 | // Note: start after subscribe 48 | err = c.Start() 49 | if err != nil { 50 | fmt.Println(err.Error()) 51 | os.Exit(-1) 52 | } 53 | time.Sleep(time.Hour) 54 | err = c.Shutdown() 55 | if err != nil { 56 | fmt.Printf("Shutdown Consumer error: %s", err.Error()) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/batch/SimpleBatchProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.batch; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import org.apache.rocketmq.client.producer.DefaultMQProducer; 20 | import org.apache.rocketmq.common.message.Message; 21 | 22 | /** 23 | * Simple batch producer. 24 | */ 25 | public class SimpleBatchProducer { 26 | 27 | public static void main(String[] args) throws Exception { 28 | DefaultMQProducer producer = new DefaultMQProducer("BatchProducerGroupName"); 29 | producer.start(); 30 | 31 | //If you just send messages of no more than 1MiB at a time, it is easy to use batch 32 | //Messages of the same batch should have: same topic, same waitStoreMsgOK and no schedule support 33 | String topic = "BatchTest"; 34 | List messages = new ArrayList<>(); 35 | messages.add(new Message(topic, "Tag", "OrderID001", "Hello world 0".getBytes("GBK"))); 36 | messages.add(new Message(topic, "Tag", "OrderID002", "Hello world 1".getBytes("GBK"))); 37 | messages.add(new Message(topic, "Tag", "OrderID003", "Hello world 2".getBytes("GBK"))); 38 | 39 | producer.send(messages); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/simple/LitePullConsumerSubscribe.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.simple; 16 | 17 | import java.util.List; 18 | import org.apache.rocketmq.client.consumer.DefaultLitePullConsumer; 19 | import org.apache.rocketmq.common.consumer.ConsumeFromWhere; 20 | import org.apache.rocketmq.common.message.MessageExt; 21 | 22 | /** 23 | * Lite Pull Consumer Subscribe. 24 | */ 25 | public class LitePullConsumerSubscribe { 26 | 27 | public static volatile boolean running = true; 28 | 29 | public static void main(String[] args) throws Exception { 30 | DefaultLitePullConsumer litePullConsumer = new DefaultLitePullConsumer("lite_pull_consumer_test"); 31 | litePullConsumer.setNamesrvAddr("127.0.0.1:9876"); 32 | litePullConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); 33 | litePullConsumer.subscribe("topicTest", "*"); 34 | litePullConsumer.start(); 35 | try { 36 | while (running) { 37 | List messageExts = litePullConsumer.poll(); 38 | System.out.printf("%s%n", messageExts); 39 | } 40 | } finally { 41 | litePullConsumer.shutdown(); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/format/RopMessageFilter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.format; 16 | 17 | import io.netty.buffer.ByteBuf; 18 | import java.util.function.Predicate; 19 | import org.apache.rocketmq.common.filter.ExpressionType; 20 | import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; 21 | 22 | /** 23 | * Rop message filter. 24 | */ 25 | public class RopMessageFilter implements Predicate { 26 | 27 | protected final SubscriptionData subscriptionData; 28 | 29 | public RopMessageFilter(SubscriptionData subscriptionData) { 30 | this.subscriptionData = subscriptionData; 31 | } 32 | 33 | @Override 34 | public boolean test(ByteBuf payload) { 35 | if (this.subscriptionData != null && payload != null 36 | && ExpressionType.isTagType(subscriptionData.getExpressionType())) { 37 | if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) { 38 | return true; 39 | } 40 | 41 | long tagsCode = payload.getLong(payload.readerIndex()); 42 | return subscriptionData.getCodeSet().contains((int) tagsCode); 43 | } 44 | return true; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/simple/SenderWithTags.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.simple; 16 | 17 | import org.apache.rocketmq.client.producer.DefaultMQProducer; 18 | import org.apache.rocketmq.client.producer.SendResult; 19 | import org.apache.rocketmq.common.message.Message; 20 | 21 | /** 22 | * Sender with tags. 23 | */ 24 | public class SenderWithTags { 25 | 26 | public static void main(String[] args) throws Exception { 27 | 28 | DefaultMQProducer producer = new DefaultMQProducer("test1|InstanceTest5", "pidTest"); 29 | producer.start(); 30 | String[] tags = {"tagA", "tagB"}; 31 | for (int i = 1; i < 100; i++) { 32 | Message message = new Message("topicTest", tags[i % 2], ("Hello world — " + i) 33 | .getBytes("GBK")); 34 | try { 35 | SendResult result = producer.send(message); 36 | System.out.printf("Topic:%s send success, misId is:%s, queueId is: %s%n", message.getTopic(), 37 | result.getMsgId(), result.getMessageQueue().getQueueId()); 38 | } catch (Exception e) { 39 | e.printStackTrace(); 40 | } 41 | } 42 | producer.shutdown(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tests/src/test/java/org/streamnative/pulsar/handlers/rocketmq/PortManager.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq; 16 | 17 | import java.net.ServerSocket; 18 | 19 | /** 20 | * Port manager allows a base port to be specified on the commandline. 21 | * Tests will then use ports, counting up from this base port. 22 | * This allows multiple instances of the bookkeeper tests to run at once. 23 | */ 24 | public class PortManager { 25 | 26 | private static int nextPort = getBasePort(); 27 | 28 | public static synchronized int nextFreePort() { 29 | int exceptionCount = 0; 30 | while (true) { 31 | int port = nextPort++; 32 | try (ServerSocket ss = new ServerSocket(port)) { 33 | ss.close(); 34 | //Give it some time to truly close the connection 35 | Thread.sleep(100); 36 | return port; 37 | } catch (Exception e) { 38 | exceptionCount++; 39 | if (exceptionCount > 5) { 40 | throw new RuntimeException(e); 41 | } 42 | } 43 | } 44 | } 45 | 46 | private static int getBasePort() { 47 | return Integer.valueOf(System.getProperty("test.basePort", "15000")); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/simple/RoundRobinProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.simple; 16 | 17 | import org.apache.rocketmq.client.producer.DefaultMQProducer; 18 | import org.apache.rocketmq.client.producer.SendResult; 19 | import org.apache.rocketmq.common.message.Message; 20 | 21 | /** 22 | * Round robin producer. 23 | */ 24 | public class RoundRobinProducer { 25 | 26 | public static void main(String[] args) throws Exception { 27 | 28 | DefaultMQProducer producer = new DefaultMQProducer("test1|InstanceTest5", "pidTest"); 29 | producer.setNamesrvAddr("127.0.0.1:9876"); 30 | producer.start(); 31 | for (int i = 1; i < 100; i++) { 32 | Message message = new Message("topicTest", "tagTest", ("Hello world — " + i) 33 | .getBytes("GBK")); 34 | try { 35 | SendResult result = producer.send(message, 300000); 36 | System.out.printf("Topic:%s send success, misId is:%s, queueId is: %s%n", message.getTopic(), 37 | result.getMsgId(), result.getMessageQueue().getQueueId()); 38 | } catch (Exception e) { 39 | e.printStackTrace(); 40 | } 41 | } 42 | producer.shutdown(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/timer/TimerTask.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.timer; 16 | 17 | import org.streamnative.pulsar.handlers.rocketmq.inner.timer.TimerTaskList.TimerTaskEntry; 18 | 19 | /** 20 | * Timer task. 21 | */ 22 | public abstract class TimerTask implements Runnable { 23 | 24 | protected final long delayMs; 25 | private volatile TimerTaskEntry timerTaskEntry = null; 26 | 27 | protected TimerTask(long delayMs) { 28 | this.delayMs = delayMs; 29 | } 30 | 31 | public synchronized void cancel() { 32 | if (null != timerTaskEntry) { 33 | timerTaskEntry.remove(); 34 | timerTaskEntry = null; 35 | } 36 | } 37 | 38 | TimerTaskEntry getTimerTaskEntry() { 39 | return timerTaskEntry; 40 | } 41 | 42 | void setTimerTaskEntry(TimerTaskEntry entry) { 43 | synchronized (this) { 44 | // if this timerTask is already held by an existing timer task entry, 45 | // we will remove such an entry first. 46 | if (null != timerTaskEntry && timerTaskEntry != entry) { 47 | timerTaskEntry.remove(); 48 | } 49 | timerTaskEntry = entry; 50 | } 51 | 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /rocketmq-impl/src/test/java/org/streamnative/pulsar/handlers/rocketmq/inner/consumer/metadata/GroupOffsetKeyTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.consumer.metadata; 16 | import static org.junit.Assert.assertEquals; 17 | 18 | import java.nio.ByteBuffer; 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | import org.streamnative.pulsar.handlers.rocketmq.inner.exception.RopDecodeException; 22 | import org.streamnative.pulsar.handlers.rocketmq.inner.exception.RopEncodeException; 23 | 24 | /** 25 | * Test GroupOffsetValueTest Class. 26 | */ 27 | public class GroupOffsetKeyTest { 28 | private GroupOffsetKey groupOffsetKey; 29 | 30 | @Before 31 | public void init() { 32 | groupOffsetKey = new GroupOffsetKey(); 33 | groupOffsetKey.setVersion((short) 2); 34 | groupOffsetKey.setGroupName("my_first_group"); 35 | groupOffsetKey.setTopicName("my_first_topic"); 36 | groupOffsetKey.setPulsarPartitionId(10); 37 | } 38 | 39 | @Test 40 | public void encodeDecodeTest() throws RopEncodeException, RopDecodeException { 41 | ByteBuffer buffer = groupOffsetKey.encode(); 42 | buffer.rewind(); 43 | GroupMetaKey metaKey = GroupMetaKey.decodeKey(buffer); 44 | assertEquals(groupOffsetKey, metaKey); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/zookeeper/RopZkUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.zookeeper; 16 | 17 | /** 18 | * Rop path for zookeeper. 19 | */ 20 | public final class RopZkUtils { 21 | 22 | public static final String ROP_PATH = "/rop"; 23 | 24 | public static final String COORDINATOR_PATH = ROP_PATH + "/coordinator"; 25 | 26 | public static final String BROKERS_PATH = ROP_PATH + "/brokers"; 27 | 28 | public static final String TOPIC_BASE_PATH = ROP_PATH + "/topics"; 29 | 30 | public static final String TOPIC_BASE_PATH_MATCH = TOPIC_BASE_PATH + "/%s"; 31 | 32 | public static final String GROUP_BASE_PATH = ROP_PATH + "/groups"; 33 | 34 | public static final String GROUP_BASE_PATH_MATCH = GROUP_BASE_PATH + "/%s"; 35 | 36 | public static final String BROKER_CLUSTER_PATH = ROP_PATH + "/brokerCluster"; 37 | 38 | public static String getTopicZNodePath(String topicName) { 39 | return String.format(TOPIC_BASE_PATH_MATCH, topicName); 40 | } 41 | 42 | public static String getGroupZNodePath(String groupName) { 43 | return String.format(GROUP_BASE_PATH_MATCH, groupName); 44 | } 45 | 46 | public static String getClusterZNodePath() { 47 | return BROKER_CLUSTER_PATH; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /examples/golang/acl/producer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | 22 | "github.com/apache/rocketmq-client-go/v2" 23 | "github.com/apache/rocketmq-client-go/v2/primitive" 24 | "github.com/apache/rocketmq-client-go/v2/producer" 25 | ) 26 | 27 | func main() { 28 | p, err := rocketmq.NewProducer( 29 | producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 30 | producer.WithRetry(2), 31 | producer.WithCredentials(primitive.Credentials{ 32 | AccessKey: "RocketMQ", 33 | SecretKey: "12345678", 34 | }), 35 | producer.WithNamespace("test1|InstanceTest"), 36 | ) 37 | 38 | if err != nil { 39 | fmt.Println("init producer error: " + err.Error()) 40 | os.Exit(0) 41 | } 42 | 43 | err = p.Start() 44 | if err != nil { 45 | fmt.Printf("start producer error: %s", err.Error()) 46 | os.Exit(1) 47 | } 48 | for i := 0; i < 100000; i++ { 49 | res, err := p.SendSync(context.Background(), primitive.NewMessage("test_acl", 50 | []byte("Hello RocketMQ Go Client!"))) 51 | 52 | if err != nil { 53 | fmt.Printf("send message error: %s\n", err) 54 | } else { 55 | fmt.Printf("send message success: result=%s\n", res.String()) 56 | } 57 | } 58 | err = p.Shutdown() 59 | if err != nil { 60 | fmt.Printf("shutdown producer error: %s", err.Error()) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /rocketmq-impl/src/test/java/org/streamnative/pulsar/handlers/rocketmq/inner/consumer/metadata/GroupOffsetValueTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.consumer.metadata; 16 | import static org.junit.Assert.assertEquals; 17 | 18 | import java.nio.ByteBuffer; 19 | import java.util.Date; 20 | import org.junit.Before; 21 | import org.junit.Test; 22 | import org.streamnative.pulsar.handlers.rocketmq.inner.exception.RopDecodeException; 23 | import org.streamnative.pulsar.handlers.rocketmq.inner.exception.RopEncodeException; 24 | 25 | /** 26 | * Test GroupOffsetValueTest Class. 27 | */ 28 | public class GroupOffsetValueTest { 29 | private GroupOffsetValue groupOffsetValue; 30 | 31 | @Before 32 | public void init() { 33 | groupOffsetValue = new GroupOffsetValue(); 34 | groupOffsetValue.setOffset(1234L); 35 | groupOffsetValue.setCommitTimestamp(new Date().getTime()); 36 | groupOffsetValue.setExpireTimestamp(new Date().getTime() + 1000); 37 | } 38 | 39 | @Test 40 | public void encodeDecodeTest() throws RopEncodeException, RopDecodeException { 41 | ByteBuffer buffer = groupOffsetValue.encode(); 42 | buffer.rewind(); 43 | GroupOffsetValue temp = new GroupOffsetValue(); 44 | temp.decode(buffer); 45 | assertEquals(groupOffsetValue, temp); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tests/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /examples/golang/acl/consumer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "time" 22 | 23 | "github.com/apache/rocketmq-client-go/v2" 24 | "github.com/apache/rocketmq-client-go/v2/consumer" 25 | "github.com/apache/rocketmq-client-go/v2/primitive" 26 | ) 27 | 28 | func main() { 29 | c, err := rocketmq.NewPushConsumer( 30 | consumer.WithGroupName("testGroup"), 31 | consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 32 | consumer.WithCredentials(primitive.Credentials{ 33 | AccessKey: "RocketMQ", 34 | SecretKey: "12345678", 35 | }), 36 | consumer.WithNamespace("test1|InstanceTest"), 37 | ) 38 | if err != nil { 39 | fmt.Println("init consumer error: " + err.Error()) 40 | os.Exit(0) 41 | } 42 | 43 | err = c.Subscribe("test_acl", consumer.MessageSelector{}, func(ctx context.Context, 44 | msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) { 45 | fmt.Printf("subscribe callback: %v \n", msgs) 46 | return consumer.ConsumeSuccess, nil 47 | }) 48 | if err != nil { 49 | fmt.Println(err.Error()) 50 | } 51 | // Note: start after subscribe 52 | err = c.Start() 53 | if err != nil { 54 | fmt.Println(err.Error()) 55 | os.Exit(-1) 56 | } 57 | time.Sleep(time.Hour) 58 | err = c.Shutdown() 59 | if err != nil { 60 | fmt.Printf("Shutdown Consumer error: %s", err.Error()) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /examples/golang/orderly/consumer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "time" 22 | 23 | "github.com/apache/rocketmq-client-go/v2" 24 | "github.com/apache/rocketmq-client-go/v2/consumer" 25 | "github.com/apache/rocketmq-client-go/v2/primitive" 26 | ) 27 | 28 | func main() { 29 | c, _ := rocketmq.NewPushConsumer( 30 | consumer.WithGroupName("testGroup"), 31 | consumer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 32 | consumer.WithConsumerModel(consumer.Clustering), 33 | consumer.WithConsumeFromWhere(consumer.ConsumeFromFirstOffset), 34 | consumer.WithConsumerOrder(true), 35 | ) 36 | err := c.Subscribe("TopicTest", consumer.MessageSelector{}, func(ctx context.Context, 37 | msgs ...*primitive.MessageExt) (consumer.ConsumeResult, error) { 38 | orderlyCtx, _ := primitive.GetOrderlyCtx(ctx) 39 | fmt.Printf("orderly context: %v\n", orderlyCtx) 40 | fmt.Printf("subscribe orderly callback: %v \n", msgs) 41 | return consumer.ConsumeSuccess, nil 42 | }) 43 | if err != nil { 44 | fmt.Println(err.Error()) 45 | } 46 | // Note: start after subscribe 47 | err = c.Start() 48 | if err != nil { 49 | fmt.Println(err.Error()) 50 | os.Exit(-1) 51 | } 52 | time.Sleep(time.Hour) 53 | err = c.Shutdown() 54 | if err != nil { 55 | fmt.Printf("Shutdown Consumer error: %s", err.Error()) 56 | } 57 | } 58 | 59 | -------------------------------------------------------------------------------- /rocketmq-impl/src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /examples/golang/async/producer.go: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "fmt" 20 | "os" 21 | "sync" 22 | 23 | "github.com/apache/rocketmq-client-go/v2" 24 | "github.com/apache/rocketmq-client-go/v2/primitive" 25 | "github.com/apache/rocketmq-client-go/v2/producer" 26 | ) 27 | 28 | // Package main implements a async producer to send message. 29 | func main() { 30 | p, _ := rocketmq.NewProducer( 31 | producer.WithNsResolver(primitive.NewPassthroughResolver([]string{"127.0.0.1:9876"})), 32 | producer.WithRetry(2), 33 | producer.WithNamespace("test1|InstanceTest")) 34 | 35 | err := p.Start() 36 | if err != nil { 37 | fmt.Printf("start producer error: %s", err.Error()) 38 | os.Exit(1) 39 | } 40 | var wg sync.WaitGroup 41 | for i := 0; i < 10; i++ { 42 | wg.Add(1) 43 | err := p.SendAsync(context.Background(), 44 | func(ctx context.Context, result *primitive.SendResult, e error) { 45 | if e != nil { 46 | fmt.Printf("receive message error: %s\n", err) 47 | } else { 48 | fmt.Printf("send message success: result=%s\n", result.String()) 49 | } 50 | wg.Done() 51 | }, primitive.NewMessage("topic-1", []byte("Hello RocketMQ Go Client!"))) 52 | 53 | if err != nil { 54 | fmt.Printf("send message error: %s\n", err) 55 | } 56 | } 57 | wg.Wait() 58 | err = p.Shutdown() 59 | if err != nil { 60 | fmt.Printf("shutdown producer error: %s", err.Error()) 61 | } 62 | } 63 | 64 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/utils/OffsetSearchPredicate.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.utils; 16 | 17 | import org.apache.bookkeeper.mledger.Entry; 18 | import org.apache.pulsar.common.api.proto.BrokerEntryMetadata; 19 | import org.apache.pulsar.common.protocol.Commands; 20 | import org.checkerframework.checker.nullness.qual.Nullable; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | /** 25 | * Predicate for find position for a given offset(index). 26 | */ 27 | public class OffsetSearchPredicate implements com.google.common.base.Predicate { 28 | private static final Logger log = LoggerFactory.getLogger(OffsetSearchPredicate.class); 29 | 30 | long indexToSearch = -1; 31 | public OffsetSearchPredicate(long indexToSearch) { 32 | this.indexToSearch = indexToSearch; 33 | } 34 | 35 | @Override 36 | public boolean apply(@Nullable Entry entry) { 37 | try { 38 | BrokerEntryMetadata brokerEntryMetadata = 39 | Commands.parseBrokerEntryMetadataIfExist(entry.getDataBuffer()); 40 | return brokerEntryMetadata.getIndex() < indexToSearch; 41 | } catch (Exception e) { 42 | log.error("Error deserialize message for message position find", e); 43 | } finally { 44 | entry.release(); 45 | } 46 | return false; 47 | } 48 | } 49 | 50 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/bean/RopConsumeStats.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | package org.streamnative.pulsar.handlers.rocketmq.inner.bean; 15 | 16 | import java.util.HashMap; 17 | import java.util.Map.Entry; 18 | import org.apache.rocketmq.common.message.MessageQueue; 19 | import org.apache.rocketmq.remoting.protocol.RemotingSerializable; 20 | 21 | /** 22 | * Rop consume stats. 23 | */ 24 | public class RopConsumeStats extends RemotingSerializable { 25 | private HashMap offsetTable = new HashMap(); 26 | private double consumeTps = 0; 27 | 28 | public long computeTotalDiff() { 29 | long diffTotal = 0L; 30 | 31 | for (Entry next : this.offsetTable.entrySet()) { 32 | long diff = next.getValue().getBrokerOffset() - next.getValue().getConsumerOffset(); 33 | diffTotal += diff; 34 | } 35 | 36 | return diffTotal; 37 | } 38 | 39 | public HashMap getOffsetTable() { 40 | return offsetTable; 41 | } 42 | 43 | public void setOffsetTable(HashMap offsetTable) { 44 | this.offsetTable = offsetTable; 45 | } 46 | 47 | public double getConsumeTps() { 48 | return consumeTps; 49 | } 50 | 51 | public void setConsumeTps(double consumeTps) { 52 | this.consumeTps = consumeTps; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/coordinator/RopBroker.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.coordinator; 16 | 17 | import lombok.extern.slf4j.Slf4j; 18 | import org.apache.zookeeper.ZooKeeper; 19 | import org.streamnative.pulsar.handlers.rocketmq.inner.RocketMQBrokerController; 20 | import org.streamnative.pulsar.handlers.rocketmq.inner.zookeeper.RopZkUtils; 21 | import org.streamnative.pulsar.handlers.rocketmq.utils.ZookeeperUtils; 22 | 23 | /** 24 | * Rop broker. 25 | */ 26 | @Slf4j 27 | public class RopBroker { 28 | 29 | private final RocketMQBrokerController brokerController; 30 | private ZooKeeper zkClient; 31 | private String zkNodePath; 32 | 33 | public RopBroker(RocketMQBrokerController brokerController) { 34 | this.brokerController = brokerController; 35 | } 36 | 37 | public void start() { 38 | log.info("Start RopBroker"); 39 | this.zkClient = brokerController.getBrokerService().pulsar().getZkClient(); 40 | this.zkNodePath = RopZkUtils.BROKERS_PATH + "/" + brokerController.getBrokerAddress(); 41 | ZookeeperUtils.createEphemeralNodeIfNotExist(zkClient, zkNodePath); 42 | } 43 | 44 | public void shutdown() { 45 | log.info("Shutdown RopBroker"); 46 | try { 47 | ZookeeperUtils.deleteData(zkClient, zkNodePath); 48 | } catch (Throwable t) { 49 | log.error("Delete rop broker zk node error", t); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/format/EntryFormatter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.format; 16 | 17 | import java.util.List; 18 | import java.util.function.Predicate; 19 | import org.apache.bookkeeper.mledger.Entry; 20 | import org.apache.pulsar.client.api.Message; 21 | import org.apache.pulsar.common.naming.TopicName; 22 | import org.apache.rocketmq.common.message.MessageExt; 23 | import org.apache.rocketmq.common.message.MessageExtBatch; 24 | import org.streamnative.pulsar.handlers.rocketmq.inner.RopMessage; 25 | import org.streamnative.pulsar.handlers.rocketmq.inner.exception.RopEncodeException; 26 | 27 | /** 28 | * The formatter for conversion between Kafka records and Bookie entries. 29 | */ 30 | public interface EntryFormatter { 31 | 32 | /** 33 | * Encode RocketMQ record/Batch records to a ByteBuf. 34 | * 35 | * @param record with RocketMQ's format 36 | * @param numMessages the number of messages 37 | * @return the ByteBuf of an entry that is to be written to Bookie 38 | */ 39 | List encode(final T record, final int numMessages) throws RopEncodeException; 40 | 41 | List encodeBatch(final MessageExtBatch record, boolean traceEnable) throws RopEncodeException; 42 | 43 | List decodePulsarMessage(final List> entries, Predicate predicate); 44 | 45 | MessageExt decodeMessageByPulsarEntry(TopicName pulsarPartitionedTopic, Entry msgEntry); 46 | } 47 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/metrics/FilteringJmxReporter.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.metrics; 16 | 17 | import com.yammer.metrics.core.Metric; 18 | import com.yammer.metrics.core.MetricName; 19 | import com.yammer.metrics.core.MetricsRegistry; 20 | import com.yammer.metrics.reporting.JmxReporter; 21 | import java.util.function.Predicate; 22 | 23 | /** 24 | * Rop JMX reporter. 25 | */ 26 | public class FilteringJmxReporter extends JmxReporter { 27 | 28 | private volatile Predicate metricPredicate; 29 | 30 | public FilteringJmxReporter(MetricsRegistry registry, Predicate metricPredicate) { 31 | super(registry); 32 | this.metricPredicate = metricPredicate; 33 | } 34 | 35 | @Override 36 | public void onMetricAdded(MetricName name, Metric metric) { 37 | if (metricPredicate.test(name)) { 38 | super.onMetricAdded(name, metric); 39 | } 40 | } 41 | 42 | public void updatePredicate(Predicate predicate) { 43 | this.metricPredicate = predicate; 44 | // re-register metrics on update 45 | getMetricsRegistry() 46 | .allMetrics() 47 | .forEach((name, metric) -> { 48 | if (metricPredicate.test(name)) { 49 | super.onMetricAdded(name, metric); 50 | } else { 51 | super.onMetricRemoved(name); 52 | } 53 | } 54 | ); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/InternalProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner; 16 | 17 | import java.util.Map; 18 | import java.util.Optional; 19 | import java.util.concurrent.CompletableFuture; 20 | import lombok.extern.slf4j.Slf4j; 21 | import org.apache.pulsar.broker.service.Producer; 22 | import org.apache.pulsar.broker.service.ServerCnx; 23 | import org.apache.pulsar.broker.service.Topic; 24 | import org.apache.pulsar.common.api.proto.ProducerAccessMode; 25 | 26 | /** 27 | * InternalProducer. 28 | */ 29 | @Slf4j 30 | public class InternalProducer extends Producer { 31 | 32 | private final ServerCnx cnx; 33 | 34 | public InternalProducer(Topic topic, ServerCnx cnx, long producerId, String producerName, 35 | Map metadata) { 36 | super(topic, cnx, producerId, producerName, null, 37 | false, metadata, null, 0, false, ProducerAccessMode.Shared, Optional.empty()); 38 | this.cnx = cnx; 39 | } 40 | 41 | // this will call back by bundle unload 42 | @Override 43 | public CompletableFuture disconnect() { 44 | return CompletableFuture.completedFuture(null); 45 | } 46 | 47 | @Override 48 | public ServerCnx getCnx() { 49 | return cnx; 50 | } 51 | 52 | @Override 53 | public boolean equals(Object obj) { 54 | return super.equals(obj); 55 | } 56 | 57 | @Override 58 | public int hashCode() { 59 | return super.hashCode(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /rocketmq-impl/src/test/java/org/streamnative/pulsar/handlers/rocketmq/inner/producer/ClientGroupNameTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.producer; 16 | 17 | import static org.junit.Assert.assertEquals; 18 | 19 | import org.apache.pulsar.common.naming.TopicName; 20 | import org.junit.Test; 21 | 22 | /** 23 | * Test client group name. 24 | */ 25 | public class ClientGroupNameTest { 26 | 27 | @Test 28 | public void testClientGroupName() { 29 | String oldGroupName = "test1|InstanceTest%cidTest"; 30 | 31 | ClientGroupName clientGroupName = new ClientGroupName(oldGroupName); 32 | assertEquals("test1|InstanceTest%cidTest", clientGroupName.getRmqGroupName()); 33 | assertEquals("test1/InstanceTest/cidTest", clientGroupName.getPulsarGroupName()); 34 | } 35 | 36 | @Test 37 | public void testClientGroupNameByTopicName() { 38 | TopicName topicName = TopicName.get("test/test-ns/test-group"); 39 | ClientGroupName clientGroupName = new ClientGroupName(topicName); 40 | 41 | assertEquals("test|test-ns%test-group", clientGroupName.getRmqGroupName()); 42 | assertEquals("test/test-ns/test-group", clientGroupName.getPulsarGroupName()); 43 | 44 | TopicName topicName1 = TopicName.get("test-group"); 45 | ClientGroupName clientGroupName1 = new ClientGroupName(topicName1); 46 | assertEquals("public|default%test-group", clientGroupName1.getRmqGroupName()); 47 | assertEquals("public/default/test-group", clientGroupName1.getPulsarGroupName()); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /rocketmq-impl/src/test/java/org/streamnative/pulsar/handlers/rocketmq/inner/producer/ClientTopicNameTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.producer; 16 | 17 | import static org.junit.Assert.assertEquals; 18 | 19 | import org.apache.pulsar.common.naming.TopicName; 20 | import org.junit.Test; 21 | 22 | /** 23 | * Test client topic name. 24 | */ 25 | public class ClientTopicNameTest { 26 | 27 | @Test 28 | public void testClientTopicName() { 29 | String oldTopicName = "test1|InstanceTest%cidTest"; 30 | 31 | ClientTopicName clientTopicName = new ClientTopicName(oldTopicName); 32 | assertEquals("test1|InstanceTest%cidTest", clientTopicName.getRmqTopicName()); 33 | assertEquals("test1/InstanceTest/cidTest", clientTopicName.getPulsarTopicName()); 34 | } 35 | 36 | @Test 37 | public void testClientTopicNameByTopicName() { 38 | TopicName topicName = TopicName.get("test/test-ns/test-topic"); 39 | ClientTopicName clientTopicName = new ClientTopicName(topicName); 40 | 41 | assertEquals("test|test-ns%test-topic", clientTopicName.getRmqTopicName()); 42 | assertEquals("test/test-ns/test-topic", clientTopicName.getPulsarTopicName()); 43 | 44 | TopicName topicName1 = TopicName.get("test-topic"); 45 | ClientTopicName clientTopicName1 = new ClientTopicName(topicName1); 46 | assertEquals("public|default%test-topic", clientTopicName1.getRmqTopicName()); 47 | assertEquals("public/default/test-topic", clientTopicName1.getPulsarTopicName()); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/RopClientChannelCnx.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner; 16 | 17 | import io.netty.channel.ChannelHandlerContext; 18 | import java.util.Objects; 19 | import lombok.Getter; 20 | import lombok.extern.slf4j.Slf4j; 21 | import org.apache.rocketmq.broker.client.ClientChannelInfo; 22 | import org.apache.rocketmq.remoting.protocol.LanguageCode; 23 | 24 | /** 25 | * Rop client channel cnx. 26 | */ 27 | @Slf4j 28 | public class RopClientChannelCnx extends ClientChannelInfo { 29 | 30 | @Getter 31 | private final RocketMQBrokerController brokerController; 32 | @Getter 33 | private final RopServerCnx serverCnx; 34 | 35 | public RopClientChannelCnx(RocketMQBrokerController brokerController, ChannelHandlerContext ctx) { 36 | this(brokerController, ctx, (String) null, (LanguageCode) null, 0); 37 | } 38 | 39 | public RopClientChannelCnx(RocketMQBrokerController brokerController, ChannelHandlerContext ctx, String clientId, 40 | LanguageCode language, int version) { 41 | super(ctx.channel(), clientId, language, version); 42 | this.brokerController = brokerController; 43 | this.serverCnx = new RopServerCnx(brokerController, ctx); 44 | } 45 | 46 | public String toString() { 47 | return super.toString(); 48 | } 49 | 50 | public boolean equals(Object obj) { 51 | return super.equals(obj); 52 | } 53 | 54 | @Override 55 | public int hashCode() { 56 | return Objects.hash(super.hashCode(), brokerController, serverCnx); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/zookeeper/RopZkClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.zookeeper; 16 | 17 | import lombok.Data; 18 | import lombok.extern.slf4j.Slf4j; 19 | import org.apache.zookeeper.WatchedEvent; 20 | import org.apache.zookeeper.Watcher; 21 | import org.apache.zookeeper.ZooKeeper; 22 | import org.streamnative.pulsar.handlers.rocketmq.inner.RocketMQBrokerController; 23 | import org.streamnative.pulsar.handlers.rocketmq.utils.ZookeeperUtils; 24 | 25 | /** 26 | * Rop Zk client. 27 | */ 28 | @Slf4j 29 | @Data 30 | public class RopZkClient implements Watcher { 31 | 32 | private final RocketMQBrokerController brokerController; 33 | private ZooKeeper zooKeeper; 34 | 35 | public RopZkClient(RocketMQBrokerController brokerController) { 36 | this.brokerController = brokerController; 37 | } 38 | 39 | public void start() { 40 | this.zooKeeper = brokerController.getBrokerService().pulsar().getZkClient(); 41 | 42 | // init rop zk node 43 | ZookeeperUtils.createPersistentNodeIfNotExist(zooKeeper, RopZkUtils.ROP_PATH); 44 | 45 | // init broker zk node 46 | ZookeeperUtils.createPersistentNodeIfNotExist(zooKeeper, RopZkUtils.BROKERS_PATH); 47 | 48 | // init topic zk node 49 | ZookeeperUtils.createPersistentNodeIfNotExist(zooKeeper, RopZkUtils.TOPIC_BASE_PATH); 50 | 51 | // init group zk node 52 | ZookeeperUtils.createPersistentNodeIfNotExist(zooKeeper, RopZkUtils.GROUP_BASE_PATH); 53 | } 54 | 55 | @Override 56 | public void process(WatchedEvent event) { 57 | 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/consumer/CommitLogOffset.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.consumer; 16 | 17 | import com.google.common.base.Preconditions; 18 | import lombok.Getter; 19 | import org.streamnative.pulsar.handlers.rocketmq.utils.CommitLogOffsetUtils; 20 | 21 | /** 22 | * commit log offset is used to lookup message in retry logic. 23 | * it's made up by 3 components, isRetryTopic + partitionId + offset 24 | */ 25 | @Getter 26 | public final class CommitLogOffset { 27 | 28 | private final boolean isRetryTopic; 29 | private final int partitionId; 30 | private final long queueOffset; 31 | 32 | public CommitLogOffset(boolean isRetryTopic, int partitionId, long queueOffset) { 33 | Preconditions 34 | .checkArgument(partitionId >= 0 && queueOffset >= 0, "partition id and queue offset must be >= 0."); 35 | this.isRetryTopic = isRetryTopic; 36 | this.partitionId = partitionId; 37 | this.queueOffset = queueOffset; 38 | } 39 | 40 | public CommitLogOffset(long commitLogOffset) { 41 | this.isRetryTopic = CommitLogOffsetUtils.isRetryTopic(commitLogOffset); 42 | this.partitionId = CommitLogOffsetUtils.getPartitionId(commitLogOffset); 43 | this.queueOffset = CommitLogOffsetUtils.getQueueOffset(commitLogOffset); 44 | } 45 | 46 | public long getCommitLogOffset() { 47 | long commitLogOffset = CommitLogOffsetUtils.setRetryTopicTag(queueOffset, isRetryTopic); 48 | commitLogOffset = CommitLogOffsetUtils.setPartitionId(commitLogOffset, partitionId); 49 | return commitLogOffset; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/springboot/SpringBootPushConsumeDemo.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.springboot; 16 | 17 | import lombok.extern.slf4j.Slf4j; 18 | import org.apache.rocketmq.spring.annotation.RocketMQMessageListener; 19 | import org.apache.rocketmq.spring.core.RocketMQListener; 20 | import org.springframework.boot.SpringApplication; 21 | import org.springframework.boot.autoconfigure.SpringBootApplication; 22 | import org.springframework.stereotype.Service; 23 | 24 | /** 25 | * SpringBoot push consume demo. 26 | */ 27 | @SpringBootApplication 28 | public class SpringBootPushConsumeDemo { 29 | 30 | public static void main(String[] args) { 31 | SpringApplication.run(SpringBootPushConsumeDemo.class, args); 32 | } 33 | 34 | @Slf4j 35 | @Service 36 | @RocketMQMessageListener( 37 | /* 使用主题全名 [namespace]%[topic], 例如：rocketmq-og7vojz5e33k|test-ns%test-topic */ 38 | topic = "rocketmq-og7vojz5e33k|test-ns%test-topic", 39 | /* 使用主题全名 [namespace]%[group], 例如：rocketmq-og7vojz5e33k|test-ns%test-group */ 40 | consumerGroup = "rocketmq-og7vojz5e33k|test-ns%test-group", 41 | /* 填写绑定的角色token */ 42 | accessKey = "eyJrZXlJZCI6InJvY2tldG1xLW9nN3Zvano1ZTMzayIsImFsZyI6IkhTMjU2In0.eyJzdWIiOiJyb2NrZXRtcS1vZzd2b2p6NWUzM2tfdGVzdC1yb2xlIn0.YPYh-7dB8lzH178b6_tP_mzdNiy6dbnpRR9RFijIT3U", 43 | /* 固定填写ROP即可 */ 44 | secretKey = "ROP" 45 | ) 46 | public static class Consumer implements RocketMQListener { 47 | 48 | public void onMessage(String message) { 49 | log.info("Received message: [{}]", message); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/simple/QueueSelectorProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.simple; 16 | 17 | import java.util.List; 18 | import org.apache.rocketmq.client.producer.DefaultMQProducer; 19 | import org.apache.rocketmq.client.producer.MessageQueueSelector; 20 | import org.apache.rocketmq.client.producer.SendResult; 21 | import org.apache.rocketmq.common.message.Message; 22 | import org.apache.rocketmq.common.message.MessageQueue; 23 | 24 | /** 25 | * Queue selector producer. 26 | */ 27 | public class QueueSelectorProducer { 28 | 29 | public static void main(String[] args) throws Exception { 30 | 31 | DefaultMQProducer producer = new DefaultMQProducer("test1|InstanceTest5", "pidTest"); 32 | producer.setNamesrvAddr("127.0.0.1:9876"); 33 | producer.start(); 34 | for (int i = 1; i < 100; i++) { 35 | Message message = new Message("topicTest", "tagTest", ("Hello world — " + i) 36 | .getBytes("GBK")); 37 | try { 38 | SendResult result = producer.send(message, new MessageQueueSelector() { 39 | @Override 40 | public MessageQueue select(List list, Message message, Object o) { 41 | return list.get(0); 42 | } 43 | }, null); 44 | System.out.printf("Topic:%s send success, misId is:%s, queueId is: %s%n", message.getTopic(), 45 | result.getMsgId(), result.getMessageQueue().getQueueId()); 46 | } catch (Exception e) { 47 | e.printStackTrace(); 48 | } 49 | } 50 | producer.shutdown(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/producer/ClientGroupName.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.producer; 16 | 17 | import static org.streamnative.pulsar.handlers.rocketmq.utils.CommonUtils.PERCENTAGE_CHAR; 18 | import static org.streamnative.pulsar.handlers.rocketmq.utils.CommonUtils.SLASH_CHAR; 19 | import static org.streamnative.pulsar.handlers.rocketmq.utils.CommonUtils.VERTICAL_LINE_CHAR; 20 | 21 | import com.google.common.base.Joiner; 22 | import java.io.Serializable; 23 | import lombok.Data; 24 | import lombok.EqualsAndHashCode; 25 | import lombok.ToString; 26 | import org.apache.pulsar.common.naming.TopicName; 27 | import org.streamnative.pulsar.handlers.rocketmq.utils.CommonUtils; 28 | 29 | /** 30 | * Client Group Name, consists of rmqGroupName and pulsarGroupName. 31 | */ 32 | @Data 33 | @EqualsAndHashCode 34 | @ToString 35 | public class ClientGroupName implements Serializable { 36 | 37 | private final String rmqGroupName; 38 | private final String pulsarGroupName; 39 | 40 | public ClientGroupName(String rmqGroupName) { 41 | this.rmqGroupName = rmqGroupName; 42 | this.pulsarGroupName = CommonUtils.pulsarGroupName(this.rmqGroupName); 43 | } 44 | 45 | public ClientGroupName(TopicName pulsarGroupName) { 46 | this.pulsarGroupName = Joiner.on(SLASH_CHAR) 47 | .join(pulsarGroupName.getTenant(), pulsarGroupName.getNamespacePortion(), 48 | pulsarGroupName.getLocalName()); 49 | 50 | this.rmqGroupName = 51 | pulsarGroupName.getTenant() + VERTICAL_LINE_CHAR + pulsarGroupName.getNamespacePortion() 52 | + PERCENTAGE_CHAR + pulsarGroupName 53 | .getLocalName(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/namespace/PushConsumerWithNamespace.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.namespace; 16 | 17 | import java.io.UnsupportedEncodingException; 18 | import java.util.concurrent.atomic.AtomicLong; 19 | import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; 20 | import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; 21 | import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; 22 | 23 | /** 24 | * Push consumer with namespace. 25 | */ 26 | public class PushConsumerWithNamespace { 27 | 28 | public static void main(String[] args) throws Exception { 29 | DefaultMQPushConsumer defaultMQPushConsumer = new DefaultMQPushConsumer( 30 | "test1|InstanceTest", "cidTest"); 31 | defaultMQPushConsumer.setNamesrvAddr("127.0.0.1:9876"); 32 | defaultMQPushConsumer.subscribe("topicTest", "*"); 33 | AtomicLong count = new AtomicLong(0L); 34 | defaultMQPushConsumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> { 35 | msgs.forEach((msg) -> { 36 | try { 37 | System.out.printf("Msg payload: %s, topic is:%s, MsgId is:%s, reconsumeTimes is:%s%n", 38 | new String(msg.getBody(), "GBK"), msg.getTopic(), msg.getMsgId(), msg.getReconsumeTimes()); 39 | } catch (UnsupportedEncodingException e) { 40 | e.printStackTrace(); 41 | } 42 | count.incrementAndGet(); 43 | System.out.println("total ====> " + count.get()); 44 | }); 45 | return ConsumeConcurrentlyStatus.RECONSUME_LATER; 46 | }); 47 | 48 | defaultMQPushConsumer.start(); 49 | } 50 | } -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/TransactionalMessageCheckService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner; 16 | 17 | import lombok.extern.slf4j.Slf4j; 18 | import org.apache.rocketmq.common.ServiceThread; 19 | 20 | /** 21 | * Transactional message check service. 22 | */ 23 | @Slf4j 24 | public class TransactionalMessageCheckService extends ServiceThread { 25 | 26 | private RocketMQBrokerController brokerController; 27 | 28 | public TransactionalMessageCheckService(RocketMQBrokerController brokerController) { 29 | this.brokerController = brokerController; 30 | } 31 | 32 | @Override 33 | public String getServiceName() { 34 | return TransactionalMessageCheckService.class.getSimpleName(); 35 | } 36 | 37 | @Override 38 | public void run() { 39 | log.info("Start transaction check service thread!"); 40 | long checkInterval = brokerController.getServerConfig().getTransactionCheckInterval(); 41 | while (!this.isStopped()) { 42 | this.waitForRunning(checkInterval); 43 | } 44 | log.info("End transaction check service thread!"); 45 | } 46 | 47 | @Override 48 | protected void onWaitEnd() { 49 | long timeout = brokerController.getServerConfig().getTransactionTimeOut(); 50 | int checkMax = brokerController.getServerConfig().getTransactionCheckMax(); 51 | long begin = System.currentTimeMillis(); 52 | log.info("Begin to check prepare message, begin time:{}", begin); 53 | this.brokerController.getTransactionalMessageService() 54 | .check(timeout, checkMax, this.brokerController.getTransactionalMessageCheckListener()); 55 | log.info("End to check prepare message, consumed time:{}", System.currentTimeMillis() - begin); 56 | } 57 | 58 | } 59 | 60 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/listener/DefaultConsumerIdsChangeListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.listener; 16 | 17 | import io.netty.channel.Channel; 18 | import java.util.List; 19 | import org.apache.rocketmq.broker.client.ConsumerGroupEvent; 20 | import org.apache.rocketmq.broker.client.ConsumerIdsChangeListener; 21 | import org.streamnative.pulsar.handlers.rocketmq.inner.RocketMQBrokerController; 22 | 23 | /** 24 | * Default consumerIds change listener. 25 | */ 26 | public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListener { 27 | 28 | private final RocketMQBrokerController brokerController; 29 | 30 | public DefaultConsumerIdsChangeListener(RocketMQBrokerController brokerController) { 31 | this.brokerController = brokerController; 32 | } 33 | 34 | @Override 35 | public void handle(ConsumerGroupEvent event, String group, Object... args) { 36 | if (event == null) { 37 | return; 38 | } 39 | switch (event) { 40 | case CHANGE: 41 | if (args == null || args.length < 1) { 42 | return; 43 | } 44 | List channels = (List) args[0]; 45 | if (channels != null && brokerController.getServerConfig().isNotifyConsumerIdsChangedEnable()) { 46 | for (Channel chl : channels) { 47 | this.brokerController.getBroker2Client().notifyConsumerIdsChanged(chl, group); 48 | } 49 | } 50 | break; 51 | case UNREGISTER: 52 | case REGISTER: 53 | break; 54 | default: 55 | throw new RuntimeException("Unknown event " + event); 56 | } 57 | } 58 | } 59 | 60 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/trace/TraceContext.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.trace; 16 | 17 | import io.netty.channel.ChannelHandlerContext; 18 | import lombok.AllArgsConstructor; 19 | import lombok.Data; 20 | import lombok.NoArgsConstructor; 21 | import org.apache.rocketmq.common.protocol.header.SendMessageRequestHeader; 22 | import org.apache.rocketmq.remoting.common.RemotingHelper; 23 | import org.streamnative.pulsar.handlers.rocketmq.utils.RocketMQTopic; 24 | 25 | /** 26 | * Trace context. 27 | */ 28 | @Data 29 | @AllArgsConstructor 30 | @NoArgsConstructor 31 | public class TraceContext { 32 | 33 | private String traceId; 34 | private String operationName; 35 | private long putStartTime; 36 | private long persistStartTime; 37 | private long endTime; 38 | private long duration; 39 | 40 | private String topic; 41 | private String group; 42 | private int partitionId; 43 | private String brokerTag; 44 | private long offset; 45 | private long queueId; 46 | private String msgId; 47 | private String offsetMsgId; 48 | private String pulsarMsgId; 49 | private String msgKey; 50 | private String instanceName; 51 | private String tags; 52 | private int code; 53 | 54 | private String messageModel; 55 | 56 | private boolean fromProxy; 57 | 58 | /** 59 | * Build message trace context. 60 | */ 61 | public static TraceContext buildMsgContext(ChannelHandlerContext ctx, SendMessageRequestHeader requestHeader) { 62 | TraceContext traceContext = new TraceContext(); 63 | traceContext.setTopic(new RocketMQTopic(requestHeader.getTopic()).getOrigNoDomainTopicName()); 64 | traceContext.setInstanceName(RemotingHelper.parseChannelRemoteAddr(ctx.channel())); 65 | return traceContext; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/delaymessage/DelayProduceDemo.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.delaymessage; 16 | 17 | import java.nio.charset.StandardCharsets; 18 | import java.util.UUID; 19 | import org.apache.rocketmq.client.producer.DefaultMQProducer; 20 | import org.apache.rocketmq.client.producer.SendResult; 21 | import org.apache.rocketmq.common.message.Message; 22 | 23 | /** 24 | * Delay producer demo. 25 | */ 26 | public class DelayProduceDemo { 27 | 28 | public static void main(String[] args) throws Exception { 29 | DefaultMQProducer producer = new DefaultMQProducer("test-producer-group"); 30 | producer.setInstanceName(UUID.randomUUID().toString()); 31 | producer.setNamesrvAddr("127.0.0.1:9876"); 32 | producer.setNamespace("test-cluster|test-ns"); 33 | producer.start(); 34 | 35 | int i = 0; 36 | while (!Thread.interrupted()) { 37 | try { 38 | Message msg = new Message("test-topic", 39 | ("No." + ++i + " Hello World").getBytes(StandardCharsets.UTF_8)); 40 | 41 | // Delayed messages, in milliseconds (ms), 42 | // are delivered after the specified delay time (after the current time), 43 | // for example, the message will be delivered after 10 seconds. 44 | long delayTime = System.currentTimeMillis() + 10000; 45 | // Set the time when the message needs to be delivered. 46 | msg.putUserProperty("__STARTDELIVERTIME", String.valueOf(delayTime)); 47 | 48 | SendResult result = producer.send(msg); 49 | System.out.println("Send delay message: " + i + " result：" + result); 50 | } catch (Exception e) { 51 | e.printStackTrace(); 52 | } 53 | Thread.sleep(1000); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/broadcast/PushConsumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.broadcast; 16 | 17 | import java.util.List; 18 | import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; 19 | import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyContext; 20 | import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; 21 | import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; 22 | import org.apache.rocketmq.client.exception.MQClientException; 23 | import org.apache.rocketmq.common.consumer.ConsumeFromWhere; 24 | import org.apache.rocketmq.common.message.MessageExt; 25 | import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; 26 | 27 | /** 28 | * Push consumer with broadcast. 29 | */ 30 | public class PushConsumer { 31 | 32 | public static void main(String[] args) throws InterruptedException, MQClientException { 33 | DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_1"); 34 | 35 | consumer.setNamesrvAddr("127.0.0.1:9876"); 36 | consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); 37 | 38 | consumer.setMessageModel(MessageModel.BROADCASTING); 39 | 40 | consumer.subscribe("TopicTest", "TagA || TagC || TagD"); 41 | 42 | consumer.registerMessageListener(new MessageListenerConcurrently() { 43 | 44 | @Override 45 | public ConsumeConcurrentlyStatus consumeMessage(List msgs, 46 | ConsumeConcurrentlyContext context) { 47 | System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); 48 | return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; 49 | } 50 | }); 51 | 52 | consumer.start(); 53 | System.out.printf("Broadcast Consumer Started.%n"); 54 | } 55 | } -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/consumer/RopGetMessageResult.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.consumer; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import lombok.Data; 20 | import lombok.ToString; 21 | import lombok.extern.slf4j.Slf4j; 22 | import org.apache.rocketmq.store.GetMessageStatus; 23 | import org.streamnative.pulsar.handlers.rocketmq.inner.RopMessage; 24 | 25 | /** 26 | * Rop get message result. 27 | */ 28 | @Data 29 | @ToString(exclude = "messageBufferList") 30 | @Slf4j 31 | public class RopGetMessageResult { 32 | 33 | private final List messageBufferList = new ArrayList<>(100); 34 | private GetMessageStatus status; 35 | private long nextBeginOffset; 36 | private long minOffset; 37 | private long maxOffset; 38 | private boolean suggestPullingFromSlave = false; 39 | private int msgCount4Commercial = 0; 40 | 41 | private String pulsarTopic; 42 | private int partitionId; 43 | private String instanceName; 44 | 45 | public int getMessageCount() { 46 | return messageBufferList.size(); 47 | } 48 | 49 | public int getBufferTotalSize() { 50 | return messageBufferList.stream().reduce(0, (r, item) -> 51 | r += item.getPayload().readableBytes() 52 | , Integer::sum); 53 | } 54 | 55 | public void addMessage(final RopMessage ropMessage) { 56 | messageBufferList.add(ropMessage); 57 | } 58 | 59 | public int size() { 60 | return messageBufferList.size(); 61 | } 62 | 63 | public void release() { 64 | for (RopMessage ropMessage : this.messageBufferList) { 65 | try { 66 | ropMessage.getPayload().release(); 67 | } catch (Exception e) { 68 | log.warn("RoP release get message failed.", e); 69 | } 70 | } 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/ordermessage/OrderConsumer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.ordermessage; 16 | 17 | import java.util.List; 18 | import java.util.concurrent.atomic.AtomicLong; 19 | import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; 20 | import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyContext; 21 | import org.apache.rocketmq.client.consumer.listener.ConsumeOrderlyStatus; 22 | import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; 23 | import org.apache.rocketmq.client.exception.MQClientException; 24 | import org.apache.rocketmq.common.consumer.ConsumeFromWhere; 25 | import org.apache.rocketmq.common.message.MessageExt; 26 | 27 | /** 28 | * Order message consumer example. 29 | */ 30 | public class OrderConsumer { 31 | 32 | public static void main(String[] args) throws MQClientException { 33 | DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("test1|InstanceTest", 34 | "please_rename_unique_group_name_3"); 35 | consumer.setNamesrvAddr("127.0.0.1:9876"); 36 | 37 | consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET); 38 | 39 | consumer.subscribe("OrderTopic", "*"); 40 | 41 | consumer.registerMessageListener(new MessageListenerOrderly() { 42 | AtomicLong consumeTimes = new AtomicLong(0); 43 | 44 | @Override 45 | public ConsumeOrderlyStatus consumeMessage(List msgs, ConsumeOrderlyContext context) { 46 | context.setAutoCommit(true); 47 | System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs); 48 | this.consumeTimes.incrementAndGet(); 49 | return ConsumeOrderlyStatus.SUCCESS; 50 | } 51 | }); 52 | 53 | consumer.start(); 54 | System.out.printf("Consumer Started.%n"); 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/producer/ClientTopicName.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.producer; 16 | 17 | import static org.streamnative.pulsar.handlers.rocketmq.utils.CommonUtils.SLASH_CHAR; 18 | import static org.streamnative.pulsar.handlers.rocketmq.utils.CommonUtils.VERTICAL_LINE_CHAR; 19 | 20 | import com.google.common.base.Joiner; 21 | import java.io.Serializable; 22 | import lombok.Data; 23 | import lombok.EqualsAndHashCode; 24 | import lombok.ToString; 25 | import org.apache.logging.log4j.util.Strings; 26 | import org.apache.pulsar.common.naming.TopicName; 27 | import org.apache.rocketmq.common.protocol.NamespaceUtil; 28 | import org.streamnative.pulsar.handlers.rocketmq.utils.CommonUtils; 29 | 30 | /** 31 | * Client topic name. 32 | */ 33 | @Data 34 | @EqualsAndHashCode 35 | @ToString 36 | public class ClientTopicName implements Serializable { 37 | 38 | private final String rmqTopicName; 39 | private final String pulsarTopicName; 40 | 41 | public ClientTopicName(String rmqTopicName) { 42 | this.rmqTopicName = rmqTopicName; 43 | this.pulsarTopicName = CommonUtils.pulsarGroupName(this.rmqTopicName); 44 | } 45 | 46 | public ClientTopicName(TopicName pulsarTopicName) { 47 | TopicName tempTopic = TopicName.get(pulsarTopicName.getPartitionedTopicName()); 48 | this.pulsarTopicName = Joiner.on(SLASH_CHAR) 49 | .join(tempTopic.getTenant(), tempTopic.getNamespacePortion(), tempTopic.getLocalName()); 50 | 51 | String rmqNamespace = tempTopic.getTenant() + VERTICAL_LINE_CHAR + tempTopic.getNamespacePortion(); 52 | this.rmqTopicName = NamespaceUtil.wrapNamespace(rmqNamespace, tempTopic.getLocalName()); 53 | } 54 | 55 | public TopicName toPulsarTopicName() { 56 | return TopicName.get(this.pulsarTopicName); 57 | } 58 | 59 | public boolean isDLQTopic() { 60 | return Strings.isNotBlank(toPulsarTopicName().getLocalName()) 61 | && NamespaceUtil.isDLQTopic(toPulsarTopicName().getLocalName()); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/InternalServerCnx.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner; 16 | 17 | import java.net.InetSocketAddress; 18 | import lombok.Getter; 19 | import lombok.extern.slf4j.Slf4j; 20 | import org.apache.pulsar.broker.service.Producer; 21 | import org.apache.pulsar.broker.service.ServerCnx; 22 | 23 | /** 24 | * InternalServerCnx, this only used to construct internalProducer / internalConsumer. 25 | */ 26 | @Slf4j 27 | public class InternalServerCnx extends ServerCnx { 28 | 29 | @Getter 30 | RopServerCnx ropServerCnx; 31 | 32 | public InternalServerCnx(RopServerCnx ropServerCnx) { 33 | super(ropServerCnx.getBrokerController().getBrokerService().pulsar()); 34 | this.ropServerCnx = ropServerCnx; 35 | // this is the client address that connect to this server. 36 | this.remoteAddress = ropServerCnx.getRemoteAddress(); 37 | 38 | // mock some values, or Producer create will meet NPE. 39 | // used in test, which will not call channel.active, and not call updateCtx. 40 | if (this.remoteAddress == null) { 41 | this.remoteAddress = new InetSocketAddress("localhost", 9999); 42 | } 43 | } 44 | 45 | // this will call back by bundle unload 46 | @Override 47 | public void closeProducer(Producer producer) { 48 | } 49 | 50 | // called after channel active 51 | public void updateCtx() { 52 | this.remoteAddress = ropServerCnx.getRemoteAddress(); 53 | } 54 | 55 | @Override 56 | public void enableCnxAutoRead() { 57 | // do nothing is this mock 58 | } 59 | 60 | @Override 61 | public void disableCnxAutoRead() { 62 | // do nothing is this mock 63 | } 64 | 65 | @Override 66 | public void cancelPublishBufferLimiting() { 67 | // do nothing is this mock 68 | } 69 | 70 | @Override 71 | public boolean equals(Object o) { 72 | return super.equals(o); 73 | } 74 | 75 | @Override 76 | public int hashCode() { 77 | return super.hashCode(); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/utils/CommitLogOffsetUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.utils; 16 | 17 | import com.google.common.base.Preconditions; 18 | import lombok.extern.slf4j.Slf4j; 19 | 20 | /** 21 | * RoP CommitLogOffset encoder and decoder. 22 | */ 23 | @Slf4j 24 | public class CommitLogOffsetUtils { 25 | 26 | // 1 bits reserved bit 27 | // use 1 bits to describe whether the topic is retry-topic, 28 | // 10 bits for pulsar real partitionId, 29 | // 52 bits for message offset. 30 | public static final int RESERVED_BITS = 1; 31 | public static final int RETRY_TOPIC_TAG_BITS = 1; 32 | public static final int PULSAR_PARTITION_ID_BITS = 10; 33 | public static final int OFFSET_BITS = Long.SIZE - (RESERVED_BITS + RETRY_TOPIC_TAG_BITS + PULSAR_PARTITION_ID_BITS); 34 | 35 | public static final long setRetryTopicTag(long commitLogOffset, boolean isRetryTopic) { 36 | return isRetryTopic ? (commitLogOffset | 0x4000000000000000L) : (commitLogOffset & 0x3FFFFFFFFFFFFFFFL); 37 | } 38 | 39 | public static final boolean isRetryTopic(long commitLogOffset) { 40 | return (commitLogOffset >>> (PULSAR_PARTITION_ID_BITS + OFFSET_BITS) & 0x01L) == 0x01L; 41 | } 42 | 43 | public static final long setPartitionId(long commitLogOffset, int partitionId) { 44 | Preconditions.checkArgument(partitionId >= 0 && partitionId < (1 << PULSAR_PARTITION_ID_BITS), 45 | "partitionId must be between 0 and 1024."); 46 | return commitLogOffset | (((long) partitionId) << OFFSET_BITS); 47 | } 48 | 49 | public static final int getPartitionId(long commitLogOffset) { 50 | return (int) ((commitLogOffset >>> OFFSET_BITS) & 0x3FFL); 51 | } 52 | 53 | public static final long setQueueOffset(long commitLogOffset, long queueOffset) { 54 | Preconditions.checkArgument(queueOffset >= 0); 55 | return ((commitLogOffset >>> OFFSET_BITS) << OFFSET_BITS) | queueOffset; 56 | } 57 | 58 | public static final long getQueueOffset(long commitLogOffset) { 59 | return commitLogOffset & ((1L << OFFSET_BITS) - 1); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/zookeeper/RopClusterContent.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.zookeeper; 16 | 17 | import com.google.common.base.Preconditions; 18 | import java.util.ArrayList; 19 | import java.util.Collections; 20 | import java.util.HashMap; 21 | import java.util.List; 22 | import java.util.Map; 23 | import java.util.Map.Entry; 24 | import lombok.Data; 25 | import lombok.EqualsAndHashCode; 26 | import lombok.ToString; 27 | import lombok.extern.slf4j.Slf4j; 28 | 29 | /** 30 | * Rop broker cluster info. 31 | */ 32 | @Slf4j 33 | @Data 34 | @EqualsAndHashCode 35 | @ToString 36 | public class RopClusterContent { 37 | 38 | private String clusterName; 39 | private Map/*ip:port*/> brokerCluster; 40 | 41 | public Map> createTopicRouteMap(int partitionNum) { 42 | Preconditions 43 | .checkArgument(brokerCluster.size() > 0, "Rop Cluster config haven't be set."); 44 | Preconditions.checkArgument(partitionNum > 0, "the num of top partition must be more than zero."); 45 | int brokerGroupNum = brokerCluster.entrySet().size(); 46 | int size = partitionNum / brokerGroupNum; 47 | int res = partitionNum % brokerGroupNum; 48 | Map assignedMap = new HashMap<>(); 49 | for (String brokerTag : brokerCluster.keySet()) { 50 | assignedMap.put(brokerTag, size + ((res--) > 0 ? 1 : 0)); 51 | } 52 | Map> result = new HashMap<>(brokerGroupNum); 53 | 54 | int total = 0; 55 | List> shuffledAssignedEntries = new ArrayList<>(assignedMap.entrySet()); 56 | Collections.shuffle(shuffledAssignedEntries); 57 | for (Entry entry : shuffledAssignedEntries) { 58 | for (int i = 0; i < entry.getValue(); i++) { 59 | List tempList = result.computeIfAbsent(entry.getKey(), k -> new ArrayList(entry.getValue())); 60 | tempList.add(i + total); 61 | } 62 | total += entry.getValue(); 63 | } 64 | return result; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/utils/FileRegionEncoder.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.utils; 16 | 17 | import io.netty.buffer.ByteBuf; 18 | import io.netty.channel.ChannelHandlerContext; 19 | import io.netty.channel.FileRegion; 20 | import io.netty.handler.codec.MessageToByteEncoder; 21 | import java.io.IOException; 22 | import java.nio.ByteBuffer; 23 | import java.nio.channels.WritableByteChannel; 24 | 25 | /** 26 | *

27 | * By default, file region are directly transferred to socket channel which is known as zero copy. 28 | * This encoder ensures this 29 | * process. 30 | *

31 | */ 32 | public class FileRegionEncoder extends MessageToByteEncoder { 33 | 34 | /** 35 | * Encode a message into a {@link ByteBuf}. This method will be called for each written message that 36 | * can be handled by this encoder. 37 | * 38 | * @param ctx the {@link ChannelHandlerContext} which this {@link 39 | * MessageToByteEncoder} belongs to 40 | * @param msg the message to encode 41 | * @param out the {@link ByteBuf} into which the encoded message will be written 42 | * @throws Exception is thrown if an error occurs 43 | */ 44 | @Override 45 | protected void encode(ChannelHandlerContext ctx, FileRegion msg, final ByteBuf out) throws Exception { 46 | WritableByteChannel writableByteChannel = new WritableByteChannel() { 47 | @Override 48 | public int write(ByteBuffer src) throws IOException { 49 | out.writeBytes(src); 50 | return out.capacity(); 51 | } 52 | 53 | @Override 54 | public boolean isOpen() { 55 | return true; 56 | } 57 | 58 | @Override 59 | public void close() throws IOException { 60 | } 61 | }; 62 | 63 | long toTransfer = msg.count(); 64 | 65 | while (true) { 66 | long transferred = msg.transfered(); 67 | if (toTransfer - transferred <= 0) { 68 | break; 69 | } 70 | msg.transferTo(writableByteChannel, transferred); 71 | } 72 | } 73 | } -------------------------------------------------------------------------------- /rocketmq-impl/src/assemble/bin.xml: -------------------------------------------------------------------------------- 1 | 16 | 17 | 21 | bin 22 | 23 | tar.gz 24 | 25 | true 26 | 27 | 28 | target 29 | lib 30 | 31 | ${project.artifactId}-${project.version}.jar 32 | 33 | 34 | 35 | ${basedir}/conf 36 | 37 | 38 | ${basedir}/bin 39 | 755 40 | 41 | 42 | 43 | 44 | 45 | ${basedir}/README.md 46 | README 47 | . 48 | 644 49 | 50 | 51 | ${basedir}/../LICENSE 52 | LICENSE 53 | . 54 | 644 55 | 56 | 57 | 58 | 59 | 60 | lib 61 | false 62 | compile 63 | false 64 | 66 | ${artifact.groupId}-${artifact.artifactId}-${artifact.version}${dashClassifier?}.${artifact.extension} 67 | 68 | 70 | junit:junit 71 | org.projectlombok:lombok 72 | 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/utils/CoreUtils.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.utils; 16 | 17 | import java.util.Map; 18 | import java.util.Map.Entry; 19 | import java.util.concurrent.locks.Lock; 20 | import java.util.concurrent.locks.ReadWriteLock; 21 | import java.util.function.Function; 22 | import java.util.function.Predicate; 23 | import java.util.function.Supplier; 24 | import java.util.stream.Collectors; 25 | import lombok.experimental.UtilityClass; 26 | 27 | /** 28 | * Core utils. 29 | */ 30 | @UtilityClass 31 | public final class CoreUtils { 32 | 33 | public static T inLock(Lock lock, Supplier supplier) { 34 | lock.lock(); 35 | try { 36 | return supplier.get(); 37 | } finally { 38 | lock.unlock(); 39 | } 40 | } 41 | 42 | public static T inReadLock(ReadWriteLock lock, Supplier supplier) { 43 | return inLock(lock.readLock(), supplier); 44 | } 45 | 46 | public static T inWriteLock(ReadWriteLock lock, Supplier supplier) { 47 | return inLock(lock.writeLock(), supplier); 48 | } 49 | 50 | public static Map> partition(Map map, 51 | Predicate predicate) { 52 | return map.entrySet() 53 | .stream() 54 | .collect(Collectors.partitioningBy( 55 | e -> predicate.test(e.getKey()), 56 | Collectors.toMap(Entry::getKey, Entry::getValue) 57 | )); 58 | } 59 | 60 | public static Map mapValue(Map map, 61 | Function func) { 62 | return map.entrySet() 63 | .stream() 64 | .collect(Collectors.toMap( 65 | e -> e.getKey(), 66 | e -> func.apply(e.getValue()) 67 | )); 68 | } 69 | 70 | public static Map mapKeyValue(Map map, 71 | Function, V2> func) { 72 | return map.entrySet() 73 | .stream() 74 | .collect(Collectors.toMap( 75 | e -> e.getKey(), 76 | e -> func.apply(e) 77 | )); 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /rocketmq-impl/src/test/java/org/streamnative/pulsar/handlers/rocketmq/inner/timer/MockTimer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.timer; 16 | 17 | import java.util.Comparator; 18 | import java.util.PriorityQueue; 19 | import org.streamnative.pulsar.handlers.rocketmq.inner.timer.TimerTaskList.TimerTaskEntry; 20 | 21 | /** 22 | * A mock implementation of {@link Timer}. 23 | */ 24 | public class MockTimer implements Timer { 25 | 26 | private final MockTime time = new MockTime(); 27 | private final PriorityQueue taskQueue = new PriorityQueue<>(Comparator.reverseOrder()); 28 | 29 | @Override 30 | public void add(TimerTask timerTask) { 31 | if (timerTask.delayMs <= 0) { 32 | timerTask.run(); 33 | } else { 34 | synchronized (taskQueue) { 35 | taskQueue.add( 36 | new TimerTaskEntry( 37 | timerTask, timerTask.delayMs + time.milliseconds())); 38 | } 39 | } 40 | } 41 | 42 | @Override 43 | public boolean advanceClock(long timeoutMs) { 44 | time.sleep(timeoutMs); 45 | 46 | boolean executed = false; 47 | final long now = time.milliseconds(); 48 | boolean hasMore = true; 49 | 50 | while (hasMore) { 51 | hasMore = false; 52 | TimerTaskEntry head; 53 | synchronized (taskQueue) { 54 | head = taskQueue.peek(); 55 | if (null != head && now > head.expirationMs()) { 56 | head = taskQueue.poll(); 57 | hasMore = !taskQueue.isEmpty(); 58 | } else { 59 | head = null; 60 | } 61 | } 62 | if (null != head) { 63 | if (!head.cancelled()) { 64 | TimerTask task = head.timerTask(); 65 | task.run(); 66 | executed = true; 67 | } 68 | } 69 | } 70 | 71 | return executed; 72 | } 73 | 74 | @Override 75 | public int size() { 76 | synchronized (taskQueue) { 77 | return taskQueue.size(); 78 | } 79 | } 80 | 81 | @Override 82 | public void shutdown() { 83 | // no-op 84 | } 85 | } 86 | 87 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/ordermessage/OrderProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.ordermessage; 16 | 17 | import java.io.UnsupportedEncodingException; 18 | import java.util.List; 19 | import org.apache.rocketmq.client.exception.MQBrokerException; 20 | import org.apache.rocketmq.client.exception.MQClientException; 21 | import org.apache.rocketmq.client.producer.DefaultMQProducer; 22 | import org.apache.rocketmq.client.producer.MessageQueueSelector; 23 | import org.apache.rocketmq.client.producer.SendResult; 24 | import org.apache.rocketmq.common.message.Message; 25 | import org.apache.rocketmq.common.message.MessageQueue; 26 | import org.apache.rocketmq.remoting.common.RemotingHelper; 27 | import org.apache.rocketmq.remoting.exception.RemotingException; 28 | 29 | /** 30 | * Order message producer example. 31 | */ 32 | public class OrderProducer { 33 | 34 | public static void main(String[] args) throws UnsupportedEncodingException { 35 | try { 36 | DefaultMQProducer producer = new DefaultMQProducer("test1|InstanceTest", "please_rename_unique_group_name"); 37 | producer.setNamesrvAddr("127.0.0.1:9876"); 38 | producer.start(); 39 | 40 | String[] tags = new String[]{"TagA", "TagB", "TagC", "TagD", "TagE"}; 41 | for (int i = 0; i < 10; i++) { 42 | int orderId = i % 10; 43 | Message msg = 44 | new Message("OrderTopic", tags[i % tags.length], "KEY" + i, 45 | ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET)); 46 | SendResult sendResult = producer.send(msg, new MessageQueueSelector() { 47 | @Override 48 | public MessageQueue select(List mqs, Message msg, Object arg) { 49 | Integer id = (Integer) arg; 50 | int index = id % mqs.size(); 51 | return mqs.get(index); 52 | } 53 | }, orderId); 54 | 55 | System.out.printf("%s%n", sendResult); 56 | } 57 | 58 | producer.shutdown(); 59 | } catch (MQClientException | RemotingException | MQBrokerException | InterruptedException e) { 60 | e.printStackTrace(); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /tests/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 18 | 19 | 22 | 23 | pulsar-protocol-handler-rocketmq-parent 24 | org.streamnative.pulsar.handlers 25 | 0.3.0-SNAPSHOT 26 | 27 | 4.0.0 28 | 29 | tests 30 | StreamNative :: Pulsar Protocol Handler :: RoP Tests 31 | Tests for RocketMQ on Pulsar 32 | 33 | 34 | 8 35 | 8 36 | 37 | 38 | 39 | 40 | 41 | org.streamnative.pulsar.handlers 42 | pulsar-protocol-handler-rocketmq 43 | ${project.version} 44 | test 45 | 46 | 47 | 48 | 49 | 50 | 51 | org.apache.maven.plugins 52 | maven-antrun-plugin 53 | 54 | 55 | install 56 | 57 | run 58 | 59 | 60 | 61 | copy protocol handler 62 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/pulsar/PulsarMessageStore.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.pulsar; 16 | 17 | import java.util.concurrent.CompletableFuture; 18 | import org.apache.rocketmq.common.message.MessageExt; 19 | import org.apache.rocketmq.common.message.MessageExtBatch; 20 | import org.apache.rocketmq.common.protocol.header.ConsumerSendMsgBackRequestHeader; 21 | import org.apache.rocketmq.common.protocol.header.PullMessageRequestHeader; 22 | import org.apache.rocketmq.remoting.protocol.RemotingCommand; 23 | import org.apache.rocketmq.store.MessageExtBrokerInner; 24 | import org.streamnative.pulsar.handlers.rocketmq.inner.PutMessageCallback; 25 | import org.streamnative.pulsar.handlers.rocketmq.inner.consumer.RopGetMessageResult; 26 | import org.streamnative.pulsar.handlers.rocketmq.inner.format.RopMessageFilter; 27 | 28 | /** 29 | * Pulsar message store interface. 30 | */ 31 | public interface PulsarMessageStore { 32 | 33 | void putMessage(int partition, MessageExtBrokerInner messageExtBrokerInner, String producerGroup, 34 | PutMessageCallback callback) throws Exception; 35 | 36 | void putSendBackMsg(MessageExtBrokerInner messageExtBrokerInner, String producerGroup, RemotingCommand response, 37 | CompletableFuture cmdFuture) 38 | throws Exception; 39 | 40 | RopGetMessageResult getMessage(int partition, RemotingCommand request, PullMessageRequestHeader requestHeader, 41 | RopMessageFilter messageFilter); 42 | 43 | void putMessages(int partitionID, MessageExtBatch batchMessage, String producerGroup, PutMessageCallback callback, 44 | boolean traceEnable) throws Exception; 45 | 46 | MessageExt lookMessageByMessageId(String topic, String msgId); 47 | 48 | MessageExt lookMessageByMessageId(String topic, long offset); 49 | 50 | MessageExt lookMessageByCommitLogOffset(ConsumerSendMsgBackRequestHeader requestHeader); 51 | 52 | /** 53 | * Reset the subscription associated with this reader to a specific message publish time. 54 | * 55 | * @param topic the sub-partitioned topic(is one topic) 56 | * @param timestamp the message publish time where to reposition the reader 57 | * @return Return rocketmq MessageExt object 58 | */ 59 | MessageExt lookMessageByTimestamp(String topic, long timestamp); 60 | 61 | long now(); 62 | } 63 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/consumer/metadata/GroupOffsetConstant.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.consumer.metadata; 16 | 17 | /** 18 | * Group offset constant. 19 | */ 20 | public final class GroupOffsetConstant { 21 | //group meta fields index 22 | public static final int GROUP_META_VERSION_POS = 0; 23 | public static final int GROUP_META_KEY_TYPE_POS = GROUP_META_VERSION_POS + 2; 24 | public static final int GROUP_META_KEY_GRP_NAME_LEN_POS = GROUP_META_KEY_TYPE_POS + 4; 25 | public static final int GROUP_META_KEY_TOTAL_HEAD_LEN = GROUP_META_KEY_GRP_NAME_LEN_POS + 4; 26 | 27 | //group offset key fields index 28 | public static final int GROUP_OFFSET_KEY_PARTITION_POS = 0; 29 | public static final int GROUP_OFFSET_KEY_TOPIC_NAME_LEN_POS = GROUP_OFFSET_KEY_PARTITION_POS + 4; 30 | public static final int GROUP_OFFSET_KEY_TOTAL_HEAD_LEN = GROUP_OFFSET_KEY_TOPIC_NAME_LEN_POS + 4; 31 | 32 | //group offset value fields index 33 | public static final int GROUP_OFFSET_VALUE_VERSION_POS = 0; 34 | public static final int GROUP_OFFSET_VALUE_OFFSET_POS = GROUP_OFFSET_VALUE_VERSION_POS + 2; 35 | public static final int GROUP_OFFSET_VALUE_COMMIT_TIMESTAMP_POS = GROUP_OFFSET_VALUE_OFFSET_POS + 8; 36 | public static final int GROUP_OFFSET_VALUE_EXPIRE_TIMESTAMP_POS = GROUP_OFFSET_VALUE_COMMIT_TIMESTAMP_POS + 8; 37 | public static final int GROUP_OFFSET_VALUE_TOTAL_LEN = GROUP_OFFSET_VALUE_EXPIRE_TIMESTAMP_POS + 8; 38 | 39 | //subscription value fields index 40 | public static final int GROUP_SUBSCRIPTION_FORMAT_VERSION_POS = 0; 41 | public static final int GROUP_SUBSCRIPTION_KEY_TAG_POS = GROUP_SUBSCRIPTION_FORMAT_VERSION_POS + 2; 42 | public static final int GROUP_SUBSCRIPTION_KEY_RETRY_QUEUE_NUM_POS = GROUP_SUBSCRIPTION_KEY_TAG_POS + 2; 43 | public static final int GROUP_SUBSCRIPTION_KEY_RETRY_MAX_TIMES_POS = GROUP_SUBSCRIPTION_KEY_RETRY_QUEUE_NUM_POS + 4; 44 | public static final int GROUP_SUBSCRIPTION_KEY_BROKER_ID_POS = GROUP_SUBSCRIPTION_KEY_RETRY_MAX_TIMES_POS + 4; 45 | public static final int GROUP_SUBSCRIPTION_KEY_BROKER_SELECTED_POS = GROUP_SUBSCRIPTION_KEY_BROKER_ID_POS + 8; 46 | public static final int GROUP_SUBSCRIPTION_KEY_GRP_NAME_LEN_POS = GROUP_SUBSCRIPTION_KEY_BROKER_SELECTED_POS + 8; 47 | public static final int GROUP_SUBSCRIPTION_KEY_TOTAL_HEAD_LEN = GROUP_SUBSCRIPTION_KEY_GRP_NAME_LEN_POS + 4; 48 | 49 | } 50 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/RocketMQChannelInitializer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq; 16 | 17 | import io.netty.channel.ChannelInitializer; 18 | import io.netty.channel.socket.SocketChannel; 19 | import io.netty.handler.timeout.IdleStateHandler; 20 | import lombok.Getter; 21 | import lombok.extern.slf4j.Slf4j; 22 | import org.apache.pulsar.broker.service.BrokerService; 23 | import org.apache.rocketmq.remoting.netty.NettyDecoder; 24 | import org.streamnative.pulsar.handlers.rocketmq.inner.RocketMQBrokerController; 25 | import org.streamnative.pulsar.handlers.rocketmq.inner.RocketMQRemoteServer; 26 | 27 | /** 28 | * rocketmq output data encoder. 29 | */ 30 | @Slf4j 31 | public class RocketMQChannelInitializer extends ChannelInitializer { 32 | 33 | @Getter 34 | private final BrokerService brokerService; 35 | @Getter 36 | private final RocketMQServiceConfiguration rocketmqConfig; 37 | @Getter 38 | private final RocketMQBrokerController brokerController; 39 | @Getter 40 | private final RocketMQRemoteServer remoteServer; 41 | @Getter 42 | private final boolean enableTls; 43 | 44 | 45 | public RocketMQChannelInitializer(RocketMQServiceConfiguration rocketmqConfig, 46 | RocketMQBrokerController rocketmqBroker, BrokerService brokerService, 47 | boolean enableTls) { 48 | super(); 49 | this.rocketmqConfig = rocketmqConfig; 50 | this.brokerController = rocketmqBroker; 51 | this.brokerService = brokerService; 52 | this.enableTls = enableTls; 53 | this.remoteServer = rocketmqBroker.getRemotingServer(); 54 | } 55 | 56 | 57 | @Override 58 | protected void initChannel(SocketChannel ch) throws Exception { 59 | ch.pipeline() 60 | .addLast(remoteServer.getDefaultEventExecutorGroup(), remoteServer.HANDSHAKE_HANDLER_NAME, 61 | remoteServer.getHandshakeHandler()) 62 | .addLast(remoteServer.getDefaultEventExecutorGroup(), 63 | remoteServer.getEncoder(), 64 | new NettyDecoder(), 65 | new IdleStateHandler(0, 0, rocketmqConfig.getServerChannelMaxIdleTimeSeconds()), 66 | remoteServer.getConnectionManageHandler(), 67 | remoteServer.getServerHandler() 68 | ); 69 | 70 | log.info("Successfully init channel {}", ch.remoteAddress().getHostString()); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/quickstart/SimpleProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.quickstart; 16 | 17 | import org.apache.rocketmq.client.exception.MQClientException; 18 | import org.apache.rocketmq.client.producer.DefaultMQProducer; 19 | import org.apache.rocketmq.client.producer.SendResult; 20 | import org.apache.rocketmq.common.message.Message; 21 | import org.apache.rocketmq.remoting.common.RemotingHelper; 22 | 23 | /** 24 | * This class demonstrates how to send messages to brokers using provided {@link DefaultMQProducer}. 25 | */ 26 | public class SimpleProducer { 27 | 28 | public static void main(String[] args) throws MQClientException, InterruptedException { 29 | 30 | /* 31 | * Instantiate with a producer group name. 32 | */ 33 | DefaultMQProducer producer = new DefaultMQProducer("please_rename_unique_group_name"); 34 | 35 | /* 36 | * Specify name server addresses. 37 | *

38 | * 39 | * Alternatively, you may specify name server addresses via exporting environmental variable: NAMESRV_ADDR 40 | * 

41 |          * {@code
42 |          * producer.setNamesrvAddr("name-server1-ip:9876;name-server2-ip:9876");
43 |          * }
44 |          *
45 | */ 46 | 47 | producer.setNamesrvAddr("127.0.0.1:9876"); 48 | 49 | /* 50 | * Launch the instance. 51 | */ 52 | producer.start(); 53 | 54 | for (int i = 0; i < 200000000; i++) { 55 | try { 56 | 57 | /* 58 | * Create a message instance, specifying topic, tag and message body. 59 | */ 60 | Message msg = new Message("topicTest2" /* Topic */, 61 | "TagA" /* Tag */, 62 | ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */ 63 | ); 64 | 65 | /* 66 | * Call send message to deliver message to one of brokers. 67 | */ 68 | SendResult sendResult = producer.send(msg); 69 | 70 | System.out.printf("%s%n", sendResult); 71 | Thread.sleep(500); 72 | } catch (Exception e) { 73 | e.printStackTrace(); 74 | Thread.sleep(500); 75 | } 76 | } 77 | 78 | /* 79 | * Shut down once the producer instance is not longer in use. 80 | */ 81 | producer.shutdown(); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /rocketmq-impl/src/main/java/org/streamnative/pulsar/handlers/rocketmq/inner/trace/TraceStatsReportService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.inner.trace; 16 | 17 | import java.io.File; 18 | import java.util.concurrent.Executors; 19 | import java.util.concurrent.ScheduledFuture; 20 | import java.util.concurrent.TimeUnit; 21 | import java.util.concurrent.atomic.AtomicBoolean; 22 | import java.util.concurrent.atomic.AtomicLong; 23 | import lombok.extern.slf4j.Slf4j; 24 | import org.streamnative.pulsar.handlers.rocketmq.inner.RocketMQBrokerController; 25 | import org.testng.util.Strings; 26 | 27 | /** 28 | * Trace stats report service. 29 | */ 30 | @Slf4j 31 | public class TraceStatsReportService implements Runnable { 32 | 33 | private final AtomicLong writeDiskCounter = new AtomicLong(0); 34 | private final AtomicBoolean diskCanWrite = new AtomicBoolean(true); 35 | 36 | private ScheduledFuture scheduledReportExecutor; 37 | private File logDir; 38 | 39 | public void boot() { 40 | scheduledReportExecutor = Executors.newSingleThreadScheduledExecutor() 41 | .scheduleAtFixedRate(this, 30, 30, TimeUnit.SECONDS); 42 | } 43 | 44 | public void shutdown() { 45 | scheduledReportExecutor.cancel(true); 46 | } 47 | 48 | 49 | public void logWriteToDisk(long count) { 50 | writeDiskCounter.addAndGet(count); 51 | } 52 | 53 | public boolean canWriteDisk() { 54 | return this.diskCanWrite.get(); 55 | } 56 | 57 | @Override 58 | public void run() { 59 | try { 60 | if (logDir == null && !Strings.isNullOrEmpty(RocketMQBrokerController.ropTraceLogDir())) { 61 | logDir = new File(RocketMQBrokerController.ropTraceLogDir()); 62 | } 63 | if (logDir != null) { 64 | int remainDiskPercentage = (int) ((((double) logDir.getFreeSpace()) / logDir.getTotalSpace()) * 100); 65 | log.info("RoP trace remaining disk usage:{}%", remainDiskPercentage); 66 | if (remainDiskPercentage < 10) { 67 | diskCanWrite.set(false); 68 | log.warn("RoP trace stop writing disk due to free space is low."); 69 | } else { 70 | if (diskCanWrite.compareAndSet(false, true)) { 71 | log.info("RoP trace restore writing."); 72 | } 73 | } 74 | } 75 | } catch (Exception e) { 76 | log.info("RoP trace disk check error.", e); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /examples/src/main/java/org/streamnative/rocketmq/example/simple/AsyncProducer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.rocketmq.example.simple; 16 | 17 | import java.io.UnsupportedEncodingException; 18 | import java.util.concurrent.CountDownLatch; 19 | import java.util.concurrent.TimeUnit; 20 | import org.apache.rocketmq.client.exception.MQClientException; 21 | import org.apache.rocketmq.client.producer.DefaultMQProducer; 22 | import org.apache.rocketmq.client.producer.SendCallback; 23 | import org.apache.rocketmq.client.producer.SendResult; 24 | import org.apache.rocketmq.common.message.Message; 25 | import org.apache.rocketmq.remoting.common.RemotingHelper; 26 | 27 | /** 28 | * Async producer. 29 | */ 30 | public class AsyncProducer { 31 | 32 | public static void main( 33 | String[] args) throws MQClientException, InterruptedException, UnsupportedEncodingException { 34 | 35 | DefaultMQProducer producer = new DefaultMQProducer("Jodie_Daily_test"); 36 | producer.setNamesrvAddr("127.0.0.1:9876"); 37 | producer.start(); 38 | producer.setRetryTimesWhenSendAsyncFailed(0); 39 | 40 | int messageCount = 100; 41 | final CountDownLatch countDownLatch = new CountDownLatch(messageCount); 42 | for (int i = 0; i < messageCount; i++) { 43 | try { 44 | final int index = i; 45 | Message msg = new Message("Jodie_topic_1023", 46 | "TagA", 47 | "OrderID188", 48 | "Hello world".getBytes(RemotingHelper.DEFAULT_CHARSET)); 49 | producer.send(msg, new SendCallback() { 50 | @Override 51 | public void onSuccess(SendResult sendResult) { 52 | countDownLatch.countDown(); 53 | System.out.printf("%-10d OK %s %n", index, sendResult.getMsgId()); 54 | } 55 | 56 | @Override 57 | public void onException(Throwable e) { 58 | countDownLatch.countDown(); 59 | System.out.printf("%-10d Exception %s %n", index, e); 60 | e.printStackTrace(); 61 | } 62 | }); 63 | } catch (Exception e) { 64 | e.printStackTrace(); 65 | } 66 | } 67 | try { 68 | countDownLatch.await(5, TimeUnit.SECONDS); 69 | } catch (InterruptedException e) { 70 | e.printStackTrace(); 71 | } 72 | producer.shutdown(); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /rocketmq-impl/src/test/java/org/streamnative/pulsar/handlers/rocketmq/utils/TopicNameUtilsTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed under the Apache License, Version 2.0 (the "License"); 3 | * you may not use this file except in compliance with the License. 4 | * You may obtain a copy of the License at 5 | * 6 | * http://www.apache.org/licenses/LICENSE-2.0 7 | * 8 | * Unless required by applicable law or agreed to in writing, software 9 | * distributed under the License is distributed on an "AS IS" BASIS, 10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | * See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package org.streamnative.pulsar.handlers.rocketmq.utils; 16 | 17 | import static org.apache.pulsar.common.naming.TopicName.PARTITIONED_TOPIC_SUFFIX; 18 | import static org.testng.Assert.assertEquals; 19 | 20 | import lombok.extern.slf4j.Slf4j; 21 | import org.apache.pulsar.common.naming.NamespaceName; 22 | import org.apache.pulsar.common.naming.TopicName; 23 | import org.apache.rocketmq.common.message.MessageQueue; 24 | import org.testng.annotations.Test; 25 | 26 | /** 27 | * Validate TopicNameUtils. 28 | */ 29 | @Slf4j 30 | public class TopicNameUtilsTest { 31 | 32 | @Test(timeOut = 20000) 33 | public void testTopicNameConvert() throws Exception { 34 | String topicName = "kopTopicNameConvert"; 35 | int partitionNumber = 77; 36 | MessageQueue topicPartition = new MessageQueue(topicName, "test-cluster", partitionNumber); 37 | 38 | String tenantName = "tenant_name"; 39 | String nsName = "ns_name"; 40 | NamespaceName ns = NamespaceName.get(tenantName, nsName); 41 | String expectedPulsarName = "persistent://" + tenantName + "/" + nsName + "/" 42 | + topicName + PARTITIONED_TOPIC_SUFFIX + partitionNumber; 43 | 44 | TopicName topicName1 = TopicNameUtils.pulsarTopicName(topicPartition, ns); 45 | TopicName topicName2 = TopicNameUtils.pulsarTopicName(topicName, partitionNumber, ns); 46 | 47 | assertEquals(expectedPulsarName, topicName1.toString()); 48 | assertEquals(expectedPulsarName, topicName2.toString()); 49 | 50 | TopicName topicName3 = TopicNameUtils.pulsarTopicName(topicName, ns); 51 | String expectedPulsarName3 = "persistent://" + tenantName + "/" + nsName + "/" 52 | + topicName; 53 | assertEquals(expectedPulsarName3, topicName3.toString()); 54 | } 55 | 56 | @Test 57 | public void testParseTopicName() { 58 | String topicName1 = "test-topic-1"; 59 | String resTopic1 = TopicNameUtils.parseTopicName(topicName1); 60 | assertEquals(resTopic1, "persistent://rocketmq/__rocketmq/test-topic-1"); 61 | 62 | String topicName2 = "tenant/namespace/test-topic-2"; 63 | String resTopic2 = TopicNameUtils.parseTopicName(topicName2); 64 | assertEquals(resTopic2, "persistent://tenant/namespace/test-topic-2"); 65 | 66 | String topicName3 = "persistent://tenant/namespace/test-topic-3"; 67 | String resTopic3 = TopicNameUtils.parseTopicName(topicName3); 68 | assertEquals(resTopic3, topicName3); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /examples/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 18 | 19 | 22 | 23 | pulsar-protocol-handler-rocketmq-parent 24 | org.streamnative.pulsar.handlers 25 | 0.3.0-SNAPSHOT 26 | 27 | 4.0.0 28 | 29 | examples 30 | 31 | 32 | 8 33 | 8 34 | 35 | 36 | 37 | 38 | org.apache.rocketmq 39 | rocketmq-spring-boot-starter 40 | 2.2.0 41 | 42 | 43 | 44 | javax.validation 45 | validation-api 46 | 2.0.1.Final 47 | compile 48 | 49 | 50 | 51 | 52 | 53 | 54 | org.apache.maven.plugins 55 | maven-assembly-plugin 56 | 3.3.0 57 | 58 | 59 | jar-with-dependencies 60 | 61 | 62 | 63 | 64 | make-assembly 65 | package 66 | 67 | single 68 | 69 | 70 | 71 | 72 | 73 | org.apache.maven.plugins 74 | maven-checkstyle-plugin 75 | 76 | true 77 | 78 | 79 | 80 | 81 | 82 | --------------------------------------------------------------------------------