├── .github └── workflows │ ├── pre-commit-config.yaml │ └── rust.yml ├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── LICENSE ├── README.md ├── bin └── robustctl ├── build.rs ├── config ├── cluster │ ├── placement-center-1.toml │ ├── placement-center-2.toml │ └── placement-center-3.toml ├── log4rs.yaml ├── log4rs.yaml.template └── placement-center.toml ├── docs └── image1.png ├── licenserc.toml ├── logs ├── requests-log.log └── server.log ├── makefile ├── scripts └── integration-testing.sh ├── src ├── clients │ ├── Cargo.toml │ ├── src │ │ ├── lib.rs │ │ ├── placement │ │ │ ├── kv │ │ │ │ ├── call.rs │ │ │ │ ├── inner.rs │ │ │ │ └── mod.rs │ │ │ ├── mod.rs │ │ │ └── openraft │ │ │ │ ├── call.rs │ │ │ │ ├── inner.rs │ │ │ │ └── mod.rs │ │ └── poll.rs │ └── tests │ │ └── test.rs ├── cmd │ ├── Cargo.toml │ ├── src │ │ └── placement-center │ │ │ └── server.rs │ └── tests │ │ └── test.rs ├── common │ └── base │ │ ├── Cargo.toml │ │ ├── src │ │ ├── config │ │ │ ├── mod.rs │ │ │ └── placement_center.rs │ │ ├── errors.rs │ │ ├── http_error.rs │ │ ├── http_response.rs │ │ ├── lib.rs │ │ ├── log │ │ │ ├── mod.rs │ │ │ └── placement_center.rs │ │ └── tools.rs │ │ └── tests │ │ └── test.rs ├── placement-center │ ├── Cargo.toml │ ├── src │ │ ├── lib.rs │ │ ├── openraft │ │ │ ├── error.rs │ │ │ ├── mod.rs │ │ │ ├── network │ │ │ │ ├── connection.rs │ │ │ │ ├── mod.rs │ │ │ │ └── network.rs │ │ │ ├── raft_node.rs │ │ │ ├── route │ │ │ │ └── mod.rs │ │ │ ├── store │ │ │ │ ├── log_store.rs │ │ │ │ ├── mod.rs │ │ │ │ └── state_machine_store.rs │ │ │ └── typeconfig.rs │ │ ├── raft │ │ │ ├── apply.rs │ │ │ ├── machine.rs │ │ │ ├── metadata.rs │ │ │ ├── mod.rs │ │ │ ├── node.rs │ │ │ ├── peer.rs │ │ │ ├── route.rs │ │ │ └── storage.rs │ │ ├── requests │ │ │ └── mod.rs │ │ ├── server │ │ │ ├── grpc │ │ │ │ ├── mod.rs │ │ │ │ ├── server.rs │ │ │ │ ├── services_kv.rs │ │ │ │ ├── services_kv_new.rs │ │ │ │ ├── services_openraft.rs │ │ │ │ └── services_raft.rs │ │ │ ├── http │ │ │ │ ├── index.rs │ │ │ │ ├── mod.rs │ │ │ │ ├── openraft.rs │ │ │ │ └── server.rs │ │ │ └── mod.rs │ │ └── storage │ │ │ ├── engine.rs │ │ │ ├── keys.rs │ │ │ ├── kv.rs │ │ │ ├── mod.rs │ │ │ ├── raft.rs │ │ │ └── rocksdb.rs │ └── tests │ │ ├── kv_server.rs │ │ └── test.rs └── protocol │ ├── Cargo.toml │ ├── src │ ├── common.proto │ ├── common.rs │ ├── kv.proto │ ├── kv.rs │ ├── lib.rs │ ├── openraft.proto │ ├── openraft.rs │ ├── placement.proto │ └── placement.rs │ └── tests │ ├── protobuf.rs │ └── test.rs └── version.ini /.github/workflows/pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | fail_fast: false 2 | repos: 3 | - repo: https://github.com/pre-commit/pre-commit-hooks 4 | rev: v4.6.0 5 | hooks: 6 | - id: check-byte-order-marker 7 | - id: check-case-conflict 8 | - id: check-merge-conflict 9 | - id: check-symlinks 10 | - id: check-yaml 11 | - id: end-of-file-fixer 12 | - id: mixed-line-ending 13 | - id: trailing-whitespace 14 | - repo: https://github.com/commitizen-tools/commitizen 15 | rev: v3.28.0 16 | hooks: 17 | - id: commitizen 18 | - repo: https://github.com/psf/black 19 | rev: 24.8.0 20 | hooks: 21 | - id: black 22 | - repo: local 23 | hooks: 24 | - id: cargo-fmt 25 | name: cargo fmt 26 | description: Format files with rustfmt. 27 | entry: bash -c 'cargo fmt -- --check' 28 | language: rust 29 | files: \.rs$ 30 | args: [] 31 | - id: cargo-deny 32 | name: cargo deny check 33 | description: Check cargo dependencies 34 | entry: bash -c 'cargo deny check -d' 35 | language: rust 36 | files: \.rs$ 37 | args: [] 38 | - id: typos 39 | name: typos 40 | description: check typo 41 | entry: bash -c 'typos' 42 | language: rust 43 | files: \.*$ 44 | pass_filenames: false 45 | - id: cargo-check 46 | name: cargo check 47 | description: Check the package for errors. 48 | entry: bash -c 'cargo check --all' 49 | language: rust 50 | files: \.rs$ 51 | pass_filenames: false 52 | - id: cargo-clippy 53 | name: cargo clippy 54 | description: Lint rust sources 55 | entry: bash -c 'cargo clippy --all-targets --all-features --tests --benches -- -D warnings' 56 | language: rust 57 | files: \.rs$ 58 | pass_filenames: false 59 | - id: cargo-test 60 | name: cargo test 61 | description: unit test for the project 62 | entry: bash -c 'cargo nextest run --all-features' 63 | language: rust 64 | files: \.rs$ 65 | pass_filenames: false 66 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ "main" ] 6 | pull_request: 7 | branches: [ "main" ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Build 20 | run: cargo build --verbose 21 | - name: Run tests 22 | run: cargo test --verbose 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | src/cmd/target 3 | build 4 | .vscode 5 | ./log 6 | .DS_Store 7 | log/* 8 | src/rlog/log/server.log 9 | .idea/.gitignore 10 | .idea/modules.xml 11 | .idea/robustmq.iml 12 | .idea/vcs.xml 13 | logs -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [workspace] 16 | members = [ 17 | "src/common/base", 18 | "src/placement-center", 19 | "src/cmd", 20 | "src/protocol", 21 | "src/clients", 22 | ] 23 | 24 | resolver = "2" 25 | 26 | [workspace.package] 27 | version = "0.0.1" 28 | edition = "2021" 29 | license = "Apache-2.0" 30 | 31 | [workspace.dependencies] 32 | serde = { version = "1.0", features = ["derive"] } 33 | serde_json = "1.0" 34 | serde_yaml = "0.9" 35 | toml = "0.8.19" 36 | log = "0.4.0" 37 | log4rs = "1.2.0" 38 | thiserror = "1.0.63" 39 | axum = "0.7.5" 40 | tokio = { version = "1", features = ["full"] } 41 | tonic = "0.11.0" 42 | tonic-build = "0.11.0" 43 | prost = "0.12.3" 44 | dashmap = "6.0.1" 45 | rocksdb = "0.22.0" 46 | bincode = "1.3.3" 47 | tokio-util = { version = "0.7.9", features = ["codec"] } 48 | mobc = "0.8.4" 49 | openraft = { git = "https://github.com/databendlabs/openraft.git", features = [ 50 | "serde", 51 | "type-alias", 52 | "loosen-follower-log-revert", 53 | ] } 54 | byteorder = "1.5.0" 55 | tracing = "0.1.40" 56 | tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } 57 | 58 | 59 | ## workspaces members 60 | placement-center = { path = "src/placement-center" } 61 | cmd = { path = "src/cmd" } 62 | protocol = { path = "src/protocol" } 63 | clients = { path = "src/clients" } 64 | common-base = { path = "src/common/base" } 65 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | hello，欢迎来到极客时间课程《Rust 实战 · 手写下一代云原生消息队列》的 Demo Project，很高兴在这里遇到你，也祝你在Rust 学习路上披荆斩棘，高歌猛进。 2 | 3 | 这是一个运行的 Demo 项目，你下载完代码后，需要安装一下环境依赖，然后即可在本地运行。 4 | 5 | ## RobustMQ-Geek 和 RobustMQ的关系 6 | 7 | - RobustMQ-Geek项目是课程《Rust 实战 · 手写下一代云原生消息队列》的配套示例项目。 8 | - 《Rust 实战 · 手写下一代云原生消息队列》课程是基于开源项目 RobustMQ 的实战经验总结而来的。 9 | - RobustMQ 是一个基于Apache LiCENSE 2.0 的消息队列领域的 Rust 开源项目 10 | 11 | 所以说 RobustMQ-Geek是 RobustMQ 的子集。也就是说 RobustMQ 里面会有更完善的代码实现。感兴趣的可以查看我们主项目RobustMQ，https://github.com/robustmq/robustmq 。 12 | 13 | 既然你看到了这里，请给我们送个见面礼吧🌟🎒 🌟🎒 ，给 https://github.com/robustmq/robustmq 点个 Star 🌟先！ 14 | 15 | ## RobustMQ 期待你的参与 16 | 欢迎各位勇士加入RobustMQ的神秘领域！🌟 17 | 18 | 在这个充满魔法与代码的世界里，我们的RobustMQ还是一个初出茅庐的少年，但它拥有着无限的可能性和潜力！🚀 19 | 20 | 👩‍💻👨‍💻我们诚邀各位Rust大师，一起挥舞你们的键盘之剑，用Rust的魔法编织出强大的代码之网。在这里，我们不仅共同打造一个坚不可摧的消息队列，更是在探索技术的边界，一起成长，一起进步！ 21 | 22 | 📚学习Rust，就像是掌握一门古老的语言，它将赋予你构建安全、高效软件的力量。让我们一起深入Rust的奥秘，解锁它的全部潜力！ 23 | 24 | 🌐构建基础软件，就像是在数字世界中建立一座坚不可摧的城堡。我们的目标是让RobustMQ成为连接现实与虚拟，过去与未来的桥梁。 25 | 26 | 🎉所以，拿起你的装备，准备好你的代码，让我们一起踏上这场激动人心的冒险之旅吧！未来已来，让我们共同创造历史！ 27 | 28 | 🌈欢迎加入RobustMQ开发组，让我们一起成为改变世界的力量！🚀🌟 29 | 30 | 欢迎查看🔮💻[《RobustMQ 魔法入门手册》](https://shimo.im/docs/XKq427g9v0Tj0PAN) 31 | 32 | 33 | ## 安装环境 34 | 35 | 1. 安装 Rust 基础环境 36 | 参考文档：https://course.rs/first-try/installation.html 。 安装完成后，查看 rustc 和 cargo 版本，能看到版本即安装成功 37 | ``` 38 | FWR3KG21WF:~ $ rustc --version 39 | rustc 1.74.0 (79e9716c9 2023-11-13) 40 | FWR3KG21WF:~ $ cargo version 41 | cargo 1.74.0 (ecb9851af 2023-10-18) 42 | ``` 43 | 44 | 2. 安装 Cmake. 45 | mac 安装命令如下： 46 | ``` 47 | brew install cmake 48 | ``` 49 | 安装完成后，查看cmake版本，能看到版本即安装成功 50 | ``` 51 | FWR3KG21WF:~ $ cmake --version 52 | cmake version 3.30.2 53 | CMake suite maintained and supported by Kitware (kitware.com/cmake). 54 | ``` 55 | 56 | 3. 安装 RocksDB 57 | 参考文档：https://github.com/rust-rocksdb/rust-rocksdb 安装 rocksdb。mac 安装命令如下： 58 | ``` 59 | brew install rocksdb 60 | ``` 61 | 安装完成后，查看rocksdb 版本，能看到版本即安装成功 62 | ``` 63 | FWR3KG21WF:~$ rocksdb_ldb --version 64 | ldb from RocksDB 9.4.0 65 | ``` 66 | 67 | ## 运行项目 68 | 建议直接在VsCode 中运行项目。VsCode环境配置请参考文档：https://course.rs/first-try/editor.html 69 | 70 | 打开项目后，直接打开文件：src/cmd/src/placement-center/server.rs 。点击运行main函数即可，启动成功日志如下： 71 | ``` 72 | 2024-09-19T15:12:22.945107+08:00 INFO placement_center - PlacementCenterConfig { cluster_name: "placement-test", addr: "127.0.0.1", node_id: 1, grpc_port: 8871, nodes: {"1": String("127.0.0.1:1228")}, http_port: 8971, data_path: "/tmp/placement-center", log: Log { log_config: "./config/log4rs.yaml", log_path: "./logs" } } 73 | 2024-09-19T15:12:22.959227+08:00 INFO placement_center::server::http::server - Broker HTTP Server start. port:8971 74 | 2024-09-19T15:12:22.959334+08:00 INFO placement_center::server::grpc::server - Broker Grpc Server start. port:8871 75 | 2024-09-19T15:12:24.696262+08:00 INFO placement_center::raft::machine - Node Raft Role changes from 【Follower】 to 【Leader】 76 | >> save entry index:2, value:Entry { entry_type: EntryNormal, term: 2, index: 2, data: [], context: [], sync_log: false } 77 | 2024-09-19T15:12:24.697367+08:00 INFO placement_center::raft::machine - save hardState!!!,len:HardState { term: 2, vote: 1, commit: 1 } 78 | 2024-09-19T15:12:24.697815+08:00 INFO placement_center::raft::machine - save light rd!!!,commit:2 79 | >> commit entry index:2 80 | ``` 81 | 82 | 你可以看到启动了 HTTP 和 GRPC Server，同时 Raft 集群也选举出 Leader 了 83 | 84 | 如果运行遇到问题，欢迎添加我们的微信群进行讨论。点击添加微信群：https://jsj.top/f/Dbjfwl 85 | 86 | -------------------------------------------------------------------------------- /bin/robustctl: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | workdir=$(cd $(dirname $0); pwd) 3 | mkdir -p ${workdir}/../logs 4 | mod=$1 5 | action=$2 6 | conf=$3 7 | if [[ -z $mod ]] 8 | then 9 | echo "Enter the name of the component to start, for example: placement-center" 10 | exit 11 | fi 12 | 13 | if [[ -z $action ]] 14 | then 15 | echo "Pleasemake sure the positon variable is start or stop." 16 | exit 17 | fi 18 | 19 | if [ $action = "start" ] 20 | then 21 | if [[ -z $conf ]] 22 | then 23 | conf=${workdir}/../config/${mod}.toml 24 | fi 25 | echo "config:$conf" 26 | echo "${mod} is starting...." 27 | nohup ${workdir}/../libs/${mod} --conf=$conf >> ${workdir}/../logs/${mod}-nohub.log 2>&1 & 28 | num=` ps -ef | grep ${mod} | grep -v grep | wc -l` 29 | if [ $num -gt 1 ] 30 | then 31 | echo " 32 | _____ ____ ______ _____ ________ _ _ _____ 33 | || \\ // \\ || |||| |||| --------||\\ //|| // \\ 34 | ||----//|| ||||____// || || \\____ || || \\ // |||| || 35 | || // || |||| \\|| || || || || \\ // |||| || 36 | ||_|__\\ \\____// ||__|__||||__|__|| __|__|| || || \\// || \\___\\// 37 | \\ 38 | " 39 | echo "${mod} started successfully." 40 | fi 41 | elif [ $action = "stop" ] 42 | then 43 | no=`ps -ef | grep ${mod} | grep conf | grep -v grep | awk '{print $2}'` 44 | if [[ -n $no ]] 45 | then 46 | echo "Currently running process number: $no" 47 | kill $no 48 | num=`ps -ef | grep ${mod} | grep conf | grep -v grep | wc -l` 49 | if [ $num -eq 0 ] 50 | then 51 | echo "${mod} stop successfully." 52 | fi 53 | fi 54 | else 55 | echo "Pleasemake sure the positon variable is start or stop." 56 | fi 57 | -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | fn main() { 16 | println("build.rs running"); 17 | } -------------------------------------------------------------------------------- /config/cluster/placement-center-1.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | cluster_name = "placement-test" 16 | addr = "127.0.0.1" 17 | node_id = 2 18 | grpc_port = 1228 19 | http_port = 8972 20 | nodes = { 2 = "127.0.0.1:1228", 3 = "127.0.0.1:1238", 4 = "127.0.0.1:1248" } 21 | data_path = "/tmp/placement-center-geek/geek-2" 22 | 23 | [log] 24 | log_config = "./config/log4rs.yaml" 25 | log_path = "./logs/2" 26 | -------------------------------------------------------------------------------- /config/cluster/placement-center-2.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | cluster_name = "placement-test" 16 | addr = "127.0.0.1" 17 | node_id = 3 18 | grpc_port = 1238 19 | http_port = 8973 20 | nodes = { 2 = "127.0.0.1:1228", 3 = "127.0.0.1:1238", 4 = "127.0.0.1:1248" } 21 | data_path = "/tmp/placement-center-geek/geek-3" 22 | 23 | [log] 24 | log_config = "./config/log4rs.yaml" 25 | log_path = "./logs/3" 26 | -------------------------------------------------------------------------------- /config/cluster/placement-center-3.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | cluster_name = "placement-test" 16 | addr = "127.0.0.1" 17 | node_id = 4 18 | grpc_port = 1248 19 | http_port = 8974 20 | nodes = { 2 = "127.0.0.1:1228", 3 = "127.0.0.1:1238", 4 = "127.0.0.1:1248" } 21 | data_path = "/tmp/placement-center-geek/geek-4" 22 | 23 | [log] 24 | log_config = "./config/log4rs.yaml" 25 | log_path = "./logs/4" 26 | -------------------------------------------------------------------------------- /config/log4rs.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | appenders: 16 | stdout: 17 | kind: console 18 | 19 | server: 20 | kind: rolling_file 21 | path: "{$path}/server.log" 22 | encoder: 23 | pattern: "{d(%Y-%m-%d %H:%M:%S)} {h({l})} {m}{n}" 24 | policy: 25 | trigger: 26 | kind: size 27 | limit: 1 gb 28 | roller: 29 | kind: fixed_window 30 | pattern: "{$path}/server-{}.log" 31 | base: 0 32 | count: 50 33 | 34 | requests: 35 | kind: rolling_file 36 | path: "{$path}/requests-log.log" 37 | encoder: 38 | pattern: "{d(%Y-%m-%d %H:%M:%S)} {h({l})} {m}{n}" 39 | policy: 40 | trigger: 41 | kind: size 42 | limit: 1 gb 43 | roller: 44 | kind: fixed_window 45 | pattern: "{$path}/requests-log-{}.log" 46 | base: 0 47 | count: 50 48 | raft: 49 | kind: rolling_file 50 | path: "{$path}/raft.log" 51 | encoder: 52 | pattern: "{d(%Y-%m-%d %H:%M:%S)} {h({l})} {m}{n}" 53 | policy: 54 | trigger: 55 | kind: size 56 | limit: 1 gb 57 | roller: 58 | kind: fixed_window 59 | pattern: "{$path}/raft-{}.log" 60 | base: 0 61 | count: 50 62 | 63 | root: 64 | level: info 65 | appenders: 66 | - stdout 67 | - server 68 | 69 | loggers: 70 | openraft: 71 | level: info 72 | appenders: 73 | - stdout 74 | - raft 75 | additive: false -------------------------------------------------------------------------------- /config/log4rs.yaml.template: -------------------------------------------------------------------------------- 1 | appenders: 2 | stdout: 3 | kind: console 4 | 5 | server: 6 | kind: rolling_file 7 | path: "{$path}/server.log" 8 | encoder: 9 | pattern: "{d(%Y-%m-%d %H:%M:%S)} {h({l})} {m}{n}" 10 | policy: 11 | trigger: 12 | kind: size 13 | limit: 1 gb 14 | roller: 15 | kind: fixed_window 16 | pattern: "{$path}/server-{}.log" 17 | base: 0 18 | count: 50 19 | 20 | requests: 21 | kind: rolling_file 22 | path: "{$path}/requests-log.log" 23 | encoder: 24 | pattern: "{d(%Y-%m-%d %H:%M:%S)} {h({l})} {m}{n}" 25 | policy: 26 | trigger: 27 | kind: size 28 | limit: 1 gb 29 | roller: 30 | kind: fixed_window 31 | pattern: "{$path}/requests-log-{}.log" 32 | base: 0 33 | count: 50 34 | 35 | root: 36 | level: info 37 | appenders: 38 | - stdout 39 | - server 40 | 41 | loggers: 42 | app::backend::db: 43 | level: info 44 | 45 | placement_center::server: 46 | level: info 47 | appenders: 48 | - stdout 49 | - server 50 | additive: false 51 | 52 | placement_center::requests: 53 | level: info 54 | appenders: 55 | - stdout 56 | - requests 57 | additive: false -------------------------------------------------------------------------------- /config/placement-center.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | cluster_name = "placement-test" 16 | addr = "127.0.0.1" 17 | node_id = 1 18 | grpc_port = 8871 19 | http_port = 8971 20 | nodes = { 1 = "127.0.0.1:1228" } 21 | data_path = "/tmp/placement-center-geek/geek-local" 22 | 23 | [log] 24 | log_config = "./config/log4rs.yaml" 25 | log_path = "./logs" 26 | -------------------------------------------------------------------------------- /docs/image1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robustmq/robustmq-geek/e6ee3ef96b001c09f4745202b64c5193440185ce/docs/image1.png -------------------------------------------------------------------------------- /licenserc.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | headerPath = "Apache-2.0.txt" 16 | 17 | includes = [ 18 | "*.yml", 19 | "*.yaml", 20 | "*.toml", 21 | "*.sh", 22 | "*.rs", 23 | "*.py", 24 | ] 25 | 26 | [properties] 27 | inceptionYear = 2023 28 | copyrightOwner = "RobustMQ Team" 29 | -------------------------------------------------------------------------------- /logs/requests-log.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/robustmq/robustmq-geek/e6ee3ef96b001c09f4745202b64c5193440185ce/logs/requests-log.log -------------------------------------------------------------------------------- /logs/server.log: -------------------------------------------------------------------------------- 1 | 2024-10-04 08:58:06 INFO PlacementCenterConfig { cluster_name: "placement-test", addr: "127.0.0.1", node_id: 1, grpc_port: 8871, nodes: {"1": String("127.0.0.1:1228")}, http_port: 8971, data_path: "/tmp/placement-center-geek/geek-local", log: Log { log_config: "./config/log4rs.yaml", log_path: "./logs" } } 2 | 2024-10-04 08:58:06 INFO Raft Nodes:{1: Node { node_id: 1, rpc_addr: "127.0.0.1:1228" }} 3 | 2024-10-04 08:58:06 INFO Broker HTTP Server start. port:8971 4 | 2024-10-04 08:58:06 INFO Broker Grpc Server start. port:8871 5 | 2024-10-04 08:58:06 INFO Whether nodes should be initialized, flag=false 6 | 2024-10-04 08:58:06 INFO Node {1: Node { node_id: 1, rpc_addr: "127.0.0.1:1228" }} was initialized successfully 7 | 2024-10-04 08:58:29 INFO When ctrl + c is received, the service starts to stop 8 | 2024-10-04 08:58:29 INFO HTTP Server stopped successfully 9 | 2024-10-04 08:58:29 INFO HTTP Server stopped successfully 10 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | TARGET = robustmq-geek 2 | BUILD_FOLD = ./build 3 | VERSION:=$(shell cat version.ini) 4 | PACKAGE_FOLD_NAME = ${TARGET}-$(VERSION) 5 | 6 | release: 7 | mkdir -p ${BUILD_FOLD} 8 | cargo build --release 9 | mkdir -p $(BUILD_FOLD)/${PACKAGE_FOLD_NAME} 10 | mkdir -p $(BUILD_FOLD)/${PACKAGE_FOLD_NAME}/bin 11 | mkdir -p $(BUILD_FOLD)/${PACKAGE_FOLD_NAME}/libs 12 | mkdir -p $(BUILD_FOLD)/${PACKAGE_FOLD_NAME}/config 13 | cp -rf target/release/placement-center $(BUILD_FOLD)/${PACKAGE_FOLD_NAME}/libs 14 | cp -rf bin/* $(BUILD_FOLD)/${PACKAGE_FOLD_NAME}/bin 15 | cp -rf config/* $(BUILD_FOLD)/${PACKAGE_FOLD_NAME}/config 16 | chmod -R 777 $(BUILD_FOLD)/${PACKAGE_FOLD_NAME}/bin/* 17 | cd $(BUILD_FOLD) && tar zcvf ${PACKAGE_FOLD_NAME}.tar.gz ${PACKAGE_FOLD_NAME} && rm -rf ${PACKAGE_FOLD_NAME} 18 | echo "build release package success. ${PACKAGE_FOLD_NAME}.tar.gz " 19 | 20 | test: 21 | sh ./scripts/integration-testing.sh 22 | clean: 23 | cargo clean 24 | rm -rf build -------------------------------------------------------------------------------- /scripts/integration-testing.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Copyright 2023 RobustMQ Team 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | start_placement_center(){ 17 | nohup cargo run --package cmd --bin $placement_center_process_name -- --conf=tests/config/$placement_center_process_name.toml >/dev/null 2>&1 & 18 | sleep 3 19 | while ! ps aux | grep -v grep | grep "$placement_center_process_name" > /dev/null; do 20 | echo "Process $placement_center_process_name has not started yet, wait 1s...." 21 | sleep 1 22 | done 23 | echo "Process $placement_center_process_name starts successfully and starts running the test case" 24 | } 25 | 26 | stop_placement_center(){ 27 | pc_no=`ps aux | grep -v grep | grep "$placement_center_process_name" | awk '{print $2}'` 28 | echo "placement center num: $pc_no" 29 | kill $pc_no 30 | sleep 3 31 | 32 | while ps aux | grep -v grep | grep "$placement_center_process_name" > /dev/null; do 33 | echo "”Process $placement_center_process_name stopped successfully" 34 | sleep 1 35 | done 36 | } 37 | 38 | start_placement_center 39 | 40 | 41 | # Run Cargo Test 42 | cargo test 43 | 44 | if [ $? -ne 0 ]; then 45 | echo "Test case failed to run" 46 | stop_placement_center 47 | exit 1 48 | else 49 | echo "Test case runs successfully" 50 | stop_placement_center 51 | fi 52 | 53 | -------------------------------------------------------------------------------- /src/clients/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "clients" 17 | version.workspace = true 18 | edition.workspace = true 19 | license.workspace = true 20 | 21 | 22 | [dependencies] 23 | clap = { version = "4.4.7", features = ["derive"] } 24 | common-base.workspace = true 25 | log.workspace = true 26 | log4rs.workspace = true 27 | tokio.workspace = true 28 | protocol.workspace = true 29 | prost.workspace = true 30 | mobc.workspace = true 31 | dashmap.workspace = true 32 | tonic.workspace = true -------------------------------------------------------------------------------- /src/clients/src/lib.rs: -------------------------------------------------------------------------------- 1 | pub mod placement; 2 | pub mod poll; 3 | 4 | pub fn retry_times() -> usize { 5 | return 3; 6 | } 7 | 8 | pub fn retry_sleep_time(times: usize) -> u64 { 9 | return (times * 2) as u64; 10 | } -------------------------------------------------------------------------------- /src/clients/src/placement/kv/call.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::PlacementCenterInterface; 16 | use crate::{ 17 | placement::{retry_call, PlacementCenterService}, 18 | poll::ClientPool, 19 | }; 20 | use common_base::errors::RobustMQError; 21 | use prost::Message as _; 22 | use protocol::kv::{ 23 | CommonReply, DeleteRequest, ExistsReply, ExistsRequest, GetReply, GetRequest, SetRequest, 24 | }; 25 | use std::sync::Arc; 26 | 27 | pub async fn placement_set( 28 | client_poll: Arc, 29 | addrs: Vec, 30 | request: SetRequest, 31 | ) -> Result { 32 | let request_data = SetRequest::encode_to_vec(&request); 33 | match retry_call( 34 | PlacementCenterService::Kv, 35 | PlacementCenterInterface::Set, 36 | client_poll, 37 | addrs, 38 | request_data, 39 | ) 40 | .await 41 | { 42 | Ok(data) => match CommonReply::decode(data.as_ref()) { 43 | Ok(da) => return Ok(da), 44 | Err(e) => return Err(RobustMQError::CommmonError(e.to_string())), 45 | }, 46 | Err(e) => { 47 | return Err(e); 48 | } 49 | } 50 | } 51 | 52 | pub async fn placement_get( 53 | client_poll: Arc, 54 | addrs: Vec, 55 | request: GetRequest, 56 | ) -> Result { 57 | let request_data = GetRequest::encode_to_vec(&request); 58 | match retry_call( 59 | PlacementCenterService::Kv, 60 | PlacementCenterInterface::Get, 61 | client_poll, 62 | addrs, 63 | request_data, 64 | ) 65 | .await 66 | { 67 | Ok(data) => match GetReply::decode(data.as_ref()) { 68 | Ok(da) => return Ok(da), 69 | Err(e) => return Err(RobustMQError::CommmonError(e.to_string())), 70 | }, 71 | Err(e) => { 72 | return Err(e); 73 | } 74 | } 75 | } 76 | 77 | pub async fn placement_delete( 78 | client_poll: Arc, 79 | addrs: Vec, 80 | request: DeleteRequest, 81 | ) -> Result { 82 | let request_data = DeleteRequest::encode_to_vec(&request); 83 | match retry_call( 84 | PlacementCenterService::Kv, 85 | PlacementCenterInterface::Delete, 86 | client_poll, 87 | addrs, 88 | request_data, 89 | ) 90 | .await 91 | { 92 | Ok(data) => match CommonReply::decode(data.as_ref()) { 93 | Ok(da) => return Ok(da), 94 | Err(e) => return Err(RobustMQError::CommmonError(e.to_string())), 95 | }, 96 | Err(e) => { 97 | return Err(e); 98 | } 99 | } 100 | } 101 | 102 | pub async fn placement_exists( 103 | client_poll: Arc, 104 | addrs: Vec, 105 | request: ExistsRequest, 106 | ) -> Result { 107 | let request_data = ExistsRequest::encode_to_vec(&request); 108 | match retry_call( 109 | PlacementCenterService::Kv, 110 | PlacementCenterInterface::Exists, 111 | client_poll, 112 | addrs, 113 | request_data, 114 | ) 115 | .await 116 | { 117 | Ok(data) => match ExistsReply::decode(data.as_ref()) { 118 | Ok(da) => return Ok(da), 119 | Err(e) => return Err(RobustMQError::CommmonError(e.to_string())), 120 | }, 121 | Err(e) => { 122 | return Err(e); 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/clients/src/placement/kv/inner.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use common_base::errors::RobustMQError; 16 | use mobc::Connection; 17 | use prost::Message; 18 | use protocol::kv::{CommonReply, DeleteRequest, ExistsReply, ExistsRequest, GetReply, GetRequest, SetRequest}; 19 | use super::KvServiceManager; 20 | 21 | pub(crate) async fn inner_get( 22 | mut client: Connection, 23 | request: Vec, 24 | ) -> Result, RobustMQError> { 25 | match GetRequest::decode(request.as_ref()) { 26 | Ok(request) => match client.get(request).await { 27 | Ok(result) => { 28 | return Ok(GetReply::encode_to_vec(&result.into_inner())); 29 | } 30 | Err(e) => return Err(RobustMQError::GrpcServerStatus(e)), 31 | }, 32 | Err(e) => { 33 | return Err(RobustMQError::CommmonError(e.to_string())); 34 | } 35 | } 36 | } 37 | 38 | pub(crate) async fn inner_set( 39 | mut client: Connection, 40 | request: Vec, 41 | ) -> Result, RobustMQError> { 42 | match SetRequest::decode(request.as_ref()) { 43 | Ok(request) => match client.set(request).await { 44 | Ok(result) => { 45 | return Ok(CommonReply::encode_to_vec(&result.into_inner())); 46 | } 47 | Err(e) => return Err(RobustMQError::GrpcServerStatus(e)), 48 | }, 49 | Err(e) => { 50 | return Err(RobustMQError::CommmonError(e.to_string())); 51 | } 52 | } 53 | } 54 | 55 | pub(crate) async fn inner_delete( 56 | mut client: Connection, 57 | request: Vec, 58 | ) -> Result, RobustMQError> { 59 | match DeleteRequest::decode(request.as_ref()) { 60 | Ok(request) => match client.delete(request).await { 61 | Ok(result) => { 62 | return Ok(CommonReply::encode_to_vec(&result.into_inner())); 63 | } 64 | Err(e) => return Err(RobustMQError::GrpcServerStatus(e)), 65 | }, 66 | Err(e) => { 67 | return Err(RobustMQError::CommmonError(e.to_string())); 68 | } 69 | } 70 | } 71 | 72 | pub(crate) async fn inner_exists( 73 | mut client: Connection, 74 | request: Vec, 75 | ) -> Result, RobustMQError> { 76 | match ExistsRequest::decode(request.as_ref()) { 77 | Ok(request) => match client.exists(request).await { 78 | Ok(result) => { 79 | return Ok(ExistsReply::encode_to_vec(&result.into_inner())); 80 | } 81 | Err(e) => return Err(RobustMQError::GrpcServerStatus(e)), 82 | }, 83 | Err(e) => { 84 | return Err(RobustMQError::CommmonError(e.to_string())); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/clients/src/placement/kv/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::poll::ClientPool; 16 | use self::inner::{inner_delete, inner_exists, inner_get, inner_set}; 17 | use super::PlacementCenterInterface; 18 | use common_base::errors::RobustMQError; 19 | use mobc::{Connection, Manager}; 20 | use protocol::kv::kv_service_client::KvServiceClient; 21 | use std::sync::Arc; 22 | use tonic::transport::Channel; 23 | 24 | pub mod call; 25 | mod inner; 26 | 27 | async fn kv_client( 28 | client_poll: Arc, 29 | addr: String, 30 | ) -> Result, RobustMQError> { 31 | match client_poll.placement_center_kv_services_client(addr).await { 32 | Ok(client) => { 33 | return Ok(client); 34 | } 35 | Err(e) => { 36 | return Err(e); 37 | } 38 | } 39 | } 40 | 41 | pub(crate) async fn kv_interface_call( 42 | interface: PlacementCenterInterface, 43 | client_poll: Arc, 44 | addr: String, 45 | request: Vec, 46 | ) -> Result, RobustMQError> { 47 | match kv_client(client_poll.clone(), addr.clone()).await { 48 | Ok(client) => { 49 | let result = match interface { 50 | PlacementCenterInterface::Set => inner_set(client, request.clone()).await, 51 | PlacementCenterInterface::Delete => inner_delete(client, request.clone()).await, 52 | PlacementCenterInterface::Get => inner_get(client, request.clone()).await, 53 | PlacementCenterInterface::Exists => inner_exists(client, request.clone()).await, 54 | _ => return Err(RobustMQError::CommmonError(format!( 55 | "kv service does not support service interfaces [{:?}]", 56 | interface 57 | ))), 58 | }; 59 | match result { 60 | Ok(data) => return Ok(data), 61 | Err(e) => { 62 | return Err(e); 63 | } 64 | } 65 | } 66 | Err(e) => { 67 | return Err(e); 68 | } 69 | } 70 | } 71 | 72 | #[derive(Clone)] 73 | pub struct KvServiceManager { 74 | pub addr: String, 75 | } 76 | 77 | impl KvServiceManager { 78 | pub fn new(addr: String) -> Self { 79 | Self { addr } 80 | } 81 | } 82 | 83 | #[tonic::async_trait] 84 | impl Manager for KvServiceManager { 85 | type Connection = KvServiceClient; 86 | type Error = RobustMQError; 87 | 88 | async fn connect(&self) -> Result { 89 | match KvServiceClient::connect(format!("http://{}", self.addr.clone())).await { 90 | Ok(client) => { 91 | return Ok(client); 92 | } 93 | Err(err) => return Err(RobustMQError::CommmonError(format!( 94 | "{},{}", 95 | err.to_string(), 96 | self.addr.clone() 97 | ))), 98 | }; 99 | } 100 | 101 | async fn check(&self, conn: Self::Connection) -> Result { 102 | Ok(conn) 103 | } 104 | } 105 | 106 | #[cfg(test)] 107 | mod tests {} 108 | -------------------------------------------------------------------------------- /src/clients/src/placement/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::{poll::ClientPool, retry_sleep_time, retry_times}; 16 | use common_base::errors::RobustMQError; 17 | use kv::kv_interface_call; 18 | use log::error; 19 | use openraft::openraft_interface_call; 20 | use std::{sync::Arc, time::Duration}; 21 | use tokio::time::sleep; 22 | 23 | #[derive(Clone, Debug)] 24 | pub enum PlacementCenterService { 25 | Kv, 26 | OpenRaft, 27 | } 28 | 29 | #[derive(Clone, Debug, PartialEq, Eq, Hash)] 30 | pub enum PlacementCenterInterface { 31 | // kv interface 32 | Set, 33 | Get, 34 | Delete, 35 | Exists, 36 | 37 | // Open Raft 38 | Vote, 39 | Append, 40 | Snapshot, 41 | } 42 | 43 | pub mod kv; 44 | pub mod openraft; 45 | 46 | async fn retry_call( 47 | service: PlacementCenterService, 48 | interface: PlacementCenterInterface, 49 | client_poll: Arc, 50 | addrs: Vec, 51 | request: Vec, 52 | ) -> Result, RobustMQError> { 53 | let mut times = 1; 54 | loop { 55 | let index = times % addrs.len(); 56 | let addr = addrs.get(index).unwrap().clone(); 57 | let result = match service { 58 | PlacementCenterService::Kv => { 59 | kv_interface_call( 60 | interface.clone(), 61 | client_poll.clone(), 62 | addr.clone(), 63 | request.clone(), 64 | ) 65 | .await 66 | } 67 | 68 | PlacementCenterService::OpenRaft => { 69 | openraft_interface_call( 70 | interface.clone(), 71 | client_poll.clone(), 72 | addr.clone(), 73 | request.clone(), 74 | ) 75 | .await 76 | } 77 | }; 78 | 79 | match result { 80 | Ok(data) => { 81 | return Ok(data); 82 | } 83 | Err(e) => { 84 | error!( 85 | "{:?}@{:?}@{},{},", 86 | service.clone(), 87 | interface.clone(), 88 | addr.clone(), 89 | e 90 | ); 91 | if times > retry_times() { 92 | return Err(e); 93 | } 94 | times = times + 1; 95 | } 96 | } 97 | sleep(Duration::from_secs(retry_sleep_time(times) as u64)).await; 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/clients/src/placement/openraft/call.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::PlacementCenterInterface; 16 | use crate::{ 17 | placement::{retry_call, PlacementCenterService}, 18 | poll::ClientPool, 19 | }; 20 | use common_base::errors::RobustMQError; 21 | use prost::Message as _; 22 | use protocol::openraft::{ 23 | AppendReply, AppendRequest, SnapshotReply, SnapshotRequest, VoteReply, VoteRequest, 24 | }; 25 | use std::sync::Arc; 26 | 27 | pub async fn placement_openraft_vote( 28 | client_poll: Arc, 29 | addrs: Vec, 30 | request: VoteRequest, 31 | ) -> Result { 32 | let request_data = VoteRequest::encode_to_vec(&request); 33 | match retry_call( 34 | PlacementCenterService::OpenRaft, 35 | PlacementCenterInterface::Vote, 36 | client_poll, 37 | addrs, 38 | request_data, 39 | ) 40 | .await 41 | { 42 | Ok(data) => match VoteReply::decode(data.as_ref()) { 43 | Ok(da) => return Ok(da), 44 | Err(e) => return Err(RobustMQError::CommmonError(e.to_string())), 45 | }, 46 | Err(e) => { 47 | return Err(e); 48 | } 49 | } 50 | } 51 | 52 | pub async fn placement_openraft_append( 53 | client_poll: Arc, 54 | addrs: Vec, 55 | request: AppendRequest, 56 | ) -> Result { 57 | let request_data = AppendRequest::encode_to_vec(&request); 58 | match retry_call( 59 | PlacementCenterService::OpenRaft, 60 | PlacementCenterInterface::Append, 61 | client_poll, 62 | addrs, 63 | request_data, 64 | ) 65 | .await 66 | { 67 | Ok(data) => match AppendReply::decode(data.as_ref()) { 68 | Ok(da) => return Ok(da), 69 | Err(e) => return Err(RobustMQError::CommmonError(e.to_string())), 70 | }, 71 | Err(e) => { 72 | return Err(e); 73 | } 74 | } 75 | } 76 | 77 | pub async fn placement_openraft_snapshot( 78 | client_poll: Arc, 79 | addrs: Vec, 80 | request: SnapshotRequest, 81 | ) -> Result { 82 | let request_data = SnapshotRequest::encode_to_vec(&request); 83 | match retry_call( 84 | PlacementCenterService::OpenRaft, 85 | PlacementCenterInterface::Snapshot, 86 | client_poll, 87 | addrs, 88 | request_data, 89 | ) 90 | .await 91 | { 92 | Ok(data) => match SnapshotReply::decode(data.as_ref()) { 93 | Ok(da) => return Ok(da), 94 | Err(e) => return Err(RobustMQError::CommmonError(e.to_string())), 95 | }, 96 | Err(e) => { 97 | return Err(e); 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /src/clients/src/placement/openraft/inner.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::OpenRaftServiceManager; 16 | use common_base::errors::RobustMQError; 17 | use mobc::Connection; 18 | use prost::Message; 19 | use protocol::openraft::{ 20 | AppendReply, AppendRequest, SnapshotReply, SnapshotRequest, VoteReply, VoteRequest, 21 | }; 22 | 23 | pub(crate) async fn inner_vote( 24 | mut client: Connection, 25 | request: Vec, 26 | ) -> Result, RobustMQError> { 27 | match VoteRequest::decode(request.as_ref()) { 28 | Ok(request) => match client.vote(request).await { 29 | Ok(result) => { 30 | return Ok(VoteReply::encode_to_vec(&result.into_inner())); 31 | } 32 | Err(e) => return Err(RobustMQError::GrpcServerStatus(e)), 33 | }, 34 | Err(e) => { 35 | return Err(RobustMQError::CommmonError(e.to_string())); 36 | } 37 | } 38 | } 39 | 40 | pub(crate) async fn inner_append( 41 | mut client: Connection, 42 | request: Vec, 43 | ) -> Result, RobustMQError> { 44 | match AppendRequest::decode(request.as_ref()) { 45 | Ok(request) => match client.append(request).await { 46 | Ok(result) => { 47 | return Ok(AppendReply::encode_to_vec(&result.into_inner())); 48 | } 49 | Err(e) => return Err(RobustMQError::GrpcServerStatus(e)), 50 | }, 51 | Err(e) => { 52 | return Err(RobustMQError::CommmonError(e.to_string())); 53 | } 54 | } 55 | } 56 | 57 | pub(crate) async fn inner_snapshot( 58 | mut client: Connection, 59 | request: Vec, 60 | ) -> Result, RobustMQError> { 61 | match SnapshotRequest::decode(request.as_ref()) { 62 | Ok(request) => match client.snapshot(request).await { 63 | Ok(result) => { 64 | return Ok(SnapshotReply::encode_to_vec(&result.into_inner())); 65 | } 66 | Err(e) => return Err(RobustMQError::GrpcServerStatus(e)), 67 | }, 68 | Err(e) => { 69 | return Err(RobustMQError::CommmonError(e.to_string())); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/clients/src/placement/openraft/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::PlacementCenterInterface; 16 | use crate::poll::ClientPool; 17 | use common_base::errors::RobustMQError; 18 | use inner::{inner_append, inner_snapshot, inner_vote}; 19 | use mobc::{Connection, Manager}; 20 | use protocol::openraft::open_raft_service_client::OpenRaftServiceClient; 21 | use std::sync::Arc; 22 | use tonic::transport::Channel; 23 | 24 | pub mod call; 25 | mod inner; 26 | 27 | pub(crate) async fn openraft_interface_call( 28 | interface: PlacementCenterInterface, 29 | client_poll: Arc, 30 | addr: String, 31 | request: Vec, 32 | ) -> Result, RobustMQError> { 33 | match openraft_client(client_poll.clone(), addr.clone()).await { 34 | Ok(client) => { 35 | let result = match interface { 36 | PlacementCenterInterface::Vote => inner_vote(client, request.clone()).await, 37 | PlacementCenterInterface::Append => inner_append(client, request.clone()).await, 38 | PlacementCenterInterface::Snapshot => inner_snapshot(client, request.clone()).await, 39 | _ => { 40 | return Err(RobustMQError::CommmonError(format!( 41 | "openraft service does not support service interfaces [{:?}]", 42 | interface 43 | ))) 44 | } 45 | }; 46 | match result { 47 | Ok(data) => return Ok(data), 48 | Err(e) => { 49 | return Err(e); 50 | } 51 | } 52 | } 53 | Err(e) => { 54 | return Err(e); 55 | } 56 | } 57 | } 58 | 59 | async fn openraft_client( 60 | client_poll: Arc, 61 | addr: String, 62 | ) -> Result, RobustMQError> { 63 | match client_poll 64 | .placement_center_openraft_services_client(addr) 65 | .await 66 | { 67 | Ok(client) => { 68 | return Ok(client); 69 | } 70 | Err(e) => { 71 | return Err(e); 72 | } 73 | } 74 | } 75 | 76 | #[derive(Clone)] 77 | pub struct OpenRaftServiceManager { 78 | pub addr: String, 79 | } 80 | 81 | impl OpenRaftServiceManager { 82 | pub fn new(addr: String) -> Self { 83 | Self { addr } 84 | } 85 | } 86 | 87 | #[tonic::async_trait] 88 | impl Manager for OpenRaftServiceManager { 89 | type Connection = OpenRaftServiceClient; 90 | type Error = RobustMQError; 91 | 92 | async fn connect(&self) -> Result { 93 | let mut addr = format!("http://{}", self.addr.clone()); 94 | 95 | match OpenRaftServiceClient::connect(addr.clone()).await { 96 | Ok(client) => { 97 | return Ok(client); 98 | } 99 | Err(err) => { 100 | return Err(RobustMQError::CommmonError(format!( 101 | "{},{}", 102 | err.to_string(), 103 | addr 104 | ))) 105 | } 106 | }; 107 | } 108 | 109 | async fn check(&self, conn: Self::Connection) -> Result { 110 | Ok(conn) 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/clients/src/poll.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::placement::{kv::KvServiceManager, openraft::OpenRaftServiceManager}; 16 | use common_base::errors::RobustMQError; 17 | use dashmap::DashMap; 18 | use mobc::{Connection, Pool}; 19 | 20 | #[derive(Clone)] 21 | pub struct ClientPool { 22 | max_open_connection: u64, 23 | // placement center 24 | placement_center_kv_service_pools: DashMap>, 25 | placement_center_openraft_service_pools: DashMap>, 26 | } 27 | 28 | impl ClientPool { 29 | pub fn new(max_open_connection: u64) -> Self { 30 | Self { 31 | max_open_connection, 32 | placement_center_kv_service_pools: DashMap::with_capacity(2), 33 | placement_center_openraft_service_pools: DashMap::with_capacity(2), 34 | } 35 | } 36 | 37 | pub async fn placement_center_kv_services_client( 38 | &self, 39 | addr: String, 40 | ) -> Result, RobustMQError> { 41 | let module = "KvServices".to_string(); 42 | let key = format!("{}_{}_{}", "PlacementCenter", module, addr); 43 | 44 | if !self.placement_center_kv_service_pools.contains_key(&key) { 45 | let manager = KvServiceManager::new(addr.clone()); 46 | let pool = Pool::builder() 47 | .max_open(self.max_open_connection) 48 | .build(manager); 49 | self.placement_center_kv_service_pools 50 | .insert(key.clone(), pool); 51 | } 52 | 53 | if let Some(poll) = self.placement_center_kv_service_pools.get(&key) { 54 | match poll.get().await { 55 | Ok(conn) => { 56 | return Ok(conn); 57 | } 58 | Err(e) => { 59 | return Err(RobustMQError::NoAvailableGrpcConnection( 60 | module, 61 | e.to_string(), 62 | )); 63 | } 64 | }; 65 | } 66 | 67 | return Err(RobustMQError::NoAvailableGrpcConnection( 68 | module, 69 | "connection pool is not initialized".to_string(), 70 | )); 71 | } 72 | 73 | pub async fn placement_center_openraft_services_client( 74 | &self, 75 | addr: String, 76 | ) -> Result, RobustMQError> { 77 | let module = "OpenRaftServices".to_string(); 78 | let key = format!("{}_{}_{}", "PlacementCenter", module, addr); 79 | if !self 80 | .placement_center_openraft_service_pools 81 | .contains_key(&key) 82 | { 83 | let manager = OpenRaftServiceManager::new(addr.clone()); 84 | let pool = Pool::builder() 85 | .max_open(self.max_open_connection) 86 | .build(manager); 87 | self.placement_center_openraft_service_pools 88 | .insert(key.clone(), pool); 89 | } 90 | 91 | if let Some(poll) = self.placement_center_openraft_service_pools.get(&key) { 92 | match poll.get().await { 93 | Ok(conn) => { 94 | return Ok(conn); 95 | } 96 | Err(e) => { 97 | return Err(RobustMQError::NoAvailableGrpcConnection( 98 | module, 99 | e.to_string(), 100 | )); 101 | } 102 | }; 103 | } 104 | 105 | return Err(RobustMQError::NoAvailableGrpcConnection( 106 | module, 107 | "connection pool is not initialized".to_string(), 108 | )); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/clients/tests/test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /src/cmd/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "cmd" 17 | version.workspace = true 18 | edition.workspace = true 19 | license.workspace = true 20 | 21 | 22 | [dependencies] 23 | clap = { version = "4.4.7", features = ["derive"] } 24 | common-base.workspace = true 25 | placement-center.workspace = true 26 | log.workspace = true 27 | log4rs.workspace = true 28 | tokio.workspace = true 29 | 30 | 31 | [[bin]] 32 | name = "placement-center" 33 | path = "src/placement-center/server.rs" 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/cmd/src/placement-center/server.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use clap::command; 16 | use clap::Parser; 17 | use common_base::config::placement_center::init_placement_center_conf_by_path; 18 | use common_base::config::placement_center::placement_center_conf; 19 | use common_base::log::placement_center::init_placement_center_log; 20 | use log::info; 21 | use placement_center::start_server; 22 | use tokio::sync::broadcast; 23 | 24 | pub const DEFAULT_PLACEMENT_CENTER_CONFIG: &str = "config/placement-center.toml"; 25 | 26 | #[derive(Parser, Debug)] 27 | #[command(author="robustmq-geek", version="0.0.1", about=" RobustMQ: Next generation cloud-native converged high-performance message queue.", long_about = None)] 28 | #[command(next_line_help = true)] 29 | 30 | struct ArgsParams { 31 | /// MetaService Indicates the path of the configuration file 32 | #[arg(short, long, default_value_t=String::from(DEFAULT_PLACEMENT_CENTER_CONFIG))] 33 | conf: String, 34 | } 35 | 36 | #[tokio::main] 37 | async fn main() { 38 | let args = ArgsParams::parse(); 39 | init_placement_center_conf_by_path(&args.conf); 40 | init_placement_center_log(); 41 | 42 | let conf = placement_center_conf(); 43 | info!("{:?}", conf); 44 | 45 | let (stop_send, _) = broadcast::channel(2); 46 | start_server(stop_send).await; 47 | } 48 | -------------------------------------------------------------------------------- /src/cmd/tests/test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /src/common/base/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "common-base" 17 | version.workspace = true 18 | edition.workspace = true 19 | license.workspace = true 20 | 21 | 22 | [dependencies] 23 | serde.workspace = true 24 | toml.workspace = true 25 | log.workspace = true 26 | log4rs.workspace = true 27 | thiserror.workspace = true 28 | serde_yaml.workspace = true 29 | serde_json.workspace = true 30 | axum.workspace = true 31 | rocksdb.workspace = true 32 | tonic.workspace = true 33 | -------------------------------------------------------------------------------- /src/common/base/src/config/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod placement_center; 16 | -------------------------------------------------------------------------------- /src/common/base/src/config/placement_center.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | * Copyright (c) 2023 RobustMQ Team 17 | * 18 | * Licensed under the Apache License, Version 2.0 (the "License"); 19 | * you may not use this file except in compliance with the License. 20 | * You may obtain a copy of the License at 21 | * 22 | * http://www.apache.org/licenses/LICENSE-2.0 23 | * 24 | * Unless required by applicable law or agreed to in writing, software 25 | * distributed under the License is distributed on an "AS IS" BASIS, 26 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 27 | * See the License for the specific language governing permissions and 28 | * limitations under the License. 29 | */ 30 | 31 | use crate::tools::read_file; 32 | use serde::Deserialize; 33 | use toml::Table; 34 | use std::sync::OnceLock; 35 | 36 | #[derive(Debug, Deserialize, Clone, Default)] 37 | pub struct PlacementCenterConfig { 38 | pub cluster_name: String, 39 | pub addr: String, 40 | #[serde(default = "default_node_id")] 41 | pub node_id: u64, 42 | #[serde(default = "default_grpc_port")] 43 | pub grpc_port: usize, 44 | pub nodes: Table, 45 | pub http_port: usize, 46 | pub data_path: String, 47 | pub log: Log, 48 | } 49 | 50 | #[derive(Debug, Deserialize, Clone, Default)] 51 | pub struct Log { 52 | pub log_config: String, 53 | pub log_path: String, 54 | } 55 | 56 | pub fn default_node_id() -> u64 { 57 | 1 58 | } 59 | 60 | pub fn default_grpc_port() -> usize { 61 | 9982 62 | } 63 | 64 | static PLACEMENT_CENTER_CONF: OnceLock = OnceLock::new(); 65 | 66 | pub fn init_placement_center_conf_by_path(config_path: &String) -> &'static PlacementCenterConfig { 67 | PLACEMENT_CENTER_CONF.get_or_init(|| { 68 | let content = match read_file(config_path) { 69 | Ok(data) => data, 70 | Err(e) => { 71 | panic!("{}", e.to_string()); 72 | } 73 | }; 74 | let pc_config: PlacementCenterConfig = toml::from_str(&content).unwrap(); 75 | return pc_config; 76 | }) 77 | } 78 | 79 | pub fn placement_center_conf() -> &'static PlacementCenterConfig { 80 | match PLACEMENT_CENTER_CONF.get() { 81 | Some(config) => { 82 | return config; 83 | } 84 | None => { 85 | panic!( 86 | "Placement center configuration is not initialized, check the configuration file." 87 | ); 88 | } 89 | } 90 | } 91 | #[cfg(test)] 92 | mod tests { 93 | use crate::config::placement_center::{ 94 | init_placement_center_conf_by_path, placement_center_conf, 95 | }; 96 | 97 | #[test] 98 | fn config_init_test() { 99 | let path = format!( 100 | "{}/../../../config/placement-center.toml", 101 | env!("CARGO_MANIFEST_DIR") 102 | ); 103 | println!("{}", path); 104 | init_placement_center_conf_by_path(&path); 105 | let config = placement_center_conf(); 106 | assert_eq!(config.node_id, 1); 107 | assert_eq!(config.grpc_port, 1228); 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/common/base/src/errors.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | * Copyright (c) 2023 RobustMQ Team 17 | * 18 | * Licensed under the Apache License, Version 2.0 (the "License"); 19 | * you may not use this file except in compliance with the License. 20 | * You may obtain a copy of the License at 21 | * 22 | * http://www.apache.org/licenses/LICENSE-2.0 23 | * 24 | * Unless required by applicable law or agreed to in writing, software 25 | * distributed under the License is distributed on an "AS IS" BASIS, 26 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 27 | * See the License for the specific language governing permissions and 28 | * limitations under the License. 29 | */ 30 | 31 | use std::io; 32 | use tonic::Status; 33 | use thiserror::Error; 34 | 35 | #[derive(Error, Debug)] 36 | pub enum RobustMQError { 37 | #[error("io error")] 38 | IOJsonError(#[from] io::Error), 39 | 40 | #[error("Parameter cannot be empty, parameter name: {0}")] 41 | ParameterCannotBeNull(String), 42 | 43 | #[error("{0}")] 44 | CommmonError(String), 45 | 46 | #[error("{0}")] 47 | RocksdbError(#[from] rocksdb::Error), 48 | 49 | #[error("No available nodes in the cluster")] 50 | ClusterNoAvailableNode, 51 | 52 | #[error("{0}")] 53 | SerdeJsonError(#[from] serde_json::Error), 54 | 55 | #[error("Description The interface {0} submitted logs to the commit log")] 56 | RaftLogCommitTimeout(String), 57 | 58 | #[error("{0} connection pool has no connection information available. {1}")] 59 | NoAvailableGrpcConnection(String, String), 60 | 61 | #[error("Grpc call of the node failed,Grpc status was {0}")] 62 | GrpcServerStatus(Status), 63 | } 64 | -------------------------------------------------------------------------------- /src/common/base/src/http_error.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | * Copyright (c) 2023 RobustMQ Team 17 | * 18 | * Licensed under the Apache License, Version 2.0 (the "License"); 19 | * you may not use this file except in compliance with the License. 20 | * You may obtain a copy of the License at 21 | * 22 | * http://www.apache.org/licenses/LICENSE-2.0 23 | * 24 | * Unless required by applicable law or agreed to in writing, software 25 | * distributed under the License is distributed on an "AS IS" BASIS, 26 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 27 | * See the License for the specific language governing permissions and 28 | * limitations under the License. 29 | */ 30 | use axum::{ 31 | http::StatusCode, 32 | response::{IntoResponse, Json, Response}, 33 | }; 34 | use serde_json::json; 35 | use thiserror::Error; 36 | 37 | #[derive(Error, Debug, PartialEq)] 38 | pub enum HttpError { 39 | #[error("Not found for {0}")] 40 | NotFound(String), 41 | } 42 | 43 | impl IntoResponse for HttpError { 44 | fn into_response(self) -> Response { 45 | let (status, error_message) = match self { 46 | HttpError::NotFound(msg) => (StatusCode::NOT_FOUND, msg), 47 | }; 48 | let body = Json(json!({ 49 | "error": format!("invalid path {}", error_message), 50 | })); 51 | 52 | (status, body).into_response() 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/common/base/src/http_response.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use serde::{Deserialize, Serialize}; 16 | 17 | #[derive(Serialize, Deserialize)] 18 | pub struct Response { 19 | pub code: u64, 20 | pub data: T, 21 | } 22 | 23 | pub fn success_response(data: T) -> String { 24 | let resp = Response { 25 | code: 0, 26 | data: data, 27 | }; 28 | return serde_json::to_string(&resp).unwrap(); 29 | } 30 | 31 | pub fn error_response(err: String) -> String { 32 | let resp = Response { 33 | code: 100, 34 | data: err, 35 | }; 36 | return serde_json::to_string(&resp).unwrap(); 37 | } 38 | -------------------------------------------------------------------------------- /src/common/base/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod config; 16 | pub mod errors; 17 | pub mod http_error; 18 | pub mod http_response; 19 | pub mod log; 20 | pub mod tools; 21 | -------------------------------------------------------------------------------- /src/common/base/src/log/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod placement_center; 16 | -------------------------------------------------------------------------------- /src/common/base/src/log/placement_center.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::{ 16 | config::placement_center::placement_center_conf, 17 | tools::{create_fold, file_exists, read_file}, 18 | }; 19 | 20 | pub fn init_placement_center_log() { 21 | let conf = placement_center_conf(); 22 | if !file_exists(&conf.log.log_config) { 23 | panic!( 24 | "Logging configuration file {} does not exist", 25 | conf.log.log_config 26 | ); 27 | } 28 | 29 | match create_fold(&conf.log.log_path) { 30 | Ok(()) => {} 31 | Err(_) => { 32 | panic!("Failed to initialize log directory {}", conf.log.log_path); 33 | } 34 | } 35 | 36 | let content = match read_file(&conf.log.log_config) { 37 | Ok(data) => data, 38 | Err(e) => { 39 | panic!("{}", e.to_string()); 40 | } 41 | }; 42 | 43 | let config_content = content.replace("{$path}", &conf.log.log_path); 44 | println!("{}","log config:"); 45 | println!("{}", config_content); 46 | let config = match serde_yaml::from_str(&config_content) { 47 | Ok(data) => data, 48 | Err(e) => { 49 | panic!( 50 | "Failed to parse the contents of the config file {} with error message :{}", 51 | conf.log.log_config, 52 | e.to_string() 53 | ); 54 | } 55 | }; 56 | 57 | match log4rs::init_raw_config(config) { 58 | Ok(_) => {} 59 | Err(e) => { 60 | panic!("{}", e.to_string()); 61 | } 62 | } 63 | } 64 | 65 | #[cfg(test)] 66 | mod tests { 67 | use super::*; 68 | 69 | #[test] 70 | fn log_print() {} 71 | } 72 | -------------------------------------------------------------------------------- /src/common/base/src/tools.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::{ 16 | fs, 17 | path::{self, Path}, time::{SystemTime, UNIX_EPOCH}, 18 | }; 19 | 20 | use crate::errors::RobustMQError; 21 | 22 | pub fn create_fold(fold: &String) -> Result<(), RobustMQError> { 23 | if !Path::new(fold).exists() { 24 | fs::create_dir_all(fold)? 25 | } 26 | return Ok(()); 27 | } 28 | 29 | pub fn file_exists(path: &String) -> bool { 30 | return Path::new(path).exists(); 31 | } 32 | 33 | pub fn read_file(path: &String) -> Result { 34 | if !path::Path::new(path).exists() { 35 | return Err(RobustMQError::CommmonError(format!( 36 | "File {} does not exist", 37 | path 38 | ))); 39 | } 40 | 41 | return Ok(fs::read_to_string(&path)?); 42 | } 43 | 44 | pub fn now_second() -> u64 { 45 | return SystemTime::now() 46 | .duration_since(UNIX_EPOCH) 47 | .unwrap() 48 | .as_secs(); 49 | } -------------------------------------------------------------------------------- /src/common/base/tests/test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /src/placement-center/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "placement-center" 17 | version.workspace = true 18 | edition.workspace = true 19 | license.workspace = true 20 | 21 | 22 | [dependencies] 23 | log.workspace = true 24 | serde.workspace = true 25 | log4rs.workspace = true 26 | axum.workspace = true 27 | common-base.workspace = true 28 | tokio.workspace = true 29 | tonic.workspace = true 30 | prost.workspace = true 31 | tonic-build.workspace = true 32 | tokio-util.workspace = true 33 | protocol.workspace = true 34 | dashmap.workspace = true 35 | serde_json.workspace = true 36 | rocksdb.workspace = true 37 | raft = { git = "https://github.com/robustmq/raft-rs", features = [ 38 | "prost-codec", 39 | ], default-features = false } 40 | 41 | bincode.workspace = true 42 | slog = "2" 43 | slog-term = "2.9.0" 44 | slog-async = "2.8.0" 45 | clients.workspace = true 46 | openraft.workspace = true 47 | byteorder.workspace = true 48 | tracing.workspace = true 49 | tracing-subscriber.workspace = true 50 | mobc.workspace = true 51 | -------------------------------------------------------------------------------- /src/placement-center/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::{Arc, RwLock}; 16 | 17 | use clients::poll::ClientPool; 18 | use common_base::config::placement_center::placement_center_conf; 19 | use log::info; 20 | use openraft::raft_node::{create_raft_node, start_openraft_node}; 21 | use raft::{ 22 | apply::{RaftMachineApply, RaftMessage}, 23 | machine::RaftMachine, 24 | metadata::RaftGroupMetadata, 25 | peer::PeerMessage, 26 | route::DataRoute, 27 | }; 28 | use server::{ 29 | grpc::server::start_grpc_server, 30 | http::server::{start_http_server, HttpServerState}, 31 | }; 32 | use storage::{raft::RaftMachineStorage, rocksdb::RocksDBEngine}; 33 | use tokio::{ 34 | signal, 35 | sync::{broadcast, mpsc}, 36 | }; 37 | 38 | pub mod openraft; 39 | pub mod raft; 40 | pub mod requests; 41 | pub mod server; 42 | pub mod storage; 43 | 44 | pub async fn start_server(stop_sx: broadcast::Sender) { 45 | let config = placement_center_conf(); 46 | let (raft_message_send, raft_message_recv) = mpsc::channel::(1000); 47 | let (peer_message_send, _) = mpsc::channel::(1000); 48 | 49 | let placement_cache = Arc::new(RwLock::new(RaftGroupMetadata::new())); 50 | 51 | let placement_center_storage = Arc::new(RaftMachineApply::new(raft_message_send)); 52 | let rocksdb_engine_handler: Arc = Arc::new(RocksDBEngine::new(&config)); 53 | 54 | let raft_machine_storage = Arc::new(RwLock::new(RaftMachineStorage::new( 55 | rocksdb_engine_handler.clone(), 56 | ))); 57 | 58 | let data_route = Arc::new(RwLock::new(DataRoute::new(rocksdb_engine_handler.clone()))); 59 | 60 | let mut raft: RaftMachine = RaftMachine::new( 61 | placement_cache.clone(), 62 | data_route, 63 | peer_message_send, 64 | raft_message_recv, 65 | stop_sx.subscribe(), 66 | raft_machine_storage.clone(), 67 | ); 68 | 69 | let client_poll = Arc::new(ClientPool::new(3)); 70 | 71 | let (openraft_node, kvs) = create_raft_node(client_poll.clone()).await; 72 | 73 | let raw_stop_sx = stop_sx.clone(); 74 | let tmp_openraft_node = openraft_node.clone(); 75 | tokio::spawn(async move { 76 | start_grpc_server( 77 | client_poll, 78 | tmp_openraft_node, 79 | placement_center_storage, 80 | rocksdb_engine_handler, 81 | placement_cache, 82 | raw_stop_sx, 83 | ) 84 | .await; 85 | }); 86 | 87 | let tmp_openraft_node = openraft_node.clone(); 88 | tokio::spawn(async move { 89 | start_openraft_node(openraft_node).await; 90 | }); 91 | 92 | let raw_stop_sx = stop_sx.clone(); 93 | tokio::spawn(async move { 94 | let state = HttpServerState::new(tmp_openraft_node, kvs); 95 | start_http_server(state, raw_stop_sx).await; 96 | }); 97 | 98 | // tokio::spawn(async move { 99 | // raft.run().await; 100 | // }); 101 | 102 | awaiting_stop(stop_sx.clone()).await; 103 | } 104 | 105 | pub async fn awaiting_stop(stop_send: broadcast::Sender) { 106 | signal::ctrl_c().await.expect("failed to listen for event"); 107 | match stop_send.send(true) { 108 | Ok(_) => { 109 | info!( 110 | "{}", 111 | "When ctrl + c is received, the service starts to stop" 112 | ); 113 | } 114 | Err(e) => { 115 | panic!("{}", e); 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/error.rs: -------------------------------------------------------------------------------- 1 | use std::fmt::Display; 2 | 3 | use common_base::errors::RobustMQError; 4 | use openraft::error::{NetworkError, RPCError, Unreachable}; 5 | 6 | use super::typeconfig::TypeConfig; 7 | 8 | #[derive(Debug)] 9 | struct ErrWrap(Box); 10 | 11 | impl Display for ErrWrap { 12 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 13 | self.0.fmt(f) 14 | } 15 | } 16 | 17 | impl std::error::Error for ErrWrap {} 18 | 19 | pub fn to_error( 20 | e: RobustMQError, 21 | ) -> RPCError { 22 | RPCError::Unreachable(Unreachable::new(&e)) 23 | } 24 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod network; 2 | pub mod raft_node; 3 | pub mod route; 4 | pub mod store; 5 | pub mod typeconfig; 6 | pub mod error; -------------------------------------------------------------------------------- /src/placement-center/src/openraft/network/connection.rs: -------------------------------------------------------------------------------- 1 | use bincode::{deserialize, serialize}; 2 | use clients::{placement::openraft::OpenRaftServiceManager, poll::ClientPool}; 3 | use common_base::errors::RobustMQError; 4 | use mobc::Connection; 5 | use openraft::{ 6 | error::{InstallSnapshotError, RPCError, RaftError}, 7 | network::RPCOption, 8 | raft::{ 9 | AppendEntriesRequest, AppendEntriesResponse, InstallSnapshotRequest, 10 | InstallSnapshotResponse, VoteRequest, VoteResponse, 11 | }, 12 | RaftNetwork, 13 | }; 14 | use protocol::openraft::{AppendRequest, SnapshotRequest}; 15 | use std::sync::Arc; 16 | 17 | use crate::openraft::{error::to_error, raft_node::NodeId, typeconfig::TypeConfig}; 18 | 19 | pub struct NetworkConnection { 20 | addr: String, 21 | client_poll: Arc, 22 | target: NodeId, 23 | } 24 | impl NetworkConnection { 25 | pub fn new(addr: String, client_poll: Arc, target: NodeId) -> Self { 26 | return NetworkConnection { 27 | addr, 28 | client_poll, 29 | target, 30 | }; 31 | } 32 | 33 | async fn c(&mut self) -> Result, RobustMQError> { 34 | return Ok(self 35 | .client_poll 36 | .placement_center_openraft_services_client(self.addr.clone()) 37 | .await?); 38 | } 39 | } 40 | 41 | #[allow(clippy::blocks_in_conditions)] 42 | impl RaftNetwork for NetworkConnection { 43 | 44 | async fn append_entries( 45 | &mut self, 46 | req: AppendEntriesRequest, 47 | _option: RPCOption, 48 | ) -> Result, RPCError>> 49 | { 50 | 51 | let mut c = match self.c().await { 52 | Ok(conn) => conn, 53 | Err(e) => return Err(to_error(e)), 54 | }; 55 | 56 | let value = match serialize(&req) { 57 | Ok(data) => data, 58 | Err(e) => return Err(to_error(RobustMQError::CommmonError(e.to_string()))), 59 | }; 60 | 61 | let request = AppendRequest { value }; 62 | 63 | let reply = match c.append(request).await { 64 | Ok(reply) => reply.into_inner(), 65 | Err(e) => return Err(to_error(RobustMQError::CommmonError(e.to_string()))), 66 | }; 67 | 68 | let result = match deserialize(&reply.value) { 69 | Ok(data) => data, 70 | Err(e) => return Err(to_error(RobustMQError::CommmonError(e.to_string()))), 71 | }; 72 | 73 | return Ok(result); 74 | } 75 | 76 | async fn install_snapshot( 77 | &mut self, 78 | req: InstallSnapshotRequest, 79 | _option: RPCOption, 80 | ) -> Result< 81 | InstallSnapshotResponse, 82 | RPCError>, 83 | > { 84 | tracing::debug!(req = debug(&req), "install_snapshot"); 85 | 86 | let mut c = match self.c().await { 87 | Ok(conn) => conn, 88 | Err(e) => return Err(to_error(e)), 89 | }; 90 | 91 | let value = match serialize(&req) { 92 | Ok(data) => data, 93 | Err(e) => return Err(to_error(RobustMQError::CommmonError(e.to_string()))), 94 | }; 95 | 96 | let request = SnapshotRequest { value }; 97 | 98 | let reply = match c.snapshot(request).await { 99 | Ok(reply) => reply.into_inner(), 100 | Err(e) => return Err(to_error(RobustMQError::CommmonError(e.to_string()))), 101 | }; 102 | let result = match deserialize(&reply.value) { 103 | Ok(data) => data, 104 | Err(e) => return Err(to_error(RobustMQError::CommmonError(e.to_string()))), 105 | }; 106 | 107 | return Ok(result); 108 | } 109 | 110 | async fn vote( 111 | &mut self, 112 | req: VoteRequest, 113 | _option: RPCOption, 114 | ) -> Result, RPCError>> { 115 | tracing::debug!(req = debug(&req), "vote"); 116 | let mut c = match self.c().await { 117 | Ok(conn) => conn, 118 | Err(e) => return Err(to_error(e)), 119 | }; 120 | 121 | let value = match serialize(&req) { 122 | Ok(data) => data, 123 | Err(e) => return Err(to_error(RobustMQError::CommmonError(e.to_string()))), 124 | }; 125 | 126 | let request = protocol::openraft::VoteRequest { value }; 127 | 128 | let reply = match c.vote(request).await { 129 | Ok(reply) => reply.into_inner(), 130 | Err(e) => return Err(to_error(RobustMQError::CommmonError(e.to_string()))), 131 | }; 132 | let result = match deserialize(&reply.value) { 133 | Ok(data) => data, 134 | Err(e) => return Err(to_error(RobustMQError::CommmonError(e.to_string()))), 135 | }; 136 | 137 | return Ok(result); 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/network/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod connection; 2 | pub mod network; 3 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/network/network.rs: -------------------------------------------------------------------------------- 1 | use clients::poll::ClientPool; 2 | use log::info; 3 | use openraft::RaftNetworkFactory; 4 | use std::sync::Arc; 5 | 6 | use super::connection::NetworkConnection; 7 | use crate::openraft::{ 8 | raft_node::{Node, NodeId}, 9 | typeconfig::TypeConfig, 10 | }; 11 | 12 | pub struct Network { 13 | client_poll: Arc, 14 | } 15 | 16 | impl Network { 17 | pub fn new(client_poll: Arc) -> Network { 18 | return Network { client_poll }; 19 | } 20 | } 21 | 22 | // NOTE: This could be implemented also on `Arc`, but since it's empty, implemented 23 | // directly. 24 | impl RaftNetworkFactory for Network { 25 | type Network = NetworkConnection; 26 | 27 | #[tracing::instrument(level = "debug", skip_all)] 28 | async fn new_client(&mut self, target: NodeId, node: &Node) -> Self::Network { 29 | let addr = format!("{}", node.rpc_addr); 30 | return NetworkConnection::new(addr, self.client_poll.clone(), target); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/raft_node.rs: -------------------------------------------------------------------------------- 1 | use crate::raft::node; 2 | 3 | use super::network::network::Network; 4 | use super::store::new_storage; 5 | use super::typeconfig::TypeConfig; 6 | use clients::poll::ClientPool; 7 | use common_base::config::placement_center::placement_center_conf; 8 | use log::{error, info}; 9 | use openraft::{Config, Raft}; 10 | use std::collections::BTreeMap; 11 | use std::fmt::Display; 12 | use std::path::Path; 13 | use std::sync::Arc; 14 | use std::time::Duration; 15 | use tokio::sync::RwLock; 16 | use tokio::time::sleep; 17 | pub type NodeId = u64; 18 | 19 | #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Default)] 20 | pub struct Node { 21 | pub node_id: u64, 22 | pub rpc_addr: String, 23 | } 24 | 25 | impl Display for Node { 26 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { 27 | write!( 28 | f, 29 | "Node {{ rpc_addr: {}, node_id: {} }}", 30 | self.rpc_addr, self.node_id 31 | ) 32 | } 33 | } 34 | 35 | pub mod typ { 36 | use openraft::error::Infallible; 37 | 38 | use crate::openraft::typeconfig::TypeConfig; 39 | 40 | pub type Entry = openraft::Entry; 41 | 42 | pub type RaftError = openraft::error::RaftError; 43 | pub type RPCError = openraft::error::RPCError>; 44 | 45 | pub type ClientWriteError = openraft::error::ClientWriteError; 46 | pub type CheckIsLeaderError = openraft::error::CheckIsLeaderError; 47 | pub type ForwardToLeader = openraft::error::ForwardToLeader; 48 | pub type InitializeError = openraft::error::InitializeError; 49 | 50 | pub type ClientWriteResponse = openraft::raft::ClientWriteResponse; 51 | } 52 | 53 | pub type ExampleRaft = openraft::Raft; 54 | 55 | pub async fn start_openraft_node(raft_node: Raft) { 56 | let conf = placement_center_conf(); 57 | let mut nodes = BTreeMap::new(); 58 | for (node_id, addr) in conf.nodes.clone() { 59 | let mut addr = addr.to_string(); 60 | addr = addr.replace("\"", ""); 61 | let node = Node { 62 | rpc_addr: addr, 63 | node_id: node_id.parse().unwrap(), 64 | }; 65 | 66 | nodes.insert(node.node_id, node); 67 | } 68 | 69 | info!("Raft Nodes:{:?}", nodes); 70 | let init_node_id = calc_init_node(&nodes); 71 | if init_node_id == conf.node_id { 72 | match raft_node.is_initialized().await { 73 | Ok(flag) => { 74 | info!("Whether nodes should be initialized, flag={}", flag); 75 | if !flag { 76 | match raft_node.initialize(nodes.clone()).await { 77 | Ok(_) => { 78 | info!("Node {:?} was initialized successfully", nodes); 79 | } 80 | Err(e) => { 81 | panic!("openraft init fail,{}", e.to_string()); 82 | } 83 | } 84 | } 85 | } 86 | Err(e) => { 87 | panic!("openraft initialized fail,{}", e.to_string()); 88 | } 89 | } 90 | } 91 | 92 | // if init_node_id == conf.node_id { 93 | // if let Some(local) = nodes.get(&conf.node_id) { 94 | // info!("Start trying to initialize node:{}", local.node_id); 95 | // let mut init_nodes = BTreeMap::new(); 96 | // init_nodes.insert(local.node_id, local.clone()); 97 | // match raft_node.is_initialized().await { 98 | // Ok(flag) => { 99 | // info!("Whether nodes should be initialized, flag={}", flag); 100 | // if !flag { 101 | // match raft_node.initialize(init_nodes).await { 102 | // Ok(_) => { 103 | // info!("Node {} was initialized successfully", conf.node_id); 104 | // } 105 | // Err(e) => { 106 | // panic!("openraft init fail,{}", e.to_string()); 107 | // } 108 | // } 109 | // } 110 | // } 111 | // Err(e) => { 112 | // panic!("openraft initialized fail,{}", e.to_string()); 113 | // } 114 | // } 115 | // } 116 | 117 | // // wait learn ready 118 | // sleep(Duration::from_secs(10)).await; 119 | 120 | // for (node_id, node) in nodes { 121 | // if node_id == conf.node_id { 122 | // continue; 123 | // } 124 | // info!("Start adding learner node {}", node_id); 125 | // match raft_node.add_learner(node_id, node, true).await { 126 | // Ok(data) => { 127 | // info!( 128 | // "Learner node {} was added successfullys, res:{:?}", 129 | // node_id, data 130 | // ); 131 | // } 132 | // Err(e) => { 133 | // error!( 134 | // "Failed to add the learner node, error message :{}", 135 | // e.to_string() 136 | // ); 137 | // } 138 | // } 139 | // } 140 | // } 141 | } 142 | 143 | pub fn calc_init_node(nodes: &BTreeMap) -> u64 { 144 | let mut node_ids: Vec = nodes.keys().map(|x| x.clone()).collect(); 145 | node_ids.sort(); 146 | return node_ids.first().unwrap().clone(); 147 | } 148 | 149 | pub async fn create_raft_node( 150 | client_poll: Arc, 151 | ) -> (Raft, Arc>>) { 152 | let config = Config { 153 | heartbeat_interval: 250, 154 | election_timeout_min: 299, 155 | ..Default::default() 156 | }; 157 | 158 | let config = Arc::new(config.validate().unwrap()); 159 | let conf = placement_center_conf(); 160 | let path = format!("{}/_engine_storage", conf.data_path.clone()); 161 | let dir = Path::new(&path); 162 | let (log_store, state_machine_store) = new_storage(&dir).await; 163 | let kvs = state_machine_store.data.kvs.clone(); 164 | 165 | let network = Network::new(client_poll); 166 | let raft = openraft::Raft::new( 167 | conf.node_id, 168 | config.clone(), 169 | network, 170 | log_store, 171 | state_machine_store, 172 | ) 173 | .await 174 | .unwrap(); 175 | 176 | return (raft, kvs); 177 | } 178 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/route/mod.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | 3 | /** 4 | * Here you will set the types of request that will interact with the raft nodes. 5 | * For example the `Set` will be used to write data (key and value) to the raft database. 6 | * The `AddNode` will append a new node to the current existing shared list of nodes. 7 | * You will want to add any request that can write data in all nodes here. 8 | */ 9 | #[derive(Serialize, Deserialize, Debug, Clone)] 10 | pub enum AppRequestData { 11 | Set { key: String, value: String }, 12 | 13 | Delete { key: String }, 14 | } 15 | 16 | /** 17 | * Here you will defined what type of answer you expect from reading the data of a node. 18 | * In this example it will return a optional value from a given key in 19 | * the `ExampleRequest.Set`. 20 | * 21 | * TODO: Should we explain how to create multiple `AppDataResponse`? 22 | * 23 | */ 24 | #[derive(Serialize, Deserialize, Debug, Clone)] 25 | pub struct AppResponseData { 26 | pub value: Option, 27 | } 28 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/store/log_store.rs: -------------------------------------------------------------------------------- 1 | use super::{id_to_bin, StorageResult}; 2 | use crate::openraft::{raft_node::NodeId, store::bin_to_id, typeconfig::TypeConfig}; 3 | use openraft::{ 4 | storage::{IOFlushed, RaftLogStorage}, 5 | AnyError, Entry, ErrorSubject, ErrorVerb, LogId, LogState, OptionalSend, RaftLogReader, 6 | StorageError, Vote, 7 | }; 8 | use rocksdb::{ColumnFamily, Direction, DB}; 9 | use std::{fmt::Debug, ops::RangeBounds, sync::Arc}; 10 | 11 | #[derive(Debug, Clone)] 12 | pub struct LogStore { 13 | pub db: Arc, 14 | } 15 | 16 | impl LogStore { 17 | fn store(&self) -> &ColumnFamily { 18 | self.db.cf_handle("_raft_store").unwrap() 19 | } 20 | 21 | fn logs(&self) -> &ColumnFamily { 22 | self.db.cf_handle("_raft_logs").unwrap() 23 | } 24 | 25 | fn flush( 26 | &self, 27 | subject: ErrorSubject, 28 | verb: ErrorVerb, 29 | ) -> Result<(), StorageError> { 30 | self.db 31 | .flush_wal(true) 32 | .map_err(|e| StorageError::new(subject, verb, AnyError::new(&e)))?; 33 | Ok(()) 34 | } 35 | 36 | fn get_last_purged_(&self) -> StorageResult>> { 37 | Ok(self 38 | .db 39 | .get_cf(self.store(), b"last_purged_log_id") 40 | .map_err(|e| StorageError::read(&e))? 41 | .and_then(|v| serde_json::from_slice(&v).ok())) 42 | } 43 | 44 | fn set_last_purged_(&self, log_id: LogId) -> StorageResult<()> { 45 | self.db 46 | .put_cf( 47 | self.store(), 48 | b"last_purged_log_id", 49 | serde_json::to_vec(&log_id).unwrap().as_slice(), 50 | ) 51 | .map_err(|e| StorageError::write(&e))?; 52 | 53 | self.flush(ErrorSubject::Store, ErrorVerb::Write)?; 54 | Ok(()) 55 | } 56 | 57 | fn set_committed_( 58 | &self, 59 | committed: &Option>, 60 | ) -> Result<(), StorageError> { 61 | let json = serde_json::to_vec(committed).unwrap(); 62 | 63 | self.db 64 | .put_cf(self.store(), b"committed", json) 65 | .map_err(|e| StorageError::write(&e))?; 66 | 67 | self.flush(ErrorSubject::Store, ErrorVerb::Write)?; 68 | Ok(()) 69 | } 70 | 71 | fn get_committed_(&self) -> StorageResult>> { 72 | Ok(self 73 | .db 74 | .get_cf(self.store(), b"committed") 75 | .map_err(|e| StorageError::read(&e))? 76 | .and_then(|v| serde_json::from_slice(&v).ok())) 77 | } 78 | 79 | fn set_vote_(&self, vote: &Vote) -> StorageResult<()> { 80 | self.db 81 | .put_cf(self.store(), b"vote", serde_json::to_vec(vote).unwrap()) 82 | .map_err(|e| StorageError::write_vote(&e))?; 83 | 84 | self.flush(ErrorSubject::Vote, ErrorVerb::Write)?; 85 | Ok(()) 86 | } 87 | 88 | fn get_vote_(&self) -> StorageResult>> { 89 | Ok(self 90 | .db 91 | .get_cf(self.store(), b"vote") 92 | .map_err(|e| StorageError::write_vote(&e))? 93 | .and_then(|v| serde_json::from_slice(&v).ok())) 94 | } 95 | } 96 | 97 | impl RaftLogReader for LogStore { 98 | async fn try_get_log_entries + Clone + Debug + OptionalSend>( 99 | &mut self, 100 | range: RB, 101 | ) -> StorageResult>> { 102 | let start = match range.start_bound() { 103 | std::ops::Bound::Included(x) => id_to_bin(*x), 104 | std::ops::Bound::Excluded(x) => id_to_bin(*x + 1), 105 | std::ops::Bound::Unbounded => id_to_bin(0), 106 | }; 107 | self.db 108 | .iterator_cf( 109 | self.logs(), 110 | rocksdb::IteratorMode::From(&start, Direction::Forward), 111 | ) 112 | .map(|res| { 113 | let (id, val) = res.unwrap(); 114 | let entry: StorageResult> = 115 | serde_json::from_slice(&val).map_err(|e| StorageError::read_logs(&e)); 116 | let id = bin_to_id(&id); 117 | 118 | assert_eq!(Ok(id), entry.as_ref().map(|e| e.log_id.index)); 119 | (id, entry) 120 | }) 121 | .take_while(|(id, _)| range.contains(id)) 122 | .map(|x| x.1) 123 | .collect() 124 | } 125 | 126 | async fn read_vote(&mut self) -> Result>, StorageError> { 127 | self.get_vote_() 128 | } 129 | } 130 | 131 | impl RaftLogStorage for LogStore { 132 | type LogReader = Self; 133 | 134 | async fn get_log_state(&mut self) -> StorageResult> { 135 | let last = self 136 | .db 137 | .iterator_cf(self.logs(), rocksdb::IteratorMode::End) 138 | .next() 139 | .and_then(|res| { 140 | let (_, ent) = res.unwrap(); 141 | Some( 142 | serde_json::from_slice::>(&ent) 143 | .ok()? 144 | .log_id, 145 | ) 146 | }); 147 | 148 | let last_purged_log_id = self.get_last_purged_()?; 149 | 150 | let last_log_id = match last { 151 | None => last_purged_log_id, 152 | Some(x) => Some(x), 153 | }; 154 | Ok(LogState { 155 | last_purged_log_id, 156 | last_log_id, 157 | }) 158 | } 159 | 160 | async fn save_committed( 161 | &mut self, 162 | _committed: Option>, 163 | ) -> Result<(), StorageError> { 164 | self.set_committed_(&_committed)?; 165 | Ok(()) 166 | } 167 | 168 | async fn read_committed(&mut self) -> Result>, StorageError> { 169 | let c = self.get_committed_()?; 170 | Ok(c) 171 | } 172 | 173 | #[tracing::instrument(level = "trace", skip(self))] 174 | async fn save_vote(&mut self, vote: &Vote) -> Result<(), StorageError> { 175 | self.set_vote_(vote) 176 | } 177 | 178 | #[tracing::instrument(level = "trace", skip_all)] 179 | async fn append(&mut self, entries: I, callback: IOFlushed) -> StorageResult<()> 180 | where 181 | I: IntoIterator> + Send, 182 | I::IntoIter: Send, 183 | { 184 | for entry in entries { 185 | let id = id_to_bin(entry.log_id.index); 186 | assert_eq!(bin_to_id(&id), entry.log_id.index); 187 | self.db 188 | .put_cf( 189 | self.logs(), 190 | id, 191 | serde_json::to_vec(&entry).map_err(|e| StorageError::write_logs(&e))?, 192 | ) 193 | .map_err(|e| StorageError::write_logs(&e))?; 194 | } 195 | 196 | callback.io_completed(Ok(())); 197 | 198 | Ok(()) 199 | } 200 | 201 | #[tracing::instrument(level = "debug", skip(self))] 202 | async fn truncate(&mut self, log_id: LogId) -> StorageResult<()> { 203 | tracing::debug!("delete_log: [{:?}, +oo)", log_id); 204 | 205 | let from = id_to_bin(log_id.index); 206 | let to = id_to_bin(0xff_ff_ff_ff_ff_ff_ff_ff); 207 | self.db 208 | .delete_range_cf(self.logs(), &from, &to) 209 | .map_err(|e| StorageError::write_logs(&e)) 210 | } 211 | 212 | #[tracing::instrument(level = "debug", skip(self))] 213 | async fn purge(&mut self, log_id: LogId) -> Result<(), StorageError> { 214 | tracing::debug!("delete_log: [0, {:?}]", log_id); 215 | 216 | self.set_last_purged_(log_id)?; 217 | let from = id_to_bin(0); 218 | let to = id_to_bin(log_id.index + 1); 219 | self.db 220 | .delete_range_cf(self.logs(), &from, &to) 221 | .map_err(|e| StorageError::write_logs(&e)) 222 | } 223 | 224 | async fn get_log_reader(&mut self) -> Self::LogReader { 225 | self.clone() 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/store/mod.rs: -------------------------------------------------------------------------------- 1 | use std::path::Path; 2 | use std::sync::Arc; 3 | 4 | use super::typeconfig::TypeConfig; 5 | use byteorder::BigEndian; 6 | use byteorder::ReadBytesExt; 7 | use byteorder::WriteBytesExt; 8 | use log_store::LogStore; 9 | use openraft::{SnapshotMeta, StorageError}; 10 | use rocksdb::ColumnFamilyDescriptor; 11 | use rocksdb::Options; 12 | use rocksdb::DB; 13 | use serde::{Deserialize, Serialize}; 14 | use state_machine_store::StateMachineStore; 15 | #[derive(Serialize, Deserialize, Debug, Clone)] 16 | pub struct StoredSnapshot { 17 | pub meta: SnapshotMeta, 18 | 19 | /// The data of the state machine at the time of this snapshot. 20 | pub data: Vec, 21 | } 22 | 23 | type StorageResult = Result>; 24 | 25 | pub mod log_store; 26 | pub mod state_machine_store; 27 | 28 | /// converts an id to a byte vector for storing in the database. 29 | /// Note that we're using big endian encoding to ensure correct sorting of keys 30 | fn id_to_bin(id: u64) -> Vec { 31 | let mut buf = Vec::with_capacity(8); 32 | buf.write_u64::(id).unwrap(); 33 | buf 34 | } 35 | 36 | fn bin_to_id(buf: &[u8]) -> u64 { 37 | (&buf[0..8]).read_u64::().unwrap() 38 | } 39 | 40 | pub(crate) async fn new_storage>(db_path: P) -> (LogStore, StateMachineStore) { 41 | let mut db_opts = Options::default(); 42 | db_opts.create_missing_column_families(true); 43 | db_opts.create_if_missing(true); 44 | 45 | let store = ColumnFamilyDescriptor::new("_raft_store", Options::default()); 46 | let logs = ColumnFamilyDescriptor::new("_raft_logs", Options::default()); 47 | 48 | let db = DB::open_cf_descriptors(&db_opts, db_path, vec![store, logs]).unwrap(); 49 | let db = Arc::new(db); 50 | 51 | let log_store = LogStore { db: db.clone() }; 52 | let sm_store = StateMachineStore::new(db).await.unwrap(); 53 | 54 | (log_store, sm_store) 55 | } 56 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/store/state_machine_store.rs: -------------------------------------------------------------------------------- 1 | use openraft::{ 2 | storage::RaftStateMachine, AnyError, EntryPayload, ErrorSubject, ErrorVerb, LogId, 3 | OptionalSend, RaftSnapshotBuilder, Snapshot, SnapshotMeta, StorageError, StoredMembership, 4 | }; 5 | use rocksdb::{ColumnFamily, DB}; 6 | use std::{collections::BTreeMap, io::Cursor, sync::Arc}; 7 | use tokio::sync::RwLock; 8 | 9 | use crate::openraft::{ 10 | raft_node::{typ, NodeId}, 11 | route::{AppRequestData, AppResponseData}, 12 | typeconfig::{SnapshotData, TypeConfig}, 13 | }; 14 | 15 | use super::{StorageResult, StoredSnapshot}; 16 | 17 | #[derive(Debug, Clone)] 18 | pub struct StateMachineStore { 19 | pub data: StateMachineData, 20 | 21 | /// snapshot index is not persisted in this example. 22 | /// 23 | /// It is only used as a suffix of snapshot id, and should be globally unique. 24 | /// In practice, using a timestamp in micro-second would be good enough. 25 | snapshot_idx: u64, 26 | 27 | /// State machine stores snapshot in db. 28 | db: Arc, 29 | } 30 | 31 | #[derive(Debug, Clone)] 32 | pub struct StateMachineData { 33 | pub last_applied_log_id: Option>, 34 | 35 | pub last_membership: StoredMembership, 36 | 37 | /// State built from applying the raft logs 38 | pub kvs: Arc>>, 39 | } 40 | 41 | impl RaftSnapshotBuilder for StateMachineStore { 42 | async fn build_snapshot(&mut self) -> Result, StorageError> { 43 | let last_applied_log = self.data.last_applied_log_id; 44 | let last_membership = self.data.last_membership.clone(); 45 | 46 | let kv_json = { 47 | let kvs = self.data.kvs.read().await; 48 | serde_json::to_vec(&*kvs).map_err(|e| StorageError::read_state_machine(&e))? 49 | }; 50 | 51 | let snapshot_id = if let Some(last) = last_applied_log { 52 | format!("{}-{}-{}", last.leader_id, last.index, self.snapshot_idx) 53 | } else { 54 | format!("--{}", self.snapshot_idx) 55 | }; 56 | 57 | let meta = SnapshotMeta { 58 | last_log_id: last_applied_log, 59 | last_membership, 60 | snapshot_id, 61 | }; 62 | 63 | let snapshot = StoredSnapshot { 64 | meta: meta.clone(), 65 | data: kv_json.clone(), 66 | }; 67 | 68 | self.set_current_snapshot_(snapshot)?; 69 | 70 | Ok(Snapshot { 71 | meta, 72 | snapshot: Box::new(Cursor::new(kv_json)), 73 | }) 74 | } 75 | } 76 | 77 | impl StateMachineStore { 78 | pub async fn new(db: Arc) -> Result> { 79 | let mut sm = Self { 80 | data: StateMachineData { 81 | last_applied_log_id: None, 82 | last_membership: Default::default(), 83 | kvs: Arc::new(Default::default()), 84 | }, 85 | snapshot_idx: 0, 86 | db, 87 | }; 88 | 89 | let snapshot = sm.get_current_snapshot_()?; 90 | if let Some(snap) = snapshot { 91 | sm.update_state_machine_(snap).await?; 92 | } 93 | 94 | Ok(sm) 95 | } 96 | 97 | async fn update_state_machine_( 98 | &mut self, 99 | snapshot: StoredSnapshot, 100 | ) -> Result<(), StorageError> { 101 | let kvs: BTreeMap = serde_json::from_slice(&snapshot.data) 102 | .map_err(|e| StorageError::read_snapshot(Some(snapshot.meta.signature()), &e))?; 103 | 104 | self.data.last_applied_log_id = snapshot.meta.last_log_id; 105 | self.data.last_membership = snapshot.meta.last_membership.clone(); 106 | let mut x = self.data.kvs.write().await; 107 | *x = kvs; 108 | 109 | Ok(()) 110 | } 111 | 112 | fn get_current_snapshot_(&self) -> StorageResult> { 113 | Ok(self 114 | .db 115 | .get_cf(self.store(), b"snapshot") 116 | .map_err(|e| StorageError::read(&e))? 117 | .and_then(|v| serde_json::from_slice(&v).ok())) 118 | } 119 | 120 | fn set_current_snapshot_(&self, snap: StoredSnapshot) -> StorageResult<()> { 121 | self.db 122 | .put_cf( 123 | self.store(), 124 | b"snapshot", 125 | serde_json::to_vec(&snap).unwrap().as_slice(), 126 | ) 127 | .map_err(|e| StorageError::write_snapshot(Some(snap.meta.signature()), &e))?; 128 | self.flush( 129 | ErrorSubject::Snapshot(Some(snap.meta.signature())), 130 | ErrorVerb::Write, 131 | )?; 132 | Ok(()) 133 | } 134 | 135 | fn flush( 136 | &self, 137 | subject: ErrorSubject, 138 | verb: ErrorVerb, 139 | ) -> Result<(), StorageError> { 140 | self.db 141 | .flush_wal(true) 142 | .map_err(|e| StorageError::new(subject, verb, AnyError::new(&e)))?; 143 | Ok(()) 144 | } 145 | 146 | fn store(&self) -> &ColumnFamily { 147 | self.db.cf_handle("_raft_store").unwrap() 148 | } 149 | } 150 | 151 | impl RaftStateMachine for StateMachineStore { 152 | type SnapshotBuilder = Self; 153 | 154 | async fn applied_state( 155 | &mut self, 156 | ) -> Result<(Option>, StoredMembership), StorageError> 157 | { 158 | Ok(( 159 | self.data.last_applied_log_id, 160 | self.data.last_membership.clone(), 161 | )) 162 | } 163 | 164 | async fn apply( 165 | &mut self, 166 | entries: I, 167 | ) -> Result, StorageError> 168 | where 169 | I: IntoIterator + OptionalSend, 170 | I::IntoIter: OptionalSend, 171 | { 172 | let entries = entries.into_iter(); 173 | let mut replies = Vec::with_capacity(entries.size_hint().0); 174 | 175 | for ent in entries { 176 | self.data.last_applied_log_id = Some(ent.log_id); 177 | 178 | let mut resp_value = None; 179 | 180 | match ent.payload { 181 | EntryPayload::Blank => {} 182 | EntryPayload::Normal(req) => match req { 183 | AppRequestData::Set { key, value } => { 184 | resp_value = Some(value.clone()); 185 | 186 | let mut st = self.data.kvs.write().await; 187 | st.insert(key, value); 188 | } 189 | AppRequestData::Delete { key } => { 190 | let mut st = self.data.kvs.write().await; 191 | st.remove(&key); 192 | } 193 | }, 194 | EntryPayload::Membership(mem) => { 195 | self.data.last_membership = StoredMembership::new(Some(ent.log_id), mem); 196 | } 197 | } 198 | 199 | replies.push(AppResponseData { value: resp_value }); 200 | } 201 | Ok(replies) 202 | } 203 | 204 | async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder { 205 | self.snapshot_idx += 1; 206 | self.clone() 207 | } 208 | 209 | async fn begin_receiving_snapshot( 210 | &mut self, 211 | ) -> Result>>, StorageError> { 212 | Ok(Box::new(Cursor::new(Vec::new()))) 213 | } 214 | 215 | async fn install_snapshot( 216 | &mut self, 217 | meta: &SnapshotMeta, 218 | snapshot: Box, 219 | ) -> Result<(), StorageError> { 220 | let new_snapshot = StoredSnapshot { 221 | meta: meta.clone(), 222 | data: snapshot.into_inner(), 223 | }; 224 | 225 | self.update_state_machine_(new_snapshot.clone()).await?; 226 | 227 | self.set_current_snapshot_(new_snapshot)?; 228 | 229 | Ok(()) 230 | } 231 | 232 | async fn get_current_snapshot( 233 | &mut self, 234 | ) -> Result>, StorageError> { 235 | let x = self.get_current_snapshot_()?; 236 | Ok(x.map(|s| Snapshot { 237 | meta: s.meta.clone(), 238 | snapshot: Box::new(Cursor::new(s.data.clone())), 239 | })) 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /src/placement-center/src/openraft/typeconfig.rs: -------------------------------------------------------------------------------- 1 | use crate::openraft::raft_node::Node; 2 | use crate::openraft::route::AppRequestData; 3 | use crate::openraft::route::AppResponseData; 4 | use std::io::Cursor; 5 | 6 | pub type SnapshotData = Cursor>; 7 | 8 | openraft::declare_raft_types!( 9 | pub TypeConfig: 10 | D = AppRequestData, 11 | R = AppResponseData, 12 | Node = Node, 13 | ); 14 | -------------------------------------------------------------------------------- /src/placement-center/src/raft/apply.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use bincode::serialize; 16 | use common_base::errors::RobustMQError; 17 | use raft::eraftpb::ConfChange; 18 | use raft::eraftpb::Message as raftPreludeMessage; 19 | use serde::Deserialize; 20 | use serde::Serialize; 21 | use std::fmt; 22 | use std::time::Duration; 23 | use tokio::sync::oneshot; 24 | use tokio::sync::oneshot::Receiver; 25 | use tokio::sync::oneshot::Sender; 26 | use tokio::time::timeout; 27 | 28 | pub enum RaftResponseMesage { 29 | Success, 30 | Fail, 31 | } 32 | pub enum RaftMessage { 33 | ConfChange { 34 | change: ConfChange, 35 | chan: Sender, 36 | }, 37 | 38 | // Received a message from another node 39 | Raft { 40 | message: raftPreludeMessage, 41 | chan: Sender, 42 | }, 43 | 44 | TransferLeader { 45 | node_id: u64, 46 | chan: Sender, 47 | }, 48 | 49 | // The data sent by the client is received. Procedure 50 | Propose { 51 | data: Vec, 52 | chan: Sender, 53 | }, 54 | } 55 | 56 | #[derive(Debug, Deserialize, Serialize)] 57 | pub enum StorageDataType { 58 | // kv 59 | KvSet, 60 | KvDelete, 61 | } 62 | 63 | #[derive(Debug, Deserialize, Serialize)] 64 | pub struct StorageData { 65 | pub data_type: StorageDataType, 66 | pub value: Vec, 67 | } 68 | 69 | impl StorageData { 70 | pub fn new(data_type: StorageDataType, value: Vec) -> StorageData { 71 | return StorageData { 72 | data_type, 73 | value: value, 74 | }; 75 | } 76 | } 77 | 78 | impl fmt::Display for StorageData { 79 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 80 | write!(f, "({:?}, {:?})", self.data_type, self.value) 81 | } 82 | } 83 | pub struct RaftMachineApply { 84 | raft_status_machine_sender: tokio::sync::mpsc::Sender, 85 | } 86 | 87 | impl RaftMachineApply { 88 | pub fn new(raft_sender: tokio::sync::mpsc::Sender) -> Self { 89 | return RaftMachineApply { 90 | raft_status_machine_sender: raft_sender, 91 | }; 92 | } 93 | 94 | pub async fn transfer_leader(&self, node_id: u64) -> Result<(), RobustMQError> { 95 | let (sx, rx) = oneshot::channel::(); 96 | return Ok(self 97 | .apply_raft_status_machine_message( 98 | RaftMessage::TransferLeader { 99 | node_id: node_id, 100 | chan: sx, 101 | }, 102 | "transfer_leader".to_string(), 103 | rx, 104 | ) 105 | .await?); 106 | } 107 | 108 | pub async fn apply_propose_message( 109 | &self, 110 | data: StorageData, 111 | action: String, 112 | ) -> Result<(), RobustMQError> { 113 | let (sx, rx) = oneshot::channel::(); 114 | return Ok(self 115 | .apply_raft_status_machine_message( 116 | RaftMessage::Propose { 117 | data: serialize(&data).unwrap(), 118 | chan: sx, 119 | }, 120 | action, 121 | rx, 122 | ) 123 | .await?); 124 | } 125 | 126 | pub async fn apply_raft_message( 127 | &self, 128 | message: raftPreludeMessage, 129 | action: String, 130 | ) -> Result<(), RobustMQError> { 131 | let (sx, rx) = oneshot::channel::(); 132 | return Ok(self 133 | .apply_raft_status_machine_message( 134 | RaftMessage::Raft { 135 | message: message, 136 | chan: sx, 137 | }, 138 | action, 139 | rx, 140 | ) 141 | .await?); 142 | } 143 | 144 | pub async fn apply_conf_raft_message( 145 | &self, 146 | change: ConfChange, 147 | action: String, 148 | ) -> Result<(), RobustMQError> { 149 | let (sx, rx) = oneshot::channel::(); 150 | return Ok(self 151 | .apply_raft_status_machine_message( 152 | RaftMessage::ConfChange { change, chan: sx }, 153 | action, 154 | rx, 155 | ) 156 | .await?); 157 | } 158 | 159 | async fn apply_raft_status_machine_message( 160 | &self, 161 | message: RaftMessage, 162 | action: String, 163 | rx: Receiver, 164 | ) -> Result<(), RobustMQError> { 165 | let _ = self.raft_status_machine_sender.send(message).await; 166 | if !self.wait_recv_chan_resp(rx).await { 167 | return Err(RobustMQError::RaftLogCommitTimeout(action)); 168 | } 169 | return Ok(()); 170 | } 171 | 172 | async fn wait_recv_chan_resp(&self, rx: Receiver) -> bool { 173 | let res = timeout(Duration::from_secs(30), async { 174 | match rx.await { 175 | Ok(val) => { 176 | return val; 177 | } 178 | Err(_) => { 179 | return RaftResponseMesage::Fail; 180 | } 181 | } 182 | }); 183 | match res.await { 184 | Ok(_) => return true, 185 | Err(_) => { 186 | return false; 187 | } 188 | } 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /src/placement-center/src/raft/metadata.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use common_base::config::placement_center::placement_center_conf; 16 | use log::info; 17 | use protocol::common::ClusterType; 18 | use raft::StateRole; 19 | use std::collections::HashMap; 20 | 21 | use super::node::BrokerNode; 22 | 23 | #[derive(PartialEq, Default, Debug, Eq, PartialOrd, Ord, Clone)] 24 | pub enum NodeState { 25 | #[default] 26 | Running, 27 | Starting, 28 | #[allow(dead_code)] 29 | Stoping, 30 | #[allow(dead_code)] 31 | Stop, 32 | } 33 | 34 | #[derive(Clone, Default, Debug)] 35 | pub struct RaftGroupMetadata { 36 | pub local: BrokerNode, 37 | pub leader: Option, 38 | pub state: NodeState, 39 | pub raft_role: StateRole, 40 | pub peers: HashMap, 41 | } 42 | 43 | impl RaftGroupMetadata { 44 | pub fn new() -> RaftGroupMetadata { 45 | let config = placement_center_conf(); 46 | let mut local = BrokerNode::default(); 47 | local.cluster_type = ClusterType::PlacementCenter.as_str_name().to_string(); 48 | local.cluster_name = config.cluster_name.clone(); 49 | local.node_inner_addr = format!("{}:{}", config.addr.clone(), config.grpc_port); 50 | local.node_ip = config.addr.clone(); 51 | local.node_id = config.node_id; 52 | 53 | let mut peers = HashMap::new(); 54 | for (node_id, addr) in config.nodes.clone() { 55 | let (ip, _) = addr.as_str().unwrap().split_once(":").unwrap(); 56 | let id: u64 = node_id.to_string().trim().parse().unwrap(); 57 | let mut node = BrokerNode::default(); 58 | 59 | node.cluster_type = ClusterType::PlacementCenter.as_str_name().to_string(); 60 | node.cluster_name = config.cluster_name.clone(); 61 | node.node_inner_addr = addr.to_string(); 62 | node.node_ip = ip.to_string(); 63 | node.node_id = id; 64 | peers.insert(id, node); 65 | } 66 | 67 | RaftGroupMetadata { 68 | local, 69 | leader: None, 70 | raft_role: StateRole::Follower, 71 | state: NodeState::Starting, 72 | peers: peers, 73 | } 74 | } 75 | 76 | pub fn get_node_by_id(&self, id: u64) -> Option<&BrokerNode> { 77 | self.peers.get(&id) 78 | } 79 | 80 | // Add Meta node 81 | pub fn add_peer(&mut self, id: u64, node: BrokerNode) { 82 | info!("add peer node:{:?}", node); 83 | self.peers.insert(id, node); 84 | } 85 | 86 | // Add Meta node 87 | pub fn remove_peer(&mut self, id: u64) { 88 | info!("remove peer node id:{:?}", id); 89 | self.peers.remove(&id); 90 | } 91 | 92 | pub fn is_leader(&self) -> bool { 93 | self.raft_role == StateRole::Leader 94 | } 95 | 96 | pub fn set_role(&mut self, role: StateRole) { 97 | self.raft_role = role; 98 | } 99 | 100 | pub fn node_ids(&self) -> Vec { 101 | let mut voters = Vec::new(); 102 | for (id, _) in self.peers.iter() { 103 | voters.push(*id); 104 | } 105 | return voters; 106 | } 107 | 108 | pub fn leader_addr(&self) -> String { 109 | if let Some(leader) = self.leader.clone() { 110 | return leader.node_inner_addr; 111 | } 112 | return "".to_string(); 113 | } 114 | 115 | pub fn leader_alive(&self) -> bool { 116 | if let Some(_) = self.leader.clone() { 117 | return true; 118 | } 119 | return false; 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/placement-center/src/raft/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | * Copyright (c) 2023 RobustMQ Team 17 | * 18 | * Licensed under the Apache License, Version 2.0 (the "License"); 19 | * you may not use this file except in compliance with the License. 20 | * You may obtain a copy of the License at 21 | * 22 | * http://www.apache.org/licenses/LICENSE-2.0 23 | * 24 | * Unless required by applicable law or agreed to in writing, software 25 | * distributed under the License is distributed on an "AS IS" BASIS, 26 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 27 | * See the License for the specific language governing permissions and 28 | * limitations under the License. 29 | */ 30 | pub mod apply; 31 | pub mod machine; 32 | pub mod metadata; 33 | pub mod peer; 34 | pub mod storage; 35 | pub mod node; 36 | pub mod route; -------------------------------------------------------------------------------- /src/placement-center/src/raft/node.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use serde::{Deserialize, Serialize}; 16 | 17 | #[derive(Default, Clone, Debug, Serialize, Deserialize)] 18 | pub struct BrokerNode { 19 | pub cluster_name: String, 20 | pub cluster_type: String, 21 | pub node_id: u64, 22 | pub node_ip: String, 23 | pub node_inner_addr: String, 24 | pub extend: String, 25 | pub create_time: u128, 26 | } 27 | 28 | impl BrokerNode { 29 | pub fn encode(&self) -> Vec { 30 | return serde_json::to_vec(&self).unwrap(); 31 | } 32 | } -------------------------------------------------------------------------------- /src/placement-center/src/raft/peer.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use log::{debug, error, info}; 16 | use protocol::placement::{ 17 | placement_center_service_client::PlacementCenterServiceClient, SendRaftMessageRequest, 18 | }; 19 | use tokio::sync::mpsc; 20 | 21 | #[derive(Debug, Clone)] 22 | pub struct PeerMessage { 23 | pub to: String, 24 | pub data: Vec, 25 | } 26 | 27 | pub struct PeersManager { 28 | peer_message_recv: mpsc::Receiver, 29 | } 30 | 31 | impl PeersManager { 32 | pub fn new(peer_message_recv: mpsc::Receiver) -> PeersManager { 33 | let pm = PeersManager { peer_message_recv }; 34 | return pm; 35 | } 36 | 37 | pub async fn start(&mut self) { 38 | info!( 39 | "{}", 40 | "Starts the thread that sends Raft messages to other nodes" 41 | ); 42 | loop { 43 | if let Some(data) = self.peer_message_recv.recv().await { 44 | let addr = data.to; 45 | let request: SendRaftMessageRequest = SendRaftMessageRequest { message: data.data }; 46 | let mut client = PlacementCenterServiceClient::connect(format!("http://{}", addr)) 47 | .await 48 | .unwrap(); 49 | 50 | match client.send_raft_message(request).await { 51 | Ok(_) => debug!("Send Raft message to node {} Successful.", addr), 52 | Err(e) => error!( 53 | "Failed to send data to {}, error message: {}", 54 | addr, 55 | e.to_string() 56 | ), 57 | } 58 | } 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/placement-center/src/raft/route.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::apply::{StorageData, StorageDataType}; 16 | use crate::storage::{kv::KvStorage, rocksdb::RocksDBEngine}; 17 | use bincode::deserialize; 18 | use common_base::errors::RobustMQError; 19 | use prost::Message as _; 20 | use protocol::kv::{DeleteRequest, SetRequest}; 21 | use std::sync::Arc; 22 | 23 | pub struct DataRoute { 24 | rocksdb_engine_handler: Arc, 25 | } 26 | 27 | impl DataRoute { 28 | pub fn new(rocksdb_engine_handler: Arc) -> DataRoute { 29 | return DataRoute { 30 | rocksdb_engine_handler, 31 | }; 32 | } 33 | 34 | //Receive write operations performed by the Raft state machine and write subsequent service data after Raft state machine synchronization is complete. 35 | pub fn route(&self, data: Vec) -> Result<(), RobustMQError> { 36 | let storage_data: StorageData = deserialize(data.as_ref()).unwrap(); 37 | match storage_data.data_type { 38 | StorageDataType::KvSet => { 39 | let kv_storage = KvStorage::new(self.rocksdb_engine_handler.clone()); 40 | let req: SetRequest = SetRequest::decode(data.as_ref()).unwrap(); 41 | return kv_storage.set(req.key, req.value); 42 | } 43 | StorageDataType::KvDelete => { 44 | let kv_storage = KvStorage::new(self.rocksdb_engine_handler.clone()); 45 | let req: DeleteRequest = DeleteRequest::decode(data.as_ref()).unwrap(); 46 | return kv_storage.delete(req.key); 47 | } 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/placement-center/src/raft/storage.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use log::info; 16 | use raft::eraftpb::HardState; 17 | use raft::prelude::ConfState; 18 | use raft::prelude::Entry; 19 | use raft::prelude::Snapshot; 20 | use raft::Error; 21 | use raft::RaftState; 22 | use raft::Result as RaftResult; 23 | use raft::Storage as RaftStorage; 24 | use raft::StorageError; 25 | use std::sync::Arc; 26 | use std::sync::RwLock; 27 | use std::sync::RwLockReadGuard; 28 | use std::sync::RwLockWriteGuard; 29 | 30 | use crate::storage::raft::RaftMachineStorage; 31 | 32 | pub struct RaftRocksDBStorage { 33 | core: Arc>, 34 | } 35 | 36 | impl RaftRocksDBStorage { 37 | pub fn new(core: Arc>) -> Self { 38 | return RaftRocksDBStorage { core }; 39 | } 40 | 41 | #[allow(dead_code)] 42 | pub fn new_with_conf_state( 43 | core: Arc>, 44 | conf_state: T, 45 | ) -> RaftRocksDBStorage 46 | where 47 | ConfState: From, 48 | { 49 | let store = RaftRocksDBStorage::new(core); 50 | store.initialize_with_conf_state(conf_state); 51 | return store; 52 | } 53 | 54 | #[allow(dead_code)] 55 | pub fn initialize_with_conf_state(&self, conf_state: T) 56 | where 57 | ConfState: From, 58 | { 59 | assert!(!self.initial_state().unwrap().initialized()); 60 | let _ = self 61 | .write_lock() 62 | .save_conf_state(ConfState::from(conf_state)); 63 | } 64 | 65 | pub fn read_lock(&self) -> RwLockReadGuard<'_, RaftMachineStorage> { 66 | self.core.read().unwrap() 67 | } 68 | 69 | pub fn write_lock(&self) -> RwLockWriteGuard<'_, RaftMachineStorage> { 70 | self.core.write().unwrap() 71 | } 72 | } 73 | 74 | impl RaftRocksDBStorage { 75 | pub fn apply_snapshot(&mut self, snapshot: Snapshot) -> RaftResult<()> { 76 | let mut store = self.core.write().unwrap(); 77 | let _ = store.apply_snapshot(snapshot); 78 | Ok(()) 79 | } 80 | 81 | pub fn append(&mut self, entrys: &Vec) -> RaftResult<()> { 82 | let mut store = self.core.write().unwrap(); 83 | let _ = store.append(entrys); 84 | return Ok(()); 85 | } 86 | 87 | pub fn commmit_index(&mut self, idx: u64) -> RaftResult<()> { 88 | let mut store = self.core.write().unwrap(); 89 | let _ = store.commmit_index(idx); 90 | return Ok(()); 91 | } 92 | 93 | pub fn set_hard_state(&mut self, hs: HardState) -> RaftResult<()> { 94 | let store = self.core.write().unwrap(); 95 | let _ = store.save_hard_state(hs.clone()); 96 | return Ok(()); 97 | } 98 | 99 | pub fn set_hard_state_comit(&mut self, hs: u64) -> RaftResult<()> { 100 | let store = self.core.write().unwrap(); 101 | let _ = store.set_hard_state_commit(hs); 102 | return Ok(()); 103 | } 104 | 105 | pub fn set_conf_state(&mut self, cs: ConfState) -> RaftResult<()> { 106 | let store = self.core.write().unwrap(); 107 | let _ = store.save_conf_state(cs); 108 | return Ok(()); 109 | } 110 | 111 | pub fn create_snapshot(&mut self) -> RaftResult<()> { 112 | let mut store = self.core.write().unwrap(); 113 | let _ = store.create_snapshot(); 114 | return Ok(()); 115 | } 116 | } 117 | 118 | impl RaftStorage for RaftRocksDBStorage { 119 | /// `initial_state` is called when Raft is initialized. This interface will return a `RaftState` 120 | /// which contains `HardState` and `ConfState`. 121 | /// 122 | /// `RaftState` could be initialized or not. If it's initialized it means the `Storage` is 123 | /// created with a configuration, and its last index and term should be greater than 0. 124 | fn initial_state(&self) -> RaftResult { 125 | let core = self.read_lock(); 126 | return Ok(core.raft_state()); 127 | } 128 | 129 | /// Returns a slice of log entries in the range `[low, high)`. 130 | /// max_size limits the total size of the log entries returned if not `None`, however 131 | /// the slice of entries returned will always have length at least 1 if entries are 132 | /// found in the range. 133 | /// 134 | /// Entries are supported to be fetched asynchronously depending on the context. Async is optional. 135 | /// Storage should check context.can_async() first and decide whether to fetch entries asynchronously 136 | /// based on its own implementation. If the entries are fetched asynchronously, storage should return 137 | /// LogTemporarilyUnavailable, and application needs to call `on_entries_fetched(context)` to trigger 138 | /// re-fetch of the entries after the storage finishes fetching the entries. 139 | /// 140 | /// # Panics 141 | /// 142 | /// Panics if `high` is higher than `Storage::last_index(&self) + 1`. 143 | fn entries( 144 | &self, 145 | low: u64, 146 | high: u64, 147 | _: impl Into>, 148 | _: raft::GetEntriesContext, 149 | ) -> RaftResult> { 150 | let core = self.read_lock(); 151 | if low < core.first_index() { 152 | return Err(Error::Store(StorageError::Compacted)); 153 | } 154 | 155 | if high > core.last_index() + 1 { 156 | panic!( 157 | "index out of bound (last: {}, high: {})", 158 | core.last_index() + 1, 159 | high 160 | ) 161 | } 162 | 163 | let mut entry_list: Vec = Vec::new(); 164 | for idx in low..=high { 165 | let sret = core.entry_by_idx(idx); 166 | if sret == None { 167 | continue; 168 | } 169 | entry_list.push(sret.unwrap()); 170 | } 171 | 172 | // todo limit size 173 | 174 | return Ok(entry_list); 175 | } 176 | 177 | /// Returns the term of entry idx, which must be in the range 178 | /// [first_index()-1, last_index()]. The term of the entry before 179 | /// first_index is retained for matching purpose even though the 180 | /// rest of that entry may not be available. 181 | fn term(&self, idx: u64) -> RaftResult { 182 | let core = self.read_lock(); 183 | 184 | if idx == core.snapshot_metadata.index { 185 | return Ok(core.snapshot_metadata.index); 186 | } 187 | 188 | if idx < core.first_index() { 189 | return Err(Error::Store(StorageError::Compacted)); 190 | } 191 | 192 | if idx > core.last_index() { 193 | return Err(Error::Store(StorageError::Unavailable)); 194 | } 195 | 196 | if let Some(value) = core.entry_by_idx(idx) { 197 | return Ok(value.term); 198 | } 199 | 200 | return Ok(core.snapshot_metadata.term); 201 | } 202 | 203 | /// Returns the index of the first log entry that is possible available via entries, which will 204 | /// always equal to `truncated index` plus 1. 205 | /// 206 | /// New created (but not initialized) `Storage` can be considered as truncated at 0 so that 1 207 | /// will be returned in this case. 208 | fn first_index(&self) -> RaftResult { 209 | let core = self.read_lock(); 210 | let fi = core.first_index(); 211 | Ok(fi) 212 | } 213 | 214 | /// The index of the last entry replicated in the `Storage`. 215 | fn last_index(&self) -> RaftResult { 216 | let core = self.read_lock(); 217 | let li = core.last_index(); 218 | Ok(li) 219 | } 220 | 221 | /// Returns the most recent snapshot. 222 | /// 223 | /// If snapshot is temporarily unavailable, it should return SnapshotTemporarilyUnavailable, 224 | /// so raft state machine could know that Storage needs some time to prepare 225 | /// snapshot and call snapshot later. 226 | /// A snapshot's index must not less than the `request_index`. 227 | /// `to` indicates which peer is requesting the snapshot. 228 | fn snapshot(&self, request_index: u64, to: u64) -> RaftResult { 229 | info!("Node {} requests snapshot data", to); 230 | let mut core = self.write_lock(); 231 | if core.trigger_snap_unavailable { 232 | return Err(Error::Store(StorageError::SnapshotTemporarilyUnavailable)); 233 | } else { 234 | let mut snap = core.snapshot(); 235 | if snap.get_metadata().index < request_index { 236 | snap.mut_metadata().index = request_index; 237 | } 238 | Ok(snap) 239 | } 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /src/placement-center/src/requests/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use log::info; 16 | 17 | pub fn network() { 18 | info!("network log"); 19 | } 20 | -------------------------------------------------------------------------------- /src/placement-center/src/server/grpc/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod server; 16 | mod services_kv; 17 | mod services_openraft; 18 | mod services_raft; 19 | mod services_kv_new; 20 | -------------------------------------------------------------------------------- /src/placement-center/src/server/grpc/server.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::{ 16 | openraft::typeconfig::TypeConfig, 17 | raft::{apply::RaftMachineApply, metadata::RaftGroupMetadata}, 18 | server::grpc::{ 19 | services_kv::GrpcKvServices, services_openraft::GrpcOpenRaftServices, 20 | services_raft::GrpcRaftServices, 21 | }, 22 | storage::rocksdb::RocksDBEngine, 23 | }; 24 | 25 | use clients::poll::ClientPool; 26 | use common_base::config::placement_center::placement_center_conf; 27 | use log::info; 28 | use openraft::Raft; 29 | use protocol::{ 30 | kv::kv_service_server::KvServiceServer, 31 | openraft::open_raft_service_server::OpenRaftServiceServer, 32 | placement::placement_center_service_server::PlacementCenterServiceServer, 33 | }; 34 | use std::sync::{Arc, RwLock}; 35 | use tokio::{select, sync::broadcast}; 36 | use tonic::transport::Server; 37 | 38 | pub async fn start_grpc_server( 39 | client_poll: Arc, 40 | raft_node: Raft, 41 | placement_center_storage: Arc, 42 | rocksdb_engine_handler: Arc, 43 | placement_cluster: Arc>, 44 | stop_sx: broadcast::Sender, 45 | ) { 46 | let config = placement_center_conf(); 47 | let server = GrpcServer::new(config.grpc_port); 48 | server 49 | .start( 50 | client_poll, 51 | placement_center_storage, 52 | rocksdb_engine_handler, 53 | placement_cluster, 54 | stop_sx, 55 | raft_node, 56 | ) 57 | .await; 58 | } 59 | 60 | pub struct GrpcServer { 61 | port: usize, 62 | } 63 | 64 | impl GrpcServer { 65 | pub fn new(port: usize) -> Self { 66 | return Self { port }; 67 | } 68 | pub async fn start( 69 | &self, 70 | client_poll: Arc, 71 | placement_center_storage: Arc, 72 | rocksdb_engine_handler: Arc, 73 | placement_cluster: Arc>, 74 | stop_sx: broadcast::Sender, 75 | raft_node: Raft, 76 | ) { 77 | let addr = format!("0.0.0.0:{}", self.port).parse().unwrap(); 78 | info!("Broker Grpc Server start. port:{}", self.port); 79 | 80 | let kv_service_handler = GrpcKvServices::new( 81 | client_poll.clone(), 82 | placement_center_storage.clone(), 83 | rocksdb_engine_handler, 84 | placement_cluster, 85 | ); 86 | let raft_service_handler = GrpcRaftServices::new(placement_center_storage); 87 | 88 | let openraft_service_handler = GrpcOpenRaftServices::new(raft_node); 89 | 90 | let mut stop_rx = stop_sx.subscribe(); 91 | select! { 92 | 93 | val = stop_rx.recv() =>{ 94 | match val{ 95 | Ok(flag) => { 96 | if flag { 97 | info!("HTTP Server stopped successfully"); 98 | 99 | } 100 | } 101 | Err(_) => {} 102 | } 103 | }, 104 | 105 | val = Server::builder().add_service(KvServiceServer::new(kv_service_handler)) 106 | .add_service(PlacementCenterServiceServer::new(raft_service_handler)) 107 | .add_service(OpenRaftServiceServer::new(openraft_service_handler)) 108 | .serve(addr)=>{ 109 | match val{ 110 | Ok(()) => { 111 | }, 112 | Err(e) => { 113 | panic!("{}",e); 114 | } 115 | } 116 | } 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/placement-center/src/server/grpc/services_kv.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::{Arc, RwLock}; 16 | 17 | use crate::{ 18 | openraft::typeconfig::TypeConfig, 19 | raft::{ 20 | apply::{RaftMachineApply, StorageData, StorageDataType}, 21 | metadata::RaftGroupMetadata, 22 | }, 23 | storage::{kv::KvStorage, rocksdb::RocksDBEngine}, 24 | }; 25 | use clients::{ 26 | placement::kv::call::{placement_delete, placement_set}, 27 | poll::ClientPool, 28 | }; 29 | use common_base::errors::RobustMQError; 30 | use openraft::Raft; 31 | use prost::Message; 32 | use protocol::kv::{ 33 | kv_service_server::KvService, CommonReply, DeleteRequest, ExistsReply, ExistsRequest, GetReply, 34 | GetRequest, SetRequest, 35 | }; 36 | use tonic::{Request, Response, Status}; 37 | 38 | pub struct GrpcKvServices { 39 | client_poll: Arc, 40 | placement_center_storage: Arc, 41 | rocksdb_engine_handler: Arc, 42 | placement_cluster: Arc>, 43 | } 44 | 45 | impl GrpcKvServices { 46 | pub fn new( 47 | client_poll: Arc, 48 | placement_center_storage: Arc, 49 | rocksdb_engine_handler: Arc, 50 | placement_cluster: Arc>, 51 | ) -> Self { 52 | return GrpcKvServices { 53 | client_poll, 54 | placement_center_storage, 55 | rocksdb_engine_handler, 56 | placement_cluster, 57 | }; 58 | } 59 | 60 | pub fn is_leader(&self) -> bool { 61 | return self.placement_cluster.read().unwrap().is_leader(); 62 | } 63 | 64 | pub fn leader_addr(&self) -> String { 65 | return self.placement_cluster.read().unwrap().leader_addr(); 66 | } 67 | } 68 | 69 | #[tonic::async_trait] 70 | impl KvService for GrpcKvServices { 71 | async fn set(&self, request: Request) -> Result, Status> { 72 | let req = request.into_inner(); 73 | 74 | if req.key.is_empty() || req.value.is_empty() { 75 | return Err(Status::cancelled( 76 | RobustMQError::ParameterCannotBeNull("key or value".to_string()).to_string(), 77 | )); 78 | } 79 | 80 | if !self.is_leader() { 81 | let leader_addr = self.leader_addr(); 82 | match placement_set(self.client_poll.clone(), vec![leader_addr], req).await { 83 | Ok(reply) => { 84 | return Ok(Response::new(reply)); 85 | } 86 | Err(e) => { 87 | return Err(Status::cancelled(e.to_string())); 88 | } 89 | } 90 | } 91 | 92 | // Raft state machine is used to store Node data 93 | let data = StorageData::new(StorageDataType::KvSet, SetRequest::encode_to_vec(&req)); 94 | match self 95 | .placement_center_storage 96 | .apply_propose_message(data, "set".to_string()) 97 | .await 98 | { 99 | Ok(_) => return Ok(Response::new(CommonReply::default())), 100 | Err(e) => { 101 | return Err(Status::cancelled(e.to_string())); 102 | } 103 | } 104 | } 105 | 106 | async fn delete( 107 | &self, 108 | request: Request, 109 | ) -> Result, Status> { 110 | let req = request.into_inner(); 111 | 112 | if req.key.is_empty() { 113 | return Err(Status::cancelled( 114 | RobustMQError::ParameterCannotBeNull("key".to_string()).to_string(), 115 | )); 116 | } 117 | 118 | if !self.is_leader() { 119 | let leader_addr = self.leader_addr(); 120 | match placement_delete(self.client_poll.clone(), vec![leader_addr], req).await { 121 | Ok(reply) => { 122 | return Ok(Response::new(reply)); 123 | } 124 | Err(e) => { 125 | return Err(Status::cancelled(e.to_string())); 126 | } 127 | } 128 | } 129 | 130 | // Raft state machine is used to store Node data 131 | let data = StorageData::new( 132 | StorageDataType::KvDelete, 133 | DeleteRequest::encode_to_vec(&req), 134 | ); 135 | match self 136 | .placement_center_storage 137 | .apply_propose_message(data, "delete".to_string()) 138 | .await 139 | { 140 | Ok(_) => return Ok(Response::new(CommonReply::default())), 141 | Err(e) => { 142 | return Err(Status::cancelled(e.to_string())); 143 | } 144 | } 145 | } 146 | 147 | async fn get(&self, request: Request) -> Result, Status> { 148 | let req = request.into_inner(); 149 | 150 | if req.key.is_empty() { 151 | return Err(Status::cancelled( 152 | RobustMQError::ParameterCannotBeNull("key".to_string()).to_string(), 153 | )); 154 | } 155 | 156 | let kv_storage = KvStorage::new(self.rocksdb_engine_handler.clone()); 157 | let mut reply = GetReply::default(); 158 | match kv_storage.get(req.key) { 159 | Ok(Some(data)) => { 160 | reply.value = data; 161 | return Ok(Response::new(reply)); 162 | } 163 | Ok(None) => {} 164 | Err(e) => return Err(Status::cancelled(e.to_string())), 165 | } 166 | 167 | return Ok(Response::new(reply)); 168 | } 169 | 170 | async fn exists( 171 | &self, 172 | request: Request, 173 | ) -> Result, Status> { 174 | let req = request.into_inner(); 175 | 176 | if req.key.is_empty() { 177 | return Err(Status::cancelled( 178 | RobustMQError::ParameterCannotBeNull("key".to_string()).to_string(), 179 | )); 180 | } 181 | 182 | let kv_storage = KvStorage::new(self.rocksdb_engine_handler.clone()); 183 | match kv_storage.exists(req.key) { 184 | Ok(flag) => { 185 | return Ok(Response::new(ExistsReply { flag })); 186 | } 187 | Err(e) => { 188 | return Err(Status::cancelled(e.to_string())); 189 | } 190 | } 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /src/placement-center/src/server/grpc/services_kv_new.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::{Arc, RwLock}; 16 | 17 | use crate::{ 18 | openraft::{route::AppRequestData, typeconfig::TypeConfig}, 19 | raft::{apply::RaftMachineApply, metadata::RaftGroupMetadata}, 20 | storage::{kv::KvStorage, rocksdb::RocksDBEngine}, 21 | }; 22 | use bincode::serialize; 23 | use clients::poll::ClientPool; 24 | use common_base::errors::RobustMQError; 25 | use openraft::Raft; 26 | use protocol::kv::{ 27 | kv_service_server::KvService, CommonReply, DeleteRequest, ExistsReply, ExistsRequest, GetReply, 28 | GetRequest, SetRequest, 29 | }; 30 | use tonic::{Request, Response, Status}; 31 | 32 | pub struct GrpcKvServices { 33 | client_poll: Arc, 34 | placement_center_storage: Arc, 35 | rocksdb_engine_handler: Arc, 36 | placement_cluster: Arc>, 37 | raft_node: Raft, 38 | } 39 | 40 | impl GrpcKvServices { 41 | pub fn new( 42 | client_poll: Arc, 43 | placement_center_storage: Arc, 44 | rocksdb_engine_handler: Arc, 45 | placement_cluster: Arc>, 46 | raft_node: Raft, 47 | ) -> Self { 48 | return GrpcKvServices { 49 | client_poll, 50 | placement_center_storage, 51 | rocksdb_engine_handler, 52 | placement_cluster, 53 | raft_node, 54 | }; 55 | } 56 | } 57 | 58 | #[tonic::async_trait] 59 | impl KvService for GrpcKvServices { 60 | async fn set(&self, request: Request) -> Result, Status> { 61 | let req = request.into_inner(); 62 | 63 | if req.key.is_empty() || req.value.is_empty() { 64 | return Err(Status::cancelled( 65 | RobustMQError::ParameterCannotBeNull("key or value".to_string()).to_string(), 66 | )); 67 | } 68 | 69 | let data = AppRequestData::Set { 70 | key: "k1".to_string(), 71 | value: "v1".to_string(), 72 | }; 73 | 74 | match self.raft_node.client_write(data).await { 75 | Ok(data) => { 76 | return Ok(Response::new(CommonReply::default())); 77 | } 78 | Err(e) => { 79 | return Err(Status::cancelled(e.to_string())); 80 | } 81 | }; 82 | } 83 | 84 | async fn delete( 85 | &self, 86 | request: Request, 87 | ) -> Result, Status> { 88 | let req = request.into_inner(); 89 | 90 | if req.key.is_empty() { 91 | return Err(Status::cancelled( 92 | RobustMQError::ParameterCannotBeNull("key".to_string()).to_string(), 93 | )); 94 | } 95 | 96 | let data = AppRequestData::Delete { 97 | key: "k1".to_string(), 98 | }; 99 | 100 | match self.raft_node.client_write(data).await { 101 | Ok(data) => { 102 | return Ok(Response::new(CommonReply::default())); 103 | } 104 | Err(e) => { 105 | return Err(Status::cancelled(e.to_string())); 106 | } 107 | }; 108 | } 109 | 110 | async fn get(&self, request: Request) -> Result, Status> { 111 | let req = request.into_inner(); 112 | 113 | if req.key.is_empty() { 114 | return Err(Status::cancelled( 115 | RobustMQError::ParameterCannotBeNull("key".to_string()).to_string(), 116 | )); 117 | } 118 | 119 | let kv_storage = KvStorage::new(self.rocksdb_engine_handler.clone()); 120 | let mut reply = GetReply::default(); 121 | match kv_storage.get(req.key) { 122 | Ok(Some(data)) => { 123 | reply.value = data; 124 | return Ok(Response::new(reply)); 125 | } 126 | Ok(None) => {} 127 | Err(e) => return Err(Status::cancelled(e.to_string())), 128 | } 129 | 130 | return Ok(Response::new(reply)); 131 | } 132 | 133 | async fn exists( 134 | &self, 135 | request: Request, 136 | ) -> Result, Status> { 137 | let req = request.into_inner(); 138 | 139 | if req.key.is_empty() { 140 | return Err(Status::cancelled( 141 | RobustMQError::ParameterCannotBeNull("key".to_string()).to_string(), 142 | )); 143 | } 144 | 145 | let kv_storage = KvStorage::new(self.rocksdb_engine_handler.clone()); 146 | match kv_storage.exists(req.key) { 147 | Ok(flag) => { 148 | return Ok(Response::new(ExistsReply { flag })); 149 | } 150 | Err(e) => { 151 | return Err(Status::cancelled(e.to_string())); 152 | } 153 | } 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/placement-center/src/server/grpc/services_openraft.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use bincode::{deserialize, serialize}; 16 | use openraft::Raft; 17 | use protocol::openraft::{ 18 | open_raft_service_server::OpenRaftService, AppendReply, AppendRequest, SnapshotReply, 19 | SnapshotRequest, VoteReply, VoteRequest, 20 | }; 21 | use tonic::{Request, Response, Status}; 22 | 23 | use crate::openraft::typeconfig::TypeConfig; 24 | 25 | pub struct GrpcOpenRaftServices { 26 | raft_node: Raft, 27 | } 28 | 29 | impl GrpcOpenRaftServices { 30 | pub fn new(raft_node: Raft) -> Self { 31 | return GrpcOpenRaftServices { raft_node }; 32 | } 33 | } 34 | 35 | #[tonic::async_trait] 36 | impl OpenRaftService for GrpcOpenRaftServices { 37 | async fn vote(&self, request: Request) -> Result, Status> { 38 | let req = request.into_inner(); 39 | let vote_data = deserialize(&req.value).unwrap(); 40 | let res = match self.raft_node.vote(vote_data).await { 41 | Ok(data) => data, 42 | Err(e) => { 43 | return Err(Status::cancelled(e.to_string())); 44 | } 45 | }; 46 | 47 | let mut reply = VoteReply::default(); 48 | reply.value = match serialize(&res) { 49 | Ok(data) => data, 50 | Err(e) => { 51 | return Err(Status::cancelled(e.to_string())); 52 | } 53 | }; 54 | 55 | return Ok(Response::new(reply)); 56 | } 57 | 58 | async fn append( 59 | &self, 60 | request: Request, 61 | ) -> Result, Status> { 62 | let req = request.into_inner(); 63 | let vote_data = deserialize(&req.value).unwrap(); 64 | let res = match self.raft_node.append_entries(vote_data).await { 65 | Ok(data) => data, 66 | Err(e) => { 67 | return Err(Status::cancelled(e.to_string())); 68 | } 69 | }; 70 | let mut reply = AppendReply::default(); 71 | reply.value = match serialize(&res) { 72 | Ok(data) => data, 73 | Err(e) => { 74 | return Err(Status::cancelled(e.to_string())); 75 | } 76 | }; 77 | return Ok(Response::new(reply)); 78 | } 79 | 80 | async fn snapshot( 81 | &self, 82 | request: Request, 83 | ) -> Result, Status> { 84 | let req = request.into_inner(); 85 | let vote_data = deserialize(&req.value).unwrap(); 86 | let res = match self.raft_node.install_snapshot(vote_data).await { 87 | Ok(data) => data, 88 | Err(e) => { 89 | return Err(Status::cancelled(e.to_string())); 90 | } 91 | }; 92 | 93 | let mut reply = SnapshotReply::default(); 94 | reply.value = match serialize(&res) { 95 | Ok(data) => data, 96 | Err(e) => { 97 | return Err(Status::cancelled(e.to_string())); 98 | } 99 | }; 100 | return Ok(Response::new(reply)); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /src/placement-center/src/server/grpc/services_raft.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use std::sync::Arc; 16 | 17 | use crate::raft::apply::RaftMachineApply; 18 | use common_base::errors::RobustMQError; 19 | use prost::Message; 20 | use protocol::placement::{ 21 | placement_center_service_server::PlacementCenterService, SendRaftConfChangeReply, 22 | SendRaftConfChangeRequest, SendRaftMessageReply, SendRaftMessageRequest, 23 | }; 24 | use raft::eraftpb::{ConfChange, Message as raftPreludeMessage}; 25 | use tonic::{Request, Response, Status}; 26 | 27 | pub struct GrpcRaftServices { 28 | placement_center_storage: Arc, 29 | } 30 | 31 | impl GrpcRaftServices { 32 | pub fn new(placement_center_storage: Arc) -> Self { 33 | return GrpcRaftServices { 34 | placement_center_storage, 35 | }; 36 | } 37 | } 38 | 39 | #[tonic::async_trait] 40 | impl PlacementCenterService for GrpcRaftServices { 41 | async fn send_raft_message( 42 | &self, 43 | request: Request, 44 | ) -> Result, Status> { 45 | let message = raftPreludeMessage::decode(request.into_inner().message.as_ref()) 46 | .map_err(|e| Status::invalid_argument(e.to_string()))?; 47 | 48 | match self 49 | .placement_center_storage 50 | .apply_raft_message(message, "send_raft_message".to_string()) 51 | .await 52 | { 53 | Ok(_) => return Ok(Response::new(SendRaftMessageReply::default())), 54 | Err(e) => { 55 | return Err(Status::cancelled( 56 | RobustMQError::RaftLogCommitTimeout(e.to_string()).to_string(), 57 | )); 58 | } 59 | } 60 | } 61 | async fn send_raft_conf_change( 62 | &self, 63 | request: Request, 64 | ) -> Result, Status> { 65 | let change = ConfChange::decode(request.into_inner().message.as_ref()) 66 | .map_err(|e| Status::invalid_argument(e.to_string()))?; 67 | 68 | match self 69 | .placement_center_storage 70 | .apply_conf_raft_message(change, "send_conf_raft_message".to_string()) 71 | .await 72 | { 73 | Ok(_) => return Ok(Response::new(SendRaftConfChangeReply::default())), 74 | Err(e) => { 75 | return Err(Status::cancelled( 76 | RobustMQError::RaftLogCommitTimeout(e.to_string()).to_string(), 77 | )); 78 | } 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/placement-center/src/server/http/index.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /* 16 | * Copyright (c) 2023 RobustMQ Team 17 | * 18 | * Licensed under the Apache License, Version 2.0 (the "License"); 19 | * you may not use this file except in compliance with the License. 20 | * You may obtain a copy of the License at 21 | * 22 | * http://www.apache.org/licenses/LICENSE-2.0 23 | * 24 | * Unless required by applicable law or agreed to in writing, software 25 | * distributed under the License is distributed on an "AS IS" BASIS, 26 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 27 | * See the License for the specific language governing permissions and 28 | * limitations under the License. 29 | */ 30 | use super::server::HttpServerState; 31 | use axum::extract::State; 32 | use common_base::http_response::success_response; 33 | pub async fn index(State(_): State) -> String { 34 | 35 | return success_response("{}"); 36 | } 37 | -------------------------------------------------------------------------------- /src/placement-center/src/server/http/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod index; 16 | pub mod openraft; 17 | pub mod server; 18 | 19 | pub(crate) fn v1_path(path: &str) -> String { 20 | return format!("/v1{}", path); 21 | } 22 | 23 | pub(crate) fn path_create(path: &str) -> String { 24 | return format!("{}/create", path); 25 | } 26 | 27 | pub(crate) fn path_update(path: &str) -> String { 28 | return format!("{}/update", path); 29 | } 30 | 31 | pub(crate) fn path_delete(path: &str) -> String { 32 | return format!("{}/delete", path); 33 | } 34 | 35 | pub(crate) fn path_list(path: &str) -> String { 36 | return format!("{}/list", path); 37 | } 38 | -------------------------------------------------------------------------------- /src/placement-center/src/server/http/openraft.rs: -------------------------------------------------------------------------------- 1 | use std::collections::{BTreeMap, BTreeSet}; 2 | 3 | use axum::extract::State; 4 | use common_base::http_response::{error_response, success_response}; 5 | use openraft::{error::Infallible, RaftMetrics}; 6 | 7 | use crate::openraft::{raft_node::Node, route::AppRequestData, typeconfig::TypeConfig}; 8 | 9 | use super::server::HttpServerState; 10 | 11 | pub async fn add_leadrner(State(state): State) -> String { 12 | let node_id = 3; 13 | let node = Node { 14 | rpc_addr: "127.0.0.0:7654".to_string(), 15 | node_id: 2, 16 | }; 17 | match state.raft_node.add_learner(node_id, node, true).await { 18 | Ok(data) => { 19 | return success_response(data); 20 | } 21 | Err(e) => { 22 | return error_response(e.to_string()); 23 | } 24 | } 25 | } 26 | 27 | pub async fn change_membership(State(state): State) -> String { 28 | let mut body = BTreeSet::new(); 29 | body.insert(3); 30 | match state.raft_node.change_membership(body, true).await { 31 | Ok(data) => { 32 | return success_response(data); 33 | } 34 | Err(e) => { 35 | return error_response(e.to_string()); 36 | } 37 | } 38 | } 39 | 40 | pub async fn init(State(state): State) -> String { 41 | let node_id = 3; 42 | let node = Node { 43 | rpc_addr: "127.0.0.0:7654".to_string(), 44 | node_id: 2, 45 | }; 46 | 47 | let mut nodes = BTreeMap::new(); 48 | nodes.insert(node_id, node); 49 | 50 | match state.raft_node.initialize(nodes).await { 51 | Ok(data) => { 52 | return success_response(data); 53 | } 54 | Err(e) => { 55 | return error_response(e.to_string()); 56 | } 57 | } 58 | } 59 | 60 | pub async fn metrics(State(state): State) -> String { 61 | let metrics = state.raft_node.metrics().borrow().clone(); 62 | let res: Result, Infallible> = Ok(metrics); 63 | return success_response(res); 64 | } 65 | 66 | pub async fn set(State(state): State) -> String { 67 | let data = AppRequestData::Set { 68 | key: "k1".to_string(), 69 | value: "v1".to_string(), 70 | }; 71 | match state.raft_node.client_write(data).await { 72 | Ok(data) => { 73 | return success_response(data); 74 | } 75 | Err(e) => { 76 | return error_response(e.to_string()); 77 | } 78 | }; 79 | } 80 | 81 | pub async fn kv_get(State(state): State) -> String { 82 | let kvs = state.kvs.read().await; 83 | let key = "k1".to_string(); 84 | let value = kvs.get(&key); 85 | 86 | return success_response(value); 87 | } 88 | -------------------------------------------------------------------------------- /src/placement-center/src/server/http/server.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use crate::openraft::typeconfig::TypeConfig; 16 | 17 | use super::openraft::{add_leadrner, change_membership, init, kv_get, metrics, set}; 18 | use super::path_list; 19 | use super::{index::index, v1_path}; 20 | use axum::routing::{get, post}; 21 | use axum::Router; 22 | use common_base::config::placement_center::placement_center_conf; 23 | use log::info; 24 | use openraft::Raft; 25 | use std::collections::BTreeMap; 26 | use std::net::SocketAddr; 27 | use std::sync::Arc; 28 | use tokio::sync::RwLock; 29 | use tokio::{select, sync::broadcast}; 30 | 31 | pub const ROUTE_ROOT: &str = "/index"; 32 | pub const ROUTE_ADD_LEARNER: &str = "/add-learner"; 33 | pub const ROUTE_CHANGE_MEMBERSHIP: &str = "/change-membership"; 34 | pub const ROUTE_INIT: &str = "/init"; 35 | pub const ROUTE_METRICS: &str = "/metrics"; 36 | pub const ROUTE_SET: &str = "/set"; 37 | pub const ROUTE_GET: &str = "/get"; 38 | 39 | #[derive(Clone)] 40 | pub struct HttpServerState { 41 | pub raft_node: Raft, 42 | pub kvs: Arc>>, 43 | } 44 | 45 | impl HttpServerState { 46 | pub fn new(raft_node: Raft, kvs: Arc>>) -> Self { 47 | return Self { raft_node, kvs }; 48 | } 49 | } 50 | 51 | pub async fn start_http_server(state: HttpServerState, stop_sx: broadcast::Sender) { 52 | let config = placement_center_conf(); 53 | let ip: SocketAddr = match format!("0.0.0.0:{}", config.http_port).parse() { 54 | Ok(data) => data, 55 | Err(e) => { 56 | panic!("{}", e); 57 | } 58 | }; 59 | 60 | info!("Broker HTTP Server start. port:{}", config.http_port); 61 | let app = routes(state); 62 | 63 | let mut stop_rx = stop_sx.subscribe(); 64 | 65 | let listener = match tokio::net::TcpListener::bind(ip).await { 66 | Ok(data) => data, 67 | Err(e) => { 68 | panic!("{}", e); 69 | } 70 | }; 71 | 72 | select! { 73 | val = stop_rx.recv() =>{ 74 | match val{ 75 | Ok(flag) => { 76 | if flag { 77 | info!("HTTP Server stopped successfully"); 78 | 79 | } 80 | } 81 | Err(_) => {} 82 | } 83 | }, 84 | val = axum::serve(listener, app.clone())=>{ 85 | match val{ 86 | Ok(()) => { 87 | }, 88 | Err(e) => { 89 | panic!("{}",e); 90 | } 91 | } 92 | } 93 | } 94 | } 95 | 96 | fn routes(state: HttpServerState) -> Router { 97 | let common = Router::new() 98 | .route(&v1_path(&path_list(ROUTE_ROOT)), get(index)) 99 | .route(&v1_path(ROUTE_ADD_LEARNER), post(add_leadrner)) 100 | .route(&v1_path(ROUTE_CHANGE_MEMBERSHIP), post(change_membership)) 101 | .route(&v1_path(ROUTE_INIT), post(init)) 102 | .route(&v1_path(ROUTE_METRICS), get(metrics)) 103 | .route(&v1_path(ROUTE_SET), get(set)) 104 | .route(&v1_path(ROUTE_GET), get(kv_get)); 105 | 106 | let app = Router::new().merge(common); 107 | return app.with_state(state); 108 | } 109 | -------------------------------------------------------------------------------- /src/placement-center/src/server/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod http; 16 | pub mod grpc; -------------------------------------------------------------------------------- /src/placement-center/src/storage/engine.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use super::{ 16 | rocksdb::{RocksDBEngine, DB_COLUMN_FAMILY_CLUSTER}, 17 | StorageDataWrap, 18 | }; 19 | use common_base::errors::RobustMQError; 20 | use serde::Serialize; 21 | use std::sync::Arc; 22 | 23 | pub fn engine_save_by_cluster( 24 | rocksdb_engine_handler: Arc, 25 | key_name: String, 26 | value: T, 27 | ) -> Result<(), RobustMQError> 28 | where 29 | T: Serialize, 30 | { 31 | return engine_save( 32 | rocksdb_engine_handler, 33 | DB_COLUMN_FAMILY_CLUSTER, 34 | key_name, 35 | value, 36 | ); 37 | } 38 | 39 | pub fn engine_get_by_cluster( 40 | rocksdb_engine_handler: Arc, 41 | key_name: String, 42 | ) -> Result, RobustMQError> { 43 | return engine_get(rocksdb_engine_handler, DB_COLUMN_FAMILY_CLUSTER, key_name); 44 | } 45 | 46 | pub fn engine_exists_by_cluster( 47 | rocksdb_engine_handler: Arc, 48 | key_name: String, 49 | ) -> Result { 50 | return engine_exists(rocksdb_engine_handler, DB_COLUMN_FAMILY_CLUSTER, key_name); 51 | } 52 | 53 | pub fn engine_delete_by_cluster( 54 | rocksdb_engine_handler: Arc, 55 | key_name: String, 56 | ) -> Result<(), RobustMQError> { 57 | return engine_delete(rocksdb_engine_handler, DB_COLUMN_FAMILY_CLUSTER, key_name); 58 | } 59 | pub fn engine_prefix_list_by_cluster( 60 | rocksdb_engine_handler: Arc, 61 | prefix_key_name: String, 62 | ) -> Result, RobustMQError> { 63 | return engine_prefix_list( 64 | rocksdb_engine_handler, 65 | DB_COLUMN_FAMILY_CLUSTER, 66 | prefix_key_name, 67 | ); 68 | } 69 | 70 | fn engine_save( 71 | rocksdb_engine_handler: Arc, 72 | rocksdb_cluster: &str, 73 | key_name: String, 74 | value: T, 75 | ) -> Result<(), RobustMQError> 76 | where 77 | T: Serialize, 78 | { 79 | let cf = if rocksdb_cluster.to_string() == DB_COLUMN_FAMILY_CLUSTER.to_string() { 80 | rocksdb_engine_handler.cf_cluster() 81 | } else { 82 | return Err(RobustMQError::ClusterNoAvailableNode); 83 | }; 84 | 85 | let content = match serde_json::to_vec(&value) { 86 | Ok(data) => data, 87 | Err(e) => return Err(RobustMQError::CommmonError(e.to_string())), 88 | }; 89 | 90 | let data = StorageDataWrap::new(content); 91 | match rocksdb_engine_handler.write(cf, &key_name, &data) { 92 | Ok(_) => { 93 | return Ok(()); 94 | } 95 | Err(e) => { 96 | return Err(RobustMQError::CommmonError(e)); 97 | } 98 | } 99 | } 100 | 101 | fn engine_get( 102 | rocksdb_engine_handler: Arc, 103 | rocksdb_cluster: &str, 104 | key_name: String, 105 | ) -> Result, RobustMQError> { 106 | let cf = if rocksdb_cluster.to_string() == DB_COLUMN_FAMILY_CLUSTER.to_string() { 107 | rocksdb_engine_handler.cf_cluster() 108 | } else { 109 | return Err(RobustMQError::ClusterNoAvailableNode); 110 | }; 111 | match rocksdb_engine_handler.read::(cf, &key_name) { 112 | Ok(Some(data)) => { 113 | return Ok(Some(data)); 114 | } 115 | Ok(None) => { 116 | return Ok(None); 117 | } 118 | Err(e) => { 119 | return Err(RobustMQError::CommmonError(e)); 120 | } 121 | } 122 | } 123 | 124 | fn engine_delete( 125 | rocksdb_engine_handler: Arc, 126 | rocksdb_cluster: &str, 127 | key_name: String, 128 | ) -> Result<(), RobustMQError> { 129 | let cf = if rocksdb_cluster.to_string() == DB_COLUMN_FAMILY_CLUSTER.to_string() { 130 | rocksdb_engine_handler.cf_cluster() 131 | } else { 132 | return Err(RobustMQError::ClusterNoAvailableNode); 133 | }; 134 | 135 | rocksdb_engine_handler.delete(cf, &key_name) 136 | } 137 | 138 | fn engine_exists( 139 | rocksdb_engine_handler: Arc, 140 | rocksdb_cluster: &str, 141 | key_name: String, 142 | ) -> Result { 143 | let cf = if rocksdb_cluster.to_string() == DB_COLUMN_FAMILY_CLUSTER.to_string() { 144 | rocksdb_engine_handler.cf_cluster() 145 | } else { 146 | return Err(RobustMQError::ClusterNoAvailableNode); 147 | }; 148 | 149 | return Ok(rocksdb_engine_handler.exist(cf, &key_name)); 150 | } 151 | 152 | fn engine_prefix_list( 153 | rocksdb_engine_handler: Arc, 154 | rocksdb_cluster: &str, 155 | prefix_key_name: String, 156 | ) -> Result, RobustMQError> { 157 | let cf = if rocksdb_cluster.to_string() == DB_COLUMN_FAMILY_CLUSTER.to_string() { 158 | rocksdb_engine_handler.cf_cluster() 159 | } else { 160 | return Err(RobustMQError::ClusterNoAvailableNode); 161 | }; 162 | 163 | let data_list = rocksdb_engine_handler.read_prefix(cf, &prefix_key_name); 164 | let mut results = Vec::new(); 165 | for raw in data_list { 166 | for (_, v) in raw { 167 | match serde_json::from_slice::(v.as_ref()) { 168 | Ok(v) => results.push(v), 169 | Err(_) => { 170 | continue; 171 | } 172 | } 173 | } 174 | } 175 | return Ok(results); 176 | } 177 | -------------------------------------------------------------------------------- /src/placement-center/src/storage/keys.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | /** ===========Raft========== */ 16 | pub fn key_name_by_first_index() -> String { 17 | return "/raft/first_index".to_string(); 18 | } 19 | 20 | pub fn key_name_by_last_index() -> String { 21 | return "/raft/last_index".to_string(); 22 | } 23 | 24 | pub fn key_name_by_hard_state() -> String { 25 | return "/raft/hard_state".to_string(); 26 | } 27 | 28 | pub fn key_name_by_conf_state() -> String { 29 | return "/raft/conf_state".to_string(); 30 | } 31 | 32 | pub fn key_name_by_entry(idx: u64) -> String { 33 | return format!("/raft/entry/{}", idx); 34 | } 35 | 36 | pub fn key_name_uncommit() -> String { 37 | return "/raft/uncommit_index".to_string(); 38 | } 39 | 40 | pub fn key_name_snapshot() -> String { 41 | return "/raft/snapshot".to_string(); 42 | } 43 | 44 | /** ===========Cluster========== */ 45 | pub fn key_cluster(cluster_type: &String, cluster_name: &String) -> String { 46 | return format!("/clusters/{}/{}", cluster_type, cluster_name); 47 | } 48 | 49 | pub fn key_cluster_prefix() -> String { 50 | return format!("/clusters/"); 51 | } 52 | 53 | pub fn key_cluster_prefix_by_type(cluster_type: &String) -> String { 54 | return format!("/clusters/{}", cluster_type); 55 | } 56 | 57 | pub fn key_node(cluster_name: &String, node_id: u64) -> String { 58 | return format!("/clusters/node/{}/{}", cluster_name, node_id); 59 | } 60 | 61 | pub fn key_node_prefix(cluster_name: &String) -> String { 62 | return format!("/clusters/node/{}", cluster_name); 63 | } 64 | 65 | pub fn key_node_prefix_all() -> String { 66 | return format!("/clusters/node/"); 67 | } 68 | 69 | pub fn key_resource_config(cluster_name: String, resource_key: String) -> String { 70 | return format!("/config/{}/{}", cluster_name, resource_key); 71 | } 72 | 73 | pub fn key_resource_idempotent(cluster_name: &String, produce_id: &String, seq_num: u64) -> String { 74 | return format!("/idempotent/{}/{}/{}", cluster_name, produce_id, seq_num); 75 | } 76 | 77 | /** ===========Journal========== */ 78 | pub fn key_shard(cluster_name: &String, shard_name: &String) -> String { 79 | return format!("/journal/shard/{}/{}", cluster_name, shard_name); 80 | } 81 | 82 | #[allow(dead_code)] 83 | pub fn key_shard_prefix(cluster_name: &String) -> String { 84 | return format!("/journal/shard/{}", cluster_name); 85 | } 86 | 87 | pub fn key_segment(cluster_name: &String, shard_name: &String, segement_seq: u64) -> String { 88 | return format!( 89 | "/journal/segment/{}/{}/{}", 90 | cluster_name, shard_name, segement_seq 91 | ); 92 | } 93 | 94 | #[allow(dead_code)] 95 | pub fn key_segment_cluster_prefix(cluster_name: &String) -> String { 96 | return format!("/journal/segment/{}", cluster_name); 97 | } 98 | 99 | #[allow(dead_code)] 100 | pub fn key_segment_shard_prefix(cluster_name: &String, shard_name: &String) -> String { 101 | return format!("/journal/segment/{}/{}", cluster_name, shard_name); 102 | } 103 | 104 | /** ===========MQTT========== */ 105 | pub fn storage_key_mqtt_user(cluster_name: &String, user_name: &String) -> String { 106 | return format!("/mqtt/user/{}/{}", cluster_name, user_name); 107 | } 108 | 109 | pub fn storage_key_mqtt_user_cluster_prefix(cluster_name: &String) -> String { 110 | return format!("/mqtt/user/{}", cluster_name); 111 | } 112 | 113 | pub fn storage_key_mqtt_topic(cluster_name: &String, user_name: &String) -> String { 114 | return format!("/mqtt/topic/{}/{}", cluster_name, user_name); 115 | } 116 | 117 | pub fn storage_key_mqtt_topic_cluster_prefix(cluster_name: &String) -> String { 118 | return format!("/mqtt/topic/{}", cluster_name); 119 | } 120 | 121 | pub fn storage_key_mqtt_session(cluster_name: &String, client_id: &String) -> String { 122 | return format!("/mqtt/session/{}/{}", cluster_name, client_id); 123 | } 124 | 125 | pub fn storage_key_mqtt_session_cluster_prefix(cluster_name: &String) -> String { 126 | return format!("/mqtt/session/{}", cluster_name); 127 | } 128 | 129 | pub fn storage_key_mqtt_last_will(cluster_name: &String, client_id: &String) -> String { 130 | return format!("/mqtt/lastwill/{}/{}", cluster_name, client_id); 131 | } 132 | pub fn storage_key_mqtt_last_will_prefix(cluster_name: &String) -> String { 133 | return format!("/mqtt/lastwill/{}", cluster_name); 134 | } 135 | 136 | pub fn storage_key_mqtt_node_sub_group_leader(cluster_name: &String) -> String { 137 | return format!("/mqtt/sub_group_leader/{}", cluster_name); 138 | } 139 | 140 | pub fn storage_key_mqtt_acl( 141 | cluster_name: &String, 142 | resource_type: &String, 143 | resource_name: &String, 144 | ) -> String { 145 | return format!( 146 | "/mqtt/acl/{}/{}/{}", 147 | cluster_name, resource_type, resource_name 148 | ); 149 | } 150 | 151 | pub fn storage_key_mqtt_acl_prefix(cluster_name: &String) -> String { 152 | return format!("/mqtt/acl/{}", cluster_name); 153 | } 154 | 155 | pub fn storage_key_mqtt_blacklist( 156 | cluster_name: &String, 157 | black_list_type: &String, 158 | resource_name: &String, 159 | ) -> String { 160 | return format!( 161 | "/mqtt/blacklist/{}/{}/{}", 162 | cluster_name, black_list_type, resource_name 163 | ); 164 | } 165 | 166 | pub fn storage_key_mqtt_blacklist_prefix(cluster_name: &String) -> String { 167 | return format!("/mqtt/blacklist/{}", cluster_name); 168 | } 169 | -------------------------------------------------------------------------------- /src/placement-center/src/storage/kv.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use common_base::errors::RobustMQError; 16 | 17 | use crate::storage::{ 18 | engine::{ 19 | engine_delete_by_cluster, engine_exists_by_cluster, engine_get_by_cluster, 20 | engine_save_by_cluster, 21 | }, 22 | rocksdb::RocksDBEngine, 23 | }; 24 | use std::sync::Arc; 25 | 26 | pub struct KvStorage { 27 | rocksdb_engine_handler: Arc, 28 | } 29 | 30 | impl KvStorage { 31 | pub fn new(rocksdb_engine_handler: Arc) -> Self { 32 | KvStorage { 33 | rocksdb_engine_handler, 34 | } 35 | } 36 | 37 | pub fn set(&self, key: String, value: String) -> Result<(), RobustMQError> { 38 | return engine_save_by_cluster(self.rocksdb_engine_handler.clone(), key, value); 39 | } 40 | 41 | pub fn delete(&self, key: String) -> Result<(), RobustMQError> { 42 | return engine_delete_by_cluster(self.rocksdb_engine_handler.clone(), key); 43 | } 44 | 45 | pub fn get(&self, key: String) -> Result, RobustMQError> { 46 | match engine_get_by_cluster(self.rocksdb_engine_handler.clone(), key) { 47 | Ok(Some(data)) => match serde_json::from_slice::(&data.data) { 48 | Ok(data) => { 49 | return Ok(Some(data)); 50 | } 51 | Err(e) => { 52 | return Err(e.into()); 53 | } 54 | }, 55 | Ok(None) => { 56 | return Ok(None); 57 | } 58 | Err(e) => Err(e), 59 | } 60 | } 61 | 62 | pub fn exists(&self, key: String) -> Result { 63 | return engine_exists_by_cluster(self.rocksdb_engine_handler.clone(), key); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/placement-center/src/storage/mod.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | use common_base::tools::now_second; 16 | use serde::{Deserialize, Serialize}; 17 | 18 | pub mod engine; 19 | pub mod kv; 20 | pub mod rocksdb; 21 | pub mod raft; 22 | pub mod keys; 23 | 24 | #[derive(Serialize, Deserialize, Debug)] 25 | pub struct StorageDataWrap { 26 | pub data: Vec, 27 | pub create_time: u64, 28 | } 29 | 30 | impl StorageDataWrap { 31 | pub fn new(data: Vec) -> Self { 32 | return StorageDataWrap { 33 | data, 34 | create_time: now_second(), 35 | }; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/placement-center/tests/kv_server.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[cfg(test)] 16 | mod tests { 17 | use std::net::TcpStream; 18 | 19 | use axum::http::request; 20 | use protocol::kv::{ 21 | kv_service_client::KvServiceClient, DeleteRequest, ExistsReply, ExistsRequest, GetRequest, 22 | SetRequest, 23 | }; 24 | 25 | #[tokio::test] 26 | async fn kv_test() { 27 | let mut client = KvServiceClient::connect("http://127.0.0.1:8871") 28 | .await 29 | .unwrap(); 30 | let key = "mq".to_string(); 31 | let value = "robustmq".to_string(); 32 | let request = tonic::Request::new(SetRequest { 33 | key: key.clone(), 34 | value: value.clone(), 35 | }); 36 | 37 | let _ = client.set(request).await.unwrap(); 38 | 39 | let request = tonic::Request::new(ExistsRequest { key: key.clone() }); 40 | let exist_reply = client.exists(request).await.unwrap().into_inner(); 41 | assert!(exist_reply.flag); 42 | 43 | let request = tonic::Request::new(GetRequest { key: key.clone() }); 44 | let get_reply = client.get(request).await.unwrap().into_inner(); 45 | assert_eq!(get_reply.value, value); 46 | 47 | let request = tonic::Request::new(DeleteRequest { key: key.clone() }); 48 | let _ = client.delete(request).await.unwrap().into_inner(); 49 | 50 | let request = tonic::Request::new(ExistsRequest { key: key.clone() }); 51 | let exist_reply = client.exists(request).await.unwrap().into_inner(); 52 | assert!(!exist_reply.flag); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/placement-center/tests/test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /src/protocol/Cargo.toml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 RobustMQ Team 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | [package] 16 | name = "protocol" 17 | version.workspace = true 18 | edition.workspace = true 19 | license.workspace = true 20 | 21 | 22 | [dependencies] 23 | serde.workspace = true 24 | tonic-build.workspace = true 25 | tonic.workspace = true 26 | prost.workspace = true 27 | dashmap.workspace = true 28 | tokio-util.workspace = true -------------------------------------------------------------------------------- /src/protocol/src/common.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package common; 3 | 4 | enum ClusterType{ 5 | PlacementCenter = 0; 6 | JournalServer = 1; 7 | MQTTBrokerServer = 2; 8 | AMQPBrokerServer = 3; 9 | } 10 | 11 | message CommonReply{ 12 | 13 | } -------------------------------------------------------------------------------- /src/protocol/src/common.rs: -------------------------------------------------------------------------------- 1 | // This file is @generated by prost-build. 2 | #[allow(clippy::derive_partial_eq_without_eq)] 3 | #[derive(Clone, PartialEq, ::prost::Message)] 4 | pub struct CommonReply {} 5 | #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] 6 | #[repr(i32)] 7 | pub enum ClusterType { 8 | PlacementCenter = 0, 9 | JournalServer = 1, 10 | MqttBrokerServer = 2, 11 | AmqpBrokerServer = 3, 12 | } 13 | impl ClusterType { 14 | /// String value of the enum field names used in the ProtoBuf definition. 15 | /// 16 | /// The values are not transformed in any way and thus are considered stable 17 | /// (if the ProtoBuf definition does not change) and safe for programmatic use. 18 | pub fn as_str_name(&self) -> &'static str { 19 | match self { 20 | ClusterType::PlacementCenter => "PlacementCenter", 21 | ClusterType::JournalServer => "JournalServer", 22 | ClusterType::MqttBrokerServer => "MQTTBrokerServer", 23 | ClusterType::AmqpBrokerServer => "AMQPBrokerServer", 24 | } 25 | } 26 | /// Creates an enum from field names used in the ProtoBuf definition. 27 | pub fn from_str_name(value: &str) -> ::core::option::Option { 28 | match value { 29 | "PlacementCenter" => Some(Self::PlacementCenter), 30 | "JournalServer" => Some(Self::JournalServer), 31 | "MQTTBrokerServer" => Some(Self::MqttBrokerServer), 32 | "AMQPBrokerServer" => Some(Self::AmqpBrokerServer), 33 | _ => None, 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/protocol/src/kv.proto: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 RobustMQ Team 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | syntax = "proto3"; 18 | package kv; 19 | 20 | service KvService { 21 | // 22 | rpc set(SetRequest) returns(CommonReply){} 23 | 24 | // 25 | rpc delete(DeleteRequest) returns(CommonReply){} 26 | 27 | // 28 | rpc get(GetRequest) returns(GetReply){} 29 | 30 | // 31 | rpc exists(ExistsRequest) returns(ExistsReply){} 32 | } 33 | 34 | message SetRequest{ 35 | string key = 1; 36 | string value = 2; 37 | } 38 | 39 | message GetRequest{ 40 | string key = 1; 41 | } 42 | 43 | message GetReply{ 44 | string value = 1; 45 | } 46 | 47 | message DeleteRequest{ 48 | string key = 1; 49 | } 50 | 51 | message ExistsRequest{ 52 | string key = 1; 53 | } 54 | 55 | message ExistsReply{ 56 | bool flag = 1; 57 | } 58 | 59 | message CommonReply{ 60 | 61 | } -------------------------------------------------------------------------------- /src/protocol/src/lib.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | pub mod kv; 16 | pub mod placement; 17 | pub mod common; 18 | pub mod openraft; -------------------------------------------------------------------------------- /src/protocol/src/openraft.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package openraft; 3 | 4 | service OpenRaftService { 5 | rpc vote(VoteRequest) returns(VoteReply){} 6 | 7 | rpc append(AppendRequest) returns(AppendReply){} 8 | 9 | rpc snapshot(SnapshotRequest) returns(SnapshotReply){} 10 | } 11 | 12 | message VoteRequest{ 13 | bytes value = 1; 14 | } 15 | 16 | message VoteReply{ 17 | bytes value = 1; 18 | } 19 | 20 | message AppendRequest{ 21 | bytes value = 1; 22 | } 23 | 24 | message AppendReply{ 25 | bytes value = 1; 26 | } 27 | 28 | message SnapshotRequest{ 29 | bytes value = 1; 30 | } 31 | 32 | message SnapshotReply{ 33 | bytes value = 1; 34 | } -------------------------------------------------------------------------------- /src/protocol/src/placement.proto: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2023 RobustMQ Team 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | syntax = "proto3"; 18 | package placement; 19 | import "common.proto"; 20 | 21 | service PlacementCenterService { 22 | 23 | rpc SendRaftMessage(SendRaftMessageRequest) returns(SendRaftMessageReply) {} 24 | 25 | rpc SendRaftConfChange(SendRaftConfChangeRequest) returns(SendRaftConfChangeReply){} 26 | 27 | } 28 | 29 | message SendRaftMessageRequest{ 30 | bytes message = 1; 31 | } 32 | 33 | message SendRaftMessageReply{ 34 | 35 | } 36 | 37 | message SendRaftConfChangeRequest{ 38 | bytes message = 1; 39 | } 40 | 41 | message SendRaftConfChangeReply{ 42 | } 43 | -------------------------------------------------------------------------------- /src/protocol/tests/protobuf.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #[cfg(test)] 16 | mod tests { 17 | #[test] 18 | fn build_pb() { 19 | tonic_build::configure() 20 | .build_server(true) 21 | .out_dir("src/") // you can change the generated code's location 22 | .compile( 23 | &[ 24 | "src/common.proto", 25 | "src/kv.proto", 26 | "src/placement.proto", 27 | "src/openraft.proto", 28 | ], 29 | &["src/"], // specify the root location to search proto dependencies 30 | ) 31 | .unwrap(); 32 | 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/protocol/tests/test.rs: -------------------------------------------------------------------------------- 1 | // Copyright 2023 RobustMQ Team 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | -------------------------------------------------------------------------------- /version.ini: -------------------------------------------------------------------------------- 1 | 0.0.1-beta --------------------------------------------------------------------------------