├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── LICENSE-MPL-RabbitMQ ├── Makefile ├── README.md ├── erlang.mk ├── examples ├── python │ ├── README.md │ ├── example1.py │ ├── example2.py │ ├── example3.py │ └── requirements.txt └── ruby │ ├── Gemfile │ ├── Gemfile.lock │ ├── README.md │ ├── example1.rb │ ├── example2.rb │ └── example3.rb ├── include └── rabbitmq_consistent_hash_exchange.hrl ├── rabbitmq-components.mk ├── src ├── Elixir.RabbitMQ.CLI.Diagnostics.Commands.ConsistentHashExchangeRingStateCommand.erl └── rabbit_exchange_type_consistent_hash.erl └── test └── rabbit_exchange_type_consistent_hash_SUITE.erl /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Thank you for using RabbitMQ and for taking the time to report an 2 | issue. 3 | 4 | ## Does This Belong to GitHub or RabbitMQ Mailing List? 5 | 6 | *Important:* please first read the `CONTRIBUTING.md` document in the 7 | root of this repository. It will help you determine whether your 8 | feedback should be directed to the RabbitMQ mailing list [1] instead. 9 | 10 | ## Please Help Maintainers and Contributors Help You 11 | 12 | In order for the RabbitMQ team to investigate your issue, please provide 13 | **as much as possible** of the following details: 14 | 15 | * RabbitMQ version 16 | * Erlang version 17 | * RabbitMQ server and client application log files 18 | * A runnable code sample, terminal transcript or detailed set of 19 | instructions that can be used to reproduce the issue 20 | * RabbitMQ plugin information via `rabbitmq-plugins list` 21 | * Client library version (for all libraries used) 22 | * Operating system, version, and patch level 23 | 24 | Running the `rabbitmq-collect-env` [2] script can provide most of the 25 | information needed. Please make the archive available via a third-party 26 | service and note that **the script does not attempt to scrub any 27 | sensitive data**. 28 | 29 | If your issue involves RabbitMQ management UI or HTTP API, please also provide 30 | the following: 31 | 32 | * Browser and its version 33 | * What management UI page was used (if applicable) 34 | * How the HTTP API requests performed can be reproduced with `curl` 35 | * Operating system on which you are running your browser, and its version 36 | * Errors reported in the JavaScript console (if any) 37 | 38 | This information **greatly speeds up issue investigation** (or makes it 39 | possible to investigate it at all). Please help project maintainers and 40 | contributors to help you by providing it! 41 | 42 | 1. https://groups.google.com/forum/#!forum/rabbitmq-users 43 | 2. https://github.com/rabbitmq/support-tools/blob/master/scripts/rabbitmq-collect-env 44 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Proposed Changes 2 | 3 | Please describe the big picture of your changes here to communicate to the 4 | RabbitMQ team why we should accept this pull request. If it fixes a bug or 5 | resolves a feature request, be sure to link to that issue. 6 | 7 | A pull request that doesn't explain **why** the change was made has a much 8 | lower chance of being accepted. 9 | 10 | If English isn't your first language, don't worry about it and try to 11 | communicate the problem you are trying to solve to the best of your abilities. 12 | As long as we can understand the intent, it's all good. 13 | 14 | ## Types of Changes 15 | 16 | What types of changes does your code introduce to this project? 17 | _Put an `x` in the boxes that apply_ 18 | 19 | - [ ] Bug fix (non-breaking change which fixes issue #NNNN) 20 | - [ ] New feature (non-breaking change which adds functionality) 21 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 22 | - [ ] Documentation (correction or otherwise) 23 | - [ ] Cosmetics (whitespace, appearance) 24 | 25 | ## Checklist 26 | 27 | _Put an `x` in the boxes that apply. You can also fill these out after creating 28 | the PR. If you're unsure about any of them, don't hesitate to ask on the 29 | mailing list. We're here to help! This is simply a reminder of what we are 30 | going to look for before merging your code._ 31 | 32 | - [ ] I have read the `CONTRIBUTING.md` document 33 | - [ ] I have signed the CA (see https://cla.pivotal.io/sign/rabbitmq) 34 | - [ ] All tests pass locally with my changes 35 | - [ ] I have added tests that prove my fix is effective or that my feature works 36 | - [ ] I have added necessary documentation (if appropriate) 37 | - [ ] Any dependent changes have been merged and published in related repositories 38 | 39 | ## Further Comments 40 | 41 | If this is a relatively large or complex change, kick off the discussion by 42 | explaining why you chose the solution you did and what alternatives you 43 | considered, etc. 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .sw? 2 | .*.sw? 3 | *.beam 4 | /.erlang.mk/ 5 | /cover/ 6 | /debug/ 7 | /deps/ 8 | /doc/ 9 | /ebin/ 10 | /escript/ 11 | /escript.lock 12 | /logs/ 13 | /plugins/ 14 | /plugins.lock 15 | /sbin/ 16 | /sbin.lock 17 | /xrefr 18 | 19 | rabbitmq_consistent_hash_exchange.d 20 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # vim:sw=2:et: 2 | 3 | os: linux 4 | dist: xenial 5 | language: elixir 6 | notifications: 7 | email: 8 | recipients: 9 | - alerts@rabbitmq.com 10 | on_success: never 11 | on_failure: always 12 | addons: 13 | apt: 14 | packages: 15 | - awscli 16 | cache: 17 | apt: true 18 | env: 19 | global: 20 | - secure: gP1vHQMh1avpsW+alOoGMWz1+oAdUW2MFbOLLInmkW2woP/TXxDM6ZgZMxHPWt1d9uzdWR3IaIh0Jq7dIMOG7UgeHKu5RP3YUXjjS7rKChDTH/zTNapHahn5dJwL9NAbOGcqvYTJhJtEA4CzfqiFlgEUH0ckDDDN8UeMiYvRWMY= 21 | - secure: WlvV5Vlid2EiEZsVZ8VI8WSvwO5vevIsRNH2W2oiMI89q2qqJA2Ec0JTiqb0ve1KMUNnpdHs66fCCMN8g0TFNLErureFiVh+zobYRfDAJecSrXEdlZdJ+af9zG1H6aNOux01jLq4wLGAoppSVBDsOgtu3d1pwXiOuKezsm7LmDA= 22 | 23 | # $base_rmq_ref is used by rabbitmq-components.mk to select the 24 | # appropriate branch for dependencies. 25 | - base_rmq_ref=master 26 | 27 | elixir: 28 | - '1.9' 29 | otp_release: 30 | - '21.3' 31 | - '22.2' 32 | 33 | install: 34 | # This project being an Erlang one (we just set language to Elixir 35 | # to ensure it is installed), we don't want Travis to run mix(1) 36 | # automatically as it will break. 37 | skip 38 | 39 | script: 40 | # $current_rmq_ref is also used by rabbitmq-components.mk to select 41 | # the appropriate branch for dependencies. 42 | - make check-rabbitmq-components.mk 43 | current_rmq_ref="${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" 44 | - make xref 45 | current_rmq_ref="${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" 46 | - make tests 47 | current_rmq_ref="${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" 48 | 49 | after_failure: 50 | - | 51 | cd "$TRAVIS_BUILD_DIR" 52 | if test -d logs && test "$AWS_ACCESS_KEY_ID" && test "$AWS_SECRET_ACCESS_KEY"; then 53 | archive_name="$(basename "$TRAVIS_REPO_SLUG")-$TRAVIS_JOB_NUMBER" 54 | 55 | tar -c --transform "s/^logs/${archive_name}/" -f - logs | \ 56 | xz > "${archive_name}.tar.xz" 57 | 58 | aws s3 cp "${archive_name}.tar.xz" s3://server-release-pipeline/travis-ci-logs/ \ 59 | --region eu-west-1 \ 60 | --acl public-read 61 | fi 62 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of fostering an open 4 | and welcoming community, we pledge to respect all people who contribute through reporting 5 | issues, posting feature requests, updating documentation, submitting pull requests or 6 | patches, and other activities. 7 | 8 | We are committed to making participation in this project a harassment-free experience for 9 | everyone, regardless of level of experience, gender, gender identity and expression, 10 | sexual orientation, disability, personal appearance, body size, race, ethnicity, age, 11 | religion, or nationality. 12 | 13 | Examples of unacceptable behavior by participants include: 14 | 15 | * The use of sexualized language or imagery 16 | * Personal attacks 17 | * Trolling or insulting/derogatory comments 18 | * Public or private harassment 19 | * Publishing other's private information, such as physical or electronic addresses, 20 | without explicit permission 21 | * Other unethical or unprofessional conduct 22 | 23 | Project maintainers have the right and responsibility to remove, edit, or reject comments, 24 | commits, code, wiki edits, issues, and other contributions that are not aligned to this 25 | Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors 26 | that they deem inappropriate, threatening, offensive, or harmful. 27 | 28 | By adopting this Code of Conduct, project maintainers commit themselves to fairly and 29 | consistently applying these principles to every aspect of managing this project. Project 30 | maintainers who do not follow or enforce the Code of Conduct may be permanently removed 31 | from the project team. 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an 34 | individual is representing the project or its community. 35 | 36 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by 37 | contacting a project maintainer at [info@rabbitmq.com](mailto:info@rabbitmq.com). All complaints will 38 | be reviewed and investigated and will result in a response that is deemed necessary and 39 | appropriate to the circumstances. Maintainers are obligated to maintain confidentiality 40 | with regard to the reporter of an incident. 41 | 42 | This Code of Conduct is adapted from the 43 | [Contributor Covenant](https://contributor-covenant.org), version 1.3.0, available at 44 | [contributor-covenant.org/version/1/3/0/](https://contributor-covenant.org/version/1/3/0/) 45 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Thank you for using RabbitMQ and for taking the time to contribute to the project. 2 | This document has two main parts: 3 | 4 | * when and how to file GitHub issues for RabbitMQ projects 5 | * how to submit pull requests 6 | 7 | They intend to save you and RabbitMQ maintainers some time, so please 8 | take a moment to read through them. 9 | 10 | ## Overview 11 | 12 | ### GitHub issues 13 | 14 | The RabbitMQ team uses GitHub issues for _specific actionable items_ that 15 | engineers can work on. This assumes the following: 16 | 17 | * GitHub issues are not used for questions, investigations, root cause 18 | analysis, discussions of potential issues, etc (as defined by this team) 19 | * Enough information is provided by the reporter for maintainers to work with 20 | 21 | The team receives many questions through various venues every single 22 | day. Frequently, these questions do not include the necessary details 23 | the team needs to begin useful work. GitHub issues can very quickly 24 | turn into a something impossible to navigate and make sense 25 | of. Because of this, questions, investigations, root cause analysis, 26 | and discussions of potential features are all considered to be 27 | [mailing list][rmq-users] material. If you are unsure where to begin, 28 | the [RabbitMQ users mailing list][rmq-users] is the right place. 29 | 30 | Getting all the details necessary to reproduce an issue, make a 31 | conclusion or even form a hypothesis about what's happening can take a 32 | fair amount of time. Please help others help you by providing a way to 33 | reproduce the behavior you're observing, or at least sharing as much 34 | relevant information as possible on the [RabbitMQ users mailing 35 | list][rmq-users]. 36 | 37 | Please provide versions of the software used: 38 | 39 | * RabbitMQ server 40 | * Erlang 41 | * Operating system version (and distribution, if applicable) 42 | * All client libraries used 43 | * RabbitMQ plugins (if applicable) 44 | 45 | The following information greatly helps in investigating and reproducing issues: 46 | 47 | * RabbitMQ server logs 48 | * A code example or terminal transcript that can be used to reproduce 49 | * Full exception stack traces (a single line message is not enough!) 50 | * `rabbitmqctl report` and `rabbitmqctl environment` output 51 | * Other relevant details about the environment and workload, e.g. a traffic capture 52 | * Feel free to edit out hostnames and other potentially sensitive information. 53 | 54 | To make collecting much of this and other environment information, use 55 | the [`rabbitmq-collect-env`][rmq-collect-env] script. It will produce an archive with 56 | server logs, operating system logs, output of certain diagnostics commands and so on. 57 | Please note that **no effort is made to scrub any information that may be sensitive**. 58 | 59 | ### Pull Requests 60 | 61 | RabbitMQ projects use pull requests to discuss, collaborate on and accept code contributions. 62 | Pull requests is the primary place of discussing code changes. 63 | 64 | Here's the recommended workflow: 65 | 66 | * [Fork the repository][github-fork] or repositories you plan on contributing to. If multiple 67 | repositories are involved in addressing the same issue, please use the same branch name 68 | in each repository 69 | * Create a branch with a descriptive name in the relevant repositories 70 | * Make your changes, run tests (usually with `make tests`), commit with a 71 | [descriptive message][git-commit-msgs], push to your fork 72 | * Submit pull requests with an explanation what has been changed and **why** 73 | * Submit a filled out and signed [Contributor Agreement][ca-agreement] if needed (see below) 74 | * Be patient. We will get to your pull request eventually 75 | 76 | If what you are going to work on is a substantial change, please first 77 | ask the core team for their opinion on the [RabbitMQ users mailing list][rmq-users]. 78 | 79 | ## Code of Conduct 80 | 81 | See [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md). 82 | 83 | ## Contributor Agreement 84 | 85 | If you want to contribute a non-trivial change, please submit a signed 86 | copy of our [Contributor Agreement][ca-agreement] around the time you 87 | submit your pull request. This will make it much easier (in some 88 | cases, possible) for the RabbitMQ team at Pivotal to merge your 89 | contribution. 90 | 91 | ## Where to Ask Questions 92 | 93 | If something isn't clear, feel free to ask on our [mailing list][rmq-users]. 94 | 95 | [rmq-collect-env]: https://github.com/rabbitmq/support-tools/blob/master/scripts/rabbitmq-collect-env 96 | [git-commit-msgs]: https://chris.beams.io/posts/git-commit/ 97 | [rmq-users]: https://groups.google.com/forum/#!forum/rabbitmq-users 98 | [ca-agreement]: https://cla.pivotal.io/sign/rabbitmq 99 | [github-fork]: https://help.github.com/articles/fork-a-repo/ 100 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This package is licensed under the MPL 2.0. For the MPL 2.0, please see LICENSE-MPL-RabbitMQ. 2 | 3 | If you have any questions regarding licensing, please contact us at 4 | info@rabbitmq.com. 5 | -------------------------------------------------------------------------------- /LICENSE-MPL-RabbitMQ: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT = rabbitmq_consistent_hash_exchange 2 | PROJECT_DESCRIPTION = Consistent Hash Exchange Type 3 | 4 | define PROJECT_APP_EXTRA_KEYS 5 | {broker_version_requirements, []} 6 | endef 7 | 8 | DEPS = rabbit_common rabbit 9 | TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client 10 | 11 | DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk 12 | DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk 13 | 14 | # FIXME: Use erlang.mk patched for RabbitMQ, while waiting for PRs to be 15 | # reviewed and merged. 16 | 17 | ERLANG_MK_REPO = https://github.com/rabbitmq/erlang.mk.git 18 | ERLANG_MK_COMMIT = rabbitmq-tmp 19 | 20 | include rabbitmq-components.mk 21 | include erlang.mk 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RabbitMQ Consistent Hash Exchange Type 2 | 3 | ## This was migrated to https://github.com/rabbitmq/rabbitmq-server 4 | 5 | This repository has been moved to the main unified RabbitMQ "monorepo", including all open issues. You can find the source under [/deps/rabbitmq_consistent_hash_exchange](https://github.com/rabbitmq/rabbitmq-server/tree/master/deps/rabbitmq_consistent_hash_exchange). 6 | -------------------------------------------------------------------------------- /examples/python/README.md: -------------------------------------------------------------------------------- 1 | # RabbitMQ Consistent Hash Exchange Examples in Python 2 | 3 | This directory contains runnable Python examples for the [RabbitMQ Consistent Hash Exchange plugin](https://github.com/rabbitmq/rabbitmq-consistent-hash-exchange/). 4 | They are the same examples as in the [plugin's README](../../README.md) file. 5 | 6 | ## Prerequisites 7 | 8 | The examples assume a RabbitMQ node with the `rabbitmq_consistent_hash_exchange` plugin 9 | enabled is running on `localhost` and that default user credentials and virtual host 10 | were not deleted. 11 | 12 | ## Dependency Installation 13 | 14 | ``` sh 15 | pip install -r ./requirements.txt 16 | ``` 17 | 18 | ## Workload Details 19 | 20 | This example uses four queues: `q1` through `q4`. The first two of them are bound to a consistent hashing 21 | exchange with a weight of `1`, while the last two of them use a weight of `2`. This means 22 | that `q3` and `q4` will get roughly twice as many published messages routed to them 23 | compared to either `q1` or `q2`. 24 | 25 | ## Running the Example 26 | 27 | ``` sh 28 | # hashing on the routing key 29 | python ./example1.py 30 | 31 | # hashing on a custom header 32 | python ./example2.py 33 | 34 | # hashing on a message property 35 | python ./example3.py 36 | ``` 37 | 38 | ## Inspecting Queue States 39 | 40 | To list bindings to the exchange and their weights, use 41 | 42 | ``` shell 43 | rabbitmqctl list_bindings | grep chx 44 | ``` 45 | 46 | To list queues and the number of ready messages in them: 47 | 48 | ``` shell 49 | rabbitmqctl list_queues name messages_ready 50 | ``` -------------------------------------------------------------------------------- /examples/python/example1.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import pika 4 | import time 5 | 6 | conn = pika.BlockingConnection(pika.ConnectionParameters(host='localhost')) 7 | ch = conn.channel() 8 | 9 | ch.exchange_declare(exchange="e", exchange_type="x-consistent-hash", durable=True) 10 | 11 | for q in ["q1", "q2", "q3", "q4"]: 12 | ch.queue_declare(queue=q, durable=True) 13 | ch.queue_purge(queue=q) 14 | 15 | for q in ["q1", "q2"]: 16 | ch.queue_bind(exchange="e", queue=q, routing_key="1") 17 | 18 | for q in ["q3", "q4"]: 19 | ch.queue_bind(exchange="e", queue=q, routing_key="2") 20 | 21 | n = 100000 22 | 23 | for rk in list(map(lambda s: str(s), range(0, n))): 24 | ch.basic_publish(exchange="e", routing_key=rk, body="") 25 | print("Done publishing.") 26 | 27 | print("Waiting for routing to finish...") 28 | # in order to keep this example simpler and focused, 29 | # wait for a few seconds instead of using publisher confirms and waiting for those 30 | time.sleep(5) 31 | 32 | print("Done.") 33 | conn.close() 34 | -------------------------------------------------------------------------------- /examples/python/example2.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import pika 4 | import time 5 | 6 | conn = pika.BlockingConnection(pika.ConnectionParameters(host='localhost')) 7 | ch = conn.channel() 8 | 9 | args = {u'hash-header': u'hash-on'} 10 | ch.exchange_declare(exchange='e2', 11 | exchange_type='x-consistent-hash', 12 | arguments=args, 13 | durable=True) 14 | 15 | for q in ['q1', 'q2', 'q3', 'q4']: 16 | ch.queue_declare(queue=q, durable=True) 17 | ch.queue_purge(queue=q) 18 | 19 | for q in ['q1', 'q2']: 20 | ch.queue_bind(exchange='e2', queue=q, routing_key='1') 21 | 22 | for q in ['q3', 'q4']: 23 | ch.queue_bind(exchange='e2', queue=q, routing_key='2') 24 | 25 | n = 100000 26 | 27 | for rk in list(map(lambda s: str(s), range(0, n))): 28 | hdrs = {u'hash-on': rk} 29 | ch.basic_publish(exchange='e2', 30 | routing_key='', 31 | body='', 32 | properties=pika.BasicProperties(content_type='text/plain', 33 | delivery_mode=2, 34 | headers=hdrs)) 35 | print('Done publishing.') 36 | 37 | print('Waiting for routing to finish...') 38 | # in order to keep this example simpler and focused, 39 | # wait for a few seconds instead of using publisher confirms and waiting for those 40 | time.sleep(5) 41 | 42 | print('Done.') 43 | conn.close() 44 | -------------------------------------------------------------------------------- /examples/python/example3.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import pika 4 | import time 5 | 6 | conn = pika.BlockingConnection(pika.ConnectionParameters(host='localhost')) 7 | ch = conn.channel() 8 | 9 | args = {u'hash-property': u'message_id'} 10 | ch.exchange_declare(exchange='e3', 11 | exchange_type='x-consistent-hash', 12 | arguments=args, 13 | durable=True) 14 | 15 | for q in ['q1', 'q2', 'q3', 'q4']: 16 | ch.queue_declare(queue=q, durable=True) 17 | ch.queue_purge(queue=q) 18 | 19 | for q in ['q1', 'q2']: 20 | ch.queue_bind(exchange='e3', queue=q, routing_key='1') 21 | 22 | for q in ['q3', 'q4']: 23 | ch.queue_bind(exchange='e3', queue=q, routing_key='2') 24 | 25 | n = 100000 26 | 27 | for rk in list(map(lambda s: str(s), range(0, n))): 28 | ch.basic_publish(exchange='e3', 29 | routing_key='', 30 | body='', 31 | properties=pika.BasicProperties(content_type='text/plain', 32 | delivery_mode=2, 33 | message_id=rk)) 34 | print('Done publishing.') 35 | 36 | print('Waiting for routing to finish...') 37 | # in order to keep this example simpler and focused, 38 | # wait for a few seconds instead of using publisher confirms and waiting for those 39 | time.sleep(5) 40 | 41 | print('Done.') 42 | conn.close() 43 | -------------------------------------------------------------------------------- /examples/python/requirements.txt: -------------------------------------------------------------------------------- 1 | pika>=1.1.0 2 | -------------------------------------------------------------------------------- /examples/ruby/Gemfile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "bunny", ">= 2.15.0", "< 3.0" 6 | -------------------------------------------------------------------------------- /examples/ruby/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | amq-protocol (2.3.1) 5 | bunny (2.15.0) 6 | amq-protocol (~> 2.3, >= 2.3.1) 7 | 8 | PLATFORMS 9 | ruby 10 | 11 | DEPENDENCIES 12 | bunny (>= 2.15.0, < 3.0) 13 | 14 | BUNDLED WITH 15 | 2.1.4 16 | -------------------------------------------------------------------------------- /examples/ruby/README.md: -------------------------------------------------------------------------------- 1 | # RabbitMQ Consistent Hash Exchange Examples in Ruby 2 | 3 | This directory contains runnable Ruby examples for the [RabbitMQ Consistent Hash Exchange plugin](https://github.com/rabbitmq/rabbitmq-consistent-hash-exchange/). 4 | They are the same examples as in the [plugin's README](../../README.md) file. 5 | 6 | ## Prerequisites 7 | 8 | The examples assume a RabbitMQ node with the `rabbitmq_consistent_hash_exchange` plugin 9 | enabled is running on `localhost` and that default user credentials and virtual host 10 | were not deleted. 11 | 12 | ## Dependency Installation 13 | 14 | ``` sh 15 | bundle install 16 | ``` 17 | 18 | ## Workload Details 19 | 20 | This example uses four queues: `q1` through `q4`. The first two of them are bound to a consistent hashing 21 | exchange with a weight of `1`, while the last two of them use a weight of `2`. This means 22 | that `q3` and `q4` will get roughly twice as many published messages routed to them 23 | compared to either `q1` or `q2`. 24 | 25 | ## Running the Example 26 | 27 | ``` sh 28 | # hashing on the routing key 29 | bundle exec ruby ./example1.rb 30 | 31 | # hashing on a custom header 32 | bundle exec ruby ./example2.rb 33 | 34 | # hashing on a message property 35 | bundle exec ruby ./example3.rb 36 | ``` 37 | 38 | ## Inspecting Queue States 39 | 40 | To list bindings to the exchange and their weights, use 41 | 42 | ``` shell 43 | rabbitmqctl list_bindings | grep chx 44 | ``` 45 | 46 | To list queues and the number of ready messages in them: 47 | 48 | ``` shell 49 | rabbitmqctl list_queues name messages_ready 50 | ``` -------------------------------------------------------------------------------- /examples/ruby/example1.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'bunny' 4 | 5 | conn = Bunny.new 6 | conn.start 7 | 8 | ch = conn.create_channel 9 | ch.confirm_select 10 | 11 | q1 = ch.queue("q1", durable: true) 12 | q2 = ch.queue("q2", durable: true) 13 | q3 = ch.queue("q3", durable: true) 14 | q4 = ch.queue("q4", durable: true) 15 | 16 | [q1, q2, q3, q4]. each(&:purge) 17 | 18 | x = ch.exchange("chx", type: "x-consistent-hash", durable: true) 19 | 20 | [q1, q2].each { |q| q.bind(x, routing_key: "1") } 21 | [q3, q4].each { |q| q.bind(x, routing_key: "2") } 22 | 23 | n = 100_000 24 | n.times do |i| 25 | x.publish(i.to_s, routing_key: i.to_s) 26 | end 27 | 28 | ch.wait_for_confirms 29 | puts "Done publishing!" 30 | 31 | # wait for queue stats to be emitted so that management UI numbers 32 | # are up-to-date 33 | sleep 5 34 | conn.close 35 | puts "Done" 36 | -------------------------------------------------------------------------------- /examples/ruby/example2.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'bundler' 4 | Bundler.setup(:default, :test) 5 | require 'bunny' 6 | 7 | conn = Bunny.new 8 | conn.start 9 | 10 | ch = conn.create_channel 11 | ch.confirm_select 12 | 13 | q1 = ch.queue("q1", durable: true) 14 | q2 = ch.queue("q2", durable: true) 15 | q3 = ch.queue("q3", durable: true) 16 | q4 = ch.queue("q4", durable: true) 17 | 18 | [q1, q2, q3, q4]. each(&:purge) 19 | 20 | x = ch.exchange("x2", type: "x-consistent-hash", durable: true, arguments: {"hash-header" => "hash-on"}) 21 | 22 | [q1, q2].each { |q| q.bind(x, routing_key: "1") } 23 | [q3, q4].each { |q| q.bind(x, routing_key: "2") } 24 | 25 | n = 100_000 26 | (0..n).map(&:to_s).each do |i| 27 | x.publish(i.to_s, routing_key: rand.to_s, headers: {"hash-on": i}) 28 | end 29 | 30 | ch.wait_for_confirms 31 | puts "Done publishing!" 32 | 33 | # wait for queue stats to be emitted so that management UI numbers 34 | # are up-to-date 35 | sleep 5 36 | conn.close 37 | puts "Done" 38 | -------------------------------------------------------------------------------- /examples/ruby/example3.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require 'bundler' 4 | Bundler.setup(:default, :test) 5 | require 'bunny' 6 | 7 | conn = Bunny.new 8 | conn.start 9 | 10 | ch = conn.create_channel 11 | ch.confirm_select 12 | 13 | q1 = ch.queue("q1", durable: true) 14 | q2 = ch.queue("q2", durable: true) 15 | q3 = ch.queue("q3", durable: true) 16 | q4 = ch.queue("q4", durable: true) 17 | 18 | [q1, q2, q3, q4].each(&:purge) 19 | 20 | x = ch.exchange("x3", type: "x-consistent-hash", durable: true, arguments: {"hash-property" => "message_id"}) 21 | 22 | [q1, q2].each { |q| q.bind(x, routing_key: "1") } 23 | [q3, q4].each { |q| q.bind(x, routing_key: "2") } 24 | 25 | n = 100_000 26 | (0..n).map(&:to_s).each do |i| 27 | x.publish(i.to_s, routing_key: rand.to_s, message_id: i) 28 | end 29 | 30 | ch.wait_for_confirms 31 | puts "Done publishing!" 32 | 33 | # wait for queue stats to be emitted so that management UI numbers 34 | # are up-to-date 35 | sleep 5 36 | conn.close 37 | puts "Done" 38 | -------------------------------------------------------------------------------- /include/rabbitmq_consistent_hash_exchange.hrl: -------------------------------------------------------------------------------- 1 | -record(chx_hash_ring, { 2 | %% a resource 3 | exchange, 4 | %% a map of bucket => queue | exchange 5 | bucket_map, 6 | next_bucket_number 7 | }). 8 | -------------------------------------------------------------------------------- /rabbitmq-components.mk: -------------------------------------------------------------------------------- 1 | ifeq ($(.DEFAULT_GOAL),) 2 | # Define default goal to `all` because this file defines some targets 3 | # before the inclusion of erlang.mk leading to the wrong target becoming 4 | # the default. 5 | .DEFAULT_GOAL = all 6 | endif 7 | 8 | # PROJECT_VERSION defaults to: 9 | # 1. the version exported by rabbitmq-server-release; 10 | # 2. the version stored in `git-revisions.txt`, if it exists; 11 | # 3. a version based on git-describe(1), if it is a Git clone; 12 | # 4. 0.0.0 13 | 14 | PROJECT_VERSION := $(RABBITMQ_VERSION) 15 | 16 | ifeq ($(PROJECT_VERSION),) 17 | PROJECT_VERSION := $(shell \ 18 | if test -f git-revisions.txt; then \ 19 | head -n1 git-revisions.txt | \ 20 | awk '{print $$$(words $(PROJECT_DESCRIPTION) version);}'; \ 21 | else \ 22 | (git describe --dirty --abbrev=7 --tags --always --first-parent \ 23 | 2>/dev/null || echo rabbitmq_v0_0_0) | \ 24 | sed -e 's/^rabbitmq_v//' -e 's/^v//' -e 's/_/./g' -e 's/-/+/' \ 25 | -e 's/-/./g'; \ 26 | fi) 27 | endif 28 | 29 | # -------------------------------------------------------------------- 30 | # RabbitMQ components. 31 | # -------------------------------------------------------------------- 32 | 33 | # For RabbitMQ repositories, we want to checkout branches which match 34 | # the parent project. For instance, if the parent project is on a 35 | # release tag, dependencies must be on the same release tag. If the 36 | # parent project is on a topic branch, dependencies must be on the same 37 | # topic branch or fallback to `stable` or `master` whichever was the 38 | # base of the topic branch. 39 | 40 | dep_amqp_client = git_rmq rabbitmq-erlang-client $(current_rmq_ref) $(base_rmq_ref) master 41 | dep_amqp10_client = git_rmq rabbitmq-amqp1.0-client $(current_rmq_ref) $(base_rmq_ref) master 42 | dep_amqp10_common = git_rmq rabbitmq-amqp1.0-common $(current_rmq_ref) $(base_rmq_ref) master 43 | dep_rabbit = git_rmq rabbitmq-server $(current_rmq_ref) $(base_rmq_ref) master 44 | dep_rabbit_common = git_rmq rabbitmq-common $(current_rmq_ref) $(base_rmq_ref) master 45 | dep_rabbitmq_amqp1_0 = git_rmq rabbitmq-amqp1.0 $(current_rmq_ref) $(base_rmq_ref) master 46 | dep_rabbitmq_auth_backend_amqp = git_rmq rabbitmq-auth-backend-amqp $(current_rmq_ref) $(base_rmq_ref) master 47 | dep_rabbitmq_auth_backend_cache = git_rmq rabbitmq-auth-backend-cache $(current_rmq_ref) $(base_rmq_ref) master 48 | dep_rabbitmq_auth_backend_http = git_rmq rabbitmq-auth-backend-http $(current_rmq_ref) $(base_rmq_ref) master 49 | dep_rabbitmq_auth_backend_ldap = git_rmq rabbitmq-auth-backend-ldap $(current_rmq_ref) $(base_rmq_ref) master 50 | dep_rabbitmq_auth_backend_oauth2 = git_rmq rabbitmq-auth-backend-oauth2 $(current_rmq_ref) $(base_rmq_ref) master 51 | dep_rabbitmq_auth_mechanism_ssl = git_rmq rabbitmq-auth-mechanism-ssl $(current_rmq_ref) $(base_rmq_ref) master 52 | dep_rabbitmq_aws = git_rmq rabbitmq-aws $(current_rmq_ref) $(base_rmq_ref) master 53 | dep_rabbitmq_boot_steps_visualiser = git_rmq rabbitmq-boot-steps-visualiser $(current_rmq_ref) $(base_rmq_ref) master 54 | dep_rabbitmq_cli = git_rmq rabbitmq-cli $(current_rmq_ref) $(base_rmq_ref) master 55 | dep_rabbitmq_codegen = git_rmq rabbitmq-codegen $(current_rmq_ref) $(base_rmq_ref) master 56 | dep_rabbitmq_consistent_hash_exchange = git_rmq rabbitmq-consistent-hash-exchange $(current_rmq_ref) $(base_rmq_ref) master 57 | dep_rabbitmq_ct_client_helpers = git_rmq rabbitmq-ct-client-helpers $(current_rmq_ref) $(base_rmq_ref) master 58 | dep_rabbitmq_ct_helpers = git_rmq rabbitmq-ct-helpers $(current_rmq_ref) $(base_rmq_ref) master 59 | dep_rabbitmq_delayed_message_exchange = git_rmq rabbitmq-delayed-message-exchange $(current_rmq_ref) $(base_rmq_ref) master 60 | dep_rabbitmq_dotnet_client = git_rmq rabbitmq-dotnet-client $(current_rmq_ref) $(base_rmq_ref) master 61 | dep_rabbitmq_event_exchange = git_rmq rabbitmq-event-exchange $(current_rmq_ref) $(base_rmq_ref) master 62 | dep_rabbitmq_federation = git_rmq rabbitmq-federation $(current_rmq_ref) $(base_rmq_ref) master 63 | dep_rabbitmq_federation_management = git_rmq rabbitmq-federation-management $(current_rmq_ref) $(base_rmq_ref) master 64 | dep_rabbitmq_java_client = git_rmq rabbitmq-java-client $(current_rmq_ref) $(base_rmq_ref) master 65 | dep_rabbitmq_jms_client = git_rmq rabbitmq-jms-client $(current_rmq_ref) $(base_rmq_ref) master 66 | dep_rabbitmq_jms_cts = git_rmq rabbitmq-jms-cts $(current_rmq_ref) $(base_rmq_ref) master 67 | dep_rabbitmq_jms_topic_exchange = git_rmq rabbitmq-jms-topic-exchange $(current_rmq_ref) $(base_rmq_ref) master 68 | dep_rabbitmq_lvc_exchange = git_rmq rabbitmq-lvc-exchange $(current_rmq_ref) $(base_rmq_ref) master 69 | dep_rabbitmq_management = git_rmq rabbitmq-management $(current_rmq_ref) $(base_rmq_ref) master 70 | dep_rabbitmq_management_agent = git_rmq rabbitmq-management-agent $(current_rmq_ref) $(base_rmq_ref) master 71 | dep_rabbitmq_management_exchange = git_rmq rabbitmq-management-exchange $(current_rmq_ref) $(base_rmq_ref) master 72 | dep_rabbitmq_management_themes = git_rmq rabbitmq-management-themes $(current_rmq_ref) $(base_rmq_ref) master 73 | dep_rabbitmq_message_timestamp = git_rmq rabbitmq-message-timestamp $(current_rmq_ref) $(base_rmq_ref) master 74 | dep_rabbitmq_metronome = git_rmq rabbitmq-metronome $(current_rmq_ref) $(base_rmq_ref) master 75 | dep_rabbitmq_mqtt = git_rmq rabbitmq-mqtt $(current_rmq_ref) $(base_rmq_ref) master 76 | dep_rabbitmq_objc_client = git_rmq rabbitmq-objc-client $(current_rmq_ref) $(base_rmq_ref) master 77 | dep_rabbitmq_peer_discovery_aws = git_rmq rabbitmq-peer-discovery-aws $(current_rmq_ref) $(base_rmq_ref) master 78 | dep_rabbitmq_peer_discovery_common = git_rmq rabbitmq-peer-discovery-common $(current_rmq_ref) $(base_rmq_ref) master 79 | dep_rabbitmq_peer_discovery_consul = git_rmq rabbitmq-peer-discovery-consul $(current_rmq_ref) $(base_rmq_ref) master 80 | dep_rabbitmq_peer_discovery_etcd = git_rmq rabbitmq-peer-discovery-etcd $(current_rmq_ref) $(base_rmq_ref) master 81 | dep_rabbitmq_peer_discovery_k8s = git_rmq rabbitmq-peer-discovery-k8s $(current_rmq_ref) $(base_rmq_ref) master 82 | dep_rabbitmq_prometheus = git_rmq rabbitmq-prometheus $(current_rmq_ref) $(base_rmq_ref) master 83 | dep_rabbitmq_random_exchange = git_rmq rabbitmq-random-exchange $(current_rmq_ref) $(base_rmq_ref) master 84 | dep_rabbitmq_recent_history_exchange = git_rmq rabbitmq-recent-history-exchange $(current_rmq_ref) $(base_rmq_ref) master 85 | dep_rabbitmq_routing_node_stamp = git_rmq rabbitmq-routing-node-stamp $(current_rmq_ref) $(base_rmq_ref) master 86 | dep_rabbitmq_rtopic_exchange = git_rmq rabbitmq-rtopic-exchange $(current_rmq_ref) $(base_rmq_ref) master 87 | dep_rabbitmq_server_release = git_rmq rabbitmq-server-release $(current_rmq_ref) $(base_rmq_ref) master 88 | dep_rabbitmq_sharding = git_rmq rabbitmq-sharding $(current_rmq_ref) $(base_rmq_ref) master 89 | dep_rabbitmq_shovel = git_rmq rabbitmq-shovel $(current_rmq_ref) $(base_rmq_ref) master 90 | dep_rabbitmq_shovel_management = git_rmq rabbitmq-shovel-management $(current_rmq_ref) $(base_rmq_ref) master 91 | dep_rabbitmq_stomp = git_rmq rabbitmq-stomp $(current_rmq_ref) $(base_rmq_ref) master 92 | dep_rabbitmq_stream = git_rmq rabbitmq-stream $(current_rmq_ref) $(base_rmq_ref) master 93 | dep_rabbitmq_toke = git_rmq rabbitmq-toke $(current_rmq_ref) $(base_rmq_ref) master 94 | dep_rabbitmq_top = git_rmq rabbitmq-top $(current_rmq_ref) $(base_rmq_ref) master 95 | dep_rabbitmq_tracing = git_rmq rabbitmq-tracing $(current_rmq_ref) $(base_rmq_ref) master 96 | dep_rabbitmq_trust_store = git_rmq rabbitmq-trust-store $(current_rmq_ref) $(base_rmq_ref) master 97 | dep_rabbitmq_test = git_rmq rabbitmq-test $(current_rmq_ref) $(base_rmq_ref) master 98 | dep_rabbitmq_web_dispatch = git_rmq rabbitmq-web-dispatch $(current_rmq_ref) $(base_rmq_ref) master 99 | dep_rabbitmq_web_stomp = git_rmq rabbitmq-web-stomp $(current_rmq_ref) $(base_rmq_ref) master 100 | dep_rabbitmq_web_stomp_examples = git_rmq rabbitmq-web-stomp-examples $(current_rmq_ref) $(base_rmq_ref) master 101 | dep_rabbitmq_web_mqtt = git_rmq rabbitmq-web-mqtt $(current_rmq_ref) $(base_rmq_ref) master 102 | dep_rabbitmq_web_mqtt_examples = git_rmq rabbitmq-web-mqtt-examples $(current_rmq_ref) $(base_rmq_ref) master 103 | dep_rabbitmq_website = git_rmq rabbitmq-website $(current_rmq_ref) $(base_rmq_ref) live master 104 | dep_toke = git_rmq toke $(current_rmq_ref) $(base_rmq_ref) master 105 | 106 | dep_rabbitmq_public_umbrella = git_rmq rabbitmq-public-umbrella $(current_rmq_ref) $(base_rmq_ref) master 107 | 108 | # Third-party dependencies version pinning. 109 | # 110 | # We do that in this file, which is copied in all projects, to ensure 111 | # all projects use the same versions. It avoids conflicts and makes it 112 | # possible to work with rabbitmq-public-umbrella. 113 | 114 | dep_accept = hex 0.3.5 115 | dep_cowboy = hex 2.8.0 116 | dep_cowlib = hex 2.9.1 117 | dep_jsx = hex 2.11.0 118 | dep_lager = hex 3.8.0 119 | dep_prometheus = git https://github.com/deadtrickster/prometheus.erl.git master 120 | dep_ra = git https://github.com/rabbitmq/ra.git master 121 | dep_ranch = hex 1.7.1 122 | dep_recon = hex 2.5.1 123 | dep_observer_cli = hex 1.5.4 124 | dep_stdout_formatter = hex 0.2.4 125 | dep_sysmon_handler = hex 1.3.0 126 | 127 | RABBITMQ_COMPONENTS = amqp_client \ 128 | amqp10_common \ 129 | amqp10_client \ 130 | rabbit \ 131 | rabbit_common \ 132 | rabbitmq_amqp1_0 \ 133 | rabbitmq_auth_backend_amqp \ 134 | rabbitmq_auth_backend_cache \ 135 | rabbitmq_auth_backend_http \ 136 | rabbitmq_auth_backend_ldap \ 137 | rabbitmq_auth_backend_oauth2 \ 138 | rabbitmq_auth_mechanism_ssl \ 139 | rabbitmq_aws \ 140 | rabbitmq_boot_steps_visualiser \ 141 | rabbitmq_cli \ 142 | rabbitmq_codegen \ 143 | rabbitmq_consistent_hash_exchange \ 144 | rabbitmq_ct_client_helpers \ 145 | rabbitmq_ct_helpers \ 146 | rabbitmq_delayed_message_exchange \ 147 | rabbitmq_dotnet_client \ 148 | rabbitmq_event_exchange \ 149 | rabbitmq_federation \ 150 | rabbitmq_federation_management \ 151 | rabbitmq_java_client \ 152 | rabbitmq_jms_client \ 153 | rabbitmq_jms_cts \ 154 | rabbitmq_jms_topic_exchange \ 155 | rabbitmq_lvc_exchange \ 156 | rabbitmq_management \ 157 | rabbitmq_management_agent \ 158 | rabbitmq_management_exchange \ 159 | rabbitmq_management_themes \ 160 | rabbitmq_message_timestamp \ 161 | rabbitmq_metronome \ 162 | rabbitmq_mqtt \ 163 | rabbitmq_objc_client \ 164 | rabbitmq_peer_discovery_aws \ 165 | rabbitmq_peer_discovery_common \ 166 | rabbitmq_peer_discovery_consul \ 167 | rabbitmq_peer_discovery_etcd \ 168 | rabbitmq_peer_discovery_k8s \ 169 | rabbitmq_prometheus \ 170 | rabbitmq_random_exchange \ 171 | rabbitmq_recent_history_exchange \ 172 | rabbitmq_routing_node_stamp \ 173 | rabbitmq_rtopic_exchange \ 174 | rabbitmq_server_release \ 175 | rabbitmq_sharding \ 176 | rabbitmq_shovel \ 177 | rabbitmq_shovel_management \ 178 | rabbitmq_stomp \ 179 | rabbitmq_stream \ 180 | rabbitmq_toke \ 181 | rabbitmq_top \ 182 | rabbitmq_tracing \ 183 | rabbitmq_trust_store \ 184 | rabbitmq_web_dispatch \ 185 | rabbitmq_web_mqtt \ 186 | rabbitmq_web_mqtt_examples \ 187 | rabbitmq_web_stomp \ 188 | rabbitmq_web_stomp_examples \ 189 | rabbitmq_website 190 | 191 | # Erlang.mk does not rebuild dependencies by default, once they were 192 | # compiled once, except for those listed in the `$(FORCE_REBUILD)` 193 | # variable. 194 | # 195 | # We want all RabbitMQ components to always be rebuilt: this eases 196 | # the work on several components at the same time. 197 | 198 | FORCE_REBUILD = $(RABBITMQ_COMPONENTS) 199 | 200 | # Several components have a custom erlang.mk/build.config, mainly 201 | # to disable eunit. Therefore, we can't use the top-level project's 202 | # erlang.mk copy. 203 | NO_AUTOPATCH += $(RABBITMQ_COMPONENTS) 204 | 205 | ifeq ($(origin current_rmq_ref),undefined) 206 | ifneq ($(wildcard .git),) 207 | current_rmq_ref := $(shell (\ 208 | ref=$$(LANG=C git branch --list | awk '/^\* \(.*detached / {ref=$$0; sub(/.*detached [^ ]+ /, "", ref); sub(/\)$$/, "", ref); print ref; exit;} /^\* / {ref=$$0; sub(/^\* /, "", ref); print ref; exit}');\ 209 | if test "$$(git rev-parse --short HEAD)" != "$$ref"; then echo "$$ref"; fi)) 210 | else 211 | current_rmq_ref := master 212 | endif 213 | endif 214 | export current_rmq_ref 215 | 216 | ifeq ($(origin base_rmq_ref),undefined) 217 | ifneq ($(wildcard .git),) 218 | possible_base_rmq_ref := master 219 | ifeq ($(possible_base_rmq_ref),$(current_rmq_ref)) 220 | base_rmq_ref := $(current_rmq_ref) 221 | else 222 | base_rmq_ref := $(shell \ 223 | (git rev-parse --verify -q master >/dev/null && \ 224 | git rev-parse --verify -q $(possible_base_rmq_ref) >/dev/null && \ 225 | git merge-base --is-ancestor $$(git merge-base master HEAD) $(possible_base_rmq_ref) && \ 226 | echo $(possible_base_rmq_ref)) || \ 227 | echo master) 228 | endif 229 | else 230 | base_rmq_ref := master 231 | endif 232 | endif 233 | export base_rmq_ref 234 | 235 | # Repository URL selection. 236 | # 237 | # First, we infer other components' location from the current project 238 | # repository URL, if it's a Git repository: 239 | # - We take the "origin" remote URL as the base 240 | # - The current project name and repository name is replaced by the 241 | # target's properties: 242 | # eg. rabbitmq-common is replaced by rabbitmq-codegen 243 | # eg. rabbit_common is replaced by rabbitmq_codegen 244 | # 245 | # If cloning from this computed location fails, we fallback to RabbitMQ 246 | # upstream which is GitHub. 247 | 248 | # Macro to transform eg. "rabbit_common" to "rabbitmq-common". 249 | rmq_cmp_repo_name = $(word 2,$(dep_$(1))) 250 | 251 | # Upstream URL for the current project. 252 | RABBITMQ_COMPONENT_REPO_NAME := $(call rmq_cmp_repo_name,$(PROJECT)) 253 | RABBITMQ_UPSTREAM_FETCH_URL ?= https://github.com/rabbitmq/$(RABBITMQ_COMPONENT_REPO_NAME).git 254 | RABBITMQ_UPSTREAM_PUSH_URL ?= git@github.com:rabbitmq/$(RABBITMQ_COMPONENT_REPO_NAME).git 255 | 256 | # Current URL for the current project. If this is not a Git clone, 257 | # default to the upstream Git repository. 258 | ifneq ($(wildcard .git),) 259 | git_origin_fetch_url := $(shell git config remote.origin.url) 260 | git_origin_push_url := $(shell git config remote.origin.pushurl || git config remote.origin.url) 261 | RABBITMQ_CURRENT_FETCH_URL ?= $(git_origin_fetch_url) 262 | RABBITMQ_CURRENT_PUSH_URL ?= $(git_origin_push_url) 263 | else 264 | RABBITMQ_CURRENT_FETCH_URL ?= $(RABBITMQ_UPSTREAM_FETCH_URL) 265 | RABBITMQ_CURRENT_PUSH_URL ?= $(RABBITMQ_UPSTREAM_PUSH_URL) 266 | endif 267 | 268 | # Macro to replace the following pattern: 269 | # 1. /foo.git -> /bar.git 270 | # 2. /foo -> /bar 271 | # 3. /foo/ -> /bar/ 272 | subst_repo_name = $(patsubst %/$(1)/%,%/$(2)/%,$(patsubst %/$(1),%/$(2),$(patsubst %/$(1).git,%/$(2).git,$(3)))) 273 | 274 | # Macro to replace both the project's name (eg. "rabbit_common") and 275 | # repository name (eg. "rabbitmq-common") by the target's equivalent. 276 | # 277 | # This macro is kept on one line because we don't want whitespaces in 278 | # the returned value, as it's used in $(dep_fetch_git_rmq) in a shell 279 | # single-quoted string. 280 | dep_rmq_repo = $(if $(dep_$(2)),$(call subst_repo_name,$(PROJECT),$(2),$(call subst_repo_name,$(RABBITMQ_COMPONENT_REPO_NAME),$(call rmq_cmp_repo_name,$(2)),$(1))),$(pkg_$(1)_repo)) 281 | 282 | dep_rmq_commits = $(if $(dep_$(1)), \ 283 | $(wordlist 3,$(words $(dep_$(1))),$(dep_$(1))), \ 284 | $(pkg_$(1)_commit)) 285 | 286 | define dep_fetch_git_rmq 287 | fetch_url1='$(call dep_rmq_repo,$(RABBITMQ_CURRENT_FETCH_URL),$(1))'; \ 288 | fetch_url2='$(call dep_rmq_repo,$(RABBITMQ_UPSTREAM_FETCH_URL),$(1))'; \ 289 | if test "$$$$fetch_url1" != '$(RABBITMQ_CURRENT_FETCH_URL)' && \ 290 | git clone -q -n -- "$$$$fetch_url1" $(DEPS_DIR)/$(call dep_name,$(1)); then \ 291 | fetch_url="$$$$fetch_url1"; \ 292 | push_url='$(call dep_rmq_repo,$(RABBITMQ_CURRENT_PUSH_URL),$(1))'; \ 293 | elif git clone -q -n -- "$$$$fetch_url2" $(DEPS_DIR)/$(call dep_name,$(1)); then \ 294 | fetch_url="$$$$fetch_url2"; \ 295 | push_url='$(call dep_rmq_repo,$(RABBITMQ_UPSTREAM_PUSH_URL),$(1))'; \ 296 | fi; \ 297 | cd $(DEPS_DIR)/$(call dep_name,$(1)) && ( \ 298 | $(foreach ref,$(call dep_rmq_commits,$(1)), \ 299 | git checkout -q $(ref) >/dev/null 2>&1 || \ 300 | ) \ 301 | (echo "error: no valid pathspec among: $(call dep_rmq_commits,$(1))" \ 302 | 1>&2 && false) ) && \ 303 | (test "$$$$fetch_url" = "$$$$push_url" || \ 304 | git remote set-url --push origin "$$$$push_url") 305 | endef 306 | 307 | # -------------------------------------------------------------------- 308 | # Component distribution. 309 | # -------------------------------------------------------------------- 310 | 311 | list-dist-deps:: 312 | @: 313 | 314 | prepare-dist:: 315 | @: 316 | 317 | # -------------------------------------------------------------------- 318 | # Umbrella-specific settings. 319 | # -------------------------------------------------------------------- 320 | 321 | # If the top-level project is a RabbitMQ component, we override 322 | # $(DEPS_DIR) for this project to point to the top-level's one. 323 | # 324 | # We also verify that the guessed DEPS_DIR is actually named `deps`, 325 | # to rule out any situation where it is a coincidence that we found a 326 | # `rabbitmq-components.mk` up upper directories. 327 | 328 | possible_deps_dir_1 = $(abspath ..) 329 | possible_deps_dir_2 = $(abspath ../../..) 330 | 331 | ifeq ($(notdir $(possible_deps_dir_1)),deps) 332 | ifneq ($(wildcard $(possible_deps_dir_1)/../rabbitmq-components.mk),) 333 | deps_dir_overriden = 1 334 | DEPS_DIR ?= $(possible_deps_dir_1) 335 | DISABLE_DISTCLEAN = 1 336 | endif 337 | endif 338 | 339 | ifeq ($(deps_dir_overriden),) 340 | ifeq ($(notdir $(possible_deps_dir_2)),deps) 341 | ifneq ($(wildcard $(possible_deps_dir_2)/../rabbitmq-components.mk),) 342 | deps_dir_overriden = 1 343 | DEPS_DIR ?= $(possible_deps_dir_2) 344 | DISABLE_DISTCLEAN = 1 345 | endif 346 | endif 347 | endif 348 | 349 | ifneq ($(wildcard UMBRELLA.md),) 350 | DISABLE_DISTCLEAN = 1 351 | endif 352 | 353 | # We disable `make distclean` so $(DEPS_DIR) is not accidentally removed. 354 | 355 | ifeq ($(DISABLE_DISTCLEAN),1) 356 | ifneq ($(filter distclean distclean-deps,$(MAKECMDGOALS)),) 357 | SKIP_DEPS = 1 358 | endif 359 | endif 360 | -------------------------------------------------------------------------------- /src/Elixir.RabbitMQ.CLI.Diagnostics.Commands.ConsistentHashExchangeRingStateCommand.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. 6 | %% 7 | 8 | -module('Elixir.RabbitMQ.CLI.Diagnostics.Commands.ConsistentHashExchangeRingStateCommand'). 9 | 10 | -include_lib("rabbit_common/include/resource.hrl"). 11 | -include("rabbitmq_consistent_hash_exchange.hrl"). 12 | 13 | -behaviour('Elixir.RabbitMQ.CLI.CommandBehaviour'). 14 | 15 | -export([ 16 | usage/0, 17 | usage_additional/0, 18 | usage_doc_guides/0, 19 | flags/0, 20 | validate/2, 21 | merge_defaults/2, 22 | banner/2, 23 | run/2, 24 | switches/0, 25 | aliases/0, 26 | output/2, 27 | scopes/0, 28 | formatter/0, 29 | help_section/0, 30 | description/0 31 | ]). 32 | 33 | -import(rabbit_data_coercion, [to_binary/1]). 34 | 35 | -define(NOT_FOUND_MESSAGE, <<"Exchange does not exist or is of a different type">>). 36 | 37 | %%---------------------------------------------------------------------------- 38 | %% Callbacks 39 | %%---------------------------------------------------------------------------- 40 | 41 | scopes() -> 42 | ['diagnostics']. 43 | 44 | switches() -> 45 | []. 46 | 47 | aliases() -> 48 | []. 49 | 50 | flags() -> 51 | []. 52 | 53 | merge_defaults(Args, Opts) -> 54 | {Args, maps:merge(#{vhost => <<"/">>}, Opts)}. 55 | 56 | validate([], _Opts) -> 57 | {validation_failure, not_enough_args}; 58 | validate([_Exchange], _Opts) -> 59 | ok; 60 | validate(_, _Opts) -> 61 | {validation_failure, too_many_args}. 62 | 63 | run([Exchange], #{node := Node, vhost := VirtualHost}) -> 64 | case rabbit_misc:rpc_call(Node, rabbit_exchange_type_consistent_hash, ring_state, [VirtualHost, Exchange]) of 65 | {badrpc, _} = Error -> 66 | Error; 67 | {badrpc, _, _} = Error -> 68 | Error; 69 | {error, _} = Error -> 70 | Error; 71 | {ok, State} -> 72 | {ok, State} 73 | end. 74 | 75 | output({error, not_found}, #{node := Node, formatter := <<"json">>}) -> 76 | {error, #{ 77 | <<"result">> => <<"error">>, 78 | <<"node">> => Node, 79 | <<"message">> => ?NOT_FOUND_MESSAGE 80 | }}; 81 | output({ok, #chx_hash_ring{exchange = Resource = #resource{name = Exchange}, bucket_map = Buckets}}, #{node := Node, formatter := <<"json">>}) -> 82 | {ok, #{ 83 | <<"result">> => <<"ok">>, 84 | <<"node">> => Node, 85 | <<"exchange">> => Exchange, 86 | <<"message">> => to_binary(rabbit_misc:format("Consistent hashing ring state for ~s", 87 | [rabbit_misc:rs(Resource)])), 88 | <<"buckets">> => 89 | maps:from_list(lists:map(fun ({Key, #resource{kind = queue, name = Queue}}) -> 90 | {to_binary(Key), Queue} 91 | end, maps:to_list(Buckets))) 92 | }}; 93 | output({error, not_found}, _Opts) -> 94 | {error, 'Elixir.RabbitMQ.CLI.Core.ExitCodes':exit_dataerr(), ?NOT_FOUND_MESSAGE}; 95 | output({ok, #chx_hash_ring{bucket_map = Buckets0}}, _Opts) -> 96 | Buckets = maps:map(fun(_Key, #resource{kind = queue, name = Queue}) -> Queue end, Buckets0), 97 | {ok, (ring_state_lines(Buckets))}; 98 | output(Result, _Opts) -> 99 | 'Elixir.RabbitMQ.CLI.DefaultOutput':output(Result). 100 | 101 | usage() -> 102 | <<"consistent_hash_exchange_ring_state ">>. 103 | 104 | usage_additional() -> 105 | []. 106 | 107 | formatter() -> 108 | 'Elixir.RabbitMQ.CLI.Formatters.String'. 109 | 110 | usage_doc_guides() -> 111 | []. 112 | 113 | help_section() -> 114 | {plugin, consistent_hash_exchange}. 115 | 116 | description() -> 117 | <<"Displays consistent hashing exchange ring state">>. 118 | 119 | banner([Exchange], #{vhost := VirtualHost}) -> 120 | erlang:iolist_to_binary([<<"Inspecting consistent hashing ring state for exchange ">>, 121 | to_binary(Exchange), 122 | <<" in virtual host ">>, 123 | to_binary(rabbit_misc:format("'~s'", [VirtualHost])), 124 | <<"...">>]). 125 | 126 | %% 127 | %% Implementation 128 | %% 129 | 130 | ring_state_lines(Buckets) -> 131 | Fun = fun (Key, QName, Acc) -> 132 | [to_binary(rabbit_misc:format("Ring index: ~b, queue: '~s'~n", [Key, QName])) | Acc] 133 | end, 134 | lists:usort(maps:fold(Fun, [], Buckets)). 135 | -------------------------------------------------------------------------------- /src/rabbit_exchange_type_consistent_hash.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. 6 | %% 7 | 8 | -module(rabbit_exchange_type_consistent_hash). 9 | -include_lib("rabbit_common/include/rabbit.hrl"). 10 | -include_lib("rabbit_common/include/rabbit_framing.hrl"). 11 | 12 | -include("rabbitmq_consistent_hash_exchange.hrl"). 13 | 14 | -behaviour(rabbit_exchange_type). 15 | 16 | -export([description/0, serialise_events/0, route/2]). 17 | -export([validate/1, validate_binding/2, 18 | create/2, delete/3, policy_changed/2, 19 | add_binding/3, remove_bindings/3, assert_args_equivalence/2]). 20 | -export([init/0]). 21 | -export([info/1, info/2]). 22 | -export([ring_state/2]). 23 | 24 | -rabbit_boot_step( 25 | {rabbit_exchange_type_consistent_hash_registry, 26 | [{description, "exchange type x-consistent-hash: registry"}, 27 | {mfa, {rabbit_registry, register, 28 | [exchange, <<"x-consistent-hash">>, ?MODULE]}}, 29 | {requires, rabbit_registry}, 30 | {enables, kernel_ready}, 31 | {cleanup, {rabbit_registry, unregister, 32 | [exchange, <<"x-consistent-hash">>]}}]}). 33 | 34 | -rabbit_boot_step( 35 | {rabbit_exchange_type_consistent_hash_mnesia, 36 | [{description, "exchange type x-consistent-hash: shared state"}, 37 | {mfa, {?MODULE, init, []}}, 38 | {requires, database}, 39 | {enables, external_infrastructure}]}). 40 | 41 | %% This data model allows for efficient routing and exchange deletion 42 | %% but less efficient (linear) binding management. 43 | 44 | -define(HASH_RING_STATE_TABLE, rabbit_exchange_type_consistent_hash_ring_state). 45 | 46 | -define(PROPERTIES, [<<"correlation_id">>, <<"message_id">>, <<"timestamp">>]). 47 | 48 | %% OTP 19.3 does not support exs1024s 49 | -define(SEED_ALGORITHM, exs1024). 50 | 51 | init() -> 52 | mnesia:create_table(?HASH_RING_STATE_TABLE, [{record_name, chx_hash_ring}, 53 | {attributes, record_info(fields, chx_hash_ring)}, 54 | {type, ordered_set}]), 55 | mnesia:add_table_copy(?HASH_RING_STATE_TABLE, node(), ram_copies), 56 | mnesia:wait_for_tables([?HASH_RING_STATE_TABLE], 30000), 57 | recover(), 58 | ok. 59 | 60 | info(_X) -> []. 61 | info(_X, _) -> []. 62 | 63 | description() -> 64 | [{description, <<"Consistent Hashing Exchange">>}]. 65 | 66 | serialise_events() -> false. 67 | 68 | route(#exchange {name = Name, 69 | arguments = Args}, 70 | #delivery {message = Msg}) -> 71 | case ets:lookup(?HASH_RING_STATE_TABLE, Name) of 72 | [] -> 73 | []; 74 | [#chx_hash_ring{bucket_map = BM}] -> 75 | case maps:size(BM) of 76 | 0 -> []; 77 | N -> 78 | K = value_to_hash(hash_on(Args), Msg), 79 | SelectedBucket = jump_consistent_hash(K, N), 80 | 81 | case maps:get(SelectedBucket, BM, undefined) of 82 | undefined -> 83 | rabbit_log:warning("Bucket ~p not found", [SelectedBucket]), 84 | []; 85 | Queue -> [Queue] 86 | end 87 | end 88 | end. 89 | 90 | validate(#exchange{arguments = Args}) -> 91 | case hash_args(Args) of 92 | {undefined, undefined} -> 93 | ok; 94 | {undefined, {_Type, Value}} -> 95 | case lists:member(Value, ?PROPERTIES) of 96 | true -> ok; 97 | false -> 98 | rabbit_misc:protocol_error(precondition_failed, 99 | "Unsupported property: ~s", 100 | [Value]) 101 | end; 102 | {_, undefined} -> 103 | ok; 104 | {_, _} -> 105 | rabbit_misc:protocol_error(precondition_failed, 106 | "hash-header and hash-property are mutually exclusive", 107 | []) 108 | end. 109 | 110 | validate_binding(_X, #binding { key = K }) -> 111 | try 112 | V = list_to_integer(binary_to_list(K)), 113 | case V < 1 of 114 | true -> {error, {binding_invalid, "The binding key must be greater than 0", []}}; 115 | false -> ok 116 | end 117 | catch error:badarg -> 118 | {error, {binding_invalid, "The binding key must be an integer: ~p", [K]}} 119 | end. 120 | 121 | maybe_initialise_hash_ring_state(transaction, #exchange{name = Name}) -> 122 | maybe_initialise_hash_ring_state(transaction, Name); 123 | maybe_initialise_hash_ring_state(transaction, X = #resource{}) -> 124 | case mnesia:read(?HASH_RING_STATE_TABLE, X) of 125 | [_] -> ok; 126 | [] -> 127 | rabbit_log:debug("Consistent hashing exchange: will initialise hashing ring schema database record"), 128 | mnesia:write_lock_table(?HASH_RING_STATE_TABLE), 129 | ok = mnesia:write(?HASH_RING_STATE_TABLE, #chx_hash_ring{ 130 | exchange = X, 131 | next_bucket_number = 0, 132 | bucket_map = #{}}, write) 133 | end; 134 | 135 | maybe_initialise_hash_ring_state(_, X) -> 136 | rabbit_misc:execute_mnesia_transaction( 137 | fun() -> maybe_initialise_hash_ring_state(transaction, X) end). 138 | 139 | recover() -> 140 | %% topology recovery has already happened, we have to recover state for any durable 141 | %% consistent hash exchanges since plugin activation was moved later in boot process 142 | %% starting with RabbitMQ 3.8.4 143 | case list_exchanges() of 144 | {ok, Xs} -> 145 | rabbit_log:debug("Consistent hashing exchange: have ~b durable exchanges to recover", [length(Xs)]), 146 | [recover_exchange_and_bindings(X) || X <- lists:usort(Xs)]; 147 | {aborted, Reason} -> 148 | rabbit_log:error( 149 | "Consistent hashing exchange: failed to recover durable exchange ring state, reason: ~p", 150 | [Reason]) 151 | end. 152 | 153 | list_exchanges() -> 154 | case mnesia:transaction( 155 | fun () -> 156 | mnesia:match_object(rabbit_exchange, 157 | #exchange{durable = true, type = 'x-consistent-hash', _ = '_'}, write) 158 | end) of 159 | {atomic, Xs} -> 160 | {ok, Xs}; 161 | {aborted, Reason} -> 162 | {aborted, Reason} 163 | end. 164 | 165 | recover_exchange_and_bindings(#exchange{name = XName} = X) -> 166 | mnesia:transaction( 167 | fun () -> 168 | rabbit_log:debug("Consistent hashing exchange: will recover exchange ~s", [rabbit_misc:rs(XName)]), 169 | create(transaction, X), 170 | rabbit_log:debug("Consistent hashing exchange: recovered exchange ~s", [rabbit_misc:rs(XName)]), 171 | Bindings = rabbit_binding:list_for_source(XName), 172 | rabbit_log:debug("Consistent hashing exchange: have ~b bindings to recover for exchange ~s", 173 | [length(Bindings), rabbit_misc:rs(XName)]), 174 | [add_binding(transaction, X, B) || B <- lists:usort(Bindings)], 175 | rabbit_log:debug("Consistent hashing exchange: recovered bindings for exchange ~s", 176 | [rabbit_misc:rs(XName)]) 177 | end). 178 | 179 | create(transaction, X) -> 180 | maybe_initialise_hash_ring_state(transaction, X); 181 | create(Tx, X) -> 182 | maybe_initialise_hash_ring_state(Tx, X). 183 | 184 | delete(transaction, #exchange{name = Name}, _Bs) -> 185 | mnesia:write_lock_table(?HASH_RING_STATE_TABLE), 186 | 187 | ok = mnesia:delete({?HASH_RING_STATE_TABLE, Name}); 188 | delete(_Tx, _X, _Bs) -> 189 | ok. 190 | 191 | policy_changed(_X1, _X2) -> ok. 192 | 193 | add_binding(transaction, X, 194 | B = #binding{source = S, destination = D, key = K}) -> 195 | Weight = rabbit_data_coercion:to_integer(K), 196 | rabbit_log:debug("Consistent hashing exchange: adding binding from " 197 | "exchange ~s to destination ~s with routing key '~s'", [rabbit_misc:rs(S), rabbit_misc:rs(D), K]), 198 | 199 | case mnesia:read(?HASH_RING_STATE_TABLE, S) of 200 | [State0 = #chx_hash_ring{bucket_map = BM0, 201 | next_bucket_number = NexN0}] -> 202 | NextN = NexN0 + Weight, 203 | %% hi/lo bucket counters are 0-based but weight is 1-based 204 | Range = lists:seq(NexN0, (NextN - 1)), 205 | BM = lists:foldl(fun(Key, Acc) -> 206 | maps:put(Key, D, Acc) 207 | end, BM0, Range), 208 | State = State0#chx_hash_ring{bucket_map = BM, 209 | next_bucket_number = NextN}, 210 | 211 | ok = mnesia:write(?HASH_RING_STATE_TABLE, State, write), 212 | ok; 213 | [] -> 214 | maybe_initialise_hash_ring_state(transaction, S), 215 | add_binding(transaction, X, B) 216 | end; 217 | add_binding(none, _X, _B) -> 218 | ok. 219 | 220 | remove_bindings(transaction, _X, Bindings) -> 221 | [remove_binding(B) || B <- Bindings], 222 | 223 | ok; 224 | remove_bindings(none, X, Bindings) -> 225 | rabbit_misc:execute_mnesia_transaction( 226 | fun() -> remove_bindings(transaction, X, Bindings) end), 227 | ok. 228 | 229 | remove_binding(#binding{source = S, destination = D, key = RK}) -> 230 | Weight = rabbit_data_coercion:to_integer(RK), 231 | rabbit_log:debug("Consistent hashing exchange: removing binding " 232 | "from exchange '~p' to destination '~p' with routing key '~s'", 233 | [rabbit_misc:rs(S), rabbit_misc:rs(D), RK]), 234 | 235 | case mnesia:read(?HASH_RING_STATE_TABLE, S) of 236 | [State0 = #chx_hash_ring{bucket_map = BM0, 237 | next_bucket_number = NexN0}] -> 238 | %% Buckets with lower numbers stay as is; buckets that 239 | %% belong to this binding are removed; buckets with 240 | %% greater numbers are updated (their numbers are adjusted downwards by weight) 241 | BucketsOfThisBinding = maps:filter(fun (_K, V) -> V =:= D end, BM0), 242 | case maps:size(BucketsOfThisBinding) of 243 | 0 -> ok; 244 | N when N >= 1 -> 245 | KeysOfThisBinding = lists:usort(maps:keys(BucketsOfThisBinding)), 246 | LastBucket = lists:last(KeysOfThisBinding), 247 | FirstBucket = hd(KeysOfThisBinding), 248 | BucketsDownTheRing = maps:filter(fun (K, _) -> K > LastBucket end, BM0), 249 | UnchangedBuckets = maps:filter(fun (K, _) -> K < FirstBucket end, BM0), 250 | 251 | %% final state with "down the ring" buckets updated 252 | NewBucketsDownTheRing = maps:fold( 253 | fun(K0, V, Acc) -> 254 | maps:put(K0 - Weight, V, Acc) 255 | end, #{}, BucketsDownTheRing), 256 | BM1 = maps:merge(UnchangedBuckets, NewBucketsDownTheRing), 257 | NextN = NexN0 - Weight, 258 | State = State0#chx_hash_ring{bucket_map = BM1, 259 | next_bucket_number = NextN}, 260 | 261 | ok = mnesia:write(?HASH_RING_STATE_TABLE, State, write) 262 | end; 263 | [] -> 264 | rabbit_log:warning("Can't remove binding: hash ring state for exchange ~s wasn't found", 265 | [rabbit_misc:rs(S)]), 266 | ok 267 | end. 268 | 269 | ring_state(VirtualHost, Exchange) -> 270 | Resource = rabbit_misc:r(VirtualHost, exchange, Exchange), 271 | case mnesia:dirty_read(?HASH_RING_STATE_TABLE, Resource) of 272 | [] -> {error, not_found}; 273 | [Row] -> {ok, Row} 274 | end. 275 | 276 | assert_args_equivalence(X, Args) -> 277 | rabbit_exchange:assert_args_equivalence(X, Args). 278 | 279 | %% 280 | %% Jump-consistent hashing. 281 | %% 282 | 283 | jump_consistent_hash(_Key, 1) -> 284 | 0; 285 | jump_consistent_hash(KeyList, NumberOfBuckets) when is_list(KeyList) -> 286 | jump_consistent_hash(hd(KeyList), NumberOfBuckets); 287 | jump_consistent_hash(Key, NumberOfBuckets) when is_integer(Key) -> 288 | SeedState = rand:seed_s(?SEED_ALGORITHM, {Key, Key, Key}), 289 | jump_consistent_hash_value(-1, 0, NumberOfBuckets, SeedState); 290 | jump_consistent_hash(Key, NumberOfBuckets) -> 291 | jump_consistent_hash(erlang:phash2(Key), NumberOfBuckets). 292 | 293 | jump_consistent_hash_value(B, J, NumberOfBuckets, _SeedState) when J >= NumberOfBuckets -> 294 | B; 295 | 296 | jump_consistent_hash_value(_B0, J0, NumberOfBuckets, SeedState0) -> 297 | B = J0, 298 | {R, SeedState} = rand:uniform_s(SeedState0), 299 | J = trunc((B + 1) / R), 300 | jump_consistent_hash_value(B, J, NumberOfBuckets, SeedState). 301 | 302 | value_to_hash(undefined, #basic_message { routing_keys = Routes }) -> 303 | Routes; 304 | value_to_hash({header, Header}, #basic_message { content = Content }) -> 305 | Headers = rabbit_basic:extract_headers(Content), 306 | case Headers of 307 | undefined -> undefined; 308 | _ -> rabbit_misc:table_lookup(Headers, Header) 309 | end; 310 | value_to_hash({property, Property}, #basic_message { content = Content }) -> 311 | #content{properties = #'P_basic'{ correlation_id = CorrId, 312 | message_id = MsgId, 313 | timestamp = Timestamp }} = 314 | rabbit_binary_parser:ensure_content_decoded(Content), 315 | case Property of 316 | <<"correlation_id">> -> CorrId; 317 | <<"message_id">> -> MsgId; 318 | <<"timestamp">> -> 319 | case Timestamp of 320 | undefined -> undefined; 321 | _ -> integer_to_binary(Timestamp) 322 | end 323 | end. 324 | 325 | hash_args(Args) -> 326 | Header = 327 | case rabbit_misc:table_lookup(Args, <<"hash-header">>) of 328 | undefined -> undefined; 329 | {longstr, V1} -> {header, V1} 330 | end, 331 | Property = 332 | case rabbit_misc:table_lookup(Args, <<"hash-property">>) of 333 | undefined -> undefined; 334 | {longstr, V2} -> {property, V2} 335 | end, 336 | {Header, Property}. 337 | 338 | hash_on(Args) -> 339 | case hash_args(Args) of 340 | {undefined, undefined} -> undefined; 341 | {Header, undefined} -> Header; 342 | {undefined, Property} -> Property 343 | end. 344 | -------------------------------------------------------------------------------- /test/rabbit_exchange_type_consistent_hash_SUITE.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. 6 | %% 7 | 8 | -module(rabbit_exchange_type_consistent_hash_SUITE). 9 | 10 | -compile(export_all). 11 | 12 | -include("rabbitmq_consistent_hash_exchange.hrl"). 13 | -include_lib("common_test/include/ct.hrl"). 14 | -include_lib("amqp_client/include/amqp_client.hrl"). 15 | -include_lib("eunit/include/eunit.hrl"). 16 | 17 | all() -> 18 | [ 19 | {group, routing_tests}, 20 | {group, hash_ring_management_tests} 21 | ]. 22 | 23 | groups() -> 24 | [ 25 | {routing_tests, [], [ 26 | routing_key_hashing_test, 27 | custom_header_hashing_test, 28 | message_id_hashing_test, 29 | correlation_id_hashing_test, 30 | timestamp_hashing_test, 31 | other_routing_test 32 | ]}, 33 | {hash_ring_management_tests, [], [ 34 | test_durable_exchange_hash_ring_recovery_between_node_restarts, 35 | test_hash_ring_updates_when_queue_is_deleted, 36 | test_hash_ring_updates_when_multiple_queues_are_deleted, 37 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure, 38 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case2, 39 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case3, 40 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case4, 41 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case5, 42 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case6, 43 | test_hash_ring_updates_when_exchange_is_deleted, 44 | test_hash_ring_updates_when_queue_is_unbound 45 | ]} 46 | ]. 47 | 48 | %% ------------------------------------------------------------------- 49 | %% Test suite setup/teardown 50 | %% ------------------------------------------------------------------- 51 | 52 | init_per_suite(Config) -> 53 | rabbit_ct_helpers:log_environment(), 54 | Config1 = rabbit_ct_helpers:set_config(Config, [ 55 | {rmq_nodename_suffix, ?MODULE} 56 | ]), 57 | rabbit_ct_helpers:run_setup_steps(Config1, 58 | rabbit_ct_broker_helpers:setup_steps() ++ 59 | rabbit_ct_client_helpers:setup_steps()). 60 | 61 | end_per_suite(Config) -> 62 | rabbit_ct_helpers:run_teardown_steps(Config, 63 | rabbit_ct_client_helpers:teardown_steps() ++ 64 | rabbit_ct_broker_helpers:teardown_steps()). 65 | 66 | init_per_group(_, Config) -> 67 | Config. 68 | 69 | end_per_group(_, Config) -> 70 | Config. 71 | 72 | init_per_testcase(Testcase, Config) -> 73 | clean_up_test_topology(Config), 74 | rabbit_ct_helpers:testcase_started(Config, Testcase). 75 | 76 | end_per_testcase(Testcase, Config) -> 77 | rabbit_ct_helpers:testcase_finished(Config, Testcase). 78 | 79 | %% ------------------------------------------------------------------- 80 | %% Test cases 81 | %% ------------------------------------------------------------------- 82 | 83 | -define(AllQs, [<<"q0">>, <<"q1">>, <<"q2">>, <<"q3">>, <<"q4">>, <<"q5">>, <<"q6">>, 84 | <<"e-q0">>, <<"e-q1">>, <<"e-q2">>, <<"e-q3">>, <<"e-q4">>, <<"e-q5">>, <<"e-q6">>, 85 | <<"d-q0">>, <<"d-q1">>, <<"d-q2">>, <<"d-q3">>, <<"d-q4">>, <<"d-q5">>, <<"d-q6">>]). 86 | -define(RoutingTestQs, [<<"q0">>, <<"q1">>, <<"q2">>, <<"q3">>]). 87 | 88 | %% N.B. lowering this value below 100K increases the probability 89 | %% of failing the Chi squared test in some environments 90 | -define(DEFAULT_SAMPLE_COUNT, 150000). 91 | 92 | routing_key_hashing_test(Config) -> 93 | ok = test_with_rk(Config, ?RoutingTestQs). 94 | 95 | custom_header_hashing_test(Config) -> 96 | ok = test_with_header(Config, ?RoutingTestQs). 97 | 98 | message_id_hashing_test(Config) -> 99 | ok = test_with_message_id(Config, ?RoutingTestQs). 100 | 101 | correlation_id_hashing_test(Config) -> 102 | ok = test_with_correlation_id(Config, ?RoutingTestQs). 103 | 104 | timestamp_hashing_test(Config) -> 105 | ok = test_with_timestamp(Config, ?RoutingTestQs). 106 | 107 | other_routing_test(Config) -> 108 | ok = test_binding_with_negative_routing_key(Config), 109 | ok = test_binding_with_non_numeric_routing_key(Config), 110 | ok = test_non_supported_property(Config), 111 | ok = test_mutually_exclusive_arguments(Config), 112 | ok. 113 | 114 | 115 | %% ------------------------------------------------------------------- 116 | %% Implementation 117 | %% ------------------------------------------------------------------- 118 | 119 | test_with_rk(Config, Qs) -> 120 | test0(Config, fun (E) -> 121 | #'basic.publish'{exchange = E, routing_key = rnd()} 122 | end, 123 | fun() -> 124 | #amqp_msg{props = #'P_basic'{}, payload = <<>>} 125 | end, [], Qs). 126 | 127 | test_with_header(Config, Qs) -> 128 | test0(Config, fun (E) -> 129 | #'basic.publish'{exchange = E} 130 | end, 131 | fun() -> 132 | H = [{<<"hashme">>, longstr, rnd()}], 133 | #amqp_msg{props = #'P_basic'{headers = H}, payload = <<>>} 134 | end, [{<<"hash-header">>, longstr, <<"hashme">>}], Qs). 135 | 136 | test_with_correlation_id(Config, Qs) -> 137 | test0(Config, fun(E) -> 138 | #'basic.publish'{exchange = E} 139 | end, 140 | fun() -> 141 | #amqp_msg{props = #'P_basic'{correlation_id = rnd()}, payload = <<>>} 142 | end, [{<<"hash-property">>, longstr, <<"correlation_id">>}], Qs). 143 | 144 | test_with_message_id(Config, Qs) -> 145 | test0(Config, fun(E) -> 146 | #'basic.publish'{exchange = E} 147 | end, 148 | fun() -> 149 | #amqp_msg{props = #'P_basic'{message_id = rnd()}, payload = <<>>} 150 | end, [{<<"hash-property">>, longstr, <<"message_id">>}], Qs). 151 | 152 | test_with_timestamp(Config, Qs) -> 153 | test0(Config, fun(E) -> 154 | #'basic.publish'{exchange = E} 155 | end, 156 | fun() -> 157 | #amqp_msg{props = #'P_basic'{timestamp = rnd_int()}, payload = <<>>} 158 | end, [{<<"hash-property">>, longstr, <<"timestamp">>}], Qs). 159 | 160 | test_mutually_exclusive_arguments(Config) -> 161 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 162 | 163 | process_flag(trap_exit, true), 164 | Cmd = #'exchange.declare'{ 165 | exchange = <<"fail">>, 166 | type = <<"x-consistent-hash">>, 167 | arguments = [{<<"hash-header">>, longstr, <<"foo">>}, 168 | {<<"hash-property">>, longstr, <<"bar">>}] 169 | }, 170 | ?assertExit(_, amqp_channel:call(Chan, Cmd)), 171 | 172 | rabbit_ct_client_helpers:close_channel(Chan), 173 | ok. 174 | 175 | test_non_supported_property(Config) -> 176 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 177 | 178 | process_flag(trap_exit, true), 179 | Cmd = #'exchange.declare'{ 180 | exchange = <<"fail">>, 181 | type = <<"x-consistent-hash">>, 182 | arguments = [{<<"hash-property">>, longstr, <<"app_id">>}] 183 | }, 184 | ?assertExit(_, amqp_channel:call(Chan, Cmd)), 185 | 186 | rabbit_ct_client_helpers:close_channel(Chan), 187 | ok. 188 | 189 | rnd() -> 190 | list_to_binary(integer_to_list(rnd_int())). 191 | 192 | rnd_int() -> 193 | rand:uniform(10000000). 194 | 195 | test0(Config, MakeMethod, MakeMsg, DeclareArgs, Queues) -> 196 | test0(Config, MakeMethod, MakeMsg, DeclareArgs, Queues, ?DEFAULT_SAMPLE_COUNT). 197 | 198 | test0(Config, MakeMethod, MakeMsg, DeclareArgs, [Q1, Q2, Q3, Q4] = Queues, IterationCount) -> 199 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 200 | #'confirm.select_ok'{} = amqp_channel:call(Chan, #'confirm.select'{}), 201 | 202 | CHX = <<"e">>, 203 | 204 | clean_up_test_topology(Config, CHX, Queues), 205 | 206 | #'exchange.declare_ok'{} = 207 | amqp_channel:call(Chan, 208 | #'exchange.declare' { 209 | exchange = CHX, 210 | type = <<"x-consistent-hash">>, 211 | auto_delete = true, 212 | arguments = DeclareArgs 213 | }), 214 | [#'queue.declare_ok'{} = 215 | amqp_channel:call(Chan, #'queue.declare' { 216 | queue = Q, exclusive = true }) || Q <- Queues], 217 | [#'queue.bind_ok'{} = 218 | amqp_channel:call(Chan, #'queue.bind' {queue = Q, 219 | exchange = CHX, 220 | routing_key = <<"1">>}) 221 | || Q <- [Q1, Q2]], 222 | [#'queue.bind_ok'{} = 223 | amqp_channel:call(Chan, #'queue.bind' {queue = Q, 224 | exchange = CHX, 225 | routing_key = <<"2">>}) 226 | || Q <- [Q3, Q4]], 227 | 228 | [amqp_channel:call(Chan, 229 | MakeMethod(CHX), 230 | MakeMsg()) || _ <- lists:duplicate(IterationCount, const)], 231 | amqp_channel:wait_for_confirms(Chan, 300), 232 | timer:sleep(500), 233 | 234 | Counts = 235 | [begin 236 | #'queue.declare_ok'{message_count = M} = 237 | amqp_channel:call(Chan, #'queue.declare' {queue = Q, 238 | exclusive = true}), 239 | M 240 | end || Q <- Queues], 241 | ?assertEqual(IterationCount, lists:sum(Counts)), %% All messages got routed 242 | 243 | %% Chi-square test 244 | %% H0: routing keys are not evenly distributed according to weight 245 | Expected = [IterationCount div 6, IterationCount div 6, (IterationCount div 6) * 2, (IterationCount div 6) * 2], 246 | Obs = lists:zip(Counts, Expected), 247 | Chi = lists:sum([((O - E) * (O - E)) / E || {O, E} <- Obs]), 248 | ct:pal("Chi-square test for 3 degrees of freedom is ~p, p = 0.01 is 11.35, observations (counts, expected): ~p", 249 | [Chi, Obs]), 250 | 251 | clean_up_test_topology(Config, CHX, Queues), 252 | rabbit_ct_client_helpers:close_channel(Chan), 253 | ok. 254 | 255 | test_binding_with_negative_routing_key(Config) -> 256 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 257 | X = <<"bind-fail">>, 258 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 259 | 260 | Declare1 = #'exchange.declare'{exchange = X, 261 | type = <<"x-consistent-hash">>}, 262 | #'exchange.declare_ok'{} = amqp_channel:call(Chan, Declare1), 263 | Q = <<"test-queue">>, 264 | Declare2 = #'queue.declare'{queue = Q}, 265 | #'queue.declare_ok'{} = amqp_channel:call(Chan, Declare2), 266 | process_flag(trap_exit, true), 267 | Cmd = #'queue.bind'{exchange = <<"bind-fail">>, 268 | routing_key = <<"-1">>}, 269 | ?assertExit(_, amqp_channel:call(Chan, Cmd)), 270 | Ch2 = rabbit_ct_client_helpers:open_channel(Config, 0), 271 | amqp_channel:call(Ch2, #'queue.delete'{queue = Q}), 272 | 273 | rabbit_ct_client_helpers:close_channel(Chan), 274 | rabbit_ct_client_helpers:close_channel(Ch2), 275 | ok. 276 | 277 | test_binding_with_non_numeric_routing_key(Config) -> 278 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 279 | X = <<"bind-fail">>, 280 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 281 | 282 | Declare1 = #'exchange.declare'{exchange = X, 283 | type = <<"x-consistent-hash">>}, 284 | #'exchange.declare_ok'{} = amqp_channel:call(Chan, Declare1), 285 | Q = <<"test-queue">>, 286 | Declare2 = #'queue.declare'{queue = Q}, 287 | #'queue.declare_ok'{} = amqp_channel:call(Chan, Declare2), 288 | process_flag(trap_exit, true), 289 | Cmd = #'queue.bind'{exchange = <<"bind-fail">>, 290 | routing_key = <<"not-a-number">>}, 291 | ?assertExit(_, amqp_channel:call(Chan, Cmd)), 292 | 293 | Ch2 = rabbit_ct_client_helpers:open_channel(Config, 0), 294 | amqp_channel:call(Ch2, #'queue.delete'{queue = Q}), 295 | 296 | rabbit_ct_client_helpers:close_channel(Chan), 297 | ok. 298 | 299 | %% 300 | %% Hash Ring management 301 | %% 302 | 303 | test_durable_exchange_hash_ring_recovery_between_node_restarts(Config) -> 304 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 305 | 306 | X = <<"test_hash_ring_recovery_between_node_restarts">>, 307 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 308 | 309 | Declare = #'exchange.declare'{exchange = X, 310 | durable = true, 311 | type = <<"x-consistent-hash">>}, 312 | #'exchange.declare_ok'{} = amqp_channel:call(Chan, Declare), 313 | 314 | Queues = [<<"d-q1">>, <<"d-q2">>, <<"d-q3">>], 315 | [#'queue.declare_ok'{} = 316 | amqp_channel:call(Chan, #'queue.declare'{ 317 | queue = Q, durable = true, exclusive = false}) || Q <- Queues], 318 | [#'queue.bind_ok'{} = 319 | amqp_channel:call(Chan, #'queue.bind'{queue = Q, 320 | exchange = X, 321 | routing_key = <<"3">>}) 322 | || Q <- Queues], 323 | 324 | ?assertEqual(9, count_buckets_of_exchange(Config, X)), 325 | assert_ring_consistency(Config, X), 326 | 327 | rabbit_ct_broker_helpers:restart_node(Config, 0), 328 | rabbit_ct_helpers:await_condition( 329 | fun () -> count_buckets_of_exchange(Config, X) > 0 end, 15000), 330 | 331 | ?assertEqual(9, count_buckets_of_exchange(Config, X)), 332 | assert_ring_consistency(Config, X), 333 | 334 | clean_up_test_topology(Config, X, Queues), 335 | rabbit_ct_client_helpers:close_channel(Chan), 336 | ok. 337 | 338 | test_hash_ring_updates_when_queue_is_deleted(Config) -> 339 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 340 | 341 | X = <<"test_hash_ring_updates_when_queue_is_deleted">>, 342 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 343 | 344 | Declare = #'exchange.declare'{exchange = X, 345 | type = <<"x-consistent-hash">>}, 346 | #'exchange.declare_ok'{} = amqp_channel:call(Chan, Declare), 347 | 348 | Q = <<"d-q">>, 349 | #'queue.declare_ok'{} = 350 | amqp_channel:call(Chan, #'queue.declare'{ 351 | queue = Q, durable = true, exclusive = false}), 352 | #'queue.bind_ok'{} = 353 | amqp_channel:call(Chan, #'queue.bind'{queue = Q, 354 | exchange = X, 355 | routing_key = <<"1">>}), 356 | 357 | ?assertEqual(1, count_buckets_of_exchange(Config, X)), 358 | assert_ring_consistency(Config, X), 359 | 360 | amqp_channel:call(Chan, #'queue.delete' {queue = Q}), 361 | ?assertEqual(0, count_buckets_of_exchange(Config, X)), 362 | 363 | clean_up_test_topology(Config, X, [Q]), 364 | rabbit_ct_client_helpers:close_channel(Chan), 365 | ok. 366 | 367 | test_hash_ring_updates_when_multiple_queues_are_deleted(Config) -> 368 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 369 | 370 | X = <<"test_hash_ring_updates_when_multiple_queues_are_deleted">>, 371 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 372 | 373 | Declare = #'exchange.declare'{exchange = X, 374 | type = <<"x-consistent-hash">>}, 375 | #'exchange.declare_ok'{} = amqp_channel:call(Chan, Declare), 376 | 377 | Queues = [<<"d-q1">>, <<"d-q2">>, <<"d-q3">>], 378 | [#'queue.declare_ok'{} = 379 | amqp_channel:call(Chan, #'queue.declare'{ 380 | queue = Q, durable = true, exclusive = false}) || Q <- Queues], 381 | [#'queue.bind_ok'{} = 382 | amqp_channel:call(Chan, #'queue.bind'{queue = Q, 383 | exchange = X, 384 | routing_key = <<"3">>}) 385 | || Q <- Queues], 386 | 387 | ?assertEqual(9, count_buckets_of_exchange(Config, X)), 388 | assert_ring_consistency(Config, X), 389 | 390 | amqp_channel:call(Chan, #'queue.delete' {queue = <<"d-q1">>}), 391 | ?assertEqual(6, count_buckets_of_exchange(Config, X)), 392 | assert_ring_consistency(Config, X), 393 | 394 | amqp_channel:call(Chan, #'queue.delete' {queue = <<"d-q2">>}), 395 | amqp_channel:call(Chan, #'queue.delete' {queue = <<"d-q3">>}), 396 | ?assertEqual(0, count_buckets_of_exchange(Config, X)), 397 | 398 | clean_up_test_topology(Config, X, Queues), 399 | rabbit_ct_client_helpers:close_channel(Chan), 400 | ok. 401 | 402 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure(Config) -> 403 | Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), 404 | {ok, Chan} = amqp_connection:open_channel(Conn), 405 | 406 | X = <<"test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure">>, 407 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 408 | 409 | Declare = #'exchange.declare'{exchange = X, 410 | type = <<"x-consistent-hash">>}, 411 | #'exchange.declare_ok'{} = amqp_channel:call(Chan, Declare), 412 | 413 | Queues = [<<"e-q1">>, <<"e-q2">>, <<"e-q3">>, <<"e-q4">>, <<"e-q5">>, <<"e-q6">>], 414 | [#'queue.declare_ok'{} = 415 | amqp_channel:call(Chan, #'queue.declare' { 416 | queue = Q, exclusive = true }) || Q <- Queues], 417 | [#'queue.bind_ok'{} = 418 | amqp_channel:call(Chan, #'queue.bind' {queue = Q, 419 | exchange = X, 420 | routing_key = <<"3">>}) 421 | || Q <- Queues], 422 | 423 | ct:pal("all hash ring rows: ~p", [hash_ring_rows(Config)]), 424 | 425 | ?assertEqual(18, count_buckets_of_exchange(Config, X)), 426 | assert_ring_consistency(Config, X), 427 | ok = amqp_connection:close(Conn), 428 | timer:sleep(500), 429 | 430 | ct:pal("all hash ring rows after connection closure: ~p", [hash_ring_rows(Config)]), 431 | 432 | ?assertEqual(0, count_buckets_of_exchange(Config, X)), 433 | clean_up_test_topology(Config, X, []), 434 | ok. 435 | 436 | %% rabbitmq/rabbitmq-consistent-has-exchange#40, uses higher weights 437 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case2(Config) -> 438 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case(Config, ?FUNCTION_NAME, 50). 439 | 440 | %% rabbitmq/rabbitmq-consistent-has-exchange#40 441 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case3(Config) -> 442 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case(Config, ?FUNCTION_NAME, 34). 443 | 444 | %% rabbitmq/rabbitmq-consistent-has-exchange#40 445 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case4(Config) -> 446 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case(Config, ?FUNCTION_NAME, 100). 447 | 448 | %% rabbitmq/rabbitmq-consistent-has-exchange#40 449 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case5(Config) -> 450 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case(Config, ?FUNCTION_NAME, 268). 451 | 452 | %% rabbitmq/rabbitmq-consistent-has-exchange#40 453 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case6(Config) -> 454 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case(Config, ?FUNCTION_NAME, 1937). 455 | 456 | test_hash_ring_updates_when_exclusive_queues_are_deleted_due_to_connection_closure_case(Config, XAsList, Key) -> 457 | Conn = rabbit_ct_client_helpers:open_unmanaged_connection(Config), 458 | {ok, Chan} = amqp_connection:open_channel(Conn), 459 | 460 | X = atom_to_binary(XAsList, utf8), 461 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 462 | 463 | Declare = #'exchange.declare'{exchange = X, 464 | type = <<"x-consistent-hash">>}, 465 | #'exchange.declare_ok'{} = amqp_channel:call(Chan, Declare), 466 | 467 | NumQueues = 6, 468 | Queues = [begin 469 | #'queue.declare_ok'{queue = Q} = 470 | amqp_channel:call(Chan, #'queue.declare' {exclusive = true }), 471 | Q 472 | end || _ <- lists:seq(1, NumQueues)], 473 | [#'queue.bind_ok'{} = 474 | amqp_channel:call(Chan, #'queue.bind' {queue = Q, 475 | exchange = X, 476 | routing_key = integer_to_binary(Key)}) 477 | || Q <- Queues], 478 | 479 | ct:pal("all hash ring rows: ~p", [hash_ring_rows(Config)]), 480 | 481 | %% NumQueues x 'Key' buckets per binding 482 | ?assertEqual(NumQueues * Key, count_buckets_of_exchange(Config, X)), 483 | assert_ring_consistency(Config, X), 484 | ok = amqp_connection:close(Conn), 485 | timer:sleep(1000), 486 | 487 | ct:pal("all hash ring rows after connection closure (~p): ~p", [XAsList, hash_ring_rows(Config)]), 488 | 489 | ?assertEqual(0, count_buckets_of_exchange(Config, X)), 490 | clean_up_test_topology(Config, X, []), 491 | ok. 492 | 493 | test_hash_ring_updates_when_exchange_is_deleted(Config) -> 494 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 495 | 496 | X = <<"test_hash_ring_updates_when_exchange_is_deleted">>, 497 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 498 | 499 | Declare = #'exchange.declare'{exchange = X, 500 | type = <<"x-consistent-hash">>}, 501 | #'exchange.declare_ok'{} = amqp_channel:call(Chan, Declare), 502 | 503 | Queues = [<<"e-q1">>, <<"e-q2">>, <<"e-q3">>, <<"e-q4">>, <<"e-q5">>, <<"e-q6">>], 504 | [#'queue.declare_ok'{} = 505 | amqp_channel:call(Chan, #'queue.declare' { 506 | queue = Q, exclusive = true }) || Q <- Queues], 507 | [#'queue.bind_ok'{} = 508 | amqp_channel:call(Chan, #'queue.bind' {queue = Q, 509 | exchange = X, 510 | routing_key = <<"2">>}) 511 | || Q <- Queues], 512 | 513 | ?assertEqual(12, count_buckets_of_exchange(Config, X)), 514 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 515 | 516 | ?assertEqual(0, count_buckets_of_exchange(Config, X)), 517 | 518 | amqp_channel:call(Chan, #'queue.delete' {queue = <<"e-q1">>}), 519 | ?assertEqual(0, count_buckets_of_exchange(Config, X)), 520 | 521 | clean_up_test_topology(Config, X, Queues), 522 | rabbit_ct_client_helpers:close_channel(Chan), 523 | ok. 524 | 525 | test_hash_ring_updates_when_queue_is_unbound(Config) -> 526 | Chan = rabbit_ct_client_helpers:open_channel(Config, 0), 527 | 528 | X = <<"test_hash_ring_updates_when_queue_is_unbound">>, 529 | amqp_channel:call(Chan, #'exchange.delete' {exchange = X}), 530 | 531 | Declare = #'exchange.declare'{exchange = X, 532 | type = <<"x-consistent-hash">>}, 533 | #'exchange.declare_ok'{} = amqp_channel:call(Chan, Declare), 534 | 535 | Queues = [<<"e-q1">>, <<"e-q2">>, <<"e-q3">>, <<"e-q4">>, <<"e-q5">>, <<"e-q6">>], 536 | [#'queue.declare_ok'{} = 537 | amqp_channel:call(Chan, #'queue.declare' { 538 | queue = Q, exclusive = true }) || Q <- Queues], 539 | [#'queue.bind_ok'{} = 540 | amqp_channel:call(Chan, #'queue.bind' {queue = Q, 541 | exchange = X, 542 | routing_key = <<"2">>}) 543 | || Q <- Queues], 544 | 545 | ?assertEqual(12, count_buckets_of_exchange(Config, X)), 546 | amqp_channel:call(Chan, #'queue.unbind'{exchange = X, 547 | queue = <<"e-q2">>, 548 | routing_key = <<"2">>}), 549 | 550 | ?assertEqual(10, count_buckets_of_exchange(Config, X)), 551 | amqp_channel:call(Chan, #'queue.unbind'{exchange = X, 552 | queue = <<"e-q6">>, 553 | routing_key = <<"2">>}), 554 | ?assertEqual(8, count_buckets_of_exchange(Config, X)), 555 | 556 | clean_up_test_topology(Config, X, Queues), 557 | rabbit_ct_client_helpers:close_channel(Chan), 558 | ok. 559 | 560 | 561 | %% 562 | %% Helpers 563 | %% 564 | 565 | hash_ring_state(Config, X) -> 566 | rabbit_ct_broker_helpers:rpc( 567 | Config, 0, ets, lookup, 568 | [rabbit_exchange_type_consistent_hash_ring_state, 569 | rabbit_misc:r(<<"/">>, exchange, X)]). 570 | 571 | hash_ring_rows(Config) -> 572 | rabbit_ct_broker_helpers:rpc( 573 | Config, 0, ets, tab2list, [rabbit_exchange_type_consistent_hash_ring_state]). 574 | 575 | assert_ring_consistency(Config, X) -> 576 | [#chx_hash_ring{bucket_map = M}] = hash_ring_state(Config, X), 577 | Buckets = lists:usort(maps:keys(M)), 578 | Hi = lists:last(Buckets), 579 | 580 | %% bucket numbers form a sequence without gaps or duplicates 581 | ?assertEqual(lists:seq(0, Hi), lists:usort(Buckets)). 582 | 583 | count_buckets_of_exchange(Config, X) -> 584 | case hash_ring_state(Config, X) of 585 | [#chx_hash_ring{bucket_map = M}] -> maps:size(M); 586 | [] -> 0 587 | end. 588 | 589 | count_all_hash_ring_buckets(Config) -> 590 | Rows = hash_ring_rows(Config), 591 | lists:foldl(fun(#chx_hash_ring{bucket_map = M}, Acc) -> Acc + maps:size(M) end, 0, Rows). 592 | 593 | clean_up_test_topology(Config) -> 594 | clean_up_test_topology(Config, none, ?AllQs). 595 | 596 | clean_up_test_topology(Config, none, Qs) -> 597 | Ch = rabbit_ct_client_helpers:open_channel(Config, 0), 598 | [amqp_channel:call(Ch, #'queue.delete' {queue = Q}) || Q <- Qs], 599 | rabbit_ct_client_helpers:close_channel(Ch); 600 | 601 | clean_up_test_topology(Config, X, Qs) -> 602 | Ch = rabbit_ct_client_helpers:open_channel(Config, 0), 603 | amqp_channel:call(Ch, #'exchange.delete' {exchange = X}), 604 | [amqp_channel:call(Ch, #'queue.delete' {queue = Q}) || Q <- Qs], 605 | rabbit_ct_client_helpers:close_channel(Ch). 606 | --------------------------------------------------------------------------------