├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── LICENSE-MPL-RabbitMQ ├── Makefile ├── README.md ├── erlang.mk ├── include └── rabbit_shovel.hrl ├── rabbitmq-components.mk ├── src ├── Elixir.RabbitMQ.CLI.Ctl.Commands.DeleteShovelCommand.erl ├── Elixir.RabbitMQ.CLI.Ctl.Commands.RestartShovelCommand.erl ├── Elixir.RabbitMQ.CLI.Ctl.Commands.ShovelStatusCommand.erl ├── rabbit_amqp091_shovel.erl ├── rabbit_amqp10_shovel.erl ├── rabbit_shovel.erl ├── rabbit_shovel_behaviour.erl ├── rabbit_shovel_config.erl ├── rabbit_shovel_dyn_worker_sup.erl ├── rabbit_shovel_dyn_worker_sup_sup.erl ├── rabbit_shovel_parameters.erl ├── rabbit_shovel_status.erl ├── rabbit_shovel_sup.erl ├── rabbit_shovel_util.erl ├── rabbit_shovel_worker.erl └── rabbit_shovel_worker_sup.erl └── test ├── amqp10_SUITE.erl ├── amqp10_dynamic_SUITE.erl ├── amqp10_shovel_SUITE.erl ├── config_SUITE.erl ├── configuration_SUITE.erl ├── delete_shovel_command_SUITE.erl ├── dynamic_SUITE.erl ├── parameters_SUITE.erl ├── shovel_status_command_SUITE.erl └── shovel_test_utils.erl /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Thank you for using RabbitMQ and for taking the time to report an 2 | issue. 3 | 4 | ## Does This Belong to GitHub or RabbitMQ Mailing List? 5 | 6 | *Important:* please first read the `CONTRIBUTING.md` document in the 7 | root of this repository. It will help you determine whether your 8 | feedback should be directed to the RabbitMQ mailing list [1] instead. 9 | 10 | ## Please Help Maintainers and Contributors Help You 11 | 12 | In order for the RabbitMQ team to investigate your issue, please provide 13 | **as much as possible** of the following details: 14 | 15 | * RabbitMQ version 16 | * Erlang version 17 | * RabbitMQ server and client application log files 18 | * A runnable code sample, terminal transcript or detailed set of 19 | instructions that can be used to reproduce the issue 20 | * RabbitMQ plugin information via `rabbitmq-plugins list` 21 | * Client library version (for all libraries used) 22 | * Operating system, version, and patch level 23 | 24 | Running the `rabbitmq-collect-env` [2] script can provide most of the 25 | information needed. Please make the archive available via a third-party 26 | service and note that **the script does not attempt to scrub any 27 | sensitive data**. 28 | 29 | If your issue involves RabbitMQ management UI or HTTP API, please also provide 30 | the following: 31 | 32 | * Browser and its version 33 | * What management UI page was used (if applicable) 34 | * How the HTTP API requests performed can be reproduced with `curl` 35 | * Operating system on which you are running your browser, and its version 36 | * Errors reported in the JavaScript console (if any) 37 | 38 | This information **greatly speeds up issue investigation** (or makes it 39 | possible to investigate it at all). Please help project maintainers and 40 | contributors to help you by providing it! 41 | 42 | 1. https://groups.google.com/forum/#!forum/rabbitmq-users 43 | 2. https://github.com/rabbitmq/support-tools/blob/master/scripts/rabbitmq-collect-env 44 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Proposed Changes 2 | 3 | Please describe the big picture of your changes here to communicate to the 4 | RabbitMQ team why we should accept this pull request. If it fixes a bug or 5 | resolves a feature request, be sure to link to that issue. 6 | 7 | A pull request that doesn't explain **why** the change was made has a much 8 | lower chance of being accepted. 9 | 10 | If English isn't your first language, don't worry about it and try to 11 | communicate the problem you are trying to solve to the best of your abilities. 12 | As long as we can understand the intent, it's all good. 13 | 14 | ## Types of Changes 15 | 16 | What types of changes does your code introduce to this project? 17 | _Put an `x` in the boxes that apply_ 18 | 19 | - [ ] Bugfix (non-breaking change which fixes issue #NNNN) 20 | - [ ] New feature (non-breaking change which adds functionality) 21 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 22 | - [ ] Documentation (correction or otherwise) 23 | - [ ] Cosmetics (whitespace, appearance) 24 | 25 | ## Checklist 26 | 27 | _Put an `x` in the boxes that apply. You can also fill these out after creating 28 | the PR. If you're unsure about any of them, don't hesitate to ask on the 29 | mailing list. We're here to help! This is simply a reminder of what we are 30 | going to look for before merging your code._ 31 | 32 | - [ ] I have read the `CONTRIBUTING.md` document 33 | - [ ] I have signed the CA (see https://cla.pivotal.io/sign/rabbitmq) 34 | - [ ] All tests pass locally with my changes 35 | - [ ] I have added tests that prove my fix is effective or that my feature works 36 | - [ ] I have added necessary documentation (if appropriate) 37 | - [ ] Any dependent changes have been merged and published in related repositories 38 | 39 | ## Further Comments 40 | 41 | If this is a relatively large or complex change, kick off the discussion by 42 | explaining why you chose the solution you did and what alternatives you 43 | considered, etc. 44 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .sw? 2 | .*.sw? 3 | *.beam 4 | *.plt 5 | /.erlang.mk/ 6 | /cover/ 7 | /deps/ 8 | /doc/ 9 | /ebin/ 10 | /escript/ 11 | /escript.lock 12 | /logs/ 13 | /plugins/ 14 | /plugins.lock 15 | /sbin/ 16 | /sbin.lock 17 | /xrefr 18 | 19 | elvis 20 | elvis.config 21 | 22 | /rabbitmq_shovel.d 23 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # vim:sw=2:et: 2 | 3 | os: linux 4 | dist: xenial 5 | language: elixir 6 | notifications: 7 | email: 8 | recipients: 9 | - alerts@rabbitmq.com 10 | on_success: never 11 | on_failure: always 12 | addons: 13 | apt: 14 | packages: 15 | - awscli 16 | cache: 17 | apt: true 18 | env: 19 | global: 20 | - secure: gGj9z50r1UhYRLmm3RL2fTMJ56gWmLxusB+uZLteJXMm8mWVnh8cEy6o4IKpo+7zR89eDLciNrSBmIohh/VMPKMLLr20gI9D5gIXHZmxMWavkc2Pnyk4KEqzoYebsWKUBtvktafGlaW+W9E/DZdAhEh3xuqUN7SppypUfSag/yM= 21 | - secure: FZTb3CpPbv816P5pFQUiyYsFHsRNTJDZ28bBrW7ZfAzCVYuu6PysCb+cofxSfPA0OMhUB46WtFeprx8sMgER1lEvUsIAyB8T5avPegOuFV9c4Wk1xAgK346eB+TH9ORjQZwx8beiMAu39aWwBu/zykcl2rWHozZMQOOgCK2Jzdc= 22 | 23 | # $base_rmq_ref is used by rabbitmq-components.mk to select the 24 | # appropriate branch for dependencies. 25 | - base_rmq_ref=master 26 | 27 | elixir: 28 | - '1.9' 29 | otp_release: 30 | - '21.3' 31 | - '22.2' 32 | 33 | install: 34 | # This project being an Erlang one (we just set language to Elixir 35 | # to ensure it is installed), we don't want Travis to run mix(1) 36 | # automatically as it will break. 37 | skip 38 | 39 | script: 40 | # $current_rmq_ref is also used by rabbitmq-components.mk to select 41 | # the appropriate branch for dependencies. 42 | - make check-rabbitmq-components.mk 43 | current_rmq_ref="${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" 44 | - make xref 45 | current_rmq_ref="${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" 46 | - make tests 47 | current_rmq_ref="${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" 48 | 49 | after_failure: 50 | - | 51 | cd "$TRAVIS_BUILD_DIR" 52 | if test -d logs && test "$AWS_ACCESS_KEY_ID" && test "$AWS_SECRET_ACCESS_KEY"; then 53 | archive_name="$(basename "$TRAVIS_REPO_SLUG")-$TRAVIS_JOB_NUMBER" 54 | 55 | tar -c --transform "s/^logs/${archive_name}/" -f - logs | \ 56 | xz > "${archive_name}.tar.xz" 57 | 58 | aws s3 cp "${archive_name}.tar.xz" s3://server-release-pipeline/travis-ci-logs/ \ 59 | --region eu-west-1 \ 60 | --acl public-read 61 | fi 62 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | As contributors and maintainers of this project, and in the interest of fostering an open 4 | and welcoming community, we pledge to respect all people who contribute through reporting 5 | issues, posting feature requests, updating documentation, submitting pull requests or 6 | patches, and other activities. 7 | 8 | We are committed to making participation in this project a harassment-free experience for 9 | everyone, regardless of level of experience, gender, gender identity and expression, 10 | sexual orientation, disability, personal appearance, body size, race, ethnicity, age, 11 | religion, or nationality. 12 | 13 | Examples of unacceptable behavior by participants include: 14 | 15 | * The use of sexualized language or imagery 16 | * Personal attacks 17 | * Trolling or insulting/derogatory comments 18 | * Public or private harassment 19 | * Publishing other's private information, such as physical or electronic addresses, 20 | without explicit permission 21 | * Other unethical or unprofessional conduct 22 | 23 | Project maintainers have the right and responsibility to remove, edit, or reject comments, 24 | commits, code, wiki edits, issues, and other contributions that are not aligned to this 25 | Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors 26 | that they deem inappropriate, threatening, offensive, or harmful. 27 | 28 | By adopting this Code of Conduct, project maintainers commit themselves to fairly and 29 | consistently applying these principles to every aspect of managing this project. Project 30 | maintainers who do not follow or enforce the Code of Conduct may be permanently removed 31 | from the project team. 32 | 33 | This Code of Conduct applies both within project spaces and in public spaces when an 34 | individual is representing the project or its community. 35 | 36 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by 37 | contacting a project maintainer at [info@rabbitmq.com](mailto:info@rabbitmq.com). All complaints will 38 | be reviewed and investigated and will result in a response that is deemed necessary and 39 | appropriate to the circumstances. Maintainers are obligated to maintain confidentiality 40 | with regard to the reporter of an incident. 41 | 42 | This Code of Conduct is adapted from the 43 | [Contributor Covenant](https://contributor-covenant.org), version 1.3.0, available at 44 | [contributor-covenant.org/version/1/3/0/](https://contributor-covenant.org/version/1/3/0/) 45 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Thank you for using RabbitMQ and for taking the time to contribute to the project. 2 | This document has two main parts: 3 | 4 | * when and how to file GitHub issues for RabbitMQ projects 5 | * how to submit pull requests 6 | 7 | They intend to save you and RabbitMQ maintainers some time, so please 8 | take a moment to read through them. 9 | 10 | ## Overview 11 | 12 | ### GitHub issues 13 | 14 | The RabbitMQ team uses GitHub issues for _specific actionable items_ that 15 | engineers can work on. This assumes the following: 16 | 17 | * GitHub issues are not used for questions, investigations, root cause 18 | analysis, discussions of potential issues, etc (as defined by this team) 19 | * Enough information is provided by the reporter for maintainers to work with 20 | 21 | The team receives many questions through various venues every single 22 | day. Frequently, these questions do not include the necessary details 23 | the team needs to begin useful work. GitHub issues can very quickly 24 | turn into a something impossible to navigate and make sense 25 | of. Because of this, questions, investigations, root cause analysis, 26 | and discussions of potential features are all considered to be 27 | [mailing list][rmq-users] material. If you are unsure where to begin, 28 | the [RabbitMQ users mailing list][rmq-users] is the right place. 29 | 30 | Getting all the details necessary to reproduce an issue, make a 31 | conclusion or even form a hypothesis about what's happening can take a 32 | fair amount of time. Please help others help you by providing a way to 33 | reproduce the behavior you're observing, or at least sharing as much 34 | relevant information as possible on the [RabbitMQ users mailing 35 | list][rmq-users]. 36 | 37 | Please provide versions of the software used: 38 | 39 | * RabbitMQ server 40 | * Erlang 41 | * Operating system version (and distribution, if applicable) 42 | * All client libraries used 43 | * RabbitMQ plugins (if applicable) 44 | 45 | The following information greatly helps in investigating and reproducing issues: 46 | 47 | * RabbitMQ server logs 48 | * A code example or terminal transcript that can be used to reproduce 49 | * Full exception stack traces (a single line message is not enough!) 50 | * `rabbitmqctl report` and `rabbitmqctl environment` output 51 | * Other relevant details about the environment and workload, e.g. a traffic capture 52 | * Feel free to edit out hostnames and other potentially sensitive information. 53 | 54 | To make collecting much of this and other environment information, use 55 | the [`rabbitmq-collect-env`][rmq-collect-env] script. It will produce an archive with 56 | server logs, operating system logs, output of certain diagnostics commands and so on. 57 | Please note that **no effort is made to scrub any information that may be sensitive**. 58 | 59 | ### Pull Requests 60 | 61 | RabbitMQ projects use pull requests to discuss, collaborate on and accept code contributions. 62 | Pull requests is the primary place of discussing code changes. 63 | 64 | Here's the recommended workflow: 65 | 66 | * [Fork the repository][github-fork] or repositories you plan on contributing to. If multiple 67 | repositories are involved in addressing the same issue, please use the same branch name 68 | in each repository 69 | * Create a branch with a descriptive name in the relevant repositories 70 | * Make your changes, run tests (usually with `make tests`), commit with a 71 | [descriptive message][git-commit-msgs], push to your fork 72 | * Submit pull requests with an explanation what has been changed and **why** 73 | * Submit a filled out and signed [Contributor Agreement][ca-agreement] if needed (see below) 74 | * Be patient. We will get to your pull request eventually 75 | 76 | If what you are going to work on is a substantial change, please first 77 | ask the core team for their opinion on the [RabbitMQ users mailing list][rmq-users]. 78 | 79 | ## Code of Conduct 80 | 81 | See [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md). 82 | 83 | ## Contributor Agreement 84 | 85 | If you want to contribute a non-trivial change, please submit a signed 86 | copy of our [Contributor Agreement][ca-agreement] around the time you 87 | submit your pull request. This will make it much easier (in some 88 | cases, possible) for the RabbitMQ team at Pivotal to merge your 89 | contribution. 90 | 91 | ## Where to Ask Questions 92 | 93 | If something isn't clear, feel free to ask on our [mailing list][rmq-users]. 94 | 95 | [rmq-collect-env]: https://github.com/rabbitmq/support-tools/blob/master/scripts/rabbitmq-collect-env 96 | [git-commit-msgs]: https://chris.beams.io/posts/git-commit/ 97 | [rmq-users]: https://groups.google.com/forum/#!forum/rabbitmq-users 98 | [ca-agreement]: https://cla.pivotal.io/sign/rabbitmq 99 | [github-fork]: https://help.github.com/articles/fork-a-repo/ 100 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This package is licensed under the MPL 2.0. For the MPL 2.0, please see LICENSE-MPL-RabbitMQ. 2 | 3 | If you have any questions regarding licensing, please contact us at 4 | info@rabbitmq.com. 5 | -------------------------------------------------------------------------------- /LICENSE-MPL-RabbitMQ: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT = rabbitmq_shovel 2 | PROJECT_DESCRIPTION = Data Shovel for RabbitMQ 3 | PROJECT_MOD = rabbit_shovel 4 | 5 | define PROJECT_ENV 6 | [ 7 | {defaults, [ 8 | {prefetch_count, 1000}, 9 | {ack_mode, on_confirm}, 10 | {publish_fields, []}, 11 | {publish_properties, []}, 12 | {reconnect_delay, 5} 13 | ]} 14 | ] 15 | endef 16 | 17 | define PROJECT_APP_EXTRA_KEYS 18 | {broker_version_requirements, []} 19 | endef 20 | 21 | DEPS = rabbit_common rabbit amqp_client amqp10_client 22 | dep_amqp10_client = git https://github.com/rabbitmq/rabbitmq-amqp1.0-client.git master 23 | 24 | LOCAL_DEPS = crypto 25 | 26 | TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers rabbitmq_amqp1_0 meck 27 | 28 | DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk 29 | DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk elvis_mk 30 | dep_elvis_mk = git https://github.com/inaka/elvis.mk.git master 31 | 32 | 33 | # FIXME: Use erlang.mk patched for RabbitMQ, while waiting for PRs to be 34 | # reviewed and merged. 35 | 36 | ERLANG_MK_REPO = https://github.com/rabbitmq/erlang.mk.git 37 | ERLANG_MK_COMMIT = rabbitmq-tmp 38 | 39 | include rabbitmq-components.mk 40 | include erlang.mk 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## RabbitMQ Shovel 2 | 3 | 4 | ## This was migrated to https://github.com/rabbitmq/rabbitmq-server 5 | 6 | This repository has been moved to the main unified RabbitMQ "monorepo", including all open issues. You can find the source under [/deps/rabbitmq_shovel](https://github.com/rabbitmq/rabbitmq-server/tree/master/deps/rabbitmq_shovel). 7 | All issues have been transferred. 8 | 9 | ## Overview 10 | 11 | RabbitMQ Shovel is a WAN-friendly tool for moving messages from 12 | a queue to an exchange, typically between different nodes. It is implemented 13 | as a RabbitMQ plugin and has a [management UI extension](https://github.com/rabbitmq/rabbitmq-shovel-management/). 14 | 15 | 16 | ## Supported RabbitMQ Versions 17 | 18 | This plugin ships with RabbitMQ, there is no need to 19 | install it separately. 20 | 21 | 22 | ## Documentation 23 | 24 | See [RabbitMQ shovel plugin](https://www.rabbitmq.com/shovel.html) on rabbitmq.com. 25 | 26 | 27 | ## License and Copyright 28 | 29 | Released under [the same license as RabbitMQ](https://www.rabbitmq.com/mpl.html). 30 | 31 | 2007-2020 (c) 2007-2020 VMware, Inc. or its affiliates. 32 | -------------------------------------------------------------------------------- /include/rabbit_shovel.hrl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. 6 | %% 7 | 8 | -record(endpoint, 9 | {uris, 10 | resource_declaration 11 | }). 12 | 13 | -record(shovel, 14 | {sources, 15 | destinations, 16 | prefetch_count, 17 | ack_mode, 18 | publish_fields, 19 | publish_properties, 20 | queue, 21 | reconnect_delay, 22 | delete_after = never 23 | }). 24 | 25 | -define(SHOVEL_USER, <<"rmq-shovel">>). 26 | 27 | -define(DEFAULT_PREFETCH, 1000). 28 | -define(DEFAULT_ACK_MODE, on_confirm). 29 | -define(DEFAULT_RECONNECT_DELAY, 5). 30 | 31 | -define(SHOVEL_GUIDE_URL, <<"https://rabbitmq.com/shovel.html">>). 32 | -------------------------------------------------------------------------------- /rabbitmq-components.mk: -------------------------------------------------------------------------------- 1 | ifeq ($(.DEFAULT_GOAL),) 2 | # Define default goal to `all` because this file defines some targets 3 | # before the inclusion of erlang.mk leading to the wrong target becoming 4 | # the default. 5 | .DEFAULT_GOAL = all 6 | endif 7 | 8 | # PROJECT_VERSION defaults to: 9 | # 1. the version exported by rabbitmq-server-release; 10 | # 2. the version stored in `git-revisions.txt`, if it exists; 11 | # 3. a version based on git-describe(1), if it is a Git clone; 12 | # 4. 0.0.0 13 | 14 | PROJECT_VERSION := $(RABBITMQ_VERSION) 15 | 16 | ifeq ($(PROJECT_VERSION),) 17 | PROJECT_VERSION := $(shell \ 18 | if test -f git-revisions.txt; then \ 19 | head -n1 git-revisions.txt | \ 20 | awk '{print $$$(words $(PROJECT_DESCRIPTION) version);}'; \ 21 | else \ 22 | (git describe --dirty --abbrev=7 --tags --always --first-parent \ 23 | 2>/dev/null || echo rabbitmq_v0_0_0) | \ 24 | sed -e 's/^rabbitmq_v//' -e 's/^v//' -e 's/_/./g' -e 's/-/+/' \ 25 | -e 's/-/./g'; \ 26 | fi) 27 | endif 28 | 29 | # -------------------------------------------------------------------- 30 | # RabbitMQ components. 31 | # -------------------------------------------------------------------- 32 | 33 | # For RabbitMQ repositories, we want to checkout branches which match 34 | # the parent project. For instance, if the parent project is on a 35 | # release tag, dependencies must be on the same release tag. If the 36 | # parent project is on a topic branch, dependencies must be on the same 37 | # topic branch or fallback to `stable` or `master` whichever was the 38 | # base of the topic branch. 39 | 40 | dep_amqp_client = git_rmq rabbitmq-erlang-client $(current_rmq_ref) $(base_rmq_ref) master 41 | dep_amqp10_client = git_rmq rabbitmq-amqp1.0-client $(current_rmq_ref) $(base_rmq_ref) master 42 | dep_amqp10_common = git_rmq rabbitmq-amqp1.0-common $(current_rmq_ref) $(base_rmq_ref) master 43 | dep_rabbit = git_rmq rabbitmq-server $(current_rmq_ref) $(base_rmq_ref) master 44 | dep_rabbit_common = git_rmq rabbitmq-common $(current_rmq_ref) $(base_rmq_ref) master 45 | dep_rabbitmq_amqp1_0 = git_rmq rabbitmq-amqp1.0 $(current_rmq_ref) $(base_rmq_ref) master 46 | dep_rabbitmq_auth_backend_amqp = git_rmq rabbitmq-auth-backend-amqp $(current_rmq_ref) $(base_rmq_ref) master 47 | dep_rabbitmq_auth_backend_cache = git_rmq rabbitmq-auth-backend-cache $(current_rmq_ref) $(base_rmq_ref) master 48 | dep_rabbitmq_auth_backend_http = git_rmq rabbitmq-auth-backend-http $(current_rmq_ref) $(base_rmq_ref) master 49 | dep_rabbitmq_auth_backend_ldap = git_rmq rabbitmq-auth-backend-ldap $(current_rmq_ref) $(base_rmq_ref) master 50 | dep_rabbitmq_auth_backend_oauth2 = git_rmq rabbitmq-auth-backend-oauth2 $(current_rmq_ref) $(base_rmq_ref) master 51 | dep_rabbitmq_auth_mechanism_ssl = git_rmq rabbitmq-auth-mechanism-ssl $(current_rmq_ref) $(base_rmq_ref) master 52 | dep_rabbitmq_aws = git_rmq rabbitmq-aws $(current_rmq_ref) $(base_rmq_ref) master 53 | dep_rabbitmq_boot_steps_visualiser = git_rmq rabbitmq-boot-steps-visualiser $(current_rmq_ref) $(base_rmq_ref) master 54 | dep_rabbitmq_cli = git_rmq rabbitmq-cli $(current_rmq_ref) $(base_rmq_ref) master 55 | dep_rabbitmq_codegen = git_rmq rabbitmq-codegen $(current_rmq_ref) $(base_rmq_ref) master 56 | dep_rabbitmq_consistent_hash_exchange = git_rmq rabbitmq-consistent-hash-exchange $(current_rmq_ref) $(base_rmq_ref) master 57 | dep_rabbitmq_ct_client_helpers = git_rmq rabbitmq-ct-client-helpers $(current_rmq_ref) $(base_rmq_ref) master 58 | dep_rabbitmq_ct_helpers = git_rmq rabbitmq-ct-helpers $(current_rmq_ref) $(base_rmq_ref) master 59 | dep_rabbitmq_delayed_message_exchange = git_rmq rabbitmq-delayed-message-exchange $(current_rmq_ref) $(base_rmq_ref) master 60 | dep_rabbitmq_dotnet_client = git_rmq rabbitmq-dotnet-client $(current_rmq_ref) $(base_rmq_ref) master 61 | dep_rabbitmq_event_exchange = git_rmq rabbitmq-event-exchange $(current_rmq_ref) $(base_rmq_ref) master 62 | dep_rabbitmq_federation = git_rmq rabbitmq-federation $(current_rmq_ref) $(base_rmq_ref) master 63 | dep_rabbitmq_federation_management = git_rmq rabbitmq-federation-management $(current_rmq_ref) $(base_rmq_ref) master 64 | dep_rabbitmq_java_client = git_rmq rabbitmq-java-client $(current_rmq_ref) $(base_rmq_ref) master 65 | dep_rabbitmq_jms_client = git_rmq rabbitmq-jms-client $(current_rmq_ref) $(base_rmq_ref) master 66 | dep_rabbitmq_jms_cts = git_rmq rabbitmq-jms-cts $(current_rmq_ref) $(base_rmq_ref) master 67 | dep_rabbitmq_jms_topic_exchange = git_rmq rabbitmq-jms-topic-exchange $(current_rmq_ref) $(base_rmq_ref) master 68 | dep_rabbitmq_lvc_exchange = git_rmq rabbitmq-lvc-exchange $(current_rmq_ref) $(base_rmq_ref) master 69 | dep_rabbitmq_management = git_rmq rabbitmq-management $(current_rmq_ref) $(base_rmq_ref) master 70 | dep_rabbitmq_management_agent = git_rmq rabbitmq-management-agent $(current_rmq_ref) $(base_rmq_ref) master 71 | dep_rabbitmq_management_exchange = git_rmq rabbitmq-management-exchange $(current_rmq_ref) $(base_rmq_ref) master 72 | dep_rabbitmq_management_themes = git_rmq rabbitmq-management-themes $(current_rmq_ref) $(base_rmq_ref) master 73 | dep_rabbitmq_message_timestamp = git_rmq rabbitmq-message-timestamp $(current_rmq_ref) $(base_rmq_ref) master 74 | dep_rabbitmq_metronome = git_rmq rabbitmq-metronome $(current_rmq_ref) $(base_rmq_ref) master 75 | dep_rabbitmq_mqtt = git_rmq rabbitmq-mqtt $(current_rmq_ref) $(base_rmq_ref) master 76 | dep_rabbitmq_objc_client = git_rmq rabbitmq-objc-client $(current_rmq_ref) $(base_rmq_ref) master 77 | dep_rabbitmq_peer_discovery_aws = git_rmq rabbitmq-peer-discovery-aws $(current_rmq_ref) $(base_rmq_ref) master 78 | dep_rabbitmq_peer_discovery_common = git_rmq rabbitmq-peer-discovery-common $(current_rmq_ref) $(base_rmq_ref) master 79 | dep_rabbitmq_peer_discovery_consul = git_rmq rabbitmq-peer-discovery-consul $(current_rmq_ref) $(base_rmq_ref) master 80 | dep_rabbitmq_peer_discovery_etcd = git_rmq rabbitmq-peer-discovery-etcd $(current_rmq_ref) $(base_rmq_ref) master 81 | dep_rabbitmq_peer_discovery_k8s = git_rmq rabbitmq-peer-discovery-k8s $(current_rmq_ref) $(base_rmq_ref) master 82 | dep_rabbitmq_prometheus = git_rmq rabbitmq-prometheus $(current_rmq_ref) $(base_rmq_ref) master 83 | dep_rabbitmq_random_exchange = git_rmq rabbitmq-random-exchange $(current_rmq_ref) $(base_rmq_ref) master 84 | dep_rabbitmq_recent_history_exchange = git_rmq rabbitmq-recent-history-exchange $(current_rmq_ref) $(base_rmq_ref) master 85 | dep_rabbitmq_routing_node_stamp = git_rmq rabbitmq-routing-node-stamp $(current_rmq_ref) $(base_rmq_ref) master 86 | dep_rabbitmq_rtopic_exchange = git_rmq rabbitmq-rtopic-exchange $(current_rmq_ref) $(base_rmq_ref) master 87 | dep_rabbitmq_server_release = git_rmq rabbitmq-server-release $(current_rmq_ref) $(base_rmq_ref) master 88 | dep_rabbitmq_sharding = git_rmq rabbitmq-sharding $(current_rmq_ref) $(base_rmq_ref) master 89 | dep_rabbitmq_shovel = git_rmq rabbitmq-shovel $(current_rmq_ref) $(base_rmq_ref) master 90 | dep_rabbitmq_shovel_management = git_rmq rabbitmq-shovel-management $(current_rmq_ref) $(base_rmq_ref) master 91 | dep_rabbitmq_stomp = git_rmq rabbitmq-stomp $(current_rmq_ref) $(base_rmq_ref) master 92 | dep_rabbitmq_stream = git_rmq rabbitmq-stream $(current_rmq_ref) $(base_rmq_ref) master 93 | dep_rabbitmq_toke = git_rmq rabbitmq-toke $(current_rmq_ref) $(base_rmq_ref) master 94 | dep_rabbitmq_top = git_rmq rabbitmq-top $(current_rmq_ref) $(base_rmq_ref) master 95 | dep_rabbitmq_tracing = git_rmq rabbitmq-tracing $(current_rmq_ref) $(base_rmq_ref) master 96 | dep_rabbitmq_trust_store = git_rmq rabbitmq-trust-store $(current_rmq_ref) $(base_rmq_ref) master 97 | dep_rabbitmq_test = git_rmq rabbitmq-test $(current_rmq_ref) $(base_rmq_ref) master 98 | dep_rabbitmq_web_dispatch = git_rmq rabbitmq-web-dispatch $(current_rmq_ref) $(base_rmq_ref) master 99 | dep_rabbitmq_web_stomp = git_rmq rabbitmq-web-stomp $(current_rmq_ref) $(base_rmq_ref) master 100 | dep_rabbitmq_web_stomp_examples = git_rmq rabbitmq-web-stomp-examples $(current_rmq_ref) $(base_rmq_ref) master 101 | dep_rabbitmq_web_mqtt = git_rmq rabbitmq-web-mqtt $(current_rmq_ref) $(base_rmq_ref) master 102 | dep_rabbitmq_web_mqtt_examples = git_rmq rabbitmq-web-mqtt-examples $(current_rmq_ref) $(base_rmq_ref) master 103 | dep_rabbitmq_website = git_rmq rabbitmq-website $(current_rmq_ref) $(base_rmq_ref) live master 104 | dep_toke = git_rmq toke $(current_rmq_ref) $(base_rmq_ref) master 105 | 106 | dep_rabbitmq_public_umbrella = git_rmq rabbitmq-public-umbrella $(current_rmq_ref) $(base_rmq_ref) master 107 | 108 | # Third-party dependencies version pinning. 109 | # 110 | # We do that in this file, which is copied in all projects, to ensure 111 | # all projects use the same versions. It avoids conflicts and makes it 112 | # possible to work with rabbitmq-public-umbrella. 113 | 114 | dep_accept = hex 0.3.5 115 | dep_cowboy = hex 2.8.0 116 | dep_cowlib = hex 2.9.1 117 | dep_jsx = hex 2.11.0 118 | dep_lager = hex 3.8.0 119 | dep_prometheus = git https://github.com/deadtrickster/prometheus.erl.git master 120 | dep_ra = git https://github.com/rabbitmq/ra.git master 121 | dep_ranch = hex 1.7.1 122 | dep_recon = hex 2.5.1 123 | dep_observer_cli = hex 1.5.4 124 | dep_stdout_formatter = hex 0.2.4 125 | dep_sysmon_handler = hex 1.3.0 126 | 127 | RABBITMQ_COMPONENTS = amqp_client \ 128 | amqp10_common \ 129 | amqp10_client \ 130 | rabbit \ 131 | rabbit_common \ 132 | rabbitmq_amqp1_0 \ 133 | rabbitmq_auth_backend_amqp \ 134 | rabbitmq_auth_backend_cache \ 135 | rabbitmq_auth_backend_http \ 136 | rabbitmq_auth_backend_ldap \ 137 | rabbitmq_auth_backend_oauth2 \ 138 | rabbitmq_auth_mechanism_ssl \ 139 | rabbitmq_aws \ 140 | rabbitmq_boot_steps_visualiser \ 141 | rabbitmq_cli \ 142 | rabbitmq_codegen \ 143 | rabbitmq_consistent_hash_exchange \ 144 | rabbitmq_ct_client_helpers \ 145 | rabbitmq_ct_helpers \ 146 | rabbitmq_delayed_message_exchange \ 147 | rabbitmq_dotnet_client \ 148 | rabbitmq_event_exchange \ 149 | rabbitmq_federation \ 150 | rabbitmq_federation_management \ 151 | rabbitmq_java_client \ 152 | rabbitmq_jms_client \ 153 | rabbitmq_jms_cts \ 154 | rabbitmq_jms_topic_exchange \ 155 | rabbitmq_lvc_exchange \ 156 | rabbitmq_management \ 157 | rabbitmq_management_agent \ 158 | rabbitmq_management_exchange \ 159 | rabbitmq_management_themes \ 160 | rabbitmq_message_timestamp \ 161 | rabbitmq_metronome \ 162 | rabbitmq_mqtt \ 163 | rabbitmq_objc_client \ 164 | rabbitmq_peer_discovery_aws \ 165 | rabbitmq_peer_discovery_common \ 166 | rabbitmq_peer_discovery_consul \ 167 | rabbitmq_peer_discovery_etcd \ 168 | rabbitmq_peer_discovery_k8s \ 169 | rabbitmq_prometheus \ 170 | rabbitmq_random_exchange \ 171 | rabbitmq_recent_history_exchange \ 172 | rabbitmq_routing_node_stamp \ 173 | rabbitmq_rtopic_exchange \ 174 | rabbitmq_server_release \ 175 | rabbitmq_sharding \ 176 | rabbitmq_shovel \ 177 | rabbitmq_shovel_management \ 178 | rabbitmq_stomp \ 179 | rabbitmq_stream \ 180 | rabbitmq_toke \ 181 | rabbitmq_top \ 182 | rabbitmq_tracing \ 183 | rabbitmq_trust_store \ 184 | rabbitmq_web_dispatch \ 185 | rabbitmq_web_mqtt \ 186 | rabbitmq_web_mqtt_examples \ 187 | rabbitmq_web_stomp \ 188 | rabbitmq_web_stomp_examples \ 189 | rabbitmq_website 190 | 191 | # Erlang.mk does not rebuild dependencies by default, once they were 192 | # compiled once, except for those listed in the `$(FORCE_REBUILD)` 193 | # variable. 194 | # 195 | # We want all RabbitMQ components to always be rebuilt: this eases 196 | # the work on several components at the same time. 197 | 198 | FORCE_REBUILD = $(RABBITMQ_COMPONENTS) 199 | 200 | # Several components have a custom erlang.mk/build.config, mainly 201 | # to disable eunit. Therefore, we can't use the top-level project's 202 | # erlang.mk copy. 203 | NO_AUTOPATCH += $(RABBITMQ_COMPONENTS) 204 | 205 | ifeq ($(origin current_rmq_ref),undefined) 206 | ifneq ($(wildcard .git),) 207 | current_rmq_ref := $(shell (\ 208 | ref=$$(LANG=C git branch --list | awk '/^\* \(.*detached / {ref=$$0; sub(/.*detached [^ ]+ /, "", ref); sub(/\)$$/, "", ref); print ref; exit;} /^\* / {ref=$$0; sub(/^\* /, "", ref); print ref; exit}');\ 209 | if test "$$(git rev-parse --short HEAD)" != "$$ref"; then echo "$$ref"; fi)) 210 | else 211 | current_rmq_ref := master 212 | endif 213 | endif 214 | export current_rmq_ref 215 | 216 | ifeq ($(origin base_rmq_ref),undefined) 217 | ifneq ($(wildcard .git),) 218 | possible_base_rmq_ref := master 219 | ifeq ($(possible_base_rmq_ref),$(current_rmq_ref)) 220 | base_rmq_ref := $(current_rmq_ref) 221 | else 222 | base_rmq_ref := $(shell \ 223 | (git rev-parse --verify -q master >/dev/null && \ 224 | git rev-parse --verify -q $(possible_base_rmq_ref) >/dev/null && \ 225 | git merge-base --is-ancestor $$(git merge-base master HEAD) $(possible_base_rmq_ref) && \ 226 | echo $(possible_base_rmq_ref)) || \ 227 | echo master) 228 | endif 229 | else 230 | base_rmq_ref := master 231 | endif 232 | endif 233 | export base_rmq_ref 234 | 235 | # Repository URL selection. 236 | # 237 | # First, we infer other components' location from the current project 238 | # repository URL, if it's a Git repository: 239 | # - We take the "origin" remote URL as the base 240 | # - The current project name and repository name is replaced by the 241 | # target's properties: 242 | # eg. rabbitmq-common is replaced by rabbitmq-codegen 243 | # eg. rabbit_common is replaced by rabbitmq_codegen 244 | # 245 | # If cloning from this computed location fails, we fallback to RabbitMQ 246 | # upstream which is GitHub. 247 | 248 | # Macro to transform eg. "rabbit_common" to "rabbitmq-common". 249 | rmq_cmp_repo_name = $(word 2,$(dep_$(1))) 250 | 251 | # Upstream URL for the current project. 252 | RABBITMQ_COMPONENT_REPO_NAME := $(call rmq_cmp_repo_name,$(PROJECT)) 253 | RABBITMQ_UPSTREAM_FETCH_URL ?= https://github.com/rabbitmq/$(RABBITMQ_COMPONENT_REPO_NAME).git 254 | RABBITMQ_UPSTREAM_PUSH_URL ?= git@github.com:rabbitmq/$(RABBITMQ_COMPONENT_REPO_NAME).git 255 | 256 | # Current URL for the current project. If this is not a Git clone, 257 | # default to the upstream Git repository. 258 | ifneq ($(wildcard .git),) 259 | git_origin_fetch_url := $(shell git config remote.origin.url) 260 | git_origin_push_url := $(shell git config remote.origin.pushurl || git config remote.origin.url) 261 | RABBITMQ_CURRENT_FETCH_URL ?= $(git_origin_fetch_url) 262 | RABBITMQ_CURRENT_PUSH_URL ?= $(git_origin_push_url) 263 | else 264 | RABBITMQ_CURRENT_FETCH_URL ?= $(RABBITMQ_UPSTREAM_FETCH_URL) 265 | RABBITMQ_CURRENT_PUSH_URL ?= $(RABBITMQ_UPSTREAM_PUSH_URL) 266 | endif 267 | 268 | # Macro to replace the following pattern: 269 | # 1. /foo.git -> /bar.git 270 | # 2. /foo -> /bar 271 | # 3. /foo/ -> /bar/ 272 | subst_repo_name = $(patsubst %/$(1)/%,%/$(2)/%,$(patsubst %/$(1),%/$(2),$(patsubst %/$(1).git,%/$(2).git,$(3)))) 273 | 274 | # Macro to replace both the project's name (eg. "rabbit_common") and 275 | # repository name (eg. "rabbitmq-common") by the target's equivalent. 276 | # 277 | # This macro is kept on one line because we don't want whitespaces in 278 | # the returned value, as it's used in $(dep_fetch_git_rmq) in a shell 279 | # single-quoted string. 280 | dep_rmq_repo = $(if $(dep_$(2)),$(call subst_repo_name,$(PROJECT),$(2),$(call subst_repo_name,$(RABBITMQ_COMPONENT_REPO_NAME),$(call rmq_cmp_repo_name,$(2)),$(1))),$(pkg_$(1)_repo)) 281 | 282 | dep_rmq_commits = $(if $(dep_$(1)), \ 283 | $(wordlist 3,$(words $(dep_$(1))),$(dep_$(1))), \ 284 | $(pkg_$(1)_commit)) 285 | 286 | define dep_fetch_git_rmq 287 | fetch_url1='$(call dep_rmq_repo,$(RABBITMQ_CURRENT_FETCH_URL),$(1))'; \ 288 | fetch_url2='$(call dep_rmq_repo,$(RABBITMQ_UPSTREAM_FETCH_URL),$(1))'; \ 289 | if test "$$$$fetch_url1" != '$(RABBITMQ_CURRENT_FETCH_URL)' && \ 290 | git clone -q -n -- "$$$$fetch_url1" $(DEPS_DIR)/$(call dep_name,$(1)); then \ 291 | fetch_url="$$$$fetch_url1"; \ 292 | push_url='$(call dep_rmq_repo,$(RABBITMQ_CURRENT_PUSH_URL),$(1))'; \ 293 | elif git clone -q -n -- "$$$$fetch_url2" $(DEPS_DIR)/$(call dep_name,$(1)); then \ 294 | fetch_url="$$$$fetch_url2"; \ 295 | push_url='$(call dep_rmq_repo,$(RABBITMQ_UPSTREAM_PUSH_URL),$(1))'; \ 296 | fi; \ 297 | cd $(DEPS_DIR)/$(call dep_name,$(1)) && ( \ 298 | $(foreach ref,$(call dep_rmq_commits,$(1)), \ 299 | git checkout -q $(ref) >/dev/null 2>&1 || \ 300 | ) \ 301 | (echo "error: no valid pathspec among: $(call dep_rmq_commits,$(1))" \ 302 | 1>&2 && false) ) && \ 303 | (test "$$$$fetch_url" = "$$$$push_url" || \ 304 | git remote set-url --push origin "$$$$push_url") 305 | endef 306 | 307 | # -------------------------------------------------------------------- 308 | # Component distribution. 309 | # -------------------------------------------------------------------- 310 | 311 | list-dist-deps:: 312 | @: 313 | 314 | prepare-dist:: 315 | @: 316 | 317 | # -------------------------------------------------------------------- 318 | # Umbrella-specific settings. 319 | # -------------------------------------------------------------------- 320 | 321 | # If the top-level project is a RabbitMQ component, we override 322 | # $(DEPS_DIR) for this project to point to the top-level's one. 323 | # 324 | # We also verify that the guessed DEPS_DIR is actually named `deps`, 325 | # to rule out any situation where it is a coincidence that we found a 326 | # `rabbitmq-components.mk` up upper directories. 327 | 328 | possible_deps_dir_1 = $(abspath ..) 329 | possible_deps_dir_2 = $(abspath ../../..) 330 | 331 | ifeq ($(notdir $(possible_deps_dir_1)),deps) 332 | ifneq ($(wildcard $(possible_deps_dir_1)/../rabbitmq-components.mk),) 333 | deps_dir_overriden = 1 334 | DEPS_DIR ?= $(possible_deps_dir_1) 335 | DISABLE_DISTCLEAN = 1 336 | endif 337 | endif 338 | 339 | ifeq ($(deps_dir_overriden),) 340 | ifeq ($(notdir $(possible_deps_dir_2)),deps) 341 | ifneq ($(wildcard $(possible_deps_dir_2)/../rabbitmq-components.mk),) 342 | deps_dir_overriden = 1 343 | DEPS_DIR ?= $(possible_deps_dir_2) 344 | DISABLE_DISTCLEAN = 1 345 | endif 346 | endif 347 | endif 348 | 349 | ifneq ($(wildcard UMBRELLA.md),) 350 | DISABLE_DISTCLEAN = 1 351 | endif 352 | 353 | # We disable `make distclean` so $(DEPS_DIR) is not accidentally removed. 354 | 355 | ifeq ($(DISABLE_DISTCLEAN),1) 356 | ifneq ($(filter distclean distclean-deps,$(MAKECMDGOALS)),) 357 | SKIP_DEPS = 1 358 | endif 359 | endif 360 | -------------------------------------------------------------------------------- /src/Elixir.RabbitMQ.CLI.Ctl.Commands.DeleteShovelCommand.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. 6 | %% 7 | 8 | -module('Elixir.RabbitMQ.CLI.Ctl.Commands.DeleteShovelCommand'). 9 | 10 | -include("rabbit_shovel.hrl"). 11 | 12 | -behaviour('Elixir.RabbitMQ.CLI.CommandBehaviour'). 13 | 14 | -export([ 15 | usage/0, 16 | usage_additional/0, 17 | usage_doc_guides/0, 18 | validate/2, 19 | merge_defaults/2, 20 | banner/2, 21 | run/2, 22 | switches/0, 23 | aliases/0, 24 | output/2, 25 | help_section/0, 26 | description/0 27 | ]). 28 | 29 | 30 | %%---------------------------------------------------------------------------- 31 | %% Callbacks 32 | %%---------------------------------------------------------------------------- 33 | usage() -> 34 | <<"delete_shovel [--vhost ] ">>. 35 | 36 | usage_additional() -> 37 | [ 38 | {<<"">>, <<"Shovel to delete">>} 39 | ]. 40 | 41 | usage_doc_guides() -> 42 | [?SHOVEL_GUIDE_URL]. 43 | 44 | description() -> 45 | <<"Deletes a Shovel">>. 46 | 47 | help_section() -> 48 | {plugin, shovel}. 49 | 50 | validate([], _Opts) -> 51 | {validation_failure, not_enough_args}; 52 | validate([_, _ | _], _Opts) -> 53 | {validation_failure, too_many_args}; 54 | validate([_], _Opts) -> 55 | ok. 56 | 57 | merge_defaults(A, Opts) -> 58 | {A, maps:merge(#{vhost => <<"/">>}, Opts)}. 59 | 60 | banner([Name], #{vhost := VHost}) -> 61 | erlang:list_to_binary(io_lib:format("Deleting shovel ~s in vhost ~s", 62 | [Name, VHost])). 63 | 64 | run([Name], #{node := Node, vhost := VHost}) -> 65 | ActingUser = 'Elixir.RabbitMQ.CLI.Core.Helpers':cli_acting_user(), 66 | case rabbit_misc:rpc_call(Node, rabbit_shovel_util, delete_shovel, [VHost, Name, ActingUser]) of 67 | {badrpc, _} = Error -> 68 | Error; 69 | {error, not_found} -> 70 | ErrMsg = rabbit_misc:format("Shovel with the given name was not found " 71 | "on the target node '~s' and / or virtual host '~s'", 72 | [Node, VHost]), 73 | {error, rabbit_data_coercion:to_binary(ErrMsg)}; 74 | ok -> ok 75 | end. 76 | 77 | switches() -> 78 | []. 79 | 80 | aliases() -> 81 | []. 82 | 83 | output(E, _Opts) -> 84 | 'Elixir.RabbitMQ.CLI.DefaultOutput':output(E). 85 | -------------------------------------------------------------------------------- /src/Elixir.RabbitMQ.CLI.Ctl.Commands.RestartShovelCommand.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. 6 | %% 7 | 8 | -module('Elixir.RabbitMQ.CLI.Ctl.Commands.RestartShovelCommand'). 9 | 10 | -include("rabbit_shovel.hrl"). 11 | 12 | -behaviour('Elixir.RabbitMQ.CLI.CommandBehaviour'). 13 | 14 | -export([ 15 | usage/0, 16 | usage_additional/0, 17 | usage_doc_guides/0, 18 | flags/0, 19 | validate/2, 20 | merge_defaults/2, 21 | banner/2, 22 | run/2, 23 | aliases/0, 24 | output/2, 25 | help_section/0, 26 | description/0 27 | ]). 28 | 29 | 30 | %%---------------------------------------------------------------------------- 31 | %% Callbacks 32 | %%---------------------------------------------------------------------------- 33 | 34 | flags() -> 35 | []. 36 | 37 | aliases() -> 38 | []. 39 | 40 | validate([], _Opts) -> 41 | {validation_failure, not_enough_args}; 42 | validate([_], _Opts) -> 43 | ok; 44 | validate(_, _Opts) -> 45 | {validation_failure, too_many_args}. 46 | 47 | merge_defaults(A, Opts) -> 48 | {A, maps:merge(#{vhost => <<"/">>}, Opts)}. 49 | 50 | banner([Name], #{node := Node, vhost := VHost}) -> 51 | erlang:iolist_to_binary([<<"Restarting dynamic Shovel ">>, Name, <<" in virtual host ">>, VHost, 52 | << " on node ">>, atom_to_binary(Node, utf8)]). 53 | 54 | run([Name], #{node := Node, vhost := VHost}) -> 55 | case rabbit_misc:rpc_call(Node, rabbit_shovel_util, restart_shovel, [VHost, Name]) of 56 | {badrpc, _} = Error -> 57 | Error; 58 | {error, not_found} -> 59 | ErrMsg = rabbit_misc:format("Shovel with the given name was not found " 60 | "on the target node '~s' and / or virtual host '~s'", 61 | [Node, VHost]), 62 | {error, rabbit_data_coercion:to_binary(ErrMsg)}; 63 | ok -> ok 64 | end. 65 | 66 | output(Output, _Opts) -> 67 | 'Elixir.RabbitMQ.CLI.DefaultOutput':output(Output). 68 | 69 | usage() -> 70 | <<"restart_shovel ">>. 71 | 72 | usage_additional() -> 73 | [ 74 | {<<"">>, <<"name of the Shovel to restart">>} 75 | ]. 76 | 77 | usage_doc_guides() -> 78 | [?SHOVEL_GUIDE_URL]. 79 | 80 | help_section() -> 81 | {plugin, shovel}. 82 | 83 | description() -> 84 | <<"Restarts a dynamic Shovel">>. 85 | -------------------------------------------------------------------------------- /src/Elixir.RabbitMQ.CLI.Ctl.Commands.ShovelStatusCommand.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. 6 | %% 7 | 8 | -module('Elixir.RabbitMQ.CLI.Ctl.Commands.ShovelStatusCommand'). 9 | 10 | -include("rabbit_shovel.hrl"). 11 | 12 | -behaviour('Elixir.RabbitMQ.CLI.CommandBehaviour'). 13 | 14 | -export([ 15 | usage/0, 16 | usage_doc_guides/0, 17 | flags/0, 18 | validate/2, 19 | merge_defaults/2, 20 | banner/2, 21 | run/2, 22 | switches/0, 23 | aliases/0, 24 | output/2, 25 | scopes/0, 26 | formatter/0, 27 | help_section/0, 28 | description/0 29 | ]). 30 | 31 | 32 | %%---------------------------------------------------------------------------- 33 | %% Callbacks 34 | %%---------------------------------------------------------------------------- 35 | usage() -> 36 | <<"shovel_status">>. 37 | 38 | usage_doc_guides() -> 39 | [?SHOVEL_GUIDE_URL]. 40 | 41 | description() -> 42 | <<"Displays status of Shovel on a node">>. 43 | 44 | help_section() -> 45 | {plugin, shovel}. 46 | 47 | flags() -> 48 | []. 49 | 50 | formatter() -> 51 | 'Elixir.RabbitMQ.CLI.Formatters.Table'. 52 | 53 | validate(_,_) -> 54 | ok. 55 | 56 | merge_defaults(A,O) -> 57 | {A, O}. 58 | 59 | banner(_, #{node := Node}) -> 60 | erlang:iolist_to_binary([<<"Shovel status on node ">>, 61 | atom_to_binary(Node, utf8)]). 62 | 63 | run(_Args, #{node := Node}) -> 64 | case rabbit_misc:rpc_call(Node, rabbit_shovel_status, status, []) of 65 | {badrpc, _} = Error -> 66 | Error; 67 | Status -> 68 | {stream, Status} 69 | end. 70 | 71 | switches() -> 72 | []. 73 | 74 | aliases() -> 75 | []. 76 | 77 | output({stream, ShovelStatus}, _Opts) -> 78 | Formatted = [fmt_name(Name, 79 | fmt_status(Status, 80 | #{type => Type, 81 | last_changed => fmt_ts(Timestamp)})) 82 | || {Name, Type, Status, Timestamp} <- ShovelStatus], 83 | {stream, Formatted}; 84 | output(E, _Opts) -> 85 | 'Elixir.RabbitMQ.CLI.DefaultOutput':output(E). 86 | 87 | scopes() -> 88 | ['ctl', 'diagnostics']. 89 | 90 | %%---------------------------------------------------------------------------- 91 | %% Formatting 92 | %%---------------------------------------------------------------------------- 93 | fmt_name({Vhost, Name}, Map) -> 94 | Map#{name => Name, vhost => Vhost}; 95 | fmt_name(Name, Map) -> 96 | %% Static shovel names don't contain the vhost 97 | Map#{name => Name}. 98 | 99 | fmt_ts({{YY, MM, DD}, {Hour, Min, Sec}}) -> 100 | erlang:list_to_binary( 101 | io_lib:format("~4..0w-~2..0w-~2..0w ~2..0w:~2..0w:~2..0w", 102 | [YY, MM, DD, Hour, Min, Sec])). 103 | 104 | fmt_status({'running' = St, Proplist}, Map) -> 105 | maps:merge(Map#{state => St, 106 | source_protocol => proplists:get_value(src_protocol, Proplist, 107 | undefined), 108 | source => proplists:get_value(src_uri, Proplist), 109 | destination_protocol => proplists:get_value(dest_protocol, Proplist, undefined), 110 | destination => proplists:get_value(dest_uri, Proplist), 111 | termination_reason => <<>>}, details_to_map(Proplist)); 112 | fmt_status('starting' = St, Map) -> 113 | Map#{state => St, 114 | source => <<>>, 115 | destination => <<>>, 116 | termination_reason => <<>>}; 117 | fmt_status({'terminated' = St, Reason}, Map) -> 118 | Map#{state => St, 119 | termination_reason => list_to_binary(io_lib:format("~p", [Reason])), 120 | source => <<>>, 121 | destination => <<>>}. 122 | 123 | details_to_map(Proplist) -> 124 | Keys = [{src_address, source_address}, {src_queue, source_queue}, 125 | {src_exchange, source_exchange}, {src_exchange_key, source_exchange_key}, 126 | {dest_address, destination_address}, {dest_queue, destination_queue}, 127 | {dest_exchange, destination_exchange}, {dest_exchange_key, destination_exchange_key}], 128 | maps:from_list([{New, proplists:get_value(Old, Proplist)} 129 | || {Old, New} <- Keys, proplists:is_defined(Old, Proplist)]). 130 | -------------------------------------------------------------------------------- /src/rabbit_amqp10_shovel.erl: -------------------------------------------------------------------------------- 1 | %% This Source Code Form is subject to the terms of the Mozilla Public 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/. 4 | %% 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates. All rights reserved. 6 | %% 7 | 8 | -module(rabbit_amqp10_shovel). 9 | 10 | -behaviour(rabbit_shovel_behaviour). 11 | 12 | -include_lib("amqp_client/include/amqp_client.hrl"). 13 | -include("rabbit_shovel.hrl"). 14 | 15 | -export([ 16 | parse/2, 17 | source_uri/1, 18 | dest_uri/1, 19 | source_protocol/1, 20 | dest_protocol/1, 21 | source_endpoint/1, 22 | dest_endpoint/1, 23 | connect_source/1, 24 | init_source/1, 25 | connect_dest/1, 26 | init_dest/1, 27 | handle_source/2, 28 | handle_dest/2, 29 | close_source/1, 30 | close_dest/1, 31 | ack/3, 32 | nack/3, 33 | forward/4 34 | ]). 35 | 36 | -import(rabbit_misc, [pget/2, pget/3]). 37 | -import(rabbit_data_coercion, [to_binary/1]). 38 | 39 | -define(INFO(Text, Args), error_logger:info_msg(Text, Args)). 40 | -define(LINK_CREDIT_TIMEOUT, 5000). 41 | 42 | -type state() :: rabbit_shovel_behaviour:state(). 43 | -type uri() :: rabbit_shovel_behaviour:uri(). 44 | -type tag() :: rabbit_shovel_behaviour:tag(). 45 | -type endpoint_config() :: rabbit_shovel_behaviour:source_config() 46 | | rabbit_shovel_behaviour:dest_config(). 47 | 48 | -spec parse(binary(), {source | destination, proplists:proplist()}) -> 49 | endpoint_config(). 50 | parse(_Name, {destination, Conf}) -> 51 | Uris = pget(uris, Conf), 52 | #{module => ?MODULE, 53 | uris => Uris, 54 | unacked => #{}, 55 | target_address => pget(target_address, Conf), 56 | properties => maps:from_list(pget(properties, Conf, [])), 57 | application_properties => maps:from_list(pget(application_properties, Conf, [])), 58 | delivery_annotations => maps:from_list(pget(delivery_annotations, Conf, [])), 59 | message_annotations => maps:from_list(pget(message_annotations, Conf, [])), 60 | add_forward_headers => pget(add_forward_headers, Conf, false), 61 | add_timestamp_header => pget(add_timestamp_header, Conf, false) 62 | }; 63 | parse(_Name, {source, Conf}) -> 64 | Uris = pget(uris, Conf), 65 | #{module => ?MODULE, 66 | uris => Uris, 67 | prefetch_count => pget(prefetch_count, Conf, 1000), 68 | delete_after => pget(delete_after, Conf, never), 69 | source_address => pget(source_address, Conf)}. 70 | 71 | -spec connect_source(state()) -> state(). 72 | connect_source(State = #{name := Name, 73 | ack_mode := AckMode, 74 | source := #{uris := [Uri | _], 75 | source_address := Addr} = Src}) -> 76 | AttachFun = fun amqp10_client:attach_receiver_link/5, 77 | {Conn, Sess, LinkRef} = connect(Name, AckMode, Uri, "receiver", Addr, Src, 78 | AttachFun), 79 | State#{source => Src#{current => #{conn => Conn, 80 | session => Sess, 81 | link => LinkRef, 82 | uri => Uri}}}. 83 | 84 | -spec connect_dest(state()) -> state(). 85 | connect_dest(State = #{name := Name, 86 | ack_mode := AckMode, 87 | dest := #{uris := [Uri | _], 88 | target_address := Addr} = Dst}) -> 89 | AttachFun = fun amqp10_client:attach_sender_link_sync/5, 90 | {Conn, Sess, LinkRef} = connect(Name, AckMode, Uri, "sender", Addr, Dst, 91 | AttachFun), 92 | %% wait for link credit here as if there are messages waiting we may try 93 | %% to forward before we've received credit 94 | State#{dest => Dst#{current => #{conn => Conn, 95 | session => Sess, 96 | link_state => attached, 97 | pending => [], 98 | link => LinkRef, 99 | uri => Uri}}}. 100 | 101 | connect(Name, AckMode, Uri, Postfix, Addr, Map, AttachFun) -> 102 | {ok, Config} = amqp10_client:parse_uri(Uri), 103 | {ok, Conn} = amqp10_client:open_connection(Config), 104 | {ok, Sess} = amqp10_client:begin_session(Conn), 105 | link(Conn), 106 | LinkName = begin 107 | LinkName0 = gen_unique_name(Name, Postfix), 108 | rabbit_data_coercion:to_binary(LinkName0) 109 | end, 110 | % mixed settlement mode covers all the ack_modes 111 | SettlementMode = case AckMode of 112 | no_ack -> settled; 113 | _ -> unsettled 114 | end, 115 | % needs to be sync, i.e. awaits the 'attach' event as 116 | % else we may try to use the link before it is ready 117 | Durability = maps:get(durability, Map, unsettled_state), 118 | {ok, LinkRef} = AttachFun(Sess, LinkName, Addr, 119 | SettlementMode, 120 | Durability), 121 | {Conn, Sess, LinkRef}. 122 | 123 | -spec init_source(state()) -> state(). 124 | init_source(State = #{source := #{current := #{link := Link}, 125 | prefetch_count := Prefetch} = Src}) -> 126 | {Credit, RenewAfter} = case Src of 127 | #{delete_after := R} when is_integer(R) -> 128 | {R, never}; 129 | #{prefetch_count := Pre} -> 130 | {Pre, round(Prefetch/10)} 131 | end, 132 | ok = amqp10_client:flow_link_credit(Link, Credit, RenewAfter), 133 | Remaining = case Src of 134 | #{delete_after := never} -> unlimited; 135 | #{delete_after := Rem} -> Rem; 136 | _ -> unlimited 137 | end, 138 | State#{source => Src#{remaining => Remaining, 139 | remaining_unacked => Remaining, 140 | last_acked_tag => -1}}. 141 | 142 | -spec init_dest(state()) -> state(). 143 | init_dest(#{name := Name, 144 | shovel_type := Type, 145 | dest := #{add_forward_headers := true} = Dst} = State) -> 146 | Props = #{<<"shovelled-by">> => rabbit_nodes:cluster_name(), 147 | <<"shovel-type">> => rabbit_data_coercion:to_binary(Type), 148 | <<"shovel-name">> => rabbit_data_coercion:to_binary(Name)}, 149 | State#{dest => Dst#{cached_forward_headers => Props}}; 150 | init_dest(State) -> 151 | State. 152 | 153 | -spec source_uri(state()) -> uri(). 154 | source_uri(#{source := #{current := #{uri := Uri}}}) -> Uri. 155 | 156 | -spec dest_uri(state()) -> uri(). 157 | dest_uri(#{dest := #{current := #{uri := Uri}}}) -> Uri. 158 | 159 | source_protocol(_State) -> amqp10. 160 | dest_protocol(_State) -> amqp10. 161 | 162 | source_endpoint(#{shovel_type := static}) -> 163 | []; 164 | source_endpoint(#{shovel_type := dynamic, 165 | source := #{source_address := Addr}}) -> 166 | [{src_address, Addr}]. 167 | 168 | dest_endpoint(#{shovel_type := static}) -> 169 | []; 170 | dest_endpoint(#{shovel_type := dynamic, 171 | dest := #{target_address := Addr}}) -> 172 | [{dest_address, Addr}]. 173 | 174 | -spec handle_source(Msg :: any(), state()) -> not_handled | state(). 175 | handle_source({amqp10_msg, _LinkRef, Msg}, State) -> 176 | Tag = amqp10_msg:delivery_id(Msg), 177 | Payload = amqp10_msg:body_bin(Msg), 178 | rabbit_shovel_behaviour:forward(Tag, #{}, Payload, State); 179 | handle_source({amqp10_event, {connection, Conn, opened}}, 180 | State = #{source := #{current := #{conn := Conn}}}) -> 181 | State; 182 | handle_source({amqp10_event, {connection, Conn, {closed, Why}}}, 183 | #{source := #{current := #{conn := Conn}}, 184 | name := Name}) -> 185 | ?INFO("Shovel ~s source connection closed. Reason: ~p~n", [Name, Why]), 186 | {stop, {inbound_conn_closed, Why}}; 187 | handle_source({amqp10_event, {session, Sess, begun}}, 188 | State = #{source := #{current := #{session := Sess}}}) -> 189 | State; 190 | handle_source({amqp10_event, {session, Sess, {ended, Why}}}, 191 | #{source := #{current := #{session := Sess}}}) -> 192 | {stop, {inbound_session_ended, Why}}; 193 | handle_source({amqp10_event, {link, Link, {detached, Why}}}, 194 | #{source := #{current := #{link := Link}}}) -> 195 | {stop, {inbound_link_detached, Why}}; 196 | handle_source({amqp10_event, {link, Link, _Evt}}, 197 | State= #{source := #{current := #{link := Link}}}) -> 198 | State; 199 | handle_source({'EXIT', Conn, Reason}, 200 | #{source := #{current := #{conn := Conn}}}) -> 201 | {stop, {outbound_conn_died, Reason}}; 202 | handle_source(_Msg, _State) -> 203 | not_handled. 204 | 205 | -spec handle_dest(Msg :: any(), state()) -> not_handled | state(). 206 | handle_dest({amqp10_disposition, {Result, Tag}}, 207 | State0 = #{ack_mode := on_confirm, 208 | dest := #{unacked := Unacked} = Dst, 209 | name := Name}) -> 210 | State1 = State0#{dest => Dst#{unacked => maps:remove(Tag, Unacked)}}, 211 | {Decr, State} = 212 | case {Unacked, Result} of 213 | {#{Tag := IncomingTag}, accepted} -> 214 | {1, rabbit_shovel_behaviour:ack(IncomingTag, false, State1)}; 215 | {#{Tag := IncomingTag}, rejected} -> 216 | {1, rabbit_shovel_behaviour:nack(IncomingTag, false, State1)}; 217 | _ -> % not found - this should ideally not happen 218 | error_logger:warning_msg("Shovel ~s amqp10 destination " 219 | "disposition tag not found: ~p~n", 220 | [Name, Tag]), 221 | {0, State1} 222 | end, 223 | rabbit_shovel_behaviour:decr_remaining(Decr, State); 224 | handle_dest({amqp10_event, {connection, Conn, opened}}, 225 | State = #{dest := #{current := #{conn := Conn}}}) -> 226 | State; 227 | handle_dest({amqp10_event, {connection, Conn, {closed, Why}}}, 228 | #{name := Name, 229 | dest := #{current := #{conn := Conn}}}) -> 230 | ?INFO("Shovel ~s destination connection closed. Reason: ~p~n", [Name, Why]), 231 | {stop, {outbound_conn_died, Why}}; 232 | handle_dest({amqp10_event, {session, Sess, begun}}, 233 | State = #{dest := #{current := #{session := Sess}}}) -> 234 | State; 235 | handle_dest({amqp10_event, {session, Sess, {ended, Why}}}, 236 | #{dest := #{current := #{session := Sess}}}) -> 237 | {stop, {outbound_conn_died, Why}}; 238 | handle_dest({amqp10_event, {link, Link, {detached, Why}}}, 239 | #{dest := #{current := #{link := Link}}}) -> 240 | {stop, {outbound_link_detached, Why}}; 241 | handle_dest({amqp10_event, {link, Link, credited}}, 242 | State0 = #{dest := #{current := #{link := Link}, 243 | pending := Pend} = Dst}) -> 244 | 245 | %% we have credit so can begin to forward 246 | State = State0#{dest => Dst#{link_state => credited, 247 | pending => []}}, 248 | lists:foldl(fun ({A, B, C}, S) -> 249 | forward(A, B, C, S) 250 | end, State, lists:reverse(Pend)); 251 | handle_dest({amqp10_event, {link, Link, _Evt}}, 252 | State= #{dest := #{current := #{link := Link}}}) -> 253 | State; 254 | handle_dest({'EXIT', Conn, Reason}, 255 | #{dest := #{current := #{conn := Conn}}}) -> 256 | {stop, {outbound_conn_died, Reason}}; 257 | handle_dest(_Msg, _State) -> 258 | not_handled. 259 | 260 | close_source(#{source := #{current := #{conn := Conn}}}) -> 261 | _ = amqp10_client:close_connection(Conn), 262 | ok; 263 | close_source(_Config) -> ok. 264 | 265 | close_dest(#{dest := #{current := #{conn := Conn}}}) -> 266 | _ = amqp10_client:close_connection(Conn), 267 | ok; 268 | close_dest(_Config) -> ok. 269 | 270 | -spec ack(Tag :: tag(), Multi :: boolean(), state()) -> state(). 271 | ack(Tag, true, State = #{source := #{current := #{session := Session}, 272 | last_acked_tag := LastTag} = Src}) -> 273 | First = LastTag + 1, 274 | ok = amqp10_client_session:disposition(Session, receiver, First, 275 | Tag, true, accepted), 276 | State#{source => Src#{last_acked_tag => Tag}}; 277 | ack(Tag, false, State = #{source := #{current := 278 | #{session := Session}} = Src}) -> 279 | ok = amqp10_client_session:disposition(Session, receiver, Tag, 280 | Tag, true, accepted), 281 | State#{source => Src#{last_acked_tag => Tag}}. 282 | 283 | -spec nack(Tag :: tag(), Multi :: boolean(), state()) -> state(). 284 | nack(Tag, false, State = #{source := 285 | #{current := #{session := Session}} = Src}) -> 286 | % the tag is the same as the deliveryid 287 | ok = amqp10_client_session:disposition(Session, receiver, Tag, 288 | Tag, false, rejected), 289 | State#{source => Src#{last_nacked_tag => Tag}}; 290 | nack(Tag, true, State = #{source := #{current := #{session := Session}, 291 | last_nacked_tag := LastTag} = Src}) -> 292 | First = LastTag + 1, 293 | ok = amqp10_client_session:disposition(Session, receiver, First, 294 | Tag, true, accepted), 295 | State#{source => Src#{last_nacked_tag => Tag}}. 296 | 297 | -spec forward(Tag :: tag(), Props :: #{atom() => any()}, 298 | Payload :: binary(), state()) -> state(). 299 | forward(_Tag, _Props, _Payload, 300 | #{source := #{remaining_unacked := 0}} = State) -> 301 | State; 302 | forward(Tag, Props, Payload, 303 | #{dest := #{current := #{link_state := attached}, 304 | pending := Pend0} = Dst} = State) -> 305 | %% simply cache the forward oo 306 | Pend = [{Tag, Props, Payload} | Pend0], 307 | State#{dest => Dst#{pending => {Pend}}}; 308 | forward(Tag, Props, Payload, 309 | #{dest := #{current := #{link := Link}, 310 | unacked := Unacked} = Dst, 311 | ack_mode := AckMode} = State) -> 312 | OutTag = rabbit_data_coercion:to_binary(Tag), 313 | Msg0 = new_message(OutTag, Payload, State), 314 | Msg = add_timestamp_header( 315 | State, set_message_properties( 316 | Props, add_forward_headers(State, Msg0))), 317 | ok = amqp10_client:send_msg(Link, Msg), 318 | rabbit_shovel_behaviour:decr_remaining_unacked( 319 | case AckMode of 320 | no_ack -> 321 | rabbit_shovel_behaviour:decr_remaining(1, State); 322 | on_confirm -> 323 | State#{dest => Dst#{unacked => Unacked#{OutTag => Tag}}}; 324 | on_publish -> 325 | State1 = rabbit_shovel_behaviour:ack(Tag, false, State), 326 | rabbit_shovel_behaviour:decr_remaining(1, State1) 327 | end). 328 | 329 | new_message(Tag, Payload, #{ack_mode := AckMode, 330 | dest := #{properties := Props, 331 | application_properties := AppProps, 332 | message_annotations := MsgAnns}}) -> 333 | Msg0 = amqp10_msg:new(Tag, Payload, AckMode =/= on_confirm), 334 | Msg1 = amqp10_msg:set_properties(Props, Msg0), 335 | Msg = amqp10_msg:set_message_annotations(MsgAnns, Msg1), 336 | amqp10_msg:set_application_properties(AppProps, Msg). 337 | 338 | add_timestamp_header(#{dest := #{add_timestamp_header := true}}, Msg) -> 339 | P =#{creation_time => os:system_time(milli_seconds)}, 340 | amqp10_msg:set_properties(P, Msg); 341 | add_timestamp_header(_, Msg) -> Msg. 342 | 343 | add_forward_headers(#{dest := #{cached_forward_headers := Props}}, Msg) -> 344 | amqp10_msg:set_application_properties(Props, Msg); 345 | add_forward_headers(_, Msg) -> Msg. 346 | 347 | set_message_properties(Props, Msg) -> 348 | %% this is effectively special handling properties from amqp 0.9.1 349 | maps:fold( 350 | fun(content_type, Ct, M) -> 351 | amqp10_msg:set_properties( 352 | #{content_type => to_binary(Ct)}, M); 353 | (content_encoding, Ct, M) -> 354 | amqp10_msg:set_properties( 355 | #{content_encoding => to_binary(Ct)}, M); 356 | (delivery_mode, 2, M) -> 357 | amqp10_msg:set_headers(#{durable => true}, M); 358 | (correlation_id, Ct, M) -> 359 | amqp10_msg:set_properties(#{correlation_id => to_binary(Ct)}, M); 360 | (reply_to, Ct, M) -> 361 | amqp10_msg:set_properties(#{reply_to => to_binary(Ct)}, M); 362 | (message_id, Ct, M) -> 363 | amqp10_msg:set_properties(#{message_id => to_binary(Ct)}, M); 364 | (timestamp, Ct, M) -> 365 | amqp10_msg:set_properties(#{creation_time => Ct}, M); 366 | (user_id, Ct, M) -> 367 | amqp10_msg:set_properties(#{user_id => Ct}, M); 368 | (headers, Headers0, M) when is_list(Headers0) -> 369 | %% AMPQ 0.9.1 are added as applicatin properties 370 | %% TODO: filter headers to make safe 371 | Headers = lists:foldl( 372 | fun ({K, _T, V}, Acc) -> 373 | case is_amqp10_compat(V) of 374 | true -> 375 | Acc#{to_binary(K) => V}; 376 | false -> 377 | Acc 378 | end 379 | end, #{}, Headers0), 380 | amqp10_msg:set_application_properties(Headers, M); 381 | (Key, Value, M) -> 382 | case is_amqp10_compat(Value) of 383 | true -> 384 | amqp10_msg:set_application_properties( 385 | #{to_binary(Key) => Value}, M); 386 | false -> 387 | M 388 | end 389 | end, Msg, Props). 390 | 391 | gen_unique_name(Pre0, Post0) -> 392 | Pre = to_binary(Pre0), 393 | Post = to_binary(Post0), 394 | Id = bin_to_hex(crypto:strong_rand_bytes(8)), 395 | <
>/binary, Id/binary, <<"_">>/binary, Post/binary>>.
396 | 
397 | bin_to_hex(Bin) ->
398 |     <<<= 10 -> N -10 + $a;
399 |            true  -> N + $0 end>>
400 |       || <> <= Bin>>.
401 | 
402 | is_amqp10_compat(T) ->
403 |     is_binary(T) orelse
404 |     is_number(T) orelse
405 |     %% TODO: not all lists are compatible
406 |     is_list(T) orelse
407 |     is_boolean(T).
408 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel.erl:
--------------------------------------------------------------------------------
 1 | %% This Source Code Form is subject to the terms of the Mozilla Public
 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
 4 | %%
 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
 6 | %%
 7 | 
 8 | -module(rabbit_shovel).
 9 | 
10 | -export([start/0, stop/0, start/2, stop/1]).
11 | 
12 | start() ->
13 |     _ = rabbit_shovel_sup:start_link(),
14 |     ok.
15 | 
16 | stop() -> ok.
17 | 
18 | start(normal, []) ->
19 |     rabbit_shovel_sup:start_link().
20 | 
21 | stop(_State) -> ok.
22 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_behaviour.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(rabbit_shovel_behaviour).
  9 | 
 10 | -export([
 11 |          % dynamic calls
 12 |          parse/3,
 13 |          connect_dest/1,
 14 |          connect_source/1,
 15 |          init_dest/1,
 16 |          init_source/1,
 17 |          close_dest/1,
 18 |          close_source/1,
 19 |          handle_dest/2,
 20 |          handle_source/2,
 21 |          source_uri/1,
 22 |          dest_uri/1,
 23 |          source_protocol/1,
 24 |          dest_protocol/1,
 25 |          source_endpoint/1,
 26 |          dest_endpoint/1,
 27 |          forward/4,
 28 |          ack/3,
 29 |          nack/3,
 30 |          % common functions
 31 |          decr_remaining_unacked/1,
 32 |          decr_remaining/2
 33 |         ]).
 34 | 
 35 | -type tag() :: non_neg_integer().
 36 | -type uri() :: string() | binary().
 37 | -type ack_mode() :: 'no_ack' | 'on_confirm' | 'on_publish'.
 38 | -type source_config() :: #{module => atom(),
 39 |                            uris => [uri()],
 40 |                            atom() => term()
 41 |                           }.
 42 | -type dest_config() :: #{module => atom(),
 43 |                          uris => [uri()],
 44 |                          atom() => term()
 45 |                         }.
 46 | -type state() :: #{source => source_config(),
 47 |                    dest => dest_config(),
 48 |                    ack_mode => ack_mode(),
 49 |                    atom() => term()}.
 50 | 
 51 | -export_type([state/0, source_config/0, dest_config/0, uri/0]).
 52 | 
 53 | -callback parse(binary(), {source | destination, Conf :: proplists:proplist()}) ->
 54 |     source_config() | dest_config().
 55 | 
 56 | -callback connect_source(state()) -> state().
 57 | -callback connect_dest(state()) -> state().
 58 | 
 59 | -callback init_source(state()) -> state().
 60 | -callback init_dest(state()) -> state().
 61 | 
 62 | -callback source_uri(state()) -> uri().
 63 | -callback dest_uri(state()) -> uri().
 64 | 
 65 | -callback source_protocol(state()) -> atom().
 66 | -callback dest_protocol(state()) -> atom().
 67 | 
 68 | -callback source_endpoint(state()) -> proplists:proplist().
 69 | -callback dest_endpoint(state()) -> proplists:proplist().
 70 | 
 71 | -callback close_dest(state()) -> ok.
 72 | -callback close_source(state()) -> ok.
 73 | 
 74 | -callback handle_source(Msg :: any(), state()) ->
 75 |     not_handled | state() | {stop, any()}.
 76 | -callback handle_dest(Msg :: any(), state()) ->
 77 |     not_handled | state() | {stop, any()}.
 78 | 
 79 | -callback ack(Tag :: tag(), Multi :: boolean(), state()) -> state().
 80 | -callback nack(Tag :: tag(), Multi :: boolean(), state()) -> state().
 81 | -callback forward(Tag :: tag(), Props :: #{atom() => any()},
 82 |                   Payload :: binary(), state()) -> state().
 83 | 
 84 | 
 85 | -spec parse(atom(), binary(), {source | destination, proplists:proplist()}) ->
 86 |     source_config() | dest_config().
 87 | parse(Mod, Name, Conf) ->
 88 |     Mod:parse(Name, Conf).
 89 | 
 90 | -spec connect_source(state()) -> state().
 91 | connect_source(State = #{source := #{module := Mod}}) ->
 92 |     Mod:connect_source(State).
 93 | 
 94 | -spec connect_dest(state()) -> state().
 95 | connect_dest(State = #{dest := #{module := Mod}}) ->
 96 |     Mod:connect_dest(State).
 97 | 
 98 | -spec init_source(state()) -> state().
 99 | init_source(State = #{source := #{module := Mod}}) ->
100 |     Mod:init_source(State).
101 | 
102 | -spec init_dest(state()) -> state().
103 | init_dest(State = #{dest := #{module := Mod}}) ->
104 |     Mod:init_dest(State).
105 | 
106 | -spec close_source(state()) -> ok.
107 | close_source(State = #{source := #{module := Mod}}) ->
108 |     Mod:close_source(State).
109 | 
110 | -spec close_dest(state()) -> ok.
111 | close_dest(State = #{dest := #{module := Mod}}) ->
112 |     Mod:close_dest(State).
113 | 
114 | -spec handle_source(any(), state()) ->
115 |     not_handled | state() | {stop, any()}.
116 | handle_source(Msg, State = #{source := #{module := Mod}}) ->
117 |     Mod:handle_source(Msg, State).
118 | 
119 | -spec handle_dest(any(), state()) ->
120 |     not_handled | state() | {stop, any()}.
121 | handle_dest(Msg, State = #{dest := #{module := Mod}}) ->
122 |     Mod:handle_dest(Msg, State).
123 | 
124 | source_uri(#{source := #{module := Mod}} = State) ->
125 |     Mod:source_uri(State).
126 | 
127 | dest_uri(#{dest := #{module := Mod}} = State) ->
128 |     Mod:dest_uri(State).
129 | 
130 | source_protocol(#{source := #{module := Mod}} = State) ->
131 |     Mod:source_protocol(State).
132 | 
133 | dest_protocol(#{dest := #{module := Mod}} = State) ->
134 |     Mod:dest_protocol(State).
135 | 
136 | source_endpoint(#{source := #{module := Mod}} = State) ->
137 |     Mod:source_endpoint(State).
138 | 
139 | dest_endpoint(#{dest := #{module := Mod}} = State) ->
140 |     Mod:dest_endpoint(State).
141 | 
142 | -spec forward(tag(), #{atom() => any()}, binary(), state()) -> state().
143 | forward(Tag, Props, Payload, #{dest := #{module := Mod}} = State) ->
144 |     Mod:forward(Tag, Props, Payload, State).
145 | 
146 | -spec ack(tag(), boolean(), state()) -> state().
147 | ack(Tag, Multi, #{source := #{module := Mod}} = State) ->
148 |     Mod:ack(Tag, Multi, State).
149 | 
150 | -spec nack(tag(), boolean(), state()) -> state().
151 | nack(Tag, Multi, #{source := #{module := Mod}} = State) ->
152 |     Mod:nack(Tag, Multi, State).
153 | 
154 | %% Common functions
155 | decr_remaining_unacked(State = #{source := #{remaining_unacked := unlimited}}) ->
156 |     State;
157 | decr_remaining_unacked(State = #{source := #{remaining_unacked := 0}}) ->
158 |     State;
159 | decr_remaining_unacked(State = #{source := #{remaining_unacked := N} = Src}) ->
160 |     State#{source => Src#{remaining_unacked =>  N - 1}}.
161 | 
162 | decr_remaining(_N, State = #{source := #{remaining := unlimited}}) ->
163 |     State;
164 | decr_remaining(N, State = #{source := #{remaining := M} = Src,
165 |                             name := Name}) ->
166 |     case M > N of
167 |         true  -> State#{source => Src#{remaining => M - N}};
168 |         false ->
169 |             error_logger:info_msg("shutting down shovel ~s, none remaining ~p~n",
170 |                                   [Name, State]),
171 |             exit({shutdown, autodelete})
172 |     end.
173 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_config.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(rabbit_shovel_config).
  9 | 
 10 | -export([parse/2,
 11 |          ensure_defaults/2]).
 12 | 
 13 | -include_lib("amqp_client/include/amqp_client.hrl").
 14 | -include("rabbit_shovel.hrl").
 15 | 
 16 | resolve_module(amqp091) -> rabbit_amqp091_shovel;
 17 | resolve_module(amqp10) -> rabbit_amqp10_shovel.
 18 | 
 19 | is_legacy(Config) ->
 20 |     not proplists:is_defined(source, Config).
 21 | 
 22 | get_brokers(Props) ->
 23 |     case proplists:get_value(brokers, Props) of
 24 |         undefined ->
 25 |             [get_value(broker, Props)];
 26 |         Brokers ->
 27 |             Brokers
 28 |     end.
 29 | 
 30 | convert_from_legacy(Config) ->
 31 |     S = get_value(sources, Config),
 32 |     validate(S),
 33 |     SUris = get_brokers(S),
 34 |     validate_uris(brokers, SUris),
 35 |     D = get_value(destinations, Config),
 36 |     validate(D),
 37 |     DUris = get_brokers(D),
 38 |     validate_uris(brokers, DUris),
 39 |     Q = get_value(queue, Config),
 40 |     DA = proplists:get_value(delete_after, Config, never),
 41 |     Pref = proplists:get_value(prefetch_count, Config, ?DEFAULT_PREFETCH),
 42 |     RD = proplists:get_value(reconnect_delay, Config, ?DEFAULT_RECONNECT_DELAY),
 43 |     AckMode = proplists:get_value(ack_mode, Config, ?DEFAULT_ACK_MODE),
 44 |     validate_ack_mode(AckMode),
 45 |     PubFields = proplists:get_value(publish_fields, Config, []),
 46 |     PubProps = proplists:get_value(publish_properties, Config, []),
 47 |     AFH = proplists:get_value(add_forward_headers, Config, false),
 48 |     ATH = proplists:get_value(add_timestamp_header, Config, false),
 49 |     SourceDecls = proplists:get_value(declarations, S, []),
 50 |     validate_list(SourceDecls),
 51 |     DestDecls = proplists:get_value(declarations, D, []),
 52 |     validate_list(DestDecls),
 53 |     [{source, [{protocol, amqp091},
 54 |                {uris, SUris},
 55 |                {declarations, SourceDecls},
 56 |                {queue, Q},
 57 |                {delete_after, DA},
 58 |                {prefetch_count, Pref}]},
 59 |      {destination, [{protocol, amqp091},
 60 |                     {uris, DUris},
 61 |                     {declarations, DestDecls},
 62 |                     {publish_properties, PubProps},
 63 |                     {publish_fields, PubFields},
 64 |                     {add_forward_headers, AFH},
 65 |                     {add_timestamp_header, ATH}]},
 66 |      {ack_mode, AckMode},
 67 |      {reconnect_delay, RD}].
 68 | 
 69 | parse(ShovelName, Config0) ->
 70 |     try
 71 |         validate(Config0),
 72 |         case is_legacy(Config0) of
 73 |             true ->
 74 |                 Config = convert_from_legacy(Config0),
 75 |                 parse_current(ShovelName, Config);
 76 |             false ->
 77 |                 parse_current(ShovelName, Config0)
 78 |         end
 79 |     catch throw:{error, Reason} ->
 80 |               {error, {invalid_shovel_configuration, ShovelName, Reason}};
 81 |           throw:Reason ->
 82 |               {error, {invalid_shovel_configuration, ShovelName, Reason}}
 83 |     end.
 84 | 
 85 | validate(Props) ->
 86 |     validate_proplist(Props),
 87 |     validate_duplicates(Props).
 88 | 
 89 | validate_proplist(Props) when is_list (Props) ->
 90 |     case lists:filter(fun ({_, _}) -> false;
 91 |                           (_) -> true
 92 |                       end, Props) of
 93 |         [] -> ok;
 94 |         Invalid ->
 95 |             throw({invalid_parameters, Invalid})
 96 |     end;
 97 | validate_proplist(X) ->
 98 |     throw({require_list, X}).
 99 | 
100 | validate_duplicates(Props) ->
101 |     case duplicate_keys(Props) of
102 |         [] -> ok;
103 |         Invalid ->
104 |             throw({duplicate_parameters, Invalid})
105 |     end.
106 | 
107 | validate_list(L) when is_list(L) -> ok;
108 | validate_list(L) ->
109 |     throw({require_list, L}).
110 | 
111 | validate_uris(Key, L) when not is_list(L) ->
112 |     throw({require_list, Key, L});
113 | validate_uris(Key, []) ->
114 |     throw({expected_non_empty_list, Key});
115 | validate_uris(_Key, L) ->
116 |     validate_uris0(L).
117 | 
118 | validate_uris0([Uri | Uris]) ->
119 |     case amqp_uri:parse(Uri) of
120 |         {ok, _Params} ->
121 |             validate_uris0(Uris);
122 |         {error, _} = Err ->
123 |             throw(Err)
124 |     end;
125 | validate_uris0([]) -> ok.
126 | 
127 | parse_current(ShovelName, Config) ->
128 |     {source, Source} = proplists:lookup(source, Config),
129 |     validate(Source),
130 |     SrcMod = resolve_module(proplists:get_value(protocol, Source, amqp091)),
131 |     {destination, Destination} = proplists:lookup(destination, Config),
132 |     validate(Destination),
133 |     DstMod = resolve_module(proplists:get_value(protocol, Destination, amqp091)),
134 |     AckMode = proplists:get_value(ack_mode, Config, no_ack),
135 |     validate_ack_mode(AckMode),
136 |     {ok, #{name => ShovelName,
137 |            shovel_type => static,
138 |            ack_mode => AckMode,
139 |            reconnect_delay => proplists:get_value(reconnect_delay, Config,
140 |                                                   ?DEFAULT_RECONNECT_DELAY),
141 |            source => rabbit_shovel_behaviour:parse(SrcMod, ShovelName,
142 |                                                    {source, Source}),
143 |            dest => rabbit_shovel_behaviour:parse(DstMod, ShovelName,
144 |                                                  {destination, Destination})}}.
145 | 
146 | %% ensures that any defaults that have been applied to a parsed
147 | %% shovel, are written back to the original proplist
148 | ensure_defaults(ShovelConfig, ParsedShovel) ->
149 |     lists:keystore(reconnect_delay, 1,
150 |                    ShovelConfig,
151 |                    {reconnect_delay,
152 |                     ParsedShovel#shovel.reconnect_delay}).
153 | 
154 | -spec fail(term()) -> no_return().
155 | fail(Reason) -> throw({error, Reason}).
156 | 
157 | validate_ack_mode(Val) when Val =:= no_ack orelse
158 |                                 Val =:= on_publish orelse
159 |                                 Val =:= on_confirm ->
160 |     ok;
161 | validate_ack_mode(WrongVal) ->
162 |     fail({invalid_parameter_value, ack_mode,
163 |           {ack_mode_value_requires_one_of, {no_ack, on_publish, on_confirm},
164 |           WrongVal}}).
165 | 
166 | duplicate_keys(PropList) when is_list(PropList) ->
167 |     proplists:get_keys(
168 |       lists:foldl(fun (K, L) -> lists:keydelete(K, 1, L) end, PropList,
169 |                   proplists:get_keys(PropList))).
170 | 
171 | get_value(Key, Props) ->
172 |     case proplists:get_value(Key, Props) of
173 |         undefined ->
174 |             throw({missing_parameter, Key});
175 |         V -> V
176 |     end.
177 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_dyn_worker_sup.erl:
--------------------------------------------------------------------------------
 1 | %% This Source Code Form is subject to the terms of the Mozilla Public
 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
 4 | %%
 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
 6 | %%
 7 | 
 8 | -module(rabbit_shovel_dyn_worker_sup).
 9 | -behaviour(supervisor2).
10 | 
11 | -export([start_link/2, init/1]).
12 | 
13 | -import(rabbit_misc, [pget/3]).
14 | 
15 | -include_lib("rabbit_common/include/rabbit.hrl").
16 | -include("rabbit_shovel.hrl").
17 | -define(SUPERVISOR, ?MODULE).
18 | 
19 | start_link(Name, Config) ->
20 |     supervisor2:start_link(?MODULE, [Name, Config]).
21 | 
22 | %%----------------------------------------------------------------------------
23 | 
24 | init([Name, Config0]) ->
25 |     Config  = rabbit_data_coercion:to_proplist(Config0),
26 |     Delay   = pget(<<"reconnect-delay">>, Config, ?DEFAULT_RECONNECT_DELAY),
27 |     case Name of
28 |       {VHost, ShovelName} -> rabbit_log:debug("Shovel '~s' in virtual host '~s' will use reconnection delay of ~p", [ShovelName, VHost, Delay]);
29 |       ShovelName          -> rabbit_log:debug("Shovel '~s' will use reconnection delay of ~s", [ShovelName, Delay])
30 |     end,
31 |     Restart = case Delay of
32 |         N when is_integer(N) andalso N > 0 ->
33 |           case pget(<<"src-delete-after">>, Config, pget(<<"delete-after">>, Config, <<"never">>)) of
34 |             %% always try to reconnect
35 |             <<"never">>                        -> {permanent, N};
36 |             %% this Shovel is an autodelete one
37 |               M when is_integer(M) andalso M > 0 -> {transient, N};
38 |               <<"queue-length">> -> {transient, N}
39 |           end;
40 |         %% reconnect-delay = 0 means "do not reconnect"
41 |         _                                  -> temporary
42 |     end,
43 | 
44 |     {ok, {{one_for_one, 1, ?MAX_WAIT},
45 |           [{Name,
46 |             {rabbit_shovel_worker, start_link, [dynamic, Name, Config]},
47 |             Restart,
48 |             16#ffffffff, worker, [rabbit_shovel_worker]}]}}.
49 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_dyn_worker_sup_sup.erl:
--------------------------------------------------------------------------------
 1 | %% This Source Code Form is subject to the terms of the Mozilla Public
 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
 4 | %%
 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
 6 | %%
 7 | 
 8 | -module(rabbit_shovel_dyn_worker_sup_sup).
 9 | -behaviour(mirrored_supervisor).
10 | 
11 | -export([start_link/0, init/1, adjust/2, stop_child/1]).
12 | 
13 | -import(rabbit_misc, [pget/2]).
14 | 
15 | -include("rabbit_shovel.hrl").
16 | -include_lib("rabbit_common/include/rabbit.hrl").
17 | -define(SUPERVISOR, ?MODULE).
18 | 
19 | start_link() ->
20 |     Pid = case mirrored_supervisor:start_link(
21 |                   {local, ?SUPERVISOR}, ?SUPERVISOR,
22 |                   fun rabbit_misc:execute_mnesia_transaction/1, ?MODULE, []) of
23 |             {ok, Pid0}                       -> Pid0;
24 |             {error, {already_started, Pid0}} -> Pid0
25 |           end,
26 |     Shovels = rabbit_runtime_parameters:list_component(<<"shovel">>),
27 |     [start_child({pget(vhost, Shovel), pget(name, Shovel)},
28 |                  pget(value, Shovel)) || Shovel <- Shovels],
29 |     {ok, Pid}.
30 | 
31 | adjust(Name, Def) ->
32 |     case child_exists(Name) of
33 |         true  -> stop_child(Name);
34 |         false -> ok
35 |     end,
36 |     start_child(Name, Def).
37 | 
38 | start_child(Name, Def) ->
39 |     case mirrored_supervisor:start_child(
40 |            ?SUPERVISOR,
41 |            {Name, {rabbit_shovel_dyn_worker_sup, start_link, [Name, Def]},
42 |             transient, ?WORKER_WAIT, worker, [rabbit_shovel_dyn_worker_sup]}) of
43 |         {ok,                      _Pid}  -> ok;
44 |         {error, {already_started, _Pid}} -> ok
45 |     end.
46 | 
47 | child_exists(Name) ->
48 |     lists:any(fun ({N, _, _, _}) -> N =:= Name end,
49 |               mirrored_supervisor:which_children(?SUPERVISOR)).
50 | 
51 | stop_child(Name) ->
52 |     case get(shovel_worker_autodelete) of
53 |         true -> ok; %% [1]
54 |         _    ->
55 |             ok = mirrored_supervisor:terminate_child(?SUPERVISOR, Name),
56 |             ok = mirrored_supervisor:delete_child(?SUPERVISOR, Name),
57 |             rabbit_shovel_status:remove(Name)
58 |     end.
59 | 
60 | %% [1] An autodeleting worker removes its own parameter, and thus ends
61 | %% up here via the parameter callback. It is a transient worker that
62 | %% is just about to terminate normally - so we don't need to tell the
63 | %% supervisor to stop us - and as usual if we call into our own
64 | %% supervisor we risk deadlock.
65 | %%
66 | %% See rabbit_shovel_worker:maybe_autodelete/1
67 | 
68 | %%----------------------------------------------------------------------------
69 | 
70 | init([]) ->
71 |     {ok, {{one_for_one, 3, 10}, []}}.
72 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_parameters.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(rabbit_shovel_parameters).
  9 | -behaviour(rabbit_runtime_parameter).
 10 | 
 11 | -include_lib("amqp_client/include/amqp_client.hrl").
 12 | -include("rabbit_shovel.hrl").
 13 | 
 14 | -export([validate/5, notify/5, notify_clear/4]).
 15 | -export([register/0, unregister/0, parse/3]).
 16 | 
 17 | -import(rabbit_misc, [pget/2, pget/3]).
 18 | 
 19 | -rabbit_boot_step({?MODULE,
 20 |                    [{description, "shovel parameters"},
 21 |                     {mfa, {rabbit_shovel_parameters, register, []}},
 22 |                     {cleanup, {?MODULE, unregister, []}},
 23 |                     {requires, rabbit_registry},
 24 |                     {enables, recovery}]}).
 25 | 
 26 | register() ->
 27 |     rabbit_registry:register(runtime_parameter, <<"shovel">>, ?MODULE).
 28 | 
 29 | unregister() ->
 30 |     rabbit_registry:unregister(runtime_parameter, <<"shovel">>).
 31 | 
 32 | validate(_VHost, <<"shovel">>, Name, Def0, User) ->
 33 |     Def = rabbit_data_coercion:to_proplist(Def0),
 34 |     Validations =
 35 |         shovel_validation()
 36 |         ++ src_validation(Def, User)
 37 |         ++ dest_validation(Def, User),
 38 |     validate_src(Def)
 39 |     ++ validate_dest(Def)
 40 |     ++ rabbit_parameter_validation:proplist(Name, Validations, Def);
 41 | 
 42 | validate(_VHost, _Component, Name, _Term, _User) ->
 43 |     {error, "name not recognised: ~p", [Name]}.
 44 | 
 45 | pget2(K1, K2, Defs) -> case {pget(K1, Defs), pget(K2, Defs)} of
 46 |                            {undefined, undefined} -> zero;
 47 |                            {undefined, _}         -> one;
 48 |                            {_,         undefined} -> one;
 49 |                            {_,         _}         -> both
 50 |                        end.
 51 | 
 52 | notify(VHost, <<"shovel">>, Name, Definition, _Username) ->
 53 |     rabbit_shovel_dyn_worker_sup_sup:adjust({VHost, Name}, Definition).
 54 | 
 55 | notify_clear(VHost, <<"shovel">>, Name, _Username) ->
 56 |     rabbit_shovel_dyn_worker_sup_sup:stop_child({VHost, Name}).
 57 | 
 58 | %%----------------------------------------------------------------------------
 59 | 
 60 | validate_src(Def) ->
 61 |     case protocols(Def)  of
 62 |         {amqp091, _} -> validate_amqp091_src(Def);
 63 |         {amqp10, _} -> []
 64 |     end.
 65 | 
 66 | validate_dest(Def) ->
 67 |     case protocols(Def)  of
 68 |         {_, amqp091} -> validate_amqp091_dest(Def);
 69 |         {_, amqp10} -> []
 70 |     end.
 71 | 
 72 | validate_amqp091_src(Def) ->
 73 |     [case pget2(<<"src-exchange">>, <<"src-queue">>, Def) of
 74 |          zero -> {error, "Must specify 'src-exchange' or 'src-queue'", []};
 75 |          one  -> ok;
 76 |          both -> {error, "Cannot specify 'src-exchange' and 'src-queue'", []}
 77 |      end,
 78 |      case {pget(<<"src-delete-after">>, Def, pget(<<"delete-after">>, Def)), pget(<<"ack-mode">>, Def)} of
 79 |          {N, <<"no-ack">>} when is_integer(N) ->
 80 |              {error, "Cannot specify 'no-ack' and numerical 'delete-after'", []};
 81 |          _ ->
 82 |              ok
 83 |      end].
 84 | 
 85 | validate_amqp091_dest(Def) ->
 86 |     [case pget2(<<"dest-exchange">>, <<"dest-queue">>, Def) of
 87 |          zero -> ok;
 88 |          one  -> ok;
 89 |          both -> {error, "Cannot specify 'dest-exchange' and 'dest-queue'", []}
 90 |      end].
 91 | 
 92 | shovel_validation() ->
 93 |     [{<<"reconnect-delay">>, fun rabbit_parameter_validation:number/2,optional},
 94 |      {<<"ack-mode">>, rabbit_parameter_validation:enum(
 95 |                         ['no-ack', 'on-publish', 'on-confirm']), optional},
 96 |      {<<"src-protocol">>,
 97 |       rabbit_parameter_validation:enum(['amqp10', 'amqp091']), optional},
 98 |      {<<"dest-protocol">>,
 99 |       rabbit_parameter_validation:enum(['amqp10', 'amqp091']), optional}
100 |     ].
101 | 
102 | src_validation(Def, User) ->
103 |     case protocols(Def)  of
104 |         {amqp091, _} -> amqp091_src_validation(Def, User);
105 |         {amqp10, _} -> amqp10_src_validation(Def, User)
106 |     end.
107 | 
108 | 
109 | amqp10_src_validation(_Def, User) ->
110 |     [
111 |      {<<"src-uri">>, validate_uri_fun(User), mandatory},
112 |      {<<"src-address">>, fun rabbit_parameter_validation:binary/2, mandatory},
113 |      {<<"src-prefetch-count">>, fun rabbit_parameter_validation:number/2, optional},
114 |      {<<"src-delete-after">>, fun validate_delete_after/2, optional}
115 |     ].
116 | 
117 | amqp091_src_validation(_Def, User) ->
118 |     [
119 |      {<<"src-uri">>,         validate_uri_fun(User), mandatory},
120 |      {<<"src-exchange">>,    fun rabbit_parameter_validation:binary/2,optional},
121 |      {<<"src-exchange-key">>,fun rabbit_parameter_validation:binary/2,optional},
122 |      {<<"src-queue">>,       fun rabbit_parameter_validation:binary/2,optional},
123 |      {<<"prefetch-count">>,  fun rabbit_parameter_validation:number/2,optional},
124 |      {<<"src-prefetch-count">>,  fun rabbit_parameter_validation:number/2,optional},
125 |      %% a deprecated pre-3.7 setting
126 |      {<<"delete-after">>, fun validate_delete_after/2, optional},
127 |      %% currently used multi-protocol friend name, introduced in 3.7
128 |      {<<"src-delete-after">>, fun validate_delete_after/2, optional}
129 |     ].
130 | 
131 | dest_validation(Def0, User) ->
132 |     Def = rabbit_data_coercion:to_proplist(Def0),
133 |     case protocols(Def)  of
134 |         {_, amqp091} -> amqp091_dest_validation(Def, User);
135 |         {_, amqp10} -> amqp10_dest_validation(Def, User)
136 |     end.
137 | 
138 | amqp10_dest_validation(_Def, User) ->
139 |     [{<<"dest-uri">>, validate_uri_fun(User), mandatory},
140 |      {<<"dest-address">>, fun rabbit_parameter_validation:binary/2, mandatory},
141 |      {<<"dest-add-forward-headers">>, fun rabbit_parameter_validation:boolean/2, optional},
142 |      {<<"dest-add-timestamp-header">>, fun rabbit_parameter_validation:boolean/2, optional},
143 |      {<<"dest-application-properties">>, fun validate_amqp10_map/2, optional},
144 |      {<<"dest-message-annotations">>, fun validate_amqp10_map/2, optional},
145 |      % TODO: restrict to allowed fields
146 |      {<<"dest-properties">>, fun validate_amqp10_map/2, optional}
147 |     ].
148 | 
149 | amqp091_dest_validation(_Def, User) ->
150 |     [{<<"dest-uri">>,        validate_uri_fun(User), mandatory},
151 |      {<<"dest-exchange">>,   fun rabbit_parameter_validation:binary/2,optional},
152 |      {<<"dest-exchange-key">>,fun rabbit_parameter_validation:binary/2,optional},
153 |      {<<"dest-queue">>,      fun rabbit_parameter_validation:binary/2,optional},
154 |      {<<"add-forward-headers">>, fun rabbit_parameter_validation:boolean/2,optional},
155 |      {<<"add-timestamp-header">>, fun rabbit_parameter_validation:boolean/2,optional},
156 |      {<<"dest-add-forward-headers">>, fun rabbit_parameter_validation:boolean/2,optional},
157 |      {<<"dest-add-timestamp-header">>, fun rabbit_parameter_validation:boolean/2,optional},
158 |      {<<"publish-properties">>, fun validate_properties/2,  optional},
159 |      {<<"dest-publish-properties">>, fun validate_properties/2,  optional}
160 |     ].
161 | 
162 | validate_uri_fun(User) ->
163 |     fun (Name, Term) -> validate_uri(Name, Term, User) end.
164 | 
165 | validate_uri(Name, Term, User) when is_binary(Term) ->
166 |     case rabbit_parameter_validation:binary(Name, Term) of
167 |         ok -> case amqp_uri:parse(binary_to_list(Term)) of
168 |                   {ok, P}    -> validate_params_user(P, User);
169 |                   {error, E} -> {error, "\"~s\" not a valid URI: ~p", [Term, E]}
170 |               end;
171 |         E  -> E
172 |     end;
173 | validate_uri(Name, Term, User) ->
174 |     case rabbit_parameter_validation:list(Name, Term) of
175 |         ok -> case [V || URI <- Term,
176 |                          V <- [validate_uri(Name, URI, User)],
177 |                          element(1, V) =:= error] of
178 |                   []      -> ok;
179 |                   [E | _] -> E
180 |               end;
181 |         E  -> E
182 |     end.
183 | 
184 | validate_params_user(#amqp_params_direct{}, none) ->
185 |     ok;
186 | validate_params_user(#amqp_params_direct{virtual_host = VHost},
187 |                      User = #user{username = Username}) ->
188 |     VHostAccess = case catch rabbit_access_control:check_vhost_access(User, VHost, undefined, #{}) of
189 |                       ok -> ok;
190 |                       NotOK ->
191 |                           rabbit_log:debug("rabbit_access_control:check_vhost_access result: ~p", [NotOK]),
192 |                           NotOK
193 |                   end,
194 |     case rabbit_vhost:exists(VHost) andalso VHostAccess of
195 |         ok -> ok;
196 |         _ ->
197 |             {error, "user \"~s\" may not connect to vhost \"~s\"", [Username, VHost]}
198 |     end;
199 | validate_params_user(#amqp_params_network{}, _User) ->
200 |     ok.
201 | 
202 | validate_delete_after(_Name, <<"never">>)          -> ok;
203 | validate_delete_after(_Name, <<"queue-length">>)   -> ok;
204 | validate_delete_after(_Name, N) when is_integer(N) -> ok;
205 | validate_delete_after(Name,  Term) ->
206 |     {error, "~s should be number, \"never\" or \"queue-length\", actually was "
207 |      "~p", [Name, Term]}.
208 | 
209 | validate_amqp10_map(Name, Terms0) ->
210 |     Terms = rabbit_data_coercion:to_proplist(Terms0),
211 |     Str = fun rabbit_parameter_validation:binary/2,
212 |     Validation = [{K, Str, optional} || {K, _} <- Terms],
213 |     rabbit_parameter_validation:proplist(Name, Validation, Terms).
214 | 
215 | %% TODO headers?
216 | validate_properties(Name, Term0) ->
217 |     Term = case Term0 of
218 |                T when is_map(T)  ->
219 |                    rabbit_data_coercion:to_proplist(Term0);
220 |                T when is_list(T) ->
221 |                    rabbit_data_coercion:to_proplist(Term0);
222 |                Other -> Other
223 |            end,
224 |     Str = fun rabbit_parameter_validation:binary/2,
225 |     Num = fun rabbit_parameter_validation:number/2,
226 |     rabbit_parameter_validation:proplist(
227 |       Name, [{<<"content_type">>,     Str, optional},
228 |              {<<"content_encoding">>, Str, optional},
229 |              {<<"delivery_mode">>,    Num, optional},
230 |              {<<"priority">>,         Num, optional},
231 |              {<<"correlation_id">>,   Str, optional},
232 |              {<<"reply_to">>,         Str, optional},
233 |              {<<"expiration">>,       Str, optional},
234 |              {<<"message_id">>,       Str, optional},
235 |              {<<"timestamp">>,        Num, optional},
236 |              {<<"type">>,             Str, optional},
237 |              {<<"user_id">>,          Str, optional},
238 |              {<<"app_id">>,           Str, optional},
239 |              {<<"cluster_id">>,       Str, optional}], Term).
240 | 
241 | %%----------------------------------------------------------------------------
242 | 
243 | parse({VHost, Name}, ClusterName, Def) ->
244 |     {Source, SourceHeaders} = parse_source(Def),
245 |     {ok, #{name => Name,
246 |            shovel_type => dynamic,
247 |            source => Source,
248 |            dest => parse_dest({VHost, Name}, ClusterName, Def,
249 |                                       SourceHeaders),
250 |            ack_mode => translate_ack_mode(pget(<<"ack-mode">>, Def, <<"on-confirm">>)),
251 |            reconnect_delay => pget(<<"reconnect-delay">>, Def,
252 |                                    ?DEFAULT_RECONNECT_DELAY)}}.
253 | 
254 | parse_source(Def) ->
255 |     case protocols(Def) of
256 |         {amqp10, _} -> parse_amqp10_source(Def);
257 |         {amqp091, _} -> parse_amqp091_source(Def)
258 |     end.
259 | 
260 | parse_dest(VHostName, ClusterName, Def, SourceHeaders) ->
261 |     case protocols(Def) of
262 |         {_, amqp10} ->
263 |             parse_amqp10_dest(VHostName, ClusterName, Def, SourceHeaders);
264 |         {_, amqp091} ->
265 |             parse_amqp091_dest(VHostName, ClusterName, Def, SourceHeaders)
266 |     end.
267 | 
268 | parse_amqp10_dest({_VHost, _Name}, _ClusterName, Def, SourceHeaders) ->
269 |     Uris = get_uris(<<"dest-uri">>, Def),
270 |     Address = pget(<<"dest-address">>, Def),
271 |     Properties =
272 |         rabbit_data_coercion:to_proplist(
273 |             pget(<<"dest-properties">>, Def, [])),
274 |     AppProperties =
275 |         rabbit_data_coercion:to_proplist(
276 |             pget(<<"dest-application-properties">>, Def, [])),
277 |     MessageAnns =
278 |         rabbit_data_coercion:to_proplist(
279 |             pget(<<"dest-message-annotations">>, Def, [])),
280 |     #{module => rabbit_amqp10_shovel,
281 |       uris => Uris,
282 |       target_address => Address,
283 |       message_annotations => maps:from_list(MessageAnns),
284 |       application_properties => maps:from_list(AppProperties ++ SourceHeaders),
285 |       properties => maps:from_list(
286 |                       lists:map(fun({K, V}) ->
287 |                                         {rabbit_data_coercion:to_atom(K), V}
288 |                                 end, Properties)),
289 |       add_timestamp_header => pget(<<"dest-add-timestamp-header">>, Def, false),
290 |       add_forward_headers => pget(<<"dest-add-forward-headers">>, Def, false),
291 |       unacked => #{}
292 |      }.
293 | 
294 | parse_amqp091_dest({VHost, Name}, ClusterName, Def, SourceHeaders) ->
295 |     DestURIs = get_uris(<<"dest-uri">>,      Def),
296 |     DestX    = pget(<<"dest-exchange">>,     Def, none),
297 |     DestXKey = pget(<<"dest-exchange-key">>, Def, none),
298 |     DestQ    = pget(<<"dest-queue">>,        Def, none),
299 |     DestDeclFun = fun (Conn, _Ch) ->
300 |                       case DestQ of
301 |                           none -> ok;
302 |                           _ -> ensure_queue(Conn, DestQ)
303 |                       end
304 |               end,
305 |     {X, Key} = case DestQ of
306 |                    none -> {DestX, DestXKey};
307 |                    _    -> {<<>>,  DestQ}
308 |                end,
309 |     Table2 = [{K, V} || {K, V} <- [{<<"dest-exchange">>,     DestX},
310 |                                    {<<"dest-exchange-key">>, DestXKey},
311 |                                    {<<"dest-queue">>,        DestQ}],
312 |                         V =/= none],
313 |     PubFun = fun (_SrcURI, _DestURI, P0) ->
314 |                      P1 = case X of
315 |                               none -> P0;
316 |                               _    -> P0#'basic.publish'{exchange = X}
317 |                           end,
318 |                      case Key of
319 |                          none -> P1;
320 |                          _    -> P1#'basic.publish'{routing_key = Key}
321 |                      end
322 |              end,
323 |     AddHeadersLegacy = pget(<<"add-forward-headers">>, Def, false),
324 |     AddHeaders = pget(<<"dest-add-forward-headers">>, Def, AddHeadersLegacy),
325 |     Table0 = [{<<"shovelled-by">>, ClusterName},
326 |               {<<"shovel-type">>,  <<"dynamic">>},
327 |               {<<"shovel-name">>,  Name},
328 |               {<<"shovel-vhost">>, VHost}],
329 |     SetProps = lookup_indices(pget(<<"dest-publish-properties">>, Def,
330 |                                    pget(<<"publish-properties">>, Def, [])),
331 |                               record_info(fields, 'P_basic')),
332 |     AddTimestampHeaderLegacy = pget(<<"add-timestamp-header">>, Def, false),
333 |     AddTimestampHeader = pget(<<"dest-add-timestamp-header">>, Def,
334 |                               AddTimestampHeaderLegacy),
335 |     PubPropsFun = fun (SrcURI, DestURI, P0) ->
336 |                       P  = set_properties(P0, SetProps),
337 |                       P1 = case AddHeaders of
338 |                                true -> rabbit_shovel_util:update_headers(
339 |                                           Table0, SourceHeaders ++ Table2,
340 |                                           SrcURI, DestURI, P);
341 |                                false -> P
342 |                       end,
343 |                       case AddTimestampHeader of
344 |                           true  -> rabbit_shovel_util:add_timestamp_header(P1);
345 |                           false -> P1
346 |                       end
347 |                   end,
348 |     %% Details are only used for status report in rabbitmqctl, as vhost is not
349 |     %% available to query the runtime parameters.
350 |     Details = maps:from_list([{K, V} || {K, V} <- [{dest_exchange, DestX},
351 |                                                    {dest_exchange_key, DestXKey},
352 |                                                    {dest_queue, DestQ}],
353 |                                         V =/= none]),
354 |     maps:merge(#{module => rabbit_amqp091_shovel,
355 |                  uris => DestURIs,
356 |                  resource_decl => DestDeclFun,
357 |                  fields_fun => PubFun,
358 |                  props_fun => PubPropsFun
359 |                 }, Details).
360 | 
361 | parse_amqp10_source(Def) ->
362 |     Uris = get_uris(<<"src-uri">>, Def),
363 |     Address = pget(<<"src-address">>, Def),
364 |     DeleteAfter = pget(<<"src-delete-after">>, Def, <<"never">>),
365 |     PrefetchCount = pget(<<"src-prefetch-count">>, Def, 1000),
366 |     Headers = [],
367 |     {#{module => rabbit_amqp10_shovel,
368 |        uris => Uris,
369 |        source_address => Address,
370 |        delete_after => opt_b2a(DeleteAfter),
371 |        prefetch_count => PrefetchCount}, Headers}.
372 | 
373 | parse_amqp091_source(Def) ->
374 |     SrcURIs = get_uris(<<"src-uri">>, Def),
375 |     SrcX = pget(<<"src-exchange">>,Def, none),
376 |     SrcXKey = pget(<<"src-exchange-key">>, Def, <<>>), %% [1]
377 |     SrcQ = pget(<<"src-queue">>, Def, none),
378 |     {SrcDeclFun, Queue, DestHeaders} =
379 |     case SrcQ of
380 |         none -> {fun (_Conn, Ch) ->
381 |                          Ms = [#'queue.declare'{exclusive = true},
382 |                                #'queue.bind'{routing_key = SrcXKey,
383 |                                              exchange    = SrcX}],
384 |                          [amqp_channel:call(Ch, M) || M <- Ms]
385 |                  end, <<>>, [{<<"src-exchange">>,     SrcX},
386 |                              {<<"src-exchange-key">>, SrcXKey}]};
387 |         _ -> {fun (Conn, _Ch) ->
388 |                       ensure_queue(Conn, SrcQ)
389 |               end, SrcQ, [{<<"src-queue">>, SrcQ}]}
390 |     end,
391 |     DeleteAfter = pget(<<"src-delete-after">>, Def,
392 |                        pget(<<"delete-after">>, Def, <<"never">>)),
393 |     PrefetchCount = pget(<<"src-prefetch-count">>, Def,
394 |                          pget(<<"prefetch-count">>, Def, 1000)),
395 |     %% Details are only used for status report in rabbitmqctl, as vhost is not
396 |     %% available to query the runtime parameters.
397 |     Details = maps:from_list([{K, V} || {K, V} <- [{source_exchange, SrcX},
398 |                                                    {source_exchange_key, SrcXKey}],
399 |                                         V =/= none]),
400 |     {maps:merge(#{module => rabbit_amqp091_shovel,
401 |                   uris => SrcURIs,
402 |                   resource_decl => SrcDeclFun,
403 |                   queue => Queue,
404 |                   delete_after => opt_b2a(DeleteAfter),
405 |                   prefetch_count => PrefetchCount
406 |                  }, Details), DestHeaders}.
407 | 
408 | get_uris(Key, Def) ->
409 |     URIs = case pget(Key, Def) of
410 |                B when is_binary(B) -> [B];
411 |                L when is_list(L)   -> L
412 |            end,
413 |     [binary_to_list(URI) || URI <- URIs].
414 | 
415 | translate_ack_mode(<<"on-confirm">>) -> on_confirm;
416 | translate_ack_mode(<<"on-publish">>) -> on_publish;
417 | translate_ack_mode(<<"no-ack">>)     -> no_ack.
418 | 
419 | ensure_queue(Conn, Queue) ->
420 |     {ok, Ch} = amqp_connection:open_channel(Conn),
421 |     try
422 |         amqp_channel:call(Ch, #'queue.declare'{queue   = Queue,
423 |                                                passive = true})
424 |     catch exit:{{shutdown, {server_initiated_close, ?NOT_FOUND, _Text}}, _} ->
425 |             {ok, Ch2} = amqp_connection:open_channel(Conn),
426 |             amqp_channel:call(Ch2, #'queue.declare'{queue   = Queue,
427 |                                                     durable = true}),
428 |             catch amqp_channel:close(Ch2)
429 | 
430 |     after
431 |         catch amqp_channel:close(Ch)
432 |     end.
433 | 
434 | opt_b2a(B) when is_binary(B) -> list_to_atom(binary_to_list(B));
435 | opt_b2a(N)                   -> N.
436 | 
437 | set_properties(Props, []) ->
438 |     Props;
439 | set_properties(Props, [{Ix, V} | Rest]) ->
440 |     set_properties(setelement(Ix, Props, V), Rest).
441 | 
442 | lookup_indices(KVs0, L) ->
443 |     KVs = rabbit_data_coercion:to_proplist(KVs0),
444 |     [{1 + list_find(list_to_atom(binary_to_list(K)), L), V} || {K, V} <- KVs].
445 | 
446 | list_find(K, L) -> list_find(K, L, 1).
447 | 
448 | list_find(K, [K|_], N) -> N;
449 | list_find(K, [],   _N) -> exit({not_found, K});
450 | list_find(K, [_|L], N) -> list_find(K, L, N + 1).
451 | 
452 | protocols(Def) when is_map(Def) ->
453 |     protocols(rabbit_data_coercion:to_proplist(Def));
454 | protocols(Def) ->
455 |     Src = case lists:keyfind(<<"src-protocol">>, 1, Def) of
456 |               {_, SrcProtocol} ->
457 |                   rabbit_data_coercion:to_atom(SrcProtocol);
458 |               false -> amqp091
459 |           end,
460 |     Dst = case lists:keyfind(<<"dest-protocol">>, 1, Def) of
461 |               {_, DstProtocol} ->
462 |                   rabbit_data_coercion:to_atom(DstProtocol);
463 |               false -> amqp091
464 |           end,
465 |     {Src, Dst}.
466 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_status.erl:
--------------------------------------------------------------------------------
 1 | %% This Source Code Form is subject to the terms of the Mozilla Public
 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
 4 | %%
 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
 6 | %%
 7 | 
 8 | -module(rabbit_shovel_status).
 9 | -behaviour(gen_server).
10 | 
11 | -export([start_link/0]).
12 | 
13 | -export([report/3, remove/1, status/0, lookup/1]).
14 | 
15 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
16 |          terminate/2, code_change/3]).
17 | 
18 | -define(SERVER, ?MODULE).
19 | -define(ETS_NAME, ?MODULE).
20 | 
21 | -record(state, {}).
22 | -record(entry, {name, type, info, timestamp}).
23 | 
24 | start_link() ->
25 |     gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
26 | 
27 | report(Name, Type, Info) ->
28 |     gen_server:cast(?SERVER, {report, Name, Type, Info, calendar:local_time()}).
29 | 
30 | remove(Name) ->
31 |     gen_server:cast(?SERVER, {remove, Name}).
32 | 
33 | status() ->
34 |     gen_server:call(?SERVER, status, infinity).
35 | 
36 | lookup(Name) ->
37 |     gen_server:call(?SERVER, {lookup, Name}, infinity).
38 | 
39 | init([]) ->
40 |     ?ETS_NAME = ets:new(?ETS_NAME,
41 |                         [named_table, {keypos, #entry.name}, private]),
42 |     {ok, #state{}}.
43 | 
44 | handle_call(status, _From, State) ->
45 |     Entries = ets:tab2list(?ETS_NAME),
46 |     {reply, [{Entry#entry.name, Entry#entry.type, Entry#entry.info,
47 |               Entry#entry.timestamp}
48 |              || Entry <- Entries], State};
49 | 
50 | handle_call({lookup, Name}, _From, State) ->
51 |     Link = case ets:lookup(?ETS_NAME, Name) of
52 |                [Entry] -> [{name, Name},
53 |                            {type, Entry#entry.type},
54 |                            {info, Entry#entry.info},
55 |                            {timestamp, Entry#entry.timestamp}];
56 |                [] -> not_found
57 |            end,
58 |     {reply, Link, State}.
59 | 
60 | handle_cast({report, Name, Type, Info, Timestamp}, State) ->
61 |     true = ets:insert(?ETS_NAME, #entry{name = Name, type = Type, info = Info,
62 |                                         timestamp = Timestamp}),
63 |     rabbit_event:notify(shovel_worker_status,
64 |                         split_name(Name) ++ split_status(Info)),
65 |     {noreply, State};
66 | 
67 | handle_cast({remove, Name}, State) ->
68 |     true = ets:delete(?ETS_NAME, Name),
69 |     rabbit_event:notify(shovel_worker_removed, split_name(Name)),
70 |     {noreply, State}.
71 | 
72 | handle_info(_Info, State) ->
73 |     {noreply, State}.
74 | 
75 | terminate(_Reason, _State) ->
76 |     ok.
77 | 
78 | code_change(_OldVsn, State, _Extra) ->
79 |     {ok, State}.
80 | 
81 | split_status({running, MoreInfo})         -> [{status, running} | MoreInfo];
82 | split_status({terminated, Reason})        -> [{status, terminated},
83 |                                               {reason, Reason}];
84 | split_status(Status) when is_atom(Status) -> [{status, Status}].
85 | 
86 | split_name({VHost, Name})           -> [{name,  Name},
87 |                                         {vhost, VHost}];
88 | split_name(Name) when is_atom(Name) -> [{name, Name}].
89 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_sup.erl:
--------------------------------------------------------------------------------
 1 | %% This Source Code Form is subject to the terms of the Mozilla Public
 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
 4 | %%
 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
 6 | %%
 7 | 
 8 | -module(rabbit_shovel_sup).
 9 | -behaviour(supervisor2).
10 | 
11 | -export([start_link/0, init/1]).
12 | 
13 | -import(rabbit_shovel_config, []).
14 | 
15 | -include("rabbit_shovel.hrl").
16 | 
17 | start_link() ->
18 |     case parse_configuration(application:get_env(shovels)) of
19 |         {ok, Configurations} ->
20 |             supervisor2:start_link({local, ?MODULE}, ?MODULE, [Configurations]);
21 |         {error, Reason} ->
22 |             {error, Reason}
23 |     end.
24 | 
25 | init([Configurations]) ->
26 |     Len = dict:size(Configurations),
27 |     ChildSpecs = [{rabbit_shovel_status,
28 |                    {rabbit_shovel_status, start_link, []},
29 |                    transient, 16#ffffffff, worker,
30 |                    [rabbit_shovel_status]},
31 |                   {rabbit_shovel_dyn_worker_sup_sup,
32 |                    {rabbit_shovel_dyn_worker_sup_sup, start_link, []},
33 |                    transient, 16#ffffffff, supervisor,
34 |                    [rabbit_shovel_dyn_worker_sup_sup]} |
35 |                   make_child_specs(Configurations)],
36 |     {ok, {{one_for_one, 2*Len, 2}, ChildSpecs}}.
37 | 
38 | make_child_specs(Configurations) ->
39 |     dict:fold(
40 |       fun (ShovelName, ShovelConfig, Acc) ->
41 |               [{ShovelName,
42 |                 {rabbit_shovel_worker_sup, start_link,
43 |                     [ShovelName, ShovelConfig]},
44 |                 permanent,
45 |                 16#ffffffff,
46 |                 supervisor,
47 |                 [rabbit_shovel_worker_sup]} | Acc]
48 |       end, [], Configurations).
49 | 
50 | parse_configuration(undefined) ->
51 |     {ok, dict:new()};
52 | parse_configuration({ok, Env}) ->
53 |     {ok, Defaults} = application:get_env(defaults),
54 |     parse_configuration(Defaults, Env, dict:new()).
55 | 
56 | parse_configuration(_Defaults, [], Acc) ->
57 |     {ok, Acc};
58 | parse_configuration(Defaults, [{ShovelName, ShovelConfig} | Env], Acc)
59 |   when is_atom(ShovelName) andalso is_list(ShovelConfig) ->
60 |     case dict:is_key(ShovelName, Acc) of
61 |         true  -> {error, {duplicate_shovel_definition, ShovelName}};
62 |         false -> case validate_shovel_config(ShovelName, ShovelConfig) of
63 |                      {ok, Shovel} ->
64 |                          %% make sure the config we accumulate has any
65 |                          %% relevant default values (discovered during
66 |                          %% validation), applied back to it
67 |                          Acc2 = dict:store(ShovelName, Shovel, Acc),
68 |                          parse_configuration(Defaults, Env, Acc2);
69 |                      Error ->
70 |                          Error
71 |                  end
72 |     end;
73 | parse_configuration(_Defaults, _, _Acc) ->
74 |     {error, require_list_of_shovel_configurations}.
75 | 
76 | validate_shovel_config(ShovelName, ShovelConfig) ->
77 |     rabbit_shovel_config:parse(ShovelName, ShovelConfig).
78 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_util.erl:
--------------------------------------------------------------------------------
 1 | %% This Source Code Form is subject to the terms of the Mozilla Public
 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
 4 | %%
 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
 6 | %%
 7 | 
 8 | -module(rabbit_shovel_util).
 9 | 
10 | -export([update_headers/5,
11 |          add_timestamp_header/1,
12 |          delete_shovel/3,
13 |          restart_shovel/2]).
14 | 
15 | -include_lib("rabbit_common/include/rabbit_framing.hrl").
16 | 
17 | -define(ROUTING_HEADER, <<"x-shovelled">>).
18 | -define(TIMESTAMP_HEADER, <<"x-shovelled-timestamp">>).
19 | 
20 | update_headers(Prefix, Suffix, SrcURI, DestURI,
21 |                Props = #'P_basic'{headers = Headers}) ->
22 |     Table = Prefix ++ [{<<"src-uri">>,  SrcURI},
23 |                        {<<"dest-uri">>, DestURI}] ++ Suffix,
24 |     Headers2 = rabbit_basic:prepend_table_header(
25 |                  ?ROUTING_HEADER, [{K, longstr, V} || {K, V} <- Table],
26 |                  Headers),
27 |     Props#'P_basic'{headers = Headers2}.
28 | 
29 | add_timestamp_header(Props = #'P_basic'{headers = undefined}) ->
30 |     add_timestamp_header(Props#'P_basic'{headers = []});
31 | add_timestamp_header(Props = #'P_basic'{headers = Headers}) ->
32 |     Headers2 = rabbit_misc:set_table_value(Headers,
33 |                                            ?TIMESTAMP_HEADER,
34 |                                            long,
35 |                                            os:system_time(seconds)),
36 |     Props#'P_basic'{headers = Headers2}.
37 | 
38 | delete_shovel(VHost, Name, ActingUser) ->
39 |     case rabbit_shovel_status:lookup({VHost, Name}) of
40 |         not_found ->
41 |             {error, not_found};
42 |         _Obj ->
43 |             ok = rabbit_runtime_parameters:clear(VHost, <<"shovel">>, Name, ActingUser)
44 |     end.
45 | 
46 | restart_shovel(VHost, Name) ->
47 |     case rabbit_shovel_status:lookup({VHost, Name}) of
48 |         not_found ->
49 |             {error, not_found};
50 |         _Obj ->
51 |             ok = rabbit_shovel_dyn_worker_sup_sup:stop_child({VHost, Name}),
52 |             {ok, _} = rabbit_shovel_dyn_worker_sup_sup:start_link(),
53 |             ok
54 |     end.
55 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_worker.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(rabbit_shovel_worker).
  9 | -behaviour(gen_server2).
 10 | 
 11 | -export([start_link/3]).
 12 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, terminate/2,
 13 |          code_change/3]).
 14 | 
 15 | %% for testing purposes
 16 | -export([get_connection_name/1]).
 17 | 
 18 | -include_lib("amqp_client/include/amqp_client.hrl").
 19 | -include("rabbit_shovel.hrl").
 20 | 
 21 | -record(state, {inbound_conn, inbound_ch, outbound_conn, outbound_ch,
 22 |                 name, type, config, inbound_uri, outbound_uri, unacked,
 23 |                 remaining, %% [1]
 24 |                 remaining_unacked}). %% [2]
 25 | 
 26 | %% [1] Counts down until we shut down in all modes
 27 | %% [2] Counts down until we stop publishing in on-confirm mode
 28 | 
 29 | start_link(Type, Name, Config) ->
 30 |     ok = rabbit_shovel_status:report(Name, Type, starting),
 31 |     gen_server2:start_link(?MODULE, [Type, Name, Config], []).
 32 | 
 33 | %%---------------------------
 34 | %% Gen Server Implementation
 35 | %%---------------------------
 36 | 
 37 | init([Type, Name, Config0]) ->
 38 |     Config = case Type of
 39 |                 static ->
 40 |                      Config0;
 41 |                 dynamic ->
 42 |                     ClusterName = rabbit_nodes:cluster_name(),
 43 |                     {ok, Conf} = rabbit_shovel_parameters:parse(Name,
 44 |                                                                 ClusterName,
 45 |                                                                 Config0),
 46 |                     Conf
 47 |             end,
 48 |     rabbit_log_shovel:debug("Initialising a Shovel ~s of type '~s'", [human_readable_name(Name), Type]),
 49 |     gen_server2:cast(self(), init),
 50 |     {ok, #state{name = Name, type = Type, config = Config}}.
 51 | 
 52 | handle_call(_Msg, _From, State) ->
 53 |     {noreply, State}.
 54 | 
 55 | handle_cast(init, State = #state{config = Config0}) ->
 56 |     try rabbit_shovel_behaviour:connect_source(Config0) of
 57 |       Config ->
 58 |         rabbit_log_shovel:debug("Shovel ~s connected to source", [human_readable_name(maps:get(name, Config))]),
 59 |         %% this makes sure that connection pid is updated in case
 60 |         %% any of the subsequent connection/init steps fail. See
 61 |         %% rabbitmq/rabbitmq-shovel#54 for context.
 62 |         gen_server2:cast(self(), connect_dest),
 63 |         {noreply, State#state{config = Config}}
 64 |     catch _:_ ->
 65 |       rabbit_log_shovel:error("Shovel ~s could not connect to source", [human_readable_name(maps:get(name, Config0))]),
 66 |       {stop, shutdown, State}
 67 |     end;
 68 | handle_cast(connect_dest, State = #state{config = Config0}) ->
 69 |     try rabbit_shovel_behaviour:connect_dest(Config0) of
 70 |       Config ->
 71 |         rabbit_log_shovel:debug("Shovel ~s connected to destination", [human_readable_name(maps:get(name, Config))]),
 72 |         gen_server2:cast(self(), init_shovel),
 73 |         {noreply, State#state{config = Config}}
 74 |     catch _:_ ->
 75 |       rabbit_log_shovel:error("Shovel ~s could not connect to destination", [human_readable_name(maps:get(name, Config0))]),
 76 |       {stop, shutdown, State}
 77 |     end;
 78 | handle_cast(init_shovel, State = #state{config = Config}) ->
 79 |     %% Don't trap exits until we have established connections so that
 80 |     %% if we try to shut down while waiting for a connection to be
 81 |     %% established then we don't block
 82 |     process_flag(trap_exit, true),
 83 |     Config1 = rabbit_shovel_behaviour:init_dest(Config),
 84 |     Config2 = rabbit_shovel_behaviour:init_source(Config1),
 85 |     rabbit_log_shovel:debug("Shovel ~s has finished setting up its topology", [human_readable_name(maps:get(name, Config2))]),
 86 |     State1 = State#state{config = Config2},
 87 |     ok = report_running(State1),
 88 |     {noreply, State1}.
 89 | 
 90 | 
 91 | handle_info(Msg, State = #state{config = Config, name = Name}) ->
 92 |     case rabbit_shovel_behaviour:handle_source(Msg, Config) of
 93 |         not_handled ->
 94 |             case rabbit_shovel_behaviour:handle_dest(Msg, Config) of
 95 |                 not_handled ->
 96 |                     rabbit_log_shovel:warning("Shovel ~s could not handle a destination message ~p", [human_readable_name(Name), Msg]),
 97 |                     {noreply, State};
 98 |                 {stop, {outbound_conn_died, heartbeat_timeout}} ->
 99 |                     rabbit_log_shovel:error("Shovel ~s detected missed heartbeats on destination connection", [human_readable_name(Name)]),
100 |                     {stop, {shutdown, heartbeat_timeout}, State};
101 |                 {stop, {outbound_conn_died, Reason}} ->
102 |                     rabbit_log_shovel:error("Shovel ~s detected destination connection failure: ~p", [human_readable_name(Name), Reason]),
103 |                     {stop, Reason, State};
104 |                 {stop, Reason} ->
105 |                     rabbit_log_shovel:debug("Shovel ~s decided to stop due a message from destination: ~p", [human_readable_name(Name), Reason]),
106 |                     {stop, Reason, State};
107 |                 Config1 ->
108 |                     {noreply, State#state{config = Config1}}
109 |             end;
110 |         {stop, {inbound_conn_died, heartbeat_timeout}} ->
111 |             rabbit_log_shovel:error("Shovel ~s detected missed heartbeats on source connection", [human_readable_name(Name)]),
112 |             {stop, {shutdown, heartbeat_timeout}, State};
113 |         {stop, {inbound_conn_died, Reason}} ->
114 |             rabbit_log_shovel:error("Shovel ~s detected source connection failure: ~p", [human_readable_name(Name), Reason]),
115 |             {stop, Reason, State};
116 |         {stop, Reason} ->
117 |             rabbit_log_shovel:error("Shovel ~s decided to stop due a message from source: ~p", [human_readable_name(Name), Reason]),
118 |             {stop, Reason, State};
119 |         Config1 ->
120 |             {noreply, State#state{config = Config1}}
121 |     end.
122 | 
123 | terminate({shutdown, autodelete}, State = #state{name = {VHost, Name},
124 |                                                  type = dynamic}) ->
125 |     rabbit_log_shovel:info("Shovel '~s' is stopping (it was configured to autodelete and transfer is completed)",
126 |                            [human_readable_name({VHost, Name})]),
127 |     close_connections(State),
128 |     %% See rabbit_shovel_dyn_worker_sup_sup:stop_child/1
129 |     put(shovel_worker_autodelete, true),
130 |     _ = rabbit_runtime_parameters:clear(VHost, <<"shovel">>, Name, ?SHOVEL_USER),
131 |     rabbit_shovel_status:remove({VHost, Name}),
132 |     ok;
133 | terminate(shutdown, State) ->
134 |     close_connections(State),
135 |     ok;
136 | terminate(socket_closed_unexpectedly, State) ->
137 |     close_connections(State),
138 |     ok;
139 | terminate({'EXIT', heartbeat_timeout}, State = #state{name = Name}) ->
140 |     rabbit_log_shovel:error("Shovel ~s is stopping because of a heartbeat timeout", [human_readable_name(Name)]),
141 |     rabbit_shovel_status:report(State#state.name, State#state.type,
142 |                                 {terminated, "heartbeat timeout"}),
143 |     close_connections(State),
144 |     ok;
145 | terminate({'EXIT', outbound_conn_died}, State = #state{name = Name}) ->
146 |     rabbit_log_shovel:error("Shovel ~s is stopping because destination connection failed", [human_readable_name(Name)]),
147 |     rabbit_shovel_status:report(State#state.name, State#state.type,
148 |                                 {terminated, "destination connection failed"}),
149 |     close_connections(State),
150 |     ok;
151 | terminate({'EXIT', inbound_conn_died}, State = #state{name = Name}) ->
152 |     rabbit_log_shovel:error("Shovel ~s is stopping because destination connection failed", [human_readable_name(Name)]),
153 |     rabbit_shovel_status:report(State#state.name, State#state.type,
154 |                                 {terminated, "source connection failed"}),
155 |     close_connections(State),
156 |     ok;
157 | terminate({shutdown, heartbeat_timeout}, State = #state{name = Name}) ->
158 |     rabbit_log_shovel:error("Shovel ~s is stopping because of a heartbeat timeout", [human_readable_name(Name)]),
159 |     rabbit_shovel_status:report(State#state.name, State#state.type,
160 |                                 {terminated, "heartbeat timeout"}),
161 |     close_connections(State),
162 |     ok;
163 | terminate({shutdown, restart}, State = #state{name = Name}) ->
164 |     rabbit_log_shovel:error("Shovel ~s is stopping to restart", [human_readable_name(Name)]),
165 |     rabbit_shovel_status:report(State#state.name, State#state.type,
166 |                                 {terminated, "needed a restart"}),
167 |     close_connections(State),
168 |     ok;
169 | terminate(Reason, State = #state{name = Name}) ->
170 |     rabbit_log_shovel:error("Shovel ~s is stopping, reason: ~p", [human_readable_name(Name), Reason]),
171 |     rabbit_shovel_status:report(State#state.name, State#state.type,
172 |                                 {terminated, Reason}),
173 |     close_connections(State),
174 |     ok.
175 | 
176 | code_change(_OldVsn, State, _Extra) ->
177 |     {ok, State}.
178 | 
179 | %%---------------------------
180 | %% Helpers
181 | %%---------------------------
182 | 
183 | human_readable_name(Name) ->
184 |   case Name of
185 |     {VHost, ShovelName} -> rabbit_misc:format("'~s' in virtual host '~s'", [ShovelName, VHost]);
186 |     ShovelName          -> rabbit_misc:format("'~s'", [ShovelName])
187 |   end.
188 | 
189 | report_running(#state{config = Config} = State) ->
190 |     InUri = rabbit_shovel_behaviour:source_uri(Config),
191 |     OutUri = rabbit_shovel_behaviour:dest_uri(Config),
192 |     InProto = rabbit_shovel_behaviour:source_protocol(Config),
193 |     OutProto = rabbit_shovel_behaviour:dest_protocol(Config),
194 |     InEndpoint = rabbit_shovel_behaviour:source_endpoint(Config),
195 |     OutEndpoint = rabbit_shovel_behaviour:dest_endpoint(Config),
196 |     rabbit_shovel_status:report(State#state.name, State#state.type,
197 |                                 {running, [{src_uri,  rabbit_data_coercion:to_binary(InUri)},
198 |                                            {src_protocol, rabbit_data_coercion:to_binary(InProto)},
199 |                                            {dest_protocol, rabbit_data_coercion:to_binary(OutProto)},
200 |                                            {dest_uri, rabbit_data_coercion:to_binary(OutUri)}]
201 |                                  ++ props_to_binary(InEndpoint) ++ props_to_binary(OutEndpoint)
202 |                                 }).
203 | 
204 | props_to_binary(Props) ->
205 |     [{K, rabbit_data_coercion:to_binary(V)} || {K, V} <- Props].
206 | 
207 | %% for static shovels, name is an atom from the configuration file
208 | get_connection_name(ShovelName) when is_atom(ShovelName) ->
209 |     Prefix = <<"Shovel ">>,
210 |     ShovelNameAsBinary = atom_to_binary(ShovelName, utf8),
211 |     <>;
212 | 
213 | %% for dynamic shovels, name is a tuple with a binary
214 | get_connection_name({_, Name}) when is_binary(Name) ->
215 |     Prefix = <<"Shovel ">>,
216 |     <>;
217 | 
218 | %% fallback
219 | get_connection_name(_) ->
220 |     <<"Shovel">>.
221 | 
222 | close_connections(#state{config = Conf}) ->
223 |     ok = rabbit_shovel_behaviour:close_source(Conf),
224 |     ok = rabbit_shovel_behaviour:close_dest(Conf).
225 | 


--------------------------------------------------------------------------------
/src/rabbit_shovel_worker_sup.erl:
--------------------------------------------------------------------------------
 1 | %% This Source Code Form is subject to the terms of the Mozilla Public
 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
 4 | %%
 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
 6 | %%
 7 | 
 8 | -module(rabbit_shovel_worker_sup).
 9 | -behaviour(mirrored_supervisor).
10 | 
11 | -export([start_link/2, init/1]).
12 | 
13 | -include("rabbit_shovel.hrl").
14 | -include_lib("rabbit_common/include/rabbit.hrl").
15 | 
16 | start_link(ShovelName, ShovelConfig) ->
17 |     mirrored_supervisor:start_link({local, ShovelName}, ShovelName,
18 |                                    fun rabbit_misc:execute_mnesia_transaction/1,
19 |                                    ?MODULE, [ShovelName, ShovelConfig]).
20 | 
21 | init([Name, Config]) ->
22 |     ChildSpecs = [{Name,
23 |                    {rabbit_shovel_worker, start_link, [static, Name, Config]},
24 |                    case Config of
25 |                        #{reconnect_delay := N}
26 |                          when is_integer(N) andalso N > 0 -> {permanent, N};
27 |                        _ -> temporary
28 |                    end,
29 |                    16#ffffffff,
30 |                    worker,
31 |                    [rabbit_shovel_worker]}],
32 |     {ok, {{one_for_one, 1, ?MAX_WAIT}, ChildSpecs}}.
33 | 


--------------------------------------------------------------------------------
/test/amqp10_SUITE.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(amqp10_SUITE).
  9 | 
 10 | -include_lib("common_test/include/ct.hrl").
 11 | -include_lib("amqp_client/include/amqp_client.hrl").
 12 | 
 13 | -compile(export_all).
 14 | 
 15 | -define(EXCHANGE,    <<"test_exchange">>).
 16 | -define(TO_SHOVEL,   <<"to_the_shovel">>).
 17 | -define(FROM_SHOVEL, <<"from_the_shovel">>).
 18 | -define(UNSHOVELLED, <<"unshovelled">>).
 19 | -define(SHOVELLED,   <<"shovelled">>).
 20 | -define(TIMEOUT,     1000).
 21 | 
 22 | all() ->
 23 |     [
 24 |       {group, tests}
 25 |     ].
 26 | 
 27 | groups() ->
 28 |     [
 29 |       {tests, [], [
 30 |           amqp10_destination_no_ack,
 31 |           amqp10_destination_on_publish,
 32 |           amqp10_destination_on_confirm,
 33 |           amqp10_source_no_ack,
 34 |           amqp10_source_on_publish,
 35 |           amqp10_source_on_confirm
 36 |         ]}
 37 |     ].
 38 | 
 39 | %% -------------------------------------------------------------------
 40 | %% Testsuite setup/teardown.
 41 | %% -------------------------------------------------------------------
 42 | 
 43 | init_per_suite(Config) ->
 44 |     {ok, _} = application:ensure_all_started(amqp10_client),
 45 |     rabbit_ct_helpers:log_environment(),
 46 |     Config1 = rabbit_ct_helpers:set_config(Config, [
 47 |         {rmq_nodename_suffix, ?MODULE}
 48 |       ]),
 49 |     rabbit_ct_helpers:run_setup_steps(Config1,
 50 |       rabbit_ct_broker_helpers:setup_steps() ++
 51 |       rabbit_ct_client_helpers:setup_steps() ++
 52 |       [fun stop_shovel_plugin/1]).
 53 | 
 54 | end_per_suite(Config) ->
 55 |     application:stop(amqp10_client),
 56 |     rabbit_ct_helpers:run_teardown_steps(Config,
 57 |       rabbit_ct_client_helpers:teardown_steps() ++
 58 |       rabbit_ct_broker_helpers:teardown_steps()).
 59 | 
 60 | init_per_group(_, Config) ->
 61 |     Config.
 62 | 
 63 | end_per_group(_, Config) ->
 64 |     Config.
 65 | 
 66 | init_per_testcase(Testcase, Config) ->
 67 |     rabbit_ct_helpers:testcase_started(Config, Testcase).
 68 | 
 69 | end_per_testcase(Testcase, Config) ->
 70 |     rabbit_ct_helpers:testcase_finished(Config, Testcase).
 71 | 
 72 | stop_shovel_plugin(Config) ->
 73 |     ok = rabbit_ct_broker_helpers:rpc(Config, 0,
 74 |                                       application, stop, [rabbitmq_shovel]),
 75 |     Config.
 76 | 
 77 | %% -------------------------------------------------------------------
 78 | %% Testcases.
 79 | %% -------------------------------------------------------------------
 80 | 
 81 | amqp10_destination_no_ack(Config) ->
 82 |     amqp10_destination(Config, no_ack).
 83 | 
 84 | amqp10_destination_on_publish(Config) ->
 85 |     amqp10_destination(Config, on_publish).
 86 | 
 87 | amqp10_destination_on_confirm(Config) ->
 88 |     amqp10_destination(Config, on_confirm).
 89 | 
 90 | amqp10_destination(Config, AckMode) ->
 91 |     TargetQ =  <<"a-queue">>,
 92 |     ok = setup_amqp10_destination_shovel(Config, TargetQ, AckMode),
 93 |     Hostname = ?config(rmq_hostname, Config),
 94 |     Port = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp),
 95 |     {ok, Conn} = amqp10_client:open_connection(Hostname, Port),
 96 |     {ok, Sess} = amqp10_client:begin_session(Conn),
 97 |     {ok, Receiver} = amqp10_client:attach_receiver_link(Sess,
 98 |                                                         <<"amqp-destination-receiver">>,
 99 |                                                         TargetQ, settled, unsettled_state),
100 |     ok = amqp10_client:flow_link_credit(Receiver, 5, never),
101 |     Chan = rabbit_ct_client_helpers:open_channel(Config, 0),
102 |     Timestamp = erlang:system_time(millisecond),
103 |     Msg = #amqp_msg{payload = <<42>>,
104 |                     props = #'P_basic'{delivery_mode = 2,
105 |                                        headers = [{<<"header1">>, long, 1},
106 |                                                   {<<"header2">>, longstr, <<"h2">>}],
107 |                                        content_encoding = ?UNSHOVELLED,
108 |                                        content_type = ?UNSHOVELLED,
109 |                                        correlation_id = ?UNSHOVELLED,
110 |                                        %% needs to be guest here
111 |                                        user_id = <<"guest">>,
112 |                                        message_id = ?UNSHOVELLED,
113 |                                        reply_to = ?UNSHOVELLED,
114 |                                        timestamp = Timestamp,
115 |                                        type = ?UNSHOVELLED
116 |                                       }},
117 |     publish(Chan, Msg, ?EXCHANGE, ?TO_SHOVEL),
118 | 
119 |     receive
120 |         {amqp10_msg, Receiver, InMsg} ->
121 |             [<<42>>] = amqp10_msg:body(InMsg),
122 |             #{content_type := ?UNSHOVELLED,
123 |               content_encoding := ?UNSHOVELLED,
124 |               correlation_id := ?UNSHOVELLED,
125 |               user_id := <<"guest">>,
126 |               message_id := ?UNSHOVELLED,
127 |               reply_to := ?UNSHOVELLED
128 |               %% timestamp gets overwritten
129 |               % creation_time := Timestamp
130 |              } = amqp10_msg:properties(InMsg),
131 |             #{<<"routing_key">> := ?TO_SHOVEL,
132 |               <<"type">> := ?UNSHOVELLED,
133 |               <<"header1">> := 1,
134 |               <<"header2">> := <<"h2">>
135 |              } = amqp10_msg:application_properties(InMsg),
136 |             #{durable := true} = amqp10_msg:headers(InMsg),
137 |             ok
138 |     after ?TIMEOUT ->
139 |               throw(timeout_waiting_for_deliver1)
140 |     end,
141 | 
142 |     [{test_shovel, static, {running, _Info}, _Time}] =
143 |         rabbit_ct_broker_helpers:rpc(Config, 0,
144 |           rabbit_shovel_status, status, []),
145 |     amqp10_client:detach_link(Receiver),
146 |     amqp10_client:close_connection(Conn),
147 |     rabbit_ct_client_helpers:close_channel(Chan).
148 | 
149 | amqp10_source_no_ack(Config) ->
150 |     amqp10_source(Config, no_ack).
151 | 
152 | amqp10_source_on_publish(Config) ->
153 |     amqp10_source(Config, on_publish).
154 | 
155 | amqp10_source_on_confirm(Config) ->
156 |     amqp10_source(Config, on_confirm).
157 | 
158 | amqp10_source(Config, AckMode) ->
159 |     SourceQ =  <<"source-queue">>,
160 |     DestQ =  <<"dest-queue">>,
161 |     ok = setup_amqp10_source_shovel(Config, SourceQ, DestQ, AckMode),
162 |     Chan = rabbit_ct_client_helpers:open_channel(Config, 0),
163 |     CTag = consume(Chan, DestQ, AckMode =:= no_ack),
164 |     Msg = #amqp_msg{payload = <<42>>,
165 |                     props = #'P_basic'{delivery_mode = 2,
166 |                                        content_type = ?UNSHOVELLED}},
167 |     % publish to source
168 |     publish(Chan, Msg, <<>>, SourceQ),
169 | 
170 |     receive
171 |         {#'basic.deliver'{consumer_tag = CTag, delivery_tag = AckTag},
172 |          #amqp_msg{payload = <<42>>,
173 |                    props = #'P_basic'{%delivery_mode = 2,
174 |                                       %content_type = ?SHOVELLED,
175 |                                       headers = [{<<"x-shovelled">>, _, _},
176 |                                                  {<<"x-shovelled-timestamp">>,
177 |                                                   long, _}]}}} ->
178 |             case AckMode of
179 |                 no_ack -> ok;
180 |                 _      -> ok = amqp_channel:call(
181 |                                  Chan, #'basic.ack'{delivery_tag = AckTag})
182 |             end
183 |     after ?TIMEOUT -> throw(timeout_waiting_for_deliver1)
184 |     end,
185 | 
186 |     [{test_shovel, static, {running, _Info}, _Time}] =
187 |         rabbit_ct_broker_helpers:rpc(Config, 0,
188 |           rabbit_shovel_status, status, []),
189 |     rabbit_ct_client_helpers:close_channel(Chan).
190 | 
191 | setup_amqp10_source_shovel(Config, SourceQueue, DestQueue, AckMode) ->
192 |     Hostname = ?config(rmq_hostname, Config),
193 |     Port = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp),
194 |     Shovel = [{test_shovel,
195 |                [{source,
196 |                  [{protocol, amqp10},
197 |                   {uris, [rabbit_misc:format("amqp://~s:~b",
198 |                                              [Hostname, Port])]},
199 |                   {source_address, SourceQueue}]
200 |                 },
201 |                 {destination,
202 |                  [{uris, [rabbit_misc:format("amqp://~s:~b/%2f?heartbeat=5",
203 |                                              [Hostname, Port])]},
204 |                   {declarations,
205 |                    [{'queue.declare', [{queue, DestQueue}, auto_delete]}]},
206 |                   {publish_fields, [{exchange, <<>>},
207 |                                     {routing_key, DestQueue}]},
208 |                   {publish_properties, [{delivery_mode, 2},
209 |                                         {content_type,  ?SHOVELLED}]},
210 |                   {add_forward_headers, true},
211 |                   {add_timestamp_header, true}]},
212 |                 {queue, <<>>},
213 |                 {ack_mode, AckMode}
214 |                ]}],
215 |     ok = rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, setup_shovel,
216 |                                       [Shovel]).
217 | 
218 | setup_amqp10_destination_shovel(Config, Queue, AckMode) ->
219 |     Hostname = ?config(rmq_hostname, Config),
220 |     Port = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp),
221 |     Shovel = [{test_shovel,
222 |                [{source,
223 |                  [{uris, [rabbit_misc:format("amqp://~s:~b/%2f?heartbeat=5",
224 |                                              [Hostname, Port])]},
225 |                   {declarations,
226 |                    [{'queue.declare', [exclusive, auto_delete]},
227 |                     {'exchange.declare', [{exchange, ?EXCHANGE}, auto_delete]},
228 |                     {'queue.bind', [{queue, <<>>}, {exchange, ?EXCHANGE},
229 |                                     {routing_key, ?TO_SHOVEL}]}]},
230 |                   {queue, <<>>}]},
231 |                 {destination,
232 |                  [{protocol, amqp10},
233 |                   {uris, [rabbit_misc:format("amqp://~s:~b",
234 |                                              [Hostname, Port])]},
235 |                   {add_forward_headers, true},
236 |                   {add_timestamp_header, true},
237 |                   {target_address, Queue}]
238 |                 },
239 |                 {ack_mode, AckMode}]}],
240 |     ok = rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, setup_shovel,
241 |                                       [Shovel]).
242 | setup_amqp10_shovel(Config, SourceQueue, DestQueue, AckMode) ->
243 |     Hostname = ?config(rmq_hostname, Config),
244 |     Port = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp),
245 |     Shovel = [{test_shovel,
246 |                [{source,
247 |                  [{protocol, amqp10},
248 |                   {uris, [rabbit_misc:format("amqp://~s:~b",
249 |                                              [Hostname, Port])]},
250 |                   {source_address, SourceQueue}]},
251 |                 {destination,
252 |                  [{protocol, amqp10},
253 |                   {uris, [rabbit_misc:format("amqp://~s:~b",
254 |                                              [Hostname, Port])]},
255 |                   {add_forward_headers, true},
256 |                   {add_timestamp_header, true},
257 |                   {target_address, DestQueue}]
258 |                 },
259 |                 {ack_mode, AckMode}]}],
260 |     ok = rabbit_ct_broker_helpers:rpc(Config, 0, ?MODULE, setup_shovel,
261 |                                       [Shovel]).
262 | 
263 | setup_shovel(ShovelConfig) ->
264 |     _ = application:stop(rabbitmq_shovel),
265 |     application:set_env(rabbitmq_shovel, shovels, ShovelConfig, infinity),
266 |     ok = application:start(rabbitmq_shovel),
267 |     await_running_shovel(test_shovel).
268 | 
269 | await_running_shovel(Name) ->
270 |     case [N || {N, _, {running, _}, _}
271 |                       <- rabbit_shovel_status:status(),
272 |                          N =:= Name] of
273 |         [_] -> ok;
274 |         _   -> timer:sleep(100),
275 |                await_running_shovel(Name)
276 |     end.
277 | 
278 | consume(Chan, Queue, NoAck) ->
279 |     #'basic.consume_ok'{consumer_tag = CTag} =
280 |         amqp_channel:subscribe(Chan, #'basic.consume'{queue = Queue,
281 |                                                       no_ack = NoAck,
282 |                                                       exclusive = false},
283 |                                self()),
284 |     receive
285 |         #'basic.consume_ok'{consumer_tag = CTag} -> ok
286 |     after ?TIMEOUT -> throw(timeout_waiting_for_consume_ok)
287 |     end,
288 |     CTag.
289 | 
290 | publish(Chan, Msg, Exchange, RoutingKey) ->
291 |     ok = amqp_channel:call(Chan, #'basic.publish'{exchange = Exchange,
292 |                                                   routing_key = RoutingKey},
293 |                            Msg).
294 | 


--------------------------------------------------------------------------------
/test/amqp10_dynamic_SUITE.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(amqp10_dynamic_SUITE).
  9 | 
 10 | -include_lib("common_test/include/ct.hrl").
 11 | -include_lib("eunit/include/eunit.hrl").
 12 | -include_lib("amqp_client/include/amqp_client.hrl").
 13 | 
 14 | -compile(export_all).
 15 | 
 16 | all() ->
 17 |     [
 18 |       {group, non_parallel_tests},
 19 |       {group, with_map_config}
 20 |     ].
 21 | 
 22 | groups() ->
 23 |     [
 24 |       {non_parallel_tests, [], [
 25 |           simple,
 26 |           change_definition,
 27 |           autodelete_amqp091_src_on_confirm,
 28 |           autodelete_amqp091_src_on_publish,
 29 |           autodelete_amqp091_dest_on_confirm,
 30 |           autodelete_amqp091_dest_on_publish,
 31 |           simple_amqp10_dest,
 32 |           simple_amqp10_src
 33 |         ]},
 34 |       {with_map_config, [], [
 35 |           simple,
 36 |           simple_amqp10_dest,
 37 |           simple_amqp10_src
 38 |       ]}
 39 |     ].
 40 | 
 41 | %% -------------------------------------------------------------------
 42 | %% Testsuite setup/teardown.
 43 | %% -------------------------------------------------------------------
 44 | 
 45 | init_per_suite(Config0) ->
 46 |     {ok, _} = application:ensure_all_started(amqp10_client),
 47 |     rabbit_ct_helpers:log_environment(),
 48 |     Config = rabbit_ct_helpers:merge_app_env(Config0,
 49 |                                              [{lager, [{error_logger_hwm, 200}]}]),
 50 |     Config1 = rabbit_ct_helpers:set_config(Config, [
 51 |         {rmq_nodename_suffix, ?MODULE}
 52 |       ]),
 53 |     rabbit_ct_helpers:run_setup_steps(Config1,
 54 |       rabbit_ct_broker_helpers:setup_steps() ++
 55 |       rabbit_ct_client_helpers:setup_steps()).
 56 | 
 57 | end_per_suite(Config) ->
 58 |     application:stop(amqp10_client),
 59 |     rabbit_ct_helpers:run_teardown_steps(Config,
 60 |       rabbit_ct_client_helpers:teardown_steps() ++
 61 |       rabbit_ct_broker_helpers:teardown_steps()).
 62 | 
 63 | init_per_group(with_map_config, Config) ->
 64 |     rabbit_ct_helpers:set_config(Config, [{map_config, true}]);
 65 | init_per_group(_, Config) ->
 66 |     rabbit_ct_helpers:set_config(Config, [{map_config, false}]).
 67 | 
 68 | end_per_group(_, Config) ->
 69 |     Config.
 70 | 
 71 | init_per_testcase(Testcase, Config0) ->
 72 |     SrcQ = list_to_binary(atom_to_list(Testcase) ++ "_src"),
 73 |     DestQ = list_to_binary(atom_to_list(Testcase) ++ "_dest"),
 74 |     DestQ2 = list_to_binary(atom_to_list(Testcase) ++ "_dest2"),
 75 |     Config = [{srcq, SrcQ}, {destq, DestQ}, {destq2, DestQ2} | Config0],
 76 |     rabbit_ct_helpers:testcase_started(Config, Testcase).
 77 | 
 78 | end_per_testcase(Testcase, Config) ->
 79 |     rabbit_ct_helpers:testcase_finished(Config, Testcase).
 80 | 
 81 | %% -------------------------------------------------------------------
 82 | %% Testcases.
 83 | %% -------------------------------------------------------------------
 84 | 
 85 | simple(Config) ->
 86 |     Src = ?config(srcq, Config),
 87 |     Dest = ?config(destq, Config),
 88 |     with_session(Config,
 89 |       fun (Sess) ->
 90 |               test_amqp10_destination(Config, Src, Dest, Sess, <<"amqp10">>,
 91 |                                       <<"src-address">>)
 92 |       end).
 93 | 
 94 | simple_amqp10_dest(Config) ->
 95 |     Src = ?config(srcq, Config),
 96 |     Dest = ?config(destq, Config),
 97 |     with_session(Config,
 98 |       fun (Sess) ->
 99 |               test_amqp10_destination(Config, Src, Dest, Sess, <<"amqp091">>,
100 |                                       <<"src-queue">>)
101 |       end).
102 | 
103 | test_amqp10_destination(Config, Src, Dest, Sess, Protocol, ProtocolSrc) ->
104 |     MapConfig = ?config(map_config, Config),
105 |     shovel_test_utils:set_param(Config, <<"test">>,
106 |                                 [{<<"src-protocol">>, Protocol},
107 |                                  {ProtocolSrc, Src},
108 |                                  {<<"dest-protocol">>, <<"amqp10">>},
109 |                                  {<<"dest-address">>, Dest},
110 |                                  {<<"dest-add-forward-headers">>, true},
111 |                                  {<<"dest-add-timestamp-header">>, true},
112 |                                  {<<"dest-application-properties">>,
113 |                                   case MapConfig of
114 |                                      true ->
115 |                                         #{<<"app-prop-key">> => <<"app-prop-value">>};
116 |                                      _ ->
117 |                                         [{<<"app-prop-key">>, <<"app-prop-value">>}]
118 |                                   end},
119 |                                  {<<"dest-properties">>,
120 |                                   case MapConfig of
121 |                                      true ->
122 |                                          #{<<"user_id">> => <<"guest">>};
123 |                                      _ ->
124 |                                          [{<<"user_id">>, <<"guest">>}]
125 |                                   end},
126 |                                  {<<"dest-message-annotations">>,
127 |                                   case MapConfig of
128 |                                      true ->
129 |                                          #{<<"message-ann-key">> =>
130 |                                                <<"message-ann-value">>};
131 |                                      _ ->
132 |                                          [{<<"message-ann-key">>,
133 |                                            <<"message-ann-value">>}]
134 |                                   end}]),
135 |     Msg = publish_expect(Sess, Src, Dest, <<"tag1">>, <<"hello">>),
136 |     ?assertMatch((#{user_id := <<"guest">>, creation_time := _}),
137 |                  (amqp10_msg:properties(Msg))),
138 |     ?assertMatch((#{<<"shovel-name">> := <<"test">>,
139 |                     <<"shovel-type">> := <<"dynamic">>, <<"shovelled-by">> := _,
140 |                     <<"app-prop-key">> := <<"app-prop-value">>}),
141 |                  (amqp10_msg:application_properties(Msg))),
142 |     ?assertMatch((#{<<"message-ann-key">> := <<"message-ann-value">>}),
143 |                  (amqp10_msg:message_annotations(Msg))).
144 | 
145 | simple_amqp10_src(Config) ->
146 |     MapConfig = ?config(map_config, Config),
147 |     Src = ?config(srcq, Config),
148 |     Dest = ?config(destq, Config),
149 |     with_session(Config,
150 |       fun (Sess) ->
151 |               shovel_test_utils:set_param(
152 |                 Config,
153 |                 <<"test">>, [{<<"src-protocol">>, <<"amqp10">>},
154 |                              {<<"src-address">>,  Src},
155 |                              {<<"dest-protocol">>, <<"amqp091">>},
156 |                              {<<"dest-queue">>, Dest},
157 |                              {<<"add-forward-headers">>, true},
158 |                              {<<"dest-add-timestamp-header">>, true},
159 |                              {<<"publish-properties">>,
160 |                                 case MapConfig of
161 |                                     true -> #{<<"cluster_id">> => <<"x">>};
162 |                                     _    -> [{<<"cluster_id">>, <<"x">>}]
163 |                                 end}
164 |                             ]),
165 |               _Msg = publish_expect(Sess, Src, Dest, <<"tag1">>,
166 |                                     <<"hello">>),
167 |               % the fidelity loss is quite high when consuming using the amqp10
168 |               % plugin. For example custom headers aren't current translated.
169 |               % This isn't due to the shovel though.
170 |               ok
171 |       end).
172 | 
173 | change_definition(Config) ->
174 |     Src = ?config(srcq, Config),
175 |     Dest = ?config(destq, Config),
176 |     Dest2 = ?config(destq2, Config),
177 |     with_session(Config,
178 |       fun (Sess) ->
179 |               shovel_test_utils:set_param(Config, <<"test">>,
180 |                                           [{<<"src-address">>,  Src},
181 |                                            {<<"src-protocol">>, <<"amqp10">>},
182 |                                            {<<"dest-protocol">>, <<"amqp10">>},
183 |                                            {<<"dest-address">>, Dest}]),
184 |               publish_expect(Sess, Src, Dest, <<"tag2">>,<<"hello">>),
185 |               shovel_test_utils:set_param(Config, <<"test">>,
186 |                                           [{<<"src-address">>,  Src},
187 |                                            {<<"src-protocol">>, <<"amqp10">>},
188 |                                            {<<"dest-protocol">>, <<"amqp10">>},
189 |                                            {<<"dest-address">>, Dest2}]),
190 |               publish_expect(Sess, Src, Dest2, <<"tag3">>, <<"hello">>),
191 |               expect_empty(Sess, Dest),
192 |               shovel_test_utils:clear_param(Config, <<"test">>),
193 |               publish_expect(Sess, Src, Src, <<"tag4">>, <<"hello2">>),
194 |               expect_empty(Sess, Dest),
195 |               expect_empty(Sess, Dest2)
196 |       end).
197 | 
198 | autodelete_amqp091_src_on_confirm(Config) ->
199 |     autodelete_case(Config, {<<"on-confirm">>, 50, 50, 50},
200 |                     fun autodelete_amqp091_src/2),
201 |     ok.
202 | 
203 | autodelete_amqp091_src_on_publish(Config) ->
204 |     autodelete_case(Config, {<<"on-publish">>, 50, 50, 50},
205 |                     fun autodelete_amqp091_src/2),
206 |     ok.
207 | 
208 | autodelete_amqp091_dest_on_confirm(Config) ->
209 |     autodelete_case(Config, {<<"on-confirm">>, 50, 50, 50},
210 |                     fun autodelete_amqp091_dest/2),
211 |     ok.
212 | 
213 | autodelete_amqp091_dest_on_publish(Config) ->
214 |     autodelete_case(Config, {<<"on-publish">>, 50, 50, 50},
215 |                     fun autodelete_amqp091_dest/2),
216 |     ok.
217 | 
218 | autodelete_case(Config, Args, CaseFun) ->
219 |     with_session(Config, CaseFun(Config, Args)).
220 | 
221 | autodelete_do(Config, {AckMode, After, ExpSrc, ExpDest}) ->
222 |     Src = ?config(srcq, Config),
223 |     Dest = ?config(destq, Config),
224 |     fun (Session) ->
225 |             publish_count(Session, Src, <<"hello">>, 100),
226 |             shovel_test_utils:set_param_nowait(
227 |               Config,
228 |               <<"test">>, [{<<"src-address">>,    Src},
229 |                            {<<"src-protocol">>,   <<"amqp10">>},
230 |                            {<<"src-delete-after">>, After},
231 |                            {<<"src-prefetch-count">>, 5},
232 |                            {<<"dest-address">>,   Dest},
233 |                            {<<"dest-protocol">>,   <<"amqp10">>},
234 |                            {<<"ack-mode">>,     AckMode}
235 |                           ]),
236 |             await_autodelete(Config, <<"test">>),
237 |             expect_count(Session, Dest, <<"hello">>, ExpDest),
238 |             expect_count(Session, Src, <<"hello">>, ExpSrc)
239 |     end.
240 | 
241 | autodelete_amqp091_src(Config, {AckMode, After, ExpSrc, ExpDest}) ->
242 |     Src = ?config(srcq, Config),
243 |     Dest = ?config(destq, Config),
244 |     fun (Session) ->
245 |             publish_count(Session, Src, <<"hello">>, 100),
246 |             shovel_test_utils:set_param_nowait(
247 |               Config,
248 |               <<"test">>, [{<<"src-queue">>, Src},
249 |                            {<<"src-protocol">>, <<"amqp091">>},
250 |                            {<<"src-delete-after">>, After},
251 |                            {<<"src-prefetch-count">>, 5},
252 |                            {<<"dest-address">>, Dest},
253 |                            {<<"dest-protocol">>, <<"amqp10">>},
254 |                            {<<"ack-mode">>, AckMode}
255 |                           ]),
256 |             await_autodelete(Config, <<"test">>),
257 |             expect_count(Session, Dest, <<"hello">>, ExpDest),
258 |             expect_count(Session, Src, <<"hello">>, ExpSrc)
259 |     end.
260 | 
261 | autodelete_amqp091_dest(Config, {AckMode, After, ExpSrc, ExpDest}) ->
262 |     Src = ?config(srcq, Config),
263 |     Dest = ?config(destq, Config),
264 |     fun (Session) ->
265 |             publish_count(Session, Src, <<"hello">>, 100),
266 |             shovel_test_utils:set_param_nowait(
267 |               Config,
268 |               <<"test">>, [{<<"src-address">>, Src},
269 |                            {<<"src-protocol">>, <<"amqp10">>},
270 |                            {<<"src-delete-after">>, After},
271 |                            {<<"src-prefetch-count">>, 5},
272 |                            {<<"dest-queue">>, Dest},
273 |                            {<<"dest-protocol">>, <<"amqp091">>},
274 |                            {<<"ack-mode">>, AckMode}
275 |                           ]),
276 |             await_autodelete(Config, <<"test">>),
277 |             expect_count(Session, Dest, <<"hello">>, ExpDest),
278 |             expect_count(Session, Src, <<"hello">>, ExpSrc)
279 |     end.
280 | 
281 | %%----------------------------------------------------------------------------
282 | 
283 | with_session(Config, Fun) ->
284 |     Hostname = ?config(rmq_hostname, Config),
285 |     Port = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp),
286 |     {ok, Conn} = amqp10_client:open_connection(Hostname, Port),
287 |     {ok, Sess} = amqp10_client:begin_session(Conn),
288 |     Fun(Sess),
289 |     amqp10_client:close_connection(Conn),
290 |     ok.
291 | 
292 | publish(Sender, Tag, Payload) when is_binary(Payload) ->
293 |     Headers = #{durable => true},
294 |     Msg = amqp10_msg:set_headers(Headers,
295 |                                  amqp10_msg:new(Tag, Payload, false)),
296 |     ok = amqp10_client:send_msg(Sender, Msg),
297 |     receive
298 |         {amqp10_disposition, {accepted, Tag}} -> ok
299 |     after 3000 ->
300 |               exit(publish_disposition_not_received)
301 |     end.
302 | 
303 | publish_expect(Session, Source, Dest, Tag, Payload) ->
304 |     LinkName = <<"dynamic-sender-", Dest/binary>>,
305 |     {ok, Sender} = amqp10_client:attach_sender_link(Session, LinkName, Source,
306 |                                                     unsettled, unsettled_state),
307 |     ok = await_amqp10_event(link, Sender, attached),
308 |     publish(Sender, Tag, Payload),
309 |     amqp10_client:detach_link(Sender),
310 |     expect_one(Session, Dest, Payload).
311 | 
312 | await_amqp10_event(On, Ref, Evt) ->
313 |     receive
314 |         {amqp10_event, {On, Ref, Evt}} -> ok
315 |     after 5000 ->
316 |           exit({amqp10_event_timeout, On, Ref, Evt})
317 |     end.
318 | 
319 | expect_one(Session, Dest, Payload) ->
320 |     LinkName = <<"dynamic-receiver-", Dest/binary>>,
321 |     {ok, Receiver} = amqp10_client:attach_receiver_link(Session, LinkName,
322 |                                                         Dest, settled,
323 |                                                         unsettled_state),
324 |     ok = amqp10_client:flow_link_credit(Receiver, 1, never),
325 |     Msg = expect(Receiver, Payload),
326 |     amqp10_client:detach_link(Receiver),
327 |     Msg.
328 | 
329 | expect(Receiver, _Payload) ->
330 |     receive
331 |         {amqp10_msg, Receiver, InMsg} ->
332 |             InMsg
333 |     after 4000 ->
334 |               throw(timeout_in_expect_waiting_for_delivery)
335 |     end.
336 | 
337 | expect_empty(Session, Dest) ->
338 |     {ok, Receiver} = amqp10_client:attach_receiver_link(Session,
339 |                                                         <<"dynamic-receiver">>,
340 |                                                         Dest, settled,
341 |                                                         unsettled_state),
342 |     % probably good enough given we don't currently have a means of
343 |     % echoing flow state
344 |     {error, timeout} = amqp10_client:get_msg(Receiver, 250),
345 |     amqp10_client:detach_link(Receiver).
346 | 
347 | publish_count(Session, Address, Payload, Count) ->
348 |     LinkName = <<"dynamic-sender-", Address/binary>>,
349 |     {ok, Sender} = amqp10_client:attach_sender_link(Session, LinkName,
350 |                                                     Address, unsettled,
351 |                                                     unsettled_state),
352 |     ok = await_amqp10_event(link, Sender, attached),
353 |     [begin
354 |          Tag = rabbit_data_coercion:to_binary(I),
355 |          publish(Sender, Tag, <>)
356 |      end || I <- lists:seq(1, Count)],
357 |      amqp10_client:detach_link(Sender).
358 | 
359 | expect_count(Session, Address, Payload, Count) ->
360 |     {ok, Receiver} = amqp10_client:attach_receiver_link(Session,
361 |                                                         <<"dynamic-receiver",
362 |                                                           Address/binary>>,
363 |                                                         Address, settled,
364 |                                                         unsettled_state),
365 |     ok = amqp10_client:flow_link_credit(Receiver, Count, never),
366 |     [begin
367 |          expect(Receiver, Payload)
368 |      end || _ <- lists:seq(1, Count)],
369 |     expect_empty(Session, Address),
370 |     amqp10_client:detach_link(Receiver).
371 | 
372 | 
373 | invalid_param(Config, Value, User) ->
374 |     {error_string, _} = rabbit_ct_broker_helpers:rpc(Config, 0,
375 |       rabbit_runtime_parameters, set,
376 |       [<<"/">>, <<"shovel">>, <<"invalid">>, Value, User]).
377 | 
378 | valid_param(Config, Value, User) ->
379 |     rabbit_ct_broker_helpers:rpc(Config, 0,
380 |       ?MODULE, valid_param1, [Config, Value, User]).
381 | 
382 | valid_param1(_Config, Value, User) ->
383 |     ok = rabbit_runtime_parameters:set(
384 |            <<"/">>, <<"shovel">>, <<"a">>, Value, User),
385 |     ok = rabbit_runtime_parameters:clear(<<"/">>, <<"shovel">>, <<"a">>, <<"acting-user">>).
386 | 
387 | invalid_param(Config, Value) -> invalid_param(Config, Value, none).
388 | valid_param(Config, Value) -> valid_param(Config, Value, none).
389 | 
390 | lookup_user(Config, Name) ->
391 |     {ok, User} = rabbit_ct_broker_helpers:rpc(Config, 0,
392 |       rabbit_access_control, check_user_login, [Name, []]),
393 |     User.
394 | 
395 | await_autodelete(Config, Name) ->
396 |     rabbit_ct_broker_helpers:rpc(Config, 0,
397 |       ?MODULE, await_autodelete1, [Config, Name], 10000).
398 | 
399 | await_autodelete1(_Config, Name) ->
400 |     shovel_test_utils:await(
401 |       fun () -> not lists:member(Name, shovels_from_parameters()) end),
402 |     shovel_test_utils:await(
403 |       fun () ->
404 |               not lists:member(Name,
405 |                                shovel_test_utils:shovels_from_status())
406 |       end).
407 | 
408 | shovels_from_parameters() ->
409 |     L = rabbit_runtime_parameters:list(<<"/">>, <<"shovel">>),
410 |     [rabbit_misc:pget(name, Shovel) || Shovel <- L].
411 | 


--------------------------------------------------------------------------------
/test/amqp10_shovel_SUITE.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(amqp10_shovel_SUITE).
  9 | 
 10 | -compile(export_all).
 11 | 
 12 | -export([
 13 |          ]).
 14 | 
 15 | -include_lib("common_test/include/ct.hrl").
 16 | -include_lib("eunit/include/eunit.hrl").
 17 | -include_lib("amqp10_common/include/amqp10_framing.hrl").
 18 | 
 19 | %%%===================================================================
 20 | %%% Common Test callbacks
 21 | %%%===================================================================
 22 | 
 23 | all() ->
 24 |     [
 25 |      {group, tests}
 26 |     ].
 27 | 
 28 | 
 29 | all_tests() ->
 30 |     [
 31 |      amqp_encoded_data_list,
 32 |      amqp_encoded_amqp_value
 33 |     ].
 34 | 
 35 | groups() ->
 36 |     [
 37 |      {tests, [], all_tests()}
 38 |     ].
 39 | 
 40 | init_per_suite(Config) ->
 41 |     Config.
 42 | 
 43 | end_per_suite(_Config) ->
 44 |     ok.
 45 | 
 46 | init_per_group(_Group, Config) ->
 47 |     Config.
 48 | 
 49 | end_per_group(_Group, _Config) ->
 50 |     ok.
 51 | 
 52 | init_per_testcase(_TestCase, Config) ->
 53 |     Config.
 54 | 
 55 | end_per_testcase(_TestCase, _Config) ->
 56 |     meck:unload(),
 57 |     ok.
 58 | 
 59 | %%%===================================================================
 60 | %%% Test cases
 61 | %%%===================================================================
 62 | 
 63 | amqp_encoded_data_list(_Config) ->
 64 |     meck:new(rabbit_shovel_behaviour, [passthrough]),
 65 |     meck:expect(rabbit_shovel_behaviour, forward,
 66 |                 fun (_, _, Pay, S) ->
 67 |                         ?assert(erlang:is_binary(Pay)),
 68 |                         S
 69 |                 end),
 70 |     %% fake some shovel state
 71 |     State = #{source => #{},
 72 |               dest => #{module => rabbit_amqp10_shovel},
 73 |               ack_mode => no_ack},
 74 |     Body = [
 75 |             #'v1_0.data'{content = <<"one">>},
 76 |             #'v1_0.data'{content = <<"two">>}
 77 |            ],
 78 |     Msg = amqp10_msg:new(55, Body),
 79 |     rabbit_amqp10_shovel:handle_source({amqp10_msg, linkref, Msg}, State),
 80 | 
 81 |     ?assert(meck:validate(rabbit_shovel_behaviour)),
 82 |     ok.
 83 | 
 84 | amqp_encoded_amqp_value(_Config) ->
 85 |     meck:new(rabbit_shovel_behaviour, [passthrough]),
 86 |     meck:expect(rabbit_shovel_behaviour, forward,
 87 |                 fun (_, _, Pay, S) ->
 88 |                         ?assert(erlang:is_binary(Pay)),
 89 |                         S
 90 |                 end),
 91 |     %% fake some shovel state
 92 |     State = #{source => #{},
 93 |               dest => #{module => rabbit_amqp10_shovel},
 94 |               ack_mode => no_ack},
 95 |     Body = #'v1_0.amqp_value'{content = {utf8, <<"hi">>}},
 96 |     Msg = amqp10_msg:new(55, Body),
 97 |     rabbit_amqp10_shovel:handle_source({amqp10_msg, linkref, Msg}, State),
 98 | 
 99 |     ?assert(meck:validate(rabbit_shovel_behaviour)),
100 |     ok.
101 | 
102 | %% Utility
103 | 


--------------------------------------------------------------------------------
/test/config_SUITE.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(config_SUITE).
  9 | 
 10 | -compile(export_all).
 11 | 
 12 | -include_lib("eunit/include/eunit.hrl").
 13 | 
 14 | -define(EXCHANGE,    <<"test_exchange">>).
 15 | -define(TO_SHOVEL,   <<"to_the_shovel">>).
 16 | -define(FROM_SHOVEL, <<"from_the_shovel">>).
 17 | -define(UNSHOVELLED, <<"unshovelled">>).
 18 | -define(SHOVELLED,   <<"shovelled">>).
 19 | -define(TIMEOUT,     1000).
 20 | 
 21 | all() ->
 22 |     [
 23 |       {group, tests}
 24 |     ].
 25 | 
 26 | groups() ->
 27 |     [
 28 |       {tests, [parallel], [
 29 |           parse_amqp091,
 30 |           parse_amqp10_mixed
 31 |         ]}
 32 |     ].
 33 | 
 34 | %% -------------------------------------------------------------------
 35 | %% Testsuite setup/teardown.
 36 | %% -------------------------------------------------------------------
 37 | 
 38 | init_per_suite(Config) ->
 39 |     Config.
 40 | 
 41 | end_per_suite(Config) ->
 42 |     Config.
 43 | 
 44 | init_per_group(_, Config) ->
 45 |     Config.
 46 | 
 47 | end_per_group(_, Config) ->
 48 |     Config.
 49 | 
 50 | init_per_testcase(_Testcase, Config) -> Config.
 51 | 
 52 | end_per_testcase(_Testcase, Config) -> Config.
 53 | 
 54 | 
 55 | %% -------------------------------------------------------------------
 56 | %% Testcases.
 57 | %% -------------------------------------------------------------------
 58 | 
 59 | parse_amqp091(_Config) ->
 60 |     Amqp091Src = {source, [{protocol, amqp091},
 61 |                            {uris, ["ampq://myhost:5672/vhost"]},
 62 |                            {declarations, []},
 63 |                            {queue, <<"the-queue">>},
 64 |                            {delete_after, never},
 65 |                            {prefetch_count, 10}]},
 66 |     Amqp091Dst = {destination, [{protocol, amqp091},
 67 |                                 {uris, ["ampq://myhost:5672"]},
 68 |                                 {declarations, []},
 69 |                                 {publish_properties, [{delivery_mode, 1}]},
 70 |                                 {publish_fields, []},
 71 |                                 {add_forward_headers, true}]},
 72 |     In = [Amqp091Src,
 73 |           Amqp091Dst,
 74 |           {ack_mode, on_confirm},
 75 |           {reconnect_delay, 2}],
 76 | 
 77 |     ?assertMatch(
 78 |        {ok, #{name := my_shovel,
 79 |               ack_mode := on_confirm,
 80 |               reconnect_delay := 2,
 81 |               dest := #{module := rabbit_amqp091_shovel,
 82 |                         uris := ["ampq://myhost:5672"],
 83 |                         fields_fun := _PubFields,
 84 |                         props_fun := _PubProps,
 85 |                         resource_decl := _DDecl,
 86 |                         add_timestamp_header := false,
 87 |                         add_forward_headers := true},
 88 |               source := #{module := rabbit_amqp091_shovel,
 89 |                           uris := ["ampq://myhost:5672/vhost"],
 90 |                           queue := <<"the-queue">>,
 91 |                           prefetch_count := 10,
 92 |                           delete_after := never,
 93 |                           resource_decl := _SDecl}}},
 94 |         rabbit_shovel_config:parse(my_shovel, In)),
 95 |     ok.
 96 | 
 97 | parse_amqp10_mixed(_Config) ->
 98 |     Amqp10Src = {source, [{protocol, amqp10},
 99 |                           {uris, ["ampq://myotherhost:5672"]},
100 |                           {source_address, <<"the-queue">>}
101 |                          ]},
102 |     Amqp10Dst = {destination, [{protocol, amqp10},
103 |                                {uris, ["ampq://myhost:5672"]},
104 |                                {target_address, <<"targe-queue">>},
105 |                                {message_annotations, [{soma_ann, <<"some-info">>}]},
106 |                                {properties, [{user_id, <<"some-user">>}]},
107 |                                {application_properties, [{app_prop_key, <<"app_prop_value">>}]},
108 |                                {add_forward_headers, true}
109 |                               ]},
110 |     In = [Amqp10Src,
111 |           Amqp10Dst,
112 |           {ack_mode, on_confirm},
113 |           {reconnect_delay, 2}],
114 | 
115 |     ?assertMatch(
116 |        {ok, #{name := my_shovel,
117 |               ack_mode := on_confirm,
118 |               source := #{module := rabbit_amqp10_shovel,
119 |                           uris := ["ampq://myotherhost:5672"],
120 |                           source_address := <<"the-queue">>
121 |                           },
122 |               dest := #{module := rabbit_amqp10_shovel,
123 |                         uris := ["ampq://myhost:5672"],
124 |                         target_address := <<"targe-queue">>,
125 |                         properties := #{user_id := <<"some-user">>},
126 |                         application_properties := #{app_prop_key := <<"app_prop_value">>},
127 |                         message_annotations := #{soma_ann := <<"some-info">>},
128 |                         add_forward_headers := true}}},
129 |         rabbit_shovel_config:parse(my_shovel, In)),
130 |     ok.
131 | 


--------------------------------------------------------------------------------
/test/configuration_SUITE.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(configuration_SUITE).
  9 | 
 10 | -include_lib("common_test/include/ct.hrl").
 11 | -include_lib("amqp_client/include/amqp_client.hrl").
 12 | 
 13 | -compile(export_all).
 14 | 
 15 | -define(EXCHANGE,    <<"test_exchange">>).
 16 | -define(TO_SHOVEL,   <<"to_the_shovel">>).
 17 | -define(FROM_SHOVEL, <<"from_the_shovel">>).
 18 | -define(UNSHOVELLED, <<"unshovelled">>).
 19 | -define(SHOVELLED,   <<"shovelled">>).
 20 | -define(TIMEOUT,     1000).
 21 | 
 22 | all() ->
 23 |     [
 24 |       {group, non_parallel_tests}
 25 |     ].
 26 | 
 27 | groups() ->
 28 |     [
 29 |       {non_parallel_tests, [], [
 30 |           zero_shovels,
 31 |           invalid_legacy_configuration,
 32 |           valid_legacy_configuration,
 33 |           valid_configuration
 34 |         ]}
 35 |     ].
 36 | 
 37 | %% -------------------------------------------------------------------
 38 | %% Testsuite setup/teardown.
 39 | %% -------------------------------------------------------------------
 40 | 
 41 | init_per_suite(Config) ->
 42 |     rabbit_ct_helpers:log_environment(),
 43 |     Config1 = rabbit_ct_helpers:set_config(Config, [
 44 |         {rmq_nodename_suffix, ?MODULE}
 45 |       ]),
 46 |     rabbit_ct_helpers:run_setup_steps(Config1,
 47 |       rabbit_ct_broker_helpers:setup_steps() ++
 48 |       rabbit_ct_client_helpers:setup_steps() ++
 49 |       [fun stop_shovel_plugin/1]).
 50 | 
 51 | end_per_suite(Config) ->
 52 |     rabbit_ct_helpers:run_teardown_steps(Config,
 53 |       rabbit_ct_client_helpers:teardown_steps() ++
 54 |       rabbit_ct_broker_helpers:teardown_steps()).
 55 | 
 56 | init_per_group(_, Config) ->
 57 |     Config.
 58 | 
 59 | end_per_group(_, Config) ->
 60 |     Config.
 61 | 
 62 | init_per_testcase(Testcase, Config) ->
 63 |     rabbit_ct_helpers:testcase_started(Config, Testcase).
 64 | 
 65 | end_per_testcase(Testcase, Config) ->
 66 |     rabbit_ct_helpers:testcase_finished(Config, Testcase).
 67 | 
 68 | stop_shovel_plugin(Config) ->
 69 |     ok = rabbit_ct_broker_helpers:rpc(Config, 0,
 70 |       application, stop, [rabbitmq_shovel]),
 71 |     Config.
 72 | 
 73 | %% -------------------------------------------------------------------
 74 | %% Testcases.
 75 | %% -------------------------------------------------------------------
 76 | 
 77 | zero_shovels(Config) ->
 78 |     passed = rabbit_ct_broker_helpers:rpc(Config, 0,
 79 |       ?MODULE, zero_shovels1, [Config]).
 80 | 
 81 | zero_shovels1(_Config) ->
 82 |     %% shovel can be started with zero shovels configured
 83 |     ok = application:start(rabbitmq_shovel),
 84 |     ok = application:stop(rabbitmq_shovel),
 85 |     passed.
 86 | 
 87 | invalid_legacy_configuration(Config) ->
 88 |     passed = rabbit_ct_broker_helpers:rpc(Config, 0,
 89 |       ?MODULE, invalid_legacy_configuration1, [Config]).
 90 | 
 91 | invalid_legacy_configuration1(_Config) ->
 92 |     %% various ways of breaking the config
 93 |     require_list_of_shovel_configurations =
 94 |         test_broken_shovel_configs(invalid_config),
 95 | 
 96 |     require_list_of_shovel_configurations =
 97 |         test_broken_shovel_configs([{test_shovel, invalid_shovel_config}]),
 98 | 
 99 |     Config = [{sources, [{broker, "amqp://"}]},
100 |               {destinations, [{broker, "amqp://"}]},
101 |               {queue, <<"">>}],
102 | 
103 |     {duplicate_shovel_definition, test_shovel} =
104 |         test_broken_shovel_configs(
105 |           [{test_shovel, Config}, {test_shovel, Config}]),
106 | 
107 |     {invalid_parameters, [{invalid, invalid, invalid}]} =
108 |         test_broken_shovel_config([{invalid, invalid, invalid} | Config]),
109 | 
110 |     {duplicate_parameters, [queue]} =
111 |         test_broken_shovel_config([{queue, <<"">>} | Config]),
112 | 
113 |     {missing_parameter, _} =
114 |         test_broken_shovel_config([]),
115 | 
116 |     {require_list, invalid} =
117 |         test_broken_shovel_sources(invalid),
118 | 
119 |     {missing_parameter, broker} =
120 |         test_broken_shovel_sources([]),
121 | 
122 |     {require_list, brokers, invalid} =
123 |         test_broken_shovel_sources([{brokers, invalid}]),
124 | 
125 |     {expected_string_uri, 42} =
126 |         test_broken_shovel_sources([{brokers, [42]}]),
127 | 
128 |     {{unexpected_uri_scheme, "invalid"}, "invalid://"} =
129 |         test_broken_shovel_sources([{broker, "invalid://"}]),
130 | 
131 |     {{unable_to_parse_uri, no_scheme}, "invalid"} =
132 |         test_broken_shovel_sources([{broker, "invalid"}]),
133 | 
134 |     {require_list, invalid} =
135 |         test_broken_shovel_sources([{broker, "amqp://"},
136 |                                     {declarations, invalid}]),
137 |     {unknown_method_name, 42} =
138 |         test_broken_shovel_sources([{broker, "amqp://"},
139 |                                     {declarations, [42]}]),
140 | 
141 |     {expected_method_field_list, 'queue.declare', 42} =
142 |         test_broken_shovel_sources([{broker, "amqp://"},
143 |                                     {declarations, [{'queue.declare', 42}]}]),
144 | 
145 |     {unknown_fields, 'queue.declare', [invalid]} =
146 |         test_broken_shovel_sources(
147 |           [{broker, "amqp://"},
148 |            {declarations, [{'queue.declare', [invalid]}]}]),
149 | 
150 |     {{invalid_amqp_params_parameter, heartbeat, "text",
151 |       [{"heartbeat", "text"}], {not_an_integer, "text"}}, _} =
152 |         test_broken_shovel_sources(
153 |           [{broker, "amqp://localhost/?heartbeat=text"}]),
154 | 
155 |     {{invalid_amqp_params_parameter, username, "text",
156 |       [{"username", "text"}],
157 |       {parameter_unconfigurable_in_query, username, "text"}}, _} =
158 |         test_broken_shovel_sources([{broker, "amqp://?username=text"}]),
159 | 
160 |     {invalid_parameter_value, prefetch_count,
161 |      {require_non_negative_integer, invalid}} =
162 |         test_broken_shovel_config([{prefetch_count, invalid} | Config]),
163 | 
164 |     {invalid_parameter_value, ack_mode,
165 |      {ack_mode_value_requires_one_of,
166 |       {no_ack, on_publish, on_confirm}, invalid}} =
167 |         test_broken_shovel_config([{ack_mode, invalid} | Config]),
168 | 
169 |     {invalid_parameter_value, queue,
170 |      {require_binary, invalid}} =
171 |         test_broken_shovel_config([{sources, [{broker, "amqp://"}]},
172 |                                    {destinations, [{broker, "amqp://"}]},
173 |                                    {queue, invalid}]),
174 | 
175 |     {invalid_parameter_value, publish_properties,
176 |      {require_list, invalid}} =
177 |         test_broken_shovel_config([{publish_properties, invalid} | Config]),
178 | 
179 |     {invalid_parameter_value, publish_properties,
180 |      {unexpected_fields, [invalid], _}} =
181 |         test_broken_shovel_config([{publish_properties, [invalid]} | Config]),
182 | 
183 |     {{invalid_ssl_parameter, fail_if_no_peer_cert, "42", _,
184 |       {require_boolean, '42'}}, _} =
185 |         test_broken_shovel_sources([{broker, "amqps://username:password@host:5673/vhost?cacertfile=/path/to/cacert.pem&certfile=/path/to/certfile.pem&keyfile=/path/to/keyfile.pem&verify=verify_peer&fail_if_no_peer_cert=42"}]),
186 | 
187 |     passed.
188 | 
189 | test_broken_shovel_configs(Configs) ->
190 |     application:set_env(rabbitmq_shovel, shovels, Configs),
191 |     {error, {Error, _}} = application:start(rabbitmq_shovel),
192 |     Error.
193 | 
194 | test_broken_shovel_config(Config) ->
195 |     {invalid_shovel_configuration, test_shovel, Error} =
196 |         test_broken_shovel_configs([{test_shovel, Config}]),
197 |     Error.
198 | 
199 | test_broken_shovel_sources(Sources) ->
200 |     test_broken_shovel_config([{sources, Sources},
201 |                                {destinations, [{broker, "amqp://"}]},
202 |                                {queue, <<"">>}]).
203 | 
204 | valid_legacy_configuration(Config) ->
205 |     ok = setup_legacy_shovels(Config),
206 |     run_valid_test(Config).
207 | 
208 | valid_configuration(Config) ->
209 |     ok = setup_shovels(Config),
210 |     run_valid_test(Config).
211 | 
212 | run_valid_test(Config) ->
213 |     Chan = rabbit_ct_client_helpers:open_channel(Config, 0),
214 | 
215 |     #'queue.declare_ok'{ queue = Q } =
216 |         amqp_channel:call(Chan, #'queue.declare' { exclusive = true }),
217 |     #'queue.bind_ok'{} =
218 |         amqp_channel:call(Chan, #'queue.bind' { queue = Q, exchange = ?EXCHANGE,
219 |                                                 routing_key = ?FROM_SHOVEL }),
220 |     #'queue.bind_ok'{} =
221 |         amqp_channel:call(Chan, #'queue.bind' { queue = Q, exchange = ?EXCHANGE,
222 |                                                 routing_key = ?TO_SHOVEL }),
223 | 
224 |     #'basic.consume_ok'{ consumer_tag = CTag } =
225 |         amqp_channel:subscribe(Chan,
226 |                                #'basic.consume' { queue = Q, exclusive = true },
227 |                                self()),
228 |     receive
229 |         #'basic.consume_ok'{ consumer_tag = CTag } -> ok
230 |     after ?TIMEOUT -> throw(timeout_waiting_for_consume_ok)
231 |     end,
232 | 
233 |     ok = amqp_channel:call(Chan,
234 |                            #'basic.publish' { exchange    = ?EXCHANGE,
235 |                                               routing_key = ?TO_SHOVEL },
236 |                            #amqp_msg { payload = <<42>>,
237 |                                        props   = #'P_basic' {
238 |                                          delivery_mode = 2,
239 |                                          content_type  = ?UNSHOVELLED }
240 |                                      }),
241 | 
242 |     receive
243 |         {#'basic.deliver' { consumer_tag = CTag, delivery_tag = AckTag,
244 |                             routing_key = ?FROM_SHOVEL },
245 |          #amqp_msg { payload = <<42>>,
246 |                      props   = #'P_basic' {
247 |                         delivery_mode = 2,
248 |                         content_type  = ?SHOVELLED,
249 |                         headers       = [{<<"x-shovelled">>, _, _},
250 |                                          {<<"x-shovelled-timestamp">>,
251 |                                           long, _}]}
252 |                    }} ->
253 |             ok = amqp_channel:call(Chan, #'basic.ack'{ delivery_tag = AckTag })
254 |     after ?TIMEOUT -> throw(timeout_waiting_for_deliver1)
255 |     end,
256 | 
257 |     [{test_shovel, static, {running, _Info}, _Time}] =
258 |         rabbit_ct_broker_helpers:rpc(Config, 0,
259 |           rabbit_shovel_status, status, []),
260 | 
261 |     receive
262 |         {#'basic.deliver' { consumer_tag = CTag, delivery_tag = AckTag1,
263 |                             routing_key = ?TO_SHOVEL },
264 |          #amqp_msg { payload = <<42>>,
265 |                      props   = #'P_basic' { delivery_mode = 2,
266 |                                             content_type  = ?UNSHOVELLED }
267 |                    }} ->
268 |             ok = amqp_channel:call(Chan, #'basic.ack'{ delivery_tag = AckTag1 })
269 |     after ?TIMEOUT -> throw(timeout_waiting_for_deliver2)
270 |     end,
271 | 
272 |     rabbit_ct_client_helpers:close_channel(Chan).
273 | 
274 | setup_legacy_shovels(Config) ->
275 |     ok = rabbit_ct_broker_helpers:rpc(Config, 0,
276 |       ?MODULE, setup_legacy_shovels1, [Config]).
277 | 
278 | setup_shovels(Config) ->
279 |     ok = rabbit_ct_broker_helpers:rpc(Config, 0,
280 |       ?MODULE, setup_shovels1, [Config]).
281 | 
282 | setup_legacy_shovels1(Config) ->
283 |     _ = application:stop(rabbitmq_shovel),
284 |     Hostname = ?config(rmq_hostname, Config),
285 |     TcpPort = rabbit_ct_broker_helpers:get_node_config(Config, 0,
286 |       tcp_port_amqp),
287 |     %% a working config
288 |     application:set_env(
289 |       rabbitmq_shovel,
290 |       shovels,
291 |       [{test_shovel,
292 |         [{sources,
293 |           [{broker, rabbit_misc:format("amqp://~s:~b/%2f?heartbeat=5",
294 |                                        [Hostname, TcpPort])},
295 |            {declarations,
296 |             [{'queue.declare',    [exclusive, auto_delete]},
297 |              {'exchange.declare', [{exchange, ?EXCHANGE}, auto_delete]},
298 |              {'queue.bind',       [{queue, <<>>}, {exchange, ?EXCHANGE},
299 |                                    {routing_key, ?TO_SHOVEL}]}
300 |             ]}]},
301 |          {destinations,
302 |           [{broker, rabbit_misc:format("amqp://~s:~b/%2f",
303 |                                        [Hostname, TcpPort])}]},
304 |          {queue, <<>>},
305 |          {ack_mode, on_confirm},
306 |          {publish_fields, [{exchange, ?EXCHANGE}, {routing_key, ?FROM_SHOVEL}]},
307 |          {publish_properties, [{delivery_mode, 2},
308 |                                {cluster_id,    <<"my-cluster">>},
309 |                                {content_type,  ?SHOVELLED}]},
310 |          {add_forward_headers, true},
311 |          {add_timestamp_header, true}
312 |         ]}],
313 |       infinity),
314 | 
315 |     ok = application:start(rabbitmq_shovel),
316 |     await_running_shovel(test_shovel).
317 | 
318 | setup_shovels1(Config) ->
319 |     _ = application:stop(rabbitmq_shovel),
320 |     Hostname = ?config(rmq_hostname, Config),
321 |     TcpPort = rabbit_ct_broker_helpers:get_node_config(Config, 0,
322 |       tcp_port_amqp),
323 |     %% a working config
324 |     application:set_env(
325 |       rabbitmq_shovel,
326 |       shovels,
327 |       [{test_shovel,
328 |         [{source,
329 |           [{uris, [rabbit_misc:format("amqp://~s:~b/%2f?heartbeat=5",
330 |                                       [Hostname, TcpPort])]},
331 |            {declarations,
332 |             [{'queue.declare',    [exclusive, auto_delete]},
333 |              {'exchange.declare', [{exchange, ?EXCHANGE}, auto_delete]},
334 |              {'queue.bind',       [{queue, <<>>}, {exchange, ?EXCHANGE},
335 |                                    {routing_key, ?TO_SHOVEL}]}]},
336 |            {queue, <<>>}]},
337 |          {destination,
338 |           [{uris, [rabbit_misc:format("amqp://~s:~b/%2f",
339 |                                       [Hostname, TcpPort])]},
340 |            {publish_fields, [{exchange, ?EXCHANGE}, {routing_key, ?FROM_SHOVEL}]},
341 |            {publish_properties, [{delivery_mode, 2},
342 |                                  {cluster_id,    <<"my-cluster">>},
343 |                                  {content_type,  ?SHOVELLED}]},
344 |            {add_forward_headers, true},
345 |            {add_timestamp_header, true}]},
346 |          {ack_mode, on_confirm}]}],
347 |       infinity),
348 | 
349 |     ok = application:start(rabbitmq_shovel),
350 |     await_running_shovel(test_shovel).
351 | 
352 | await_running_shovel(Name) ->
353 |     case [N || {N, _, {running, _}, _}
354 |                       <- rabbit_shovel_status:status(),
355 |                          N =:= Name] of
356 |         [_] -> ok;
357 |         _   -> timer:sleep(100),
358 |                await_running_shovel(Name)
359 |     end.
360 | 


--------------------------------------------------------------------------------
/test/delete_shovel_command_SUITE.erl:
--------------------------------------------------------------------------------
 1 | %% This Source Code Form is subject to the terms of the Mozilla Public
 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
 4 | %%
 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
 6 | %%
 7 | 
 8 | -module(delete_shovel_command_SUITE).
 9 | 
10 | -include_lib("common_test/include/ct.hrl").
11 | -include_lib("amqp_client/include/amqp_client.hrl").
12 | 
13 | -compile(export_all).
14 | 
15 | -define(CMD, 'Elixir.RabbitMQ.CLI.Ctl.Commands.DeleteShovelCommand').
16 | 
17 | all() ->
18 |     [
19 |       {group, non_parallel_tests}
20 |     ].
21 | 
22 | groups() ->
23 |     [
24 |      {non_parallel_tests, [], [
25 |                                delete_not_found,
26 |                                delete
27 |                               ]}
28 |     ].
29 | 
30 | %% -------------------------------------------------------------------
31 | %% Testsuite setup/teardown.
32 | %% -------------------------------------------------------------------
33 | 
34 | init_per_suite(Config) ->
35 |     rabbit_ct_helpers:log_environment(),
36 |     Config1 = rabbit_ct_helpers:set_config(Config, [
37 |         {rmq_nodename_suffix, ?MODULE}
38 |       ]),
39 |     Config2 = rabbit_ct_helpers:run_setup_steps(Config1,
40 |       rabbit_ct_broker_helpers:setup_steps() ++
41 |       rabbit_ct_client_helpers:setup_steps()),
42 |     Config2.
43 | 
44 | end_per_suite(Config) ->
45 |     rabbit_ct_helpers:run_teardown_steps(Config,
46 |       rabbit_ct_client_helpers:teardown_steps() ++
47 |       rabbit_ct_broker_helpers:teardown_steps()).
48 | 
49 | init_per_group(_, Config) ->
50 |     Config.
51 | 
52 | end_per_group(_, Config) ->
53 |     Config.
54 | 
55 | init_per_testcase(Testcase, Config) ->
56 |     rabbit_ct_helpers:testcase_started(Config, Testcase).
57 | 
58 | end_per_testcase(Testcase, Config) ->
59 |     rabbit_ct_helpers:testcase_finished(Config, Testcase).
60 | 
61 | %% -------------------------------------------------------------------
62 | %% Testcases.
63 | %% -------------------------------------------------------------------
64 | delete_not_found(Config) ->
65 |     [A] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
66 |     Opts = #{node => A, vhost => <<"/">>},
67 |     {error, _} = ?CMD:run([<<"myshovel">>], Opts).
68 | 
69 | delete(Config) ->
70 |     shovel_test_utils:set_param(
71 |       Config,
72 |       <<"myshovel">>, [{<<"src-queue">>,  <<"src">>},
73 |                        {<<"dest-queue">>, <<"dest">>}]),
74 |     [A] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
75 |     Opts = #{node => A, vhost => <<"/">>},
76 |     ok = ?CMD:run([<<"myshovel">>], Opts),
77 |     [] = rabbit_ct_broker_helpers:rpc(Config, 0, rabbit_shovel_status,
78 |                                       status, []).
79 | 


--------------------------------------------------------------------------------
/test/parameters_SUITE.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(parameters_SUITE).
  9 | 
 10 | -compile(export_all).
 11 | 
 12 | -include_lib("eunit/include/eunit.hrl").
 13 | -include_lib("rabbit_common/include/rabbit_framing.hrl").
 14 | 
 15 | -define(EXCHANGE,    <<"test_exchange">>).
 16 | -define(TO_SHOVEL,   <<"to_the_shovel">>).
 17 | -define(FROM_SHOVEL, <<"from_the_shovel">>).
 18 | -define(UNSHOVELLED, <<"unshovelled">>).
 19 | -define(SHOVELLED,   <<"shovelled">>).
 20 | -define(TIMEOUT,     1000).
 21 | 
 22 | all() ->
 23 |     [
 24 |       {group, tests}
 25 |     ].
 26 | 
 27 | groups() ->
 28 |     [
 29 |       {tests, [parallel], [
 30 |           parse_amqp091_maps,
 31 |           parse_amqp091_proplists,
 32 |           parse_amqp091_empty_maps,
 33 |           parse_amqp091_empty_proplists,
 34 |           parse_amqp10,
 35 |           parse_amqp10_minimal,
 36 |           validate_amqp10,
 37 |           validate_amqp10_with_a_map
 38 |         ]}
 39 |     ].
 40 | 
 41 | %% -------------------------------------------------------------------
 42 | %% Testsuite setup/teardown.
 43 | %% -------------------------------------------------------------------
 44 | 
 45 | init_per_suite(Config) ->
 46 |     Config.
 47 | 
 48 | end_per_suite(Config) ->
 49 |     Config.
 50 | 
 51 | init_per_group(_, Config) ->
 52 |     Config.
 53 | 
 54 | end_per_group(_, Config) ->
 55 |     Config.
 56 | 
 57 | init_per_testcase(_Testcase, Config) -> Config.
 58 | 
 59 | end_per_testcase(_Testcase, Config) -> Config.
 60 | 
 61 | 
 62 | %% -------------------------------------------------------------------
 63 | %% Testcases.
 64 | %% -------------------------------------------------------------------
 65 | 
 66 | parse_amqp091_maps(_Config) ->
 67 |     Params =
 68 |         [{<<"src-uri">>, <<"amqp://localhost:5672">>},
 69 |          {<<"src-protocol">>, <<"amqp091">>},
 70 |          {<<"dest-protocol">>, <<"amqp091">>},
 71 |          {<<"dest-uri">>, <<"amqp://remotehost:5672">>},
 72 |          {<<"add-forward-headers">>, true},
 73 |          {<<"add-timestamp-header">>, true},
 74 |          {<<"publish-properties">>, #{<<"cluster_id">> => <<"x">>,
 75 |                                       <<"delivery_mode">> => 2}},
 76 |          {<<"ack-mode">>, <<"on-publish">>},
 77 |          {<<"src-delete-after">>, <<"queue-length">>},
 78 |          {<<"prefetch-count">>, 30},
 79 |          {<<"reconnect-delay">>, 1001},
 80 |          {<<"src-queue">>, <<"a-src-queue">>},
 81 |          {<<"dest-queue">>, <<"a-dest-queue">>}
 82 |         ],
 83 | 
 84 |     test_parse_amqp091(Params).
 85 | 
 86 | parse_amqp091_proplists(_Config) ->
 87 |     Params =
 88 |         [{<<"src-uri">>, <<"amqp://localhost:5672">>},
 89 |          {<<"src-protocol">>, <<"amqp091">>},
 90 |          {<<"dest-protocol">>, <<"amqp091">>},
 91 |          {<<"dest-uri">>, <<"amqp://remotehost:5672">>},
 92 |          {<<"dest-add-forward-headers">>, true},
 93 |          {<<"dest-add-timestamp-header">>, true},
 94 |          {<<"dest-publish-properties">>, [{<<"cluster_id">>, <<"x">>},
 95 |                                           {<<"delivery_mode">>, 2}]},
 96 |          {<<"ack-mode">>, <<"on-publish">>},
 97 |          {<<"src-delete-after">>, <<"queue-length">>},
 98 |          {<<"src-prefetch-count">>, 30},
 99 |          {<<"reconnect-delay">>, 1001},
100 |          {<<"src-queue">>, <<"a-src-queue">>},
101 |          {<<"dest-queue">>, <<"a-dest-queue">>}
102 |         ],
103 |     test_parse_amqp091(Params).
104 | 
105 | parse_amqp091_empty_maps(_Config) ->
106 |     Params =
107 |         [{<<"src-uri">>, <<"amqp://localhost:5672">>},
108 |          {<<"src-protocol">>, <<"amqp091">>},
109 |          {<<"dest-protocol">>, <<"amqp091">>},
110 |          {<<"dest-uri">>, <<"amqp://remotehost:5672">>},
111 |          {<<"dest-add-forward-headers">>, true},
112 |          {<<"dest-add-timestamp-header">>, true},
113 |          {<<"dest-publish-properties">>, #{}},
114 |          {<<"ack-mode">>, <<"on-publish">>},
115 |          {<<"src-delete-after">>, <<"queue-length">>},
116 |          {<<"src-prefetch-count">>, 30},
117 |          {<<"reconnect-delay">>, 1001},
118 |          {<<"src-queue">>, <<"a-src-queue">>},
119 |          {<<"dest-queue">>, <<"a-dest-queue">>}
120 |         ],
121 |     test_parse_amqp091_with_blank_proprties(Params).
122 | 
123 | parse_amqp091_empty_proplists(_Config) ->
124 |     Params =
125 |         [{<<"src-uri">>, <<"amqp://localhost:5672">>},
126 |          {<<"src-protocol">>, <<"amqp091">>},
127 |          {<<"dest-protocol">>, <<"amqp091">>},
128 |          {<<"dest-uri">>, <<"amqp://remotehost:5672">>},
129 |          {<<"dest-add-forward-headers">>, true},
130 |          {<<"dest-add-timestamp-header">>, true},
131 |          {<<"dest-publish-properties">>, []},
132 |          {<<"ack-mode">>, <<"on-publish">>},
133 |          {<<"src-delete-after">>, <<"queue-length">>},
134 |          {<<"src-prefetch-count">>, 30},
135 |          {<<"reconnect-delay">>, 1001},
136 |          {<<"src-queue">>, <<"a-src-queue">>},
137 |          {<<"dest-queue">>, <<"a-dest-queue">>}
138 |         ],
139 |     test_parse_amqp091_with_blank_proprties(Params).
140 | 
141 | 
142 | test_parse_amqp091(Params) ->
143 |     {ok, Result} = rabbit_shovel_parameters:parse({"vhost", "name"},
144 |                                                   "my-cluster", Params),
145 |     #{ack_mode := on_publish,
146 |       name := "name",
147 |       reconnect_delay := 1001,
148 |       dest := #{module := rabbit_amqp091_shovel,
149 |                 uris := ["amqp://remotehost:5672"],
150 |                 props_fun := PropsFun
151 |                },
152 |       source := #{module := rabbit_amqp091_shovel,
153 |                   uris := ["amqp://localhost:5672"],
154 |                   prefetch_count := 30,
155 |                   queue := <<"a-src-queue">>,
156 |                   delete_after := 'queue-length'}
157 |      } = Result,
158 | 
159 |     #'P_basic'{headers = ActualHeaders,
160 |                delivery_mode = 2,
161 |                cluster_id = <<"x">>} = PropsFun("amqp://localhost:5672",
162 |                                                 "amqp://remotehost:5672",
163 |                                                 #'P_basic'{headers = undefined}),
164 |     assert_amqp901_headers(ActualHeaders),
165 |     ok.
166 | 
167 | test_parse_amqp091_with_blank_proprties(Params) ->
168 |     {ok, Result} = rabbit_shovel_parameters:parse({"vhost", "name"},
169 |                                                   "my-cluster", Params),
170 |     #{ack_mode := on_publish,
171 |       name := "name",
172 |       reconnect_delay := 1001,
173 |       dest := #{module := rabbit_amqp091_shovel,
174 |                 uris := ["amqp://remotehost:5672"],
175 |                 props_fun := PropsFun
176 |                },
177 |       source := #{module := rabbit_amqp091_shovel,
178 |                   uris := ["amqp://localhost:5672"],
179 |                   prefetch_count := 30,
180 |                   queue := <<"a-src-queue">>,
181 |                   delete_after := 'queue-length'}
182 |      } = Result,
183 | 
184 |     #'P_basic'{headers = ActualHeaders} = PropsFun("amqp://localhost:5672",
185 |                                                    "amqp://remotehost:5672",
186 |                                                    #'P_basic'{headers = undefined}),
187 |     assert_amqp901_headers(ActualHeaders),
188 |     ok.
189 | 
190 | assert_amqp901_headers(ActualHeaders) ->
191 |     {_, array, [{table, Shovelled}]} = lists:keyfind(<<"x-shovelled">>, 1, ActualHeaders),
192 |     {_, long, _} = lists:keyfind(<<"x-shovelled-timestamp">>, 1, ActualHeaders),
193 | 
194 |     ExpectedHeaders =
195 |     [{<<"shovelled-by">>, "my-cluster"},
196 |      {<<"shovel-type">>, <<"dynamic">>},
197 |      {<<"shovel-name">>, "name"},
198 |      {<<"shovel-vhost">>, "vhost"},
199 |      {<<"src-uri">>,"amqp://localhost:5672"},
200 |      {<<"dest-uri">>,"amqp://remotehost:5672"},
201 |      {<<"src-queue">>,<<"a-src-queue">>},
202 |      {<<"dest-queue">>,<<"a-dest-queue">>}],
203 |     lists:foreach(fun({K, V}) ->
204 |                           ?assertMatch({K, _, V},
205 |                                        lists:keyfind(K, 1, Shovelled))
206 |                   end, ExpectedHeaders),
207 |     ok.
208 | 
209 | parse_amqp10(_Config) ->
210 |     Params =
211 |         [
212 |          {<<"ack-mode">>, <<"on-publish">>},
213 |          {<<"reconnect-delay">>, 1001},
214 | 
215 |          {<<"src-protocol">>, <<"amqp10">>},
216 |          {<<"src-uri">>, <<"amqp://localhost:5672">>},
217 |          {<<"src-address">>, <<"a-src-queue">>},
218 |          {<<"src-delete-after">>, <<"never">>},
219 |          {<<"src-prefetch-count">>, 30},
220 | 
221 |          {<<"dest-protocol">>, <<"amqp10">>},
222 |          {<<"dest-uri">>, <<"amqp://remotehost:5672">>},
223 |          {<<"dest-address">>, <<"a-dest-queue">>},
224 |          {<<"dest-add-forward-headers">>, true},
225 |          {<<"dest-add-timestamp-header">>, true},
226 |          {<<"dest-application-properties">>, [{<<"some-app-prop">>,
227 |                                                <<"app-prop-value">>}]},
228 |          {<<"dest-message-annotations">>, [{<<"some-message-ann">>,
229 |                                             <<"message-ann-value">>}]},
230 |          {<<"dest-properties">>, [{<<"user_id">>, <<"some-user">>}]}
231 |         ],
232 | 
233 |     ?assertMatch(
234 |        {ok, #{name := "my_shovel",
235 |               ack_mode := on_publish,
236 |               source := #{module := rabbit_amqp10_shovel,
237 |                           uris := ["amqp://localhost:5672"],
238 |                           delete_after := never,
239 |                           prefetch_count := 30,
240 |                           source_address := <<"a-src-queue">>
241 |                           },
242 |               dest := #{module := rabbit_amqp10_shovel,
243 |                         uris := ["amqp://remotehost:5672"],
244 |                         target_address := <<"a-dest-queue">>,
245 |                         message_annotations := #{<<"some-message-ann">> :=
246 |                                                  <<"message-ann-value">>},
247 |                         application_properties := #{<<"some-app-prop">> :=
248 |                                                     <<"app-prop-value">>},
249 |                         properties := #{user_id := <<"some-user">>},
250 |                         add_timestamp_header := true,
251 |                         add_forward_headers := true
252 |                        }
253 |              }},
254 |         rabbit_shovel_parameters:parse({"vhost", "my_shovel"}, "my-cluster",
255 |                                        Params)),
256 |     ok.
257 | 
258 | parse_amqp10_minimal(_Config) ->
259 |     Params =
260 |         [
261 |          {<<"src-protocol">>, <<"amqp10">>},
262 |          {<<"src-uri">>, <<"amqp://localhost:5672">>},
263 |          {<<"src-address">>, <<"a-src-queue">>},
264 | 
265 |          {<<"dest-protocol">>, <<"amqp10">>},
266 |          {<<"dest-uri">>, <<"amqp://remotehost:5672">>},
267 |          {<<"dest-address">>, <<"a-dest-queue">>}
268 |         ],
269 |     ?assertMatch(
270 |        {ok, #{name := "my_shovel",
271 |               ack_mode := on_confirm,
272 |               source := #{module := rabbit_amqp10_shovel,
273 |                           uris := ["amqp://localhost:5672"],
274 |                           delete_after := never,
275 |                           source_address := <<"a-src-queue">>
276 |                           },
277 |               dest := #{module := rabbit_amqp10_shovel,
278 |                         uris := ["amqp://remotehost:5672"],
279 |                         unacked := #{},
280 |                         target_address := <<"a-dest-queue">>
281 |                        }
282 |              }},
283 |         rabbit_shovel_parameters:parse({"vhost", "my_shovel"}, "my-cluster",
284 |                                        Params)),
285 |     ok.
286 | 
287 | validate_amqp10(_Config) ->
288 |     Params =
289 |         [
290 |          {<<"ack-mode">>, <<"on-publish">>},
291 |          {<<"reconnect-delay">>, 1001},
292 | 
293 |          {<<"src-protocol">>, <<"amqp10">>},
294 |          {<<"src-uri">>, <<"amqp://localhost:5672">>},
295 |          {<<"src-address">>, <<"a-src-queue">>},
296 |          {<<"src-delete-after">>, <<"never">>},
297 |          {<<"src-prefetch-count">>, 30},
298 | 
299 |          {<<"dest-protocol">>, <<"amqp10">>},
300 |          {<<"dest-uri">>, <<"amqp://remotehost:5672">>},
301 |          {<<"dest-address">>, <<"a-dest-queue">>},
302 |          {<<"dest-add-forward-headers">>, true},
303 |          {<<"dest-add-timestamp-header">>, true},
304 |          {<<"dest-application-properties">>, [{<<"some-app-prop">>,
305 |                                                <<"app-prop-value">>}]},
306 |          {<<"dest-message-annotations">>, [{<<"some-message-ann">>,
307 |                                                <<"message-ann-value">>}]},
308 |          {<<"dest-properties">>, [{<<"user_id">>, <<"some-user">>}]}
309 |         ],
310 | 
311 |         Res = rabbit_shovel_parameters:validate("my-vhost", <<"shovel">>,
312 |                                                 "my-shovel", Params, none),
313 |         [] = validate_ok(Res),
314 |         ok.
315 | 
316 | validate_amqp10_with_a_map(_Config) ->
317 |     Params =
318 |         #{
319 |          <<"ack-mode">> => <<"on-publish">>,
320 |          <<"reconnect-delay">> => 1001,
321 | 
322 |          <<"src-protocol">> => <<"amqp10">>,
323 |          <<"src-uri">> => <<"amqp://localhost:5672">>,
324 |          <<"src-address">> => <<"a-src-queue">>,
325 |          <<"src-delete-after">> => <<"never">>,
326 |          <<"src-prefetch-count">> => 30,
327 | 
328 |          <<"dest-protocol">> => <<"amqp10">>,
329 |          <<"dest-uri">> => <<"amqp://remotehost:5672">>,
330 |          <<"dest-address">> => <<"a-dest-queue">>,
331 |          <<"dest-add-forward-headers">> => true,
332 |          <<"dest-add-timestamp-header">> => true,
333 |          <<"dest-application-properties">> => [{<<"some-app-prop">>,
334 |                                                 <<"app-prop-value">>}],
335 |          <<"dest-message-annotations">> => [{<<"some-message-ann">>, <<"message-ann-value">>}],
336 |          <<"dest-properties">> => #{<<"user_id">> => <<"some-user">>}
337 |         },
338 | 
339 |         Res = rabbit_shovel_parameters:validate("my-vhost", <<"shovel">>,
340 |                                                 "my-shovel", Params, none),
341 |         [] = validate_ok(Res),
342 |         ok.
343 | 
344 | validate_ok([ok | T]) ->
345 |     validate_ok(T);
346 | validate_ok([[_|_] = L | T]) ->
347 |     validate_ok(L) ++ validate_ok(T);
348 | validate_ok([]) -> [];
349 | validate_ok(X) ->
350 |     exit({not_ok, X}).
351 | 


--------------------------------------------------------------------------------
/test/shovel_status_command_SUITE.erl:
--------------------------------------------------------------------------------
  1 | %% This Source Code Form is subject to the terms of the Mozilla Public
  2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
  3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
  4 | %%
  5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
  6 | %%
  7 | 
  8 | -module(shovel_status_command_SUITE).
  9 | 
 10 | -include_lib("common_test/include/ct.hrl").
 11 | -include_lib("amqp_client/include/amqp_client.hrl").
 12 | 
 13 | -compile(export_all).
 14 | 
 15 | -define(CMD, 'Elixir.RabbitMQ.CLI.Ctl.Commands.ShovelStatusCommand').
 16 | 
 17 | all() ->
 18 |     [
 19 |       {group, non_parallel_tests}
 20 |     ].
 21 | 
 22 | groups() ->
 23 |     [
 24 |      {non_parallel_tests, [], [
 25 |                                run_not_started,
 26 |                                output_not_started,
 27 |                                run_starting,
 28 |                                output_starting,
 29 |                                run_running,
 30 |                                output_running
 31 |         ]}
 32 |     ].
 33 | 
 34 | %% -------------------------------------------------------------------
 35 | %% Testsuite setup/teardown.
 36 | %% -------------------------------------------------------------------
 37 | 
 38 | init_per_suite(Config) ->
 39 |     rabbit_ct_helpers:log_environment(),
 40 |     Config1 = rabbit_ct_helpers:set_config(Config, [
 41 |         {rmq_nodename_suffix, ?MODULE}
 42 |       ]),
 43 |     Config2 = rabbit_ct_helpers:run_setup_steps(Config1,
 44 |       rabbit_ct_broker_helpers:setup_steps() ++
 45 |       rabbit_ct_client_helpers:setup_steps()),
 46 |     Config2.
 47 | 
 48 | end_per_suite(Config) ->
 49 |     rabbit_ct_helpers:run_teardown_steps(Config,
 50 |       rabbit_ct_client_helpers:teardown_steps() ++
 51 |       rabbit_ct_broker_helpers:teardown_steps()).
 52 | 
 53 | init_per_group(_, Config) ->
 54 |     Config.
 55 | 
 56 | end_per_group(_, Config) ->
 57 |     Config.
 58 | 
 59 | init_per_testcase(Testcase, Config) ->
 60 |     rabbit_ct_helpers:testcase_started(Config, Testcase).
 61 | 
 62 | end_per_testcase(Testcase, Config) ->
 63 |     rabbit_ct_helpers:testcase_finished(Config, Testcase).
 64 | 
 65 | %% -------------------------------------------------------------------
 66 | %% Testcases.
 67 | %% -------------------------------------------------------------------
 68 | run_not_started(Config) ->
 69 |     [A] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 70 |     Opts = #{node => A},
 71 |     {stream, []} = ?CMD:run([], Opts).
 72 | 
 73 | output_not_started(Config) ->
 74 |     [A] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 75 |     Opts = #{node => A},
 76 |     {stream, []} = ?CMD:output({stream, []}, Opts).
 77 | 
 78 | run_starting(Config) ->
 79 |     shovel_test_utils:set_param_nowait(
 80 |       Config,
 81 |       <<"test">>, [{<<"src-queue">>,  <<"src">>},
 82 |                    {<<"dest-queue">>, <<"dest">>}]),
 83 |     [A] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 84 |     Opts = #{node => A},
 85 |     case ?CMD:run([], Opts) of
 86 |         {stream, [{{<<"/">>, <<"test">>}, dynamic, starting, _}]} ->
 87 |             ok;
 88 |         {stream, []} ->
 89 |             throw(shovel_not_found);
 90 |         {stream, [{{<<"/">>, <<"test">>}, dynamic, {running, _}, _}]} ->
 91 |             ct:pal("Shovel is already running, starting could not be tested!")
 92 |     end,
 93 |     shovel_test_utils:clear_param(Config, <<"test">>).
 94 | 
 95 | output_starting(Config) ->
 96 |     [A] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
 97 |     Opts = #{node => A},
 98 |     {stream, [#{vhost := <<"/">>, name := <<"test">>, type := dynamic,
 99 |                 state := starting, last_changed := <<"2016-11-17 10:00:00">>}]}
100 |         = ?CMD:output({stream, [{{<<"/">>, <<"test">>}, dynamic, starting,
101 |                                  {{2016, 11, 17}, {10, 00, 00}}}]}, Opts),
102 |     shovel_test_utils:clear_param(Config, <<"test">>).
103 | 
104 | run_running(Config) ->
105 |     shovel_test_utils:set_param(
106 |       Config,
107 |       <<"test">>, [{<<"src-queue">>,  <<"src">>},
108 |                    {<<"dest-queue">>, <<"dest">>}]),
109 |     [A] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
110 |     Opts = #{node => A},
111 |     {stream, [{{<<"/">>, <<"test">>}, dynamic, {running, _}, _}]}
112 |         = ?CMD:run([], Opts),
113 |     shovel_test_utils:clear_param(Config, <<"test">>).
114 | 
115 | output_running(Config) ->
116 |     [A] = rabbit_ct_broker_helpers:get_node_configs(Config, nodename),
117 |     Opts = #{node => A},
118 |     {stream, [#{vhost := <<"/">>, name := <<"test">>, type := dynamic,
119 |                 state := running, source := <<"amqp://server-1">>,
120 |                 destination := <<"amqp://server-2">>,
121 |                 termination_reason := <<>>,
122 |                 last_changed := <<"2016-11-17 10:00:00">>}]} =
123 |         ?CMD:output({stream, [{{<<"/">>, <<"test">>}, dynamic,
124 |                                {running, [{src_uri, <<"amqp://server-1">>},
125 |                                           {dest_uri, <<"amqp://server-2">>}]},
126 |                                {{2016, 11, 17}, {10, 00, 00}}}]}, Opts),
127 |     shovel_test_utils:clear_param(Config, <<"test">>).
128 | 


--------------------------------------------------------------------------------
/test/shovel_test_utils.erl:
--------------------------------------------------------------------------------
 1 | %% This Source Code Form is subject to the terms of the Mozilla Public
 2 | %% License, v. 2.0. If a copy of the MPL was not distributed with this
 3 | %% file, You can obtain one at https://mozilla.org/MPL/2.0/.
 4 | %%
 5 | %% Copyright (c) 2007-2020 VMware, Inc. or its affiliates.  All rights reserved.
 6 | %%
 7 | 
 8 | -module(shovel_test_utils).
 9 | 
10 | -include_lib("common_test/include/ct.hrl").
11 | -export([set_param/3, set_param_nowait/3, await_shovel/2, await_shovel1/2,
12 |          shovels_from_status/0, await/1, clear_param/2]).
13 | 
14 | make_uri(Config) ->
15 |     Hostname = ?config(rmq_hostname, Config),
16 |     Port = rabbit_ct_broker_helpers:get_node_config(Config, 0, tcp_port_amqp),
17 |     list_to_binary(lists:flatten(io_lib:format("amqp://~s:~b",
18 |                                                [Hostname, Port]))).
19 | set_param(Config, Name, Value) ->
20 |     set_param_nowait(Config, Name, Value),
21 |     await_shovel(Config, Name).
22 | 
23 | set_param_nowait(Config, Name, Value) ->
24 |     Uri = make_uri(Config),
25 |     ok = rabbit_ct_broker_helpers:rpc(Config, 0,
26 |       rabbit_runtime_parameters, set, [
27 |         <<"/">>, <<"shovel">>, Name, [{<<"src-uri">>,  Uri},
28 |                                       {<<"dest-uri">>, [Uri]} |
29 |                                       Value], none]).
30 | 
31 | await_shovel(Config, Name) ->
32 |     rabbit_ct_broker_helpers:rpc(Config, 0,
33 |       ?MODULE, await_shovel1, [Config, Name]).
34 | 
35 | await_shovel1(_Config, Name) ->
36 |     await(fun () -> lists:member(Name, shovels_from_status()) end).
37 | 
38 | shovels_from_status() ->
39 |     S = rabbit_shovel_status:status(),
40 |     [N || {{<<"/">>, N}, dynamic, {running, _}, _} <- S].
41 | 
42 | await(Pred) ->
43 |     case Pred() of
44 |         true  -> ok;
45 |         false -> timer:sleep(100),
46 |                  await(Pred)
47 |     end.
48 | 
49 | clear_param(Config, Name) ->
50 |     rabbit_ct_broker_helpers:rpc(Config, 0,
51 |       rabbit_runtime_parameters, clear, [<<"/">>, <<"shovel">>, Name, <<"acting-user">>]).
52 | 


--------------------------------------------------------------------------------