├── .github └── workflows │ └── action.yml ├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── config └── config.exs ├── erlang.mk ├── lib ├── rabbitmq_message_deduplication.ex └── rabbitmq_message_deduplication │ ├── cache.ex │ ├── cache_manager.ex │ ├── common.ex │ ├── rabbit_message_deduplication_exchange.ex │ ├── rabbit_message_deduplication_policies.ex │ ├── rabbit_message_deduplication_policy_event.ex │ └── rabbit_message_deduplication_queue.ex ├── mix.exs ├── rabbitmq-components.mk └── test ├── cache_manager_test.exs ├── cache_test.exs ├── exchange_SUITE.erl ├── policies_test.exs ├── queue_SUITE.erl └── test_helper.exs /.github/workflows/action.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: [push, pull_request] 3 | env: 4 | SERVER_FOLDER: rabbitmq-server 5 | PLUGIN_FOLDER: rabbitmq-server/deps/rabbitmq-message-deduplication 6 | jobs: 7 | build: 8 | runs-on: ubuntu-24.04 9 | strategy: 10 | matrix: 11 | PLATFORM: 12 | - { RMQREF: "v3.13.x", ERLVER: "26.2", ELXVER: "1.16" } 13 | - { RMQREF: "v4.0.x", ERLVER: "26.2", ELXVER: "1.16" } 14 | - { RMQREF: "v4.1.x", ERLVER: "27", ELXVER: "1.18" } 15 | name: "Broker: ${{ matrix.PLATFORM.RMQREF }} - Erlang: ${{ matrix.PLATFORM.ERLVER }} - Elixir: ${{ matrix.PLATFORM.ELXVER }}" 16 | steps: 17 | - name: Checkout RabbitMQ Server 18 | uses: actions/checkout@v4 19 | with: 20 | repository: rabbitmq/rabbitmq-server 21 | path: ${{ env.SERVER_FOLDER }} 22 | - name: Checkout Plugin 23 | uses: actions/checkout@v4 24 | with: 25 | path: ${{ env.PLUGIN_FOLDER }} 26 | - name: Install Erlang and Elixir 27 | uses: erlef/setup-beam@v1 28 | with: 29 | otp-version: ${{ matrix.PLATFORM.ERLVER }} 30 | elixir-version: ${{ matrix.PLATFORM.ELXVER }} 31 | - name: Run tests 32 | working-directory: ${{ env.PLUGIN_FOLDER }} 33 | run: | 34 | make tests RABBITMQ_VERSION=${{ matrix.PLATFORM.RMQREF }} MIX_ENV=test 35 | - name: Build distribution files 36 | working-directory: ${{ env.PLUGIN_FOLDER }} 37 | run: | 38 | make dist RABBITMQ_VERSION=${{ matrix.PLATFORM.RMQREF }} MIX_ENV=prod DIST_AS_EZS=yes 39 | - name: Store test artifacts 40 | uses: actions/upload-artifact@v4 41 | with: 42 | name: tests-rmq${{ matrix.PLATFORM.RMQREF }}-erl${{ matrix.PLATFORM.ERLVER }}-elx${{ matrix.PLATFORM.ELXVER }} 43 | path: ${{ env.SERVER_FOLDER }}/logs/ 44 | - name: Store build artifacts 45 | uses: actions/upload-artifact@v4 46 | with: 47 | name: plugins-rmq${{ matrix.PLATFORM.RMQREF }}-erl${{ matrix.PLATFORM.ERLVER }}-elx${{ matrix.PLATFORM.ELXVER }} 48 | path: | 49 | ${{ env.PLUGIN_FOLDER }}/plugins/elixir-${{ matrix.PLATFORM.ELXVER }}.*.ez 50 | ${{ env.PLUGIN_FOLDER }}/plugins/rabbitmq_message_deduplication-[0-9].*.ez 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | deps 3 | .erlang.mk 4 | *.beam 5 | log* 6 | *.plt 7 | mix.lock 8 | plugins 9 | GPATH 10 | GRTAGS 11 | GTAGS 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: focal 2 | language: elixir 3 | elixir: '1.10.4' 4 | otp_release: 5 | - '23.1.2' 6 | install: 7 | - wget https://github.com/rabbitmq/mix_task_archive_deps/releases/download/0.5.0/mix_task_archive_deps-0.5.0.ez 8 | - mix archive.install --force ./mix_task_archive_deps-0.5.0.ez 9 | script: make tests current_rmq_ref=$TRAVIS_BRANCH 10 | branches: 11 | only: 12 | - master 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Mozilla Public License Version 2.0 2 | ================================== 3 | 4 | 1. Definitions 5 | -------------- 6 | 7 | 1.1. "Contributor" 8 | means each individual or legal entity that creates, contributes to 9 | the creation of, or owns Covered Software. 10 | 11 | 1.2. "Contributor Version" 12 | means the combination of the Contributions of others (if any) used 13 | by a Contributor and that particular Contributor's Contribution. 14 | 15 | 1.3. "Contribution" 16 | means Covered Software of a particular Contributor. 17 | 18 | 1.4. "Covered Software" 19 | means Source Code Form to which the initial Contributor has attached 20 | the notice in Exhibit A, the Executable Form of such Source Code 21 | Form, and Modifications of such Source Code Form, in each case 22 | including portions thereof. 23 | 24 | 1.5. "Incompatible With Secondary Licenses" 25 | means 26 | 27 | (a) that the initial Contributor has attached the notice described 28 | in Exhibit B to the Covered Software; or 29 | 30 | (b) that the Covered Software was made available under the terms of 31 | version 1.1 or earlier of the License, but not also under the 32 | terms of a Secondary License. 33 | 34 | 1.6. "Executable Form" 35 | means any form of the work other than Source Code Form. 36 | 37 | 1.7. "Larger Work" 38 | means a work that combines Covered Software with other material, in 39 | a separate file or files, that is not Covered Software. 40 | 41 | 1.8. "License" 42 | means this document. 43 | 44 | 1.9. "Licensable" 45 | means having the right to grant, to the maximum extent possible, 46 | whether at the time of the initial grant or subsequently, any and 47 | all of the rights conveyed by this License. 48 | 49 | 1.10. "Modifications" 50 | means any of the following: 51 | 52 | (a) any file in Source Code Form that results from an addition to, 53 | deletion from, or modification of the contents of Covered 54 | Software; or 55 | 56 | (b) any new file in Source Code Form that contains any Covered 57 | Software. 58 | 59 | 1.11. "Patent Claims" of a Contributor 60 | means any patent claim(s), including without limitation, method, 61 | process, and apparatus claims, in any patent Licensable by such 62 | Contributor that would be infringed, but for the grant of the 63 | License, by the making, using, selling, offering for sale, having 64 | made, import, or transfer of either its Contributions or its 65 | Contributor Version. 66 | 67 | 1.12. "Secondary License" 68 | means either the GNU General Public License, Version 2.0, the GNU 69 | Lesser General Public License, Version 2.1, the GNU Affero General 70 | Public License, Version 3.0, or any later versions of those 71 | licenses. 72 | 73 | 1.13. "Source Code Form" 74 | means the form of the work preferred for making modifications. 75 | 76 | 1.14. "You" (or "Your") 77 | means an individual or a legal entity exercising rights under this 78 | License. For legal entities, "You" includes any entity that 79 | controls, is controlled by, or is under common control with You. For 80 | purposes of this definition, "control" means (a) the power, direct 81 | or indirect, to cause the direction or management of such entity, 82 | whether by contract or otherwise, or (b) ownership of more than 83 | fifty percent (50%) of the outstanding shares or beneficial 84 | ownership of such entity. 85 | 86 | 2. License Grants and Conditions 87 | -------------------------------- 88 | 89 | 2.1. Grants 90 | 91 | Each Contributor hereby grants You a world-wide, royalty-free, 92 | non-exclusive license: 93 | 94 | (a) under intellectual property rights (other than patent or trademark) 95 | Licensable by such Contributor to use, reproduce, make available, 96 | modify, display, perform, distribute, and otherwise exploit its 97 | Contributions, either on an unmodified basis, with Modifications, or 98 | as part of a Larger Work; and 99 | 100 | (b) under Patent Claims of such Contributor to make, use, sell, offer 101 | for sale, have made, import, and otherwise transfer either its 102 | Contributions or its Contributor Version. 103 | 104 | 2.2. Effective Date 105 | 106 | The licenses granted in Section 2.1 with respect to any Contribution 107 | become effective for each Contribution on the date the Contributor first 108 | distributes such Contribution. 109 | 110 | 2.3. Limitations on Grant Scope 111 | 112 | The licenses granted in this Section 2 are the only rights granted under 113 | this License. No additional rights or licenses will be implied from the 114 | distribution or licensing of Covered Software under this License. 115 | Notwithstanding Section 2.1(b) above, no patent license is granted by a 116 | Contributor: 117 | 118 | (a) for any code that a Contributor has removed from Covered Software; 119 | or 120 | 121 | (b) for infringements caused by: (i) Your and any other third party's 122 | modifications of Covered Software, or (ii) the combination of its 123 | Contributions with other software (except as part of its Contributor 124 | Version); or 125 | 126 | (c) under Patent Claims infringed by Covered Software in the absence of 127 | its Contributions. 128 | 129 | This License does not grant any rights in the trademarks, service marks, 130 | or logos of any Contributor (except as may be necessary to comply with 131 | the notice requirements in Section 3.4). 132 | 133 | 2.4. Subsequent Licenses 134 | 135 | No Contributor makes additional grants as a result of Your choice to 136 | distribute the Covered Software under a subsequent version of this 137 | License (see Section 10.2) or under the terms of a Secondary License (if 138 | permitted under the terms of Section 3.3). 139 | 140 | 2.5. Representation 141 | 142 | Each Contributor represents that the Contributor believes its 143 | Contributions are its original creation(s) or it has sufficient rights 144 | to grant the rights to its Contributions conveyed by this License. 145 | 146 | 2.6. Fair Use 147 | 148 | This License is not intended to limit any rights You have under 149 | applicable copyright doctrines of fair use, fair dealing, or other 150 | equivalents. 151 | 152 | 2.7. Conditions 153 | 154 | Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted 155 | in Section 2.1. 156 | 157 | 3. Responsibilities 158 | ------------------- 159 | 160 | 3.1. Distribution of Source Form 161 | 162 | All distribution of Covered Software in Source Code Form, including any 163 | Modifications that You create or to which You contribute, must be under 164 | the terms of this License. You must inform recipients that the Source 165 | Code Form of the Covered Software is governed by the terms of this 166 | License, and how they can obtain a copy of this License. You may not 167 | attempt to alter or restrict the recipients' rights in the Source Code 168 | Form. 169 | 170 | 3.2. Distribution of Executable Form 171 | 172 | If You distribute Covered Software in Executable Form then: 173 | 174 | (a) such Covered Software must also be made available in Source Code 175 | Form, as described in Section 3.1, and You must inform recipients of 176 | the Executable Form how they can obtain a copy of such Source Code 177 | Form by reasonable means in a timely manner, at a charge no more 178 | than the cost of distribution to the recipient; and 179 | 180 | (b) You may distribute such Executable Form under the terms of this 181 | License, or sublicense it under different terms, provided that the 182 | license for the Executable Form does not attempt to limit or alter 183 | the recipients' rights in the Source Code Form under this License. 184 | 185 | 3.3. Distribution of a Larger Work 186 | 187 | You may create and distribute a Larger Work under terms of Your choice, 188 | provided that You also comply with the requirements of this License for 189 | the Covered Software. If the Larger Work is a combination of Covered 190 | Software with a work governed by one or more Secondary Licenses, and the 191 | Covered Software is not Incompatible With Secondary Licenses, this 192 | License permits You to additionally distribute such Covered Software 193 | under the terms of such Secondary License(s), so that the recipient of 194 | the Larger Work may, at their option, further distribute the Covered 195 | Software under the terms of either this License or such Secondary 196 | License(s). 197 | 198 | 3.4. Notices 199 | 200 | You may not remove or alter the substance of any license notices 201 | (including copyright notices, patent notices, disclaimers of warranty, 202 | or limitations of liability) contained within the Source Code Form of 203 | the Covered Software, except that You may alter any license notices to 204 | the extent required to remedy known factual inaccuracies. 205 | 206 | 3.5. Application of Additional Terms 207 | 208 | You may choose to offer, and to charge a fee for, warranty, support, 209 | indemnity or liability obligations to one or more recipients of Covered 210 | Software. However, You may do so only on Your own behalf, and not on 211 | behalf of any Contributor. You must make it absolutely clear that any 212 | such warranty, support, indemnity, or liability obligation is offered by 213 | You alone, and You hereby agree to indemnify every Contributor for any 214 | liability incurred by such Contributor as a result of warranty, support, 215 | indemnity or liability terms You offer. You may include additional 216 | disclaimers of warranty and limitations of liability specific to any 217 | jurisdiction. 218 | 219 | 4. Inability to Comply Due to Statute or Regulation 220 | --------------------------------------------------- 221 | 222 | If it is impossible for You to comply with any of the terms of this 223 | License with respect to some or all of the Covered Software due to 224 | statute, judicial order, or regulation then You must: (a) comply with 225 | the terms of this License to the maximum extent possible; and (b) 226 | describe the limitations and the code they affect. Such description must 227 | be placed in a text file included with all distributions of the Covered 228 | Software under this License. Except to the extent prohibited by statute 229 | or regulation, such description must be sufficiently detailed for a 230 | recipient of ordinary skill to be able to understand it. 231 | 232 | 5. Termination 233 | -------------- 234 | 235 | 5.1. The rights granted under this License will terminate automatically 236 | if You fail to comply with any of its terms. However, if You become 237 | compliant, then the rights granted under this License from a particular 238 | Contributor are reinstated (a) provisionally, unless and until such 239 | Contributor explicitly and finally terminates Your grants, and (b) on an 240 | ongoing basis, if such Contributor fails to notify You of the 241 | non-compliance by some reasonable means prior to 60 days after You have 242 | come back into compliance. Moreover, Your grants from a particular 243 | Contributor are reinstated on an ongoing basis if such Contributor 244 | notifies You of the non-compliance by some reasonable means, this is the 245 | first time You have received notice of non-compliance with this License 246 | from such Contributor, and You become compliant prior to 30 days after 247 | Your receipt of the notice. 248 | 249 | 5.2. If You initiate litigation against any entity by asserting a patent 250 | infringement claim (excluding declaratory judgment actions, 251 | counter-claims, and cross-claims) alleging that a Contributor Version 252 | directly or indirectly infringes any patent, then the rights granted to 253 | You by any and all Contributors for the Covered Software under Section 254 | 2.1 of this License shall terminate. 255 | 256 | 5.3. In the event of termination under Sections 5.1 or 5.2 above, all 257 | end user license agreements (excluding distributors and resellers) which 258 | have been validly granted by You or Your distributors under this License 259 | prior to termination shall survive termination. 260 | 261 | ************************************************************************ 262 | * * 263 | * 6. Disclaimer of Warranty * 264 | * ------------------------- * 265 | * * 266 | * Covered Software is provided under this License on an "as is" * 267 | * basis, without warranty of any kind, either expressed, implied, or * 268 | * statutory, including, without limitation, warranties that the * 269 | * Covered Software is free of defects, merchantable, fit for a * 270 | * particular purpose or non-infringing. The entire risk as to the * 271 | * quality and performance of the Covered Software is with You. * 272 | * Should any Covered Software prove defective in any respect, You * 273 | * (not any Contributor) assume the cost of any necessary servicing, * 274 | * repair, or correction. This disclaimer of warranty constitutes an * 275 | * essential part of this License. No use of any Covered Software is * 276 | * authorized under this License except under this disclaimer. * 277 | * * 278 | ************************************************************************ 279 | 280 | ************************************************************************ 281 | * * 282 | * 7. Limitation of Liability * 283 | * -------------------------- * 284 | * * 285 | * Under no circumstances and under no legal theory, whether tort * 286 | * (including negligence), contract, or otherwise, shall any * 287 | * Contributor, or anyone who distributes Covered Software as * 288 | * permitted above, be liable to You for any direct, indirect, * 289 | * special, incidental, or consequential damages of any character * 290 | * including, without limitation, damages for lost profits, loss of * 291 | * goodwill, work stoppage, computer failure or malfunction, or any * 292 | * and all other commercial damages or losses, even if such party * 293 | * shall have been informed of the possibility of such damages. This * 294 | * limitation of liability shall not apply to liability for death or * 295 | * personal injury resulting from such party's negligence to the * 296 | * extent applicable law prohibits such limitation. Some * 297 | * jurisdictions do not allow the exclusion or limitation of * 298 | * incidental or consequential damages, so this exclusion and * 299 | * limitation may not apply to You. * 300 | * * 301 | ************************************************************************ 302 | 303 | 8. Litigation 304 | ------------- 305 | 306 | Any litigation relating to this License may be brought only in the 307 | courts of a jurisdiction where the defendant maintains its principal 308 | place of business and such litigation shall be governed by laws of that 309 | jurisdiction, without reference to its conflict-of-law provisions. 310 | Nothing in this Section shall prevent a party's ability to bring 311 | cross-claims or counter-claims. 312 | 313 | 9. Miscellaneous 314 | ---------------- 315 | 316 | This License represents the complete agreement concerning the subject 317 | matter hereof. If any provision of this License is held to be 318 | unenforceable, such provision shall be reformed only to the extent 319 | necessary to make it enforceable. Any law or regulation which provides 320 | that the language of a contract shall be construed against the drafter 321 | shall not be used to construe this License against a Contributor. 322 | 323 | 10. Versions of the License 324 | --------------------------- 325 | 326 | 10.1. New Versions 327 | 328 | Mozilla Foundation is the license steward. Except as provided in Section 329 | 10.3, no one other than the license steward has the right to modify or 330 | publish new versions of this License. Each version will be given a 331 | distinguishing version number. 332 | 333 | 10.2. Effect of New Versions 334 | 335 | You may distribute the Covered Software under the terms of the version 336 | of the License under which You originally received the Covered Software, 337 | or under the terms of any subsequent version published by the license 338 | steward. 339 | 340 | 10.3. Modified Versions 341 | 342 | If you create software not governed by this License, and you want to 343 | create a new license for such software, you may create and use a 344 | modified version of this License if you rename the license and remove 345 | any references to the name of the license steward (except to note that 346 | such modified license differs from this License). 347 | 348 | 10.4. Distributing Source Code Form that is Incompatible With Secondary 349 | Licenses 350 | 351 | If You choose to distribute Source Code Form that is Incompatible With 352 | Secondary Licenses under the terms of this version of the License, the 353 | notice described in Exhibit B of this License must be attached. 354 | 355 | Exhibit A - Source Code Form License Notice 356 | ------------------------------------------- 357 | 358 | This Source Code Form is subject to the terms of the Mozilla Public 359 | License, v. 2.0. If a copy of the MPL was not distributed with this 360 | file, You can obtain one at http://mozilla.org/MPL/2.0/. 361 | 362 | If it is not possible or desirable to put the notice in a particular 363 | file, then You may include the notice in a location (such as a LICENSE 364 | file in a relevant directory) where a recipient would be likely to look 365 | for such a notice. 366 | 367 | You may add additional accurate notices of copyright ownership. 368 | 369 | Exhibit B - "Incompatible With Secondary Licenses" Notice 370 | --------------------------------------------------------- 371 | 372 | This Source Code Form is "Incompatible With Secondary Licenses", as 373 | defined by the Mozilla Public License, v. 2.0. 374 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT = rabbitmq_message_deduplication 2 | 3 | RABBITMQ_VERSION ?= v4.1.x 4 | current_rmq_ref = $(RABBITMQ_VERSION) 5 | 6 | # The Application needs to depend on `rabbit` in order to be detected as a plugin. 7 | DEPS = rabbit 8 | TEST_DEPS = rabbitmq_ct_helpers rabbitmq_ct_client_helpers amqp_client 9 | 10 | DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk 11 | DEP_EARLY_PLUGINS = rabbit_common/mk/rabbitmq-early-plugin.mk 12 | 13 | # Mix customizations 14 | MIX_ENV ?= dev 15 | override MIX := mix 16 | 17 | # RMQ `dist` target removes anything within the `plugins` folder 18 | # which is not managed by erlang.mk. 19 | # We need to instruct the `rabbitmq-dist:do-dist` target to not 20 | # remove our plugin and related dependencies. 21 | EXTRA_DIST_EZS = $(shell find $(DIST_DIR) -name '*.ez') 22 | 23 | app:: deps 24 | $(MIX) make_app 25 | 26 | tests:: app test-build 27 | $(MIX) make_tests 28 | 29 | dist:: app 30 | mkdir -p $(DIST_DIR) 31 | $(MIX) make_archives 32 | 33 | clean:: 34 | @rm -fr _build 35 | 36 | include rabbitmq-components.mk 37 | include erlang.mk 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RabbitMQ Message Deduplication Plugin 2 | 3 | [![Build Status](https://github.com/noxdafox/rabbitmq-message-deduplication/actions/workflows/action.yml/badge.svg)](https://github.com/noxdafox/rabbitmq-message-deduplication/actions/workflows/action.yml) 4 | 5 | A plugin for filtering duplicate messages. 6 | 7 | Messages can be deduplicated when published into an exchange or enqueued to a queue. 8 | 9 | ## Installing 10 | 11 | Download the `.ez` files from the chosen [release](https://github.com/noxdafox/rabbitmq-message-deduplication/releases) and copy them into the [RabbitMQ plugins directory](http://www.rabbitmq.com/relocate.html). 12 | 13 | Check the Release notes for minimum supported versions. 14 | 15 | Enable the plugin: 16 | 17 | ```bash 18 | [sudo] rabbitmq-plugins enable rabbitmq_message_deduplication 19 | ``` 20 | 21 | ## Building from Source 22 | 23 | Please see RabbitMQ Plugin Development guide. 24 | 25 | To build the plugin: 26 | 27 | ```bash 28 | git clone https://github.com/noxdafox/rabbitmq-message-deduplication.git 29 | cd rabbitmq-message-deduplication 30 | make dist 31 | ``` 32 | 33 | Then copy all the *.ez files inside the plugins folder to the [RabbitMQ plugins directory](http://www.rabbitmq.com/relocate.html) and enable the plugin: 34 | 35 | ```bash 36 | [sudo] rabbitmq-plugins enable rabbitmq_message_deduplication 37 | ``` 38 | 39 | ## Version requirements 40 | 41 | The latest version of the plugin requires RabbitMQ 3.13.0. 42 | 43 | Earlier RabbitMQ versions are supported by 0.6.2. 44 | 45 | ## Exchange level deduplication 46 | 47 | The exchange type `x-message-deduplication` allows to filter message duplicates before any routing rule is applied. 48 | 49 | Each message containing the `x-deduplication-header` header will not be routed if its value has been submitted previously. The amount of time a given message will be guaranteed to be unique can be controlled via the `x-cache-ttl` exchange argument or message header. 50 | 51 | > **_NOTE;_** This exchange acts like a [`fanout` exchange](https://www.rabbitmq.com/tutorials/amqp-concepts.html#exchange-fanout), so routing rules are not applied. 52 | 53 | ### Declare an exchange 54 | 55 | To create a message deduplication exchange, just declare it providing the type `x-message-deduplication`. 56 | 57 | Required arguments: 58 | 59 | * `x-cache-size`: maximum number of entries for the deduplication cache. If the deduplication cache fills up, unspecified existing entries will be removed to give space to new ones. 60 | 61 | Optional arguments: 62 | 63 | * `x-cache-ttl`: amount of time in milliseconds duplicate headers are kept in cache. 64 | * `x-cache-persistence`: whether the duplicates cache will persist on disk or in memory. 65 | Default persistence type is `memory`. 66 | 67 | ### Message headers 68 | 69 | * `x-deduplication-header`: messages will be deduplicated based on the content of this header. If the header is not provided, the message will not be checked against duplicates. 70 | * `x-cache-ttl`: this header is optional and will override the default value provided during the exchange declaration. This header controls for how many milliseconds to deduplicate the message. After the TTL expires, a new message with the same header will be routed again. 71 | 72 | ## Queue level deduplication 73 | 74 | A queue declared with the `x-message-deduplication` parameter enabled will filter message duplicates before they are published within. 75 | 76 | Each message containing the `x-deduplication-header` header will not be enqueued if another message with the same header is already present within the queue. 77 | 78 | > **_NOTE:_** Mirrored and Quorum queues are currently not supported. 79 | 80 | ### Declare a queue 81 | 82 | When declaring a queue, it is possible to enable message deduplication via the `x-message-deduplication` boolean argument. 83 | 84 | ### Message headers 85 | 86 | * `x-deduplication-header`: messages will be deduplicated based on the content of this header. If the header is not provided, the message will not be checked against duplicates. 87 | 88 | ## Disabling the Plugin 89 | 90 | It is possible to disable the plugin via the command: 91 | 92 | ```bash 93 | [sudo] rabbitmq-plugins disable rabbitmq_message_deduplication 94 | ``` 95 | 96 | All deduplication exchanges and queues will be rendered non functional. It is responsibility of the User to remove them. 97 | 98 | ## Running the tests 99 | 100 | ```bash 101 | make tests 102 | ``` 103 | 104 | ## License 105 | 106 | See the LICENSE file. 107 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | alias :timer, as: Timer 4 | 5 | config :rabbitmq_message_deduplication, 6 | cache_wait_time: Timer.seconds(30), 7 | cache_cleanup_period: Timer.seconds(3) 8 | -------------------------------------------------------------------------------- /erlang.mk: -------------------------------------------------------------------------------- 1 | # Automated update. 2 | 3 | ERLANG_MK_BUILD_CONFIG ?= build.config 4 | ERLANG_MK_BUILD_DIR ?= .erlang.mk.build 5 | 6 | erlang.mk: bootstrap 7 | git clone https://github.com/ninenines/erlang.mk $(ERLANG_MK_BUILD_DIR) 8 | if [ -f $(ERLANG_MK_BUILD_CONFIG) ]; then cp $(ERLANG_MK_BUILD_CONFIG) $(ERLANG_MK_BUILD_DIR); fi 9 | cd $(ERLANG_MK_BUILD_DIR) && $(MAKE) 10 | cp $(ERLANG_MK_BUILD_DIR)/erlang.mk ./erlang.mk 11 | rm -rf $(ERLANG_MK_BUILD_DIR) 12 | 13 | .PHONY: bootstrap 14 | bootstrap: ; 15 | -------------------------------------------------------------------------------- /lib/rabbitmq_message_deduplication.ex: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2025, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication do 9 | 10 | use Application 11 | 12 | # Start a dummy supervisor to enable the Application behaviour. 13 | # https://erlang.org/pipermail/erlang-questions/2010-April/050508.html 14 | @impl true 15 | def start(_, _) do 16 | Supervisor.start_link(__MODULE__, [], []) 17 | end 18 | 19 | @impl true 20 | def stop(_) do 21 | RabbitMQMessageDeduplication.Exchange.unregister() 22 | RabbitMQMessageDeduplication.Queue.disable() 23 | RabbitMQMessageDeduplication.PolicyEvent.disable() 24 | end 25 | 26 | def init([]) do 27 | Supervisor.init([], strategy: :one_for_one) 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /lib/rabbitmq_message_deduplication/cache.ex: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2025, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.Cache do 9 | @moduledoc """ 10 | Simple cache implemented on top of Mnesia. 11 | 12 | Entries can be stored within the cache with a given TTL. 13 | After the TTL expires the entrys will be transparently removed. 14 | 15 | When the cache is full, a random element is removed to make space to a new one. 16 | A FIFO approach would be preferrable but impractical by now due to Mnesia limitations. 17 | 18 | """ 19 | alias :os, as: Os 20 | alias :erlang, as: Erlang 21 | alias :mnesia, as: Mnesia 22 | alias RabbitMQMessageDeduplication.Common, as: Common 23 | 24 | @options [:size, :ttl, :distributed, :limit, :default_ttl] 25 | 26 | @doc """ 27 | Create a new cache with the given name and options. 28 | 29 | A distributed cache is replicated across multiple nodes. 30 | 31 | """ 32 | @spec create(atom, boolean, list) :: :ok | { :error, any } 33 | def create(cache, distributed, options) do 34 | Mnesia.start() 35 | 36 | case cache_create(cache, distributed, options) do 37 | {_, reason} -> {:error, reason} 38 | result -> result 39 | end 40 | end 41 | 42 | @doc """ 43 | Insert the given entry into the cache if it doesn't exist. 44 | The TTL controls the lifetime in milliseconds of the entry. 45 | 46 | If the cache is full, an entry will be removed to make space. 47 | 48 | """ 49 | @spec insert(atom, any, integer | nil) :: 50 | { :ok, :inserted | :exists } | { :error, any } 51 | def insert(cache, entry, ttl \\ nil) do 52 | function = fn -> 53 | if cache_member?(cache, entry) do 54 | :exists 55 | else 56 | if cache_full?(cache) do 57 | cache_delete_first(cache) 58 | end 59 | 60 | Mnesia.write({cache, entry, entry_expiration(cache, ttl)}) 61 | 62 | :inserted 63 | end 64 | end 65 | 66 | case Mnesia.transaction(function) do 67 | {:atomic, result} -> {:ok, result} 68 | {:aborted, reason} -> {:error, reason} 69 | end 70 | end 71 | 72 | @doc """ 73 | Delete the given entry from the cache. 74 | """ 75 | @spec delete(atom, any) :: :ok | { :error, any } 76 | def delete(cache, entry) do 77 | case Mnesia.transaction(fn -> Mnesia.delete({cache, entry}) end) do 78 | {:atomic, :ok} -> :ok 79 | {:aborted, reason} -> {:error, reason} 80 | end 81 | end 82 | 83 | @doc """ 84 | Flush the cache content. 85 | """ 86 | @spec flush(atom) :: :ok | { :error, any } 87 | def flush(cache) do 88 | case Mnesia.clear_table(cache) do 89 | {:atomic, :ok} -> :ok 90 | {:aborted, reason} -> {:error, reason} 91 | end 92 | end 93 | 94 | @doc """ 95 | Drop the cache with all its content. 96 | """ 97 | @spec drop(atom) :: :ok | { :error, any } 98 | def drop(cache) do 99 | case Mnesia.delete_table(cache) do 100 | {:atomic, :ok} -> :ok 101 | {:aborted, reason} -> {:error, reason} 102 | end 103 | end 104 | 105 | @doc """ 106 | Remove all entries which TTL has expired. 107 | """ 108 | @spec delete_expired_entries(atom) :: :ok | { :error, any } 109 | def delete_expired_entries(cache) do 110 | select = fn -> 111 | Mnesia.select(cache, [{{cache, :"$1", :"$2"}, 112 | [{:>, Os.system_time(:millisecond), :"$2"}], 113 | [:"$1"]}]) 114 | end 115 | 116 | delete = fn x -> Enum.each(x, fn e -> Mnesia.delete({cache, e}) end) end 117 | 118 | case Mnesia.transaction(select) do 119 | {:atomic, expired} -> case Mnesia.transaction(delete, [expired], 1) do 120 | {:atomic, :ok} -> :ok 121 | {:aborted, reason} -> {:error, reason} 122 | end 123 | {:aborted, {:no_exists, _}} -> {:error, :no_cache} 124 | end 125 | end 126 | 127 | @doc """ 128 | Return information related to the given cache. 129 | """ 130 | @spec info(atom) :: list 131 | def info(cache) do 132 | with entries when is_integer(entries) <- Mnesia.table_info(cache, :size), 133 | words when is_integer(words) <- Mnesia.table_info(cache, :memory) 134 | do 135 | {_, nodes} = cache_layout(cache) 136 | bytes = words * Erlang.system_info(:wordsize) 137 | 138 | case cache_property(cache, :size) do 139 | nil -> [entries: entries, bytes: bytes, nodes: nodes] 140 | size -> [entries: entries, bytes: bytes, nodes: nodes, size: size] 141 | end 142 | else 143 | :undefined -> [] 144 | end 145 | end 146 | 147 | @doc """ 148 | Rebalance cache replicas. 149 | """ 150 | @spec rebalance_replicas(atom) :: any 151 | def rebalance_replicas(cache) do 152 | if cache_property(cache, :distributed) do 153 | cache_rebalance(cache) 154 | end 155 | end 156 | 157 | @doc """ 158 | Change cache options. 159 | """ 160 | @spec change_option(atom, atom, any) :: :ok | { :error, any } 161 | def change_option(cache, option, value) when option in @options do 162 | :ok = cache_property(cache, option, value) 163 | end 164 | def change_option(_, option, _), do: {:error, {:invalid, option}} 165 | 166 | ## Utility functions 167 | 168 | # Mnesia cache table creation. 169 | defp cache_create(cache, distributed, options) do 170 | persistence = case Keyword.get(options, :persistence) do 171 | :disk -> :disc_copies 172 | :memory -> :ram_copies 173 | end 174 | replicas = if distributed, do: cache_replicas(), else: [Node.self()] 175 | options = [{:attributes, [:entry, :expiration]}, 176 | {persistence, replicas}, 177 | {:index, [:expiration]}, 178 | {:user_properties, [{:distributed, distributed}, 179 | {:size, Keyword.get(options, :size)}, 180 | {:ttl, Keyword.get(options, :ttl)}]}] 181 | 182 | case Mnesia.create_table(cache, options) do 183 | {:atomic, :ok} -> 184 | wait_for_cache(cache) 185 | {:aborted, reason} when elem(reason, 0) == :already_exists -> 186 | maybe_reconfigure(cache, distributed) 187 | error -> 188 | error 189 | end 190 | end 191 | 192 | # Wait for the table to be loaded and force it in case of timeout 193 | defp wait_for_cache(cache) do 194 | case Mnesia.wait_for_tables([cache], Common.cache_wait_time()) do 195 | {:timeout, [cache]} -> Mnesia.force_load_table(cache) 196 | result -> result 197 | end 198 | end 199 | 200 | # Lookup the entry within the cache, deletes the entry if expired 201 | # Must be included within transaction. 202 | defp cache_member?(cache, entry) do 203 | case cache |> Mnesia.read(entry) |> List.keyfind(entry, 1) do 204 | {_, _, expiration} -> if expiration <= Os.system_time(:millisecond) do 205 | Mnesia.delete({cache, entry}) 206 | false 207 | else 208 | true 209 | end 210 | nil -> false 211 | end 212 | end 213 | 214 | # Delete the first element from the cache. 215 | # As the Mnesia Set is not ordered, the first element is random. 216 | # Must be included within transaction. 217 | defp cache_delete_first(cache) do 218 | Mnesia.delete({cache, Mnesia.first(cache)}) 219 | end 220 | 221 | # True if the cache is full, false otherwise. 222 | defp cache_full?(cache) do 223 | Mnesia.table_info(cache, :size) >= cache_property(cache, :size) 224 | end 225 | 226 | # Calculate the expiration given a TTL or the cache default TTL 227 | defp entry_expiration(cache, ttl) do 228 | default = cache_property(cache, :ttl) 229 | 230 | cond do 231 | ttl != nil -> Os.system_time(:millisecond) + ttl 232 | default != nil -> Os.system_time(:millisecond) + default 233 | true -> nil 234 | end 235 | end 236 | 237 | # Retrieve the given property from the Mnesia user_properties field 238 | defp cache_property(cache, property) do 239 | cache |> Mnesia.table_info(:user_properties) |> Keyword.get(property) 240 | end 241 | 242 | # Set the given Mnesia user_properties field 243 | defp cache_property(cache, property, value) when property in @options do 244 | case Mnesia.write_table_property(cache, {property, value}) do 245 | {:atomic, :ok} -> :ok 246 | {:aborted, error} -> {:error, error} 247 | end 248 | end 249 | 250 | # Rebalance a distributed cache across the cluster nodes 251 | defp cache_rebalance(cache) do 252 | {storage_type, cache_nodes} = cache_layout(cache) 253 | 254 | for node <- cache_replicas(cache_nodes) do 255 | case Mnesia.add_table_copy(cache, node, storage_type) do 256 | {:atomic, :ok} -> 257 | wait_for_cache(cache) 258 | {:aborted, reason} when elem(reason, 0) == :already_exists -> 259 | maybe_reconfigure(cache, true) 260 | end 261 | end 262 | end 263 | 264 | # List the nodes on which to create the cache replicas. 265 | # Distributed caches are replicated on two-thirds of the cluster nodes. 266 | defp cache_replicas(cache_nodes \\ []) do 267 | cluster_nodes = Mnesia.system_info(:running_db_nodes) 268 | replica_number = floor((length(cluster_nodes) * 2) / 3) 269 | 270 | Enum.take(cache_nodes ++ (cluster_nodes -- cache_nodes), replica_number) 271 | end 272 | 273 | # Returns a tuple {persistence, nodes} 274 | defp cache_layout(cache) do 275 | case Mnesia.table_info(cache, :ram_copies) do 276 | [] -> {:disc_copies, Mnesia.table_info(cache, :disc_copies)} 277 | nodes -> {:ram_copies, nodes} 278 | end 279 | end 280 | 281 | # Caches created prior to v0.6.0 need to be reconfigured. 282 | defp maybe_reconfigure(cache, distributed) do 283 | if cache_property(cache, :distributed) == nil do 284 | cache_property(cache, :distributed, distributed) 285 | cache_property(cache, :size, cache_property(cache, :limit)) 286 | cache_property(cache, :ttl, cache_property(cache, :default_ttl)) 287 | 288 | Mnesia.delete_table_property(cache, :limit) 289 | Mnesia.delete_table_property(cache, :default_ttl) 290 | end 291 | 292 | wait_for_cache(cache) 293 | end 294 | end 295 | -------------------------------------------------------------------------------- /lib/rabbitmq_message_deduplication/cache_manager.ex: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2025, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.CacheManager do 9 | @moduledoc """ 10 | The Cache Manager takes care of creating, maintaining and destroying caches. 11 | """ 12 | 13 | use GenServer 14 | 15 | require RabbitMQMessageDeduplication.Cache 16 | 17 | alias :timer, as: Timer 18 | alias :mnesia, as: Mnesia 19 | alias RabbitMQMessageDeduplication.Cache, as: Cache 20 | alias RabbitMQMessageDeduplication.Common, as: Common 21 | 22 | Module.register_attribute(__MODULE__, 23 | :rabbit_boot_step, 24 | accumulate: true, persist: true) 25 | 26 | @rabbit_boot_step { 27 | __MODULE__, 28 | [description: "message deduplication plugin cache maintenance process", 29 | mfa: {:rabbit_sup, :start_child, [__MODULE__]}, 30 | cleanup: {:rabbit_sup, :stop_child, [__MODULE__]}, 31 | requires: :database, 32 | enables: :external_infrastructure]} 33 | 34 | def start_link() do 35 | GenServer.start_link(__MODULE__, %{}, name: __MODULE__) 36 | end 37 | 38 | @doc """ 39 | Create the cache and register it within the maintenance process. 40 | """ 41 | @spec create(atom, boolean, list) :: :ok | { :error, any } 42 | def create(cache, distributed, options) do 43 | try do 44 | timeout = Common.cache_wait_time() + Timer.seconds(5) 45 | 46 | GenServer.call(__MODULE__, {:create, cache, distributed, options}, timeout) 47 | catch 48 | :exit, {:noproc, _} -> {:error, :noproc} 49 | end 50 | end 51 | 52 | @doc """ 53 | Destroy the cache and remove it from the maintenance process. 54 | """ 55 | @spec destroy(atom) :: :ok | { :error, any } 56 | def destroy(cache) do 57 | try do 58 | GenServer.call(__MODULE__, {:destroy, cache}) 59 | catch 60 | :exit, {:noproc, _} -> {:error, :noproc} 61 | end 62 | end 63 | 64 | @doc """ 65 | Disable the cache and terminate the manager process. 66 | """ 67 | def disable() do 68 | {:ok, _node} = Mnesia.unsubscribe(:system) 69 | :ok = Supervisor.terminate_child(:rabbit_sup, __MODULE__) 70 | :ok = Supervisor.delete_child(:rabbit_sup, __MODULE__) 71 | end 72 | 73 | ## Server Callbacks 74 | 75 | # Run Mnesia creation functions handling output 76 | defmacro mnesia_create(function) do 77 | quote do 78 | case unquote(function) do 79 | {:atomic, :ok} -> :ok 80 | {:aborted, {:already_exists, _}} -> :ok 81 | {:aborted, {:already_exists, _, _}} -> :ok 82 | error -> error 83 | end 84 | end 85 | end 86 | 87 | # Create the cache table and start the cleanup routine. 88 | def init(state) do 89 | Mnesia.start() 90 | 91 | with :ok <- mnesia_create(Mnesia.create_table(caches(), [])), 92 | :ok <- mnesia_create(Mnesia.add_table_copy(caches(), node(), :ram_copies)), 93 | :ok <- Mnesia.wait_for_tables([caches()], Common.cache_wait_time()), 94 | {:ok, _node} <- Mnesia.subscribe(:system) 95 | do 96 | Process.send_after(__MODULE__, :cleanup, Common.cleanup_period()) 97 | {:ok, state} 98 | else 99 | {:timeout, reason} -> {:error, reason} 100 | error -> error 101 | end 102 | end 103 | 104 | # Create the cache and add it to the Mnesia caches table 105 | def handle_call({:create, cache, distributed, options}, _from, state) do 106 | function = fn -> Mnesia.write({caches(), cache, :nil}) end 107 | 108 | with :ok <- Cache.create(cache, distributed, options), 109 | {:atomic, result} <- Mnesia.transaction(function) 110 | do 111 | {:reply, result, state} 112 | else 113 | {:aborted, reason} -> {:reply, {:error, reason}, state} 114 | error -> {:reply, error, state} 115 | end 116 | end 117 | 118 | # Drop the cache and remove it from the Mnesia caches table 119 | def handle_call({:destroy, cache}, _from, state) do 120 | function = fn -> Mnesia.delete({caches(), cache}) end 121 | 122 | with :ok <- Cache.drop(cache), 123 | {:atomic, result} <- Mnesia.transaction(function) 124 | do 125 | {:reply, result, state} 126 | else 127 | {:aborted, reason} -> {:reply, {:error, reason}, state} 128 | error -> {:reply, error, state} 129 | end 130 | end 131 | 132 | # The maintenance process deletes expired cache entries. 133 | def handle_info(:cleanup, state) do 134 | {:atomic, caches} = Mnesia.transaction(fn -> Mnesia.all_keys(caches()) end) 135 | Enum.each(caches, &Cache.delete_expired_entries/1) 136 | Process.send_after(__MODULE__, :cleanup, Common.cleanup_period()) 137 | 138 | {:noreply, state} 139 | end 140 | 141 | # On node addition distribute cache tables 142 | def handle_info({:mnesia_system_event, {:mnesia_up, _node}}, state) do 143 | {:atomic, caches} = Mnesia.transaction(fn -> Mnesia.all_keys(caches()) end) 144 | Enum.each(caches, &Cache.rebalance_replicas/1) 145 | 146 | {:noreply, state} 147 | end 148 | 149 | def handle_info({:mnesia_system_event, _event}, state) do 150 | {:noreply, state} 151 | end 152 | 153 | def caches(), do: :message_deduplication_caches 154 | end 155 | -------------------------------------------------------------------------------- /lib/rabbitmq_message_deduplication/common.ex: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2025, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.Common do 9 | @moduledoc """ 10 | Common functions shared between the exchange and the queue 11 | behaviour implementations. 12 | 13 | """ 14 | 15 | require RabbitMQMessageDeduplication.Cache 16 | 17 | alias :mc, as: MC 18 | alias :timer, as: Timer 19 | alias RabbitMQMessageDeduplication.Cache, as: Cache 20 | 21 | @default_arguments %{type: nil, default: nil} 22 | 23 | @doc """ 24 | Retrieve a configuration value from a list of arguments or policies. 25 | """ 26 | @spec rabbit_argument(List.t, String.t, List.t) :: String.t | nil 27 | def rabbit_argument(arguments, argument, opts \\ []) do 28 | %{type: type, default: default} = Enum.into(opts, @default_arguments) 29 | 30 | case type do 31 | :number -> case rabbit_keyfind(arguments, argument, default) do 32 | number when is_bitstring(number) -> String.to_integer(number) 33 | integer when is_integer(integer) -> integer 34 | ^default -> default 35 | end 36 | :atom -> case rabbit_keyfind(arguments, argument, default) do 37 | string when is_bitstring(string) -> String.to_atom(string) 38 | ^default -> default 39 | end 40 | nil -> rabbit_keyfind(arguments, argument, default) 41 | end 42 | end 43 | 44 | @doc """ 45 | Retrieve the given header from the message. 46 | """ 47 | @spec message_header(MC.state, String.t) :: String.t | integer() | float() | boolean() | :undefined | nil 48 | def message_header(message, header) do 49 | case MC.x_header(header, message) do 50 | {_type, value} when not is_list(value) and not is_tuple(value) -> 51 | # list and tuple values have type-tagged elements 52 | # that would need to be untagged recursively 53 | # we don't expect to use such headers, so those cases are not handled 54 | value 55 | :null -> 56 | # header value in AMQP message was {:void, :undefined} 57 | 58 | # pre-3.13 version of this function used rabbit_keyfind/2 59 | # which returned :undefined instead of nil or :void. We have to 60 | # keep this value as this is used in keys to cache the message 61 | # and is preserved during a rolling upgrade in a replicated 62 | # Mnesia table 63 | :undefined 64 | :undefined -> nil 65 | end 66 | end 67 | 68 | @doc """ 69 | Check if the routed/queued message is a duplicate. 70 | 71 | If not, it adds it to the cache with the corresponding name. 72 | """ 73 | @spec duplicate?(tuple, MC.state, integer | nil) :: boolean 74 | def duplicate?(name, message, ttl \\ nil) do 75 | cache = cache_name(name) 76 | 77 | case message_header(message, "x-deduplication-header") do 78 | key when not is_nil(key) -> case Cache.insert(cache, key, ttl) do 79 | {:ok, :exists} -> true 80 | {:ok, :inserted} -> false 81 | end 82 | nil -> false 83 | end 84 | end 85 | 86 | @doc """ 87 | Constructs a sanitized cache name from a tuple containing 88 | the VHost and the exchange/queue name. 89 | """ 90 | @spec cache_name({:resource, String.t, :exchange | :queue, String.t}) :: atom 91 | def cache_name({:resource, resource, :exchange, exchange}) do 92 | resource = sanitize_string(resource) 93 | exchange = sanitize_string(exchange) 94 | 95 | String.to_atom("cache_exchange_#{resource}_#{exchange}") 96 | end 97 | 98 | def cache_name({:resource, resource, :queue, queue}) do 99 | resource = sanitize_string(resource) 100 | queue = sanitize_string(queue) 101 | 102 | String.to_atom("cache_queue_#{resource}_#{queue}") 103 | end 104 | 105 | def rabbit_keyfind(list, key, default \\ nil) do 106 | case List.keyfind(list, key, 0) do 107 | {_key, _type, value} -> value 108 | {_key, value} -> value 109 | _ -> default 110 | end 111 | end 112 | 113 | @doc """ 114 | Retrieve 115 | """ 116 | def cache_wait_time() do 117 | Application.get_env(appname(), :cache_wait_time, Timer.seconds(30)) 118 | end 119 | 120 | def cleanup_period() do 121 | Application.get_env(appname(), :cache_cleanup_period, Timer.seconds(3)) 122 | end 123 | 124 | defp sanitize_string(string) do 125 | string 126 | |> String.replace(~r/[-\. ]/, "_") 127 | |> String.replace("/", "") 128 | |> String.downcase() 129 | end 130 | 131 | defp appname(), do: Application.get_application(__MODULE__) 132 | end 133 | -------------------------------------------------------------------------------- /lib/rabbitmq_message_deduplication/rabbit_message_deduplication_exchange.ex: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2025, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.Exchange do 9 | @moduledoc """ 10 | This module adds support for deduplication exchanges. 11 | 12 | Messages carrying the `x-deduplication-header` header will be deduplicated 13 | if a message with the same header value was already seen before. 14 | 15 | When a message is routed within the exchange, it's checked against duplicates. 16 | If no duplicate is found, the message is routed and its deduplication header 17 | cached. If a TTL was set, the header is removed once expired. 18 | If the deduplication cache fills up, old elements will be removed 19 | to make space to new onces. 20 | 21 | This module implements the `rabbit_exchange_type` behaviour. 22 | 23 | """ 24 | 25 | import Record, only: [defrecord: 2, extract: 2] 26 | 27 | require RabbitMQMessageDeduplication.Cache 28 | require RabbitMQMessageDeduplication.Common 29 | 30 | alias :rabbit_log, as: RabbitLog 31 | alias :rabbit_misc, as: RabbitMisc 32 | alias :rabbit_policy, as: RabbitPolicy 33 | alias :rabbit_router, as: RabbitRouter 34 | alias :rabbit_exchange, as: RabbitExchange 35 | alias :rabbit_registry, as: RabbitRegistry 36 | alias RabbitMQMessageDeduplication.Common, as: Common 37 | alias RabbitMQMessageDeduplication.Cache, as: Cache 38 | alias RabbitMQMessageDeduplication.CacheManager, as: CacheManager 39 | 40 | @behaviour :rabbit_exchange_type 41 | 42 | Module.register_attribute(__MODULE__, 43 | :rabbit_boot_step, 44 | accumulate: true, persist: true) 45 | 46 | @exchange_type <<"x-message-deduplication">> 47 | @rabbit_boot_step {__MODULE__, 48 | [{:description, "exchange type x-message-deduplication"}, 49 | {:mfa, {__MODULE__, :register, []}}, 50 | {:requires, :rabbit_registry}, 51 | {:enables, :kernel_ready}]} 52 | 53 | defrecord :exchange, extract( 54 | :exchange, from_lib: "rabbit_common/include/rabbit.hrl") 55 | 56 | @doc """ 57 | Register the exchange type within the Broker. 58 | """ 59 | @spec register() :: :ok 60 | def register() do 61 | RabbitRegistry.register(:exchange, @exchange_type, __MODULE__) 62 | maybe_reconfigure_caches() 63 | end 64 | 65 | @doc """ 66 | Unregister the exchange type from the Broker. 67 | """ 68 | @spec unregister() :: :ok 69 | def unregister() do 70 | RabbitRegistry.unregister(:exchange, @exchange_type) 71 | end 72 | 73 | @impl :rabbit_exchange_type 74 | def description() do 75 | [ 76 | {:name, @exchange_type}, 77 | {:description, <<"Message Deduplication Exchange.">>} 78 | ] 79 | end 80 | 81 | @impl :rabbit_exchange_type 82 | def serialise_events() do 83 | false 84 | end 85 | 86 | @impl :rabbit_exchange_type 87 | def route(exchange(name: name), msg, _opts) do 88 | if route?(name, msg) do 89 | RabbitRouter.match_routing_key(name, [:_]) 90 | else 91 | [] 92 | end 93 | end 94 | 95 | @impl :rabbit_exchange_type 96 | def validate(exchange(arguments: args)) do 97 | case List.keyfind(args, "x-cache-size", 0) do 98 | {"x-cache-size", _, val} when is_integer(val) and val > 0 -> :ok 99 | {"x-cache-size", :longstr, val} -> 100 | case Integer.parse(val, 10) do 101 | :error -> RabbitMisc.protocol_error( 102 | :precondition_failed, 103 | "Missing or invalid argument, \ 104 | 'x-cache-size' must be an integer greater than 0", []) 105 | _ -> :ok 106 | end 107 | _ -> 108 | RabbitMisc.protocol_error( 109 | :precondition_failed, 110 | "Missing or invalid argument, \ 111 | 'x-cache-size' must be an integer greater than 0", []) 112 | end 113 | 114 | case List.keyfind(args, "x-cache-ttl", 0) do 115 | nil -> :ok 116 | {"x-cache-ttl", _, val} when is_integer(val) and val > 0 -> :ok 117 | {"x-cache-ttl", :longstr, val} -> 118 | case Integer.parse(val, 10) do 119 | :error -> RabbitMisc.protocol_error( 120 | :precondition_failed, 121 | "Invalid argument, \ 122 | 'x-cache-ttl' must be an integer greater than 0", []) 123 | _ -> :ok 124 | end 125 | _ -> RabbitMisc.protocol_error( 126 | :precondition_failed, 127 | "Invalid argument, \ 128 | 'x-cache-ttl' must be an integer greater than 0", []) 129 | end 130 | 131 | case List.keyfind(args, "x-cache-persistence", 0) do 132 | nil -> :ok 133 | {"x-cache-persistence", :longstr, "disk"} -> :ok 134 | {"x-cache-persistence", :longstr, "memory"} -> :ok 135 | _ -> RabbitMisc.protocol_error( 136 | :precondition_failed, 137 | "Invalid argument, \ 138 | 'x-cache-persistence' must be either 'disk' or 'memory'", []) 139 | end 140 | end 141 | 142 | @impl :rabbit_exchange_type 143 | def validate_binding(_ex, _bs) do 144 | :ok 145 | end 146 | 147 | @impl :rabbit_exchange_type 148 | def create(_sr, exchange(name: name, arguments: args)) do 149 | cache = Common.cache_name(name) 150 | options = format_options(args) 151 | 152 | RabbitLog.debug( 153 | "Starting exchange deduplication cache ~s with options ~p~n", 154 | [cache, options]) 155 | 156 | CacheManager.create(cache, true, options) 157 | end 158 | 159 | @impl :rabbit_exchange_type 160 | def delete(_sr, exchange(name: name)) do 161 | name |> Common.cache_name() |> CacheManager.destroy() 162 | end 163 | 164 | def delete(_tx, exchange, _bs), do: delete(:none, exchange) 165 | 166 | @impl :rabbit_exchange_type 167 | def policy_changed(_ex, exchange(name: name, arguments: args, policy: :undefined)) do 168 | cache = Common.cache_name(name) 169 | 170 | RabbitLog.debug( 171 | "All policies for exchange ~p were deleted, resetting to defaults ~p ~n", 172 | [name, format_options(args)]) 173 | 174 | reset_arguments(cache, args) 175 | end 176 | 177 | @impl :rabbit_exchange_type 178 | def policy_changed(_ex, exch = exchange(name: name, arguments: args, policy: policy)) do 179 | cache = Common.cache_name(name) 180 | 181 | RabbitLog.debug("Applying ~s policy to exchange ~p ~n", 182 | [RabbitPolicy.name(exch), name]) 183 | 184 | # We need to remove old policy before applying new one 185 | reset_arguments(cache, args) 186 | 187 | for policy_definition <- policy[:definition] do 188 | case policy_definition do 189 | {"x-cache-ttl", value} -> Cache.change_option(cache, :ttl, value) 190 | {"x-cache-size", value} -> Cache.change_option(cache, :size, value) 191 | {"x-cache-persistence", value} -> Cache.change_option(cache, :persistence, value) 192 | end 193 | end 194 | end 195 | 196 | @impl :rabbit_exchange_type 197 | def add_binding(_tx, _ex, _bs), do: :ok 198 | 199 | @impl :rabbit_exchange_type 200 | def remove_bindings(_tx, _ex, _bs), do: :ok 201 | 202 | @impl :rabbit_exchange_type 203 | def assert_args_equivalence(exch, args) do 204 | RabbitExchange.assert_args_equivalence(exch, args) 205 | end 206 | 207 | @impl :rabbit_exchange_type 208 | def info(exchange) do 209 | info(exchange, [:cache_info]) 210 | end 211 | 212 | @impl :rabbit_exchange_type 213 | def info(exchange(name: name), [:cache_info]) do 214 | [cache_info: name |> Common.cache_name() |> Cache.info()] 215 | end 216 | 217 | @impl :rabbit_exchange_type 218 | def info(_ex, _it) do 219 | [] 220 | end 221 | 222 | # Utility functions 223 | 224 | # Whether to route the message or not. 225 | defp route?(exchange_name, message) do 226 | ttl = Common.message_header(message, "x-cache-ttl") 227 | not Common.duplicate?(exchange_name, message, ttl) 228 | end 229 | 230 | # Format arguments into options 231 | defp format_options(args) do 232 | [size: Common.rabbit_argument( 233 | args, "x-cache-size", type: :number), 234 | ttl: Common.rabbit_argument( 235 | args, "x-cache-ttl", type: :number), 236 | persistence: Common.rabbit_argument( 237 | args, "x-cache-persistence", type: :atom, default: "memory")] 238 | end 239 | 240 | # Reconfigure cache to default arguments 241 | defp reset_arguments(cache, args) do 242 | for {key, value} <- format_options(args) do 243 | Cache.change_option(cache, key, value) 244 | end 245 | end 246 | 247 | # Caches created prior to v0.6.0 need to be reconfigured. 248 | defp maybe_reconfigure_caches() do 249 | RabbitLog.debug("Deduplication Exchanges startup, reconfiguring old caches") 250 | 251 | RabbitExchange.list() 252 | |> Enum.filter(fn(exchange(name: type)) -> type == @exchange_type end) 253 | |> Enum.map(fn(exchange) -> create(:none, exchange) end) 254 | 255 | :ok 256 | end 257 | end 258 | -------------------------------------------------------------------------------- /lib/rabbitmq_message_deduplication/rabbit_message_deduplication_policies.ex: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2025, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.Policies do 9 | @moduledoc """ 10 | Implement RabbitMQ policy validator behaviour to validate 11 | the plugin specific policies. 12 | """ 13 | 14 | alias :rabbit_registry, as: RabbitRegistry 15 | 16 | @behaviour :rabbit_policy_validator 17 | 18 | Module.register_attribute(__MODULE__, 19 | :rabbit_boot_step, 20 | accumulate: true, persist: true) 21 | 22 | @rabbit_boot_step {__MODULE__, 23 | [{:description, "message deduplication policy validator"}, 24 | {:mfa, {__MODULE__, :register, []}}, 25 | {:requires, :rabbit_registry}, 26 | {:enables, :recovery}]} 27 | 28 | @policy_validators [policy_validator: <<"x-cache-size">>, 29 | policy_validator: <<"x-cache-ttl">>, 30 | policy_validator: <<"x-cache-persistence">>, 31 | policy_validator: <<"x-message-deduplication">>] 32 | 33 | def register() do 34 | for {class, name} <- @policy_validators do 35 | RabbitRegistry.register(class, name, __MODULE__) 36 | end 37 | 38 | :ok 39 | end 40 | 41 | @doc """ 42 | Validate through all the policies stopping at the first error. 43 | """ 44 | @impl :rabbit_policy_validator 45 | @spec validate_policy([{Binary.t, any}]) :: :ok | {:error, String.t, [term]} 46 | def validate_policy(policies) do 47 | Enum.reduce(policies, :ok, fn {key, value}, :ok -> policy_validator(key, value) 48 | _, error -> error 49 | end) 50 | end 51 | 52 | defp policy_validator(<<"x-message-deduplication">>, true), do: :ok 53 | defp policy_validator(<<"x-message-deduplication">>, false), do: :ok 54 | defp policy_validator(<<"x-message-deduplication">>, val) do 55 | {:error, '\"x-message-deduplication\" must be a boolean, got ~tp', [val]} 56 | end 57 | 58 | defp policy_validator(<<"x-cache-size">>, val) when is_integer(val) and val > 0, do: :ok 59 | defp policy_validator(<<"x-cache-size">>, val) do 60 | {:error, '\"x-cache-size\" must be an integer greater than 0, got ~tp', [val]} 61 | end 62 | 63 | defp policy_validator(<<"x-cache-ttl">>, val) when is_integer(val) and val > 0, do: :ok 64 | defp policy_validator(<<"x-cache-ttl">>, val) do 65 | {:error, '\"x-cache-ttl\" must be an integer greater than 0, got ~tp', [val]} 66 | end 67 | 68 | defp policy_validator(<<"x-cache-persistence">>, "disk"), do: :ok 69 | defp policy_validator(<<"x-cache-persistence">>, "memory"), do: :ok 70 | defp policy_validator(<<"x-cache-persistence">>, val) do 71 | {:error, '\"x-cache-persistence\" must be either \"disk\" or \"memory\", got ~tp', [val]} 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/rabbitmq_message_deduplication/rabbit_message_deduplication_policy_event.ex: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2025, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.PolicyEvent do 9 | @moduledoc """ 10 | The `backing_queue_behaviour` is plagued by several issues which make 11 | implementing queue policy support hard to say the least. 12 | 13 | Firstly, the behaviour does not provide any callback allowing 14 | the backing queue to react to policy changes. Most importantly, policies 15 | are not stored within transient queues. 16 | 17 | This module overcomes both limitations by using :gen_event to react 18 | to internal RabbitMQ notifications related to policy changes. 19 | 20 | When a policy is changed, if it pertains the `x-message-deduplication` 21 | attribute, the module will apply the policy to all queues matching it. 22 | """ 23 | 24 | require RabbitMQMessageDeduplication.Queue 25 | 26 | alias :amqqueue, as: AMQQueue 27 | alias :gen_event, as: GenEvent 28 | alias :rabbit_log, as: RabbitLog 29 | alias :rabbit_policy, as: RabbitPolicy 30 | alias :rabbit_amqqueue, as: RabbitQueue 31 | alias RabbitMQMessageDeduplication.Queue, as: DedupQueue 32 | 33 | @behaviour :gen_event 34 | 35 | Module.register_attribute(__MODULE__, 36 | :rabbit_boot_step, 37 | accumulate: true, persist: true) 38 | 39 | @rabbit_boot_step {__MODULE__, 40 | [{:description, "message deduplication policy events"}, 41 | {:mfa, {__MODULE__, :enable, []}}, 42 | {:requires, :recovery}, 43 | {:enables, :routing_ready}]} 44 | 45 | @spec enable() :: :ok 46 | def enable() do 47 | GenEvent.add_handler(:rabbit_event, __MODULE__, []) 48 | end 49 | 50 | @spec disable() :: :ok 51 | def disable() do 52 | GenEvent.delete_handler(:rabbit_event, __MODULE__, []) 53 | end 54 | 55 | @impl :gen_event 56 | def init(_), do: {:ok, []} 57 | 58 | @impl :gen_event 59 | def handle_event({:event, :queue_policy_updated, policy, _, _}, state) do 60 | status = case List.keyfind(policy[:definition], "x-message-deduplication", 0) do 61 | {"x-message-deduplication", _} -> apply_to_queue(policy) 62 | nil -> :ok 63 | end 64 | 65 | {status, state} 66 | end 67 | 68 | @impl :gen_event 69 | def handle_event({:event, :queue_policy_cleared, policy, _, _}, state) do 70 | {apply_to_queue(policy), state} 71 | end 72 | 73 | @impl :gen_event 74 | def handle_event(_, state), do: {:ok, state} 75 | 76 | @impl :gen_event 77 | def handle_call(_Request, state), do: {:ok, :not_understood, state} 78 | 79 | # Apply new policies to matching queues 80 | defp apply_to_queue(policy) do 81 | {:ok, queue} = RabbitQueue.lookup(policy[:name]) 82 | queue = RabbitPolicy.set(queue) 83 | 84 | RabbitLog.debug("Policy change for queue ~p~n", [policy[:name]]) 85 | 86 | AMQQueue.get_pid(queue) 87 | |> RabbitQueue.run_backing_queue(DedupQueue, 88 | fn(_, state) -> 89 | state 90 | |> DedupQueue.dqstate(queue: queue) 91 | |> DedupQueue.maybe_toggle_dedup_queue() 92 | end) 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /lib/rabbitmq_message_deduplication/rabbit_message_deduplication_queue.ex: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2025, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.Queue do 9 | @moduledoc """ 10 | This module adds support for deduplication queues. 11 | 12 | Messages carrying the `x-deduplication-header` header will be deduplicated 13 | if a message with the same header value is already present within the queue. 14 | 15 | When a message is published within the queue, it's checked against duplicates. 16 | If no duplicate is found, the message is inserted and its deduplication header 17 | cached. Once the message is acknowledged or dropped, the header is removed 18 | from the cache. 19 | 20 | This module implements the `rabbit_backing_queue` behaviour delegating 21 | all the queue related operation to the underlying backing queue. 22 | 23 | """ 24 | 25 | import Record, only: [defrecord: 2] 26 | 27 | require RabbitMQMessageDeduplication.Cache 28 | require RabbitMQMessageDeduplication.Common 29 | 30 | alias :mc, as: MC 31 | alias :amqqueue, as: AMQQueue 32 | alias :rabbit_log, as: RabbitLog 33 | alias :rabbit_amqqueue, as: RabbitQueue 34 | alias RabbitMQMessageDeduplication.Common, as: Common 35 | alias RabbitMQMessageDeduplication.Cache, as: Cache 36 | alias RabbitMQMessageDeduplication.CacheManager, as: CacheManager 37 | 38 | @behaviour :rabbit_backing_queue 39 | 40 | Module.register_attribute(__MODULE__, 41 | :rabbit_boot_step, 42 | accumulate: true, persist: true) 43 | 44 | @rabbit_boot_step {__MODULE__, 45 | [{:description, "message deduplication queue"}, 46 | {:mfa, {__MODULE__, :enable, []}}, 47 | {:requires, :kernel_ready}, 48 | {:enables, :core_initialized}]} 49 | 50 | defrecord :dqack, [:tag, :header] 51 | defrecord :dqstate, [:queue, :queue_state, dedup_enabled: false] 52 | 53 | # The passthrough macros call the underlying backing queue functions 54 | # The suffixes indicate the arity of the return values 55 | # of the backing queue functions they are wrapping. 56 | 57 | # Backing queue functions returning one value, does not change the queue state 58 | defmacrop passthrough(do: function) do 59 | quote do 60 | backing_queue = Application.get_env(__MODULE__, :backing_queue_module) 61 | backing_queue.unquote(function) 62 | end 63 | end 64 | 65 | # Backing queue functions returning the state 66 | defmacrop passthrough1(state, do: function) do 67 | quote do 68 | backing_queue = Application.get_env(__MODULE__, :backing_queue_module) 69 | dqstate(queue: dqstate(unquote(state), :queue), 70 | queue_state: backing_queue.unquote(function), 71 | dedup_enabled: dqstate(unquote(state), :dedup_enabled)) 72 | end 73 | end 74 | 75 | # Backing queue functions returning a tuple {result, state} 76 | defmacrop passthrough2(state, do: function) do 77 | quote do 78 | backing_queue = Application.get_env(__MODULE__, :backing_queue_module) 79 | {result, queue_state} = backing_queue.unquote(function) 80 | {result, dqstate(queue: dqstate(unquote(state), :queue), 81 | queue_state: queue_state, 82 | dedup_enabled: dqstate(unquote(state), :dedup_enabled))} 83 | end 84 | end 85 | 86 | # Backing queue functions returning a tuple {result1, result2, state} 87 | defmacrop passthrough3(state, do: function) do 88 | quote do 89 | backing_queue = Application.get_env(__MODULE__, :backing_queue_module) 90 | {result1, result2, queue_state} = backing_queue.unquote(function) 91 | {result1, result2, dqstate(queue: dqstate(unquote(state), :queue), 92 | queue_state: queue_state, 93 | dedup_enabled: dqstate(unquote(state), :dedup_enabled))} 94 | end 95 | end 96 | 97 | @doc """ 98 | Enable deduplication queues support. 99 | 100 | Replace the original backing queue module with this one. 101 | 102 | """ 103 | @spec enable() :: :ok 104 | def enable() do 105 | case Application.get_env(:rabbit, :backing_queue_module) do 106 | __MODULE__ -> :ok 107 | backing_queue -> 108 | RabbitLog.info( 109 | "Deduplication queues enabled, real BQ is ~s~n", [backing_queue]) 110 | Application.put_env(__MODULE__, :backing_queue_module, backing_queue) 111 | Application.put_env(:rabbit, :backing_queue_module, __MODULE__) 112 | maybe_reconfigure_caches() 113 | end 114 | end 115 | 116 | @doc """ 117 | Disable deduplication queues support. 118 | 119 | Revert to the original backing queue module. 120 | 121 | """ 122 | @spec disable() :: :ok 123 | def disable() do 124 | case Application.get_env(:rabbit, :backing_queue_module) do 125 | __MODULE__ -> 126 | backing_queue = Application.get_env(__MODULE__, :backing_queue_module) 127 | RabbitLog.info( 128 | "Deduplication queues disabled, real BQ is ~s~n", [backing_queue]) 129 | Application.put_env(:rabbit, :backing_queue_module, backing_queue) 130 | _ -> :ok 131 | end 132 | end 133 | 134 | @impl :rabbit_backing_queue 135 | def start(vhost, queues) do 136 | passthrough do: start(vhost, queues) 137 | end 138 | 139 | @impl :rabbit_backing_queue 140 | def stop(vhost) do 141 | passthrough do: stop(vhost) 142 | end 143 | 144 | @impl :rabbit_backing_queue 145 | def init(queue, recovery, callback) do 146 | state = maybe_toggle_dedup_queue(dqstate(queue: queue)) 147 | 148 | passthrough1(state) do 149 | init(queue, recovery, callback) 150 | end 151 | end 152 | 153 | @impl :rabbit_backing_queue 154 | def terminate(any, state = dqstate(queue_state: qs)) do 155 | passthrough1(state, do: terminate(any, qs)) 156 | end 157 | 158 | @impl :rabbit_backing_queue 159 | def delete_and_terminate(any, state = dqstate(queue: queue, queue_state: qs)) do 160 | if dedup_queue?(state) do 161 | :ok = delete_cache(queue) 162 | end 163 | 164 | passthrough1(state) do 165 | delete_and_terminate(any, qs) 166 | end 167 | end 168 | 169 | @impl :rabbit_backing_queue 170 | def delete_crashed(queue) do 171 | passthrough do: delete_crashed(queue) 172 | end 173 | 174 | @impl :rabbit_backing_queue 175 | def purge(state = dqstate(queue: queue, queue_state: qs)) do 176 | if dedup_queue?(state) do 177 | :ok = AMQQueue.get_name(queue) |> Common.cache_name() |> Cache.flush() 178 | end 179 | 180 | passthrough2(state, do: purge(qs)) 181 | end 182 | 183 | @impl :rabbit_backing_queue 184 | def purge_acks(state = dqstate(queue_state: qs)) do 185 | passthrough1(state, do: purge_acks(qs)) 186 | end 187 | 188 | @impl :rabbit_backing_queue 189 | def publish(message, properties, boolean, pid, 190 | state = dqstate(queue_state: qs)) do 191 | passthrough1(state) do 192 | publish(message, properties, boolean, pid, qs) 193 | end 194 | end 195 | # v3.13.x 196 | def publish(message, properties, boolean, pid, flow, 197 | state = dqstate(queue_state: qs)) do 198 | passthrough1(state) do 199 | publish(message, properties, boolean, pid, flow, qs) 200 | end 201 | end 202 | 203 | # Optimization for cases in which the queue is empty and the message 204 | # is delivered straight to the client. Acknowledgement is enabled. 205 | @impl :rabbit_backing_queue 206 | def publish_delivered(message, properties, pid, state) do 207 | dqstate(queue_state: qs) = state 208 | 209 | {ack_tag, state} = passthrough2(state) do 210 | publish_delivered(message, properties, pid, qs) 211 | end 212 | 213 | if dedup_queue?(state) do 214 | head = Common.message_header(message, "x-deduplication-header") 215 | {dqack(tag: ack_tag, header: head), state} 216 | else 217 | {ack_tag, state} 218 | end 219 | end 220 | # v3.13.x 221 | def publish_delivered(message, properties, pid, flow, state) do 222 | dqstate(queue_state: qs) = state 223 | 224 | {ack_tag, state} = passthrough2(state) do 225 | publish_delivered(message, properties, pid, flow, qs) 226 | end 227 | 228 | if dedup_queue?(state) do 229 | head = Common.message_header(message, "x-deduplication-header") 230 | {dqack(tag: ack_tag, header: head), state} 231 | else 232 | {ack_tag, state} 233 | end 234 | end 235 | 236 | # v4.0.x 237 | def discard(msg_id, pid, state = dqstate(queue_state: qs)) when is_binary(msg_id) do 238 | passthrough1(state, do: discard(msg_id, pid, qs)) 239 | end 240 | @impl :rabbit_backing_queue 241 | def discard(message, pid, state = dqstate(queue: queue, queue_state: qs)) do 242 | if dedup_queue?(state) do 243 | maybe_delete_cache_entry(queue, message) 244 | end 245 | 246 | passthrough1(state, do: discard(message, pid, qs)) 247 | end 248 | # v3.13.x 249 | def discard(msg_id, pid, flow, state = dqstate(queue_state: qs)) do 250 | passthrough1(state, do: discard(msg_id, pid, flow, qs)) 251 | end 252 | 253 | @impl :rabbit_backing_queue 254 | def drain_confirmed(state = dqstate(queue_state: qs)) do 255 | passthrough2(state, do: drain_confirmed(qs)) 256 | end 257 | 258 | # The dropwhile callback handles message TTL expiration. 259 | # The duplicates cache TTL mechanism is used instead. 260 | @impl :rabbit_backing_queue 261 | def dropwhile(msg_pred, state = dqstate(queue_state: qs)) do 262 | passthrough2(state, do: dropwhile(msg_pred, qs)) 263 | end 264 | 265 | # The fetchwhile callback handles message TTL dead lettering. 266 | # The duplicates cache TTL mechanism is used instead. 267 | @impl :rabbit_backing_queue 268 | def fetchwhile(msg_pred, msg_fun, acc, state = dqstate(queue_state: qs)) do 269 | passthrough3(state, do: fetchwhile(msg_pred, msg_fun, acc, qs)) 270 | end 271 | 272 | @impl :rabbit_backing_queue 273 | def fetch(needs_ack, state = dqstate(queue: queue, queue_state: qs)) do 274 | case passthrough2(state, do: fetch(needs_ack, qs)) do 275 | {:empty, state} -> {:empty, state} 276 | {{message, delivery, ack_tag}, state} -> 277 | if dedup_queue?(state) do 278 | if needs_ack do 279 | head = Common.message_header(message, "x-deduplication-header") 280 | {{message, delivery, dqack(tag: ack_tag, header: head)}, state} 281 | else 282 | maybe_delete_cache_entry(queue, message) 283 | {{message, delivery, ack_tag}, state} 284 | end 285 | else 286 | {{message, delivery, ack_tag}, state} 287 | end 288 | end 289 | end 290 | 291 | # TODO: this is a bit of a hack. 292 | # As the drop callback returns only the message id, we can't retrieve 293 | # the message deduplication header. As a workaround `fetch` is used. 294 | # This assumes the backing queue drop and fetch behaviours are the same. 295 | # A better solution would be to store the message IDs in a dedicated index. 296 | @impl :rabbit_backing_queue 297 | def drop(need_ack, state = dqstate(queue: queue, queue_state: qs)) do 298 | if dedup_queue?(state) do 299 | case fetch(need_ack, state) do 300 | {:empty, state} -> {:empty, state} 301 | {{message, _, ack_tag}, state} -> 302 | maybe_delete_cache_entry(queue, message) 303 | id = MC.get_annotation(:id, message) 304 | 305 | {{id, ack_tag}, state} 306 | end 307 | else 308 | passthrough2(state, do: drop(need_ack, qs)) 309 | end 310 | end 311 | 312 | @impl :rabbit_backing_queue 313 | def ack(acks = [dqack() | _], state) do 314 | dqstate(queue: queue, queue_state: qs) = state 315 | acks = Enum.map(acks, fn(dqack(tag: ack_tag, header: header)) -> 316 | maybe_delete_cache_entry(queue, header) 317 | ack_tag 318 | end) 319 | 320 | passthrough2(state, do: ack(acks, qs)) 321 | end 322 | 323 | @impl :rabbit_backing_queue 324 | def ack(acks, state = dqstate(queue_state: qs)) do 325 | passthrough2(state, do: ack(acks, qs)) 326 | end 327 | 328 | @impl :rabbit_backing_queue 329 | def requeue(acks = [dqack() | _], state = dqstate(queue_state: qs)) do 330 | acks = Enum.map(acks, fn(dqack(tag: ack_tag)) -> ack_tag end) 331 | 332 | passthrough2(state, do: requeue(acks, qs)) 333 | end 334 | 335 | @impl :rabbit_backing_queue 336 | def requeue(acks, state = dqstate(queue_state: qs)) do 337 | passthrough2(state, do: requeue(acks, qs)) 338 | end 339 | 340 | @impl :rabbit_backing_queue 341 | def ackfold(function, acc, state, acks = [dqack() | _]) do 342 | dqstate(queue: queue, queue_state: qs) = state 343 | acks = Enum.map(acks, fn(dqack(tag: ack_tag, header: header)) -> 344 | maybe_delete_cache_entry(queue, header) 345 | ack_tag 346 | end) 347 | 348 | passthrough2(state, do: ackfold(function, acc, qs, acks)) 349 | end 350 | 351 | @impl :rabbit_backing_queue 352 | def ackfold(function, acc, state = dqstate(queue_state: qs), acks) do 353 | passthrough2(state, do: ackfold(function, acc, qs, acks)) 354 | end 355 | 356 | @impl :rabbit_backing_queue 357 | def fold(function, acc, state = dqstate(queue_state: qs)) do 358 | passthrough2(state, do: fold(function, acc, qs)) 359 | end 360 | 361 | @impl :rabbit_backing_queue 362 | def len(dqstate(queue_state: qs)) do 363 | passthrough do: len(qs) 364 | end 365 | 366 | @impl :rabbit_backing_queue 367 | def is_empty(dqstate(queue_state: qs)) do 368 | passthrough do: is_empty(qs) 369 | end 370 | 371 | @impl :rabbit_backing_queue 372 | def depth(dqstate(queue_state: qs)) do 373 | passthrough do: depth(qs) 374 | end 375 | 376 | @impl :rabbit_backing_queue 377 | def update_rates(state = dqstate(queue_state: qs)) do 378 | passthrough1(state, do: update_rates(qs)) 379 | end 380 | 381 | @impl :rabbit_backing_queue 382 | def needs_timeout(dqstate(queue_state: qs)) do 383 | passthrough do: needs_timeout(qs) 384 | end 385 | 386 | @impl :rabbit_backing_queue 387 | def timeout(state = dqstate(queue_state: qs)) do 388 | passthrough1(state, do: timeout(qs)) 389 | end 390 | 391 | @impl :rabbit_backing_queue 392 | def handle_pre_hibernate(state = dqstate(queue_state: qs)) do 393 | passthrough1(state, do: handle_pre_hibernate(qs)) 394 | end 395 | 396 | @impl :rabbit_backing_queue 397 | def resume(state = dqstate(queue_state: qs)) do 398 | passthrough1(state, do: resume(qs)) 399 | end 400 | 401 | @impl :rabbit_backing_queue 402 | def msg_rates(dqstate(queue_state: qs)) do 403 | passthrough do: msg_rates(qs) 404 | end 405 | 406 | @impl :rabbit_backing_queue 407 | def info(:backing_queue_status, state = dqstate(queue: queue, queue_state: qs)) do 408 | args = AMQQueue.get_arguments(queue) 409 | queue_info = passthrough do: info(:backing_queue_status, qs) 410 | priority = Common.rabbit_argument(args, "x-max-priority", default: false) 411 | 412 | if dedup_queue?(state) and !priority do 413 | [message_deduplication_cache_info: cache_info(queue)] ++ queue_info 414 | else 415 | queue_info 416 | end 417 | end 418 | 419 | @impl :rabbit_backing_queue 420 | def info(atom, dqstate(queue_state: qs)) do 421 | passthrough do: info(atom, qs) 422 | end 423 | 424 | @impl :rabbit_backing_queue 425 | def invoke(__MODULE__, function, state) do 426 | function.(__MODULE__, state) 427 | end 428 | 429 | @impl :rabbit_backing_queue 430 | def invoke(module, function, state = dqstate(queue_state: qs)) do 431 | passthrough1(state, do: invoke(module, function, qs)) 432 | end 433 | 434 | @impl :rabbit_backing_queue 435 | def is_duplicate(message, state = dqstate(queue: queue, queue_state: qs)) do 436 | case passthrough2(state, do: is_duplicate(message, qs)) do 437 | {true, state} -> {true, state} 438 | {false, state} -> if dedup_queue?(state) do 439 | {duplicate?(queue, message), state} 440 | else 441 | {false, state} 442 | end 443 | end 444 | end 445 | 446 | @impl :rabbit_backing_queue 447 | def set_queue_mode(queue_mode, state = dqstate(queue_state: qs)) do 448 | passthrough1(state, do: set_queue_mode(queue_mode, qs)) 449 | end 450 | 451 | @impl :rabbit_backing_queue 452 | def set_queue_version(queue_version, state = dqstate(queue_state: qs)) do 453 | passthrough1(state, do: set_queue_version(queue_version, qs)) 454 | end 455 | 456 | @impl :rabbit_backing_queue 457 | def zip_msgs_and_acks(delivered_publish, acks = [dqack() | _], acc, state) do 458 | dqstate(queue_state: qs) = state 459 | acks = Enum.map(acks, fn(dqack(tag: ack_tag)) -> ack_tag end) 460 | 461 | passthrough do: zip_msgs_and_acks(delivered_publish, acks, acc, qs) 462 | end 463 | 464 | @impl :rabbit_backing_queue 465 | def zip_msgs_and_acks(delivered_publish, acks, acc, state) do 466 | dqstate(queue_state: qs) = state 467 | 468 | passthrough do: zip_msgs_and_acks(delivered_publish, acks, acc, qs) 469 | end 470 | 471 | # Not present anymore in recent versions of RMQ, keeping for old ones 472 | def handle_info(term, state = dqstate(queue_state: qs)) do 473 | passthrough1(state, do: set_queue_mode(term, qs)) 474 | end 475 | 476 | # v3.13.x 477 | def batch_publish(batch, pid, flow, state = dqstate(queue_state: qs)) do 478 | passthrough1(state, do: batch_publish(batch, pid, flow, qs)) 479 | end 480 | 481 | # v3.13.x 482 | def batch_publish_delivered(batch, pid, flow, state) do 483 | dqstate(queue_state: qs) = state 484 | 485 | passthrough2(state) do 486 | batch_publish_delivered(batch, pid, flow, qs) 487 | end 488 | end 489 | 490 | # v3.13.x 491 | def set_ram_duration_target(duration, state = dqstate(queue_state: qs)) do 492 | passthrough1(state, do: set_ram_duration_target(duration, qs)) 493 | end 494 | 495 | # v3.13.x 496 | def ram_duration(state = dqstate(queue_state: qs)) do 497 | passthrough2(state, do: ram_duration(qs)) 498 | end 499 | 500 | # Utility functions 501 | 502 | # Enable/disable queue-level deduplication 503 | def maybe_toggle_dedup_queue(state = dqstate(queue: queue, queue_state: qs)) do 504 | cond do 505 | enable_dedup_queue?(state) -> 506 | :ok = init_cache(queue) 507 | dqstate(queue: queue, queue_state: qs, dedup_enabled: true) 508 | disable_dedup_queue?(state) -> 509 | :ok = delete_cache(queue) 510 | dqstate(queue: queue, queue_state: qs, dedup_enabled: false) 511 | true -> 512 | dqstate(queue: queue, queue_state: qs, dedup_enabled: false) 513 | end 514 | end 515 | 516 | # Caches created prior to v0.6.0 need to be reconfigured. 517 | defp maybe_reconfigure_caches() do 518 | RabbitLog.debug("Deduplication Queues startup, reconfiguring old caches~n") 519 | 520 | RabbitQueue.list() 521 | |> Enum.filter(&dedup_arg?/1) 522 | |> Enum.map(&init_cache/1) 523 | 524 | :ok 525 | end 526 | 527 | # Initialize the deduplication cache 528 | defp init_cache(queue) do 529 | cache = queue |> AMQQueue.get_name() |> Common.cache_name() 530 | ttl = queue 531 | |> AMQQueue.get_arguments() 532 | |> Common.rabbit_argument("x-message-ttl", type: :number) 533 | options = [ttl: ttl, persistence: :memory] 534 | 535 | RabbitLog.debug( 536 | "Starting queue deduplication cache ~s with options ~p~n", 537 | [cache, options]) 538 | 539 | case CacheManager.create(cache, false, options) do 540 | :ok -> Cache.flush(cache) 541 | {:error, {:already_exists, ^cache}} -> Cache.flush(cache) 542 | error -> error 543 | end 544 | end 545 | 546 | # Remove the cache and all its content 547 | defp delete_cache(queue) do 548 | cache = queue |> AMQQueue.get_name() |> Common.cache_name() 549 | 550 | RabbitLog.debug("Deleting queue deduplication cache ~s~n", [cache]) 551 | 552 | CacheManager.destroy(cache) 553 | end 554 | 555 | # Returns true if the message is a duplicate. 556 | defp duplicate?(queue, message) do 557 | queue 558 | |> AMQQueue.get_name() 559 | |> Common.duplicate?(message, message_expiration(message)) 560 | end 561 | 562 | # Returns the expiration property of the given message 563 | defp message_expiration(message) do 564 | case MC.ttl(message) do 565 | :undefined -> nil 566 | ttl -> ttl 567 | end 568 | end 569 | 570 | # Removes the message deduplication header from the cache 571 | defp maybe_delete_cache_entry(queue, msg) when is_tuple(msg) do 572 | header = Common.message_header(msg, "x-deduplication-header") 573 | maybe_delete_cache_entry(queue, header) 574 | end 575 | 576 | defp maybe_delete_cache_entry(queue, header) when not is_nil(header) do 577 | queue 578 | |> AMQQueue.get_name() 579 | |> Common.cache_name() 580 | |> Cache.delete(header) 581 | end 582 | 583 | defp maybe_delete_cache_entry(_queue, header) when is_nil(header) do end 584 | 585 | # Returns the cache information 586 | defp cache_info(queue) do 587 | name = AMQQueue.get_name(queue) 588 | cache = Common.cache_name(name) 589 | 590 | try do 591 | Cache.info(cache) 592 | catch 593 | :exit, {:aborted, {:no_exists, ^cache, _}} -> [] 594 | :exit, {:noproc, {GenServer, :call, [^cache | _]}} -> [] 595 | end 596 | end 597 | 598 | # True if `x-message-deduplication` is present within queue arguments or policy 599 | defp dedup_arg?(queue) do 600 | queue_policy(queue) 601 | ++ AMQQueue.get_arguments(queue) 602 | |> Common.rabbit_argument("x-message-deduplication", default: false) 603 | end 604 | 605 | # Return the list of policy arguments assigned to the queue 606 | defp queue_policy(queue) do 607 | case AMQQueue.get_policy(queue) do 608 | :undefined -> [] 609 | policy -> policy[:definition] 610 | end 611 | end 612 | 613 | # True if it's an active deduplication queue 614 | defp dedup_queue?(dqstate(dedup_enabled: val)), do: val 615 | 616 | # True if deduplication should be enabled for the queue 617 | defp enable_dedup_queue?(dqstate(dedup_enabled: true)), do: false 618 | defp enable_dedup_queue?(dqstate(queue: q, dedup_enabled: false)), do: dedup_arg?(q) 619 | 620 | # True if deduplication should be disabled for the queue 621 | defp disable_dedup_queue?(dqstate(dedup_enabled: false)), do: false 622 | defp disable_dedup_queue?(dqstate(queue: q, dedup_enabled: true)), do: not dedup_arg?(q) 623 | end 624 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule RabbitMQ.MessageDeduplicationPlugin.Mixfile do 2 | use Mix.Project 3 | 4 | def project() do 5 | [ 6 | app: :rabbitmq_message_deduplication, 7 | version: "0.7.1", 8 | build_embedded: Mix.env == :prod, 9 | start_permanent: Mix.env == :prod, 10 | deps: deps(), 11 | deps_path: System.get_env("DEPS_DIR", "deps"), 12 | aliases: aliases() 13 | ] 14 | end 15 | 16 | def application() do 17 | [ 18 | # The Application needs to depend on `rabbit` in order to be detected as a plugin. 19 | extra_applications: [:mnesia, :rabbit], 20 | mod: {RabbitMQMessageDeduplication, []}, 21 | registered: [RabbitMQMessageDeduplication], 22 | broker_version_requirements: if Mix.env == :prod do 23 | ["3.13.0", "4.0.0", "4.1.0"] 24 | else 25 | [] 26 | end 27 | ] 28 | end 29 | 30 | defp deps() do 31 | [ 32 | {:mix_task_archive_deps, github: "rabbitmq/mix_task_archive_deps", runtime: false} 33 | ] 34 | end 35 | 36 | defp aliases() do 37 | [ 38 | make_app: [ 39 | "deps.get", 40 | "deps.compile", 41 | "compile" 42 | ], 43 | make_archives: [ 44 | "archive.build.deps --destination=#{dist_dir()}", 45 | "archive.build.elixir --destination=#{dist_dir()}", 46 | "archive.build.all --destination=#{dist_dir()}" 47 | ], 48 | make_tests: [ 49 | "deps.get", 50 | "test" 51 | ], 52 | # Do not start the application during unit tests 53 | test: "test --no-start" 54 | ] 55 | end 56 | 57 | defp dist_dir() do 58 | System.get_env("DIST_DIR", "plugins") 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /rabbitmq-components.mk: -------------------------------------------------------------------------------- 1 | # Inspired by erlang.mk bootstrap Makefile. 2 | # Fetch updated rabbitmq-components.mk from rabbitmq-common. 3 | 4 | RABBITMQ_COMMON_DIR ?= .rabbitmq-components.mk.build 5 | 6 | rabbitmq-components.mk: rabbitmq-components-bootstrap 7 | git clone --depth 1 https://github.com/rabbitmq/rabbitmq-server $(RABBITMQ_COMMON_DIR) 8 | cp $(RABBITMQ_COMMON_DIR)/rabbitmq-components.mk ./rabbitmq-components.mk 9 | rm -rf $(RABBITMQ_COMMON_DIR) 10 | 11 | .PHONY: rabbitmq-components-bootstrap 12 | rabbitmq-components-bootstrap: ; 13 | -------------------------------------------------------------------------------- /test/cache_manager_test.exs: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2025, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.CacheManager.Test do 9 | use ExUnit.Case 10 | 11 | alias :timer, as: Timer 12 | alias :mnesia, as: Mnesia 13 | alias RabbitMQMessageDeduplication.Cache, as: Cache 14 | alias RabbitMQMessageDeduplication.CacheManager, as: CacheManager 15 | 16 | setup do 17 | start_supervised!(%{id: :cache_manager, 18 | start: {CacheManager, 19 | :start_link, 20 | []}}) 21 | 22 | %{} 23 | end 24 | 25 | test "cache creation", %{} do 26 | options = [persistence: :memory] 27 | 28 | CacheManager.create(:cache, true, options) 29 | {:atomic, [:cache]} = Mnesia.transaction(fn -> Mnesia.all_keys(caches()) end) 30 | CacheManager.destroy(:cache) 31 | end 32 | 33 | test "cache deletion", %{} do 34 | options = [persistence: :memory] 35 | 36 | :ok = CacheManager.create(:cache, false, options) 37 | {:atomic, [:cache]} = Mnesia.transaction(fn -> Mnesia.all_keys(caches()) end) 38 | :ok = CacheManager.destroy(:cache) 39 | {:atomic, []} = Mnesia.transaction(fn -> Mnesia.all_keys(caches()) end) 40 | end 41 | 42 | test "cache cleanup routine", %{} do 43 | options = [persistence: :memory] 44 | 45 | :ok = CacheManager.create(:cache, true, options) 46 | 47 | {:ok, :inserted} = Cache.insert(:cache, "foo", 1000) 48 | 49 | Timer.sleep(3200) 50 | 51 | {:atomic, []} = Mnesia.transaction(fn -> Mnesia.all_keys(:cache) end) 52 | {:ok, :inserted} = Cache.insert(:cache, "foo") 53 | 54 | :ok = CacheManager.destroy(:cache) 55 | end 56 | 57 | def caches(), do: :message_deduplication_caches 58 | end 59 | -------------------------------------------------------------------------------- /test/cache_test.exs: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2023, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.Cache.Test do 9 | use ExUnit.Case 10 | 11 | alias :timer, as: Timer 12 | alias :mnesia, as: Mnesia 13 | alias RabbitMQMessageDeduplication.Cache, as: Cache 14 | 15 | setup do 16 | cache = :test_cache 17 | cache_ttl = :test_cache_ttl 18 | cache_simple = :cache_simple 19 | 20 | on_exit fn -> 21 | Mnesia.delete_table(cache) 22 | Mnesia.delete_table(cache_ttl) 23 | Mnesia.delete_table(cache_simple) 24 | end 25 | 26 | cache_simple_options = [persistence: :memory] 27 | cache_options = [size: 1, ttl: nil, persistence: :memory] 28 | cache_ttl_options = [size: 1, ttl: Timer.seconds(1), persistence: :memory] 29 | 30 | :ok = Cache.create(cache, true, cache_options) 31 | :ok = Cache.create(cache_ttl, false, cache_ttl_options) 32 | :ok = Cache.create(cache_simple, true, cache_simple_options) 33 | 34 | %{cache: cache, cache_ttl: cache_ttl, cache_simple: cache_simple} 35 | end 36 | 37 | test "basic insertion", 38 | %{cache: cache, cache_ttl: _, cache_simple: _} do 39 | {:ok, :inserted} = Cache.insert(cache, "foo") 40 | {:ok, :exists} = Cache.insert(cache, "foo") 41 | end 42 | 43 | test "TTL at insertion", 44 | %{cache: cache, cache_ttl: _, cache_simple: _} do 45 | {:ok, :inserted} = Cache.insert(cache, "foo", Timer.seconds(1)) 46 | {:ok, :exists} = Cache.insert(cache, "foo") 47 | 48 | 1 |> Timer.seconds() |> Timer.sleep() 49 | 50 | :ok = Cache.delete_expired_entries(cache) 51 | 52 | {:ok, :inserted} = Cache.insert(cache, "foo") 53 | end 54 | 55 | test "TTL at table creation", 56 | %{cache: _, cache_ttl: cache, cache_simple: _} do 57 | {:ok, :inserted} = Cache.insert(cache, "foo") 58 | {:ok, :exists} = Cache.insert(cache, "foo") 59 | 60 | 1 |> Timer.seconds() |> Timer.sleep() 61 | 62 | :ok = Cache.delete_expired_entries(cache) 63 | 64 | {:ok, :inserted} = Cache.insert(cache, "foo") 65 | end 66 | 67 | test "entries are deleted after TTL", 68 | %{cache: cache, cache_ttl: _, cache_simple: _} do 69 | {:ok, :inserted} = Cache.insert(cache, "foo", Timer.seconds(1)) 70 | {:ok, :exists} = Cache.insert(cache, "foo") 71 | 72 | Timer.sleep(1200) 73 | 74 | :ok = Cache.delete_expired_entries(cache) 75 | 76 | {:atomic, []} = Mnesia.transaction(fn -> Mnesia.all_keys(cache) end) 77 | end 78 | 79 | test "entries are deleted if cache is full", 80 | %{cache: cache, cache_ttl: _, cache_simple: _} do 81 | {:ok, :inserted} = Cache.insert(cache, "foo") 82 | {:ok, :exists} = Cache.insert(cache, "foo") 83 | {:ok, :inserted} = Cache.insert(cache, "bar") 84 | {:ok, :exists} = Cache.insert(cache, "bar") 85 | 86 | {:ok, :inserted} = Cache.insert(cache, "foo") 87 | end 88 | 89 | test "cache entry deletion", %{cache: cache, cache_ttl: _, cache_simple: _} do 90 | {:ok, :inserted} = Cache.insert(cache, "foo") 91 | {:ok, :exists} = Cache.insert(cache, "foo") 92 | 93 | Cache.delete(cache, "foo") 94 | 95 | {:ok, :inserted} = Cache.insert(cache, "foo") 96 | end 97 | 98 | test "cache information", 99 | %{cache: cache, cache_ttl: _, cache_simple: _} do 100 | {:ok, :inserted} = Cache.insert(cache, "foo") 101 | 102 | [entries: 1, bytes: _, nodes: _, size: 1] = Cache.info(cache) 103 | end 104 | 105 | test "simple cache information", 106 | %{cache: _, cache_ttl: _, cache_simple: cache_simple} do 107 | {:ok, :inserted} = Cache.insert(cache_simple, "foo") 108 | 109 | [entries: 1, bytes: _, nodes: _] = Cache.info(cache_simple) 110 | end 111 | 112 | test "flush the cache", %{cache: cache, cache_ttl: _, cache_simple: _} do 113 | {:ok, :inserted} = Cache.insert(cache, "foo") 114 | {:ok, :exists} = Cache.insert(cache, "foo") 115 | 116 | :ok = Cache.flush(cache) 117 | 118 | {:ok, :inserted} = Cache.insert(cache, "foo") 119 | end 120 | 121 | test "drop the cache", %{cache: cache, cache_ttl: _, cache_simple: _} do 122 | :ok = Cache.drop(cache) 123 | 124 | assert Enum.member?(Mnesia.system_info(:tables), cache) == false 125 | end 126 | 127 | test "reconfigure the cache", %{cache: cache, cache_ttl: _, cache_simple: _} do 128 | :ok = Cache.change_option(cache, :size, 10) 129 | 130 | [entries: _, bytes: _, nodes: _, size: 10] = Cache.info(cache) 131 | 132 | {:error, {:invalid, :wrong_key}} = Cache.change_option(cache, :wrong_key, 10) 133 | end 134 | 135 | test "reconfigure old cache on creation", %{cache: cache, cache_ttl: _, cache_simple: _} do 136 | Mnesia.delete_table_property(cache, :distributed) 137 | Mnesia.delete_table_property(cache, :size) 138 | Mnesia.delete_table_property(cache, :ttl) 139 | 140 | Mnesia.write_table_property(cache, {:limit, 100}) 141 | Mnesia.write_table_property(cache, {:default_ttl, 100}) 142 | 143 | cache_options = [size: 1, ttl: nil, persistence: :memory] 144 | 145 | Cache.create(cache, true, cache_options) 146 | 147 | {:ttl, 100} = Mnesia.read_table_property(cache, :ttl) 148 | {:size, 100} = Mnesia.read_table_property(cache, :size) 149 | {:distributed, true} = Mnesia.read_table_property(cache, :distributed) 150 | end 151 | end 152 | -------------------------------------------------------------------------------- /test/exchange_SUITE.erl: -------------------------------------------------------------------------------- 1 | % This Source Code Form is subject to the terms of the Mozilla Public 2 | % License, v. 2.0. If a copy of the MPL was not distributed with this 3 | % file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | % 5 | % Copyright (c) 2017-2025, Matteo Cafasso. 6 | % All rights reserved. 7 | 8 | -module(exchange_SUITE). 9 | 10 | -include_lib("eunit/include/eunit.hrl"). 11 | -include_lib("common_test/include/ct.hrl"). 12 | -include_lib("amqp_client/include/amqp_client.hrl"). 13 | 14 | -compile(export_all). 15 | 16 | all() -> 17 | [ 18 | {group, non_parallel_tests} 19 | ]. 20 | 21 | groups() -> 22 | [ 23 | {non_parallel_tests, [], [ 24 | disable_enable, 25 | declare_exchanges, 26 | deduplicate_message, 27 | deduplicate_message_ttl, 28 | deduplicate_message_cache_overflow, 29 | exchange_policy 30 | ]} 31 | ]. 32 | 33 | %% ------------------------------------------------------------------- 34 | %% Testsuite setup/teardown. 35 | %% ------------------------------------------------------------------- 36 | 37 | init_per_suite(Config) -> 38 | rabbit_ct_helpers:log_environment(), 39 | Config1 = rabbit_ct_helpers:set_config(Config, 40 | [{rmq_nodename_suffix, ?MODULE}]), 41 | rabbit_ct_helpers:run_setup_steps(Config1, 42 | rabbit_ct_broker_helpers:setup_steps() ++ 43 | rabbit_ct_client_helpers:setup_steps()). 44 | 45 | end_per_suite(Config) -> 46 | rabbit_ct_helpers:run_teardown_steps( 47 | Config, rabbit_ct_client_helpers:teardown_steps() ++ 48 | rabbit_ct_broker_helpers:teardown_steps()). 49 | 50 | init_per_group(_, Config) -> Config. 51 | 52 | end_per_group(_, Config) -> Config. 53 | 54 | init_per_testcase(Testcase, Config) -> 55 | rabbit_ct_helpers:testcase_started(Config, Testcase). 56 | 57 | end_per_testcase(Testcase, Config) -> 58 | Channel = rabbit_ct_client_helpers:open_channel(Config), 59 | 60 | amqp_channel:call(Channel, #'exchange.delete'{exchange = <<"test">>}), 61 | amqp_channel:call(Channel, #'queue.delete'{queue = <<"test">>}), 62 | 63 | rabbit_ct_helpers:testcase_finished(Config, Testcase). 64 | 65 | %% ------------------------------------------------------------------- 66 | %% Testcases. 67 | %% ------------------------------------------------------------------- 68 | 69 | %% Basic smoke test that it is possible to disable, then enable the plugin 70 | disable_enable(Config) -> 71 | ok = rabbit_ct_broker_helpers:disable_plugin(Config, 0, rabbitmq_message_deduplication), 72 | ok = rabbit_ct_broker_helpers:enable_plugin(Config, 0, rabbitmq_message_deduplication). 73 | 74 | declare_exchanges(Config) -> 75 | Channel = rabbit_ct_client_helpers:open_channel(Config), 76 | 77 | DeclareShort = #'exchange.declare'{exchange = <<"test_exchange_short">>, 78 | type = <<"x-message-deduplication">>, 79 | auto_delete = true, 80 | arguments = [{<<"x-cache-size">>, short, 10}, 81 | {<<"x-cache-ttl">>, short, 1000}, 82 | {<<"x-cache-persistence">>, longstr, "memory"}]}, 83 | #'exchange.declare_ok'{} = amqp_channel:call(Channel, DeclareShort), 84 | 85 | DeclareLong = #'exchange.declare'{exchange = <<"test_exchange_long">>, 86 | type = <<"x-message-deduplication">>, 87 | auto_delete = true, 88 | arguments = [{<<"x-cache-size">>, long, 10}, 89 | {<<"x-cache-ttl">>, long, 1000}, 90 | {<<"x-cache-persistence">>, longstr, "memory"}]}, 91 | #'exchange.declare_ok'{} = amqp_channel:call(Channel, DeclareLong), 92 | 93 | DeclareSigned = #'exchange.declare'{exchange = <<"test_exchange_signed">>, 94 | type = <<"x-message-deduplication">>, 95 | auto_delete = true, 96 | arguments = [{<<"x-cache-size">>, signedint, 10}, 97 | {<<"x-cache-ttl">>, signedint, 1000}, 98 | {<<"x-cache-persistence">>, longstr, "memory"}]}, 99 | #'exchange.declare_ok'{} = amqp_channel:call(Channel, DeclareSigned), 100 | 101 | DeclareUnsigned = #'exchange.declare'{exchange = <<"test_exchange_unsigned">>, 102 | type = <<"x-message-deduplication">>, 103 | auto_delete = true, 104 | arguments = [{<<"x-cache-size">>, unsignedint, 10}, 105 | {<<"x-cache-ttl">>, unsignedint, 1000}, 106 | {<<"x-cache-persistence">>, longstr, "memory"}]}, 107 | #'exchange.declare_ok'{} = amqp_channel:call(Channel, DeclareUnsigned), 108 | 109 | DeclareStr = #'exchange.declare'{exchange = <<"test_exchange_strings">>, 110 | type = <<"x-message-deduplication">>, 111 | auto_delete = true, 112 | arguments = [{<<"x-cache-size">>, longstr, "10"}, 113 | {<<"x-cache-ttl">>, longstr, "1000"}, 114 | {<<"x-cache-persistence">>, longstr, "disk"}]}, 115 | #'exchange.declare_ok'{} = amqp_channel:call(Channel, DeclareStr), 116 | 117 | DeclareErr = #'exchange.declare'{exchange = <<"test_exchange_error">>, 118 | type = <<"x-message-deduplication">>, 119 | arguments = [{<<"x-cache-size">>, longstr, "foo"}, 120 | {<<"x-cache-ttl">>, longstr, "bar"}]}, 121 | ?assertExit(_, amqp_channel:call(Channel, DeclareErr)). 122 | 123 | deduplicate_message(Config) -> 124 | Get = #'basic.get'{queue = <<"test">>}, 125 | Channel = rabbit_ct_client_helpers:open_channel(Config), 126 | 127 | #'exchange.declare_ok'{} = amqp_channel:call( 128 | Channel, make_exchange(<<"test">>, 10, 10000)), 129 | bind_new_queue(Channel, <<"test">>, <<"test">>), 130 | 131 | %% Deduplication header present 132 | publish_message(Channel, <<"test">>, "deduplicate-this"), 133 | publish_message(Channel, <<"test">>, "deduplicate-this"), 134 | 135 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 136 | #'basic.get_empty'{} = amqp_channel:call(Channel, Get), 137 | 138 | %% Deduplication header absent 139 | publish_message(Channel, <<"test">>), 140 | publish_message(Channel, <<"test">>), 141 | 142 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 143 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get). 144 | 145 | deduplicate_message_ttl(Config) -> 146 | Get = #'basic.get'{queue = <<"test">>}, 147 | Channel = rabbit_ct_client_helpers:open_channel(Config), 148 | 149 | #'exchange.declare_ok'{} = amqp_channel:call( 150 | Channel, make_exchange(<<"test">>, 10, 1000)), 151 | bind_new_queue(Channel, <<"test">>, <<"test">>), 152 | 153 | %% Exchange default TTL 154 | publish_message(Channel, <<"test">>, "deduplicate-this"), 155 | timer:sleep(2000), 156 | publish_message(Channel, <<"test">>, "deduplicate-this"), 157 | 158 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 159 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 160 | 161 | %% Message TTL override 162 | Headers = [{<<"x-cache-ttl">>, long, 500}], 163 | publish_message(Channel, <<"test">>, "deduplicate-that", Headers), 164 | timer:sleep(800), 165 | publish_message(Channel, <<"test">>, "deduplicate-that", Headers), 166 | 167 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 168 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get). 169 | 170 | deduplicate_message_cache_overflow(Config) -> 171 | Get = #'basic.get'{queue = <<"test">>}, 172 | Channel = rabbit_ct_client_helpers:open_channel(Config), 173 | 174 | #'exchange.declare_ok'{} = amqp_channel:call( 175 | Channel, make_exchange(<<"test">>, 1, 10000)), 176 | bind_new_queue(Channel, <<"test">>, <<"test">>), 177 | 178 | publish_message(Channel, <<"test">>, "deduplicate-this"), 179 | publish_message(Channel, <<"test">>, "deduplicate-that"), 180 | publish_message(Channel, <<"test">>, "deduplicate-this"), 181 | 182 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 183 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 184 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get). 185 | 186 | exchange_policy(Config) -> 187 | Get = #'basic.get'{queue = <<"test">>}, 188 | Channel = rabbit_ct_client_helpers:open_channel(Config), 189 | 190 | #'exchange.declare_ok'{} = amqp_channel:call( 191 | Channel, make_exchange(<<"test">>, 1, 10000)), 192 | bind_new_queue(Channel, <<"test">>, <<"test">>), 193 | 194 | % Cache size is increased to 2. There should not be overflow 195 | rabbit_ct_broker_helpers:set_policy(Config, 0, <<"policy-test">>, 196 | <<".*">>, <<"all">>, [{<<"x-cache-size">>, 5}]), 197 | 198 | publish_message(Channel, <<"test">>, "deduplicate-this"), 199 | publish_message(Channel, <<"test">>, "deduplicate-that"), 200 | publish_message(Channel, <<"test">>, "deduplicate-this"), 201 | 202 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 203 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 204 | #'basic.get_empty'{} = amqp_channel:call(Channel, Get), 205 | 206 | % Policy is cleared, default arguments are restored 207 | rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"policy-test">>), 208 | 209 | publish_message(Channel, <<"test">>, "deduplicate-those"), 210 | publish_message(Channel, <<"test">>, "deduplicate-those"), 211 | 212 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 213 | #'basic.get_empty'{} = amqp_channel:call(Channel, Get). 214 | 215 | 216 | %% ------------------------------------------------------------------- 217 | %% Utility functions. 218 | %% ------------------------------------------------------------------- 219 | 220 | make_exchange(Ex, Size, TTL) -> 221 | #'exchange.declare'{ 222 | exchange = Ex, 223 | type = <<"x-message-deduplication">>, 224 | arguments = [{<<"x-cache-size">>, long, Size}, 225 | {<<"x-cache-ttl">>, long, TTL}]}. 226 | 227 | bind_new_queue(Ch, Ex, Q) -> 228 | Queue = #'queue.declare'{queue = <<"test">>, auto_delete = true}, 229 | #'queue.declare_ok'{} = amqp_channel:call(Ch, Queue), 230 | 231 | Binding = #'queue.bind'{queue = Q, exchange = Ex, routing_key = <<"#">>}, 232 | #'queue.bind_ok'{} = amqp_channel:call(Ch, Binding). 233 | 234 | publish_message(Ch, Ex) -> 235 | Publish = #'basic.publish'{exchange = Ex, routing_key = <<"#">>}, 236 | Msg = #amqp_msg{payload = <<"payload">>}, 237 | amqp_channel:cast(Ch, Publish, Msg). 238 | 239 | publish_message(Ch, Ex, D) -> 240 | publish_message(Ch, Ex, D, []). 241 | 242 | publish_message(Ch, Ex, D, H) -> 243 | Headers = [{<<"x-deduplication-header">>, longstr, D}] ++ H, 244 | Publish = #'basic.publish'{exchange = Ex, routing_key = <<"#">>}, 245 | Props = #'P_basic'{headers = Headers}, 246 | Msg = #amqp_msg{props = Props, payload = <<"payload">>}, 247 | amqp_channel:cast(Ch, Publish, Msg). 248 | -------------------------------------------------------------------------------- /test/policies_test.exs: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this 3 | # file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | # 5 | # Copyright (c) 2017-2023, Matteo Cafasso. 6 | # All rights reserved. 7 | 8 | defmodule RabbitMQMessageDeduplication.Policies.Test do 9 | use ExUnit.Case 10 | 11 | alias RabbitMQMessageDeduplication.Policies, as: Policies 12 | 13 | test "queue policy validation", %{} do 14 | :ok = Policies.validate_policy([{<<"x-message-deduplication">>, true}, 15 | {<<"x-message-deduplication">>, false}]) 16 | {:error, _msg, ["true"]} = Policies.validate_policy([{<<"x-message-deduplication">>, "true"}]) 17 | end 18 | 19 | test "queue exchange validation", %{} do 20 | :ok = Policies.validate_policy([{<<"x-cache-size">>, 100}, 21 | {<<"x-cache-ttl">>, 100}, 22 | {<<"x-cache-persistence">>, "disk"}, 23 | {<<"x-cache-persistence">>, "memory"}]) 24 | {:error, _msg, ["100"]} = Policies.validate_policy([{<<"x-cache-size">>, "100"}]) 25 | {:error, _msg, [0]} = Policies.validate_policy([{<<"x-cache-size">>, 0}]) 26 | {:error, _msg, ["100"]} = Policies.validate_policy([{<<"x-cache-ttl">>, "100"}]) 27 | {:error, _msg, [0]} = Policies.validate_policy([{<<"x-cache-ttl">>, 0}]) 28 | {:error, _msg, ["true"]} = Policies.validate_policy([{<<"x-cache-persistence">>, "true"}]) 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /test/queue_SUITE.erl: -------------------------------------------------------------------------------- 1 | % This Source Code Form is subject to the terms of the Mozilla Public 2 | % License, v. 2.0. If a copy of the MPL was not distributed with this 3 | % file, You can obtain one at http://mozilla.org/MPL/2.0/. 4 | % 5 | % Copyright (c) 2017-2025, Matteo Cafasso. 6 | % All rights reserved. 7 | 8 | -module(queue_SUITE). 9 | 10 | -include_lib("eunit/include/eunit.hrl"). 11 | -include_lib("common_test/include/ct.hrl"). 12 | -include_lib("amqp_client/include/amqp_client.hrl"). 13 | 14 | -compile(export_all). 15 | 16 | all() -> 17 | [ 18 | {group, non_parallel_tests} 19 | ]. 20 | 21 | groups() -> 22 | [ 23 | {non_parallel_tests, [], [ 24 | deduplicate_message, 25 | deduplicate_message_ttl, 26 | deduplicate_message_confirm, 27 | message_acknowledged, 28 | queue_overflow, 29 | dead_letter, 30 | consume_no_ack, 31 | queue_policy 32 | ]} 33 | ]. 34 | 35 | %% ------------------------------------------------------------------- 36 | %% Testsuite setup/teardown. 37 | %% ------------------------------------------------------------------- 38 | 39 | init_per_suite(Config) -> 40 | rabbit_ct_helpers:log_environment(), 41 | Config1 = rabbit_ct_helpers:set_config(Config, 42 | [{rmq_nodename_suffix, ?MODULE}]), 43 | rabbit_ct_helpers:run_setup_steps(Config1, 44 | rabbit_ct_broker_helpers:setup_steps() ++ 45 | rabbit_ct_client_helpers:setup_steps()). 46 | 47 | end_per_suite(Config) -> 48 | rabbit_ct_helpers:run_teardown_steps( 49 | Config, rabbit_ct_client_helpers:teardown_steps() ++ 50 | rabbit_ct_broker_helpers:teardown_steps()). 51 | 52 | init_per_group(_, Config) -> Config. 53 | 54 | end_per_group(_, Config) -> Config. 55 | 56 | init_per_testcase(Testcase, Config) -> 57 | rabbit_ct_helpers:testcase_started(Config, Testcase). 58 | 59 | end_per_testcase(Testcase, Config) -> 60 | Channel = rabbit_ct_client_helpers:open_channel(Config), 61 | 62 | amqp_channel:call(Channel, #'exchange.delete'{exchange = <<"test">>}), 63 | amqp_channel:call(Channel, #'queue.delete'{queue = <<"test">>}), 64 | 65 | rabbit_ct_helpers:testcase_finished(Config, Testcase). 66 | 67 | %% ------------------------------------------------------------------- 68 | %% Testcases. 69 | %% ------------------------------------------------------------------- 70 | 71 | deduplicate_message(Config) -> 72 | Get = #'basic.get'{queue = <<"test">>}, 73 | Channel = rabbit_ct_client_helpers:open_channel(Config), 74 | 75 | #'queue.declare_ok'{} = amqp_channel:call(Channel, make_queue(<<"test">>)), 76 | bind_new_exchange(Channel, <<"test">>, <<"test">>), 77 | 78 | %% Deduplication header present 79 | %% String 80 | publish_message(Channel, <<"test">>, "deduplicate-this"), 81 | publish_message(Channel, <<"test">>, "deduplicate-this"), 82 | 83 | {#'basic.get_ok'{delivery_tag = Tag1}, _} = amqp_channel:call(Channel, Get), 84 | #'basic.get_empty'{} = amqp_channel:call(Channel, Get), 85 | amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag1}), 86 | 87 | %% Integer 88 | publish_message(Channel, <<"test">>, 42), 89 | publish_message(Channel, <<"test">>, 42), 90 | 91 | {#'basic.get_ok'{delivery_tag = Tag2}, _} = amqp_channel:call(Channel, Get), 92 | #'basic.get_empty'{} = amqp_channel:call(Channel, Get), 93 | amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag2}), 94 | 95 | %% Float 96 | publish_message(Channel, <<"test">>, 4.2), 97 | publish_message(Channel, <<"test">>, 4.2), 98 | 99 | {#'basic.get_ok'{delivery_tag = Tag3}, _} = amqp_channel:call(Channel, Get), 100 | #'basic.get_empty'{} = amqp_channel:call(Channel, Get), 101 | amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag3}), 102 | 103 | %% None/null/nil/void/undefined 104 | publish_message(Channel, <<"test">>, undefined), 105 | publish_message(Channel, <<"test">>, undefined), 106 | 107 | {#'basic.get_ok'{delivery_tag = Tag4}, _} = amqp_channel:call(Channel, Get), 108 | #'basic.get_empty'{} = amqp_channel:call(Channel, Get), 109 | amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag4}), 110 | 111 | %% Deduplication header absent 112 | publish_message(Channel, <<"test">>), 113 | publish_message(Channel, <<"test">>), 114 | 115 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 116 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get). 117 | 118 | deduplicate_message_ttl(Config) -> 119 | Get = #'basic.get'{queue = <<"test">>}, 120 | Channel = rabbit_ct_client_helpers:open_channel(Config), 121 | 122 | Args = [{<<"x-message-ttl">>, long, 1000}], 123 | #'queue.declare_ok'{} = amqp_channel:call(Channel, 124 | make_queue(<<"test">>, Args)), 125 | bind_new_exchange(Channel, <<"test">>, <<"test">>), 126 | 127 | %% Queue default TTL 128 | publish_message(Channel, <<"test">>, "deduplicate-this"), 129 | timer:sleep(2000), 130 | publish_message(Channel, <<"test">>, "deduplicate-this"), 131 | 132 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 133 | 134 | %% Message TTL override 135 | publish_message(Channel, <<"test">>, "deduplicate-that", <<"500">>), 136 | timer:sleep(800), 137 | publish_message(Channel, <<"test">>, "deduplicate-that", <<"500">>), 138 | 139 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get). 140 | 141 | deduplicate_message_confirm(Config) -> 142 | Get = #'basic.get'{queue = <<"test">>}, 143 | Channel = rabbit_ct_client_helpers:open_channel(Config), 144 | #'confirm.select_ok'{} = amqp_channel:call(Channel, #'confirm.select'{}), 145 | 146 | Args = [{<<"x-message-ttl">>, long, 1000}], 147 | #'queue.declare_ok'{} = amqp_channel:call(Channel, 148 | make_queue(<<"test">>, Args)), 149 | bind_new_exchange(Channel, <<"test">>, <<"test">>), 150 | 151 | %% Publish and wait for confirmation 152 | publish_message(Channel, <<"test">>, "deduplicate-this"), 153 | true = amqp_channel:wait_for_confirms(Channel, 3), 154 | publish_message(Channel, <<"test">>, "deduplicate-this"), 155 | false = amqp_channel:wait_for_confirms(Channel, 3), 156 | 157 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get). 158 | 159 | message_acknowledged(Config) -> 160 | Get = #'basic.get'{queue = <<"test">>}, 161 | Channel = rabbit_ct_client_helpers:open_channel(Config), 162 | 163 | #'queue.declare_ok'{} = amqp_channel:call(Channel, make_queue(<<"test">>)), 164 | bind_new_exchange(Channel, <<"test">>, <<"test">>), 165 | 166 | publish_message(Channel, <<"test">>, "deduplicate-this"), 167 | 168 | {#'basic.get_ok'{delivery_tag = Tag}, _} = amqp_channel:call(Channel, Get), 169 | 170 | amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag}), 171 | 172 | publish_message(Channel, <<"test">>, "deduplicate-this"), 173 | 174 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get). 175 | 176 | queue_overflow(Config) -> 177 | Get = #'basic.get'{queue = <<"test">>}, 178 | Channel = rabbit_ct_client_helpers:open_channel(Config), 179 | 180 | Args = [{<<"x-max-length">>, long, 1}], 181 | #'queue.declare_ok'{} = amqp_channel:call(Channel, 182 | make_queue(<<"test">>, Args)), 183 | bind_new_exchange(Channel, <<"test">>, <<"test">>), 184 | 185 | publish_message(Channel, <<"test">>, "deduplicate-this"), 186 | publish_message(Channel, <<"test">>, "deduplicate-that"), 187 | publish_message(Channel, <<"test">>, "deduplicate-this"), 188 | 189 | {#'basic.get_ok'{}, 190 | #amqp_msg{props = #'P_basic'{headers = [ 191 | {<<"x-deduplication-header">>, 192 | longstr, 193 | <<"deduplicate-this">>} 194 | ] 195 | }}} = amqp_channel:call(Channel, Get). 196 | 197 | dead_letter(Config) -> 198 | Get = #'basic.get'{queue = <<"test">>}, 199 | DLGet = #'basic.get'{queue = <<"dead-letter-queue">>}, 200 | Channel = rabbit_ct_client_helpers:open_channel(Config), 201 | 202 | #'queue.declare_ok'{} = amqp_channel:call(Channel, make_queue(<<"dead-letter-queue">>)), 203 | bind_new_exchange(Channel, <<"dead-letter-exchange">>, <<"dead-letter-queue">>), 204 | 205 | Args = [{<<"x-dead-letter-exchange">>, longstr, "dead-letter-exchange"}], 206 | #'queue.declare_ok'{} = amqp_channel:call(Channel, make_queue(<<"test">>, Args)), 207 | bind_new_exchange(Channel, <<"test">>, <<"test">>), 208 | 209 | publish_message(Channel, <<"test">>, "deduplicate-this"), 210 | {#'basic.get_ok'{delivery_tag = Tag}, _} = amqp_channel:call(Channel, Get), 211 | 212 | amqp_channel:cast(Channel, #'basic.reject'{delivery_tag = Tag, requeue = false}), 213 | 214 | publish_message(Channel, <<"test">>, "deduplicate-this"), 215 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, Get), 216 | 217 | publish_message(Channel, <<"test">>, "deduplicate-this"), 218 | {#'basic.get_ok'{}, _} = amqp_channel:call(Channel, DLGet). 219 | 220 | consume_no_ack(Config) -> 221 | Channel = rabbit_ct_client_helpers:open_channel(Config), 222 | 223 | #'queue.declare_ok'{queue = Q} = amqp_channel:call(Channel, make_queue(<<"no-ack-queue">>)), 224 | bind_new_exchange(Channel, <<"test">>, <<"no-ack-queue">>), 225 | 226 | #'basic.consume_ok'{consumer_tag = _Tag} = 227 | amqp_channel:subscribe(Channel, #'basic.consume'{queue = Q, no_ack = true}, self()), 228 | receive 229 | #'basic.consume_ok'{} -> ok 230 | end, 231 | 232 | publish_message(Channel, <<"test">>, "deduplicate-this"), 233 | receive 234 | {#'basic.deliver'{}, _} -> ok 235 | after 1000 -> 236 | error(message_not_received) 237 | end, 238 | 239 | publish_message(Channel, <<"test">>, "deduplicate-this"), 240 | receive 241 | {#'basic.deliver'{}, _} -> ok 242 | after 1000 -> 243 | error(message_not_received) 244 | end. 245 | 246 | queue_policy(Config) -> 247 | Channel = rabbit_ct_client_helpers:open_channel(Config), 248 | 249 | #'queue.declare_ok'{} = amqp_channel:call(Channel, #'queue.declare'{queue = <<"test">>}), 250 | bind_new_exchange(Channel, <<"test">>, <<"test">>), 251 | 252 | rabbit_ct_broker_helpers:set_policy(Config, 0, <<"policy-test">>, 253 | <<".*">>, <<"all">>, [{<<"x-message-deduplication">>, true}]), 254 | %% Wait for policy propagation 255 | timer:sleep(1000), 256 | 257 | publish_message(Channel, <<"test">>, "deduplicate-this"), 258 | publish_message(Channel, <<"test">>, "deduplicate-this"), 259 | 260 | Get = #'basic.get'{queue = <<"test">>}, 261 | {#'basic.get_ok'{delivery_tag = Tag}, _} = amqp_channel:call(Channel, Get), 262 | #'basic.get_empty'{} = amqp_channel:call(Channel, Get), 263 | amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag}), 264 | 265 | %% Policy is applied to new queues 266 | #'queue.declare_ok'{} = amqp_channel:call(Channel, #'queue.declare'{queue = <<"test0">>}), 267 | bind_new_exchange(Channel, <<"test0">>, <<"test0">>), 268 | 269 | publish_message(Channel, <<"test0">>, "deduplicate-this"), 270 | publish_message(Channel, <<"test0">>, "deduplicate-this"), 271 | 272 | Get0 = #'basic.get'{queue = <<"test0">>}, 273 | {#'basic.get_ok'{delivery_tag = Tag0}, _} = amqp_channel:call(Channel, Get0), 274 | #'basic.get_empty'{} = amqp_channel:call(Channel, Get0), 275 | amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag0}), 276 | 277 | amqp_channel:call(Channel, #'exchange.delete'{exchange = <<"test0">>}), 278 | amqp_channel:call(Channel, #'queue.delete'{queue = <<"test0">>}), 279 | 280 | % Policy is cleared, default arguments are restored 281 | rabbit_ct_broker_helpers:clear_policy(Config, 0, <<"policy-test">>), 282 | 283 | publish_message(Channel, <<"test">>, "deduplicate-this"), 284 | publish_message(Channel, <<"test">>, "deduplicate-this"), 285 | 286 | Get = #'basic.get'{queue = <<"test">>}, 287 | {#'basic.get_ok'{delivery_tag = Tag1}, _} = amqp_channel:call(Channel, Get), 288 | {#'basic.get_ok'{delivery_tag = Tag2}, _} = amqp_channel:call(Channel, Get), 289 | amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag1}), 290 | amqp_channel:cast(Channel, #'basic.ack'{delivery_tag = Tag2}). 291 | 292 | 293 | %% ------------------------------------------------------------------- 294 | %% Utility functions. 295 | %% ------------------------------------------------------------------- 296 | 297 | make_queue(Q) -> 298 | #'queue.declare'{ 299 | queue = Q, 300 | arguments = [{<<"x-message-deduplication">>, bool, true}]}. 301 | 302 | make_queue(Q, Args) -> 303 | #'queue.declare'{ 304 | queue = Q, 305 | arguments = [{<<"x-message-deduplication">>, bool, true} | Args]}. 306 | 307 | bind_new_exchange(Ch, Ex, Q) -> 308 | Exchange = #'exchange.declare'{exchange = Ex, type = <<"direct">>}, 309 | #'exchange.declare_ok'{} = amqp_channel:call(Ch, Exchange), 310 | 311 | Binding = #'queue.bind'{queue = Q, exchange = Ex, routing_key = <<"#">>}, 312 | #'queue.bind_ok'{} = amqp_channel:call(Ch, Binding). 313 | 314 | publish_message(Ch, Ex) -> 315 | Publish = #'basic.publish'{exchange = Ex, routing_key = <<"#">>}, 316 | Msg = #amqp_msg{payload = <<"payload">>}, 317 | amqp_channel:cast(Ch, Publish, Msg). 318 | 319 | publish_message(Ch, Ex, D) -> 320 | Type = case D of 321 | D when is_integer(D) -> long; 322 | D when is_float(D) -> float; 323 | D when is_list(D) -> longstr; 324 | undefined -> void 325 | end, 326 | Props = #'P_basic'{headers = [{<<"x-deduplication-header">>, Type, D}]}, 327 | Publish = #'basic.publish'{exchange = Ex, routing_key = <<"#">>}, 328 | Msg = #amqp_msg{props = Props, payload = <<"payload">>}, 329 | amqp_channel:cast(Ch, Publish, Msg). 330 | 331 | publish_message(Ch, Ex, D, E) -> 332 | Props = #'P_basic'{headers = [{<<"x-deduplication-header">>, longstr, D}], 333 | expiration = E}, 334 | Publish = #'basic.publish'{exchange = Ex, routing_key = <<"#">>}, 335 | Msg = #amqp_msg{props = Props, payload = <<"payload">>}, 336 | amqp_channel:cast(Ch, Publish, Msg). 337 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | --------------------------------------------------------------------------------