├── .formatter.exs ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── lib └── broadway_cloud_pub_sub │ ├── acknowledger.ex │ ├── client.ex │ ├── options.ex │ ├── producer.ex │ └── pull_client.ex ├── mix.exs ├── mix.lock └── test ├── broadway_cloud_pub_sub ├── acknowledger_test.exs ├── producer_test.exs └── pull_client_test.exs └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - main 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-20.04 12 | env: 13 | MIX_ENV: test 14 | strategy: 15 | fail-fast: false 16 | matrix: 17 | include: 18 | - pair: 19 | elixir: "1.11" 20 | otp: "21" 21 | - pair: 22 | elixir: "1.18" 23 | otp: "27" 24 | lint: lint 25 | steps: 26 | - uses: actions/checkout@v4 27 | 28 | - uses: erlef/setup-beam@v1 29 | with: 30 | otp-version: ${{matrix.pair.otp}} 31 | elixir-version: ${{matrix.pair.elixir}} 32 | 33 | - uses: actions/cache@v4 34 | with: 35 | path: | 36 | deps 37 | _build 38 | key: ${{ runner.os }}-mix-${{matrix.pair.elixir}}-${{matrix.pair.otp}}-${{ hashFiles('**/mix.lock') }} 39 | restore-keys: | 40 | ${{ runner.os }}-mix-${{matrix.pair.elixir}}-${{matrix.pair.otp}}- 41 | 42 | - run: mix deps.get 43 | 44 | - run: mix format --check-formatted 45 | if: ${{ matrix.lint }} 46 | 47 | - run: mix deps.unlock --check-unused 48 | if: ${{ matrix.lint }} 49 | 50 | - run: mix deps.compile 51 | 52 | - run: mix compile --warnings-as-errors 53 | if: ${{ matrix.lint }} 54 | 55 | - run: mix test 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where 3rd-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | broadway_cloud_pub_sub-*.tar 24 | 25 | .elixir_ls 26 | .tool-versions 27 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [0.9.1] - 2024-06-21 9 | 10 | ### Changed 11 | 12 | - Forward compatibility with Broadway v1.1 13 | 14 | - Handle messages without data 15 | 16 | ## [0.9.0] - 2023-11-06 17 | 18 | ### Changed 19 | 20 | - Requires Elixir v1.11+. 21 | 22 | - Relax `nimble_options` dependency. 23 | 24 | - Optional dependency `goth` must be v1.3+. 25 | 26 | - **Possible breaking change:** The default token generator requires the `:goth` option to fetch tokens. 27 | 28 | If you had not yet upgraded to Goth v1.3+ please follow the [upgrade guide][goth_upgrade]. 29 | Then wherever you invoke `Broadway.start_link/2`, add the `:goth` option: 30 | 31 | ```elixir 32 | producer: [ 33 | module: 34 | {BroadwayCloudPubSub.Producer, 35 | goth: MyApp.Goth, 36 | subscription: "projects//subscriptions/"} 37 | ] 38 | ``` 39 | 40 | If you had previously upgraded to Goth v1.3+ then wherever you invoke `Broadway.start_link/2`, 41 | you may have something like the following: 42 | 43 | ```elixir 44 | producer: [ 45 | module: 46 | {BroadwayCloudPubSub.Producer, 47 | subscription: "projects//subscriptions/", 48 | token_generator: {MyApp, :fetch_token, []}} 49 | ] 50 | ``` 51 | 52 | ...where `MyApp.fetch_token/0` is similar to the following: 53 | 54 | ```elixir 55 | defmodule MyApp 56 | def fetch_token do 57 | with {:ok, token} <- Goth.fetch(MyApp.Goth) do 58 | {:ok, token.token} 59 | end 60 | end 61 | end 62 | ``` 63 | 64 | You can remove your custom token generator function and replace your producer config with this: 65 | 66 | 67 | ```elixir 68 | producer: [ 69 | module: 70 | {BroadwayCloudPubSub.Producer, 71 | goth: MyApp.Goth, 72 | subscription: "projects//subscriptions/"} 73 | ] 74 | ``` 75 | 76 | ### Added 77 | 78 | - The `:goth` option specifies the [Goth][goth] server for 79 | the default token generator. 80 | 81 | ### Removed 82 | 83 | - The `:scope` option has been removed. You may set custom scopes on your own [Goth][goth] server 84 | via the `:source` option. 85 | 86 | For example, if you want to use the scope `"https://www.googleapis.com/auth/pubsub"`, then 87 | wherever you start Goth add the `:scopes` option to your authentication source: 88 | 89 | ```elixir 90 | # The `:metadata` source type retrieves credentials from Google metadata servers. 91 | # Refer to the Goth documentation for more source options. 92 | source = {:metadata, scopes: ["https://www.googleapis.com/auth/pubsub"]} 93 | 94 | children = [ 95 | {Goth, name: MyApp.Goth, source: source} 96 | ] 97 | ``` 98 | 99 | ## [0.8.0] - 2022-10-26 100 | 101 | This version moves Cloud PubSub from Tesla to Finch, so read the notes below and upgrade with care. 102 | 103 | ### Added 104 | 105 | - Use `:finch` as the HTTP client and provide a `:finch` producer option for a user-defined HTTP pool 106 | 107 | - Make HTTP requests in a separate process for cleaner shutdown 108 | 109 | - Support multiple topologies from the same Brodway module 110 | 111 | - Add telemetry events around HTTP requests 112 | 113 | - Add `:deliveryAttempt` field to metadata 114 | 115 | ### Removed 116 | 117 | - The `:pool_size` option has been removed. Define your own [Finch][finch] 118 | pool and use the `:finch` option instead. 119 | 120 | For example, if your `:pool_size` was 10, then add Finch to your application 121 | supervision tree (usually located in `lib/my_app/application.ex`): 122 | 123 | ```elixir 124 | children = 125 | [ 126 | {Finch, name: MyFinch, pools: %{:default => [size: 10]}} 127 | ] 128 | ``` 129 | 130 | ...and wherever you invoke `Broadway.start_link/2`, replace this: 131 | 132 | ```elixir 133 | producer: [ 134 | module: 135 | {BroadwayCloudPubSub.Producer, 136 | subscription: "projects//subscriptions/", 137 | pool_size: 10} 138 | ] 139 | ``` 140 | 141 | ...with this: 142 | 143 | ```elixir 144 | producer: [ 145 | module: 146 | {BroadwayCloudPubSub.Producer, 147 | subscription: "projects//subscriptions/", 148 | finch: MyFinch} 149 | ] 150 | ``` 151 | 152 | ## [0.7.1] - 2022-05-09 153 | 154 | ### Added 155 | 156 | - Add `:receive_timeout` option 157 | 158 | ## [0.7.0] - 2021-08-30 159 | 160 | ### Changed 161 | 162 | - Require Broadway 1.0 163 | 164 | ## [0.6.3] - 2021-07-19 165 | 166 | ### Changed 167 | 168 | - Remove sensitive details from error log messages 169 | 170 | ### Added 171 | 172 | - A new `:middleware` option to pass a list of custom Tesla middleware 173 | - Failures to ack are now automatically retried. Retries can be customised via the new `:retry` option 174 | 175 | ## [0.6.2] - 2021-02-24 176 | 177 | ### Changed 178 | 179 | - Fixed a bug causing malformed acknowledgement requests (#54) 180 | 181 | ## [0.6.1] - 2021-02-23 182 | 183 | ### Changed 184 | 185 | - Decreased maximum number of ackIds per request (#49) 186 | 187 | ## [0.6.0] - 2020-02-18 188 | 189 | ### Added 190 | 191 | - Support for passing a tuple for the `scope` option 192 | 193 | ### Changed 194 | 195 | - Require [Broadway v0.6.0](https://hexdocs.pm/broadway/0.6.0) 196 | 197 | ## [0.5.0] - 2019-11-06 198 | 199 | ### Added 200 | 201 | - Client options for connection pools (#37) 202 | 203 | - Support for configuring acknowledgement behavior (#36) 204 | 205 | ### Changed 206 | 207 | - Move acknowledger behaviour from `GoogleApiClient` into `ClientAcknowledger` (#39) 208 | 209 | ## [0.4.0] - 2019-08-19 210 | 211 | ### Changed 212 | 213 | - Move to Plataformatec GitHub organization and become an official Broadway connector 214 | 215 | - Rename behaviour `RestClient` to `Client` (#23) 216 | 217 | - Use hackney as the default adapter (#20) 218 | 219 | - Require Broadway 0.4.x and (optionally) Goth 1.x (#26) 220 | 221 | - Replace `:token_module` option with `:token_generator` (#29) 222 | 223 | - Hide `handle_receive_messages` function that was accidentally made public 224 | 225 | ## [0.3.0] - 2019-05-08 226 | 227 | ### Changed 228 | 229 | - **BREAKING:** The PubsubMessage struct now gets unpacked into the `%Broadway.Message{}` received in your pipeline. If you were using `message.data.data` before, you can now use `message.data`. Additional properties from the PubsubMessage can be found in the message metadata, for instance: `message.metadata.attributes` or `message.metadata.messageId`. 230 | - Requires `:broadway ~> 0.3.0` 231 | 232 | ## [0.1.3] - 2019-05-06 233 | 234 | ### Changed 235 | 236 | - Fixed `BroadwayCloudPubSub.GoogleApiClient` attempting to send an empty acknowledge request. 237 | 238 | ## [0.1.2] - 2019-04-11 239 | 240 | ### Changed 241 | 242 | - MIX_ENV for publishing releases to Hex. 243 | 244 | ## [0.1.1] - 2019-04-11 245 | 246 | ### Added 247 | 248 | - This `CHANGELOG` file to hopefully serve as an evolving example of a 249 | standardized open source project `CHANGELOG`. 250 | 251 | ### Changed 252 | 253 | - Fixed CircleCI build for publishing docs to Hex. 254 | 255 | ## [0.1.0] - 2019-04-10 256 | 257 | ### Added 258 | 259 | - `BroadwayCloudPubSub.Producer` - A GenStage producer that continuously receives messages from 260 | a Pub/Sub subscription acknowledges them after being successfully processed. 261 | - `BroadwayCloudPubSub.RestClient` - A generic behaviour to implement Pub/Sub clients using the [REST API](https://cloud.google.com/pubsub/docs/reference/rest/). 262 | - `BroadwayCloudPubSub.GoogleApiClient` - Default REST client used by `BroadwayCloudPubSub.Producer`. 263 | - `BroadwayCloudPubSub.Token` - A generic behaviour to implement token authentication for Pub/Sub clients. 264 | - `BroadwayCloudPubSub.GothToken` - Default token provider used by `BroadwayCloudPubSub.Producer`. 265 | 266 | [Unreleased]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.9.0...HEAD 267 | [0.9.0]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.8.0...v0.9.0 268 | [0.8.0]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.7.1...v0.8.0 269 | [0.7.1]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.7.0...v0.7.1 270 | [0.7.0]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.6.2...v0.7.0 271 | [0.6.3]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.6.2...v0.6.3 272 | [0.6.2]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.6.1...v0.6.2 273 | [0.6.1]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.6.0...v0.6.1 274 | [0.6.0]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.5.0...v0.6.0 275 | [0.5.0]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.4.0...v0.5.0 276 | [0.4.0]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.3.0...v0.4.0 277 | [0.3.0]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.1.3...v0.3.0 278 | [0.1.3]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.1.2...v0.1.3 279 | [0.1.2]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.1.1...v0.1.2 280 | [0.1.1]: https://github.com/dashbitco/broadway_cloud_pub_sub/compare/v0.1.0...v0.1.1 281 | [0.1.0]: https://github.com/dashbitco/broadway_cloud_pub_sub/releases/tag/v0.1.0 282 | 283 | 284 | [finch]: https://hexdocs.pm/finch 285 | [goth]: https://hexdocs.pm/goth 286 | [goth_upgrade]: https://hexdocs.pm/goth/upgrade_guide.html 287 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Michael Crumm 190 | Copyright 2020 Dashbit 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # BroadwayCloudPubSub 2 | 3 | [![CI](https://github.com/dashbitco/broadway_cloud_pub_sub/actions/workflows/ci.yml/badge.svg)](https://github.com/dashbitco/broadway_cloud_pub_sub/actions/workflows/ci.yml) 4 | 5 | A Google Cloud Pub/Sub connector for [Broadway](https://github.com/dashbitco/broadway). 6 | 7 | Documentation can be found at [https://hexdocs.pm/broadway_cloud_pub_sub](https://hexdocs.pm/broadway_cloud_pub_sub). 8 | 9 | This project provides: 10 | 11 | * `BroadwayCloudPubSub.Producer` - A GenStage producer that continuously receives messages from a Pub/Sub subscription acknowledges them after being successfully processed. 12 | * `BroadwayCloudPubSub.Client` - A generic behaviour to implement Pub/Sub clients. 13 | * `BroadwayCloudPubSub.PullClient` - Default REST client used by `BroadwayCloudPubSub.Producer`. 14 | 15 | ## Installation 16 | 17 | Add `:broadway_cloud_pub_sub` to the list of dependencies in `mix.exs`: 18 | 19 | ```elixir 20 | def deps do 21 | [ 22 | {:broadway_cloud_pub_sub, "~> 0.9.0"}, 23 | {:goth, "~> 1.3"} 24 | ] 25 | end 26 | ``` 27 | 28 | > Note the [goth](https://hexdocs.pm/goth) package, which handles Google Authentication, is required for the default token generator. 29 | 30 | ## Usage 31 | 32 | Configure Broadway with one or more producers using `BroadwayCloudPubSub.Producer`: 33 | 34 | ```elixir 35 | Broadway.start_link(MyBroadway, 36 | name: MyBroadway, 37 | producer: [ 38 | module: {BroadwayCloudPubSub.Producer, 39 | goth: MyGoth, 40 | subscription: "projects/my-project/subscriptions/my-subscription" 41 | } 42 | ] 43 | ) 44 | ``` 45 | 46 | ## License 47 | 48 | Copyright 2019 Michael Crumm \ 49 | Copyright 2020 Dashbit 50 | 51 | Licensed under the Apache License, Version 2.0 (the "License"); 52 | you may not use this file except in compliance with the License. 53 | You may obtain a copy of the License at 54 | 55 | http://www.apache.org/licenses/LICENSE-2.0 56 | 57 | Unless required by applicable law or agreed to in writing, software 58 | distributed under the License is distributed on an "AS IS" BASIS, 59 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 60 | See the License for the specific language governing permissions and 61 | limitations under the License. 62 | -------------------------------------------------------------------------------- /lib/broadway_cloud_pub_sub/acknowledger.ex: -------------------------------------------------------------------------------- 1 | defmodule BroadwayCloudPubSub.Acknowledger do 2 | @moduledoc false 3 | alias Broadway.Acknowledger 4 | alias BroadwayCloudPubSub.{Client, Options} 5 | 6 | @behaviour Acknowledger 7 | 8 | @typedoc """ 9 | Acknowledgement data for a `Broadway.Message`. 10 | """ 11 | @type ack_data :: %{ 12 | :ack_id => Client.ack_id(), 13 | optional(:on_failure) => ack_option, 14 | optional(:on_success) => ack_option 15 | } 16 | 17 | @typedoc """ 18 | An acknowledgement action. 19 | """ 20 | @type ack_option :: :ack | :noop | :nack | {:nack, Client.ack_deadline()} 21 | 22 | @type ack_ref :: reference 23 | 24 | # The maximum number of ackIds to be sent in acknowledge/modifyAckDeadline 25 | # requests. There is an API limit of 524288 bytes (512KiB) per acknowledge/modifyAckDeadline 26 | # request. ackIds have a maximum size of 184 bytes, so 524288/184 ~= 2849. 27 | # Accounting for some overhead, a maximum of 2500 ackIds per request should be safe. 28 | # See https://github.com/googleapis/nodejs-pubsub/pull/65/files#diff-3d29c4447546c72118ed5d5cbf38ab8bR34-R42 29 | @max_ack_ids_per_request 2_500 30 | 31 | @doc """ 32 | Returns an `acknowledger` to be put on a `Broadway.Message`. 33 | """ 34 | @spec builder(ack_ref) :: (Client.ack_id() -> {__MODULE__, ack_ref, ack_data}) 35 | def builder(ack_ref) do 36 | &{__MODULE__, ack_ref, %{ack_id: &1}} 37 | end 38 | 39 | @impl Acknowledger 40 | def ack(ack_ref, successful, failed) do 41 | config = :persistent_term.get(ack_ref) 42 | 43 | success_actions = group_actions_ack_ids(successful, :on_success, config) 44 | failure_actions = group_actions_ack_ids(failed, :on_failure, config) 45 | 46 | success_actions 47 | |> Map.merge(failure_actions, fn _, a, b -> a ++ b end) 48 | |> ack_messages(config) 49 | 50 | :ok 51 | end 52 | 53 | @impl Acknowledger 54 | def configure(_ack_ref, ack_data, options) do 55 | opts = NimbleOptions.validate!(options, Options.acknowledger_definition()) 56 | ack_data = Map.merge(ack_data, Map.new(opts)) 57 | 58 | {:ok, ack_data} 59 | end 60 | 61 | defp group_actions_ack_ids(messages, key, config) do 62 | Enum.group_by(messages, &group_acknowledger(&1, key, config), &extract_ack_id/1) 63 | end 64 | 65 | defp group_acknowledger(%{acknowledger: {_, _, ack_data}}, key, config) do 66 | Map.get_lazy(ack_data, key, fn -> config_action(key, config) end) 67 | end 68 | 69 | defp config_action(:on_success, %{on_success: action}), do: action 70 | defp config_action(:on_failure, %{on_failure: action}), do: action 71 | 72 | defp extract_ack_id(message) do 73 | {_, _, %{ack_id: ack_id}} = message.acknowledger 74 | ack_id 75 | end 76 | 77 | defp ack_messages(actions_and_ids, config) do 78 | Enum.each(actions_and_ids, fn {action, ack_ids} -> 79 | ack_ids 80 | |> Enum.chunk_every(@max_ack_ids_per_request) 81 | |> Enum.each(&apply_ack_func(action, &1, config)) 82 | end) 83 | end 84 | 85 | defp apply_ack_func(:noop, _ack_ids, _config), do: :ok 86 | 87 | defp apply_ack_func(:ack, ack_ids, config) do 88 | %{client: client} = config 89 | client.acknowledge(ack_ids, config) 90 | end 91 | 92 | defp apply_ack_func(:nack, ack_ids, config) do 93 | apply_ack_func({:nack, 0}, ack_ids, config) 94 | end 95 | 96 | defp apply_ack_func({:nack, deadline}, ack_ids, config) do 97 | %{client: client} = config 98 | client.put_deadline(ack_ids, deadline, config) 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/broadway_cloud_pub_sub/client.ex: -------------------------------------------------------------------------------- 1 | defmodule BroadwayCloudPubSub.Client do 2 | @moduledoc """ 3 | A generic behaviour to implement Pub/Sub Clients for `BroadwayCloudPubSub.Producer`. 4 | 5 | This module defines callbacks to normalize options and receive messages 6 | from a Cloud Pub/Sub topic. Modules that implement this behaviour should be passed 7 | as the `:client` option from `BroadwayCloudPubSub.Producer`. 8 | """ 9 | 10 | alias Broadway.Message 11 | 12 | @typedoc """ 13 | A list of `Broadway.Message` structs. 14 | """ 15 | @type messages :: [Message.t()] 16 | 17 | @typedoc """ 18 | The amount of time (in seconds) before Pub/Sub should reschedule a message. 19 | """ 20 | @type ack_deadline :: 0..600 21 | 22 | @typedoc """ 23 | The `ackId` returned by Pub/Sub to be used when acknowledging a message. 24 | """ 25 | @type ack_id :: String.t() 26 | 27 | @typedoc """ 28 | A list of `ackId` strings. 29 | """ 30 | @type ack_ids :: list(ack_id) 31 | 32 | @doc """ 33 | Invoked once by BroadwayCloudPubSub.Producer during `Broadway.start_link/2`. 34 | 35 | The goal of this task is to manipulate the producer options, 36 | if necessary at all, and introduce any new child specs that will be 37 | started in Broadway's supervision tree. 38 | """ 39 | @callback prepare_to_connect(name :: atom, producer_opts :: keyword) :: 40 | {[:supervisor.child_spec() | {module, any} | module], producer_opts :: keyword} 41 | 42 | @callback init(opts :: any) :: {:ok, normalized_opts :: any} | {:error, message :: binary} 43 | 44 | @doc """ 45 | Dispatches a [`pull`](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull) request. 46 | """ 47 | @callback receive_messages(demand :: pos_integer, ack_builder :: (ack_id -> term), opts :: any) :: 48 | messages 49 | 50 | @doc """ 51 | Dispatches a [`modifyAckDeadline`](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/modifyAckDeadline) request. 52 | """ 53 | @callback put_deadline(ack_ids, ack_deadline, opts :: any) :: any 54 | 55 | @doc """ 56 | Dispatches an [`acknowledge`](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/acknowledge) request. 57 | """ 58 | @callback acknowledge(ack_ids, opts :: any) :: any 59 | 60 | @optional_callbacks acknowledge: 2, prepare_to_connect: 2, put_deadline: 3 61 | end 62 | -------------------------------------------------------------------------------- /lib/broadway_cloud_pub_sub/options.ex: -------------------------------------------------------------------------------- 1 | defmodule BroadwayCloudPubSub.Options do 2 | @moduledoc false 3 | 4 | @default_base_url "https://pubsub.googleapis.com" 5 | 6 | @default_max_number_of_messages 10 7 | 8 | @default_receive_interval 5_000 9 | 10 | @default_receive_timeout :infinity 11 | 12 | definition = [ 13 | # Handled by Broadway. 14 | broadway: [type: :any, doc: false], 15 | client: [ 16 | type: {:or, [:atom, :mod_arg]}, 17 | default: BroadwayCloudPubSub.PullClient, 18 | doc: """ 19 | A module that implements the BroadwayCloudPubSub.Client behaviour. 20 | This module is responsible for fetching and acknowledging the messages. 21 | Pay attention that all options passed to the producer will be forwarded 22 | to the client. It's up to the client to normalize the options it needs. 23 | 24 | The BroadwayCloudPubSub.PullClient is the default client and will 25 | automatically retry the following errors [408, 500, 502, 503, 504, 522, 26 | 524] up to 10 times with a 500ms pause between retries. This can be 27 | configured by passing the module with options to the client: 28 | 29 | {BroadwayCloudPubSub.PullClient, 30 | retry_codes: [502, 503], 31 | retry_delay_ms: 300, 32 | max_retries: 5} 33 | 34 | These options will be merged with the options to the producer and passed 35 | to the client init/1 function. 36 | """ 37 | ], 38 | subscription: [ 39 | type: {:custom, __MODULE__, :type_non_empty_string, [[{:name, :subscription}]]}, 40 | required: true, 41 | doc: """ 42 | The name of the subscription, including the project. 43 | For example, if you project is named `"my-project"` and your 44 | subscription is named `"my-subscription"`, then your subscription 45 | name is `"projects/my-project/subscriptions/my-subscription"`. 46 | """ 47 | ], 48 | max_number_of_messages: [ 49 | doc: "The maximum number of messages to be fetched per request.", 50 | type: :pos_integer, 51 | default: @default_max_number_of_messages 52 | ], 53 | on_failure: [ 54 | type: 55 | {:custom, __MODULE__, :type_atom_action_or_nack_with_bounded_integer, 56 | [[{:name, :on_failure}, {:min, 0}, {:max, 600}]]}, 57 | doc: """ 58 | Configures the acking behaviour for failed messages. 59 | See the "Acknowledgements" section below for all the possible values. 60 | This option can also be changed for each message through 61 | `Broadway.Message.configure_ack/2`. 62 | """, 63 | default: :noop 64 | ], 65 | on_success: [ 66 | type: 67 | {:custom, __MODULE__, :type_atom_action_or_nack_with_bounded_integer, 68 | [[{:name, :on_success}, {:min, 0}, {:max, 600}]]}, 69 | doc: """ 70 | Configures the acking behaviour for successful messages. 71 | See the "Acknowledgements" section below for all the possible values. 72 | This option can also be changed for each message through 73 | `Broadway.Message.configure_ack/2`. 74 | """, 75 | default: :ack 76 | ], 77 | receive_interval: [ 78 | type: :integer, 79 | default: @default_receive_interval, 80 | doc: """ 81 | The duration (in milliseconds) for which the producer waits 82 | before making a request for more messages. 83 | """ 84 | ], 85 | receive_timeout: [ 86 | type: 87 | {:custom, __MODULE__, :type_positive_integer_or_infinity, [[{:name, :receive_timeout}]]}, 88 | default: @default_receive_timeout, 89 | doc: """ 90 | The maximum time (in milliseconds) to wait for a response 91 | before the pull client returns an error. 92 | """ 93 | ], 94 | goth: [ 95 | type: :atom, 96 | doc: """ 97 | The `Goth` module to use for authentication. Note that this option only applies to the 98 | default token generator. 99 | """ 100 | ], 101 | token_generator: [ 102 | type: :mfa, 103 | doc: """ 104 | An MFArgs tuple that will be called before each request 105 | to fetch an authentication token. It should return `{:ok, String.t()} | {:error, any()}`. 106 | By default this will invoke `Goth.fetch/1` with the `:goth` option. 107 | See the "Custom token generator" section below for more information. 108 | """ 109 | ], 110 | base_url: [ 111 | type: {:custom, __MODULE__, :type_non_empty_string, [[{:name, :base_url}]]}, 112 | default: @default_base_url, 113 | doc: """ 114 | The base URL for the Cloud Pub/Sub service. 115 | This option is mostly useful for testing via the Pub/Sub emulator. 116 | """ 117 | ], 118 | finch: [ 119 | type: :atom, 120 | default: nil, 121 | doc: """ 122 | The name of the `Finch` pool. If no name is provided, then a default 123 | pool will be started by the pipeline's supervisor. 124 | """ 125 | ], 126 | return_immediately: [ 127 | doc: false, 128 | deprecated: "Google Pub/Sub will remove this option in the future.", 129 | type: :boolean 130 | ], 131 | # Testing Options 132 | test_pid: [ 133 | type: :pid, 134 | doc: false 135 | ], 136 | message_server: [ 137 | type: :pid, 138 | doc: false 139 | ], 140 | prepare_to_connect_ref: [ 141 | type: :any, 142 | doc: false 143 | ] 144 | ] 145 | 146 | @definition NimbleOptions.new!(definition) 147 | 148 | def definition do 149 | @definition 150 | end 151 | 152 | @acknowledger_definition definition 153 | |> Keyword.take([:on_failure, :on_success]) 154 | |> NimbleOptions.new!() 155 | 156 | def acknowledger_definition do 157 | @acknowledger_definition 158 | end 159 | 160 | @doc """ 161 | Builds an MFArgs tuple for a token generator. 162 | """ 163 | def make_token_generator(opts) do 164 | goth = Keyword.get(opts, :goth) 165 | 166 | unless goth do 167 | require Logger 168 | 169 | Logger.error(""" 170 | the :goth option is required for the default authentication token generator 171 | 172 | If you are upgrading from an earlier version of Goth, refer to the 173 | upgrade guide for more information: 174 | 175 | https://hexdocs.pm/goth/upgrade_guide.html 176 | """) 177 | end 178 | 179 | ensure_goth_loaded() 180 | {__MODULE__, :generate_goth_token, [goth]} 181 | end 182 | 183 | defp ensure_goth_loaded() do 184 | unless Code.ensure_loaded?(Goth) do 185 | require Logger 186 | 187 | Logger.error(""" 188 | the default authentication token generator uses the Goth library but it's not available 189 | 190 | Add goth to your dependencies: 191 | 192 | defp deps do 193 | {:goth, "~> 1.3"} 194 | end 195 | 196 | Or provide your own token generator: 197 | 198 | Broadway.start_link( 199 | producers: [ 200 | default: [ 201 | module: {BroadwayCloudPubSub.Producer, 202 | token_generator: {MyGenerator, :generate, ["foo"]} 203 | } 204 | ] 205 | ] 206 | ) 207 | """) 208 | end 209 | end 210 | 211 | def type_atom_action_or_nack_with_bounded_integer(:ack, [{:name, _}, {:min, _}, {:max, _}]) do 212 | {:ok, :ack} 213 | end 214 | 215 | def type_atom_action_or_nack_with_bounded_integer(:noop, [{:name, _}, {:min, _}, {:max, _}]) do 216 | {:ok, :noop} 217 | end 218 | 219 | def type_atom_action_or_nack_with_bounded_integer(:nack, [{:name, _}, {:min, _}, {:max, _}]) do 220 | {:ok, {:nack, 0}} 221 | end 222 | 223 | def type_atom_action_or_nack_with_bounded_integer( 224 | {:nack, value}, 225 | [{:name, _}, {:min, min}, {:max, max}] 226 | ) 227 | when is_integer(value) and value >= min and value <= max do 228 | {:ok, {:nack, value}} 229 | end 230 | 231 | def type_atom_action_or_nack_with_bounded_integer(value, [ 232 | {:name, name}, 233 | {:min, min}, 234 | {:max, max} 235 | ]) do 236 | {:error, 237 | "expected :#{name} to be one of :ack, :noop, :nack, or {:nack, integer} where " <> 238 | "integer is between #{min} and #{max}, got: #{inspect(value)}"} 239 | end 240 | 241 | def type_non_empty_string("", [{:name, name}]) do 242 | {:error, "expected :#{name} to be a non-empty string, got: \"\""} 243 | end 244 | 245 | def type_non_empty_string(value, _) 246 | when not is_nil(value) and is_binary(value) do 247 | {:ok, value} 248 | end 249 | 250 | def type_non_empty_string(value, [{:name, name}]) do 251 | {:error, "expected :#{name} to be a non-empty string, got: #{inspect(value)}"} 252 | end 253 | 254 | def type_non_empty_string_or_tagged_tuple(value, [{:name, name}]) 255 | when not (is_binary(value) or is_tuple(value)) or (value == "" or value == {}) do 256 | {:error, "expected :#{name} to be a non-empty string or tuple, got: #{inspect(value)}"} 257 | end 258 | 259 | def type_non_empty_string_or_tagged_tuple(value, _) do 260 | {:ok, value} 261 | end 262 | 263 | def type_positive_integer_or_infinity(value, _) when is_integer(value) and value > 0 do 264 | {:ok, value} 265 | end 266 | 267 | def type_positive_integer_or_infinity(:infinity, _) do 268 | {:ok, :infinity} 269 | end 270 | 271 | def type_positive_integer_or_infinity(value, [{:name, name}]) do 272 | {:error, "expected :#{name} to be a positive integer or :infinity, got: #{inspect(value)}"} 273 | end 274 | 275 | @doc false 276 | def generate_goth_token(goth_name) do 277 | with {:ok, %{token: token}} <- Goth.fetch(goth_name) do 278 | {:ok, token} 279 | end 280 | end 281 | end 282 | -------------------------------------------------------------------------------- /lib/broadway_cloud_pub_sub/producer.ex: -------------------------------------------------------------------------------- 1 | defmodule BroadwayCloudPubSub.Producer do 2 | @moduledoc """ 3 | A GenStage producer that continuously receives messages from a Google Cloud Pub/Sub 4 | topic and acknowledges them after being successfully processed. 5 | 6 | By default this producer uses `BroadwayCloudPubSub.PullClient` to talk to Cloud 7 | Pub/Sub, but you can provide your client by implementing the `BroadwayCloudPubSub.Client` 8 | behaviour. 9 | 10 | For a quick getting started on using Broadway with Cloud Pub/Sub, please see 11 | the [Google Cloud Pub/Sub Guide](https://hexdocs.pm/broadway/google-cloud-pubsub.html). 12 | 13 | ## Options 14 | 15 | Aside from `:receive_interval` and `:client` which are generic and apply to all 16 | producers (regardless of the client implementation), all other options are specific to 17 | `BroadwayCloudPubSub.PullClient`, which is the default client. 18 | 19 | #{NimbleOptions.docs(BroadwayCloudPubSub.Options.definition())} 20 | 21 | ### Custom token generator 22 | 23 | A custom token generator can be given as a MFArgs tuple. 24 | 25 | For example, define a `MyApp.fetch_token/0` function: 26 | 27 | defmodule MyApp do 28 | 29 | @doc "Fetches a Google auth token" 30 | def fetch_token do 31 | with {:ok, token} <- Goth.fetch(MyApp.Goth) 32 | {:ok, token.token} 33 | end 34 | end 35 | end 36 | 37 | and configure the producer to use it: 38 | 39 | token_generator: {MyApp, :fetch_token, []} 40 | 41 | ## Acknowledgements 42 | 43 | You can use the `:on_success` and `:on_failure` options to control how 44 | messages are acknowledged with the Pub/Sub system. 45 | 46 | By default successful messages are acknowledged and failed messages are ignored. 47 | You can set `:on_success` and `:on_failure` when starting this producer, 48 | or change them for each message through `Broadway.Message.configure_ack/2`. 49 | 50 | The following values are supported by both `:on_success` and `:on_failure`: 51 | 52 | * `:ack` - Acknowledge the message. Pub/Sub can remove the message from 53 | the subscription. 54 | 55 | * `:noop` - Do nothing. No requests will be made to Pub/Sub, and the 56 | message will be rescheduled according to the subscription-level 57 | `ackDeadlineSeconds`. 58 | 59 | * `:nack` - Make a request to Pub/Sub to set `ackDeadlineSeconds` to `0`, 60 | which may cause the message to be immediately redelivered to another 61 | connected consumer. Note that this does not modify the subscription-level 62 | `ackDeadlineSeconds` used for subsequent messages. 63 | 64 | * `{:nack, integer}` - Modifies the `ackDeadlineSeconds` for a particular 65 | message. Note that this does not modify the subscription-level 66 | `ackDeadlineSeconds` used for subsequent messages. 67 | 68 | ### Batching 69 | 70 | Even if you are not interested in working with Broadway batches via the 71 | `handle_batch/3` callback, we recommend all Broadway pipelines with Pub/Sub 72 | producers to define a default batcher with `batch_size` set to 10, so 73 | messages can be acknowledged in batches, which improves the performance 74 | and reduces the cost of integrating with Google Cloud Pub/Sub. In addition, 75 | you should ensure that `batch_timeout` is set to a value less than 76 | the acknowledgement deadline on the subscription. Otherwise you could 77 | potentially have messages that remain in the subscription and are never 78 | acknowledged successfully. 79 | 80 | ### Example 81 | 82 | Broadway.start_link(MyBroadway, 83 | name: MyBroadway, 84 | producer: [ 85 | module: {BroadwayCloudPubSub.Producer, 86 | goth: MyApp.Goth, 87 | subscription: "projects/my-project/subscriptions/my_subscription" 88 | } 89 | ], 90 | processors: [ 91 | default: [] 92 | ], 93 | batchers: [ 94 | default: [ 95 | batch_size: 10, 96 | batch_timeout: 2_000 97 | ] 98 | ] 99 | ) 100 | 101 | The above configuration will set up a producer that continuously receives 102 | messages from `"projects/my-project/subscriptions/my_subscription"` and sends 103 | them downstream. 104 | 105 | ## Telemetry 106 | 107 | This producer emits a few [Telemetry](https://github.com/beam-telemetry/telemetry) 108 | events which are listed below. 109 | 110 | * `[:broadway_cloud_pub_sub, :pull_client, :receive_messages, :start | :stop | :exception]` spans - 111 | these events are emitted in "span style" when executing pull requests to GCP PubSub. 112 | See `:telemetry.span/3`. 113 | 114 | All these events have the measurements described in `:telemetry.span/3`. The events 115 | contain the following metadata: 116 | 117 | * `:max_messages` - the number of messages requested after applying the `max_messages` 118 | config option to the existing demand 119 | * `:demand` - the total demand accumulated into the producer 120 | * `:name` - the name of the Broadway topology 121 | 122 | * `[:broadway_cloud_pub_sub, :pull_client, :ack, :start | :stop | :exception]` span - these events 123 | are emitted in "span style" when acking messages on GCP PubSub. See `:telemetry.span/3`. 124 | 125 | All these events have the measurements described in `:telemetry.span/3`. The events 126 | contain the following metadata: 127 | 128 | * `:name` - the name of the Broadway topology 129 | """ 130 | 131 | use GenStage 132 | alias Broadway.Producer 133 | alias BroadwayCloudPubSub.{Acknowledger, Options} 134 | 135 | @behaviour Producer 136 | 137 | @impl GenStage 138 | def init(opts) do 139 | receive_interval = opts[:receive_interval] 140 | client = opts[:client] 141 | 142 | {:ok, config} = client.init(opts) 143 | ack_ref = opts[:broadway][:name] 144 | 145 | {:producer, 146 | %{ 147 | demand: 0, 148 | draining: false, 149 | receive_timer: nil, 150 | receive_interval: receive_interval, 151 | client: {client, config}, 152 | ack_ref: ack_ref, 153 | worker_task: nil 154 | }} 155 | end 156 | 157 | @impl Producer 158 | def prepare_for_start(_module, broadway_opts) do 159 | {producer_module, client_opts} = broadway_opts[:producer][:module] 160 | 161 | opts = NimbleOptions.validate!(client_opts, Options.definition()) 162 | 163 | ack_ref = broadway_opts[:name] 164 | 165 | {client, opts} = 166 | case opts[:client] do 167 | {client, client_opts} -> {client, Keyword.merge(opts, client_opts)} 168 | client -> {client, opts} 169 | end 170 | 171 | opts = Keyword.put(opts, :client, client) 172 | 173 | opts = 174 | Keyword.put_new_lazy(opts, :token_generator, fn -> 175 | Options.make_token_generator(opts) 176 | end) 177 | 178 | {specs, opts} = prepare_to_connect(broadway_opts[:name], client, opts) 179 | 180 | ack_opts = Keyword.put(opts, :topology_name, broadway_opts[:name]) 181 | 182 | :persistent_term.put(ack_ref, Map.new(ack_opts)) 183 | 184 | broadway_opts_with_defaults = 185 | put_in(broadway_opts, [:producer, :module], {producer_module, opts}) 186 | 187 | {specs, broadway_opts_with_defaults} 188 | end 189 | 190 | defp prepare_to_connect(module, client, producer_opts) do 191 | if Code.ensure_loaded?(client) and function_exported?(client, :prepare_to_connect, 2) do 192 | client.prepare_to_connect(module, producer_opts) 193 | else 194 | {[], producer_opts} 195 | end 196 | end 197 | 198 | @impl true 199 | def handle_demand(incoming_demand, %{demand: demand} = state) do 200 | handle_receive_messages(%{state | demand: demand + incoming_demand}) 201 | end 202 | 203 | @impl true 204 | def handle_info(:receive_messages, state) do 205 | handle_receive_messages(%{state | receive_timer: nil}) 206 | end 207 | 208 | def handle_info({ref, messages}, %{demand: demand, worker_task: %{ref: ref}} = state) do 209 | new_demand = demand - length(messages) 210 | 211 | receive_timer = 212 | case {messages, new_demand} do 213 | {[], _} -> 214 | schedule_receive_messages(state.receive_interval) 215 | 216 | {_, 0} -> 217 | nil 218 | 219 | _ -> 220 | schedule_receive_messages(0) 221 | end 222 | 223 | {:noreply, messages, 224 | %{state | demand: new_demand, receive_timer: receive_timer, worker_task: nil}} 225 | end 226 | 227 | @impl true 228 | def handle_info(_, state) do 229 | {:noreply, [], state} 230 | end 231 | 232 | @impl Producer 233 | def prepare_for_draining(state) do 234 | if state.worker_task do 235 | Task.shutdown(state.worker_task, :brutal_kill) 236 | end 237 | 238 | {:noreply, [], %{state | worker_task: nil, draining: true}} 239 | end 240 | 241 | defp handle_receive_messages(%{draining: true} = state) do 242 | {:noreply, [], state} 243 | end 244 | 245 | defp handle_receive_messages(%{receive_timer: nil, demand: demand, worker_task: nil} = state) 246 | when demand > 0 do 247 | task = receive_messages_from_pubsub(state, demand) 248 | 249 | {:noreply, [], %{state | worker_task: task}} 250 | end 251 | 252 | defp handle_receive_messages(state) do 253 | {:noreply, [], state} 254 | end 255 | 256 | defp receive_messages_from_pubsub(state, total_demand) do 257 | %{client: {client, opts}, ack_ref: ack_ref} = state 258 | 259 | Task.async(fn -> 260 | client.receive_messages(total_demand, Acknowledger.builder(ack_ref), opts) 261 | end) 262 | end 263 | 264 | defp schedule_receive_messages(interval) do 265 | Process.send_after(self(), :receive_messages, interval) 266 | end 267 | end 268 | -------------------------------------------------------------------------------- /lib/broadway_cloud_pub_sub/pull_client.ex: -------------------------------------------------------------------------------- 1 | defmodule BroadwayCloudPubSub.PullClient do 2 | @moduledoc """ 3 | A subscriptions [pull client](https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull) built on `Finch`. 4 | """ 5 | alias Broadway.Message 6 | alias BroadwayCloudPubSub.Client 7 | alias Finch.Response 8 | 9 | require Logger 10 | 11 | @behaviour Client 12 | 13 | @default_retry_codes [408, 500, 502, 503, 504, 522, 524] 14 | @default_retry_delay_ms 500 15 | @default_max_retries 10 16 | 17 | @impl Client 18 | def prepare_to_connect(name, producer_opts) do 19 | case Keyword.fetch(producer_opts, :finch) do 20 | {:ok, nil} -> 21 | prepare_finch(name, producer_opts) 22 | 23 | {:ok, _} -> 24 | {[], producer_opts} 25 | 26 | :error -> 27 | prepare_finch(name, producer_opts) 28 | end 29 | end 30 | 31 | defp prepare_finch(name, producer_opts) do 32 | finch = Module.concat(name, __MODULE__) 33 | 34 | specs = [ 35 | {Finch, name: finch} 36 | ] 37 | 38 | producer_opts = Keyword.put(producer_opts, :finch, finch) 39 | 40 | {specs, producer_opts} 41 | end 42 | 43 | @impl Client 44 | def init(opts) do 45 | {:ok, Map.new(opts)} 46 | end 47 | 48 | @impl Client 49 | def receive_messages(demand, ack_builder, config) do 50 | max_messages = min(demand, config.max_number_of_messages) 51 | 52 | :telemetry.span( 53 | [:broadway_cloud_pub_sub, :pull_client, :receive_messages], 54 | %{ 55 | max_messages: max_messages, 56 | demand: demand, 57 | name: config.broadway[:name] 58 | }, 59 | fn -> 60 | result = 61 | config 62 | |> execute(:pull, %{"maxMessages" => max_messages}) 63 | |> handle_response(:receive_messages) 64 | |> wrap_received_messages(ack_builder) 65 | 66 | {result, %{name: config.broadway[:name], max_messages: max_messages, demand: demand}} 67 | end 68 | ) 69 | end 70 | 71 | @impl Client 72 | def put_deadline(ack_ids, ack_deadline_seconds, config) do 73 | payload = %{ 74 | "ackIds" => ack_ids, 75 | "ackDeadlineSeconds" => ack_deadline_seconds 76 | } 77 | 78 | config 79 | |> execute(:modack, payload) 80 | |> handle_response(:put_deadline) 81 | end 82 | 83 | @impl Client 84 | def acknowledge(ack_ids, config) do 85 | :telemetry.span( 86 | [:broadway_cloud_pub_sub, :pull_client, :ack], 87 | %{name: config.topology_name}, 88 | fn -> 89 | result = 90 | config 91 | |> execute(:acknowledge, %{"ackIds" => ack_ids}) 92 | |> handle_response(:acknowledge) 93 | 94 | {result, %{name: config.topology_name}} 95 | end 96 | ) 97 | end 98 | 99 | defp handle_response({:ok, response}, :receive_messages) do 100 | case response do 101 | %{"receivedMessages" => received_messages} -> received_messages 102 | _ -> [] 103 | end 104 | end 105 | 106 | defp handle_response({:ok, _}, _) do 107 | :ok 108 | end 109 | 110 | defp handle_response({:error, reason}, :receive_messages) do 111 | Logger.error("Unable to fetch events from Cloud Pub/Sub - reason: #{reason}") 112 | [] 113 | end 114 | 115 | defp handle_response({:error, reason}, :acknowledge) do 116 | Logger.error("Unable to acknowledge messages with Cloud Pub/Sub - reason: #{reason}") 117 | :ok 118 | end 119 | 120 | defp handle_response({:error, reason}, :put_deadline) do 121 | Logger.error("Unable to put new ack deadline with Cloud Pub/Sub - reason: #{reason}") 122 | :ok 123 | end 124 | 125 | defp wrap_received_messages(pub_sub_messages, ack_builder) do 126 | Enum.map(pub_sub_messages, fn pub_sub_msg -> 127 | pub_sub_msg_to_broadway_msg(pub_sub_msg, ack_builder) 128 | end) 129 | end 130 | 131 | defp pub_sub_msg_to_broadway_msg(pub_sub_msg, ack_builder) do 132 | %{"ackId" => ack_id, "message" => message} = pub_sub_msg 133 | 134 | # 2022-09-21 (MC) The docs falsely claim the following: 135 | # "If a DeadLetterPolicy is not set on the subscription, this will be 0." 136 | # In reality, if DeadLetterPolicy is not set, neither is the deliveryAttempt field. 137 | # https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull#receivedmessage 138 | delivery_attempt = Map.get(pub_sub_msg, "deliveryAttempt") 139 | 140 | {data, metadata} = 141 | message 142 | |> decode_message() 143 | |> Map.pop("data") 144 | 145 | metadata = %{ 146 | attributes: metadata["attributes"], 147 | deliveryAttempt: delivery_attempt, 148 | messageId: metadata["messageId"], 149 | orderingKey: metadata["orderingKey"], 150 | publishTime: parse_datetime(metadata["publishTime"]) 151 | } 152 | 153 | %Message{ 154 | data: data, 155 | metadata: metadata, 156 | acknowledger: ack_builder.(ack_id) 157 | } 158 | end 159 | 160 | defp parse_datetime(nil), do: nil 161 | 162 | defp parse_datetime(str) when is_binary(str) do 163 | case DateTime.from_iso8601(str) do 164 | {:ok, dt, _} -> 165 | dt 166 | 167 | err -> 168 | raise """ 169 | invalid datetime string: #{inspect(err)} 170 | """ 171 | end 172 | end 173 | 174 | defp decode_message(%{"data" => encoded_data} = message) when is_binary(encoded_data) do 175 | %{message | "data" => Base.decode64!(encoded_data)} 176 | end 177 | 178 | defp decode_message(%{"data" => nil} = message), do: message 179 | defp decode_message(%{} = message) when not is_map_key(message, "data"), do: message 180 | 181 | defp headers(config) do 182 | token = get_token(config) 183 | [{"authorization", "Bearer #{token}"}, {"content-type", "application/json"}] 184 | end 185 | 186 | @mod_ack_action "modifyAckDeadline" 187 | defp url(config, :modack), do: url(config, @mod_ack_action) 188 | 189 | defp url(config, action) do 190 | sub = URI.encode(config.subscription) 191 | path = "/v1/" <> sub <> ":" <> to_string(action) 192 | config.base_url <> path 193 | end 194 | 195 | defp execute(config, action, payload) do 196 | url = url(config, action) 197 | body = Jason.encode!(payload) 198 | headers = headers(config) 199 | execute(url, body, headers, config, action, payload, max_retries(config)) 200 | end 201 | 202 | defp execute(url, body, headers, config, action, payload, retries_left) do 203 | case finch_request(config.finch, url, body, headers, config.receive_timeout) do 204 | {:ok, %Response{status: 200, body: body}} -> 205 | {:ok, Jason.decode!(body)} 206 | 207 | {:ok, %Response{} = resp} -> 208 | maybe_retry(resp, url, body, headers, config, action, payload, retries_left) 209 | 210 | {:error, err} -> 211 | {:error, format_error(url, err)} 212 | end 213 | end 214 | 215 | defp maybe_retry(resp, url, body, headers, config, action, payload, retries_left) do 216 | if should_retry(resp, config, retries_left) do 217 | config |> retry_delay() |> Process.sleep() 218 | execute(url, body, headers, config, action, payload, retries_left - 1) 219 | else 220 | {:error, format_error(url, resp)} 221 | end 222 | end 223 | 224 | defp should_retry(%Response{status: status}, config, retries_left) do 225 | retries_left > 0 and status in retry_codes(config) 226 | end 227 | 228 | defp max_retries(config) do 229 | config[:max_retries] || @default_max_retries 230 | end 231 | 232 | defp retry_codes(config) do 233 | config[:retry_codes] || @default_retry_codes 234 | end 235 | 236 | defp retry_delay(config) do 237 | config[:retry_delay_ms] || @default_retry_delay_ms 238 | end 239 | 240 | defp finch_request(finch, url, body, headers, timeout) do 241 | :post 242 | |> Finch.build(url, headers, body) 243 | |> Finch.request(finch, receive_timeout: timeout) 244 | end 245 | 246 | defp get_token(%{token_generator: {m, f, a}}) do 247 | {:ok, token} = apply(m, f, a) 248 | token 249 | end 250 | 251 | defp format_error(url, %Response{status: status, body: body}) do 252 | """ 253 | \nRequest to #{inspect(url)} failed with status #{inspect(status)}, got: 254 | #{inspect(body)} 255 | """ 256 | end 257 | 258 | defp format_error(url, err) do 259 | inspect(%{url: url, error: err}) 260 | end 261 | end 262 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule BroadwayCloudPubSub.MixProject do 2 | use Mix.Project 3 | 4 | @version "0.9.1" 5 | @description "A Google Cloud Pub/Sub connector for Broadway" 6 | @repo_url "https://github.com/dashbitco/broadway_cloud_pub_sub" 7 | 8 | def project do 9 | [ 10 | app: :broadway_cloud_pub_sub, 11 | version: @version, 12 | elixir: "~> 1.11", 13 | elixirc_paths: elixirc_paths(Mix.env()), 14 | name: "BroadwayCloudPubSub", 15 | description: @description, 16 | start_permanent: Mix.env() == :prod, 17 | deps: deps(), 18 | docs: docs(), 19 | package: package() 20 | ] 21 | end 22 | 23 | def application do 24 | [ 25 | extra_applications: [:logger] 26 | ] 27 | end 28 | 29 | # Specifies which paths to compile per environment. 30 | defp elixirc_paths(:test), do: ["lib", "test/support"] 31 | defp elixirc_paths(_), do: ["lib"] 32 | 33 | defp deps do 34 | [ 35 | {:broadway, "~> 1.0"}, 36 | {:finch, "~> 0.9"}, 37 | {:jason, "~> 1.0"}, 38 | {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0"}, 39 | {:telemetry, "~> 0.4.3 or ~> 1.0"}, 40 | {:goth, "~> 1.3", optional: true}, 41 | {:ex_doc, "~> 0.23", only: :docs}, 42 | {:bypass, "~> 2.1", only: :test} 43 | ] 44 | end 45 | 46 | defp docs do 47 | [ 48 | main: "BroadwayCloudPubSub.Producer", 49 | nest_modules_by_prefix: [BroadwayCloudPubSub], 50 | source_ref: "v#{@version}", 51 | source_url: @repo_url, 52 | extras: [ 53 | "README.md", 54 | "CHANGELOG.md" 55 | ] 56 | ] 57 | end 58 | 59 | defp package do 60 | %{ 61 | licenses: ["Apache-2.0"], 62 | links: %{"GitHub" => @repo_url} 63 | } 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "broadway": {:hex, :broadway, "1.0.0", "da99ca10aa221a9616ccff8cb8124510b7e063112d4593c3bae50448b37bbc90", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b86ebd492f687edc9ad44d0f9e359da70f305b6d090e92a06551cef71ec41324"}, 3 | "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, 4 | "castore": {:hex, :castore, "0.1.18", "deb5b9ab02400561b6f5708f3e7660fc35ca2d51bfc6a940d2f513f89c2975fc", [:mix], [], "hexpm", "61bbaf6452b782ef80b33cdb45701afbcf0a918a45ebe7e73f1130d661e66a06"}, 5 | "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, 6 | "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, 7 | "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, 8 | "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, 9 | "ex_doc": {:hex, :ex_doc, "0.34.1", "9751a0419bc15bc7580c73fde506b17b07f6402a1e5243be9e0f05a68c723368", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "d441f1a86a235f59088978eff870de2e815e290e44a8bd976fe5d64470a4c9d2"}, 10 | "finch": {:hex, :finch, "0.9.0", "8b772324aebafcaba763f1dffaa3e7f52f8c4e52485f50f48bbb2f42219a2e87", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.5", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a93bfcad9ca50fa3cb2d459f27667d9a87cfbb7fecf9b29b2e78a50bc2ab445d"}, 11 | "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, 12 | "goth": {:hex, :goth, "1.4.2", "a598dfbce6fe65db3f5f43b1ab2ce8fbe3b2fe20a7569ad62d71c11c0ddc3f41", [:mix], [{:finch, "~> 0.9", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.11", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "d51bb6544dc551fe5754ab72e6cf194120b3c06d924282aaa3321a516ed3b98a"}, 13 | "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, 14 | "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, 15 | "jose": {:hex, :jose, "1.11.6", "613fda82552128aa6fb804682e3a616f4bc15565a048dabd05b1ebd5827ed965", [:mix, :rebar3], [], "hexpm", "6275cb75504f9c1e60eeacb771adfeee4905a9e182103aa59b53fed651ff9738"}, 16 | "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, 17 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, 18 | "makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"}, 19 | "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, 20 | "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"}, 21 | "nimble_options": {:hex, :nimble_options, "0.3.7", "1e52dd7673d36138b1a5dede183b5d86dff175dc46d104a8e98e396b85b04670", [:mix], [], "hexpm", "2086907e6665c6b6579be54ef5001928df5231f355f71ed258f80a55e9f63633"}, 22 | "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, 23 | "nimble_pool": {:hex, :nimble_pool, "0.2.6", "91f2f4c357da4c4a0a548286c84a3a28004f68f05609b4534526871a22053cde", [:mix], [], "hexpm", "1c715055095d3f2705c4e236c18b618420a35490da94149ff8b580a2144f653f"}, 24 | "plug": {:hex, :plug, "1.11.1", "f2992bac66fdae679453c9e86134a4201f6f43a687d8ff1cd1b2862d53c80259", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "23524e4fefbb587c11f0833b3910bfb414bf2e2534d61928e920f54e3a1b881f"}, 25 | "plug_cowboy": {:hex, :plug_cowboy, "2.5.0", "51c998f788c4e68fc9f947a5eba8c215fbb1d63a520f7604134cab0270ea6513", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5b2c8925a5e2587446f33810a58c01e66b3c345652eeec809b76ba007acde71a"}, 26 | "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, 27 | "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, 28 | "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, 29 | } 30 | -------------------------------------------------------------------------------- /test/broadway_cloud_pub_sub/acknowledger_test.exs: -------------------------------------------------------------------------------- 1 | defmodule BroadwayCloudPubSub.AcknowledgerTest do 2 | use ExUnit.Case 3 | alias Broadway.Message 4 | alias BroadwayCloudPubSub.Client 5 | alias BroadwayCloudPubSub.Acknowledger 6 | 7 | defmodule CallerClient do 8 | alias BroadwayCloudPubSub.Acknowledger 9 | 10 | @behaviour Client 11 | 12 | @impl Client 13 | def init(opts) do 14 | {:ok, %{test_pid: opts[:test_pid]}} 15 | end 16 | 17 | @impl Client 18 | def receive_messages(_demand, _ack_builder, _opts), do: [] 19 | 20 | @impl Client 21 | def acknowledge(ack_ids, config) do 22 | send(config.test_pid, {:acknowledge, length(ack_ids)}) 23 | end 24 | 25 | @impl Client 26 | def put_deadline(ack_ids, deadline, config) do 27 | send(config.test_pid, {:put_deadline, length(ack_ids), deadline}) 28 | end 29 | end 30 | 31 | defp init_with_ack_ref(opts) do 32 | ack_ref = opts[:broadway][:name] 33 | 34 | :persistent_term.put(ack_ref, %{ 35 | base_url: "http://localhost:8085", 36 | client: CallerClient, 37 | on_failure: opts[:on_failure] || :noop, 38 | on_success: opts[:on_success] || :ack, 39 | subscription: "projects/test/subscriptions/test-subscription", 40 | # Required for the CallerClient 41 | test_pid: opts[:test_pid], 42 | token_generator: {Token, :generate, []} 43 | }) 44 | 45 | {:ok, _config} = CallerClient.init(opts) 46 | 47 | ack_ref 48 | end 49 | 50 | describe "configure/3" do 51 | test "raise on unsupported configure option" do 52 | assert_raise(NimbleOptions.ValidationError, ~r/unknown options \[:on_other\]/, fn -> 53 | Acknowledger.configure(:ack_ref, %{}, on_other: :ack) 54 | end) 55 | end 56 | 57 | test "raise on unsupported on_success value" do 58 | assert_raise( 59 | NimbleOptions.ValidationError, 60 | ~r/expected :on_success to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: :unknown/, 61 | fn -> 62 | Acknowledger.configure(:ack_ref, %{}, on_success: :unknown) 63 | end 64 | ) 65 | end 66 | 67 | test "raise on unsupported on_failure value" do 68 | assert_raise( 69 | NimbleOptions.ValidationError, 70 | ~r/expected :on_failure to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: :unknown/, 71 | fn -> 72 | Acknowledger.configure(:ack_ref, %{}, on_failure: :unknown) 73 | end 74 | ) 75 | end 76 | 77 | test "sets defaults" do 78 | ack_data = %{ack_id: "1"} 79 | expected = %{ack_id: "1", on_success: :ack, on_failure: :noop} 80 | 81 | assert {:ok, expected} == Acknowledger.configure(:ack_ref, ack_data, []) 82 | end 83 | 84 | test "set on_success with ignore" do 85 | ack_data = %{ack_id: "1"} 86 | expected = %{ack_id: "1", on_success: :noop, on_failure: :noop} 87 | 88 | assert {:ok, expected} == 89 | Acknowledger.configure(:ack_ref, ack_data, on_success: :noop) 90 | end 91 | 92 | test "set on_failure with deadline 0" do 93 | ack_data = %{ack_id: "1"} 94 | expected = %{ack_id: "1", on_success: :ack, on_failure: {:nack, 0}} 95 | 96 | assert {:ok, expected} == 97 | Acknowledger.configure(:ack_ref, ack_data, on_failure: :nack) 98 | end 99 | 100 | test "set on_failure with custom deadline" do 101 | ack_data = %{ack_id: "1"} 102 | expected = %{ack_id: "1", on_success: :ack, on_failure: {:nack, 60}} 103 | 104 | assert {:ok, expected} == 105 | Acknowledger.configure(:ack_ref, ack_data, on_failure: {:nack, 60}) 106 | end 107 | end 108 | 109 | describe "ack/3" do 110 | setup do 111 | producer_opts = [ 112 | # will be injected by Broadway at runtime 113 | broadway: [name: :Broadway4], 114 | client: CallerClient, 115 | test_pid: self() 116 | ] 117 | 118 | {:ok, producer_opts: producer_opts} 119 | end 120 | 121 | test "with defaults, only successful messages are acknowledged", %{producer_opts: opts} do 122 | ack_ref = init_with_ack_ref(opts) 123 | 124 | messages = build_messages(6, ack_ref) 125 | 126 | {successful, failed} = Enum.split(messages, 3) 127 | 128 | Acknowledger.ack(ack_ref, successful, failed) 129 | 130 | assert_received {:acknowledge, 3} 131 | end 132 | 133 | test "overriding default on_success", %{producer_opts: opts} do 134 | ack_ref = init_with_ack_ref([on_success: :noop] ++ opts) 135 | 136 | messages = build_messages(6, ack_ref) 137 | 138 | {successful, failed} = Enum.split(messages, 3) 139 | 140 | Acknowledger.ack(ack_ref, successful, failed) 141 | 142 | refute_received {:acknowledge, 3} 143 | end 144 | 145 | test "overriding default on_failure", %{producer_opts: opts} do 146 | ack_ref = init_with_ack_ref([on_failure: :ack] ++ opts) 147 | 148 | messages = build_messages(6, ack_ref) 149 | 150 | {successful, failed} = Enum.split(messages, 3) 151 | 152 | Acknowledger.ack(ack_ref, successful, failed) 153 | 154 | assert_received {:acknowledge, 6} 155 | end 156 | 157 | test "overriding message on_success", %{producer_opts: opts} do 158 | ack_ref = init_with_ack_ref(opts) 159 | 160 | [first | rest] = build_messages(3, ack_ref) 161 | 162 | first = Message.configure_ack(first, on_success: :nack) 163 | 164 | Acknowledger.ack(ack_ref, [first | rest], []) 165 | 166 | assert_received({:acknowledge, 2}) 167 | assert_received({:put_deadline, 1, 0}) 168 | end 169 | 170 | test "overriding message on_failure", %{producer_opts: opts} do 171 | ack_ref = init_with_ack_ref(opts) 172 | 173 | [first | rest] = build_messages(3, ack_ref) 174 | 175 | first = Message.configure_ack(first, on_failure: :nack) 176 | 177 | Acknowledger.ack(ack_ref, rest, [first]) 178 | 179 | assert_received({:acknowledge, 2}) 180 | assert_received({:put_deadline, 1, 0}) 181 | end 182 | 183 | test "groups successful and failed messages by action", %{producer_opts: opts} do 184 | ack_ref = init_with_ack_ref([on_failure: :ack] ++ opts) 185 | 186 | messages = build_messages(6, ack_ref) 187 | 188 | {successful, failed} = Enum.split(messages, 3) 189 | 190 | Acknowledger.ack(ack_ref, successful, failed) 191 | 192 | assert_received({:acknowledge, 6}) 193 | end 194 | 195 | test "configuring message treats :nack as {:nack, 0}", %{producer_opts: opts} do 196 | ack_ref = init_with_ack_ref([on_success: {:nack, 0}, on_failure: {:nack, 0}] ++ opts) 197 | 198 | [first | messages] = build_messages(6, ack_ref) 199 | 200 | first = Message.configure_ack(first, on_success: :nack) 201 | 202 | {successful, failed} = Enum.split([first | messages], 3) 203 | 204 | Acknowledger.ack(ack_ref, successful, failed) 205 | 206 | assert_received({:put_deadline, 6, 0}) 207 | end 208 | 209 | test "chunks actions every 3_000 ack_ids", %{producer_opts: opts} do 210 | ack_ref = init_with_ack_ref([on_failure: :nack] ++ opts) 211 | 212 | messages = build_messages(10_000, ack_ref) 213 | 214 | {successful, failed} = Enum.split(messages, 3_500) 215 | 216 | Acknowledger.ack(ack_ref, successful, failed) 217 | 218 | assert_received({:acknowledge, 2_500}) 219 | assert_received({:acknowledge, 1_000}) 220 | assert_received({:put_deadline, 2_500, 0}) 221 | assert_received({:put_deadline, 2_500, 0}) 222 | assert_received({:put_deadline, 1_500, 0}) 223 | end 224 | end 225 | 226 | defp build_messages(n, ack_ref) when is_integer(n) and n > 1 do 227 | Enum.map(1..n, &build_message(&1, ack_ref)) 228 | end 229 | 230 | defp build_message(data, ack_ref) do 231 | acknowledger = Acknowledger.builder(ack_ref).("Ack_#{inspect(data)}") 232 | %Message{data: "Message_#{inspect(data)}", acknowledger: acknowledger} 233 | end 234 | end 235 | -------------------------------------------------------------------------------- /test/broadway_cloud_pub_sub/producer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule BroadwayCloudPubSub.ProducerTest do 2 | use ExUnit.Case 3 | 4 | alias Broadway.Message 5 | alias NimbleOptions.ValidationError 6 | 7 | defmodule MessageServer do 8 | use GenServer 9 | 10 | def start_link do 11 | GenServer.start_link(__MODULE__, []) 12 | end 13 | 14 | def push_messages(server, messages, timeout \\ 5000) do 15 | GenServer.call(server, {:messages, messages}, timeout) 16 | end 17 | 18 | def take_messages(server, amount, timeout \\ 5000) do 19 | GenServer.call(server, {:take, amount}, timeout) 20 | end 21 | 22 | def init(_opts) do 23 | {:ok, %{queue: [], awaiting: nil}} 24 | end 25 | 26 | def handle_call({:messages, messages}, _from, state) do 27 | messages = Enum.to_list(messages) 28 | 29 | if state.awaiting do 30 | {to_return, to_keep} = Enum.split(state.queue ++ messages, state.awaiting.amount) 31 | GenServer.reply(state.awaiting.from, to_return) 32 | 33 | {:reply, :ok, %{state | awaiting: nil, queue: state.queue ++ to_keep}} 34 | else 35 | {:reply, :ok, %{state | queue: state.queue ++ messages}} 36 | end 37 | end 38 | 39 | def handle_call({:take, amount}, from, state) do 40 | if length(state.queue) != 0 do 41 | {to_return, to_keep} = Enum.split(state.queue, amount) 42 | 43 | {:reply, to_return, %{state | queue: to_keep}} 44 | else 45 | state = %{state | awaiting: %{from: from, amount: amount}} 46 | {:noreply, state} 47 | end 48 | end 49 | end 50 | 51 | defmodule FakeClient do 52 | alias BroadwayCloudPubSub.Client 53 | alias Broadway.Acknowledger 54 | 55 | @behaviour Client 56 | @behaviour Acknowledger 57 | 58 | @impl Client 59 | def init(opts), do: {:ok, opts} 60 | 61 | @impl Client 62 | def receive_messages(amount, _builder, opts) do 63 | messages = MessageServer.take_messages(opts[:message_server], amount) 64 | send(opts[:test_pid], {:messages_received, length(messages)}) 65 | 66 | for msg <- messages do 67 | ack_data = %{ 68 | receipt: %{id: "Id_#{msg}", receipt_handle: "ReceiptHandle_#{msg}"}, 69 | test_pid: opts[:test_pid] 70 | } 71 | 72 | %Message{data: msg, acknowledger: {__MODULE__, :ack_ref, ack_data}} 73 | end 74 | end 75 | 76 | @impl Acknowledger 77 | def ack(_ack_ref, successful, _failed) do 78 | [%Message{acknowledger: {_, _, %{test_pid: test_pid}}} | _] = successful 79 | send(test_pid, {:messages_deleted, length(successful)}) 80 | end 81 | end 82 | 83 | defmodule FakePrepareToConnectClient do 84 | alias BroadwayCloudPubSub.Client 85 | 86 | @behaviour Client 87 | 88 | @impl Client 89 | def prepare_to_connect(_module, opts) do 90 | ref = 91 | opts[:prepare_to_connect_ref] || 92 | raise "expected :prepare_to_connect_ref option to be set, but it was not" 93 | 94 | test_pid = opts[:test_pid] 95 | 96 | task = 97 | {Task, 98 | fn -> 99 | send(test_pid, {:prepare_to_connect_spec, ref}) 100 | Process.sleep(:infinity) 101 | end} 102 | 103 | {[task], Keyword.put(opts, :prepare_to_connect_ref, ref)} 104 | end 105 | 106 | @impl Client 107 | def init(opts) do 108 | send(opts[:test_pid], {:prepare_to_connect_opts, opts[:prepare_to_connect_ref]}) 109 | 110 | {:ok, opts} 111 | end 112 | 113 | @impl Client 114 | def receive_messages(_amount, _builder, _opts), do: [] 115 | end 116 | 117 | defmodule Forwarder do 118 | use Broadway 119 | 120 | def handle_message(_, message, %{test_pid: test_pid}) do 121 | send(test_pid, {:message_handled, message.data}) 122 | message 123 | end 124 | 125 | def handle_batch(_, messages, _, _) do 126 | messages 127 | end 128 | end 129 | 130 | defp prepare_for_start_module_opts(module_opts) do 131 | {:ok, message_server} = MessageServer.start_link() 132 | {:ok, pid} = start_broadway(message_server) 133 | 134 | try do 135 | BroadwayCloudPubSub.Producer.prepare_for_start(Forwarder, 136 | producer: [ 137 | module: {BroadwayCloudPubSub.Producer, module_opts}, 138 | concurrency: 1 139 | ], 140 | name: __MODULE__ 141 | ) 142 | after 143 | stop_broadway(pid) 144 | end 145 | end 146 | 147 | describe "prepare_for_start/2 validation" do 148 | test ":subscription should be a string" do 149 | assert_raise( 150 | ValidationError, 151 | "required option :subscription not found, received options: [:client]", 152 | fn -> 153 | prepare_for_start_module_opts([]) 154 | end 155 | ) 156 | 157 | assert_raise( 158 | ValidationError, 159 | ~r/expected :subscription to be a non-empty string, got: nil/, 160 | fn -> 161 | prepare_for_start_module_opts(subscription: nil) 162 | end 163 | ) 164 | 165 | assert_raise( 166 | ValidationError, 167 | ~r/expected :subscription to be a non-empty string, got: \"\"/, 168 | fn -> 169 | prepare_for_start_module_opts(subscription: "") 170 | end 171 | ) 172 | 173 | assert_raise( 174 | ValidationError, 175 | ~r/expected :subscription to be a non-empty string, got: :foo/, 176 | fn -> 177 | prepare_for_start_module_opts(subscription: :foo) 178 | end 179 | ) 180 | 181 | assert { 182 | _, 183 | [ 184 | producer: [ 185 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 186 | concurrency: 1 187 | ], 188 | name: __MODULE__ 189 | ] 190 | } = 191 | prepare_for_start_module_opts( 192 | goth: FakeAuth, 193 | subscription: "projects/foo/subscriptions/bar" 194 | ) 195 | 196 | assert producer_opts[:subscription] == "projects/foo/subscriptions/bar" 197 | end 198 | 199 | test ":goth is optional, but logs an error with the default token generator" do 200 | assert ExUnit.CaptureLog.capture_log(fn -> 201 | assert {_, 202 | [ 203 | producer: [ 204 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 205 | concurrency: 1 206 | ], 207 | name: __MODULE__ 208 | ]} = 209 | prepare_for_start_module_opts( 210 | subscription: "projects/foo/subscriptions/bar" 211 | ) 212 | 213 | assert Keyword.fetch(producer_opts, :goth) == :error 214 | end) =~ "the :goth option is required for the default authentication token generator" 215 | end 216 | 217 | test ":max_number_of_messages is optional with default value 10" do 218 | assert {_, 219 | [ 220 | producer: [ 221 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 222 | concurrency: 1 223 | ], 224 | name: __MODULE__ 225 | ]} = 226 | prepare_for_start_module_opts( 227 | goth: FakeAuth, 228 | subscription: "projects/foo/subscriptions/bar" 229 | ) 230 | 231 | assert producer_opts[:max_number_of_messages] == 10 232 | end 233 | 234 | test ":max_number_of_messages should be a positive integer" do 235 | assert {_, 236 | [ 237 | producer: [ 238 | module: {BroadwayCloudPubSub.Producer, result_module_opts}, 239 | concurrency: 1 240 | ], 241 | name: __MODULE__ 242 | ]} = 243 | prepare_for_start_module_opts( 244 | goth: FakeAuth, 245 | subscription: "projects/foo/subscriptions/bar", 246 | max_number_of_messages: 1 247 | ) 248 | 249 | assert result_module_opts[:max_number_of_messages] == 1 250 | 251 | assert {_, 252 | [ 253 | producer: [ 254 | module: {BroadwayCloudPubSub.Producer, result_module_opts}, 255 | concurrency: 1 256 | ], 257 | name: __MODULE__ 258 | ]} = 259 | prepare_for_start_module_opts( 260 | goth: FakeAuth, 261 | subscription: "projects/foo/subscriptions/bar", 262 | max_number_of_messages: 10 263 | ) 264 | 265 | assert result_module_opts[:max_number_of_messages] == 10 266 | 267 | assert_raise( 268 | ValidationError, 269 | ~r/expected :max_number_of_messages to be a positive integer, got: 0/, 270 | fn -> 271 | prepare_for_start_module_opts( 272 | goth: FakeAuth, 273 | subscription: "projects/foo/subscriptions/bar", 274 | max_number_of_messages: 0 275 | ) 276 | end 277 | ) 278 | 279 | assert_raise( 280 | ValidationError, 281 | ~r/expected :max_number_of_messages to be a positive integer, got: -1/, 282 | fn -> 283 | prepare_for_start_module_opts( 284 | goth: FakeAuth, 285 | subscription: "projects/foo/subscriptions/bar", 286 | max_number_of_messages: -1 287 | ) 288 | end 289 | ) 290 | end 291 | 292 | test ":token_generator defaults to using Goth with the :goth option" do 293 | assert {_, 294 | [ 295 | producer: [ 296 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 297 | concurrency: 1 298 | ], 299 | name: __MODULE__ 300 | ]} = 301 | prepare_for_start_module_opts( 302 | goth: FakeAuth, 303 | subscription: "projects/foo/subscriptions/bar" 304 | ) 305 | 306 | assert producer_opts[:token_generator] == 307 | {BroadwayCloudPubSub.Options, :generate_goth_token, [FakeAuth]} 308 | end 309 | 310 | test ":token_generator should be a tuple {Mod, Fun, Args}" do 311 | token_generator = {Token, :fetch, []} 312 | 313 | assert {_, 314 | [ 315 | producer: [ 316 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 317 | concurrency: 1 318 | ], 319 | name: __MODULE__ 320 | ]} = 321 | prepare_for_start_module_opts( 322 | subscription: "projects/foo/subscriptions/bar", 323 | token_generator: token_generator 324 | ) 325 | 326 | assert producer_opts[:token_generator] == token_generator 327 | 328 | assert_raise ValidationError, 329 | ~r/expected :token_generator to be a tuple {Mod, Fun, Args}, got: {1, 1, 1}/, 330 | fn -> 331 | prepare_for_start_module_opts( 332 | subscription: "projects/foo/subscriptions/bar", 333 | token_generator: {1, 1, 1} 334 | ) 335 | end 336 | 337 | assert_raise ValidationError, 338 | ~r/expected :token_generator to be a tuple {Mod, Fun, Args}, got: SomeModule/, 339 | fn -> 340 | prepare_for_start_module_opts( 341 | subscription: "projects/foo/subscriptions/bar", 342 | token_generator: SomeModule 343 | ) 344 | end 345 | end 346 | 347 | test ":receive_timeout is optional with default value :infinity" do 348 | assert {_, 349 | [ 350 | producer: [ 351 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 352 | concurrency: 1 353 | ], 354 | name: __MODULE__ 355 | ]} = 356 | prepare_for_start_module_opts( 357 | goth: FakeAuth, 358 | subscription: "projects/foo/subscriptions/bar" 359 | ) 360 | 361 | assert producer_opts[:receive_timeout] == :infinity 362 | end 363 | 364 | test ":receive_timeout should be a positive integer or :infinity" do 365 | for value <- [0, -1, :an_atom, SomeModule] do 366 | assert_raise ValidationError, 367 | ~r/expected :receive_timeout to be a positive integer or :infinity, got: #{inspect(value)}/, 368 | fn -> 369 | prepare_for_start_module_opts( 370 | goth: FakeAuth, 371 | subscription: "projects/foo/subscriptions/bar", 372 | receive_timeout: value 373 | ) 374 | end 375 | end 376 | 377 | assert {_, 378 | [ 379 | producer: [ 380 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 381 | concurrency: 1 382 | ], 383 | name: __MODULE__ 384 | ]} = 385 | prepare_for_start_module_opts( 386 | goth: FakeAuth, 387 | subscription: "projects/foo/subscriptions/bar", 388 | receive_timeout: 15_000 389 | ) 390 | 391 | assert producer_opts[:receive_timeout] == 15_000 392 | end 393 | 394 | test ":on_success defaults to :ack" do 395 | assert {_, 396 | [ 397 | producer: [ 398 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 399 | concurrency: 1 400 | ], 401 | name: __MODULE__ 402 | ]} = 403 | prepare_for_start_module_opts( 404 | goth: FakeAuth, 405 | subscription: "projects/foo/subscriptions/bar" 406 | ) 407 | 408 | assert producer_opts[:on_success] == :ack 409 | end 410 | 411 | test ":on_failure defaults to :noop" do 412 | assert {_, 413 | [ 414 | producer: [ 415 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 416 | concurrency: 1 417 | ], 418 | name: __MODULE__ 419 | ]} = 420 | prepare_for_start_module_opts( 421 | goth: FakeAuth, 422 | subscription: "projects/foo/subscriptions/bar" 423 | ) 424 | 425 | assert producer_opts[:on_failure] == :noop 426 | end 427 | 428 | test ":on_success should be a valid action" do 429 | for action <- [:ack, :noop, {:nack, 0}, {:nack, 100}, {:nack, 600}] do 430 | assert {_, 431 | [ 432 | producer: [ 433 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 434 | concurrency: 1 435 | ], 436 | name: __MODULE__ 437 | ]} = 438 | prepare_for_start_module_opts( 439 | goth: FakeAuth, 440 | subscription: "projects/foo/subscriptions/bar", 441 | on_success: action 442 | ) 443 | 444 | assert producer_opts[:on_success] == action 445 | end 446 | 447 | assert_raise ValidationError, 448 | ~r/expected :on_success to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: :foo/, 449 | fn -> 450 | prepare_for_start_module_opts( 451 | goth: FakeAuth, 452 | subscription: "projects/foo/subscriptions/bar", 453 | on_success: :foo 454 | ) 455 | end 456 | 457 | assert_raise ValidationError, 458 | ~r/expected :on_success to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: "foo"/, 459 | fn -> 460 | prepare_for_start_module_opts( 461 | goth: FakeAuth, 462 | subscription: "projects/foo/subscriptions/bar", 463 | on_success: "foo" 464 | ) 465 | end 466 | 467 | assert_raise ValidationError, 468 | ~r/expected :on_success to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: 1/, 469 | fn -> 470 | prepare_for_start_module_opts( 471 | goth: FakeAuth, 472 | subscription: "projects/foo/subscriptions/bar", 473 | on_success: 1 474 | ) 475 | end 476 | 477 | assert_raise ValidationError, 478 | ~r/expected :on_success to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: SomeModule/, 479 | fn -> 480 | prepare_for_start_module_opts( 481 | goth: FakeAuth, 482 | subscription: "projects/foo/subscriptions/bar", 483 | on_success: SomeModule 484 | ) 485 | end 486 | end 487 | 488 | test ":on_failure should be a valid action" do 489 | for action <- [:ack, :noop, {:nack, 0}, {:nack, 100}, {:nack, 600}] do 490 | assert {_, 491 | [ 492 | producer: [ 493 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 494 | concurrency: 1 495 | ], 496 | name: __MODULE__ 497 | ]} = 498 | prepare_for_start_module_opts( 499 | goth: FakeAuth, 500 | subscription: "projects/foo/subscriptions/bar", 501 | on_failure: action 502 | ) 503 | 504 | assert producer_opts[:on_failure] == action 505 | end 506 | 507 | assert_raise ValidationError, 508 | ~r/expected :on_failure to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: :foo/, 509 | fn -> 510 | prepare_for_start_module_opts( 511 | goth: FakeAuth, 512 | subscription: "projects/foo/subscriptions/bar", 513 | on_failure: :foo 514 | ) 515 | end 516 | 517 | assert_raise ValidationError, 518 | ~r/expected :on_failure to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: "foo"/, 519 | fn -> 520 | prepare_for_start_module_opts( 521 | goth: FakeAuth, 522 | subscription: "projects/foo/subscriptions/bar", 523 | on_failure: "foo" 524 | ) 525 | end 526 | 527 | assert_raise ValidationError, 528 | ~r/expected :on_failure to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: 1/, 529 | fn -> 530 | prepare_for_start_module_opts( 531 | goth: FakeAuth, 532 | subscription: "projects/foo/subscriptions/bar", 533 | on_failure: 1 534 | ) 535 | end 536 | 537 | assert_raise ValidationError, 538 | ~r/expected :on_failure to be one of :ack, :noop, :nack, or {:nack, integer} where integer is between 0 and 600, got: SomeModule/, 539 | fn -> 540 | prepare_for_start_module_opts( 541 | goth: FakeAuth, 542 | subscription: "projects/foo/subscriptions/bar", 543 | on_failure: SomeModule 544 | ) 545 | end 546 | end 547 | 548 | test "custom action :nack casts to {:nack, 0}" do 549 | assert {_, 550 | [ 551 | producer: [ 552 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 553 | concurrency: 1 554 | ], 555 | name: __MODULE__ 556 | ]} = 557 | prepare_for_start_module_opts( 558 | goth: FakeAuth, 559 | on_failure: :nack, 560 | on_success: :nack, 561 | subscription: "projects/foo/subscriptions/bar" 562 | ) 563 | 564 | assert producer_opts[:on_success] == {:nack, 0} 565 | assert producer_opts[:on_failure] == {:nack, 0} 566 | end 567 | 568 | test "with :client PullClient returns a child_spec for starting a Finch pool" do 569 | assert { 570 | [ 571 | {Finch, name: BroadwayCloudPubSub.ProducerTest.BroadwayCloudPubSub.PullClient} 572 | ], 573 | [ 574 | producer: [ 575 | module: {BroadwayCloudPubSub.Producer, _producer_opts}, 576 | concurrency: 1 577 | ], 578 | name: __MODULE__ 579 | ] 580 | } = 581 | prepare_for_start_module_opts( 582 | goth: FakeAuth, 583 | subscription: "projects/foo/subscriptions/bar" 584 | ) 585 | end 586 | 587 | test "with :client PullClient and :finch returns empty specs" do 588 | assert { 589 | [], 590 | [ 591 | producer: [ 592 | module: {BroadwayCloudPubSub.Producer, producer_opts}, 593 | concurrency: 1 594 | ], 595 | name: __MODULE__ 596 | ] 597 | } = 598 | prepare_for_start_module_opts( 599 | goth: FakeAuth, 600 | subscription: "projects/foo/subscriptions/bar", 601 | finch: MyFinch 602 | ) 603 | 604 | assert producer_opts[:finch] == MyFinch 605 | end 606 | end 607 | 608 | test "receive messages when the queue has less than the demand" do 609 | {:ok, message_server} = MessageServer.start_link() 610 | {:ok, pid} = start_broadway(message_server) 611 | 612 | MessageServer.push_messages(message_server, 1..5) 613 | 614 | assert_receive {:messages_received, 5} 615 | 616 | for msg <- 1..5 do 617 | assert_receive {:message_handled, ^msg} 618 | end 619 | 620 | stop_broadway(pid) 621 | end 622 | 623 | test "keep receiving messages when the queue has more than the demand" do 624 | {:ok, message_server} = MessageServer.start_link() 625 | MessageServer.push_messages(message_server, 1..20) 626 | {:ok, pid} = start_broadway(message_server) 627 | 628 | assert_receive {:messages_received, 10} 629 | 630 | for msg <- 1..10 do 631 | assert_receive {:message_handled, ^msg} 632 | end 633 | 634 | assert_receive {:messages_received, 5} 635 | 636 | for msg <- 11..15 do 637 | assert_receive {:message_handled, ^msg} 638 | end 639 | 640 | assert_receive {:messages_received, 5} 641 | 642 | for msg <- 16..20 do 643 | assert_receive {:message_handled, ^msg} 644 | end 645 | 646 | stop_broadway(pid) 647 | end 648 | 649 | test "keep trying to receive new messages when the queue is empty" do 650 | {:ok, message_server} = MessageServer.start_link() 651 | {:ok, pid} = start_broadway(message_server) 652 | 653 | MessageServer.push_messages(message_server, [13]) 654 | assert_receive {:messages_received, 1} 655 | assert_receive {:message_handled, 13} 656 | 657 | refute_receive {:message_handled, _} 658 | 659 | MessageServer.push_messages(message_server, [14, 15]) 660 | assert_receive {:messages_received, 2} 661 | assert_receive {:message_handled, 14} 662 | assert_receive {:message_handled, 15} 663 | 664 | stop_broadway(pid) 665 | end 666 | 667 | test "stop trying to receive new messages after start draining" do 668 | {:ok, message_server} = MessageServer.start_link() 669 | broadway_name = new_unique_name() 670 | {:ok, pid} = start_broadway(broadway_name, message_server) 671 | 672 | [producer] = Broadway.producer_names(broadway_name) 673 | 674 | Broadway.Topology.ProducerStage.drain(producer) 675 | :sys.get_state(producer) 676 | 677 | MessageServer.push_messages(message_server, [14, 15]) 678 | 679 | refute_receive {:messages_received, _}, 10 680 | 681 | stop_broadway(pid) 682 | end 683 | 684 | test "delete acknowledged messages" do 685 | {:ok, message_server} = MessageServer.start_link() 686 | {:ok, pid} = start_broadway(message_server) 687 | 688 | MessageServer.push_messages(message_server, 1..20) 689 | 690 | assert_receive {:messages_deleted, 10} 691 | assert_receive {:messages_deleted, 10} 692 | 693 | stop_broadway(pid) 694 | end 695 | 696 | describe "calling Client.prepare_to_connect/2" do 697 | test "with default options" do 698 | {:ok, message_server} = MessageServer.start_link() 699 | broadway_name = new_unique_name() 700 | ref = make_ref() 701 | 702 | {:ok, pid} = 703 | start_broadway(broadway_name, message_server, FakePrepareToConnectClient, 704 | prepare_to_connect_ref: ref 705 | ) 706 | 707 | assert_receive {:prepare_to_connect_spec, ^ref}, 500 708 | assert_receive {:prepare_to_connect_opts, ^ref}, 500 709 | 710 | stop_broadway(pid) 711 | end 712 | end 713 | 714 | test "passing extra options to client" do 715 | {:ok, message_server} = MessageServer.start_link() 716 | broadway_name = new_unique_name() 717 | 718 | {:ok, pid_1} = 719 | start_broadway( 720 | broadway_name, 721 | message_server, 722 | {FakeClient, max_retries: 3, retry_codes: [502, 503], retry_delay_ms: 300} 723 | ) 724 | 725 | stop_broadway(pid_1) 726 | end 727 | 728 | test "support multiple topologies" do 729 | {:ok, message_server} = MessageServer.start_link() 730 | broadway_name = new_unique_name() 731 | {:ok, pid_1} = start_broadway(broadway_name, message_server, FakeClient) 732 | {:ok, message_server} = MessageServer.start_link() 733 | broadway_name = new_unique_name() 734 | {:ok, pid_2} = start_broadway(broadway_name, message_server, FakeClient) 735 | 736 | stop_broadway(pid_1) 737 | stop_broadway(pid_2) 738 | end 739 | 740 | defp start_broadway( 741 | broadway_name \\ new_unique_name(), 742 | message_server, 743 | client \\ FakeClient, 744 | opts \\ [] 745 | ) do 746 | Broadway.start_link( 747 | Forwarder, 748 | build_broadway_opts(broadway_name, opts, 749 | client: client, 750 | goth: DefaultGothProducerOption, 751 | subscription: "projects/my-project/subscriptions/my-subscription", 752 | receive_interval: 0, 753 | test_pid: self(), 754 | message_server: message_server 755 | ) 756 | ) 757 | end 758 | 759 | defp build_broadway_opts(broadway_name, opts, producer_opts) do 760 | [ 761 | name: broadway_name, 762 | context: %{test_pid: self()}, 763 | producer: [ 764 | module: {BroadwayCloudPubSub.Producer, Keyword.merge(producer_opts, opts)}, 765 | concurrency: 1 766 | ], 767 | processors: [ 768 | default: [concurrency: 1] 769 | ], 770 | batchers: [ 771 | default: [ 772 | batch_size: 10, 773 | batch_timeout: 50, 774 | concurrency: 1 775 | ] 776 | ] 777 | ] 778 | end 779 | 780 | defp new_unique_name() do 781 | :"Broadway#{System.unique_integer([:positive, :monotonic])}" 782 | end 783 | 784 | defp stop_broadway(pid) do 785 | ref = Process.monitor(pid) 786 | Process.exit(pid, :normal) 787 | 788 | receive do 789 | {:DOWN, ^ref, _, _, _} -> :ok 790 | end 791 | end 792 | end 793 | -------------------------------------------------------------------------------- /test/broadway_cloud_pub_sub/pull_client_test.exs: -------------------------------------------------------------------------------- 1 | defmodule BroadwayCloudPubSub.PullClientTest do 2 | use ExUnit.Case, async: true 3 | 4 | import ExUnit.CaptureLog 5 | 6 | alias BroadwayCloudPubSub.Acknowledger 7 | alias BroadwayCloudPubSub.PullClient 8 | alias Broadway.Message 9 | 10 | @pull_response """ 11 | { 12 | "receivedMessages": [ 13 | { 14 | "ackId": "1", 15 | "deliveryAttempt": 1, 16 | "message": { 17 | "data": "TWVzc2FnZTE=", 18 | "messageId": "19917247034", 19 | "attributes": { 20 | "foo": "bar", 21 | "qux": "" 22 | }, 23 | "publishTime": "2014-02-14T00:00:01Z" 24 | } 25 | }, 26 | { 27 | "ackId": "2", 28 | "deliveryAttempt": 2, 29 | "message": { 30 | "data": "TWVzc2FnZTI=", 31 | "messageId": "19917247035", 32 | "attributes": {}, 33 | "publishTime": "2014-02-14T00:00:01Z" 34 | } 35 | }, 36 | { 37 | "ackId": "3", 38 | "deliveryAttempt": 3, 39 | "message": { 40 | "data": null, 41 | "messageId": "19917247036", 42 | "attributes": { 43 | "number": "three" 44 | }, 45 | "publishTime": "2014-02-14T00:00:02Z" 46 | } 47 | }, 48 | { 49 | "ackId": "4", 50 | "message": { 51 | "data": null, 52 | "messageId": "19917247037", 53 | "attributes": {}, 54 | "publishTime": null 55 | } 56 | } 57 | ] 58 | } 59 | """ 60 | 61 | @empty_response """ 62 | {} 63 | """ 64 | 65 | @ordered_response """ 66 | { 67 | "receivedMessages": [ 68 | { 69 | "ackId": "1", 70 | "deliveryAttempt": 1, 71 | "message": { 72 | "data": "TWVzc2FnZTE=", 73 | "messageId": "19917247038", 74 | "publishTime": "2014-02-14T00:00:03Z", 75 | "orderingKey": "key1" 76 | } 77 | } 78 | ] 79 | } 80 | """ 81 | 82 | @no_payload_response """ 83 | { 84 | "receivedMessages": [ 85 | { 86 | "ackId": "1", 87 | "message": { 88 | "attributes": { 89 | "payloadFormat": "NONE" 90 | }, 91 | "messageId": "20240501001", 92 | "publishTime": "2024-05-01T13:07:41.716Z" 93 | } 94 | } 95 | ] 96 | } 97 | """ 98 | 99 | setup do 100 | server = Bypass.open() 101 | base_url = "http://localhost:#{server.port}" 102 | 103 | finch = __MODULE__.Finch 104 | _ = start_supervised({Finch, name: finch}) 105 | 106 | {:ok, server: server, base_url: base_url, finch: finch} 107 | end 108 | 109 | def on_pubsub_request(server, fun) when is_function(fun, 2) do 110 | test_pid = self() 111 | 112 | Bypass.expect(server, fn conn -> 113 | url = Plug.Conn.request_url(conn) 114 | {:ok, body, conn} = Plug.Conn.read_body(conn) 115 | body = Jason.decode!(body) 116 | 117 | send(test_pid, {:http_request_called, %{url: url, body: body}}) 118 | 119 | case fun.(url, body) do 120 | {:ok, resp_body} -> Plug.Conn.resp(conn, 200, resp_body) 121 | {:error, resp_body} -> Plug.Conn.resp(conn, 500, resp_body) 122 | {:error, status, resp_body} -> Plug.Conn.resp(conn, status, resp_body) 123 | end 124 | end) 125 | end 126 | 127 | def multiple_errors_on_pubsub(server, error_count: total_errors, error_status: error_status) do 128 | {:ok, agent} = Agent.start_link(fn -> 1 end) 129 | 130 | Bypass.expect(server, fn conn -> 131 | attempt = Agent.get_and_update(agent, fn num -> {num, num + 1} end) 132 | 133 | if attempt <= total_errors do 134 | Plug.Conn.resp(conn, error_status, @empty_response) 135 | else 136 | Agent.stop(agent) 137 | Plug.Conn.resp(conn, 200, @ordered_response) 138 | end 139 | end) 140 | end 141 | 142 | defp init_with_ack_builder(opts) do 143 | # mimics workflow from Producer.prepare_for_start/2 144 | ack_ref = opts[:broadway][:name] 145 | fill_persistent_term(ack_ref, opts) 146 | 147 | {:ok, config} = PullClient.init(opts) 148 | {ack_ref, Acknowledger.builder(ack_ref), config} 149 | end 150 | 151 | describe "receive_messages/3" do 152 | setup %{server: server, base_url: base_url, finch: finch} do 153 | test_pid = self() 154 | 155 | on_pubsub_request(server, fn _url, _body -> 156 | {:ok, @pull_response} 157 | end) 158 | 159 | %{ 160 | pid: test_pid, 161 | opts: [ 162 | # will be injected by Broadway at runtime 163 | broadway: [name: :Broadway3], 164 | base_url: base_url, 165 | finch: finch, 166 | max_number_of_messages: 10, 167 | subscription: "projects/foo/subscriptions/bar", 168 | token_generator: {__MODULE__, :generate_token, []}, 169 | receive_timeout: :infinity, 170 | max_retries: 0 171 | ] 172 | } 173 | end 174 | 175 | test "returns a list of ordered Broadway.Message with orderingKey in the :metadata", %{ 176 | opts: base_opts, 177 | server: server 178 | } do 179 | on_pubsub_request(server, fn _url, _body -> 180 | {:ok, @ordered_response} 181 | end) 182 | 183 | {:ok, opts} = PullClient.init(base_opts) 184 | 185 | assert [message] = PullClient.receive_messages(10, & &1, opts) 186 | 187 | assert message.metadata.messageId == "19917247038" 188 | assert message.metadata.orderingKey == "key1" 189 | end 190 | 191 | test "retries if the option is set", %{ 192 | opts: base_opts, 193 | server: server 194 | } do 195 | multiple_errors_on_pubsub(server, error_count: 3, error_status: 502) 196 | 197 | {:ok, opts} = 198 | base_opts 199 | |> Keyword.put(:max_retries, 3) 200 | |> Keyword.put(:retry_delay_ms, 0) 201 | |> Keyword.put(:retry_codes, [502]) 202 | |> PullClient.init() 203 | 204 | assert [_message] = PullClient.receive_messages(10, & &1, opts) 205 | end 206 | 207 | test "returns a list of Broadway.Message when payloadFormat is NONE", %{ 208 | opts: base_opts, 209 | server: server 210 | } do 211 | on_pubsub_request(server, fn _, _ -> 212 | {:ok, @no_payload_response} 213 | end) 214 | 215 | {:ok, opts} = PullClient.init(base_opts) 216 | 217 | assert [message] = PullClient.receive_messages(10, & &1, opts) 218 | assert message.metadata.messageId == "20240501001" 219 | end 220 | 221 | test "returns a list of Broadway.Message with :data and :metadata set", %{ 222 | opts: base_opts 223 | } do 224 | {:ok, opts} = PullClient.init(base_opts) 225 | 226 | [message1, message2, message3, message4] = PullClient.receive_messages(10, & &1, opts) 227 | 228 | assert %Message{data: "Message1", metadata: %{publishTime: %DateTime{}}} = message1 229 | 230 | assert message1.metadata.messageId == "19917247034" 231 | assert message1.metadata.deliveryAttempt == 1 232 | 233 | assert %{ 234 | "foo" => "bar", 235 | "qux" => "" 236 | } = message1.metadata.attributes 237 | 238 | assert message2.data == "Message2" 239 | assert message2.metadata.messageId == "19917247035" 240 | assert message2.metadata.attributes == %{} 241 | assert message2.metadata.deliveryAttempt == 2 242 | 243 | assert %Message{data: nil} = message3 244 | assert message3.metadata.deliveryAttempt == 3 245 | 246 | assert %{ 247 | "number" => "three" 248 | } = message3.metadata.attributes 249 | 250 | assert message4.metadata.publishTime == nil 251 | assert message4.metadata.deliveryAttempt == nil 252 | end 253 | 254 | test "returns an empty list when an empty response is returned by the server", %{ 255 | opts: base_opts, 256 | server: server 257 | } do 258 | on_pubsub_request(server, fn _, _ -> 259 | {:ok, @empty_response} 260 | end) 261 | 262 | {:ok, opts} = PullClient.init(base_opts) 263 | 264 | assert [] == PullClient.receive_messages(10, & &1, opts) 265 | end 266 | 267 | test "if the request fails, returns an empty list and log the error", %{ 268 | opts: base_opts, 269 | server: server 270 | } do 271 | on_pubsub_request(server, fn _, _ -> {:error, 403, @empty_response} end) 272 | 273 | {:ok, opts} = PullClient.init(base_opts) 274 | 275 | assert capture_log(fn -> 276 | assert PullClient.receive_messages(10, & &1, opts) == [] 277 | end) =~ "[error] Unable to fetch events from Cloud Pub/Sub - reason: " 278 | end 279 | 280 | test "send a projects.subscriptions.pull request with default options", %{opts: base_opts} do 281 | {:ok, opts} = PullClient.init(base_opts) 282 | PullClient.receive_messages(10, & &1, opts) 283 | 284 | assert_received {:http_request_called, %{body: body, url: url}} 285 | assert body == %{"maxMessages" => 10} 286 | assert url == base_opts[:base_url] <> "/v1/projects/foo/subscriptions/bar:pull" 287 | end 288 | 289 | test "request with custom :max_number_of_messages", %{opts: base_opts} do 290 | {:ok, opts} = base_opts |> Keyword.put(:max_number_of_messages, 5) |> PullClient.init() 291 | PullClient.receive_messages(10, & &1, opts) 292 | 293 | assert_received {:http_request_called, %{body: body, url: _url}} 294 | assert body["maxMessages"] == 5 295 | end 296 | 297 | test "exposes telemetry for pull requests", %{opts: base_opts} do 298 | :telemetry.attach( 299 | :start_handler, 300 | [:broadway_cloud_pub_sub, :pull_client, :receive_messages, :start], 301 | fn _name, _measurements, metadata, _config -> 302 | send(self(), {:start, metadata}) 303 | end, 304 | %{} 305 | ) 306 | 307 | :telemetry.attach( 308 | :stop_handler, 309 | [:broadway_cloud_pub_sub, :pull_client, :receive_messages, :stop], 310 | fn _name, measurements, _metadata, _config -> 311 | send(self(), {:stop, measurements}) 312 | end, 313 | %{} 314 | ) 315 | 316 | {:ok, opts} = base_opts |> Keyword.put(:max_number_of_messages, 5) |> PullClient.init() 317 | PullClient.receive_messages(10, & &1, opts) 318 | 319 | assert_received {:start, metadata} 320 | assert_received {:stop, measurements} 321 | assert metadata.demand == 10 322 | assert metadata.max_messages == 5 323 | assert is_integer(measurements.duration) 324 | 325 | :telemetry.detach(:start_handler) 326 | :telemetry.detach(:stop_handler) 327 | end 328 | end 329 | 330 | describe "acknowledge/2" do 331 | setup %{server: server, base_url: base_url, finch: finch} do 332 | test_pid = self() 333 | 334 | on_pubsub_request(server, fn _, _ -> 335 | {:ok, @empty_response} 336 | end) 337 | 338 | %{ 339 | pid: test_pid, 340 | opts: [ 341 | # will be injected by Broadway at runtime 342 | broadway: [name: :Broadway3], 343 | base_url: base_url, 344 | finch: finch, 345 | max_number_of_messages: 10, 346 | subscription: "projects/foo/subscriptions/bar", 347 | token_generator: {__MODULE__, :generate_token, []}, 348 | receive_timeout: :infinity, 349 | topology_name: Broadway3, 350 | max_retries: 0 351 | ] 352 | } 353 | end 354 | 355 | test "makes a projects.subscriptions.acknowledge request", %{opts: base_opts} do 356 | {:ok, opts} = PullClient.init(base_opts) 357 | 358 | PullClient.acknowledge(["1", "2", "3"], opts) 359 | 360 | assert_received {:http_request_called, %{body: body, url: url}} 361 | 362 | assert body == %{"ackIds" => ["1", "2", "3"]} 363 | base_url = base_opts[:base_url] 364 | assert url == base_url <> "/v1/projects/foo/subscriptions/bar:acknowledge" 365 | end 366 | 367 | test "if the request fails, returns :ok and logs an error", %{ 368 | opts: base_opts, 369 | server: server 370 | } do 371 | on_pubsub_request(server, fn _, _ -> 372 | {:error, 503, @empty_response} 373 | end) 374 | 375 | {:ok, opts} = PullClient.init(base_opts) 376 | 377 | assert capture_log(fn -> 378 | assert PullClient.acknowledge(["1", "2"], opts) == :ok 379 | end) =~ "[error] Unable to acknowledge messages with Cloud Pub/Sub - reason: " 380 | end 381 | 382 | test "emits telemetry events", %{opts: base_opts} do 383 | :telemetry.attach( 384 | :start_handler, 385 | [:broadway_cloud_pub_sub, :pull_client, :ack, :start], 386 | fn _name, _measurements, metadata, _config -> 387 | send(self(), {:start, metadata}) 388 | end, 389 | %{} 390 | ) 391 | 392 | :telemetry.attach( 393 | :stop_handler, 394 | [:broadway_cloud_pub_sub, :pull_client, :ack, :stop], 395 | fn _name, measurements, metadata, _config -> 396 | send(self(), {:stop, measurements, metadata}) 397 | end, 398 | %{} 399 | ) 400 | 401 | {:ok, opts} = PullClient.init(base_opts) 402 | 403 | PullClient.acknowledge(["1", "2", "3"], opts) 404 | 405 | assert_received {:start, metadata} 406 | assert metadata.name == Broadway3 407 | assert_received {:stop, measurements, metadata} 408 | assert is_integer(measurements.duration) 409 | assert metadata.name == Broadway3 410 | 411 | :telemetry.detach(:start_handler) 412 | :telemetry.detach(:stop_handler) 413 | end 414 | end 415 | 416 | describe "put_deadline/3" do 417 | setup %{server: server, base_url: base_url, finch: finch} do 418 | test_pid = self() 419 | 420 | on_pubsub_request(server, fn _, _ -> 421 | {:ok, @empty_response} 422 | end) 423 | 424 | %{ 425 | pid: test_pid, 426 | opts: [ 427 | # will be injected by Broadway at runtime 428 | broadway: [name: :Broadway3], 429 | base_url: base_url, 430 | finch: finch, 431 | max_number_of_messages: 10, 432 | subscription: "projects/foo/subscriptions/bar", 433 | token_generator: {__MODULE__, :generate_token, []}, 434 | receive_timeout: :infinity, 435 | topology_name: Broadway3, 436 | max_retries: 0 437 | ] 438 | } 439 | end 440 | 441 | test "makes a projects.subscriptions.modifyAckDeadline request", %{ 442 | opts: base_opts 443 | } do 444 | {:ok, opts} = PullClient.init(base_opts) 445 | 446 | ack_ids = ["1", "2"] 447 | PullClient.put_deadline(ack_ids, 30, opts) 448 | 449 | assert_received {:http_request_called, %{body: body, url: url}} 450 | assert body == %{"ackIds" => ack_ids, "ackDeadlineSeconds" => 30} 451 | 452 | assert url == base_opts[:base_url] <> "/v1/projects/foo/subscriptions/bar:modifyAckDeadline" 453 | end 454 | 455 | test "if the request fails, returns :ok and logs an error", 456 | %{opts: base_opts, server: server} do 457 | on_pubsub_request(server, fn _, _ -> 458 | {:error, 503, @empty_response} 459 | end) 460 | 461 | {:ok, opts} = PullClient.init(base_opts) 462 | 463 | assert capture_log(fn -> 464 | assert PullClient.put_deadline(["1", "2"], 60, opts) == :ok 465 | end) =~ "[error] Unable to put new ack deadline with Cloud Pub/Sub - reason: " 466 | end 467 | end 468 | 469 | describe "prepare_to_connect/2" do 470 | test "returns a child_spec for starting a Finch http pool " do 471 | {[pool_spec], opts} = PullClient.prepare_to_connect(SomePipeline, []) 472 | assert pool_spec == {Finch, name: SomePipeline.BroadwayCloudPubSub.PullClient} 473 | assert opts == [finch: SomePipeline.BroadwayCloudPubSub.PullClient] 474 | end 475 | 476 | test "allows custom finch" do 477 | {specs, opts} = PullClient.prepare_to_connect(SomePipeline, finch: Foo) 478 | 479 | assert specs == [] 480 | assert opts == [finch: Foo] 481 | end 482 | end 483 | 484 | describe "integration with BroadwayCloudPubSub.Acknowledger" do 485 | setup %{server: server, base_url: base_url, finch: finch} do 486 | test_pid = self() 487 | 488 | on_pubsub_request(server, fn url, body -> 489 | action = 490 | url 491 | |> String.split(":") 492 | |> List.last() 493 | 494 | case action do 495 | "pull" -> 496 | send(test_pid, {:pull_dispatched, %{url: url, body: body}}) 497 | {:ok, @pull_response} 498 | 499 | "acknowledge" -> 500 | %{"ackIds" => ack_ids} = body 501 | 502 | send(test_pid, {:acknowledge_dispatched, length(ack_ids), ack_ids}) 503 | {:ok, @empty_response} 504 | 505 | "modifyAckDeadline" -> 506 | %{"ackIds" => ack_ids, "ackDeadlineSeconds" => deadline} = body 507 | 508 | send( 509 | test_pid, 510 | {:modack_dispatched, length(ack_ids), deadline} 511 | ) 512 | 513 | {:ok, @empty_response} 514 | end 515 | end) 516 | 517 | {:ok, 518 | %{ 519 | pid: test_pid, 520 | opts: [ 521 | # will be injected by Broadway at runtime 522 | broadway: [name: :Broadway3], 523 | base_url: base_url, 524 | client: PullClient, 525 | finch: finch, 526 | max_number_of_messages: 10, 527 | subscription: "projects/foo/subscriptions/bar", 528 | token_generator: {__MODULE__, :generate_token, []}, 529 | receive_timeout: :infinity, 530 | toplogy_name: Broadway3 531 | ] 532 | }} 533 | end 534 | 535 | test "returns a list of Broadway.Message structs with ack builder", %{ 536 | opts: base_opts 537 | } do 538 | {:ok, opts} = PullClient.init(base_opts) 539 | 540 | [message1, message2, message3, message4] = 541 | PullClient.receive_messages(10, &{:ack, &1}, opts) 542 | 543 | assert {:ack, _} = message1.acknowledger 544 | assert {:ack, _} = message2.acknowledger 545 | assert {:ack, _} = message3.acknowledger 546 | assert {:ack, _} = message4.acknowledger 547 | end 548 | 549 | test "with defaults successful messages are acknowledged, and failed messages are ignored", %{ 550 | opts: base_opts 551 | } do 552 | {ack_ref, builder, opts} = init_with_ack_builder(base_opts) 553 | 554 | messages = PullClient.receive_messages(10, builder, opts) 555 | 556 | {successful, failed} = Enum.split(messages, 1) 557 | 558 | Acknowledger.ack(ack_ref, successful, failed) 559 | 560 | assert_receive {:acknowledge_dispatched, 1, ["1"]} 561 | end 562 | 563 | test "when :on_success is :noop, acknowledgement is a no-op", %{ 564 | opts: base_opts 565 | } do 566 | {ack_ref, builder, opts} = 567 | base_opts 568 | |> Keyword.put(:on_success, :noop) 569 | |> init_with_ack_builder() 570 | 571 | [_, _, _, _] = messages = PullClient.receive_messages(10, builder, opts) 572 | 573 | Acknowledger.ack(ack_ref, messages, []) 574 | 575 | refute_receive {:acknowledge_dispatched, 4, _} 576 | end 577 | 578 | test "when :on_success is :nack, dispatches modifyAckDeadline", %{ 579 | opts: base_opts 580 | } do 581 | {ack_ref, builder, opts} = 582 | base_opts 583 | |> Keyword.put(:on_success, :nack) 584 | |> init_with_ack_builder() 585 | 586 | [_, _, _, _] = messages = PullClient.receive_messages(10, builder, opts) 587 | 588 | Acknowledger.ack(ack_ref, messages, []) 589 | 590 | assert_receive {:modack_dispatched, 4, 0} 591 | refute_receive {:acknowledge_dispatched, 4, _} 592 | end 593 | 594 | test "when :on_success is {:nack, integer}, dispatches modifyAckDeadline", %{ 595 | opts: base_opts 596 | } do 597 | {ack_ref, builder, opts} = 598 | base_opts 599 | |> Keyword.put(:on_success, {:nack, 300}) 600 | |> init_with_ack_builder() 601 | 602 | [_, _, _, _] = messages = PullClient.receive_messages(10, builder, opts) 603 | 604 | Acknowledger.ack(ack_ref, messages, []) 605 | 606 | assert_receive {:modack_dispatched, 4, 300} 607 | refute_receive {:acknowledge_dispatched, 4, _} 608 | end 609 | 610 | test "with default :on_failure, failed messages are ignored", %{opts: base_opts} do 611 | {ack_ref, builder, opts} = init_with_ack_builder(base_opts) 612 | 613 | [_, _, _, _] = messages = PullClient.receive_messages(10, builder, opts) 614 | 615 | Acknowledger.ack(ack_ref, [], messages) 616 | 617 | refute_receive {:acknowledge_dispatched, 4, _} 618 | end 619 | 620 | test "when :on_failure is :nack, dispatches modifyAckDeadline", %{ 621 | opts: base_opts 622 | } do 623 | {ack_ref, builder, opts} = 624 | base_opts 625 | |> Keyword.put(:on_failure, :nack) 626 | |> init_with_ack_builder() 627 | 628 | [_, _, _, _] = messages = PullClient.receive_messages(10, builder, opts) 629 | 630 | Acknowledger.ack(ack_ref, [], messages) 631 | 632 | assert_receive {:modack_dispatched, 4, 0} 633 | end 634 | 635 | test "when :on_failure is {:nack, integer}, dispatches modifyAckDeadline", %{ 636 | opts: base_opts 637 | } do 638 | {ack_ref, builder, opts} = 639 | base_opts 640 | |> Keyword.put(:on_failure, {:nack, 60}) 641 | |> init_with_ack_builder() 642 | 643 | [_, _, _, _] = messages = PullClient.receive_messages(10, builder, opts) 644 | 645 | Acknowledger.ack(ack_ref, [], messages) 646 | 647 | assert_receive {:modack_dispatched, 4, 60} 648 | end 649 | end 650 | 651 | def generate_token, do: {:ok, "token.#{System.os_time(:second)}"} 652 | 653 | defp fill_persistent_term(ack_ref, base_opts) do 654 | :persistent_term.put(ack_ref, %{ 655 | base_url: Keyword.fetch!(base_opts, :base_url), 656 | client: PullClient, 657 | finch: Keyword.fetch!(base_opts, :finch), 658 | on_failure: base_opts[:on_failure] || :noop, 659 | on_success: base_opts[:on_success] || :ack, 660 | subscription: "projects/test/subscriptions/test-subscription", 661 | token_generator: {__MODULE__, :generate_token, []}, 662 | receive_timeout: base_opts[:receive_timeout] || :infinity, 663 | topology_name: Broadway3 664 | }) 665 | end 666 | end 667 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | --------------------------------------------------------------------------------