├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── config.yml │ └── feature_request.md ├── pull_request_template.md └── workflows │ ├── master.yml │ ├── pr-examples.yml │ ├── pr.yml │ └── tests.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── Makefile ├── README.md ├── RELEASE-PROCEDURE.md ├── UPGRADE-0.3.md ├── UPGRADE-0.4.md ├── UPGRADE-1.0.md ├── _examples ├── basic │ ├── 1-your-first-app │ │ ├── .validate_example.yml │ │ ├── README.md │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── 2-realtime-feed │ │ ├── .validate_example_subscribing.yml │ │ ├── README.md │ │ ├── consumer │ │ │ ├── go.mod │ │ │ ├── go.sum │ │ │ └── main.go │ │ ├── docker-compose.yml │ │ └── producer │ │ │ ├── go.mod │ │ │ ├── go.sum │ │ │ └── main.go │ ├── 3-router │ │ ├── .validate_example.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── 4-metrics │ │ ├── .validate_example.yml │ │ ├── README.md │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ ├── main.go │ │ └── prometheus.yml │ ├── 5-cqrs-protobuf │ │ ├── .validate_example.yml │ │ ├── Makefile │ │ ├── README.md │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ ├── main.go │ │ ├── messages.pb.go │ │ └── proto │ │ │ └── messages.proto │ └── 6-cqrs-ordered-events │ │ ├── .validate_example.yml │ │ ├── Makefile │ │ ├── README.md │ │ ├── activity.go │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ ├── main.go │ │ ├── message.go │ │ ├── messages.pb.go │ │ ├── proto │ │ └── messages.proto │ │ └── subscribers.go ├── pubsubs │ ├── amqp │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── aws-sns │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── aws-sqs │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── go-channel │ │ ├── .validate_example.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── googlecloud │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── kafka │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── nats-core │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── nats-jetstream │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── nats-streaming │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── redisstream │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ └── sql │ │ ├── .validate_example.yml │ │ ├── docker-compose.yml │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go └── real-world-examples │ ├── consumer-groups │ ├── README.md │ ├── api │ │ ├── http.go │ │ ├── main.go │ │ ├── public │ │ │ └── index.html │ │ └── storage.go │ ├── common │ │ ├── events.go │ │ └── messaging.go │ ├── crm-service │ │ └── main.go │ ├── docker-compose.yml │ ├── docs │ │ ├── screen1.png │ │ └── screen2.png │ ├── go.mod │ ├── go.sum │ └── newsletter-service │ │ └── main.go │ ├── delayed-messages │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ └── main.go │ ├── delayed-requeue │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ └── main.go │ ├── exactly-once-delivery-counter │ ├── README.md │ ├── architecture.jpg │ ├── at-least-once-delivery.jpg │ ├── docker-compose.yml │ ├── run.go │ ├── schema.sql │ ├── server │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ └── worker │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── persistent-event-log │ ├── .validate_example.yml │ ├── README.md │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ └── main.go │ ├── receiving-webhooks │ ├── .validate_example.yml │ ├── README.md │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ └── main.go │ ├── sending-webhooks │ ├── .validate_example.yml │ ├── README.md │ ├── docker-compose.yml │ ├── producer │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ ├── router │ │ ├── go.mod │ │ ├── go.sum │ │ └── main.go │ └── webhooks-server │ │ └── main.go │ ├── server-sent-events-htmx │ ├── Dockerfile │ ├── README.md │ ├── docker-compose.yml │ ├── docker │ │ ├── Dockerfile │ │ └── reflex.conf │ ├── events.go │ ├── go.mod │ ├── go.sum │ ├── http.go │ ├── main.go │ ├── models.go │ ├── repository.go │ └── views │ │ ├── base.templ │ │ ├── base_templ.go │ │ ├── pages.templ │ │ └── pages_templ.go │ ├── server-sent-events │ ├── README.md │ ├── diagram.jpg │ ├── docker-compose.yml │ ├── schema.sql │ ├── screen.gif │ ├── screenshot.png │ └── server │ │ ├── diagram.jpg │ │ ├── event_handlers.go │ │ ├── feeds_storage.go │ │ ├── go.mod │ │ ├── go.sum │ │ ├── http.go │ │ ├── main.go │ │ ├── models.go │ │ ├── posts_storage.go │ │ └── public │ │ └── index.html │ ├── synchronizing-databases │ ├── .validate_example.yml │ ├── README.md │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ ├── main.go │ ├── mysql.go │ └── postgres.go │ ├── transactional-events-forwarder │ ├── .validate_example.yml │ ├── README.md │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ └── main.go │ └── transactional-events │ ├── .validate_example.yml │ ├── README.md │ ├── docker-compose.yml │ ├── go.mod │ ├── go.sum │ └── main.go ├── codecov.yml ├── components ├── cqrs │ ├── command_bus.go │ ├── command_bus_test.go │ ├── command_handler.go │ ├── command_handler_test.go │ ├── command_processor.go │ ├── command_processor_test.go │ ├── cqrs.go │ ├── cqrs_test.go │ ├── ctx.go │ ├── doc.go │ ├── event_bus.go │ ├── event_bus_test.go │ ├── event_handler.go │ ├── event_handler_test.go │ ├── event_processor.go │ ├── event_processor_group.go │ ├── event_processor_group_test.go │ ├── event_processor_test.go │ ├── marshaler.go │ ├── marshaler_json.go │ ├── marshaler_json_test.go │ ├── marshaler_protobuf.go │ ├── marshaler_protobuf_events_new_test.go │ ├── marshaler_protobuf_events_test.go │ ├── marshaler_protobuf_gogo.go │ ├── marshaler_protobuf_gogo_test.go │ ├── marshaler_protobuf_test.go │ ├── name.go │ ├── name_test.go │ ├── object.go │ └── testdata │ │ └── events.proto ├── delay │ ├── delay.go │ ├── publisher.go │ └── publisher_test.go ├── fanin │ ├── fanin.go │ └── fanin_test.go ├── forwarder │ ├── envelope.go │ ├── envelope_test.go │ ├── forwarder.go │ ├── forwarder_test.go │ └── publisher.go ├── metrics │ ├── builder.go │ ├── ctx.go │ ├── handler.go │ ├── http.go │ ├── http_test.go │ ├── labels.go │ ├── publisher.go │ └── subscriber.go ├── requestreply │ ├── backend_pubsub.go │ ├── backend_pubsub_marshaler.go │ ├── command_bus.go │ ├── handler.go │ ├── requestreply.go │ └── requestreply_test.go └── requeuer │ ├── requeuer.go │ └── requeuer_test.go ├── dev ├── consolidate-gomods │ └── main.go ├── coverage.sh ├── prometheus.yml ├── update-examples-deps │ ├── go.mod │ ├── go.sum │ └── main.go └── validate-examples │ ├── go.mod │ ├── go.sum │ └── main.go ├── doc.go ├── docs ├── .hugo_build.lock ├── .npmignore ├── .npmrc ├── .prettierignore ├── .prettierrc.yaml ├── DEVELOP.md ├── assets │ ├── favicon.ico │ ├── favicon.png │ ├── images │ │ └── .gitkeep │ ├── js │ │ └── custom.js │ ├── jsconfig.json │ ├── mask-icon.svg │ ├── scss │ │ └── common │ │ │ ├── _custom.scss │ │ │ └── _variables-custom.scss │ └── svgs │ │ └── .gitkeep ├── build.sh ├── config │ ├── _default │ │ ├── hugo.toml │ │ ├── languages.toml │ │ ├── markup.toml │ │ ├── menus │ │ │ └── menus.en.toml │ │ ├── module.toml │ │ └── params.toml │ ├── babel.config.js │ ├── next │ │ └── hugo.toml │ ├── postcss.config.js │ └── production │ │ └── hugo.toml ├── content │ ├── _index.md │ ├── advanced │ │ ├── delayed-messages.md │ │ ├── fanin.md │ │ ├── fanout.md │ │ ├── forwarder.md │ │ ├── metrics.md │ │ └── requeuing-after-error.md │ ├── development │ │ ├── benchmark.md │ │ ├── contributing.md │ │ ├── pub-sub-implementing.md │ │ └── releases.md │ ├── docs │ │ ├── _index.md │ │ ├── articles.md │ │ ├── awesome.md │ │ ├── cqrs.md │ │ ├── getting-started.md │ │ ├── message.md │ │ ├── message │ │ │ ├── .validate_example.yml │ │ │ ├── go.mod │ │ │ ├── go.sum │ │ │ └── receiving-ack.go │ │ ├── messages-router.md │ │ ├── middlewares.md │ │ ├── pub-sub.md │ │ ├── snippets │ │ │ ├── amqp-consumer-groups │ │ │ │ ├── .validate_example.yml │ │ │ │ ├── docker-compose.yml │ │ │ │ ├── go.mod │ │ │ │ ├── go.sum │ │ │ │ └── main.go │ │ │ └── tail-log-file │ │ │ │ ├── go.mod │ │ │ │ ├── go.sum │ │ │ │ └── main.go │ │ └── troubleshooting.md │ ├── pubsubs │ │ ├── _index.md │ │ ├── amqp.md │ │ ├── aws.md │ │ ├── bolt.md │ │ ├── firestore.md │ │ ├── gochannel.md │ │ ├── googlecloud.md │ │ ├── http.md │ │ ├── io.md │ │ ├── kafka.md │ │ ├── nats.md │ │ ├── redisstream.md │ │ └── sql.md │ └── support.md ├── extract_middleware_godocs.py ├── layouts │ ├── _default │ │ └── _markup │ │ │ └── render-link.html │ ├── index.html │ ├── partials │ │ ├── footer │ │ │ ├── footer.html │ │ │ └── script-footer-custom.html │ │ ├── head │ │ │ ├── custom-head.html │ │ │ ├── resource-hints.html │ │ │ └── script-header.html │ │ ├── header │ │ │ └── header.html │ │ ├── main │ │ │ └── edit-page.html │ │ ├── private │ │ │ └── has-headings.html │ │ └── seo │ │ │ ├── opengraph.html │ │ │ └── twitter.html │ └── shortcodes │ │ ├── load-snippet-partial.html │ │ ├── load-snippet.html │ │ ├── readfile.html │ │ ├── tab.html │ │ └── tabs.html ├── package-lock.json ├── package.json ├── resources │ └── _gen │ │ ├── assets │ │ └── scss │ │ │ ├── app.scss_901a6e181e810c5c7347a10d84f037ab.content │ │ │ ├── app.scss_901a6e181e810c5c7347a10d84f037ab.json │ │ │ ├── app.scss_cdf9d7c9eb97e4550ded64a8776dd9e8.content │ │ │ └── app.scss_cdf9d7c9eb97e4550ded64a8776dd9e8.json │ │ └── images │ │ ├── _hu70523d59fb738bec5c06336403a46531_39940_c0fe83760c1bebd5a39d4ddb7fce622e.webp │ │ ├── _hu89002a090cbbd5e6897ae6b591dddabc_33739_cb9243f2e37f830fb14160ae4284ce39.webp │ │ ├── cqrs-example-storming_7615831582150998571_huaaacfa63d5a84cf464fbe57abe466f11_96906_1549x914_resize_q85_h2_lanczos_3.webp │ │ ├── favicon_hufb12268f494215628cd81cc4fa356d3c_100807_180x180_resize_lanczos_3.png │ │ ├── favicon_hufb12268f494215628cd81cc4fa356d3c_100807_192x192_resize_lanczos_3.png │ │ ├── favicon_hufb12268f494215628cd81cc4fa356d3c_100807_32x32_resize_lanczos_3.png │ │ ├── favicon_hufb12268f494215628cd81cc4fa356d3c_100807_512x512_resize_lanczos_3.png │ │ └── grafana_import_dashboard_6707854648249907356_huaaa3b6f44cdba346c1f24c07e8af91ec_92673_1024x786_resize_q85_h2_lanczos_3.webp └── static │ ├── .gitkeep │ ├── fonts │ └── quicksand │ │ ├── quicksand-v31-latin-500.woff2 │ │ ├── quicksand-v31-latin-700.woff2 │ │ └── quicksand-v31-latin-regular.woff2 │ └── img │ ├── forwarder-envelope.svg │ ├── gopher.svg │ ├── publishing-with-forwarder.svg │ ├── pyramid.png │ └── watermill-router.svg ├── go.mod ├── go.sum ├── internal ├── channel.go ├── channel_test.go ├── name.go ├── name_test.go ├── norace.go ├── publisher │ ├── errors.go │ ├── retry.go │ └── retry_test.go ├── race.go └── subscriber │ └── multiplier.go ├── log.go ├── log_test.go ├── message ├── decorator.go ├── decorator_bench_test.go ├── decorator_test.go ├── message.go ├── message_test.go ├── messages.go ├── messages_test.go ├── metadata.go ├── pubsub.go ├── router.go ├── router │ ├── middleware │ │ ├── circuit_breaker.go │ │ ├── circuit_breaker_test.go │ │ ├── correlation.go │ │ ├── correlation_test.go │ │ ├── deduplicator.go │ │ ├── deduplicator_test.go │ │ ├── delay_on_error.go │ │ ├── delay_on_error_test.go │ │ ├── duplicator.go │ │ ├── duplicator_test.go │ │ ├── ignore_errors.go │ │ ├── ignore_errors_test.go │ │ ├── instant_ack.go │ │ ├── instant_ack_test.go │ │ ├── message_test.go │ │ ├── poison.go │ │ ├── poison_test.go │ │ ├── randomfail.go │ │ ├── randomfail_test.go │ │ ├── recoverer.go │ │ ├── recoverer_test.go │ │ ├── retry.go │ │ ├── retry_test.go │ │ ├── throttle.go │ │ ├── throttle_test.go │ │ ├── timeout.go │ │ └── timeout_test.go │ └── plugin │ │ └── signals.go ├── router_context.go ├── router_context_test.go ├── router_test.go └── subscriber │ ├── read.go │ └── read_test.go ├── netlify.toml ├── pubsub ├── doc.go ├── gochannel │ ├── doc.go │ ├── fanout.go │ ├── fanout_test.go │ ├── pubsub.go │ ├── pubsub_bench_test.go │ ├── pubsub_stress_test.go │ └── pubsub_test.go ├── sync │ ├── waitgroup.go │ └── waitgroup_test.go └── tests │ ├── bench_pubsub.go │ ├── test_asserts.go │ ├── test_pubsub.go │ └── test_pubsub_stress.go ├── slog.go ├── slog_test.go ├── tools ├── mill │ ├── .default-config.yml │ ├── Makefile │ ├── README.md │ ├── cmd │ │ ├── amqp.go │ │ ├── consume.go │ │ ├── googlecloud.go │ │ ├── internal │ │ │ └── indent.go │ │ ├── kafka.go │ │ ├── produce.go │ │ └── root.go │ ├── go.mod │ ├── go.sum │ └── main.go └── pq │ ├── README.md │ ├── backend │ └── postgres.go │ ├── cli │ ├── backend.go │ ├── message.go │ └── model.go │ ├── go.mod │ ├── go.sum │ └── main.go ├── uuid.go └── uuid_test.go /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | 15 | 16 | ### Steps to reproduce 17 | 18 | 23 | 24 | docker-compose.yml 25 | ```yaml 26 | 27 | ``` 28 | 29 | ```go 30 | // Your reproduction code goes here 31 | ``` 32 | 33 | ### Expected behavior 34 | 35 | 36 | 37 | ### Actual behavior 38 | 39 | 40 | 41 | ### Possible solution 42 | 43 | 44 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: true 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Feature request 11 | 12 | 17 | 18 | ### Description 19 | 20 | 21 | 22 | ### Example use case 23 | 24 | 28 | 29 | #### How it can look like in code 30 | 31 | ```go 32 | 33 | ``` 34 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 10 | 11 | ### Motivation / Background 12 | 13 | 22 | 23 | ### Detail 24 | 25 | 26 | ### Alternative approaches considered (if applicable) 27 | 28 | 29 | 30 | ### Checklist 31 | 32 | The resources of our team are limited. **There are a couple of things that you can do to help us merge your PR faster**: 33 | 34 | - [ ] I wrote tests for the changes. 35 | - [ ] All tests are passing. 36 | - If you are testing a Pub/Sub, you can start Docker with `make up`. 37 | - You can start with `make test_short` for a quick check. 38 | - If you want to run all tests, use `make test`. 39 | - [ ] Code has no breaking changes. 40 | - [ ] _(If applicable)_ documentation on [watermill.io](https://watermill.io/) is updated. 41 | - Documentation is built in the [github.com/ThreeDotsLabs/watermill/docs](https://github.com/ThreeDotsLabs/watermill/tree/master/docs). 42 | - You can find development instructions in the [DEVELOP.md](https://github.com/ThreeDotsLabs/watermill/tree/master/docs/DEVELOP.md). -------------------------------------------------------------------------------- /.github/workflows/master.yml: -------------------------------------------------------------------------------- 1 | name: master 2 | on: 3 | push: 4 | branches: 5 | - master 6 | jobs: 7 | ci: 8 | uses: ThreeDotsLabs/watermill/.github/workflows/tests.yml@master 9 | with: 10 | stress-tests: true 11 | codecov: true 12 | secrets: 13 | codecov_token: ${{ secrets.CODECOV_TOKEN }} 14 | -------------------------------------------------------------------------------- /.github/workflows/pr-examples.yml: -------------------------------------------------------------------------------- 1 | name: pr-examples 2 | on: 3 | pull_request: 4 | paths: 5 | - '_examples/**/*' 6 | 7 | jobs: 8 | validate-examples: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | - uses: actions/setup-go@v4 13 | with: 14 | go-version: '^1.21.1' 15 | - run: make validate_examples 16 | timeout-minutes: 30 -------------------------------------------------------------------------------- /.github/workflows/pr.yml: -------------------------------------------------------------------------------- 1 | name: pr 2 | on: 3 | pull_request: 4 | jobs: 5 | ci: 6 | uses: ThreeDotsLabs/watermill/.github/workflows/tests.yml@master 7 | with: 8 | codecov: true 9 | secrets: 10 | codecov_token: ${{ secrets.CODECOV_TOKEN }} 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | vendor 3 | docs/themes/ 4 | docs/node_modules/ 5 | docs/public/ 6 | docs/content/src-link 7 | docs/content/middleware 8 | docs/hugo_stats.json 9 | *.out 10 | *.log 11 | .mod-cache 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Three Dots Labs 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | up: 2 | 3 | test: 4 | go test ./... 5 | 6 | test_v: 7 | go test -v ./... 8 | 9 | test_short: 10 | go test ./... -short 11 | 12 | test_race: 13 | go test ./... -short -race 14 | 15 | test_stress: 16 | go test -tags=stress -timeout=30m ./... 17 | 18 | test_codecov: 19 | go test -coverprofile=coverage.out -covermode=atomic ./... 20 | 21 | test_reconnect: 22 | go test -tags=reconnect ./... 23 | 24 | build: 25 | go build ./... 26 | 27 | wait: 28 | 29 | fmt: 30 | go fmt ./... 31 | goimports -l -w . 32 | 33 | generate_gomod: 34 | rm go.mod go.sum || true 35 | go mod init github.com/ThreeDotsLabs/watermill 36 | 37 | go install ./... 38 | sed -i '\|go |d' go.mod 39 | go mod edit -fmt 40 | 41 | update_examples_deps: 42 | go run dev/update-examples-deps/main.go 43 | 44 | validate_examples: 45 | (cd dev/validate-examples/ && go run main.go) 46 | -------------------------------------------------------------------------------- /RELEASE-PROCEDURE.md: -------------------------------------------------------------------------------- 1 | # Release procedure 2 | 3 | 1. Generate clean go.mod: `make generate_gomod` 4 | 2. Push to master 5 | 3. Update missing documentation 6 | 4. Check snippets in documentation (sometimes `first_line_contains` or `last_line_contains` can change position and load too much) 7 | 5. Add breaking changes to `UPGRADE-[new-version].md` 8 | 6. Push to master 9 | 7. [Add release in GitHub](https://github.com/ThreeDotsLabs/watermill/releases) 10 | 8. Update Pub/Subs versions 11 | 9. Update and validate examples: `make validate_examples` 12 | -------------------------------------------------------------------------------- /UPGRADE-0.3.md: -------------------------------------------------------------------------------- 1 | # UPGRADE FROM 0.2.x to 0.3 2 | 3 | # `watermill/message` 4 | 5 | - `message.Message.Ack` and `message.Message.Nack` now return `bool` instead of `error` 6 | - `message.Subscriber.Subscribe` now accepts `context.Context` as the first argument 7 | - `message.Subscriber.Subscribe` now returns `<-chan *Message` instead of `chan *Message` 8 | - `message.Router.AddHandler` and `message.Router.AddNoPublisherHandler` now panic, instead of returning error 9 | 10 | # `watermill/message/infrastructure` 11 | 12 | - updated all Pub/Subs to new `message.Subscriber` interface 13 | - `gochannel.NewGoChannel` now accepts `gochannel.Config`, instead of positional parameters 14 | - `http.NewSubscriber` now accepts `http.SubscriberConfig`, instead of positional parameters 15 | 16 | # `watermill/message/router/middleware` 17 | 18 | - `metrics.NewMetrics` is removed, please use the [metrics](components/metrics) component instead 19 | 20 | # `watermill` 21 | 22 | - `watermill.LoggerAdapter` interface now requires a `With(fields LogFields) LoggerAdapter` method 23 | -------------------------------------------------------------------------------- /UPGRADE-1.0.md: -------------------------------------------------------------------------------- 1 | # Upgrade instructions from v0.4.X 2 | 3 | In v1.0.0 we introduced a couple of breaking changes, to keep a stable API until version v2. 4 | 5 | ## Migrating Pub/Subs 6 | 7 | All Pub/Subs (excluding go-channel implementation) were moved to separated repositories. 8 | You can replace all import paths, with provided `sed`: 9 | 10 | find . -type f -iname '*.go' -exec sed -i -E "s/github\.com\/ThreeDotsLabs\/watermill\/message\/infrastructure\/(amqp|googlecloud|http|io|kafka|nats|sql)/github.com\/ThreeDotsLabs\/watermill-\1\/pkg\/\1/" "{}" +; 11 | find . -type f -iname '*.go' -exec sed -i -E "s/github\.com\/ThreeDotsLabs\/watermill\/message\/infrastructure\/gochannel/github\.com\/ThreeDotsLabs\/watermill\/pubsub\/gochannel/" "{}" +; 12 | 13 | # Breaking changes 14 | - `message.PubSub` interface was removed 15 | - `message.NewPubSub` constructor was removed 16 | - `message.NoPublishHandlerFunc` is now passed to `message.Router.AddNoPublisherHandler`, instead of `message.HandlerFunc`. 17 | - `message.Router.Run` now requires `context.Context` in parameter 18 | - `PrometheusMetricsBuilder.DecoratePubSub` was removed (because of `message.PubSub` interface removal) 19 | - `cars.ObjectName` was renamed to `cqrs.FullyQualifiedStructName` 20 | - `github.com/ThreeDotsLabs/watermill/message/infrastructure/gochannel` was moved to `github.com/ThreeDotsLabs/watermill/pubsub/gochannel` 21 | - `middleware.Retry` configuration parameters have been renamed 22 | - Universal Pub/Sub tests have been moved from `github.com/ThreeDotsLabs/watermill/message/infrastructure` to `github.com/ThreeDotsLabs/watermill/pubsub/tests` 23 | - All universal tests require now `TestContext`. 24 | - Removed `context` from `googlecloud.NewPublisher` 25 | -------------------------------------------------------------------------------- /_examples/basic/1-your-first-app/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "received event {ID:[0-9]+}" 5 | -------------------------------------------------------------------------------- /_examples/basic/1-your-first-app/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | volumes: 6 | - .:/app 7 | - $GOPATH/pkg/mod:/go/pkg/mod 8 | working_dir: /app 9 | command: > 10 | /bin/sh -c "go install github.com/ThreeDotsLabs/watermill/tools/mill@latest && 11 | go run main.go" 12 | 13 | kafka: 14 | image: bitnami/kafka:3.5.0 15 | restart: unless-stopped 16 | environment: 17 | ALLOW_PLAINTEXT_LISTENER: yes 18 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 19 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 20 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 21 | -------------------------------------------------------------------------------- /_examples/basic/2-realtime-feed/.validate_example_subscribing.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "Adding to feed" 5 | -------------------------------------------------------------------------------- /_examples/basic/2-realtime-feed/README.md: -------------------------------------------------------------------------------- 1 | # Realtime Feed 2 | 3 | This example features a very busy blogging platform, with thousands of messages showing up on your feed. 4 | 5 | There are two separate applications (microservices) integrating over a Kafka topic. The [`producer`](producer/) generates 6 | thousands of "posts" and publishes them to the topic. The [`consumer`](consumer/) subscribes to this topic and 7 | displays each post on the standard output. 8 | 9 | The consumer has a throttling middleware enabled, so you have a chance to actually read the posts. 10 | 11 | To understand the background and internals, see [getting started guide](https://watermill.io/docs/getting-started/). 12 | 13 | ## Requirements 14 | 15 | To run this example you will need Docker and docker-compose installed. See the [installation guide](https://docs.docker.com/compose/install/). 16 | 17 | ## Running 18 | 19 | ```bash 20 | docker-compose up 21 | ``` 22 | 23 | You should see the live feed of posts on the standard output. 24 | 25 | ## Exercises 26 | 27 | 1. Peek into the posts counter published on `posts_count` topic. 28 | 29 | ``` 30 | docker-compose exec consumer mill kafka consume -b kafka:9092 -t posts_count 31 | ``` 32 | 33 | 2. Add a persistent storage for incoming posts in the consumer service, instead of displaying them. 34 | Consider using the [SQL Publisher](https://github.com/ThreeDotsLabs/watermill-sql). 35 | -------------------------------------------------------------------------------- /_examples/basic/2-realtime-feed/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | producer: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - kafka 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app/producer/ 11 | command: go run main.go 12 | 13 | consumer: 14 | image: golang:1.23 15 | restart: unless-stopped 16 | depends_on: 17 | - kafka 18 | volumes: 19 | - .:/app 20 | - $GOPATH/pkg/mod:/go/pkg/mod 21 | working_dir: /app/consumer/ 22 | command: > 23 | /bin/sh -c "go install github.com/ThreeDotsLabs/watermill/tools/mill@latest && 24 | go run main.go" 25 | 26 | zookeeper: 27 | image: confluentinc/cp-zookeeper:7.3.1 28 | restart: unless-stopped 29 | environment: 30 | ZOOKEEPER_CLIENT_PORT: 2181 31 | logging: 32 | driver: none 33 | 34 | kafka: 35 | image: confluentinc/cp-kafka:7.3.1 36 | restart: unless-stopped 37 | logging: 38 | driver: none 39 | depends_on: 40 | - zookeeper 41 | environment: 42 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 43 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 44 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 45 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 46 | -------------------------------------------------------------------------------- /_examples/basic/3-router/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "go run main.go" 2 | timeout: 120 3 | expected_output: "Received message: [0-9a-f\\-]+" 4 | -------------------------------------------------------------------------------- /_examples/basic/3-router/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require github.com/ThreeDotsLabs/watermill v1.4.4 8 | 9 | require ( 10 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 11 | github.com/google/uuid v1.6.0 // indirect 12 | github.com/hashicorp/errwrap v1.1.0 // indirect 13 | github.com/hashicorp/go-multierror v1.1.1 // indirect 14 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 15 | github.com/oklog/ulid v1.3.1 // indirect 16 | github.com/pkg/errors v0.9.1 // indirect 17 | github.com/sony/gobreaker v1.0.0 // indirect 18 | ) 19 | -------------------------------------------------------------------------------- /_examples/basic/4-metrics/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "msg=\"Message acked\"" 5 | -------------------------------------------------------------------------------- /_examples/basic/4-metrics/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | golang: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | ports: 6 | - 8080:8080 7 | - 8081:8081 8 | depends_on: 9 | - prometheus 10 | volumes: 11 | - ../../../:/watermill 12 | - $GOPATH/pkg/mod:/go/pkg/mod 13 | working_dir: /watermill/_examples/basic/4-metrics 14 | command: go run main.go -metrics :8081 -delay 0.1 15 | 16 | prometheus: 17 | image: prom/prometheus 18 | restart: unless-stopped 19 | network_mode: host 20 | volumes: 21 | - ./prometheus.yml:/etc/prometheus/prometheus.yml 22 | 23 | grafana: 24 | image: grafana/grafana:5.2.4 25 | network_mode: host 26 | 27 | -------------------------------------------------------------------------------- /_examples/basic/4-metrics/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require github.com/ThreeDotsLabs/watermill v1.4.4 8 | 9 | require ( 10 | github.com/beorn7/perks v1.0.1 // indirect 11 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 12 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 13 | github.com/go-chi/chi/v5 v5.2.0 // indirect 14 | github.com/google/uuid v1.6.0 // indirect 15 | github.com/hashicorp/errwrap v1.1.0 // indirect 16 | github.com/hashicorp/go-multierror v1.1.1 // indirect 17 | github.com/klauspost/compress v1.17.11 // indirect 18 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 19 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 20 | github.com/oklog/ulid v1.3.1 // indirect 21 | github.com/pkg/errors v0.9.1 // indirect 22 | github.com/prometheus/client_golang v1.20.5 // indirect 23 | github.com/prometheus/client_model v0.6.1 // indirect 24 | github.com/prometheus/common v0.61.0 // indirect 25 | github.com/prometheus/procfs v0.15.1 // indirect 26 | github.com/sony/gobreaker v1.0.0 // indirect 27 | golang.org/x/sys v0.29.0 // indirect 28 | google.golang.org/protobuf v1.36.3 // indirect 29 | ) 30 | 31 | // uncomment to use local sources 32 | // replace github.com/ThreeDotsLabs/watermill => ../../../../watermill 33 | -------------------------------------------------------------------------------- /_examples/basic/4-metrics/prometheus.yml: -------------------------------------------------------------------------------- 1 | # my global config 2 | global: 3 | scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. 4 | evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. 5 | # scrape_timeout is set to the global default (10s). 6 | 7 | scrape_configs: 8 | # # The job name is added as a label `job=` to any timeseries scraped from this config. 9 | # - job_name: 'prometheus' 10 | # 11 | # # metrics_path defaults to '/metrics' 12 | # # scheme defaults to 'http'. 13 | # 14 | # static_configs: 15 | # - targets: ['localhost:9090'] 16 | 17 | - job_name: 'metrics_example' 18 | 19 | static_configs: 20 | - targets: ['localhost:8081'] 21 | -------------------------------------------------------------------------------- /_examples/basic/5-cqrs-protobuf/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 120 4 | expected_outputs: 5 | - "beers ordered to room \\d+" 6 | - "Already booked rooms for \\$\\d{2,}" 7 | -------------------------------------------------------------------------------- /_examples/basic/5-cqrs-protobuf/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: proto 2 | 3 | proto: 4 | protoc --proto_path=proto --go_out=. --go_opt=paths=source_relative proto/messages.proto 5 | -------------------------------------------------------------------------------- /_examples/basic/5-cqrs-protobuf/README.md: -------------------------------------------------------------------------------- 1 | # Example Golang CQRS application 2 | 3 | This application is using [Watermill CQRS](http://watermill.io/docs/cqrs) component. 4 | 5 | Detailed documentation for CQRS can be found in Watermill's docs: [http://watermill.io/docs/cqrs#usage](http://watermill.io/docs/cqrs). 6 | 7 | ![CQRS Event Storming](https://threedots.tech/watermill-io/cqrs-example-storming.png) 8 | 9 | ```mermaid 10 | sequenceDiagram 11 | participant M as Main 12 | participant CB as CommandBus 13 | participant BRH as BookRoomHandler 14 | participant EB as EventBus 15 | participant OBRB as OrderBeerOnRoomBooked 16 | participant OBH as OrderBeerHandler 17 | participant BFR as BookingsFinancialReport 18 | 19 | Note over M,BFR: Commands use AMQP queue, Events use AMQP pub/sub 20 | 21 | M->>CB: Send(BookRoom)
topic: commands.BookRoom 22 | CB->>BRH: Handle(BookRoom) 23 | 24 | BRH->>EB: Publish(RoomBooked)
topic: events.RoomBooked 25 | 26 | par Process RoomBooked Event 27 | EB->>OBRB: Handle(RoomBooked) 28 | OBRB->>CB: Send(OrderBeer)
topic: commands.OrderBeer 29 | CB->>OBH: Handle(OrderBeer) 30 | OBH->>EB: Publish(BeerOrdered)
topic: events.BeerOrdered 31 | 32 | EB->>BFR: Handle(RoomBooked) 33 | Note over BFR: Updates financial report 34 | end 35 | ``` 36 | 37 | 38 | ## Running 39 | 40 | ```bash 41 | docker-compose up 42 | ``` 43 | -------------------------------------------------------------------------------- /_examples/basic/5-cqrs-protobuf/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | golang: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | ports: 6 | - 8080:8080 7 | depends_on: 8 | - rabbitmq 9 | links: 10 | - rabbitmq 11 | volumes: 12 | - .:/app 13 | - $GOPATH/pkg/mod:/go/pkg/mod 14 | working_dir: /app 15 | command: go run . 16 | 17 | rabbitmq: 18 | image: rabbitmq:3.7 19 | restart: unless-stopped 20 | attach: false 21 | -------------------------------------------------------------------------------- /_examples/basic/5-cqrs-protobuf/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | require ( 4 | github.com/ThreeDotsLabs/watermill v1.4.4 5 | github.com/ThreeDotsLabs/watermill-amqp/v3 v3.0.0 6 | github.com/golang/protobuf v1.5.4 7 | google.golang.org/protobuf v1.36.3 8 | ) 9 | 10 | require ( 11 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 12 | github.com/gogo/protobuf v1.3.2 // indirect 13 | github.com/google/uuid v1.6.0 // indirect 14 | github.com/hashicorp/errwrap v1.1.0 // indirect 15 | github.com/hashicorp/go-multierror v1.1.1 // indirect 16 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 17 | github.com/oklog/ulid v1.3.1 // indirect 18 | github.com/pkg/errors v0.9.1 // indirect 19 | github.com/rabbitmq/amqp091-go v1.10.0 // indirect 20 | github.com/sony/gobreaker v1.0.0 // indirect 21 | ) 22 | 23 | go 1.23 24 | 25 | toolchain go1.23.4 26 | -------------------------------------------------------------------------------- /_examples/basic/5-cqrs-protobuf/proto/messages.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package main; 3 | 4 | option go_package = "./main"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | 8 | message BookRoom { 9 | string room_id = 1; 10 | string guest_name = 2; 11 | 12 | google.protobuf.Timestamp start_date = 4; 13 | google.protobuf.Timestamp end_date = 5; 14 | } 15 | 16 | message RoomBooked { 17 | string reservation_id = 1; 18 | string room_id = 2; 19 | string guest_name = 3; 20 | int64 price = 4; 21 | 22 | google.protobuf.Timestamp start_date = 5; 23 | google.protobuf.Timestamp end_date = 6; 24 | } 25 | 26 | message OrderBeer { 27 | string room_id = 1; 28 | int64 count = 2; 29 | } 30 | 31 | 32 | message BeerOrdered { 33 | string room_id = 1; 34 | int64 count = 2; 35 | } -------------------------------------------------------------------------------- /_examples/basic/6-cqrs-ordered-events/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 120 4 | expected_outputs: 5 | - "Subscriber added subscriber_id" 6 | - "Subscriber updated subscriber_id" 7 | - "Subscriber removed subscriber_id" 8 | - "\\[ACTIVITY\\] activity_type=SUBSCRIBED" 9 | - "\\[ACTIVITY\\] activity_type=UNSUBSCRIBED" 10 | - "\\[ACTIVITY\\] activity_type=EMAIL_UPDATED" 11 | -------------------------------------------------------------------------------- /_examples/basic/6-cqrs-ordered-events/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: proto 2 | 3 | proto: 4 | protoc --proto_path=proto --go_out=. --go_opt=paths=source_relative proto/messages.proto 5 | -------------------------------------------------------------------------------- /_examples/basic/6-cqrs-ordered-events/README.md: -------------------------------------------------------------------------------- 1 | # Example Golang CQRS application - ordered events with Kafka 2 | 3 | This application is using [Watermill CQRS](http://watermill.io/docs/cqrs) component. 4 | 5 | Detailed documentation for CQRS can be found in Watermill's docs: [http://watermill.io/docs/cqrs#usage](http://watermill.io/docs/cqrs). 6 | 7 | This example, uses event groups to keep order for multiple events. You can read more about them in the [Watermill documentation](https://watermill.io/docs/cqrs/). 8 | We are also using Kafka's partitioning keys to increase processing throughput without losing order of events. 9 | 10 | 11 | ## What does this application do? 12 | 13 | This application manages an email subscription system where users can: 14 | 15 | 1. Subscribe to receive emails by providing their email address 16 | 2. Update their email address after subscribing 17 | 3. Unsubscribe from the mailing list 18 | 19 | The system maintains: 20 | - A current list of all active subscribers 21 | - A timeline of all subscription-related activities 22 | 23 | In this example, keeping order of events is crucial. 24 | If events won't be ordered, and `SubscriberSubscribed` would arrive after `SubscriberUnsubscribed` event, 25 | the subscriber will be still subscribed. 26 | 27 | ## Possible improvements 28 | 29 | In this example, we are using global `events` and `commands` topics. 30 | You can consider splitting them into smaller topics, for example, per aggregate type. 31 | 32 | Thanks to that, you can scale your application horizontally and increase the throughput and processing less events. 33 | 34 | ## Running 35 | 36 | ```bash 37 | docker-compose up 38 | ``` 39 | -------------------------------------------------------------------------------- /_examples/basic/6-cqrs-ordered-events/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | golang: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | ports: 6 | - 8080:8080 7 | depends_on: 8 | - kafka 9 | - zookeeper 10 | links: 11 | - kafka 12 | - zookeeper 13 | volumes: 14 | - .:/app 15 | - $GOPATH/pkg/mod:/go/pkg/mod 16 | working_dir: /app 17 | command: go run . 18 | 19 | zookeeper: 20 | container_name: zk 21 | attach: false 22 | image: confluentinc/cp-zookeeper:7.7.1 23 | environment: 24 | ZOOKEEPER_CLIENT_PORT: 2181 25 | ZOOKEEPER_TICK_TIME: 2000 26 | ports: 27 | - 2181:2181 28 | 29 | kafka: 30 | container_name: kafka 31 | attach: false 32 | image: confluentinc/cp-kafka:7.7.1 33 | depends_on: 34 | - zookeeper 35 | ports: 36 | - 9093:9093 37 | environment: 38 | KAFKA_BROKER_ID: 1 39 | KAFKA_ZOOKEEPER_CONNECT: zk:2181 40 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:9093 41 | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT 42 | KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT 43 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 -------------------------------------------------------------------------------- /_examples/basic/6-cqrs-ordered-events/message.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log/slog" 6 | 7 | "github.com/ThreeDotsLabs/watermill/components/cqrs" 8 | "github.com/ThreeDotsLabs/watermill/message" 9 | "google.golang.org/protobuf/types/known/timestamppb" 10 | ) 11 | 12 | func GenerateMessageMetadata(partitionKey string) *MessageMetadata { 13 | return &MessageMetadata{ 14 | PartitionKey: partitionKey, 15 | CreatedAt: timestamppb.Now(), 16 | } 17 | } 18 | 19 | type CqrsMarshalerDecorator struct { 20 | cqrs.ProtoMarshaler 21 | } 22 | 23 | const PartitionKeyMetadataField = "partition_key" 24 | 25 | func (c CqrsMarshalerDecorator) Marshal(v interface{}) (*message.Message, error) { 26 | msg, err := c.ProtoMarshaler.Marshal(v) 27 | if err != nil { 28 | return nil, err 29 | } 30 | 31 | pm, ok := v.(ProtoMessage) 32 | if !ok { 33 | return nil, fmt.Errorf("%T does not implement ProtoMessage and can't be marshaled", v) 34 | } 35 | 36 | metadata := pm.GetMetadata() 37 | if metadata == nil { 38 | return nil, fmt.Errorf("%T.GetMetadata returned nil", v) 39 | } 40 | 41 | msg.Metadata.Set(PartitionKeyMetadataField, metadata.PartitionKey) 42 | msg.Metadata.Set("created_at", metadata.CreatedAt.AsTime().String()) 43 | 44 | return msg, nil 45 | } 46 | 47 | type ProtoMessage interface { 48 | GetMetadata() *MessageMetadata 49 | } 50 | 51 | // GenerateKafkaPartitionKey is a function that generates a partition key for Kafka messages. 52 | func GenerateKafkaPartitionKey(topic string, msg *message.Message) (string, error) { 53 | slog.Debug("Setting partition key", "topic", topic, "msg_metadata", msg.Metadata) 54 | 55 | return msg.Metadata.Get(PartitionKeyMetadataField), nil 56 | } 57 | -------------------------------------------------------------------------------- /_examples/basic/6-cqrs-ordered-events/proto/messages.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package main; 3 | 4 | option go_package = "./main"; 5 | 6 | import "google/protobuf/timestamp.proto"; 7 | 8 | message MessageMetadata { 9 | string partition_key = 1; 10 | google.protobuf.Timestamp created_at = 2; 11 | } 12 | 13 | // Commands 14 | message Subscribe { 15 | MessageMetadata metadata = 1; 16 | 17 | string subscriber_id = 2; 18 | string email = 3; 19 | } 20 | 21 | message Unsubscribe { 22 | MessageMetadata metadata = 1; 23 | 24 | string subscriber_id = 2; 25 | } 26 | 27 | message UpdateEmail { 28 | MessageMetadata metadata = 1; 29 | 30 | string subscriber_id = 2; 31 | string new_email = 3; 32 | } 33 | 34 | // Events 35 | message SubscriberSubscribed { 36 | MessageMetadata metadata = 1; 37 | 38 | string subscriber_id = 2; 39 | string email = 3; 40 | } 41 | 42 | message SubscriberUnsubscribed { 43 | MessageMetadata metadata = 1; 44 | 45 | string subscriber_id = 2; 46 | } 47 | 48 | message SubscriberEmailUpdated { 49 | MessageMetadata metadata = 1; 50 | 51 | string subscriber_id = 2; 52 | string new_email = 3; 53 | } -------------------------------------------------------------------------------- /_examples/basic/6-cqrs-ordered-events/subscribers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log/slog" 6 | "sync" 7 | ) 8 | 9 | type SubscriberReadModel struct { 10 | subscribers map[string]string // map[subscriberID]email 11 | lock sync.RWMutex 12 | } 13 | 14 | func NewSubscriberReadModel() *SubscriberReadModel { 15 | return &SubscriberReadModel{ 16 | subscribers: make(map[string]string), 17 | } 18 | } 19 | 20 | func (m *SubscriberReadModel) OnSubscribed(ctx context.Context, event *SubscriberSubscribed) error { 21 | m.lock.Lock() 22 | defer m.lock.Unlock() 23 | 24 | m.subscribers[event.SubscriberId] = event.Email 25 | 26 | slog.Info( 27 | "Subscriber added", 28 | "subscriber_id", event.SubscriberId, 29 | "email", event.Email, 30 | ) 31 | 32 | return nil 33 | } 34 | 35 | func (m *SubscriberReadModel) OnUnsubscribed(ctx context.Context, event *SubscriberUnsubscribed) error { 36 | m.lock.Lock() 37 | defer m.lock.Unlock() 38 | 39 | delete(m.subscribers, event.SubscriberId) 40 | 41 | slog.Info( 42 | "Subscriber removed", 43 | "subscriber_id", event.SubscriberId, 44 | ) 45 | 46 | return nil 47 | } 48 | 49 | func (m *SubscriberReadModel) OnEmailUpdated(ctx context.Context, event *SubscriberEmailUpdated) error { 50 | m.lock.Lock() 51 | defer m.lock.Unlock() 52 | 53 | m.subscribers[event.SubscriberId] = event.NewEmail 54 | 55 | slog.Info( 56 | "Subscriber updated", 57 | "subscriber_id", event.SubscriberId, 58 | "email", event.NewEmail, 59 | ) 60 | 61 | return nil 62 | } 63 | 64 | func (m *SubscriberReadModel) GetSubscriberCount() int { 65 | m.lock.RLock() 66 | defer m.lock.RUnlock() 67 | return len(m.subscribers) 68 | } 69 | -------------------------------------------------------------------------------- /_examples/pubsubs/amqp/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 120 4 | expected_output: "received message: [0-9a-f\\-]+, payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/amqp/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - rabbitmq 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: go run main.go 12 | 13 | rabbitmq: 14 | image: rabbitmq:3.7 15 | restart: unless-stopped 16 | -------------------------------------------------------------------------------- /_examples/pubsubs/amqp/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | require ( 4 | github.com/ThreeDotsLabs/watermill v1.4.4 5 | github.com/ThreeDotsLabs/watermill-amqp/v3 v3.0.0 6 | ) 7 | 8 | require ( 9 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 10 | github.com/google/uuid v1.6.0 // indirect 11 | github.com/hashicorp/errwrap v1.1.0 // indirect 12 | github.com/hashicorp/go-multierror v1.1.1 // indirect 13 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 14 | github.com/oklog/ulid v1.3.1 // indirect 15 | github.com/pkg/errors v0.9.1 // indirect 16 | github.com/rabbitmq/amqp091-go v1.10.0 // indirect 17 | ) 18 | 19 | go 1.23 20 | 21 | toolchain go1.23.4 22 | -------------------------------------------------------------------------------- /_examples/pubsubs/aws-sns/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 120 4 | expected_output: "A received message: [0-9a-f\\-]+, payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/aws-sns/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | volumes: 6 | - .:/app 7 | - $GOPATH/pkg/mod:/go/pkg/mod 8 | working_dir: /app 9 | command: go run main.go 10 | 11 | localstack: 12 | image: localstack/localstack:3.8 13 | environment: 14 | - SERVICES=sqs,sns 15 | - AWS_DEFAULT_REGION=us-east-1 16 | - EDGE_PORT=4566 17 | ports: 18 | - "4566-4597:4566-4597" 19 | healthcheck: 20 | test: awslocal sqs list-queues && awslocal sns list-topics 21 | interval: 5s 22 | timeout: 5s 23 | retries: 5 24 | start_period: 30s 25 | -------------------------------------------------------------------------------- /_examples/pubsubs/aws-sns/go.mod: -------------------------------------------------------------------------------- 1 | module main 2 | 3 | require ( 4 | github.com/ThreeDotsLabs/watermill v1.4.4 5 | github.com/ThreeDotsLabs/watermill-aws v1.0.0 6 | github.com/aws/aws-sdk-go-v2 v1.33.0 7 | github.com/aws/aws-sdk-go-v2/service/sns v1.33.12 8 | github.com/aws/aws-sdk-go-v2/service/sqs v1.37.8 9 | github.com/aws/smithy-go v1.22.1 10 | github.com/samber/lo v1.47.0 11 | ) 12 | 13 | require ( 14 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect 15 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect 16 | github.com/google/uuid v1.6.0 // indirect 17 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 18 | github.com/oklog/ulid v1.3.1 // indirect 19 | github.com/pkg/errors v0.9.1 // indirect 20 | golang.org/x/text v0.21.0 // indirect 21 | ) 22 | 23 | go 1.23 24 | 25 | toolchain go1.23.4 26 | -------------------------------------------------------------------------------- /_examples/pubsubs/aws-sqs/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 120 4 | expected_output: "received message: [0-9a-f\\-]+, payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/aws-sqs/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | volumes: 6 | - .:/app 7 | - $GOPATH/pkg/mod:/go/pkg/mod 8 | working_dir: /app 9 | command: go run main.go 10 | 11 | localstack: 12 | image: localstack/localstack:latest 13 | environment: 14 | - SERVICES=sqs,sns 15 | - AWS_DEFAULT_REGION=us-east-1 16 | - EDGE_PORT=4566 17 | ports: 18 | - "4566-4597:4566-4597" 19 | healthcheck: 20 | test: awslocal sqs list-queues && awslocal sns list-topics 21 | interval: 5s 22 | timeout: 5s 23 | retries: 5 24 | start_period: 30s 25 | -------------------------------------------------------------------------------- /_examples/pubsubs/aws-sqs/go.mod: -------------------------------------------------------------------------------- 1 | module main 2 | 3 | require ( 4 | github.com/ThreeDotsLabs/watermill v1.4.4 5 | github.com/ThreeDotsLabs/watermill-aws v1.0.0 6 | github.com/aws/aws-sdk-go-v2 v1.33.0 7 | github.com/aws/aws-sdk-go-v2/service/sqs v1.37.8 8 | github.com/aws/smithy-go v1.22.1 9 | github.com/samber/lo v1.47.0 10 | ) 11 | 12 | require ( 13 | github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect 14 | github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect 15 | github.com/google/uuid v1.6.0 // indirect 16 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 17 | github.com/oklog/ulid v1.3.1 // indirect 18 | github.com/pkg/errors v0.9.1 // indirect 19 | golang.org/x/text v0.21.0 // indirect 20 | ) 21 | 22 | go 1.23 23 | 24 | toolchain go1.23.4 25 | -------------------------------------------------------------------------------- /_examples/pubsubs/go-channel/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "go run main.go" 2 | timeout: 30 3 | expected_output: "payload: Hello, world!" 4 | -------------------------------------------------------------------------------- /_examples/pubsubs/go-channel/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | require github.com/ThreeDotsLabs/watermill v1.4.4 4 | 5 | require ( 6 | github.com/google/uuid v1.6.0 // indirect 7 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 8 | github.com/oklog/ulid v1.3.1 // indirect 9 | github.com/pkg/errors v0.9.1 // indirect 10 | ) 11 | 12 | go 1.23 13 | 14 | toolchain go1.23.4 15 | -------------------------------------------------------------------------------- /_examples/pubsubs/go-channel/main.go: -------------------------------------------------------------------------------- 1 | // Sources for https://watermill.io/docs/getting-started/ 2 | package main 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/ThreeDotsLabs/watermill" 10 | "github.com/ThreeDotsLabs/watermill/message" 11 | "github.com/ThreeDotsLabs/watermill/pubsub/gochannel" 12 | ) 13 | 14 | func main() { 15 | pubSub := gochannel.NewGoChannel( 16 | gochannel.Config{}, 17 | watermill.NewStdLogger(false, false), 18 | ) 19 | 20 | messages, err := pubSub.Subscribe(context.Background(), "example.topic") 21 | if err != nil { 22 | panic(err) 23 | } 24 | 25 | go process(messages) 26 | 27 | publishMessages(pubSub) 28 | } 29 | 30 | func publishMessages(publisher message.Publisher) { 31 | for { 32 | msg := message.NewMessage(watermill.NewUUID(), []byte("Hello, world!")) 33 | 34 | if err := publisher.Publish("example.topic", msg); err != nil { 35 | panic(err) 36 | } 37 | 38 | time.Sleep(time.Second) 39 | } 40 | } 41 | 42 | func process(messages <-chan *message.Message) { 43 | for msg := range messages { 44 | fmt.Printf("received message: %s, payload: %s\n", msg.UUID, string(msg.Payload)) 45 | 46 | // we need to Acknowledge that we received and processed the message, 47 | // otherwise, it will be resent over and over again. 48 | msg.Ack() 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /_examples/pubsubs/googlecloud/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/googlecloud/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - googlecloud 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | environment: 11 | # use local emulator instead of google cloud engine 12 | PUBSUB_EMULATOR_HOST: "googlecloud:8085" 13 | working_dir: /app 14 | command: go run main.go 15 | 16 | googlecloud: 17 | image: google/cloud-sdk:414.0.0 18 | entrypoint: gcloud --quiet beta emulators pubsub start --host-port=0.0.0.0:8085 --verbosity=debug --log-http 19 | restart: unless-stopped 20 | -------------------------------------------------------------------------------- /_examples/pubsubs/kafka/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/kafka/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - kafka 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: go run main.go 12 | 13 | zookeeper: 14 | image: confluentinc/cp-zookeeper:7.3.1 15 | restart: unless-stopped 16 | logging: 17 | driver: none 18 | environment: 19 | ZOOKEEPER_CLIENT_PORT: 2181 20 | 21 | kafka: 22 | image: confluentinc/cp-kafka:7.3.1 23 | restart: unless-stopped 24 | depends_on: 25 | - zookeeper 26 | logging: 27 | driver: none 28 | environment: 29 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 30 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 31 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 32 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 33 | -------------------------------------------------------------------------------- /_examples/pubsubs/nats-core/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/nats-core/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - nats 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: go run main.go 12 | 13 | nats: 14 | image: nats:2 15 | ports: 16 | - "0.0.0.0:4222:4222" 17 | restart: unless-stopped 18 | command: ["-js"] 19 | ulimits: 20 | nofile: 21 | soft: 65536 22 | hard: 65536 23 | -------------------------------------------------------------------------------- /_examples/pubsubs/nats-core/go.mod: -------------------------------------------------------------------------------- 1 | module main 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-nats/v2 v2.1.2 10 | github.com/nats-io/nats.go v1.38.0 11 | ) 12 | 13 | require ( 14 | github.com/google/uuid v1.6.0 // indirect 15 | github.com/klauspost/compress v1.17.11 // indirect 16 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 17 | github.com/nats-io/nkeys v0.4.9 // indirect 18 | github.com/nats-io/nuid v1.0.1 // indirect 19 | github.com/oklog/ulid v1.3.1 // indirect 20 | github.com/pkg/errors v0.9.1 // indirect 21 | golang.org/x/crypto v0.32.0 // indirect 22 | golang.org/x/sys v0.29.0 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /_examples/pubsubs/nats-jetstream/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/nats-jetstream/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - nats 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: go run main.go 12 | 13 | nats: 14 | image: nats:2 15 | ports: 16 | - "0.0.0.0:4222:4222" 17 | restart: unless-stopped 18 | command: ["-js"] 19 | ulimits: 20 | nofile: 21 | soft: 65536 22 | hard: 65536 23 | -------------------------------------------------------------------------------- /_examples/pubsubs/nats-jetstream/go.mod: -------------------------------------------------------------------------------- 1 | module main 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-nats/v2 v2.1.2 10 | github.com/nats-io/nats.go v1.38.0 11 | ) 12 | 13 | require ( 14 | github.com/google/uuid v1.6.0 // indirect 15 | github.com/klauspost/compress v1.17.11 // indirect 16 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 17 | github.com/nats-io/nkeys v0.4.9 // indirect 18 | github.com/nats-io/nuid v1.0.1 // indirect 19 | github.com/oklog/ulid v1.3.1 // indirect 20 | github.com/pkg/errors v0.9.1 // indirect 21 | golang.org/x/crypto v0.32.0 // indirect 22 | golang.org/x/sys v0.29.0 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /_examples/pubsubs/nats-streaming/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/nats-streaming/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - nats-streaming 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: go run main.go 12 | 13 | nats-streaming: 14 | image: nats-streaming:0.11.2 15 | restart: unless-stopped 16 | -------------------------------------------------------------------------------- /_examples/pubsubs/nats-streaming/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | require ( 4 | github.com/ThreeDotsLabs/watermill v1.4.4 5 | github.com/ThreeDotsLabs/watermill-nats v1.0.7 6 | github.com/nats-io/stan.go v0.10.4 7 | ) 8 | 9 | require ( 10 | github.com/gogo/protobuf v1.3.2 // indirect 11 | github.com/google/uuid v1.6.0 // indirect 12 | github.com/hashicorp/go-hclog v1.4.0 // indirect 13 | github.com/hashicorp/go-msgpack v0.5.5 // indirect 14 | github.com/hashicorp/raft v1.3.11 // indirect 15 | github.com/klauspost/compress v1.17.11 // indirect 16 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 17 | github.com/minio/highwayhash v1.0.2 // indirect 18 | github.com/nats-io/jwt/v2 v2.3.0 // indirect 19 | github.com/nats-io/nats.go v1.38.0 // indirect 20 | github.com/nats-io/nkeys v0.4.9 // indirect 21 | github.com/nats-io/nuid v1.0.1 // indirect 22 | github.com/oklog/ulid v1.3.1 // indirect 23 | github.com/pkg/errors v0.9.1 // indirect 24 | go.etcd.io/bbolt v1.3.6 // indirect 25 | golang.org/x/crypto v0.32.0 // indirect 26 | golang.org/x/sys v0.29.0 // indirect 27 | golang.org/x/time v0.3.0 // indirect 28 | ) 29 | 30 | go 1.23 31 | 32 | toolchain go1.23.4 33 | -------------------------------------------------------------------------------- /_examples/pubsubs/redisstream/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/redisstream/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - redis 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: go run main.go 12 | 13 | redis: 14 | image: redis:7 15 | ports: 16 | - 6379:6379 17 | restart: unless-stopped 18 | -------------------------------------------------------------------------------- /_examples/pubsubs/redisstream/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-redisstream v1.4.2 10 | github.com/redis/go-redis/v9 v9.7.0 11 | ) 12 | 13 | require ( 14 | github.com/Rican7/retry v0.3.1 // indirect 15 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 16 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 17 | github.com/golang/protobuf v1.5.4 // indirect 18 | github.com/google/uuid v1.6.0 // indirect 19 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 20 | github.com/oklog/ulid v1.3.1 // indirect 21 | github.com/pkg/errors v0.9.1 // indirect 22 | github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect 23 | google.golang.org/appengine v1.6.8 // indirect 24 | google.golang.org/protobuf v1.36.3 // indirect 25 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 26 | ) 27 | -------------------------------------------------------------------------------- /_examples/pubsubs/sql/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "Hello, world!" 5 | -------------------------------------------------------------------------------- /_examples/pubsubs/sql/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - mysql 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: go run main.go 12 | 13 | mysql: 14 | image: mysql:8.0 15 | restart: unless-stopped 16 | ports: 17 | - 3306:3306 18 | environment: 19 | MYSQL_DATABASE: watermill 20 | MYSQL_ALLOW_EMPTY_PASSWORD: "yes" 21 | -------------------------------------------------------------------------------- /_examples/pubsubs/sql/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-sql/v3 v3.1.0 10 | github.com/go-sql-driver/mysql v1.8.1 11 | ) 12 | 13 | require ( 14 | filippo.io/edwards25519 v1.1.0 // indirect 15 | github.com/google/uuid v1.6.0 // indirect 16 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 17 | github.com/oklog/ulid v1.3.1 // indirect 18 | github.com/pkg/errors v0.9.1 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /_examples/real-world-examples/consumer-groups/README.md: -------------------------------------------------------------------------------- 1 | # Interactive Consumer Groups Example (Routing Events) 2 | 3 | This example shows how Customer Groups work, i.e. how to decide which handlers receive which events. 4 | 5 | Consumer Group is a concept used in Apache Kafka®, but many other Pub/Subs use a similar mechanism. 6 | 7 | The example uses Watermill and Redis Streams Pub/Sub, but the same idea applies to other Pub/Subs as well. 8 | 9 | ## Live video 10 | 11 | This example was showcased on the Watermill v1.2 Launch Event. You can see the [recording on YouTube](https://www.youtube.com/live/wjnd0Hj6CaM?t=1020) (starts at 17:00). 12 | 13 | [![Live Recording](https://img.youtube.com/vi/wjnd0Hj6CaM/0.jpg)](https://www.youtube.com/live/wjnd0Hj6CaM?t=1020) 14 | 15 | ## Running 16 | 17 | ``` 18 | docker-compose up 19 | ``` 20 | 21 | Then visit [localhost:8080](http://localhost:8080) and check the examples in each tab. 22 | 23 | ## Screenshots 24 | 25 | ![](docs/screen2.png) 26 | 27 | ![](docs/screen1.png) 28 | 29 | ## Code 30 | 31 | See [crm-service](crm-service) and [newsletter-service](newsletter-service) for the Watermill handlers setup. 32 | 33 | ## How does it work? 34 | 35 | This example uses SSE for pushing events to the frontend UI. See the [other example on SEE](../server-sent-events) for more details. 36 | -------------------------------------------------------------------------------- /_examples/real-world-examples/consumer-groups/api/storage.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/ThreeDotsLabs/watermill-routing-example/server/common" 7 | ) 8 | 9 | type storage struct { 10 | lock *sync.Mutex 11 | receivedMessages map[string][]common.MessageReceived 12 | } 13 | 14 | func (s *storage) Append(message common.MessageReceived) { 15 | s.lock.Lock() 16 | defer s.lock.Unlock() 17 | 18 | for k, v := range s.receivedMessages { 19 | s.receivedMessages[k] = append(v, message) 20 | } 21 | } 22 | 23 | func (s *storage) PopAll(key string) []common.MessageReceived { 24 | s.lock.Lock() 25 | defer s.lock.Unlock() 26 | 27 | if _, ok := s.receivedMessages[key]; !ok { 28 | s.receivedMessages[key] = []common.MessageReceived{} 29 | return []common.MessageReceived{} 30 | } 31 | 32 | messages := s.receivedMessages[key] 33 | s.receivedMessages[key] = []common.MessageReceived{} 34 | return messages 35 | } 36 | -------------------------------------------------------------------------------- /_examples/real-world-examples/consumer-groups/common/events.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type UserSignedUp struct { 4 | UserID string `json:"id"` 5 | Consents Consents `json:"consents"` 6 | } 7 | 8 | type Consents struct { 9 | Marketing bool `json:"marketing"` 10 | News bool `json:"news"` 11 | } 12 | 13 | type MessageReceived struct { 14 | ID string `json:"id"` 15 | Service string `json:"service"` 16 | Handler string `json:"handler"` 17 | Topic string `json:"topic"` 18 | } 19 | -------------------------------------------------------------------------------- /_examples/real-world-examples/consumer-groups/common/messaging.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | 7 | "github.com/ThreeDotsLabs/watermill" 8 | "github.com/ThreeDotsLabs/watermill/message" 9 | ) 10 | 11 | const UpdatesTopic = "updates" 12 | 13 | func NotifyMiddleware(pub message.Publisher, serviceName string) func(message.HandlerFunc) message.HandlerFunc { 14 | return func(next message.HandlerFunc) message.HandlerFunc { 15 | return func(msg *message.Message) ([]*message.Message, error) { 16 | topic := message.SubscribeTopicFromCtx(msg.Context()) 17 | handler := strings.Split(message.HandlerNameFromCtx(msg.Context()), "-")[0] 18 | 19 | msgs, err := next(msg) 20 | if err != nil { 21 | return msgs, err 22 | } 23 | 24 | payload := MessageReceived{ 25 | ID: msg.UUID, 26 | Service: serviceName, 27 | Handler: handler, 28 | Topic: topic, 29 | } 30 | 31 | jsonPayload, err := json.Marshal(payload) 32 | if err != nil { 33 | return nil, err 34 | } 35 | 36 | newMsg := message.NewMessage(watermill.NewUUID(), jsonPayload) 37 | 38 | err = pub.Publish(UpdatesTopic, newMsg) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | return msgs, nil 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /_examples/real-world-examples/consumer-groups/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | api: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - redis 7 | volumes: 8 | - .:/app 9 | working_dir: /app/api 10 | ports: 11 | - "8080:8080" 12 | command: go run . 13 | 14 | newsletter-1: 15 | image: golang:1.23 16 | restart: unless-stopped 17 | depends_on: 18 | - redis 19 | volumes: 20 | - .:/app 21 | working_dir: /app/newsletter-service 22 | command: go run . 23 | environment: 24 | REPLICA: 1 25 | 26 | newsletter-2: 27 | image: golang:1.23 28 | restart: unless-stopped 29 | depends_on: 30 | - redis 31 | volumes: 32 | - .:/app 33 | working_dir: /app/newsletter-service 34 | command: go run . 35 | environment: 36 | REPLICA: 2 37 | 38 | crm-1: 39 | image: golang:1.23 40 | restart: unless-stopped 41 | depends_on: 42 | - redis 43 | volumes: 44 | - .:/app 45 | working_dir: /app/crm-service 46 | command: go run . 47 | environment: 48 | REPLICA: 1 49 | 50 | crm-2: 51 | image: golang:1.23 52 | restart: unless-stopped 53 | depends_on: 54 | - redis 55 | volumes: 56 | - .:/app 57 | working_dir: /app/crm-service 58 | command: go run . 59 | environment: 60 | REPLICA: 2 61 | 62 | redis: 63 | image: redis:7 64 | ports: 65 | - "6379:6379" 66 | restart: unless-stopped 67 | -------------------------------------------------------------------------------- /_examples/real-world-examples/consumer-groups/docs/screen1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/_examples/real-world-examples/consumer-groups/docs/screen1.png -------------------------------------------------------------------------------- /_examples/real-world-examples/consumer-groups/docs/screen2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/_examples/real-world-examples/consumer-groups/docs/screen2.png -------------------------------------------------------------------------------- /_examples/real-world-examples/consumer-groups/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ThreeDotsLabs/watermill-routing-example/server 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-http v1.1.4 10 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 11 | github.com/go-chi/chi/v5 v5.2.0 12 | github.com/go-chi/render v1.0.3 // indirect 13 | github.com/google/uuid v1.6.0 // indirect 14 | github.com/hashicorp/errwrap v1.1.0 // indirect 15 | github.com/hashicorp/go-multierror v1.1.1 // indirect 16 | ) 17 | 18 | require ( 19 | github.com/ThreeDotsLabs/watermill-redisstream v1.4.2 20 | github.com/redis/go-redis/v9 v9.7.0 21 | ) 22 | 23 | require ( 24 | github.com/Rican7/retry v0.3.1 // indirect 25 | github.com/ajg/form v1.5.1 // indirect 26 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 27 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 28 | github.com/go-chi/chi v4.1.2+incompatible // indirect 29 | github.com/gogo/protobuf v1.3.2 // indirect 30 | github.com/golang/protobuf v1.5.4 // indirect 31 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 32 | github.com/oklog/ulid v1.3.1 // indirect 33 | github.com/pkg/errors v0.9.1 // indirect 34 | github.com/sony/gobreaker v1.0.0 // indirect 35 | github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect 36 | google.golang.org/appengine v1.6.8 // indirect 37 | google.golang.org/protobuf v1.36.3 // indirect 38 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 39 | ) 40 | -------------------------------------------------------------------------------- /_examples/real-world-examples/delayed-messages/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | volumes: 6 | - .:/app 7 | - $GOPATH/pkg/mod:/go/pkg/mod 8 | working_dir: /app 9 | command: go run main.go 10 | 11 | redis: 12 | image: redis:7 13 | ports: 14 | - 6379:6379 15 | restart: unless-stopped 16 | 17 | postgres: 18 | image: postgres:15 19 | restart: unless-stopped 20 | ports: 21 | - 5432:5432 22 | environment: 23 | POSTGRES_USER: watermill 24 | POSTGRES_DB: watermill 25 | POSTGRES_PASSWORD: "password" 26 | -------------------------------------------------------------------------------- /_examples/real-world-examples/delayed-messages/go.mod: -------------------------------------------------------------------------------- 1 | module delayed-messages 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-redisstream v1.4.2 10 | github.com/ThreeDotsLabs/watermill-sql/v4 v4.0.0-rc.2 11 | github.com/brianvoe/gofakeit/v6 v6.28.0 12 | github.com/google/uuid v1.6.0 13 | github.com/lib/pq v1.10.9 14 | github.com/redis/go-redis/v9 v9.7.0 15 | ) 16 | 17 | require ( 18 | github.com/Rican7/retry v0.3.1 // indirect 19 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 20 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 21 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 22 | github.com/gogo/protobuf v1.3.2 // indirect 23 | github.com/golang/protobuf v1.5.4 // indirect 24 | github.com/hashicorp/errwrap v1.1.0 // indirect 25 | github.com/hashicorp/go-multierror v1.1.1 // indirect 26 | github.com/jackc/pgpassfile v1.0.0 // indirect 27 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 28 | github.com/jackc/pgx/v5 v5.7.2 // indirect 29 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 30 | github.com/oklog/ulid v1.3.1 // indirect 31 | github.com/pkg/errors v0.9.1 // indirect 32 | github.com/sony/gobreaker v1.0.0 // indirect 33 | github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect 34 | golang.org/x/crypto v0.32.0 // indirect 35 | golang.org/x/text v0.21.0 // indirect 36 | google.golang.org/appengine v1.6.8 // indirect 37 | google.golang.org/protobuf v1.36.3 // indirect 38 | ) 39 | -------------------------------------------------------------------------------- /_examples/real-world-examples/delayed-requeue/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | volumes: 6 | - .:/app 7 | - $GOPATH/pkg/mod:/go/pkg/mod 8 | working_dir: /app 9 | command: go run main.go 10 | 11 | redis: 12 | image: redis:7 13 | ports: 14 | - 6379:6379 15 | restart: unless-stopped 16 | 17 | postgres: 18 | image: postgres:15 19 | restart: unless-stopped 20 | ports: 21 | - 5432:5432 22 | environment: 23 | POSTGRES_USER: watermill 24 | POSTGRES_DB: watermill 25 | POSTGRES_PASSWORD: "password" 26 | -------------------------------------------------------------------------------- /_examples/real-world-examples/delayed-requeue/go.mod: -------------------------------------------------------------------------------- 1 | module delayed-requeue 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-redisstream v1.4.2 10 | github.com/ThreeDotsLabs/watermill-sql/v4 v4.0.0-rc.2 11 | github.com/brianvoe/gofakeit/v6 v6.28.0 12 | github.com/lib/pq v1.10.9 13 | github.com/redis/go-redis/v9 v9.7.0 14 | ) 15 | 16 | require ( 17 | github.com/Rican7/retry v0.3.1 // indirect 18 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 19 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 20 | github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect 21 | github.com/go-sql-driver/mysql v1.8.1 // indirect 22 | github.com/gogo/protobuf v1.3.2 // indirect 23 | github.com/golang/protobuf v1.5.4 // indirect 24 | github.com/google/uuid v1.6.0 // indirect 25 | github.com/hashicorp/errwrap v1.1.0 // indirect 26 | github.com/hashicorp/go-multierror v1.1.1 // indirect 27 | github.com/jackc/pgpassfile v1.0.0 // indirect 28 | github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect 29 | github.com/jackc/pgx/v5 v5.7.2 // indirect 30 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 31 | github.com/oklog/ulid v1.3.1 // indirect 32 | github.com/pkg/errors v0.9.1 // indirect 33 | github.com/sony/gobreaker v1.0.0 // indirect 34 | github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect 35 | golang.org/x/crypto v0.32.0 // indirect 36 | golang.org/x/text v0.21.0 // indirect 37 | google.golang.org/appengine v1.6.8 // indirect 38 | google.golang.org/protobuf v1.36.3 // indirect 39 | ) 40 | -------------------------------------------------------------------------------- /_examples/real-world-examples/exactly-once-delivery-counter/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/_examples/real-world-examples/exactly-once-delivery-counter/architecture.jpg -------------------------------------------------------------------------------- /_examples/real-world-examples/exactly-once-delivery-counter/at-least-once-delivery.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/_examples/real-world-examples/exactly-once-delivery-counter/at-least-once-delivery.jpg -------------------------------------------------------------------------------- /_examples/real-world-examples/exactly-once-delivery-counter/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | ports: 6 | - 8080:8080 7 | volumes: 8 | - ./server:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: 'go run .' 12 | 13 | worker: 14 | image: golang:1.23 15 | restart: unless-stopped 16 | volumes: 17 | - ./worker:/app 18 | - $GOPATH/pkg/mod:/go/pkg/mod 19 | working_dir: /app 20 | command: 'go run .' 21 | 22 | mysql: 23 | image: mysql:8.0 24 | restart: unless-stopped 25 | ports: 26 | - 3306:3306 27 | environment: 28 | MYSQL_DATABASE: example 29 | MYSQL_ALLOW_EMPTY_PASSWORD: "yes" 30 | volumes: 31 | - ./schema.sql:/docker-entrypoint-initdb.d/schema.sql 32 | -------------------------------------------------------------------------------- /_examples/real-world-examples/exactly-once-delivery-counter/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE counter ( 2 | id VARCHAR(36) NOT NULL UNIQUE, 3 | value int NOT NULL 4 | ); 5 | -------------------------------------------------------------------------------- /_examples/real-world-examples/exactly-once-delivery-counter/server/go.mod: -------------------------------------------------------------------------------- 1 | module exactly-once-delivery 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-sql/v3 v3.1.0 10 | github.com/go-chi/chi/v5 v5.2.0 11 | github.com/go-sql-driver/mysql v1.8.1 12 | ) 13 | 14 | require ( 15 | filippo.io/edwards25519 v1.1.0 // indirect 16 | github.com/google/uuid v1.6.0 // indirect 17 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 18 | github.com/oklog/ulid v1.3.1 // indirect 19 | github.com/pkg/errors v0.9.1 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /_examples/real-world-examples/exactly-once-delivery-counter/worker/go.mod: -------------------------------------------------------------------------------- 1 | module exactly-once-delivery 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-sql/v3 v3.1.0 10 | github.com/go-sql-driver/mysql v1.8.1 11 | github.com/pkg/errors v0.9.1 12 | ) 13 | 14 | require ( 15 | filippo.io/edwards25519 v1.1.0 // indirect 16 | github.com/google/uuid v1.6.0 // indirect 17 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 18 | github.com/oklog/ulid v1.3.1 // indirect 19 | ) 20 | -------------------------------------------------------------------------------- /_examples/real-world-examples/persistent-event-log/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "received event" 5 | -------------------------------------------------------------------------------- /_examples/real-world-examples/persistent-event-log/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - mysql 7 | - googlecloud 8 | volumes: 9 | - .:/app 10 | - $GOPATH/pkg/mod:/go/pkg/mod 11 | working_dir: /app 12 | environment: 13 | PUBSUB_EMULATOR_HOST: googlecloud:8085 14 | command: go run main.go 15 | 16 | mysql: 17 | image: mysql:8.0 18 | logging: 19 | driver: none 20 | restart: unless-stopped 21 | ports: 22 | - 3306:3306 23 | environment: 24 | MYSQL_DATABASE: watermill 25 | MYSQL_ALLOW_EMPTY_PASSWORD: "yes" 26 | 27 | googlecloud: 28 | image: google/cloud-sdk:228.0.0 29 | logging: 30 | driver: none 31 | entrypoint: gcloud --quiet beta emulators pubsub start --host-port=0.0.0.0:8085 --verbosity=debug --log-http 32 | ports: 33 | - 8085:8085 34 | environment: 35 | PUBSUB_EMULATOR_HOST: googlecloud:8085 36 | restart: unless-stopped 37 | -------------------------------------------------------------------------------- /_examples/real-world-examples/receiving-webhooks/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "Starting handler" 5 | -------------------------------------------------------------------------------- /_examples/real-world-examples/receiving-webhooks/README.md: -------------------------------------------------------------------------------- 1 | # Receiving webhooks (HTTP to Kafka) 2 | 3 | This example showcases the use of the **HTTP Subscriber** to receive webhooks with HTTP POST requests. 4 | 5 | Received messages are then published to a Kafka topic. 6 | 7 | ## Requirements 8 | 9 | To run this example you will need Docker and docker-compose installed. See installation guide at https://docs.docker.com/compose/install/ 10 | 11 | ## Running 12 | 13 | To run all services, execute: 14 | 15 | ``` 16 | docker-compose up 17 | ``` 18 | -------------------------------------------------------------------------------- /_examples/real-world-examples/receiving-webhooks/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | golang: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | ports: 6 | - 8080:8080 7 | depends_on: 8 | - kafka 9 | volumes: 10 | - .:/app 11 | - $GOPATH/pkg/mod:/go/pkg/mod 12 | working_dir: /app 13 | command: go run main.go -kafka kafka:9092 -http :8080 14 | 15 | zookeeper: 16 | image: confluentinc/cp-zookeeper:7.3.1 17 | restart: unless-stopped 18 | environment: 19 | ZOOKEEPER_CLIENT_PORT: 2181 20 | logging: 21 | driver: none 22 | 23 | kafka: 24 | image: confluentinc/cp-kafka:7.3.1 25 | restart: unless-stopped 26 | logging: 27 | driver: none 28 | depends_on: 29 | - zookeeper 30 | environment: 31 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 32 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 33 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 34 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 35 | 36 | -------------------------------------------------------------------------------- /_examples/real-world-examples/sending-webhooks/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "POST /foo_or_bar: message" 5 | -------------------------------------------------------------------------------- /_examples/real-world-examples/sending-webhooks/README.md: -------------------------------------------------------------------------------- 1 | # Sending webhooks (Kafka to HTTP) 2 | 3 | This example showcases the use of the **HTTP Publisher** to call webhooks with HTTP POST requests. It consists of three services: 4 | 5 | 1. `producer` publishes messages on Kafka. The messages come in three varieties: `Foo`, `Bar`, and `Baz`. The event type is encoded in metadata, under the key `event_type`. 6 | 1. `webhook_server` is a HTTP server that listens for requests and prints the path, method, and payload on stdout. 7 | 1. `router` consumes the Kafka messages, and uses the HTTP producer to send requests to `webhook_server`. To illustrate how one message can spawn multiple webhooks, the following paths are called based on `event_type`: 8 | 1. `/foo` for events of type `Foo` 9 | 1. `/foo_or_bar` for events of type `Foo` or `Bar` 10 | 1. `/all` for all events. 11 | 12 | Additionally, services `zookeeper` and `kafka` are present to provide backend for the Kafka producer and subscriber. 13 | 14 | ## Requirements 15 | 16 | To run this example you will need Docker and docker-compose installed. See installation guide at https://docs.docker.com/compose/install/ 17 | 18 | ## Running 19 | 20 | To run all services, execute: 21 | 22 | ``` 23 | docker-compose up 24 | ``` 25 | 26 | To filter messages from a specific service, execute: 27 | 28 | ``` 29 | docker-compose logs [-f] {service} 30 | ``` 31 | 32 | in a separate terminal window while the services are running. Use the `-f` flag to emulate `tail -f` behavior, i.e. follow the output. 33 | -------------------------------------------------------------------------------- /_examples/real-world-examples/sending-webhooks/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | webhooks-server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | volumes: 6 | - .:/app 7 | - $GOPATH/pkg/mod:/go/pkg/mod 8 | working_dir: /app/webhooks-server/ 9 | command: go run main.go 10 | 11 | router: 12 | image: golang:1.23 13 | restart: unless-stopped 14 | depends_on: 15 | - kafka 16 | volumes: 17 | - .:/app 18 | - $GOPATH/pkg/mod:/go/pkg/mod 19 | working_dir: /app/router/ 20 | command: go run main.go 21 | 22 | producer: 23 | image: golang:1.23 24 | restart: unless-stopped 25 | depends_on: 26 | - kafka 27 | - webhooks-server 28 | - router 29 | volumes: 30 | - .:/app 31 | - $GOPATH/pkg/mod:/go/pkg/mod 32 | working_dir: /app/producer/ 33 | command: go run main.go 34 | 35 | zookeeper: 36 | image: confluentinc/cp-zookeeper:7.3.1 37 | restart: unless-stopped 38 | environment: 39 | ZOOKEEPER_CLIENT_PORT: 2181 40 | logging: 41 | driver: none 42 | 43 | kafka: 44 | image: confluentinc/cp-kafka:7.3.1 45 | restart: unless-stopped 46 | logging: 47 | driver: none 48 | depends_on: 49 | - zookeeper 50 | environment: 51 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 52 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 53 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 54 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 55 | -------------------------------------------------------------------------------- /_examples/real-world-examples/sending-webhooks/producer/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "time" 7 | 8 | "github.com/ThreeDotsLabs/watermill" 9 | "github.com/ThreeDotsLabs/watermill-kafka/v3/pkg/kafka" 10 | "github.com/ThreeDotsLabs/watermill/message" 11 | ) 12 | 13 | var ( 14 | brokers = []string{"kafka:9092"} 15 | logger = watermill.NewStdLogger(false, false) 16 | ) 17 | 18 | type eventType string 19 | 20 | const ( 21 | Foo eventType = "Foo" 22 | Bar eventType = "Bar" 23 | Baz eventType = "Baz" 24 | ) 25 | 26 | func main() { 27 | pub, err := kafka.NewPublisher( 28 | kafka.PublisherConfig{ 29 | Brokers: brokers, 30 | Marshaler: kafka.DefaultMarshaler{}, 31 | }, 32 | logger, 33 | ) 34 | if err != nil { 35 | panic(err) 36 | } 37 | 38 | eventTypes := []eventType{Foo, Bar, Baz} 39 | 40 | for { 41 | eventType := eventTypes[rand.Intn(3)] 42 | msg := message.NewMessage(watermill.NewUUID(), []byte("message")) 43 | msg.Metadata.Set("event_type", string(eventType)) 44 | 45 | fmt.Printf("%s Publishing %s\n\n", time.Now().String(), eventType) 46 | if err := pub.Publish("kafka_to_http_example", msg); err != nil { 47 | panic(err) 48 | } 49 | time.Sleep(time.Second) 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /_examples/real-world-examples/sending-webhooks/webhooks-server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "time" 8 | ) 9 | 10 | // handler receives the webhook requests and logs them in stdout. 11 | func handler(w http.ResponseWriter, r *http.Request) { 12 | body, err := ioutil.ReadAll(r.Body) 13 | if err != nil { 14 | w.WriteHeader(http.StatusBadRequest) 15 | return 16 | } 17 | fmt.Printf( 18 | "[%s] %s %s: %s\n\n", 19 | time.Now().String(), 20 | r.Method, 21 | r.URL.String(), 22 | string(body), 23 | ) 24 | w.WriteHeader(http.StatusOK) 25 | } 26 | 27 | func main() { 28 | http.HandleFunc("/", handler) 29 | http.ListenAndServe(":8001", http.DefaultServeMux) 30 | } 31 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events-htmx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23 AS builder 2 | 3 | COPY . /src 4 | WORKDIR /src/ 5 | 6 | RUN CGO_ENABLED=0 go build -ldflags="-s -w" -trimpath -o /main . 7 | 8 | FROM alpine 9 | RUN apk add --no-cache ca-certificates 10 | COPY --from=builder /main /main 11 | CMD ["/main"] 12 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events-htmx/README.md: -------------------------------------------------------------------------------- 1 | # Server Sent Events (htmx) 2 | 3 | This is an example project described in [Live website updates with Go, SSE, and htmx](https://threedots.tech/post/live-website-updates-go-sse-htmx/). 4 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events-htmx/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | build: 4 | context: docker 5 | volumes: 6 | - ./:/src 7 | - go_pkg:/go/pkg 8 | - go_cache:/go-cache 9 | working_dir: /src 10 | ports: 11 | - '8080:8080' 12 | environment: 13 | - PORT=8080 14 | - DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable 15 | - PUBSUB_PROJECT_ID=local 16 | - PUBSUB_EMULATOR_HOST=pubsub:8681 17 | restart: unless-stopped 18 | networks: 19 | - sse 20 | 21 | postgres: 22 | image: postgres:15 23 | restart: unless-stopped 24 | environment: 25 | - POSTGRES_PASSWORD=postgres 26 | ports: 27 | - 5432:5432 28 | networks: 29 | - sse 30 | 31 | pubsub: 32 | image: messagebird/gcloud-pubsub-emulator:latest 33 | restart: unless-stopped 34 | ports: 35 | - '8681:8681' 36 | networks: 37 | - sse 38 | 39 | networks: 40 | sse: 41 | 42 | volumes: 43 | go_pkg: 44 | go_cache: 45 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events-htmx/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.23 2 | 3 | RUN go install github.com/cespare/reflex@latest 4 | RUN go install github.com/a-h/templ/cmd/templ@latest 5 | 6 | COPY reflex.conf / 7 | 8 | ENTRYPOINT ["/go/bin/reflex", "-c", "/reflex.conf"] 9 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events-htmx/docker/reflex.conf: -------------------------------------------------------------------------------- 1 | -r '(\.go$|go\.mod$)' -s go run . 2 | -r '\.templ$' templ generate 3 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events-htmx/models.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "time" 4 | 5 | var allReactions = []Reaction{ 6 | { 7 | ID: "fire", 8 | Label: "🔥", 9 | }, 10 | { 11 | ID: "thinking", 12 | Label: "🤔", 13 | }, 14 | { 15 | ID: "heart", 16 | Label: "🩵", 17 | }, 18 | { 19 | ID: "laugh", 20 | Label: "😂", 21 | }, 22 | { 23 | ID: "sad", 24 | Label: "😢", 25 | }, 26 | } 27 | 28 | func mustReactionByID(id string) Reaction { 29 | for _, r := range allReactions { 30 | if r.ID == id { 31 | return r 32 | } 33 | } 34 | 35 | panic("reaction not found") 36 | } 37 | 38 | type Reaction struct { 39 | ID string 40 | Label string 41 | } 42 | 43 | type Post struct { 44 | ID int 45 | Author string 46 | Content string 47 | CreatedAt time.Time 48 | Views int 49 | Reactions map[string]int 50 | } 51 | 52 | type PostStats struct { 53 | ID int 54 | Views int 55 | ViewsUpdated bool 56 | Reactions map[string]int 57 | ReactionUpdated *string 58 | } 59 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events/diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/_examples/real-world-examples/server-sent-events/diagram.jpg -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | ports: 6 | - 8080:8080 7 | volumes: 8 | - ./server:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: 'go run .' 12 | 13 | mysql: 14 | image: mysql:8.0 15 | restart: unless-stopped 16 | environment: 17 | MYSQL_DATABASE: example 18 | MYSQL_ALLOW_EMPTY_PASSWORD: "yes" 19 | volumes: 20 | - ./schema.sql:/docker-entrypoint-initdb.d/schema.sql 21 | 22 | mongo: 23 | image: mongo:3.6 24 | restart: unless-stopped 25 | environment: 26 | MONGO_INITDB_ROOT_USERNAME: root 27 | MONGO_INITDB_ROOT_PASSWORD: password 28 | 29 | nats-streaming: 30 | image: nats-streaming:0.11.2 31 | restart: unless-stopped 32 | logging: 33 | driver: none 34 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE example.posts ( 2 | id VARCHAR(36) NOT NULL PRIMARY KEY, 3 | title VARCHAR(255) NOT NULL DEFAULT '', 4 | content TEXT NOT NULL, 5 | author VARCHAR(255) NOT NULL DEFAULT '' 6 | ); 7 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events/screen.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/_examples/real-world-examples/server-sent-events/screen.gif -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/_examples/real-world-examples/server-sent-events/screenshot.png -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events/server/diagram.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/_examples/real-world-examples/server-sent-events/server/diagram.jpg -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events/server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/ThreeDotsLabs/watermill" 7 | ) 8 | 9 | func main() { 10 | logger := watermill.NewStdLogger(false, false) 11 | 12 | postsStorage := NewPostsStorage() 13 | feedsStorage := NewFeedsStorage() 14 | 15 | pub, sub, err := SetupMessageRouter(feedsStorage, logger) 16 | if err != nil { 17 | panic(err) 18 | } 19 | 20 | httpRouter := Router{ 21 | Subscriber: sub, 22 | Publisher: Publisher{publisher: pub}, 23 | PostsStorage: postsStorage, 24 | FeedsStorage: feedsStorage, 25 | Logger: logger, 26 | } 27 | 28 | mux := httpRouter.Mux() 29 | 30 | err = http.ListenAndServe(":8080", mux) 31 | if err != nil { 32 | panic(err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /_examples/real-world-examples/server-sent-events/server/posts_storage.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/go-sql-driver/mysql" 10 | ) 11 | 12 | type PostsStorage struct { 13 | db *sql.DB 14 | } 15 | 16 | func NewPostsStorage() PostsStorage { 17 | conf := mysql.NewConfig() 18 | conf.Net = "tcp" 19 | conf.User = "root" 20 | conf.Addr = "mysql" 21 | conf.DBName = "example" 22 | 23 | db, err := sql.Open("mysql", conf.FormatDSN()) 24 | if err != nil { 25 | panic(err) 26 | } 27 | 28 | for { 29 | err = db.Ping() 30 | if err == nil { 31 | break 32 | } else { 33 | fmt.Println("Could not connect to MySQL, retrying...") 34 | time.Sleep(time.Second * 3) 35 | } 36 | } 37 | 38 | return PostsStorage{ 39 | db: db, 40 | } 41 | } 42 | 43 | func (s PostsStorage) ByID(ctx context.Context, id string) (Post, error) { 44 | query := "SELECT title, content, author FROM posts WHERE id=?" 45 | row := s.db.QueryRowContext(ctx, query, id) 46 | 47 | var title, content, author string 48 | err := row.Scan(&title, &content, &author) 49 | if err != nil { 50 | return Post{}, err 51 | } 52 | 53 | return NewPost(id, title, content, author), nil 54 | } 55 | 56 | func (s PostsStorage) Add(ctx context.Context, post Post) error { 57 | query := "INSERT INTO posts (id, title, content, author) VALUES (?, ?, ?, ?)" 58 | _, err := s.db.ExecContext(ctx, query, post.ID, post.Title, post.Content, post.Author) 59 | return err 60 | } 61 | 62 | func (s PostsStorage) Update(ctx context.Context, post Post) error { 63 | query := "UPDATE posts SET title=?, content=?, author=? WHERE id=?" 64 | _, err := s.db.ExecContext(ctx, query, post.Title, post.Content, post.Author, post.ID) 65 | return err 66 | } 67 | -------------------------------------------------------------------------------- /_examples/real-world-examples/synchronizing-databases/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "received user:" 5 | -------------------------------------------------------------------------------- /_examples/real-world-examples/synchronizing-databases/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - mysql 7 | - postgres 8 | volumes: 9 | - .:/app 10 | - $GOPATH/pkg/mod:/go/pkg/mod 11 | working_dir: /app 12 | command: go run . 13 | 14 | mysql: 15 | image: mysql:8.0 16 | restart: unless-stopped 17 | ports: 18 | - 3306:3306 19 | environment: 20 | MYSQL_DATABASE: watermill 21 | MYSQL_ALLOW_EMPTY_PASSWORD: "yes" 22 | 23 | postgres: 24 | image: postgres:11 25 | restart: unless-stopped 26 | ports: 27 | - 5432:5432 28 | environment: 29 | POSTGRES_USER: watermill 30 | POSTGRES_DB: watermill 31 | POSTGRES_PASSWORD: "password" 32 | -------------------------------------------------------------------------------- /_examples/real-world-examples/synchronizing-databases/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/ThreeDotsLabs/watermill v1.4.4 9 | github.com/ThreeDotsLabs/watermill-sql/v3 v3.1.0 10 | github.com/brianvoe/gofakeit/v6 v6.28.0 11 | github.com/go-sql-driver/mysql v1.8.1 12 | github.com/lib/pq v1.10.9 13 | ) 14 | 15 | require ( 16 | filippo.io/edwards25519 v1.1.0 // indirect 17 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 18 | github.com/google/uuid v1.6.0 // indirect 19 | github.com/hashicorp/errwrap v1.1.0 // indirect 20 | github.com/hashicorp/go-multierror v1.1.1 // indirect 21 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 22 | github.com/oklog/ulid v1.3.1 // indirect 23 | github.com/pkg/errors v0.9.1 // indirect 24 | github.com/sony/gobreaker v1.0.0 // indirect 25 | ) 26 | -------------------------------------------------------------------------------- /_examples/real-world-examples/transactional-events-forwarder/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "Sending a prize to the winner" 5 | -------------------------------------------------------------------------------- /_examples/real-world-examples/transactional-events-forwarder/README.md: -------------------------------------------------------------------------------- 1 | # Publishing events in transactions with help of Forwarder component (MySQL to Google Pub/Sub) 2 | 3 | While working with an event-driven application, you may in some point need to store an application state and publish a message 4 | telling the rest of the system about what just happened. As it may look trivial at a first glance, it could become 5 | a bit tricky if we consider what can go wrong in case we won't pay enough attention to details. 6 | 7 | ## Solution 8 | 9 | This example presents a solution to this problem: saving events in transaction along with persisting application state. 10 | It also compares two other approaches which lack transactional publishing therefore expose application to a risk 11 | of inconsistency across the system. 12 | 13 | ## Requirements 14 | 15 | To run this example you will need Docker and docker-compose installed. See installation guide at https://docs.docker.com/compose/install/ 16 | 17 | ## Running 18 | 19 | ```bash 20 | docker-compose up 21 | ``` 22 | -------------------------------------------------------------------------------- /_examples/real-world-examples/transactional-events-forwarder/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | environment: 5 | - PUBSUB_EMULATOR_HOST=googlecloud:8085 6 | depends_on: 7 | - mysql 8 | - googlecloud 9 | restart: unless-stopped 10 | volumes: 11 | - .:/app 12 | - $GOPATH/pkg/mod:/go/pkg/mod 13 | working_dir: /app 14 | command: go run . 15 | 16 | mysql: 17 | image: mysql:8.0 18 | restart: unless-stopped 19 | logging: 20 | driver: none 21 | ports: 22 | - 3306:3306 23 | environment: 24 | MYSQL_DATABASE: watermill 25 | MYSQL_ALLOW_EMPTY_PASSWORD: "yes" 26 | 27 | googlecloud: 28 | image: google/cloud-sdk:414.0.0 29 | logging: 30 | driver: none 31 | entrypoint: gcloud --quiet beta emulators pubsub start --host-port=0.0.0.0:8085 --verbosity=debug --log-http 32 | ports: 33 | - 8085:8085 34 | restart: unless-stopped 35 | -------------------------------------------------------------------------------- /_examples/real-world-examples/transactional-events/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 180 4 | expected_output: "received event" 5 | -------------------------------------------------------------------------------- /_examples/real-world-examples/transactional-events/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - mysql 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: > 12 | /bin/sh -c "go install github.com/ThreeDotsLabs/watermill/tools/mill@latest && 13 | go run main.go" 14 | 15 | mysql: 16 | image: mysql:8.0 17 | restart: unless-stopped 18 | ports: 19 | - 3306:3306 20 | environment: 21 | MYSQL_DATABASE: watermill 22 | MYSQL_ALLOW_EMPTY_PASSWORD: "yes" 23 | 24 | zookeeper: 25 | image: confluentinc/cp-zookeeper:7.3.1 26 | logging: 27 | driver: none 28 | restart: unless-stopped 29 | environment: 30 | ZOOKEEPER_CLIENT_PORT: 2181 31 | 32 | kafka: 33 | image: confluentinc/cp-kafka:7.3.1 34 | logging: 35 | driver: none 36 | restart: unless-stopped 37 | depends_on: 38 | - zookeeper 39 | environment: 40 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 41 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 42 | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 43 | KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" 44 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - "pubsub/tests" # test helpers used to test Pub/Subs 3 | 4 | comment: no # do not comment PR with the result 5 | 6 | coverage: 7 | precision: 0 8 | status: 9 | patch: false # do not run coverage on patch nor changes 10 | project: 11 | default: 12 | target: auto 13 | threshold: 5% 14 | -------------------------------------------------------------------------------- /components/cqrs/command_handler_test.go: -------------------------------------------------------------------------------- 1 | package cqrs_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/ThreeDotsLabs/watermill/components/cqrs" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | type SomeCommand struct { 13 | Foo string 14 | } 15 | 16 | func TestNewCommandHandler(t *testing.T) { 17 | cmdToSend := &SomeCommand{"bar"} 18 | 19 | ch := cqrs.NewCommandHandler( 20 | "some_handler", 21 | func(ctx context.Context, cmd *SomeCommand) error { 22 | assert.Equal(t, cmdToSend, cmd) 23 | return fmt.Errorf("some error") 24 | }, 25 | ) 26 | 27 | assert.Equal(t, "some_handler", ch.HandlerName()) 28 | assert.Equal(t, &SomeCommand{}, ch.NewCommand()) 29 | 30 | err := ch.Handle(context.Background(), cmdToSend) 31 | assert.EqualError(t, err, "some error") 32 | } 33 | -------------------------------------------------------------------------------- /components/cqrs/ctx.go: -------------------------------------------------------------------------------- 1 | package cqrs 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | ) 8 | 9 | type ctxKey string 10 | 11 | const ( 12 | originalMessage ctxKey = "original_message" 13 | ) 14 | 15 | // OriginalMessageFromCtx returns the original message that was received by the event/command handler. 16 | func OriginalMessageFromCtx(ctx context.Context) *message.Message { 17 | val, ok := ctx.Value(originalMessage).(*message.Message) 18 | if !ok { 19 | return nil 20 | } 21 | return val 22 | } 23 | 24 | // CtxWithOriginalMessage returns a new context with the original message attached. 25 | func CtxWithOriginalMessage(ctx context.Context, msg *message.Message) context.Context { 26 | return context.WithValue(ctx, originalMessage, msg) 27 | } 28 | -------------------------------------------------------------------------------- /components/cqrs/doc.go: -------------------------------------------------------------------------------- 1 | // Detailed CQRS documentation can be found in https://watermill.io/docs/cqrs/ 2 | 3 | package cqrs 4 | -------------------------------------------------------------------------------- /components/cqrs/event_handler_test.go: -------------------------------------------------------------------------------- 1 | package cqrs_test 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/ThreeDotsLabs/watermill/components/cqrs" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | type SomeEvent struct { 13 | Foo string 14 | } 15 | 16 | func TestNewEventHandler(t *testing.T) { 17 | cmdToSend := &SomeEvent{"bar"} 18 | 19 | ch := cqrs.NewEventHandler( 20 | "some_handler", 21 | func(ctx context.Context, cmd *SomeEvent) error { 22 | assert.Equal(t, cmdToSend, cmd) 23 | return fmt.Errorf("some error") 24 | }, 25 | ) 26 | 27 | assert.Equal(t, "some_handler", ch.HandlerName()) 28 | assert.Equal(t, &SomeEvent{}, ch.NewEvent()) 29 | 30 | err := ch.Handle(context.Background(), cmdToSend) 31 | assert.EqualError(t, err, "some error") 32 | } 33 | 34 | func TestNewGroupEventHandler(t *testing.T) { 35 | cmdToSend := &SomeEvent{"bar"} 36 | 37 | ch := cqrs.NewGroupEventHandler( 38 | func(ctx context.Context, cmd *SomeEvent) error { 39 | assert.Equal(t, cmdToSend, cmd) 40 | return fmt.Errorf("some error") 41 | }, 42 | ) 43 | 44 | assert.Equal(t, &SomeEvent{}, ch.NewEvent()) 45 | 46 | err := ch.Handle(context.Background(), cmdToSend) 47 | assert.EqualError(t, err, "some error") 48 | } 49 | -------------------------------------------------------------------------------- /components/cqrs/marshaler.go: -------------------------------------------------------------------------------- 1 | package cqrs 2 | 3 | import ( 4 | "github.com/ThreeDotsLabs/watermill/message" 5 | ) 6 | 7 | // CommandEventMarshaler marshals Commands and Events to Watermill's messages and vice versa. 8 | // Payload of the command needs to be marshaled to []bytes. 9 | type CommandEventMarshaler interface { 10 | // Marshal marshals Command or Event to Watermill's message. 11 | Marshal(v interface{}) (*message.Message, error) 12 | 13 | // Unmarshal unmarshals watermill's message to v Command or Event. 14 | Unmarshal(msg *message.Message, v interface{}) (err error) 15 | 16 | // Name returns the name of Command or Event. 17 | // Name is used to determine, that received command or event is event which we want to handle. 18 | Name(v interface{}) string 19 | 20 | // NameFromMessage returns the name of Command or Event from Watermill's message (generated by Marshal). 21 | // 22 | // When we have Command or Event marshaled to Watermill's message, 23 | // we should use NameFromMessage instead of Name to avoid unnecessary unmarshaling. 24 | NameFromMessage(msg *message.Message) string 25 | } 26 | -------------------------------------------------------------------------------- /components/cqrs/marshaler_json.go: -------------------------------------------------------------------------------- 1 | package cqrs 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/ThreeDotsLabs/watermill" 7 | "github.com/ThreeDotsLabs/watermill/message" 8 | ) 9 | 10 | type JSONMarshaler struct { 11 | NewUUID func() string 12 | GenerateName func(v interface{}) string 13 | } 14 | 15 | func (m JSONMarshaler) Marshal(v interface{}) (*message.Message, error) { 16 | b, err := json.Marshal(v) 17 | if err != nil { 18 | return nil, err 19 | } 20 | 21 | msg := message.NewMessage( 22 | m.newUUID(), 23 | b, 24 | ) 25 | msg.Metadata.Set("name", m.Name(v)) 26 | 27 | return msg, nil 28 | } 29 | 30 | func (m JSONMarshaler) newUUID() string { 31 | if m.NewUUID != nil { 32 | return m.NewUUID() 33 | } 34 | 35 | // default 36 | return watermill.NewUUID() 37 | } 38 | 39 | func (JSONMarshaler) Unmarshal(msg *message.Message, v interface{}) (err error) { 40 | return json.Unmarshal(msg.Payload, v) 41 | } 42 | 43 | func (m JSONMarshaler) Name(cmdOrEvent interface{}) string { 44 | if m.GenerateName != nil { 45 | return m.GenerateName(cmdOrEvent) 46 | } 47 | 48 | return FullyQualifiedStructName(cmdOrEvent) 49 | } 50 | 51 | func (m JSONMarshaler) NameFromMessage(msg *message.Message) string { 52 | return msg.Metadata.Get("name") 53 | } 54 | -------------------------------------------------------------------------------- /components/cqrs/marshaler_json_test.go: -------------------------------------------------------------------------------- 1 | package cqrs_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/ThreeDotsLabs/watermill" 11 | "github.com/ThreeDotsLabs/watermill/components/cqrs" 12 | ) 13 | 14 | var jsonEventToMarshal = TestEvent{ 15 | ID: watermill.NewULID(), 16 | When: time.Date(2016, time.August, 15, 14, 13, 12, 0, time.UTC), 17 | } 18 | 19 | func TestJsonMarshaler(t *testing.T) { 20 | marshaler := cqrs.JSONMarshaler{} 21 | 22 | msg, err := marshaler.Marshal(jsonEventToMarshal) 23 | require.NoError(t, err) 24 | 25 | eventToUnmarshal := TestEvent{} 26 | err = marshaler.Unmarshal(msg, &eventToUnmarshal) 27 | require.NoError(t, err) 28 | 29 | assert.EqualValues(t, jsonEventToMarshal, eventToUnmarshal) 30 | } 31 | 32 | func TestJSONMarshaler_Marshal_new_uuid_set(t *testing.T) { 33 | marshaler := cqrs.JSONMarshaler{ 34 | NewUUID: func() string { 35 | return "foo" 36 | }, 37 | } 38 | 39 | msg, err := marshaler.Marshal(jsonEventToMarshal) 40 | require.NoError(t, err) 41 | 42 | assert.Equal(t, msg.UUID, "foo") 43 | } 44 | 45 | func TestJSONMarshaler_Marshal_generate_name(t *testing.T) { 46 | marshaler := cqrs.JSONMarshaler{ 47 | GenerateName: func(v interface{}) string { 48 | return "foo" 49 | }, 50 | } 51 | 52 | msg, err := marshaler.Marshal(jsonEventToMarshal) 53 | require.NoError(t, err) 54 | 55 | assert.Equal(t, msg.Metadata.Get("name"), "foo") 56 | } 57 | -------------------------------------------------------------------------------- /components/cqrs/name.go: -------------------------------------------------------------------------------- 1 | package cqrs 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // FullyQualifiedStructName name returns object name in format [package].[type name]. 9 | // For example, for the struct: 10 | // 11 | // package events 12 | // type UserCreated struct {} 13 | // 14 | // it will return "events.UserCreated". 15 | // 16 | // It ignores if the value is a pointer or not. 17 | func FullyQualifiedStructName(v interface{}) string { 18 | s := fmt.Sprintf("%T", v) 19 | s = strings.TrimLeft(s, "*") 20 | 21 | return s 22 | } 23 | 24 | // StructName name returns struct name in format [type name]. 25 | // For example, for the struct: 26 | // 27 | // package events 28 | // type UserCreated struct {} 29 | // 30 | // it will return "UserCreated". 31 | // 32 | // It ignores if the value is a pointer or not. 33 | func StructName(v interface{}) string { 34 | segments := strings.Split(fmt.Sprintf("%T", v), ".") 35 | 36 | return segments[len(segments)-1] 37 | } 38 | 39 | type namedStruct interface { 40 | Name() string 41 | } 42 | 43 | // NamedStruct returns the name from a message implementing the following interface: 44 | // 45 | // type namedStruct interface { 46 | // Name() string 47 | // } 48 | // 49 | // It ignores if the value is a pointer or not. 50 | func NamedStruct(fallback func(v interface{}) string) func(v interface{}) string { 51 | return func(v interface{}) string { 52 | if v, ok := v.(namedStruct); ok { 53 | return v.Name() 54 | } 55 | 56 | return fallback(v) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /components/cqrs/name_test.go: -------------------------------------------------------------------------------- 1 | package cqrs_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/ThreeDotsLabs/watermill/components/cqrs" 9 | ) 10 | 11 | func TestFullyQualifiedStructName(t *testing.T) { 12 | type Object struct{} 13 | 14 | assert.Equal(t, "cqrs_test.Object", cqrs.FullyQualifiedStructName(Object{})) 15 | assert.Equal(t, "cqrs_test.Object", cqrs.FullyQualifiedStructName(&Object{})) 16 | } 17 | 18 | func BenchmarkFullyQualifiedStructName(b *testing.B) { 19 | type Object struct{} 20 | o := Object{} 21 | 22 | for i := 0; i < b.N; i++ { 23 | cqrs.FullyQualifiedStructName(o) 24 | } 25 | } 26 | 27 | func TestStructName(t *testing.T) { 28 | type Object struct{} 29 | 30 | assert.Equal(t, "Object", cqrs.StructName(Object{})) 31 | assert.Equal(t, "Object", cqrs.StructName(&Object{})) 32 | } 33 | 34 | func TestNamedStruct(t *testing.T) { 35 | assert.Equal(t, "named object", cqrs.NamedStruct(cqrs.StructName)(namedObject{})) 36 | assert.Equal(t, "named object", cqrs.NamedStruct(cqrs.StructName)(&namedObject{})) 37 | 38 | // Test fallback 39 | type Object struct{} 40 | 41 | assert.Equal(t, "Object", cqrs.NamedStruct(cqrs.StructName)(Object{})) 42 | assert.Equal(t, "Object", cqrs.NamedStruct(cqrs.StructName)(&Object{})) 43 | } 44 | 45 | type namedObject struct{} 46 | 47 | func (namedObject) Name() string { 48 | return "named object" 49 | } 50 | -------------------------------------------------------------------------------- /components/cqrs/object.go: -------------------------------------------------------------------------------- 1 | package cqrs 2 | 3 | import ( 4 | "reflect" 5 | ) 6 | 7 | func isPointer(v interface{}) error { 8 | rv := reflect.ValueOf(v) 9 | 10 | if rv.Kind() != reflect.Ptr || rv.IsNil() { 11 | return NonPointerError{rv.Type()} 12 | } 13 | 14 | return nil 15 | } 16 | 17 | type NonPointerError struct { 18 | Type reflect.Type 19 | } 20 | 21 | func (e NonPointerError) Error() string { 22 | return "non-pointer command: " + e.Type.String() + ", handler.NewCommand() should return pointer to the command" 23 | } 24 | -------------------------------------------------------------------------------- /components/cqrs/testdata/events.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package cqrs_test; 3 | option go_package = "./cqrs_test"; 4 | 5 | import "google/protobuf/timestamp.proto"; 6 | 7 | message TestProtobufLegacyEvent { 8 | string id = 1; 9 | google.protobuf.Timestamp when = 3; 10 | } 11 | 12 | enum Status { 13 | STATUS_UNSPECIFIED = 0; 14 | ACTIVE = 1; 15 | DELETED = 2; 16 | } 17 | 18 | message SubEvent { 19 | repeated string tags = 1; 20 | map flags = 2; 21 | } 22 | 23 | message TestComplexProtobufEvent { 24 | string id = 1; 25 | bytes data = 2; 26 | google.protobuf.Timestamp when = 3; 27 | 28 | map nested_map = 4; 29 | repeated SubEvent events = 5; 30 | 31 | oneof result { 32 | SubEvent success = 6; 33 | string error = 7; 34 | Status fallback = 8; 35 | } 36 | 37 | reserved 23 to 30; 38 | } 39 | -------------------------------------------------------------------------------- /components/forwarder/envelope_test.go: -------------------------------------------------------------------------------- 1 | package forwarder 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/ThreeDotsLabs/watermill" 8 | "github.com/ThreeDotsLabs/watermill/message" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | type contextKey string 14 | 15 | func TestEnvelope(t *testing.T) { 16 | expectedUUID := watermill.NewUUID() 17 | expectedPayload := message.Payload("msg content") 18 | expectedMetadata := message.Metadata{"key": "value"} 19 | expectedDestinationTopic := "dest_topic" 20 | 21 | ctx := context.WithValue(context.Background(), contextKey("key"), "value") 22 | 23 | msg := message.NewMessage(expectedUUID, expectedPayload) 24 | msg.Metadata = expectedMetadata 25 | msg.SetContext(ctx) 26 | 27 | wrappedMsg, err := wrapMessageInEnvelope(expectedDestinationTopic, msg) 28 | require.NoError(t, err) 29 | require.NotNil(t, wrappedMsg) 30 | v, ok := wrappedMsg.Context().Value(contextKey("key")).(string) 31 | require.True(t, ok) 32 | require.Equal(t, "value", v) 33 | 34 | destinationTopic, unwrappedMsg, err := unwrapMessageFromEnvelope(wrappedMsg) 35 | require.NoError(t, err) 36 | require.NotNil(t, unwrappedMsg) 37 | assert.Equal(t, expectedUUID, unwrappedMsg.UUID) 38 | assert.Equal(t, expectedPayload, unwrappedMsg.Payload) 39 | assert.Equal(t, expectedMetadata, unwrappedMsg.Metadata) 40 | assert.Equal(t, expectedDestinationTopic, destinationTopic) 41 | 42 | v, ok = unwrappedMsg.Context().Value(contextKey("key")).(string) 43 | require.True(t, ok) 44 | require.Equal(t, "value", v) 45 | } 46 | -------------------------------------------------------------------------------- /components/metrics/ctx.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import "context" 4 | 5 | type contextValue int 6 | 7 | const ( 8 | publishObserved contextValue = iota 9 | subscribeObserved 10 | ) 11 | 12 | // setPublishObservedToCtx is used to achieve metrics idempotency in case of double applied middleware 13 | func setPublishObservedToCtx(ctx context.Context) context.Context { 14 | return context.WithValue(ctx, publishObserved, true) 15 | } 16 | 17 | func publishAlreadyObserved(ctx context.Context) bool { 18 | return ctx.Value(publishObserved) != nil 19 | } 20 | 21 | // setSubscribeObservedToCtx is used to achieve metrics idempotency in case of double applied middleware 22 | func setSubscribeObservedToCtx(ctx context.Context) context.Context { 23 | return context.WithValue(ctx, subscribeObserved, true) 24 | } 25 | 26 | func subscribeAlreadyObserved(ctx context.Context) bool { 27 | return ctx.Value(subscribeObserved) != nil 28 | } 29 | -------------------------------------------------------------------------------- /components/metrics/http.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "net/http" 5 | 6 | "github.com/go-chi/chi/v5" 7 | "github.com/prometheus/client_golang/prometheus" 8 | "github.com/prometheus/client_golang/prometheus/promhttp" 9 | ) 10 | 11 | // CreateRegistryAndServeHTTP establishes an HTTP server that exposes the /metrics endpoint for Prometheus at the given address. 12 | // It returns a new prometheus registry (to register the metrics on) and a canceling function that ends the server. 13 | func CreateRegistryAndServeHTTP(addr string) (registry *prometheus.Registry, cancel func()) { 14 | registry = prometheus.NewRegistry() 15 | return registry, ServeHTTP(addr, registry) 16 | } 17 | 18 | // ServeHTTP establishes an HTTP server that exposes the /metrics endpoint for Prometheus at the given address. 19 | // It takes an existing Prometheus registry and returns a canceling function that ends the server. 20 | func ServeHTTP(addr string, registry *prometheus.Registry) (cancel func()) { 21 | router := chi.NewRouter() 22 | 23 | handler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) 24 | router.Get("/metrics", func(w http.ResponseWriter, r *http.Request) { 25 | handler.ServeHTTP(w, r) 26 | }) 27 | server := http.Server{ 28 | Addr: addr, 29 | Handler: router, 30 | } 31 | 32 | go func() { 33 | err := server.ListenAndServe() 34 | if err != http.ErrServerClosed { 35 | panic(err) 36 | } 37 | }() 38 | 39 | return func() { _ = server.Close() } 40 | } 41 | -------------------------------------------------------------------------------- /components/metrics/labels.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | 8 | "github.com/prometheus/client_golang/prometheus" 9 | ) 10 | 11 | const ( 12 | labelKeyHandlerName = "handler_name" 13 | labelKeyPublisherName = "publisher_name" 14 | labelKeySubscriberName = "subscriber_name" 15 | labelSuccess = "success" 16 | labelAcked = "acked" 17 | 18 | labelValueNoHandler = "" 19 | ) 20 | 21 | var ( 22 | labelGetters = map[string]func(context.Context) string{ 23 | labelKeyHandlerName: message.HandlerNameFromCtx, 24 | labelKeyPublisherName: message.PublisherNameFromCtx, 25 | labelKeySubscriberName: message.SubscriberNameFromCtx, 26 | } 27 | ) 28 | 29 | func labelsFromCtx(ctx context.Context, labels ...string) prometheus.Labels { 30 | ctxLabels := map[string]string{} 31 | 32 | for _, l := range labels { 33 | k := l 34 | ctxLabels[l] = "" 35 | 36 | getter, ok := labelGetters[k] 37 | if !ok { 38 | continue 39 | } 40 | 41 | v := getter(ctx) 42 | if v != "" { 43 | ctxLabels[l] = v 44 | } 45 | } 46 | 47 | return ctxLabels 48 | } 49 | -------------------------------------------------------------------------------- /components/metrics/subscriber.go: -------------------------------------------------------------------------------- 1 | package metrics 2 | 3 | import ( 4 | "github.com/ThreeDotsLabs/watermill/message" 5 | "github.com/prometheus/client_golang/prometheus" 6 | ) 7 | 8 | var ( 9 | subscriberLabelKeys = []string{ 10 | labelKeyHandlerName, 11 | labelKeySubscriberName, 12 | } 13 | ) 14 | 15 | // SubscriberPrometheusMetricsDecorator decorates a subscriber to capture Prometheus metrics. 16 | type SubscriberPrometheusMetricsDecorator struct { 17 | message.Subscriber 18 | subscriberName string 19 | subscriberMessagesReceivedTotal *prometheus.CounterVec 20 | closing chan struct{} 21 | } 22 | 23 | func (s SubscriberPrometheusMetricsDecorator) recordMetrics(msg *message.Message) { 24 | if msg == nil { 25 | return 26 | } 27 | 28 | ctx := msg.Context() 29 | labels := labelsFromCtx(ctx, subscriberLabelKeys...) 30 | if labels[labelKeySubscriberName] == "" { 31 | labels[labelKeySubscriberName] = s.subscriberName 32 | } 33 | if labels[labelKeyHandlerName] == "" { 34 | labels[labelKeyHandlerName] = labelValueNoHandler 35 | } 36 | 37 | go func() { 38 | if subscribeAlreadyObserved(ctx) { 39 | // decorator idempotency when applied decorator multiple times 40 | return 41 | } 42 | 43 | select { 44 | case <-msg.Acked(): 45 | labels[labelAcked] = "acked" 46 | case <-msg.Nacked(): 47 | labels[labelAcked] = "nacked" 48 | } 49 | s.subscriberMessagesReceivedTotal.With(labels).Inc() 50 | }() 51 | 52 | msg.SetContext(setSubscribeObservedToCtx(msg.Context())) 53 | } 54 | -------------------------------------------------------------------------------- /dev/consolidate-gomods/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | ) 10 | 11 | // simple script to consolidate all gomods to one gomod 12 | // required for GolangCI linter 13 | func main() { 14 | bigFatGomod := "" 15 | 16 | for _, fileName := range getGomods() { 17 | dir := filepath.Dir(fileName) 18 | if dir == "." { 19 | continue 20 | } 21 | 22 | file, err := os.Open(fileName) 23 | if err != nil { 24 | panic(err) 25 | } 26 | 27 | fileMod := "" 28 | 29 | scanner := bufio.NewScanner(file) 30 | for scanner.Scan() { 31 | txt := scanner.Text() 32 | if strings.HasPrefix(txt, "go ") { 33 | continue 34 | } 35 | if strings.HasPrefix(txt, "module ") { 36 | continue 37 | } 38 | 39 | fileMod += txt + "\n" 40 | } 41 | 42 | if err := scanner.Err(); err != nil { 43 | panic(err) 44 | } 45 | 46 | if fileMod != "" { 47 | bigFatGomod += "// " + fileName + "\n" 48 | bigFatGomod += fileMod + "\n" 49 | } 50 | 51 | _ = file.Close() 52 | 53 | // gomod is stupid, and go vendor removes all deps that are not needed 54 | // (and they are not needed if they are already meet in sub go.mods) 55 | if err := os.Remove(fileName); err != nil { 56 | panic(err) 57 | } 58 | } 59 | 60 | fmt.Println(bigFatGomod) 61 | } 62 | 63 | func getGomods() []string { 64 | var fileList []string 65 | 66 | err := filepath.Walk(".", func(path string, f os.FileInfo, err error) error { 67 | if strings.Contains(path, "/vendor/") { 68 | return nil 69 | } 70 | 71 | if strings.Contains(path, "go.mod") { 72 | fileList = append(fileList, path) 73 | } 74 | return nil 75 | }) 76 | 77 | if err != nil { 78 | panic(err) 79 | } 80 | 81 | return fileList 82 | } 83 | -------------------------------------------------------------------------------- /dev/coverage.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ######## 3 | # Source: https://gist.github.com/lwolf/3764a3b6cd08387e80aa6ca3b9534b8a 4 | # originally from https://github.com/mlafeldt/chef-runner/blob/v0.7.0/script/coverage 5 | ####### 6 | # Generate test coverage statistics for Go packages. 7 | # 8 | # Works around the fact that `go test -coverprofile` currently does not work 9 | # with multiple packages, see https://code.google.com/p/go/issues/detail?id=6909 10 | # 11 | # Usage: script/coverage [--html|--coveralls] 12 | # 13 | # --html Additionally create HTML report and open it in browser 14 | # --coveralls Push coverage statistics to coveralls.io 15 | # 16 | 17 | set -e 18 | 19 | workdir=.cover 20 | profile="$workdir/cover.out" 21 | mode=count 22 | 23 | generate_cover_data() { 24 | rm -rf "$workdir" 25 | mkdir "$workdir" 26 | 27 | for pkg in "$@"; do 28 | f="$workdir/$(echo $pkg | tr / -).cover" 29 | go test -covermode="$mode" -coverprofile="$f" "$pkg" 30 | done 31 | 32 | echo "mode: $mode" >"$profile" 33 | grep -h -v "^mode:" "$workdir"/*.cover >>"$profile" 34 | } 35 | 36 | show_cover_report() { 37 | go tool cover -${1}="$profile" 38 | } 39 | 40 | push_to_coveralls() { 41 | echo "Pushing coverage statistics to coveralls.io" 42 | goveralls -coverprofile="$profile" 43 | } 44 | 45 | generate_cover_data $(go list ./... | grep -v /vendor/) 46 | show_cover_report func 47 | case "$1" in 48 | "") 49 | ;; 50 | --html) 51 | show_cover_report html ;; 52 | --coveralls) 53 | push_to_coveralls ;; 54 | *) 55 | echo >&2 "error: invalid option: $1"; exit 1 ;; 56 | esac -------------------------------------------------------------------------------- /dev/prometheus.yml: -------------------------------------------------------------------------------- 1 | # for Watermill development purposes. 2 | # there is one sample scrape target; add your own if needed. 3 | global: 4 | scrape_interval: 15s 5 | evaluation_interval: 15s 6 | 7 | scrape_configs: 8 | - job_name: 'prometheus' 9 | static_configs: 10 | - targets: ['localhost:9090'] 11 | 12 | - job_name: 'some_metrics_endpoint' 13 | static_configs: 14 | - targets: ['localhost:8080'] 15 | -------------------------------------------------------------------------------- /dev/update-examples-deps/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ThreeDotsLabs/watermill/dev/update-examples-deps 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | -------------------------------------------------------------------------------- /dev/update-examples-deps/go.sum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/dev/update-examples-deps/go.sum -------------------------------------------------------------------------------- /dev/validate-examples/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ThreeDotsLabs/watermill/dev/validate-examples 2 | 3 | go 1.23 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/fatih/color v1.18.0 9 | gopkg.in/yaml.v2 v2.4.0 10 | ) 11 | 12 | require ( 13 | github.com/mattn/go-colorable v0.1.14 // indirect 14 | github.com/mattn/go-isatty v0.0.20 // indirect 15 | golang.org/x/sys v0.29.0 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /dev/validate-examples/go.sum: -------------------------------------------------------------------------------- 1 | github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= 2 | github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= 3 | github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= 4 | github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= 5 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 6 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 7 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 8 | golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= 9 | golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 10 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 11 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 12 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 13 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 14 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Watermill is a Golang library for working efficiently with message streams. 2 | // 3 | // It is intended for building event driven applications, 4 | // enabling event sourcing, RPC over messages, sagas 5 | // and basically whatever else comes to your mind. 6 | // 7 | // You can use conventional pub/sub implementations 8 | // like Kafka or RabbitMQ, but also HTTP or MySQL binlog if that fits your use case. 9 | // 10 | // Website with detailed documentation: https://watermill.io/ 11 | // 12 | // Getting started guide: https://watermill.io/docs/getting-started/ 13 | package watermill 14 | -------------------------------------------------------------------------------- /docs/.hugo_build.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/.hugo_build.lock -------------------------------------------------------------------------------- /docs/.npmignore: -------------------------------------------------------------------------------- 1 | .env 2 | .netlify 3 | .hugo_build.lock 4 | node_modules 5 | public 6 | resources 7 | -------------------------------------------------------------------------------- /docs/.npmrc: -------------------------------------------------------------------------------- 1 | enable-pre-post-scripts=true 2 | auto-install-peers=true 3 | node-linker=hoisted 4 | prefer-symlinked-executables=false 5 | -------------------------------------------------------------------------------- /docs/.prettierignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.ico 3 | *.png 4 | *.jp*g 5 | *.toml 6 | *.*ignore 7 | *.svg 8 | *.xml 9 | LICENSE 10 | .npmrc 11 | .gitkeep 12 | *.woff* 13 | -------------------------------------------------------------------------------- /docs/.prettierrc.yaml: -------------------------------------------------------------------------------- 1 | # Default config 2 | tabWidth: 4 3 | endOfLine: crlf 4 | singleQuote: true 5 | printWidth: 100000 6 | trailingComma: none 7 | bracketSameLine: true 8 | quoteProps: consistent 9 | experimentalTernaries: true 10 | 11 | # Overridden config 12 | overrides: 13 | - files: ["*.md", "*.json", "*.yaml"] 14 | options: 15 | tabWidth: 2 16 | singleQuote: false 17 | - files: ["*.scss"] 18 | options: 19 | singleQuote: false 20 | -------------------------------------------------------------------------------- /docs/DEVELOP.md: -------------------------------------------------------------------------------- 1 | ## How to Develop watermill.io docs? 2 | 3 | ### Building & running 4 | 5 | ```bash 6 | ./build.sh 7 | npm run dev 8 | ``` 9 | 10 | ### Useful resources 11 | 12 | - [Available shortcodes](https://getdoks.org/docs/basics/shortcodes/) 13 | - [Diagrams](https://getdoks.org/docs/built-ins/diagrams/) (we recommend [Mermaid](https://getdoks.org/docs/built-ins/diagrams/#mermaid)) 14 | - [Codeglocks](https://getdoks.org/docs/built-ins/code-blocks/) 15 | -------------------------------------------------------------------------------- /docs/assets/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/assets/favicon.ico -------------------------------------------------------------------------------- /docs/assets/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/assets/favicon.png -------------------------------------------------------------------------------- /docs/assets/images/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/assets/images/.gitkeep -------------------------------------------------------------------------------- /docs/assets/js/custom.js: -------------------------------------------------------------------------------- 1 | // Put your custom JS code here 2 | 3 | // a bit hacky way to force dark mode by default 4 | // it sets local storage item used by docs/node_modules/@thulite/doks-core/assets/js/color-mode.js 5 | if (!localStorage.getItem('theme')) { 6 | localStorage.setItem('theme', 'dark'); 7 | } 8 | 9 | import { render } from 'github-buttons'; 10 | 11 | let renderGitHubButton= () => { 12 | let oldButton = document.getElementById("github-button"); 13 | if (oldButton) { 14 | oldButton.remove(); 15 | } 16 | 17 | let options = { 18 | "href": "https://github.com/ThreeDotsLabs/watermill", 19 | "data-show-count": true, 20 | "data-size": "large", 21 | "data-color-scheme": localStorage.getItem('theme'), 22 | } 23 | 24 | render(options, function (el) { 25 | let menu = document.getElementById("offcanvasNavMain").querySelector(".offcanvas-body"); 26 | let searchToggle = document.getElementById("searchToggleDesktop"); 27 | 28 | el.setAttribute("id", "github-button"); 29 | el.classList.add("nav-link", "px-2", "mx-auto"); 30 | el.setAttribute("style", "margin-top: 12px;"); 31 | 32 | 33 | menu.insertBefore(el, searchToggle); 34 | }) 35 | } 36 | renderGitHubButton() 37 | -------------------------------------------------------------------------------- /docs/assets/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", 4 | "paths": { 5 | "*": ["*", "..\\node_modules\\@thulite\\doks-core\\assets\\*"] 6 | } 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /docs/assets/mask-icon.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/assets/scss/common/_custom.scss: -------------------------------------------------------------------------------- 1 | // Put your custom SCSS code here 2 | * { 3 | -webkit-font-smoothing: antialiased; 4 | } 5 | 6 | h1, h2, h3, h4, h5, .navbar-brand { 7 | font-family: Quicksand, sans-serif; 8 | font-weight: 700; 9 | } 10 | 11 | [data-bs-theme="dark"] .only-light { 12 | display: none; 13 | } 14 | 15 | [data-bs-theme="light"] .only-dark { 16 | display: none; 17 | } 18 | -------------------------------------------------------------------------------- /docs/assets/scss/common/_variables-custom.scss: -------------------------------------------------------------------------------- 1 | // Put your custom SCSS variables here 2 | $font-family-sans-serif: 3 | "Heebo", 4 | "sans-serif", 5 | system-ui, 6 | -apple-system, 7 | "Segoe UI", 8 | Roboto, 9 | "Helvetica Neue", 10 | "Noto Sans", 11 | "Liberation Sans", 12 | Arial, 13 | sans-serif, 14 | "Apple Color Emoji", 15 | "Segoe UI Emoji", 16 | "Segoe UI Symbol", 17 | "Noto Color Emoji"; 18 | 19 | $container-max-widths: ( 20 | sm: 540px, 21 | md: 720px, 22 | lg: 960px, 23 | xl: 1240px, 24 | xxl: 1820px 25 | ); 26 | 27 | 28 | $primary: #4f46e5; 29 | -------------------------------------------------------------------------------- /docs/assets/svgs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/assets/svgs/.gitkeep -------------------------------------------------------------------------------- /docs/config/_default/languages.toml: -------------------------------------------------------------------------------- 1 | [en] 2 | languageName = "English" 3 | contentDir = "content/en" 4 | weight = 10 5 | [en.params] 6 | languageISO = "EN" 7 | languageTag = "en-US" 8 | footer = '' 9 | alertText = '' 10 | -------------------------------------------------------------------------------- /docs/config/_default/markup.toml: -------------------------------------------------------------------------------- 1 | defaultMarkdownHandler = "goldmark" 2 | 3 | [goldmark] 4 | [goldmark.extensions] 5 | linkify = true 6 | [goldmark.parser] 7 | autoHeadingID = true 8 | autoHeadingIDType = "github" 9 | [goldmark.parser.attribute] 10 | block = true 11 | title = true 12 | [goldmark.renderer] 13 | unsafe = true 14 | 15 | [highlight] 16 | anchorLineNos = false 17 | codeFences = true 18 | guessSyntax = false 19 | hl_Lines = '' 20 | hl_inline = false 21 | lineAnchors = '' 22 | lineNoStart = 1 23 | lineNos = false 24 | lineNumbersInTable = false 25 | noClasses = false 26 | noHl = false 27 | style = 'monokai' 28 | tabWidth = 2 29 | 30 | [tableOfContents] 31 | endLevel = 3 32 | ordered = false 33 | startLevel = 2 34 | -------------------------------------------------------------------------------- /docs/config/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | [ 4 | '@babel/preset-env', 5 | { 6 | targets: { 7 | browsers: [ 8 | // Best practice: https://github.com/babel/babel/issues/7789 9 | '>=1%', 10 | 'not ie 11', 11 | 'not op_mini all' 12 | ] 13 | } 14 | } 15 | ] 16 | ] 17 | }; 18 | -------------------------------------------------------------------------------- /docs/config/next/hugo.toml: -------------------------------------------------------------------------------- 1 | # Overrides for next environment 2 | baseurl = "/" 3 | -------------------------------------------------------------------------------- /docs/config/production/hugo.toml: -------------------------------------------------------------------------------- 1 | # Overrides for production environment 2 | -------------------------------------------------------------------------------- /docs/content/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Watermill" 3 | description: "Building event-driven applications the easy way in Go." 4 | lead: "Building event-driven applications the easy way in Go." 5 | date: 2023-09-07T16:33:54+02:00 6 | lastmod: 2023-09-07T16:33:54+02:00 7 | draft: false 8 | seo: 9 | title: "Watermill" # custom title (optional) 10 | description: "" # custom description (recommended) 11 | canonical: "" # custom canonical URL (optional) 12 | noindex: false # false (default) or true 13 | --- 14 | -------------------------------------------------------------------------------- /docs/content/advanced/fanin.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "FanIn (merging topics)" 3 | description = "Merging two topics into one with the FanIn component" 4 | date = 2023-01-21T12:47:30+01:00 5 | weight = -100 6 | draft = false 7 | bref = "Merging two topics into one with the FanIn component" 8 | +++ 9 | 10 | ## FanIn component 11 | 12 | The FanIn component merges two topics into one. 13 | 14 | ### Configuring 15 | 16 | {{% load-snippet-partial file="src-link/components/fanin/fanin.go" first_line_contains="type Config struct {" last_line_contains="CloseTimeout time.Duration" padding_after="1" %}} 17 | 18 | ### Running 19 | 20 | You need to provide a Publisher and a Subscriber implementation for the FanIn component. 21 | 22 | You can find the list of supported Pub/Subs on [Supported Pub/Subs page](/pubsubs/). 23 | The Publisher and subscriber can be implemented by different message brokers (for example, you can merge a Kafka topic with a RabbitMQ topic). 24 | 25 | ```go 26 | 27 | logger := watermill.NewStdLogger(false, false) 28 | 29 | // create Publisher and Subscriber 30 | pub, err := // ... 31 | sub, err := // ... 32 | 33 | fi, err := fanin.NewFanIn( 34 | sub, 35 | pub, 36 | fanin.Config{ 37 | SourceTopics: upstreamTopics, 38 | TargetTopic: downstreamTopic, 39 | }, 40 | logger, 41 | ) 42 | if err != nil { 43 | panic(err) 44 | } 45 | 46 | if err := fi.Run(context.Background()); err != nil { 47 | panic(err) 48 | } 49 | ``` 50 | 51 | ### Controlling FanIn component 52 | 53 | The FanIn component can be stopped by cancelling the context passed to the `Run` method or by calling the `Close` method. 54 | 55 | {{% load-snippet-partial file="src-link/components/fanin/fanin.go" first_line_contains="func (f *FanIn) Run" last_line_contains=" Close() error" padding_after="2" %}} 56 | -------------------------------------------------------------------------------- /docs/content/advanced/fanout.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "FanOut (multiplying messages)" 3 | description = "FanOut is a component that receives messages from the subscriber and passes them to all publishers." 4 | date = 2024-10-09T02:47:30+01:00 5 | weight = -50 6 | draft = false 7 | bref = "FanOut is a component that receives messages from the subscriber and passes them to all publishers." 8 | +++ 9 | 10 | ## FanOut component 11 | 12 | FanOut is a component that receives messages from a topic and passes them to all subscribers. In effect, messages are "multiplied". 13 | 14 | A typical use case for using FanOut is having one external subscription and multiple workers 15 | inside the process. 16 | 17 | ### Configuring 18 | 19 | {{% load-snippet-partial file="src-link/pubsub/gochannel/fanout.go" first_line_contains="// NewFanOut" last_line_contains=")" padding_after="0" %}} 20 | 21 | You need to call AddSubscription method for all topics that you want to listen to. 22 | This needs to be done *before* starting the FanOut. 23 | 24 | {{% load-snippet-partial file="src-link/pubsub/gochannel/fanout.go" first_line_contains="// AddSubscription" last_line_contains=")" padding_after="0" %}} 25 | 26 | ### Running 27 | 28 | {{% load-snippet-partial file="src-link/pubsub/gochannel/fanout.go" first_line_contains="// Run" last_line_contains=")" padding_after="0" %}} 29 | 30 | Then, use it as any other `message.Subscriber`. 31 | -------------------------------------------------------------------------------- /docs/content/development/benchmark.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "Benchmark" 3 | description = "Watermill Benchmark" 4 | weight = 250 5 | draft = false 6 | bref = "Watermill Benchmark" 7 | +++ 8 | 9 | You can find benchmarking tools and results in the [github.com/ThreeDotsLabs/watermill-benchmark](https://github.com/ThreeDotsLabs/watermill-benchmark) repository. 10 | 11 | **Note they are meant as rough estimations and should not be used to decide which Pub/Sub is the best pick.** 12 | Performance depends on many factors and configurations, and it's always best to test it in your environment. 13 | -------------------------------------------------------------------------------- /docs/content/development/releases.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "Releases" 3 | description = "Watermill Releases" 4 | weight = 300 5 | bref = "Watermill Releases" 6 | +++ 7 | 8 | You can read about the historical Watermill releases in [the posts on the Three Dots Labs blog](https://threedots.tech/series/watermill-release-post/). 9 | 10 | 11 | -------------------------------------------------------------------------------- /docs/content/docs/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Docs" 3 | description: "" 4 | summary: "" 5 | date: 2023-09-07T16:12:03+02:00 6 | lastmod: 2023-09-07T16:12:03+02:00 7 | draft: false 8 | weight: 999 9 | toc: true 10 | seo: 11 | title: "" # custom title (optional) 12 | description: "" # custom description (recommended) 13 | canonical: "" # custom canonical URL (optional) 14 | noindex: false # false (default) or true 15 | --- 16 | -------------------------------------------------------------------------------- /docs/content/docs/articles.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "Articles" 3 | description = "In-depth articles mentioning Watermill" 4 | weight = 100 5 | draft = false 6 | bref = "In-depth articles mentioning Watermill" 7 | +++ 8 | 9 | You can find more in-depth tips on Watermill in these articles: 10 | 11 | * [Distributed Transactions in Go: Read Before You Try](https://threedots.tech/post/distributed-transactions-in-go/) 12 | * [Live website updates with Go, SSE, and htmx](https://threedots.tech/post/live-website-updates-go-sse-htmx/) 13 | * [Using MySQL as a Pub/Sub](https://threedots.tech/post/when-sql-database-makes-great-pub-sub/) 14 | * [Creating local Go dev environment with Docker and live code reloading](https://threedots.tech/post/go-docker-dev-environment-with-go-modules-and-live-code-reloading/) 15 | 16 | -------------------------------------------------------------------------------- /docs/content/docs/message.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "Message" 3 | description = "Message is one of core parts of Watermill" 4 | date = 2018-12-05T12:42:40+01:00 5 | weight = -1000 6 | draft = false 7 | bref = "Message is one of core parts of Watermill" 8 | +++ 9 | 10 | Message is one of core parts of Watermill. Messages are emitted by [*Publishers*]({{< ref "/docs/pub-sub#publisher" >}}) and received by [*Subscribers*]({{< ref "/docs/pub-sub#subscriber" >}}). 11 | When a message is processed, you should send an [`Ack()`]({{< ref "#ack" >}}) or a [`Nack()`]({{< ref "#ack" >}}) when the processing failed. 12 | 13 | `Acks` and `Nacks` are processed by Subscribers (in default implementations, the subscribers are waiting for an `Ack` or a `Nack`). 14 | 15 | {{% load-snippet-partial file="src-link/message/message.go" first_line_contains="type Message struct {" last_line_contains="ctx context.Context" padding_after="2" %}} 16 | 17 | ## Ack 18 | 19 | #### Sending `Ack` 20 | 21 | {{% load-snippet-partial file="src-link/message/message.go" first_line_contains="// Ack" last_line_contains="func (m *Message) Ack() bool {" padding_after="0" %}} 22 | 23 | 24 | ## Nack 25 | 26 | {{% load-snippet-partial file="src-link/message/message.go" first_line_contains="// Nack" last_line_contains="func (m *Message) Nack() bool {" padding_after="0" %}} 27 | 28 | #### Receiving `Ack/Nack` 29 | 30 | {{% load-snippet-partial file="docs/message/receiving-ack.go" first_line_contains="select {" last_line_contains="}" padding_after="0" %}} 31 | 32 | 33 | ## Context 34 | 35 | Message contains the standard library context, just like an HTTP request. 36 | 37 | {{% load-snippet-partial file="src-link/message/message.go" first_line_contains="// Context" last_line_contains="func (m *Message) SetContext" padding_after="2" %}} 38 | -------------------------------------------------------------------------------- /docs/content/docs/message/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "go run receiving-ack.go" 2 | timeout: 30 3 | expected_output: "ack received" 4 | -------------------------------------------------------------------------------- /docs/content/docs/message/go.mod: -------------------------------------------------------------------------------- 1 | module receiving-ack.go 2 | 3 | require github.com/ThreeDotsLabs/watermill v1.2.0-rc.11 4 | 5 | require ( 6 | github.com/google/uuid v1.3.0 // indirect 7 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 8 | github.com/oklog/ulid v1.3.1 // indirect 9 | github.com/pkg/errors v0.9.1 // indirect 10 | ) 11 | 12 | go 1.21 13 | -------------------------------------------------------------------------------- /docs/content/docs/message/go.sum: -------------------------------------------------------------------------------- 1 | github.com/ThreeDotsLabs/watermill v1.2.0-rc.11 h1:tQJ3L/AnfliXaxaq+ElHOfzi0Vx+AN8cAnIOLcUTrxo= 2 | github.com/ThreeDotsLabs/watermill v1.2.0-rc.11/go.mod h1:QLZSaklpSZ/7yv288LL2DFOgCEi86VYEmQvzmaMlHoA= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 5 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 6 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 7 | github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= 8 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 9 | github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= 10 | github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= 11 | github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= 12 | github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= 13 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 14 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 15 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 16 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 17 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 18 | -------------------------------------------------------------------------------- /docs/content/docs/message/receiving-ack.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "time" 6 | 7 | "github.com/ThreeDotsLabs/watermill/message" 8 | ) 9 | 10 | func main() { 11 | msg := message.NewMessage("1", []byte("foo")) 12 | 13 | go func() { 14 | time.Sleep(time.Millisecond * 10) 15 | msg.Ack() 16 | }() 17 | 18 | select { 19 | case <-msg.Acked(): 20 | log.Print("ack received") 21 | case <-msg.Nacked(): 22 | log.Print("nack received") 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /docs/content/docs/middlewares.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "Middleware" 3 | description = "Add generic functionalities to your handlers in an unobtrusive way" 4 | date = 2019-06-01T19:00:00+01:00 5 | weight = -500 6 | draft = false 7 | bref = "Add functionality to handlers" 8 | +++ 9 | 10 | ## Introduction 11 | 12 | Middleware wrap handlers with functionality that is important, but not relevant for the primary handler's logic. 13 | Examples include retrying the handler after an error was returned, or recovering from panic in the handler 14 | and capturing the stacktrace. 15 | 16 | Middleware wrap the handler function like this: 17 | 18 | {{% load-snippet-partial file="src-link/message/router.go" first_line_contains="// HandlerMiddleware" last_line_contains="type HandlerMiddleware" %}} 19 | 20 | ## Usage 21 | 22 | Middleware can be executed for all as well as for a specific handler in a router. When middleware is added directly 23 | to a router it will be executed for all handlers provided for a router. If a middleware should be executed only 24 | for a specific handler, it needs to be added to handler in the router. 25 | 26 | Example usage is shown below: 27 | 28 | {{% load-snippet-partial file="src-link/_examples/basic/3-router/main.go" first_line_contains="router, err := message.NewRouter(message.RouterConfig{}, logger)" last_line_contains="// Now that all handlers are registered, we're running the Router." padding_after="1" %}} 29 | 30 | ## Available middleware 31 | 32 | Below are the middleware provided by Watermill and ready to use. You can also easily implement your own. 33 | For example, if you'd like to store every received message in some kind of log, it's the best way to do it. 34 | 35 | {{% readfile file="/content/src-link/middleware-defs.md" %}} 36 | 37 | -------------------------------------------------------------------------------- /docs/content/docs/snippets/amqp-consumer-groups/.validate_example.yml: -------------------------------------------------------------------------------- 1 | validation_cmd: "docker compose up" 2 | teardown_cmd: "docker compose down" 3 | timeout: 120 4 | expected_output: "payload: Hello, world!" 5 | -------------------------------------------------------------------------------- /docs/content/docs/snippets/amqp-consumer-groups/docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | server: 3 | image: golang:1.23 4 | restart: unless-stopped 5 | depends_on: 6 | - rabbitmq 7 | volumes: 8 | - .:/app 9 | - $GOPATH/pkg/mod:/go/pkg/mod 10 | working_dir: /app 11 | command: go run main.go 12 | 13 | rabbitmq: 14 | image: rabbitmq:3.7 15 | restart: unless-stopped 16 | -------------------------------------------------------------------------------- /docs/content/docs/snippets/amqp-consumer-groups/go.mod: -------------------------------------------------------------------------------- 1 | module main.go 2 | 3 | require ( 4 | github.com/ThreeDotsLabs/watermill v1.2.0-rc.11 5 | github.com/ThreeDotsLabs/watermill-amqp/v2 v2.0.7 6 | ) 7 | 8 | require ( 9 | github.com/cenkalti/backoff/v3 v3.2.2 // indirect 10 | github.com/google/uuid v1.3.0 // indirect 11 | github.com/hashicorp/errwrap v1.1.0 // indirect 12 | github.com/hashicorp/go-multierror v1.1.1 // indirect 13 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 14 | github.com/oklog/ulid v1.3.1 // indirect 15 | github.com/pkg/errors v0.9.1 // indirect 16 | github.com/rabbitmq/amqp091-go v1.6.1 // indirect 17 | ) 18 | 19 | go 1.21 20 | -------------------------------------------------------------------------------- /docs/content/docs/snippets/tail-log-file/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ThreeDotsLabs/watermill/docs/content/docs/snippets/tail-log-file 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/ThreeDotsLabs/watermill v1.2.0-rc.11 7 | github.com/ThreeDotsLabs/watermill-io v1.0.3 8 | ) 9 | 10 | require ( 11 | github.com/google/uuid v1.3.0 // indirect 12 | github.com/hashicorp/errwrap v1.1.0 // indirect 13 | github.com/hashicorp/go-multierror v1.1.1 // indirect 14 | github.com/lithammer/shortuuid/v3 v3.0.7 // indirect 15 | github.com/oklog/ulid v1.3.1 // indirect 16 | github.com/pkg/errors v0.9.1 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /docs/content/docs/snippets/tail-log-file/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | 8 | "github.com/ThreeDotsLabs/watermill" 9 | "github.com/ThreeDotsLabs/watermill-io/pkg/io" 10 | "github.com/ThreeDotsLabs/watermill/message" 11 | ) 12 | 13 | // this will `tail -f` a log file and publish an alert if a line fulfils some criterion 14 | 15 | func main() { 16 | // if an alert is raised, the offending line will be published on this 17 | // this would be set to an actual publisher 18 | var alertPublisher message.Publisher 19 | 20 | if len(os.Args) < 2 { 21 | panic( 22 | fmt.Errorf("usage: %s /path/to/file.log", os.Args[0]), 23 | ) 24 | } 25 | logFile, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0444) 26 | if err != nil { 27 | panic(err) 28 | } 29 | 30 | sub, err := io.NewSubscriber(logFile, io.SubscriberConfig{ 31 | UnmarshalFunc: io.PayloadUnmarshalFunc, 32 | }, watermill.NewStdLogger(true, false)) 33 | if err != nil { 34 | panic(err) 35 | } 36 | 37 | // for io.Subscriber, topic does not matter 38 | lines, err := sub.Subscribe(context.Background(), "") 39 | if err != nil { 40 | panic(err) 41 | } 42 | 43 | for line := range lines { 44 | if criterion(string(line.Payload)) { 45 | _ = alertPublisher.Publish("alerts", line) 46 | } 47 | } 48 | } 49 | 50 | func criterion(line string) bool { 51 | // decide whether an action needs to be taken 52 | return false 53 | } 54 | -------------------------------------------------------------------------------- /docs/content/pubsubs/_index.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "Supported Pub/Subs" 3 | bref = "Watermill supports these Pub/Sub adapters out of the box:" 4 | +++ 5 | -------------------------------------------------------------------------------- /docs/content/pubsubs/gochannel.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "Go Channel" 3 | description = "A Pub/Sub implemented on Golang goroutines and channels" 4 | date = 2019-07-06T22:30:00+02:00 5 | bref = "A Pub/Sub implemented on Golang goroutines and channels" 6 | weight = 40 7 | +++ 8 | 9 | {{% load-snippet-partial file="src-link/pubsub/gochannel/pubsub.go" first_line_contains="// GoChannel" last_line_contains="type GoChannel struct {" %}} 10 | 11 | You can find a fully functional example with Go Channels in the [Watermill examples](https://github.com/ThreeDotsLabs/watermill/tree/master/_examples/pubsubs/go-channel). 12 | 13 | ### Characteristics 14 | 15 | | Feature | Implements | Note | 16 | | ------- | ---------- | ---- | 17 | | ConsumerGroups | no | | 18 | | ExactlyOnceDelivery | yes | | 19 | | GuaranteedOrder | yes | | 20 | | Persistent | no| | 21 | 22 | ### Configuration 23 | 24 | You can inject configuration via the constructor. 25 | 26 | {{% load-snippet-partial file="src-link/pubsub/gochannel/pubsub.go" first_line_contains="func NewGoChannel" last_line_contains="logger:" %}} 27 | 28 | ### Publishing 29 | 30 | {{% load-snippet-partial file="src-link/pubsub/gochannel/pubsub.go" first_line_contains="// Publish" last_line_contains="func (g *GoChannel) Publish" %}} 31 | 32 | ### Subscribing 33 | 34 | {{% load-snippet-partial file="src-link/pubsub/gochannel/pubsub.go" first_line_contains="// Subscribe" last_line_contains="func (g *GoChannel) Subscribe" %}} 35 | 36 | ### Marshaler 37 | 38 | No marshaling is needed when sending messages within the process. 39 | -------------------------------------------------------------------------------- /docs/content/support.md: -------------------------------------------------------------------------------- 1 | +++ 2 | title = "Support" 3 | description = "" 4 | +++ 5 | 6 | ### Community Support 7 | 8 | Join us on the `#watermill` channel on the [Three Dots Labs discord](https://discord.gg/QV6VFg4YQE). 9 | 10 | ### Professional Support 11 | 12 | For enterprise support, please contact us by e-mail: contact@threedotslabs.com 13 | 14 | You can also use the [contact form on our website](https://threedots.tech/contact/?utm_source=watermill-docs). 15 | -------------------------------------------------------------------------------- /docs/layouts/_default/_markup/render-link.html: -------------------------------------------------------------------------------- 1 | {{ .Text | safeHTML }} 2 | -------------------------------------------------------------------------------- /docs/layouts/partials/footer/footer.html: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 |
5 | Check our online hands-on training 6 | 7 |
8 | 9 |
10 |
11 |
12 |
13 | 14 | Three Dots Labs 15 | Three Dots Labs 16 | 17 |
18 |
19 |

© Three Dots Labs 2014 — {{ dateFormat "2006" now }}

20 |
21 |
22 |
23 | 24 |
25 |

26 |

27 | 28 |

29 | Watermill is open-source software and is not backed by venture capital. 30 |
31 | We are an independent, bootstrapped company. 32 | 33 |

34 |
35 |
36 | -------------------------------------------------------------------------------- /docs/layouts/partials/footer/script-footer-custom.html: -------------------------------------------------------------------------------- 1 | {{/* Put your custom tags here */}} 2 | 3 | {{/* EXAMPLE - only load script for production 4 | {{ if eq (hugo.Environment) "production" -}} 5 | {{ partial "footer/esbuild" (dict "src" "js/instantpage.js" "load" "async" "transpile" false) -}} 6 | {{ end -}} 7 | */}} 8 | 9 | {{/* EXAMPLE - only load script for a page type e.g. contact or gallery 10 | {{ if eq .Type "gallery" -}} 11 | {{ partial "footer/esbuild" (dict "src" "js/gallery.js" "load" "async" "transpile" false) -}} 12 | {{ end -}} 13 | */}} 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /docs/layouts/partials/head/custom-head.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {{ $pf:= "Heebo:wght@400;600" }} 5 | {{ $sf:= "Quicksand:wght@700" }} 6 | 7 | 8 | 17 | -------------------------------------------------------------------------------- /docs/layouts/partials/head/resource-hints.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /docs/layouts/partials/head/script-header.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docs/layouts/partials/main/edit-page.html: -------------------------------------------------------------------------------- 1 | {{ $parts := slice site.Params.doks.docsRepo }} 2 | 3 | {{ if (eq site.Params.doks.repoHost "GitHub") }} 4 | {{ $parts = $parts | append "edit" site.Params.doks.docsRepoBranch }} 5 | {{ else if (eq site.Params.doks.repoHost "Gitea") }} 6 | {{ $parts = $parts | append "_edit" site.Params.doks.docsRepoBranch }} 7 | {{ else if (eq site.Params.doks.repoHost "GitLab") }} 8 | {{ $parts = $parts | append "-/blob" site.Params.doks.docsRepoBranch }} 9 | {{ else if (eq site.Params.doks.repoHost "Bitbucket") }} 10 | {{ $parts = $parts | append "src" site.Params.doks.docsRepoBranch }} 11 | {{ else if (eq site.Params.doks.repoHost "BitbucketServer") }} 12 | {{ $parts = $parts | append "browse" site.Params.doks.docsRepoBranch }} 13 | {{ end }} 14 | 15 | {{ if isset .Site.Params "docsRepoSubPath" }} 16 | {{ if not (eq site.Params.doks.docsRepoSubPath "") }} 17 | {{ $parts = $parts | append site.Params.doks.docsRepoSubPath }} 18 | {{ end }} 19 | {{ end }} 20 | 21 | {{ $filePath := replace .File.Path "\\" "/" }} 22 | 23 | {{ $lang := "" }} 24 | {{ if site.Params.doks.multilingualMode }} 25 | {{ $lang = .Lang }} 26 | {{ end }} 27 | 28 | {{ $parts = $parts | append "docs/content" $filePath }} 29 | 30 | {{ $url := delimit $parts "/" }} 31 | 32 |
33 | 34 | 35 | 36 | 37 | Help us improve this page 38 | 39 |
40 | -------------------------------------------------------------------------------- /docs/layouts/partials/private/has-headings.html: -------------------------------------------------------------------------------- 1 | {{ $hasHeadings := false }} 2 | {{ if (isset .Fragments "Headings") }} 3 | {{ $hasHeadings = gt (len .Fragments.Headings) 0 }} 4 | {{ end }} 5 | {{ $hasHeadings }} 6 | -------------------------------------------------------------------------------- /docs/layouts/partials/seo/twitter.html: -------------------------------------------------------------------------------- 1 | {{/* Based on: https://github.com/gohugoio/hugo/blob/master/tpl/tplimpl/embedded/templates/twitter_cards.html */}} 2 | 3 | {{ $imagePermalink := (printf "https://academy-api.threedots.tech/ssr/image.png?%s" (collections.Querify "url" .Permalink) ) }} 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | {{- /* Deprecate site.Social.twitter in favor of site.Params.social.twitter */}} 12 | {{- $twitterSite := "" }} 13 | {{- with site.Params.social }} 14 | {{- if reflect.IsMap . }} 15 | {{- $twitterSite = .twitter }} 16 | {{- end }} 17 | {{- else }} 18 | {{- with site.Social.twitter }} 19 | {{- $twitterSite = . }} 20 | {{- warnf "The social key in site configuration is deprecated. Use params.social.twitter instead." }} 21 | {{- end }} 22 | {{- end }} 23 | 24 | {{- with $twitterSite }} 25 | {{- $content := . }} 26 | {{- if not (strings.HasPrefix . "@") }} 27 | {{- $content = printf "@%v" $twitterSite }} 28 | {{- end }} 29 | 30 | {{- end }} 31 | -------------------------------------------------------------------------------- /docs/layouts/shortcodes/load-snippet.html: -------------------------------------------------------------------------------- 1 | {{ $file := (.Get "file") }} 2 | {{ $content := readFile $file }} 3 | 4 | {{ $start_line := (.Get "start_line") | default "0" }} 5 | {{ $end_line := (.Get "end_line") | default "0" }} 6 | 7 | {{ $has_start_line := (ne $start_line "0") }} 8 | {{ $has_end_line := (ne $end_line "0") }} 9 | 10 | {{ $lines := slice }} 11 | 12 | {{ $linkFile := $file }} 13 | {{ $repo := "watermill" }} 14 | 15 | {{ if in $file "src-link/watermill-" }} 16 | {{ $repo = index (findRE "watermill-[a-z]+" $linkFile) 0 }} 17 | {{ $linkFile = replace $linkFile $repo "" }} 18 | {{ $linkFile = replace $linkFile "src-link//" "" }} 19 | {{ else if in $linkFile "src-link/" }} 20 | {{ $linkFile = replace $linkFile "src-link/" "" }} 21 | {{ else }} 22 | {{ $linkFile = print "docs/content/" $linkFile }} 23 | {{ end }} 24 | 25 | {{ range $elem_key, $elem_val := split $content "\n" }} 26 | {{if and (or (not $has_start_line) (ge (add $elem_key 1) ($start_line | int))) (or (not $has_end_line) (le (add $elem_key 1) ($end_line | int)))}} 27 | {{ $lines = $lines | append $elem_val }} 28 | {{ end }} 29 | {{ end }} 30 | 31 |
32 |
33 | {{ transform.Highlight (delimit $lines "\n" | safeHTML) (.Get "type" | default "go") }} 34 |
35 |
36 | 37 | Full source: [{{ $linkFile }}](https://github.com/ThreeDotsLabs/watermill/tree/master/{{ $linkFile }}) 38 | -------------------------------------------------------------------------------- /docs/layouts/shortcodes/readfile.html: -------------------------------------------------------------------------------- 1 | {{$file := .Get "file"}} 2 | {{- if eq (.Get "markdown") "true" -}} 3 | {{- $file | readFile | markdownify -}} 4 | {{- else -}} 5 | {{ $file | readFile | safeHTML }} 6 | {{- end -}} 7 | -------------------------------------------------------------------------------- /docs/layouts/shortcodes/tab.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | {{ if .Parent -}} 6 | {{ $name := .Get 0 }} 7 | {{ $slug := .Get 1 }} 8 | {{ $group := printf "tabs-%s" (.Parent.Get 0) }} 9 | 10 | {{ if not (.Parent.Scratch.Get $group) }} 11 | {{ .Parent.Scratch.Set $group slice }} 12 | {{ end }} 13 | 14 | {{ .Parent.Scratch.Add $group (dict "Name" $name "Slug" $slug "Content" .Inner) }} 15 | {{ else -}} 16 | {{ errorf "%q: 'tab' shortcode must be inside 'tabs' shortcode" .Page.Path }} 17 | {{ end -}} 18 | -------------------------------------------------------------------------------- /docs/layouts/shortcodes/tabs.html: -------------------------------------------------------------------------------- 1 | 4 | 5 | {{ if .Inner }}{{ end }} 6 | {{ $id := .Get 0 }} 7 | {{ $group := printf "tabs-%s" $id }} 8 | 9 | 18 | 19 |
20 | {{ range $index, $tab := .Scratch.Get $group -}} 21 |
22 | {{ .Content | $.Page.RenderString -}} 23 |
24 | {{ end -}} 25 |
26 | -------------------------------------------------------------------------------- /docs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "watermill-docs", 3 | "version": "0.0.0", 4 | "description": "Doks theme", 5 | "author": "Thulite", 6 | "license": "MIT", 7 | "scripts": { 8 | "create": "hugo new", 9 | "dev": "hugo server --disableFastRender --noHTTPCache", 10 | "format": "prettier **/** -w -c", 11 | "build": "hugo --minify --gc -b ${URL}", 12 | "build:branch": "hugo --minify --gc -b ${DEPLOY_URL}", 13 | "preview": "vite preview --outDir public" 14 | }, 15 | "dependencies": { 16 | "@tabler/icons": "^3.12.0", 17 | "@thulite/doks-core": "^1.7.0", 18 | "@thulite/images": "^3.3.0", 19 | "@thulite/inline-svg": "^1.1.0", 20 | "@thulite/seo": "^2.4.0", 21 | "github-buttons": "^2.29.1", 22 | "thulite": "^2.5.0" 23 | }, 24 | "devDependencies": { 25 | "prettier": "^3.3.3", 26 | "vite": "^5.4.2" 27 | }, 28 | "engines": { 29 | "node": ">=20.11.0" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /docs/resources/_gen/assets/scss/app.scss_901a6e181e810c5c7347a10d84f037ab.json: -------------------------------------------------------------------------------- 1 | {"Target":"main.a9c3f85c163b978572680a17d826de9b0ff2910f85dd3926df2255ee310d8153e4429b32fe1d95795105f91f218271aee9b9dd5cf274163b7521126bd18fdf86.css","MediaType":"text/css","Data":{"Integrity":"sha512-qcP4XBY7l4VyaAoX2Cbemw/ykQ+F3Tkm3yJV7jENgVPkQpsy/h2VeVEF+R8hgnGu6bndXPJ0Fjt1IRJr0Y/fhg=="}} -------------------------------------------------------------------------------- /docs/resources/_gen/assets/scss/app.scss_cdf9d7c9eb97e4550ded64a8776dd9e8.json: -------------------------------------------------------------------------------- 1 | {"Target":"main.3e9cfa18ef8eea818e5322c5eccce2bcedbda0100f6cc0469a0d0d315f86d01a5e46c2424f782e7e0c2e12da5c5afc98309d50b5aa60f98949293da86193c731.css","MediaType":"text/css","Data":{"Integrity":"sha512-Ppz6GO+O6oGOUyLF7MzivO29oBAPbMBGmg0NMV+G0BpeRsJCT3gufgwuEtpcWvyYMJ1Qtapg+YlJKT2oYZPHMQ=="}} -------------------------------------------------------------------------------- /docs/resources/_gen/images/_hu70523d59fb738bec5c06336403a46531_39940_c0fe83760c1bebd5a39d4ddb7fce622e.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/resources/_gen/images/_hu70523d59fb738bec5c06336403a46531_39940_c0fe83760c1bebd5a39d4ddb7fce622e.webp -------------------------------------------------------------------------------- /docs/resources/_gen/images/_hu89002a090cbbd5e6897ae6b591dddabc_33739_cb9243f2e37f830fb14160ae4284ce39.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/resources/_gen/images/_hu89002a090cbbd5e6897ae6b591dddabc_33739_cb9243f2e37f830fb14160ae4284ce39.webp -------------------------------------------------------------------------------- /docs/resources/_gen/images/cqrs-example-storming_7615831582150998571_huaaacfa63d5a84cf464fbe57abe466f11_96906_1549x914_resize_q85_h2_lanczos_3.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/resources/_gen/images/cqrs-example-storming_7615831582150998571_huaaacfa63d5a84cf464fbe57abe466f11_96906_1549x914_resize_q85_h2_lanczos_3.webp -------------------------------------------------------------------------------- /docs/resources/_gen/images/favicon_hufb12268f494215628cd81cc4fa356d3c_100807_180x180_resize_lanczos_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/resources/_gen/images/favicon_hufb12268f494215628cd81cc4fa356d3c_100807_180x180_resize_lanczos_3.png -------------------------------------------------------------------------------- /docs/resources/_gen/images/favicon_hufb12268f494215628cd81cc4fa356d3c_100807_192x192_resize_lanczos_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/resources/_gen/images/favicon_hufb12268f494215628cd81cc4fa356d3c_100807_192x192_resize_lanczos_3.png -------------------------------------------------------------------------------- /docs/resources/_gen/images/favicon_hufb12268f494215628cd81cc4fa356d3c_100807_32x32_resize_lanczos_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/resources/_gen/images/favicon_hufb12268f494215628cd81cc4fa356d3c_100807_32x32_resize_lanczos_3.png -------------------------------------------------------------------------------- /docs/resources/_gen/images/favicon_hufb12268f494215628cd81cc4fa356d3c_100807_512x512_resize_lanczos_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/resources/_gen/images/favicon_hufb12268f494215628cd81cc4fa356d3c_100807_512x512_resize_lanczos_3.png -------------------------------------------------------------------------------- /docs/resources/_gen/images/grafana_import_dashboard_6707854648249907356_huaaa3b6f44cdba346c1f24c07e8af91ec_92673_1024x786_resize_q85_h2_lanczos_3.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/resources/_gen/images/grafana_import_dashboard_6707854648249907356_huaaa3b6f44cdba346c1f24c07e8af91ec_92673_1024x786_resize_q85_h2_lanczos_3.webp -------------------------------------------------------------------------------- /docs/static/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/static/.gitkeep -------------------------------------------------------------------------------- /docs/static/fonts/quicksand/quicksand-v31-latin-500.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/static/fonts/quicksand/quicksand-v31-latin-500.woff2 -------------------------------------------------------------------------------- /docs/static/fonts/quicksand/quicksand-v31-latin-700.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/static/fonts/quicksand/quicksand-v31-latin-700.woff2 -------------------------------------------------------------------------------- /docs/static/fonts/quicksand/quicksand-v31-latin-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/static/fonts/quicksand/quicksand-v31-latin-regular.woff2 -------------------------------------------------------------------------------- /docs/static/img/pyramid.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ThreeDotsLabs/watermill/cdddd603198590141e394c3a0190301ba55f32ea/docs/static/img/pyramid.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ThreeDotsLabs/watermill 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/cenkalti/backoff/v3 v3.2.2 7 | github.com/go-chi/chi/v5 v5.1.0 8 | github.com/gogo/protobuf v1.3.2 9 | github.com/golang/protobuf v1.5.4 10 | github.com/google/uuid v1.6.0 11 | github.com/lithammer/shortuuid/v3 v3.0.7 12 | github.com/oklog/ulid v1.3.1 13 | github.com/pkg/errors v0.9.1 14 | github.com/prometheus/client_golang v1.20.2 15 | github.com/sony/gobreaker v1.0.0 16 | github.com/stretchr/testify v1.9.0 17 | google.golang.org/protobuf v1.34.2 18 | ) 19 | 20 | require ( 21 | github.com/beorn7/perks v1.0.1 // indirect 22 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 23 | github.com/davecgh/go-spew v1.1.1 // indirect 24 | github.com/klauspost/compress v1.17.9 // indirect 25 | github.com/kr/text v0.2.0 // indirect 26 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 27 | github.com/pmezard/go-difflib v1.0.0 // indirect 28 | github.com/prometheus/client_model v0.6.1 // indirect 29 | github.com/prometheus/common v0.55.0 // indirect 30 | github.com/prometheus/procfs v0.15.1 // indirect 31 | golang.org/x/sys v0.24.0 // indirect 32 | gopkg.in/yaml.v3 v3.0.1 // indirect 33 | ) 34 | -------------------------------------------------------------------------------- /internal/channel.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | // IsChannelClosed returns true if provided `chan struct{}` is closed. 4 | // IsChannelClosed panics if message is sent to this channel. 5 | func IsChannelClosed(channel chan struct{}) bool { 6 | select { 7 | case _, ok := <-channel: 8 | if ok { 9 | panic("received unexpected message") 10 | } 11 | return true 12 | default: 13 | return false 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /internal/channel_test.go: -------------------------------------------------------------------------------- 1 | package internal_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ThreeDotsLabs/watermill/internal" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestIsChannelClosed(t *testing.T) { 11 | closed := make(chan struct{}) 12 | close(closed) 13 | 14 | withSentValue := make(chan struct{}, 1) 15 | withSentValue <- struct{}{} 16 | 17 | testCases := []struct { 18 | Name string 19 | Channel chan struct{} 20 | ExpectedPanic bool 21 | ExpectedClosed bool 22 | }{ 23 | { 24 | Name: "not_closed", 25 | Channel: make(chan struct{}), 26 | ExpectedPanic: false, 27 | ExpectedClosed: false, 28 | }, 29 | { 30 | Name: "closed", 31 | Channel: closed, 32 | ExpectedPanic: false, 33 | ExpectedClosed: true, 34 | }, 35 | { 36 | Name: "with_sent_value", 37 | Channel: withSentValue, 38 | ExpectedPanic: true, 39 | ExpectedClosed: false, 40 | }, 41 | } 42 | 43 | for _, c := range testCases { 44 | t.Run(c.Name, func(t *testing.T) { 45 | testFunc := func() { 46 | closed := internal.IsChannelClosed(c.Channel) 47 | assert.EqualValues(t, c.ExpectedClosed, closed) 48 | } 49 | 50 | if c.ExpectedPanic { 51 | assert.Panics(t, testFunc) 52 | } else { 53 | assert.NotPanics(t, testFunc) 54 | } 55 | }) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /internal/name.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | // StructName returns a normalized name of the passed structure. 9 | func StructName(v interface{}) string { 10 | if s, ok := v.(fmt.Stringer); ok { 11 | return s.String() 12 | } 13 | 14 | s := fmt.Sprintf("%T", v) 15 | // trim the pointer marker, if any 16 | return strings.TrimLeft(s, "*") 17 | } 18 | -------------------------------------------------------------------------------- /internal/name_test.go: -------------------------------------------------------------------------------- 1 | package internal_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ThreeDotsLabs/watermill/internal" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | type testStruct struct{} 11 | 12 | type stringerStruct struct{} 13 | 14 | func (stringerStruct) String() string { 15 | return "stringer" 16 | } 17 | 18 | func TestStructName(t *testing.T) { 19 | testCases := []struct { 20 | Name string 21 | Struct interface{} 22 | ExpectedName string 23 | }{ 24 | { 25 | Name: "simple_struct", 26 | Struct: testStruct{}, 27 | ExpectedName: "internal_test.testStruct", 28 | }, 29 | { 30 | Name: "pointer_struct", 31 | Struct: &testStruct{}, 32 | ExpectedName: "internal_test.testStruct", 33 | }, 34 | { 35 | Name: "stringer", 36 | Struct: stringerStruct{}, 37 | ExpectedName: "stringer", 38 | }, 39 | } 40 | 41 | for _, c := range testCases { 42 | t.Run(c.Name, func(t *testing.T) { 43 | s := internal.StructName(c.Struct) 44 | assert.Equal(t, c.ExpectedName, s) 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /internal/norace.go: -------------------------------------------------------------------------------- 1 | //go:build !race 2 | // +build !race 3 | 4 | package internal 5 | 6 | const RaceEnabled = false 7 | -------------------------------------------------------------------------------- /internal/publisher/errors.go: -------------------------------------------------------------------------------- 1 | package publisher 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | ) 8 | 9 | type ErrCouldNotPublish struct { 10 | reasons map[string]error 11 | } 12 | 13 | func (e *ErrCouldNotPublish) addMsg(msg *message.Message, reason error) { 14 | e.reasons[msg.UUID] = reason 15 | } 16 | 17 | func NewErrCouldNotPublish() *ErrCouldNotPublish { 18 | return &ErrCouldNotPublish{make(map[string]error)} 19 | } 20 | 21 | func (e ErrCouldNotPublish) Len() int { 22 | return len(e.reasons) 23 | } 24 | 25 | func (e ErrCouldNotPublish) Error() string { 26 | if len(e.reasons) == 0 { 27 | return "" 28 | } 29 | b := strings.Builder{} 30 | b.WriteString("Could not publish the messages:\n") 31 | for uuid, reason := range e.reasons { 32 | b.WriteString(uuid + " : " + reason.Error() + "\n") 33 | } 34 | return b.String() 35 | } 36 | 37 | func (e ErrCouldNotPublish) Reasons() map[string]error { 38 | return e.reasons 39 | } 40 | -------------------------------------------------------------------------------- /internal/race.go: -------------------------------------------------------------------------------- 1 | //go:build race 2 | // +build race 3 | 4 | package internal 5 | 6 | const RaceEnabled = true 7 | -------------------------------------------------------------------------------- /message/messages.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | // Messages is a slice of messages. 4 | type Messages []*Message 5 | 6 | // IDs returns a slice of Messages' IDs. 7 | func (m Messages) IDs() []string { 8 | ids := make([]string, len(m)) 9 | 10 | for i, msg := range m { 11 | ids[i] = msg.UUID 12 | } 13 | 14 | return ids 15 | } 16 | -------------------------------------------------------------------------------- /message/messages_test.go: -------------------------------------------------------------------------------- 1 | package message_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestMessages_IDs(t *testing.T) { 11 | msgs := message.Messages{ 12 | message.NewMessage("1", nil), 13 | message.NewMessage("2", nil), 14 | message.NewMessage("3", nil), 15 | } 16 | 17 | assert.Equal(t, []string{"1", "2", "3"}, msgs.IDs()) 18 | } 19 | -------------------------------------------------------------------------------- /message/metadata.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | // Metadata is sent with every message to provide extra context without unmarshaling the message payload. 4 | type Metadata map[string]string 5 | 6 | // Get returns the metadata value for the given key. If the key is not found, an empty string is returned. 7 | func (m Metadata) Get(key string) string { 8 | if v, ok := m[key]; ok { 9 | return v 10 | } 11 | 12 | return "" 13 | } 14 | 15 | // Set sets the metadata key to value. 16 | func (m Metadata) Set(key, value string) { 17 | m[key] = value 18 | } 19 | -------------------------------------------------------------------------------- /message/router/middleware/circuit_breaker.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "github.com/ThreeDotsLabs/watermill/message" 5 | "github.com/sony/gobreaker" 6 | ) 7 | 8 | // CircuitBreaker is a middleware that wraps the handler in a circuit breaker. 9 | // Based on the configuration, the circuit breaker will fail fast if the handler keeps returning errors. 10 | // This is useful for preventing cascading failures. 11 | type CircuitBreaker struct { 12 | cb *gobreaker.CircuitBreaker 13 | } 14 | 15 | // NewCircuitBreaker returns a new CircuitBreaker middleware. 16 | // Refer to the gobreaker documentation for the available settings. 17 | func NewCircuitBreaker(settings gobreaker.Settings) CircuitBreaker { 18 | return CircuitBreaker{ 19 | cb: gobreaker.NewCircuitBreaker(settings), 20 | } 21 | } 22 | 23 | // Middleware returns the CircuitBreaker middleware. 24 | func (c CircuitBreaker) Middleware(h message.HandlerFunc) message.HandlerFunc { 25 | return func(msg *message.Message) ([]*message.Message, error) { 26 | out, err := c.cb.Execute(func() (interface{}, error) { 27 | return h(msg) 28 | }) 29 | 30 | var result []*message.Message 31 | if out != nil { 32 | result = out.([]*message.Message) 33 | } 34 | 35 | return result, err 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /message/router/middleware/circuit_breaker_test.go: -------------------------------------------------------------------------------- 1 | package middleware_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | 8 | "github.com/ThreeDotsLabs/watermill/message" 9 | "github.com/ThreeDotsLabs/watermill/message/router/middleware" 10 | "github.com/sony/gobreaker" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestCircuitBreaker(t *testing.T) { 15 | t.Parallel() 16 | 17 | count := 0 18 | failing := true 19 | 20 | h := middleware.NewCircuitBreaker( 21 | gobreaker.Settings{ 22 | Name: "test", 23 | Timeout: time.Millisecond * 50, 24 | }, 25 | ).Middleware(func(msg *message.Message) (messages []*message.Message, e error) { 26 | count++ 27 | 28 | if failing { 29 | return nil, errors.New("test error") 30 | } 31 | 32 | return nil, nil 33 | }) 34 | 35 | msg := message.NewMessage("1", nil) 36 | 37 | // The first 6 calls should fail and increment the count 38 | for i := 0; i < 6; i++ { 39 | _, err := h(msg) 40 | assert.Error(t, err) 41 | } 42 | 43 | assert.Equal(t, 6, count) 44 | 45 | // The next calls should fail and not increment the count (the circuit breaker is open) 46 | for i := 0; i < 4; i++ { 47 | _, err := h(msg) 48 | assert.Error(t, err) 49 | } 50 | assert.Equal(t, 6, count) 51 | 52 | time.Sleep(time.Millisecond * 100) 53 | failing = false 54 | 55 | // After a timeout, the Circuit Breaker is closed again 56 | for i := 0; i < 4; i++ { 57 | _, err := h(msg) 58 | assert.NoError(t, err) 59 | } 60 | assert.Equal(t, 10, count) 61 | } 62 | -------------------------------------------------------------------------------- /message/router/middleware/correlation.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "github.com/ThreeDotsLabs/watermill/message" 5 | ) 6 | 7 | // CorrelationIDMetadataKey is used to store the correlation ID in metadata. 8 | const CorrelationIDMetadataKey = "correlation_id" 9 | 10 | // SetCorrelationID sets a correlation ID for the message. 11 | // 12 | // SetCorrelationID should be called when the message enters the system. 13 | // When message is produced in a request (for example HTTP), 14 | // message correlation ID should be the same as the request's correlation ID. 15 | func SetCorrelationID(id string, msg *message.Message) { 16 | if MessageCorrelationID(msg) != "" { 17 | return 18 | } 19 | 20 | msg.Metadata.Set(CorrelationIDMetadataKey, id) 21 | } 22 | 23 | // MessageCorrelationID returns correlation ID from the message. 24 | func MessageCorrelationID(message *message.Message) string { 25 | return message.Metadata.Get(CorrelationIDMetadataKey) 26 | } 27 | 28 | // CorrelationID adds correlation ID to all messages produced by the handler. 29 | // ID is based on ID from message received by handler. 30 | // 31 | // To make CorrelationID working correctly, SetCorrelationID must be called to first message entering the system. 32 | func CorrelationID(h message.HandlerFunc) message.HandlerFunc { 33 | return func(message *message.Message) ([]*message.Message, error) { 34 | producedMessages, err := h(message) 35 | 36 | correlationID := MessageCorrelationID(message) 37 | for _, msg := range producedMessages { 38 | SetCorrelationID(correlationID, msg) 39 | } 40 | 41 | return producedMessages, err 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /message/router/middleware/correlation_test.go: -------------------------------------------------------------------------------- 1 | package middleware_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/pkg/errors" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/ThreeDotsLabs/watermill/message/router/middleware" 11 | 12 | "github.com/ThreeDotsLabs/watermill/message" 13 | ) 14 | 15 | func TestCorrelationID(t *testing.T) { 16 | handlerErr := errors.New("foo") 17 | 18 | handler := middleware.CorrelationID(func(msg *message.Message) ([]*message.Message, error) { 19 | return message.Messages{message.NewMessage("2", nil)}, handlerErr 20 | }) 21 | 22 | msg := message.NewMessage("1", nil) 23 | middleware.SetCorrelationID("correlation_id", msg) 24 | 25 | producedMsgs, err := handler(msg) 26 | 27 | assert.Equal(t, "2", producedMsgs[0].UUID) 28 | assert.Equal(t, middleware.MessageCorrelationID(producedMsgs[0]), "correlation_id") 29 | assert.Equal(t, handlerErr, err) 30 | } 31 | 32 | func TestSetCorrelationID_already_set(t *testing.T) { 33 | msg := message.NewMessage("", nil) 34 | 35 | middleware.SetCorrelationID("foo", msg) 36 | middleware.SetCorrelationID("bar", msg) 37 | 38 | assert.Equal(t, "foo", middleware.MessageCorrelationID(msg)) 39 | } 40 | -------------------------------------------------------------------------------- /message/router/middleware/delay_on_error.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ThreeDotsLabs/watermill/components/delay" 7 | "github.com/ThreeDotsLabs/watermill/message" 8 | ) 9 | 10 | // DelayOnError is a middleware that adds the delay metadata to the message if an error occurs. 11 | // 12 | // IMPORTANT: The delay metadata doesn't cause delays with all Pub/Subs! Using it won't have any effect on Pub/Subs that don't support it. 13 | // See the list of supported Pub/Subs in the documentation: https://watermill.io/advanced/delayed-messages/ 14 | type DelayOnError struct { 15 | // InitialInterval is the first interval between retries. Subsequent intervals will be scaled by Multiplier. 16 | InitialInterval time.Duration 17 | // MaxInterval sets the limit for the exponential backoff of retries. The interval will not be increased beyond MaxInterval. 18 | MaxInterval time.Duration 19 | // Multiplier is the factor by which the waiting interval will be multiplied between retries. 20 | Multiplier float64 21 | } 22 | 23 | func (d *DelayOnError) Middleware(h message.HandlerFunc) message.HandlerFunc { 24 | return func(msg *message.Message) ([]*message.Message, error) { 25 | msgs, err := h(msg) 26 | if err != nil { 27 | d.applyDelay(msg) 28 | } 29 | 30 | return msgs, err 31 | } 32 | } 33 | 34 | func (d *DelayOnError) applyDelay(msg *message.Message) { 35 | delayedForStr := msg.Metadata.Get(delay.DelayedForKey) 36 | delayedFor, err := time.ParseDuration(delayedForStr) 37 | if delayedForStr != "" && err == nil { 38 | delayedFor *= time.Duration(d.Multiplier) 39 | if delayedFor > d.MaxInterval { 40 | delayedFor = d.MaxInterval 41 | } 42 | 43 | delay.Message(msg, delay.For(delayedFor)) 44 | } else { 45 | delay.Message(msg, delay.For(d.InitialInterval)) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /message/router/middleware/delay_on_error_test.go: -------------------------------------------------------------------------------- 1 | package middleware_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/ThreeDotsLabs/watermill/components/delay" 11 | "github.com/ThreeDotsLabs/watermill/message" 12 | "github.com/ThreeDotsLabs/watermill/message/router/middleware" 13 | ) 14 | 15 | func TestDelayOnError(t *testing.T) { 16 | m := middleware.DelayOnError{ 17 | InitialInterval: time.Second, 18 | MaxInterval: time.Second * 10, 19 | Multiplier: 2, 20 | } 21 | 22 | msg := message.NewMessage("1", []byte("test")) 23 | 24 | getDelayFor := func(msg *message.Message) string { 25 | return msg.Metadata.Get(delay.DelayedForKey) 26 | } 27 | 28 | okHandler := func(msg *message.Message) ([]*message.Message, error) { 29 | return nil, nil 30 | } 31 | 32 | errHandler := func(msg *message.Message) ([]*message.Message, error) { 33 | return nil, errors.New("error") 34 | } 35 | 36 | assert.Equal(t, "", getDelayFor(msg)) 37 | 38 | _, _ = m.Middleware(okHandler)(msg) 39 | assert.Equal(t, "", getDelayFor(msg)) 40 | 41 | _, _ = m.Middleware(errHandler)(msg) 42 | assert.Equal(t, "1s", getDelayFor(msg)) 43 | 44 | _, _ = m.Middleware(errHandler)(msg) 45 | assert.Equal(t, "2s", getDelayFor(msg)) 46 | 47 | _, _ = m.Middleware(errHandler)(msg) 48 | assert.Equal(t, "4s", getDelayFor(msg)) 49 | 50 | _, _ = m.Middleware(errHandler)(msg) 51 | assert.Equal(t, "8s", getDelayFor(msg)) 52 | 53 | _, _ = m.Middleware(errHandler)(msg) 54 | assert.Equal(t, "10s", getDelayFor(msg)) 55 | 56 | _, _ = m.Middleware(errHandler)(msg) 57 | assert.Equal(t, "10s", getDelayFor(msg)) 58 | } 59 | -------------------------------------------------------------------------------- /message/router/middleware/duplicator.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "github.com/ThreeDotsLabs/watermill/message" 5 | ) 6 | 7 | // Duplicator is processing messages twice, to ensure that the endpoint is idempotent. 8 | func Duplicator(h message.HandlerFunc) message.HandlerFunc { 9 | return func(msg *message.Message) ([]*message.Message, error) { 10 | firstProducedMessages, firstErr := h(msg) 11 | if firstErr != nil { 12 | return nil, firstErr 13 | } 14 | 15 | secondProducedMessages, secondErr := h(msg) 16 | if secondErr != nil { 17 | return nil, secondErr 18 | } 19 | 20 | return append(firstProducedMessages, secondProducedMessages...), nil 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /message/router/middleware/duplicator_test.go: -------------------------------------------------------------------------------- 1 | package middleware_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | 9 | "github.com/ThreeDotsLabs/watermill/message" 10 | "github.com/ThreeDotsLabs/watermill/message/router/middleware" 11 | ) 12 | 13 | var ( 14 | someMsg = message.NewMessage("1", nil) 15 | ) 16 | 17 | func TestDuplicator(t *testing.T) { 18 | var executionsCount int 19 | producedMessages, err := middleware.Duplicator(func(msg *message.Message) ([]*message.Message, error) { 20 | executionsCount++ 21 | return []*message.Message{msg}, nil 22 | })(someMsg) 23 | 24 | assert.NoError(t, err) 25 | assert.Len(t, producedMessages, 2) 26 | assert.Equal(t, "1", producedMessages[0].UUID) 27 | assert.Equal(t, "1", producedMessages[1].UUID) 28 | assert.Equal(t, 2, executionsCount) 29 | } 30 | 31 | func TestDuplicator_errors(t *testing.T) { 32 | _, err := middleware.Duplicator(func(msg *message.Message) ([]*message.Message, error) { 33 | return nil, errors.New("some error") 34 | })(someMsg) 35 | assert.Error(t, err, "some error") 36 | 37 | var wasExecuted bool 38 | _, err = middleware.Duplicator(func(msg *message.Message) ([]*message.Message, error) { 39 | if wasExecuted { 40 | return nil, errors.New("some other error") 41 | } 42 | 43 | wasExecuted = true 44 | return []*message.Message{msg}, nil 45 | })(someMsg) 46 | assert.Error(t, err, "some other error") 47 | } 48 | -------------------------------------------------------------------------------- /message/router/middleware/ignore_errors.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "github.com/ThreeDotsLabs/watermill/message" 5 | "github.com/pkg/errors" 6 | ) 7 | 8 | // IgnoreErrors provides a middleware that makes the handler ignore some explicitly whitelisted errors. 9 | type IgnoreErrors struct { 10 | ignoredErrors map[string]struct{} 11 | } 12 | 13 | // NewIgnoreErrors creates a new IgnoreErrors middleware. 14 | func NewIgnoreErrors(errs []error) IgnoreErrors { 15 | errsMap := make(map[string]struct{}, len(errs)) 16 | 17 | for _, err := range errs { 18 | errsMap[err.Error()] = struct{}{} 19 | } 20 | 21 | return IgnoreErrors{errsMap} 22 | } 23 | 24 | // Middleware returns the IgnoreErrors middleware. 25 | func (i IgnoreErrors) Middleware(h message.HandlerFunc) message.HandlerFunc { 26 | return func(msg *message.Message) ([]*message.Message, error) { 27 | events, err := h(msg) 28 | if err != nil { 29 | if _, ok := i.ignoredErrors[errors.Cause(err).Error()]; ok { 30 | return events, nil 31 | } 32 | 33 | return events, err 34 | } 35 | 36 | return events, nil 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /message/router/middleware/ignore_errors_test.go: -------------------------------------------------------------------------------- 1 | package middleware_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | "github.com/ThreeDotsLabs/watermill/message/router/middleware" 8 | "github.com/pkg/errors" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestIgnoreErrors_Middleware(t *testing.T) { 13 | testCases := []struct { 14 | Name string 15 | IgnoredErrors []error 16 | TestError error 17 | ShouldBeIgnored bool 18 | }{ 19 | { 20 | Name: "ignored_error", 21 | IgnoredErrors: []error{errors.New("test")}, 22 | TestError: errors.New("test"), 23 | ShouldBeIgnored: true, 24 | }, 25 | { 26 | Name: "not_ignored_error", 27 | IgnoredErrors: []error{errors.New("test")}, 28 | TestError: errors.New("not_ignored"), 29 | ShouldBeIgnored: false, 30 | }, 31 | { 32 | Name: "wrapped_error_should_ignore", 33 | IgnoredErrors: []error{errors.New("test")}, 34 | TestError: errors.Wrap(errors.New("test"), "wrapped"), 35 | ShouldBeIgnored: true, 36 | }, 37 | } 38 | 39 | for _, c := range testCases { 40 | t.Run(c.Name, func(t *testing.T) { 41 | m := middleware.NewIgnoreErrors(c.IgnoredErrors) 42 | 43 | messagesToProduce := []*message.Message{message.NewMessage("1", nil)} 44 | 45 | producedMessages, err := m.Middleware(func(msg *message.Message) ([]*message.Message, error) { 46 | return messagesToProduce, c.TestError 47 | })(nil) 48 | 49 | if c.ShouldBeIgnored { 50 | assert.NoError(t, err) 51 | } else { 52 | assert.Equal(t, c.TestError, err) 53 | } 54 | 55 | assert.Equal(t, messagesToProduce, producedMessages) 56 | }) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /message/router/middleware/instant_ack.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import "github.com/ThreeDotsLabs/watermill/message" 4 | 5 | // InstantAck makes the handler instantly acknowledge the incoming message, regardless of any errors. 6 | // It may be used to gain throughput, but at a cost: 7 | // If you had exactly-once delivery, you may expect at-least-once instead. 8 | // If you had ordered messages, the ordering might be broken. 9 | func InstantAck(h message.HandlerFunc) message.HandlerFunc { 10 | return func(message *message.Message) ([]*message.Message, error) { 11 | message.Ack() 12 | return h(message) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /message/router/middleware/instant_ack_test.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | "github.com/pkg/errors" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestInstantAck(t *testing.T) { 12 | producedMessages := message.Messages{message.NewMessage("2", nil)} 13 | producedErr := errors.New("foo") 14 | 15 | h := InstantAck(func(msg *message.Message) (messages []*message.Message, e error) { 16 | return producedMessages, producedErr 17 | }) 18 | 19 | msg := message.NewMessage("1", nil) 20 | 21 | handlerMessages, handlerErr := h(msg) 22 | assert.EqualValues(t, producedMessages, handlerMessages) 23 | assert.Equal(t, producedErr, handlerErr) 24 | 25 | select { 26 | case <-msg.Acked(): 27 | // ok 28 | case <-msg.Nacked(): 29 | t.Fatal("expected ack, not nack") 30 | default: 31 | t.Fatal("no ack received") 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /message/router/middleware/message_test.go: -------------------------------------------------------------------------------- 1 | package middleware_test 2 | 3 | import ( 4 | "github.com/ThreeDotsLabs/watermill" 5 | "github.com/ThreeDotsLabs/watermill/message" 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type mockPublisherBehaviour int 10 | 11 | const ( 12 | BehaviourAlwaysOK mockPublisherBehaviour = iota + 1 13 | BehaviourAlwaysFail 14 | BehaviourAlwaysPanic 15 | ) 16 | 17 | var ( 18 | errClosed = errors.New("closed") 19 | errFailed = errors.New("failed") 20 | errPanicked = errors.New("panicked") 21 | ) 22 | 23 | type mockPublisher struct { 24 | behaviour mockPublisherBehaviour 25 | closed bool 26 | 27 | produced []*message.Message 28 | } 29 | 30 | func (mp *mockPublisher) Publish(topic string, messages ...*message.Message) error { 31 | if mp.closed { 32 | return errClosed 33 | } 34 | 35 | switch mp.behaviour { 36 | case BehaviourAlwaysOK: 37 | case BehaviourAlwaysFail: 38 | return errFailed 39 | case BehaviourAlwaysPanic: 40 | panic(errPanicked) 41 | } 42 | 43 | mp.produced = append(mp.produced, messages...) 44 | return nil 45 | } 46 | 47 | func (mp *mockPublisher) Close() error { 48 | mp.closed = true 49 | return nil 50 | } 51 | 52 | func (mp *mockPublisher) PopMessages() []*message.Message { 53 | defer func() { mp.produced = []*message.Message{} }() 54 | return mp.produced 55 | } 56 | 57 | var handlerFuncAlwaysOKMessages = []*message.Message{ 58 | message.NewMessage(watermill.NewUUID(), nil), 59 | message.NewMessage(watermill.NewUUID(), nil), 60 | } 61 | 62 | func handlerFuncAlwaysOK(*message.Message) ([]*message.Message, error) { 63 | return handlerFuncAlwaysOKMessages, nil 64 | } 65 | 66 | func handlerFuncAlwaysFailing(*message.Message) ([]*message.Message, error) { 67 | return nil, errFailed 68 | } 69 | -------------------------------------------------------------------------------- /message/router/middleware/randomfail.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "math/rand" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | "github.com/pkg/errors" 8 | ) 9 | 10 | func shouldFail(probability float32) bool { 11 | r := rand.Float32() 12 | return r <= probability 13 | } 14 | 15 | // RandomFail makes the handler fail with an error based on random chance. Error probability should be in the range (0,1). 16 | func RandomFail(errorProbability float32) message.HandlerMiddleware { 17 | return func(h message.HandlerFunc) message.HandlerFunc { 18 | return func(message *message.Message) ([]*message.Message, error) { 19 | if shouldFail(errorProbability) { 20 | return nil, errors.New("random fail occurred") 21 | } 22 | 23 | return h(message) 24 | } 25 | } 26 | } 27 | 28 | // RandomPanic makes the handler panic based on random chance. Panic probability should be in the range (0,1). 29 | func RandomPanic(panicProbability float32) message.HandlerMiddleware { 30 | return func(h message.HandlerFunc) message.HandlerFunc { 31 | return func(message *message.Message) ([]*message.Message, error) { 32 | if shouldFail(panicProbability) { 33 | panic("random panic occurred") 34 | } 35 | 36 | return h(message) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /message/router/middleware/randomfail_test.go: -------------------------------------------------------------------------------- 1 | package middleware_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | "github.com/ThreeDotsLabs/watermill/message/router/middleware" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestRandomFail(t *testing.T) { 12 | h := middleware.RandomFail(1)(func(msg *message.Message) (messages []*message.Message, e error) { 13 | return nil, nil 14 | }) 15 | 16 | _, err := h(message.NewMessage("1", nil)) 17 | assert.Error(t, err) 18 | } 19 | 20 | func TestRandomPanic(t *testing.T) { 21 | h := middleware.RandomPanic(1)(func(msg *message.Message) (messages []*message.Message, e error) { 22 | return nil, nil 23 | }) 24 | 25 | assert.Panics(t, func() { 26 | _, _ = h(message.NewMessage("1", nil)) 27 | }) 28 | } 29 | -------------------------------------------------------------------------------- /message/router/middleware/recoverer.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "fmt" 5 | "runtime/debug" 6 | 7 | "github.com/ThreeDotsLabs/watermill/message" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | // RecoveredPanicError holds the recovered panic's error along with the stacktrace. 12 | type RecoveredPanicError struct { 13 | V interface{} 14 | Stacktrace string 15 | } 16 | 17 | func (p RecoveredPanicError) Error() string { 18 | return fmt.Sprintf("panic occurred: %#v, stacktrace: \n%s", p.V, p.Stacktrace) 19 | } 20 | 21 | // Recoverer recovers from any panic in the handler and appends RecoveredPanicError with the stacktrace 22 | // to any error returned from the handler. 23 | func Recoverer(h message.HandlerFunc) message.HandlerFunc { 24 | return func(event *message.Message) (events []*message.Message, err error) { 25 | panicked := true 26 | 27 | defer func() { 28 | if r := recover(); r != nil || panicked { 29 | err = errors.WithStack(RecoveredPanicError{V: r, Stacktrace: string(debug.Stack())}) 30 | } 31 | }() 32 | 33 | events, err = h(event) 34 | panicked = false 35 | return events, err 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /message/router/middleware/recoverer_test.go: -------------------------------------------------------------------------------- 1 | package middleware_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | 8 | "github.com/ThreeDotsLabs/watermill/message" 9 | "github.com/ThreeDotsLabs/watermill/message/router/middleware" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestRecoverer_Panic(t *testing.T) { 14 | h := middleware.Recoverer(func(msg *message.Message) (messages []*message.Message, e error) { 15 | panic("foo") 16 | }) 17 | 18 | _, err := h(message.NewMessage("1", nil)) 19 | require.Error(t, err) 20 | assert.Contains(t, err.Error(), "message/router/middleware/recoverer.go") // stacktrace part 21 | } 22 | 23 | func TestRecoverer_PanicNil(t *testing.T) { 24 | h := middleware.Recoverer(func(msg *message.Message) (messages []*message.Message, e error) { 25 | panic(nil) 26 | }) 27 | 28 | _, err := h(message.NewMessage("1", nil)) 29 | require.Error(t, err) 30 | } 31 | 32 | func TestRecoverer_NoPanic(t *testing.T) { 33 | h := middleware.Recoverer(func(msg *message.Message) (messages []*message.Message, e error) { 34 | return nil, nil 35 | }) 36 | 37 | _, err := h(message.NewMessage("1", nil)) 38 | require.NoError(t, err) 39 | } 40 | -------------------------------------------------------------------------------- /message/router/middleware/throttle.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | ) 8 | 9 | // Throttle provides a middleware that limits the amount of messages processed per unit of time. 10 | // This may be done e.g. to prevent excessive load caused by running a handler on a long queue of unprocessed messages. 11 | type Throttle struct { 12 | ticker *time.Ticker 13 | } 14 | 15 | // NewThrottle creates a new Throttle middleware. 16 | // Example duration and count: NewThrottle(10, time.Second) for 10 messages per second 17 | func NewThrottle(count int64, duration time.Duration) *Throttle { 18 | return &Throttle{ 19 | ticker: time.NewTicker(duration / time.Duration(count)), 20 | } 21 | } 22 | 23 | // Middleware returns the Throttle middleware. 24 | func (t Throttle) Middleware(h message.HandlerFunc) message.HandlerFunc { 25 | return func(message *message.Message) ([]*message.Message, error) { 26 | // throttle is shared by multiple handlers, which will wait for their "tick" 27 | <-t.ticker.C 28 | 29 | return h(message) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /message/router/middleware/timeout.go: -------------------------------------------------------------------------------- 1 | package middleware 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "github.com/ThreeDotsLabs/watermill/message" 8 | ) 9 | 10 | // Timeout makes the handler cancel the incoming message's context after a specified time. 11 | // Any timeout-sensitive functionality of the handler should listen on msg.Context().Done() to know when to fail. 12 | func Timeout(timeout time.Duration) func(message.HandlerFunc) message.HandlerFunc { 13 | return func(h message.HandlerFunc) message.HandlerFunc { 14 | return func(msg *message.Message) ([]*message.Message, error) { 15 | ctx, cancel := context.WithTimeout(msg.Context(), timeout) 16 | defer func() { 17 | cancel() 18 | }() 19 | 20 | msg.SetContext(ctx) 21 | return h(msg) 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /message/router/middleware/timeout_test.go: -------------------------------------------------------------------------------- 1 | package middleware_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/require" 9 | 10 | "github.com/ThreeDotsLabs/watermill/message" 11 | "github.com/ThreeDotsLabs/watermill/message/router/middleware" 12 | ) 13 | 14 | func TestTimeout(t *testing.T) { 15 | timeout := middleware.Timeout(time.Millisecond * 10) 16 | 17 | h := timeout(func(msg *message.Message) ([]*message.Message, error) { 18 | delay := time.After(time.Millisecond * 100) 19 | 20 | select { 21 | case <-msg.Context().Done(): 22 | return nil, nil 23 | case <-delay: 24 | return nil, errors.New("timeout did not occur") 25 | } 26 | }) 27 | 28 | _, err := h(message.NewMessage("any-uuid", nil)) 29 | require.NoError(t, err) 30 | } 31 | -------------------------------------------------------------------------------- /message/router/plugin/signals.go: -------------------------------------------------------------------------------- 1 | package plugin 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | 9 | "github.com/ThreeDotsLabs/watermill/message" 10 | ) 11 | 12 | // SignalsHandler is a plugin that kills the router after SIGINT or SIGTERM is sent to the process. 13 | func SignalsHandler(r *message.Router) error { 14 | sigs := make(chan os.Signal, 1) 15 | signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) 16 | 17 | go func() { 18 | sig := <-sigs 19 | r.Logger().Info(fmt.Sprintf("Received %s signal, closing\n", sig), nil) 20 | 21 | err := r.Close() 22 | if err != nil { 23 | r.Logger().Error("Router close failed", err, nil) 24 | } 25 | }() 26 | return nil 27 | } 28 | -------------------------------------------------------------------------------- /message/router_context.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type ctxKey string 8 | 9 | const ( 10 | handlerNameKey ctxKey = "handler_name" 11 | publisherNameKey ctxKey = "publisher_name" 12 | subscriberNameKey ctxKey = "subscriber_name" 13 | subscribeTopicKey ctxKey = "subscribe_topic" 14 | publishTopicKey ctxKey = "publish_topic" 15 | ) 16 | 17 | func valFromCtx(ctx context.Context, key ctxKey) string { 18 | val, ok := ctx.Value(key).(string) 19 | if !ok { 20 | return "" 21 | } 22 | return val 23 | } 24 | 25 | // HandlerNameFromCtx returns the name of the message handler in the router that consumed the message. 26 | func HandlerNameFromCtx(ctx context.Context) string { 27 | return valFromCtx(ctx, handlerNameKey) 28 | } 29 | 30 | // PublisherNameFromCtx returns the name of the message publisher type that published the message in the router. 31 | // For example, for Kafka it will be `kafka.Publisher`. 32 | func PublisherNameFromCtx(ctx context.Context) string { 33 | return valFromCtx(ctx, publisherNameKey) 34 | } 35 | 36 | // SubscriberNameFromCtx returns the name of the message subscriber type that subscribed to the message in the router. 37 | // For example, for Kafka it will be `kafka.Subscriber`. 38 | func SubscriberNameFromCtx(ctx context.Context) string { 39 | return valFromCtx(ctx, subscriberNameKey) 40 | } 41 | 42 | // SubscribeTopicFromCtx returns the topic from which message was received in the router. 43 | func SubscribeTopicFromCtx(ctx context.Context) string { 44 | return valFromCtx(ctx, subscribeTopicKey) 45 | } 46 | 47 | // PublishTopicFromCtx returns the topic to which message will be published by the router. 48 | func PublishTopicFromCtx(ctx context.Context) string { 49 | return valFromCtx(ctx, publishTopicKey) 50 | } 51 | -------------------------------------------------------------------------------- /message/subscriber/read.go: -------------------------------------------------------------------------------- 1 | package subscriber 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ThreeDotsLabs/watermill/message" 7 | ) 8 | 9 | // BulkRead reads provided amount of messages from the provided channel, until a timeout occurs or the limit is reached. 10 | func BulkRead(messagesCh <-chan *message.Message, limit int, timeout time.Duration) (receivedMessages message.Messages, all bool) { 11 | MessagesLoop: 12 | for len(receivedMessages) < limit { 13 | select { 14 | case msg, ok := <-messagesCh: 15 | if !ok { 16 | break MessagesLoop 17 | } 18 | 19 | receivedMessages = append(receivedMessages, msg) 20 | msg.Ack() 21 | case <-time.After(timeout): 22 | break MessagesLoop 23 | } 24 | } 25 | 26 | return receivedMessages, len(receivedMessages) == limit 27 | } 28 | 29 | // BulkReadWithDeduplication reads provided number of messages from the provided channel, ignoring duplicates, 30 | // until a timeout occurs or the limit is reached. 31 | func BulkReadWithDeduplication(messagesCh <-chan *message.Message, limit int, timeout time.Duration) (receivedMessages message.Messages, all bool) { 32 | receivedIDs := map[string]struct{}{} 33 | 34 | MessagesLoop: 35 | for len(receivedMessages) < limit { 36 | select { 37 | case msg, ok := <-messagesCh: 38 | if !ok { 39 | break MessagesLoop 40 | } 41 | 42 | if _, ok := receivedIDs[msg.UUID]; !ok { 43 | receivedIDs[msg.UUID] = struct{}{} 44 | receivedMessages = append(receivedMessages, msg) 45 | } 46 | msg.Ack() 47 | case <-time.After(timeout): 48 | break MessagesLoop 49 | } 50 | } 51 | 52 | return receivedMessages, len(receivedMessages) == limit 53 | } 54 | -------------------------------------------------------------------------------- /netlify.toml: -------------------------------------------------------------------------------- 1 | [build] 2 | command = "./build.sh --copy && npm run build" 3 | base = "docs/" 4 | publish = "docs/public/" 5 | 6 | [build.environment] 7 | NODE_VERSION = "20.11.0" 8 | NPM_VERSION = "10.2.4" 9 | HUGO_VERSION = "0.127.0" 10 | 11 | [context.deploy-preview] 12 | command = "./build.sh --copy && npm run build:branch" 13 | 14 | [context.branch-deploy] 15 | command = "./build.sh --copy && npm run build:branch" 16 | 17 | [[redirects]] 18 | from = "/api/event" 19 | to = "https://academy-api.threedots.tech/api/event" 20 | force = true 21 | status = 200 22 | 23 | [[redirects]] 24 | from = "/docs/fanin" 25 | to = "/advanced/fanin/" 26 | status = 301 27 | 28 | [[redirects]] 29 | from = "/docs/forwarder" 30 | to = "/advanced/forwarder/" 31 | status = 301 32 | 33 | [[redirects]] 34 | from = "/docs/metrics" 35 | to = "/advanced/metrics/" 36 | status = 301 37 | 38 | [[redirects]] 39 | from = "/docs/pub-sub-implementing" 40 | to = "/development/pub-sub-implementing/" 41 | status = 301 42 | 43 | [[redirects]] 44 | from = "/pubsubs/amazonsqs/" 45 | to = "/pubsubs/aws/" 46 | status = 301 47 | -------------------------------------------------------------------------------- /pubsub/doc.go: -------------------------------------------------------------------------------- 1 | // Infrastructure directory contains Pub/Subs implementations. 2 | // 3 | // Detailed Pub/Subs docs: https://watermill.io/pubsubs/ 4 | // Getting started guide: https://watermill.io/docs/getting-started/ 5 | 6 | package pubsub 7 | -------------------------------------------------------------------------------- /pubsub/gochannel/doc.go: -------------------------------------------------------------------------------- 1 | // This is just the simplest Pub/Sub implementation 2 | // 3 | // All Pub/Sub implementations can be found at https://watermill.io/pubsubs/ 4 | 5 | package gochannel 6 | -------------------------------------------------------------------------------- /pubsub/gochannel/pubsub_bench_test.go: -------------------------------------------------------------------------------- 1 | package gochannel_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/ThreeDotsLabs/watermill/pubsub/gochannel" 7 | 8 | "github.com/ThreeDotsLabs/watermill/pubsub/tests" 9 | 10 | "github.com/ThreeDotsLabs/watermill" 11 | "github.com/ThreeDotsLabs/watermill/message" 12 | ) 13 | 14 | func BenchmarkSubscriber(b *testing.B) { 15 | tests.BenchSubscriber(b, func(n int) (message.Publisher, message.Subscriber) { 16 | pubSub := gochannel.NewGoChannel( 17 | gochannel.Config{OutputChannelBuffer: int64(n)}, watermill.NopLogger{}, 18 | ) 19 | return pubSub, pubSub 20 | }) 21 | } 22 | 23 | func BenchmarkSubscriberPersistent(b *testing.B) { 24 | tests.BenchSubscriber(b, func(n int) (message.Publisher, message.Subscriber) { 25 | pubSub := gochannel.NewGoChannel( 26 | gochannel.Config{ 27 | OutputChannelBuffer: int64(n), 28 | Persistent: true, 29 | }, 30 | watermill.NopLogger{}, 31 | ) 32 | return pubSub, pubSub 33 | }) 34 | } 35 | -------------------------------------------------------------------------------- /pubsub/gochannel/pubsub_stress_test.go: -------------------------------------------------------------------------------- 1 | //go:build stress 2 | // +build stress 3 | 4 | package gochannel_test 5 | 6 | import ( 7 | "testing" 8 | 9 | "github.com/ThreeDotsLabs/watermill/pubsub/tests" 10 | ) 11 | 12 | func TestPublishSubscribe_stress(t *testing.T) { 13 | tests.TestPubSubStressTest( 14 | t, 15 | tests.Features{ 16 | ConsumerGroups: false, 17 | ExactlyOnceDelivery: true, 18 | GuaranteedOrder: false, 19 | Persistent: false, 20 | RequireSingleInstance: true, 21 | }, 22 | createPersistentPubSub, 23 | nil, 24 | ) 25 | } 26 | -------------------------------------------------------------------------------- /pubsub/sync/waitgroup.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | ) 7 | 8 | // WaitGroupTimeout adds timeout feature for sync.WaitGroup.Wait(). 9 | // It returns true, when timed out. 10 | func WaitGroupTimeout(wg *sync.WaitGroup, timeout time.Duration) bool { 11 | wgClosed := make(chan struct{}, 1) 12 | go func() { 13 | wg.Wait() 14 | wgClosed <- struct{}{} 15 | }() 16 | 17 | select { 18 | case <-wgClosed: 19 | return false 20 | case <-time.After(timeout): 21 | return true 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pubsub/sync/waitgroup_test.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestWaitGroupTimeout_no_timeout(t *testing.T) { 12 | wg := &sync.WaitGroup{} 13 | 14 | timedout := WaitGroupTimeout(wg, time.Millisecond*100) 15 | assert.False(t, timedout) 16 | } 17 | 18 | func TestWaitGroupTimeout_timeout(t *testing.T) { 19 | wg := &sync.WaitGroup{} 20 | wg.Add(1) 21 | 22 | timedout := WaitGroupTimeout(wg, time.Millisecond*100) 23 | assert.True(t, timedout) 24 | } 25 | -------------------------------------------------------------------------------- /pubsub/tests/bench_pubsub.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/ThreeDotsLabs/watermill/message" 9 | "github.com/ThreeDotsLabs/watermill/message/subscriber" 10 | ) 11 | 12 | // BenchmarkPubSubConstructor is a function that creates a Publisher and Subscriber to be used for benchmarks. 13 | type BenchmarkPubSubConstructor func(n int) (message.Publisher, message.Subscriber) 14 | 15 | // BenchSubscriber runs benchmark on a message Subscriber. 16 | func BenchSubscriber(b *testing.B, pubSubConstructor BenchmarkPubSubConstructor) { 17 | pub, sub := pubSubConstructor(b.N) 18 | topicName := testTopicName(TestContext{TestID: NewTestID()}) 19 | 20 | messages, err := sub.Subscribe(context.Background(), topicName) 21 | if err != nil { 22 | b.Fatal(err) 23 | } 24 | 25 | go func() { 26 | for i := 0; i < b.N; i++ { 27 | msg := message.NewMessage("1", nil) 28 | err := pub.Publish(topicName, msg) 29 | if err != nil { 30 | panic(err) 31 | } 32 | } 33 | }() 34 | 35 | b.ResetTimer() 36 | 37 | consumedMessages, all := subscriber.BulkRead(messages, b.N, time.Second*60) 38 | if !all { 39 | b.Fatalf("not all messages received, have %d, expected %d", len(consumedMessages), b.N) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /pubsub/tests/test_pubsub_stress.go: -------------------------------------------------------------------------------- 1 | //go:build stress 2 | // +build stress 3 | 4 | package tests 5 | 6 | import ( 7 | "runtime" 8 | ) 9 | 10 | func init() { 11 | // stress tests may work a bit slower 12 | defaultTimeout *= 5 13 | 14 | // Set GOMAXPROCS to double the number of CPUs 15 | runtime.GOMAXPROCS(runtime.GOMAXPROCS(0) * 2) 16 | } 17 | -------------------------------------------------------------------------------- /tools/mill/.default-config.yml: -------------------------------------------------------------------------------- 1 | # These are the current default settings for the Watermill CLI tool. 2 | # Use them as a template for your own config 3 | log: false 4 | trace: false 5 | debug: false 6 | 7 | amqp: 8 | uri: "" 9 | durable: true 10 | consume: 11 | exchange: "" 12 | queue: "" 13 | produce: 14 | exchange: "" 15 | exchangetype: fanout 16 | routingkey: "" 17 | 18 | googlecloud: 19 | projectid: "" 20 | topic: "" 21 | consume: 22 | subscription: "" 23 | produce: 24 | subscription: 25 | add: 26 | ackdeadline: 10s 27 | labels: "" 28 | retainacked: false 29 | retentionduration: 168h0m0s 30 | rm: 31 | # no flags for rm yet 32 | 33 | kafka: 34 | brokers: [] 35 | topic: "" 36 | consume: 37 | consumergroup: "" 38 | frombeginning: false 39 | produce: 40 | # no flags for produce yet 41 | -------------------------------------------------------------------------------- /tools/mill/Makefile: -------------------------------------------------------------------------------- 1 | mill: 2 | go build -o mill main.go 3 | -------------------------------------------------------------------------------- /tools/mill/cmd/internal/indent.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import "strings" 4 | 5 | // Indent indents all lines in the given string with a given prefix. 6 | func Indent(s, prefix string) string { 7 | endsWithNewline := strings.HasSuffix(s, "\n") 8 | split := strings.Split(s, "\n") 9 | 10 | for i, ss := range split { 11 | split[i] = prefix + ss 12 | } 13 | joined := strings.Join(split, "\n") 14 | if endsWithNewline { 15 | joined += "\n" 16 | } 17 | 18 | return joined 19 | } 20 | -------------------------------------------------------------------------------- /tools/mill/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "github.com/ThreeDotsLabs/watermill/tools/mill/cmd" 4 | 5 | func main() { 6 | cmd.Execute() 7 | } 8 | 9 | // TODO: alternative input/output modes: json, gob, protobuf... (?) 10 | -------------------------------------------------------------------------------- /tools/pq/README.md: -------------------------------------------------------------------------------- 1 | # pq 2 | 3 | pq is a CLI tool for working with delayed messages on poison queues. 4 | 5 | For now, it supports the PostgreSQL Pub/Sub implementation. 6 | 7 | ## Install 8 | 9 | ```bash 10 | go install github.com/ThreeDotsLabs/watermill/tools/pq@latest 11 | ``` 12 | 13 | ## Usage 14 | 15 | Set the `DATABASE_URL` environment variable to your PostgreSQL connection string. 16 | 17 | For example, to connect to the database used for the [delayed requeue example](../../_examples/real-world-examples/delayed-requeue): 18 | 19 | ```bash 20 | export DATABASE_URL="postgres://watermill:password@postgres:5432/watermill?sslmode=disable" 21 | ``` 22 | 23 | ```bash 24 | pq -backend postgres -topic requeue 25 | ``` 26 | 27 | This will use the default `watermill_` prefix, so will use the `watermill_requeue` table. 28 | 29 | If you use a custom prefix, use the `-raw-topic` flag instead: 30 | 31 | ```bash 32 | pq -backend postgres -raw-topic my_prefix_requeue 33 | ``` 34 | 35 | ## Commands 36 | 37 | - Requeue — Updates the `_watermill_delayed_until` metadata to the current time, so the message will be instantly requeued. 38 | - Ack — Removes the message from the queue (be careful — you will lose the message forever). 39 | -------------------------------------------------------------------------------- /tools/pq/cli/backend.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | type BackendConfig struct { 10 | Topic string 11 | RawTopic string 12 | } 13 | 14 | func (c BackendConfig) Validate() error { 15 | if c.Topic == "" && c.RawTopic == "" { 16 | return errors.New("topic or raw topic must be provided") 17 | } 18 | 19 | if c.Topic != "" && c.RawTopic != "" { 20 | return errors.New("only one of topic or raw topic must be provided") 21 | } 22 | 23 | return nil 24 | } 25 | 26 | type BackendConstructor func(ctx context.Context, cfg BackendConfig) (Backend, error) 27 | 28 | type Backend interface { 29 | AllMessages(ctx context.Context) ([]Message, error) 30 | Requeue(ctx context.Context, msg Message) error 31 | Ack(ctx context.Context, msg Message) error 32 | } 33 | -------------------------------------------------------------------------------- /tools/pq/cli/message.go: -------------------------------------------------------------------------------- 1 | package cli 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/ThreeDotsLabs/watermill/components/delay" 7 | "github.com/ThreeDotsLabs/watermill/message/router/middleware" 8 | ) 9 | 10 | type Message struct { 11 | // ID is a unique message ID across the Pub/Sub's topic. 12 | ID string 13 | UUID string 14 | Payload string 15 | Metadata map[string]string 16 | 17 | OriginalTopic string 18 | DelayedUntil string 19 | DelayedFor string 20 | RequeueIn time.Duration 21 | } 22 | 23 | func NewMessage(id string, uuid string, payload string, metadata map[string]string) (Message, error) { 24 | originalTopic := metadata[middleware.PoisonedTopicKey] 25 | 26 | // Calculate the time until the message should be requeued 27 | delayedUntil, err := time.Parse(time.RFC3339, metadata[delay.DelayedUntilKey]) 28 | if err != nil { 29 | return Message{}, err 30 | } 31 | 32 | delayedFor := metadata[delay.DelayedForKey] 33 | requeueIn := delayedUntil.Sub(time.Now().UTC()).Round(time.Second) 34 | 35 | return Message{ 36 | ID: id, 37 | UUID: uuid, 38 | Payload: payload, 39 | Metadata: metadata, 40 | OriginalTopic: originalTopic, 41 | DelayedUntil: delayedUntil.String(), 42 | DelayedFor: delayedFor, 43 | RequeueIn: requeueIn, 44 | }, nil 45 | } 46 | -------------------------------------------------------------------------------- /tools/pq/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "log" 7 | 8 | "github.com/ThreeDotsLabs/watermill/tools/pq/backend" 9 | "github.com/ThreeDotsLabs/watermill/tools/pq/cli" 10 | 11 | tea "github.com/charmbracelet/bubbletea" 12 | ) 13 | 14 | var ( 15 | backendFlag = flag.String("backend", "", "backend to use") 16 | topicFlag = flag.String("topic", "", "topic to use") 17 | rawTopicFlag = flag.String("raw-topic", "", "raw topic to use") 18 | ) 19 | 20 | func main() { 21 | flag.Parse() 22 | 23 | config := cli.BackendConfig{ 24 | Topic: *topicFlag, 25 | RawTopic: *rawTopicFlag, 26 | } 27 | 28 | err := config.Validate() 29 | if err != nil { 30 | log.Fatal(err) 31 | } 32 | 33 | var b cli.Backend 34 | switch *backendFlag { 35 | case "postgres": 36 | b, err = backend.NewPostgresBackend(context.Background(), config) 37 | if err != nil { 38 | log.Fatal(err) 39 | } 40 | default: 41 | log.Fatalf("unknown backend: %s", *backendFlag) 42 | } 43 | 44 | m := cli.NewModel(b) 45 | 46 | p := tea.NewProgram(m, tea.WithAltScreen()) 47 | _, err = p.Run() 48 | if err != nil { 49 | log.Fatal(err) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /uuid.go: -------------------------------------------------------------------------------- 1 | package watermill 2 | 3 | import ( 4 | "crypto/rand" 5 | 6 | "github.com/google/uuid" 7 | "github.com/lithammer/shortuuid/v3" 8 | "github.com/oklog/ulid" 9 | ) 10 | 11 | // NewUUID returns a new UUID Version 4. 12 | func NewUUID() string { 13 | return uuid.New().String() 14 | } 15 | 16 | // NewShortUUID returns a new short UUID. 17 | func NewShortUUID() string { 18 | return shortuuid.New() 19 | } 20 | 21 | // NewULID returns a new ULID. 22 | func NewULID() string { 23 | return ulid.MustNew(ulid.Now(), rand.Reader).String() 24 | } 25 | -------------------------------------------------------------------------------- /uuid_test.go: -------------------------------------------------------------------------------- 1 | package watermill_test 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | 7 | "github.com/ThreeDotsLabs/watermill" 8 | ) 9 | 10 | func testUniqueness(t *testing.T, genFunc func() string) { 11 | producers := 100 12 | uuidsPerProducer := 10000 13 | 14 | if testing.Short() { 15 | producers = 10 16 | uuidsPerProducer = 1000 17 | } 18 | 19 | uuidsCount := producers * uuidsPerProducer 20 | 21 | uuids := make(chan string, uuidsCount) 22 | allGenerated := sync.WaitGroup{} 23 | allGenerated.Add(producers) 24 | 25 | for i := 0; i < producers; i++ { 26 | go func() { 27 | for j := 0; j < uuidsPerProducer; j++ { 28 | uuids <- genFunc() 29 | } 30 | allGenerated.Done() 31 | }() 32 | } 33 | 34 | uniqueUUIDs := make(map[string]struct{}, uuidsCount) 35 | 36 | allGenerated.Wait() 37 | close(uuids) 38 | 39 | for uuid := range uuids { 40 | if _, ok := uniqueUUIDs[uuid]; ok { 41 | t.Error(uuid, " has duplicate") 42 | } 43 | uniqueUUIDs[uuid] = struct{}{} 44 | } 45 | } 46 | 47 | func TestUUID(t *testing.T) { 48 | testUniqueness(t, watermill.NewUUID) 49 | } 50 | 51 | func TestShortUUID(t *testing.T) { 52 | testUniqueness(t, watermill.NewShortUUID) 53 | } 54 | 55 | func TestULID(t *testing.T) { 56 | testUniqueness(t, watermill.NewULID) 57 | } 58 | --------------------------------------------------------------------------------