├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── changelog.md ├── include └── brucke_int.hrl ├── priv ├── brucke.yml └── ssl │ ├── ca.crt │ ├── client.crt │ └── client.key ├── rebar.config ├── rebar.config.script ├── rel ├── sys.config └── vm.args ├── rpm └── brucke.spec ├── scripts ├── cover-print-not-covered-lines.escript ├── start-test-brokers.sh └── vsn-check.sh ├── src ├── brucke.app.src ├── brucke.erl ├── brucke_app.erl ├── brucke_backlog.erl ├── brucke_config.erl ├── brucke_filter.erl ├── brucke_http.erl ├── brucke_http_healthcheck_handler.erl ├── brucke_http_ping_handler.erl ├── brucke_lib.erl ├── brucke_member.erl ├── brucke_metrics.erl ├── brucke_ratelimiter.erl ├── brucke_ratelimiter_http_handler.erl ├── brucke_routes.erl ├── brucke_subscriber.erl └── brucke_sup.erl └── test ├── brucke_SUITE.erl ├── brucke_backlog_tests.erl ├── brucke_config_tests.erl ├── brucke_http_SUITE.erl └── brucke_test_filter.erl /.gitignore: -------------------------------------------------------------------------------- 1 | log/ 2 | doc/ 3 | erl_crash.dump 4 | deps/ 5 | logs/ 6 | ebin/ 7 | cover/ 8 | test/*.beam 9 | test/ct.cover.spec 10 | *.d 11 | *.plt 12 | eunit.coverdata 13 | ct.coverdata 14 | xrefr 15 | _build/ 16 | _rel/ 17 | rebar.lock 18 | .eunit/ 19 | .rebar/ 20 | *.tmp 21 | relx 22 | x86_64/ 23 | *.coverdata 24 | *.iml 25 | .idea/ 26 | brod-*/ 27 | brod.zip 28 | brod.tar.gz 29 | *.log 30 | *.crashdump 31 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: erlang 2 | 3 | sudo: required 4 | 5 | services: 6 | - docker 7 | 8 | before_install: 9 | - git clone https://github.com/erlang/rebar3.git; cd rebar3; ./bootstrap; sudo mv rebar3 /usr/local/bin/; cd .. 10 | - sudo docker info 11 | 12 | notifications: 13 | email: false 14 | 15 | otp_release: 16 | - 21.0 17 | 18 | script: 19 | - make xref 20 | - make dialyzer 21 | - make edoc 22 | - make test-env 23 | - make t 24 | - make cover 25 | 26 | after_success: 27 | - make coveralls 28 | 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PROJECT = brucke 2 | PROJECT_DESCRIPTION = Inter-cluster bridge of kafka topics 3 | PROJECT_VERSION = $(shell erl -noshell -eval "{ok, [{_, _, L}]} = file:consult('src/brucke.app.src'), io:put_chars([V || {vsn, V} <- L]), halt(0)") 4 | 5 | all: compile 6 | 7 | rebar ?= $(shell which rebar3) 8 | rebar_cmd = $(rebar) $(profile:%=as %) 9 | 10 | .PHONY: compile 11 | compile: 12 | @$(rebar_cmd) compile 13 | 14 | .PHONY: xref 15 | xref: 16 | @$(rebar_cmd) xref 17 | 18 | .PHONY: clean 19 | clean: 20 | @$(rebar_cmd) clean 21 | 22 | .PHONY: distclean 23 | distclean: 24 | @$(rebar_cmd) clean 25 | @rm -rf _build 26 | 27 | .PHONY: eunit 28 | eunit: 29 | @$(rebar_cmd) eunit -v 30 | 31 | .PHONY: ct 32 | ct: 33 | @$(rebar_cmd) ct 34 | 35 | .PHONY: edoc 36 | edoc: profile=edown 37 | edoc: 38 | @$(rebar_cmd) edoc 39 | 40 | .PHONY: shell 41 | shell: profile=dev 42 | shell: 43 | @$(rebar_cmd) shell --apps brod 44 | 45 | .PHONY: dialyzer 46 | dialyzer: compile 47 | @$(rebar_cmd) dialyzer 48 | 49 | .PHONY: cover 50 | cover: 51 | @$(rebar_cmd) cover -v 52 | 53 | .PHONY: t 54 | t: eunit ct cover 55 | 56 | .PHONY: test-env 57 | test-env: 58 | ./scripts/start-test-brokers.sh 59 | 60 | .PHONY: rel 61 | rel: profile=prod 62 | rel: all 63 | @$(rebar_cmd) release 64 | 65 | .PHONY: run 66 | run: profile=dev 67 | run: 68 | @$(rebar_cmd) release 69 | @_build/dev/rel/brucke/bin/brucke console 70 | 71 | TOPDIR = /tmp/brucke-rpm 72 | PWD = $(shell pwd) 73 | 74 | .PHONY: rpm 75 | rpm: profile=prod 76 | rpm: rel 77 | @rpmbuild -v -bb \ 78 | --define "_sourcedir $(PWD)" \ 79 | --define "_builddir $(PWD)" \ 80 | --define "_rpmdir $(PWD)" \ 81 | --define "_topdir $(TOPDIR)" \ 82 | --define "_name $(PROJECT)" \ 83 | --define "_description $(PROJECT_DESCRIPTION)" \ 84 | --define "_version $(PROJECT_VERSION)" \ 85 | rpm/brucke.spec 86 | 87 | .PHONY: hex-publish 88 | hex-publish: distclean 89 | @$(rebar_cmd) hex publish 90 | 91 | .PHONY: coveralls 92 | coveralls: 93 | @$(rebar_cmd) coveralls send 94 | 95 | .PHONY: escript 96 | escript: 97 | @$(rebar_cmd) escriptize 98 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Inter-cluster bridge of kakfa topics 2 | Copyright 2016 Klarna AB 3 | 4 | This product includes software developed by 5 | Klarna AB (https://www.klarna.com) 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/klarna/brucke.svg)](https://travis-ci.org/klarna/brucke) [![Coverage Status](https://coveralls.io/repos/github/klarna/brucke/badge.svg?branch=master)](https://coveralls.io/github/klarna/brucke?branch=master) 2 | 3 | # Brucke - Inter-cluster bridge of kafka topics 4 | Brucke is a Inter-cluster bridge of kafka topics powered by [Brod](https://github.com/klarna/brod) 5 | 6 | Brucke bridges messages from upstream topic to downstream topic with 7 | configurable re-partitionning strategy and message filter/transform plugins 8 | 9 | # Configuration 10 | 11 | A brucke config file is a YAML file. 12 | 13 | Config file path should be set in config_file variable of brucke app config, or via `BRUCKE_CONFIG_FILE` OS env variable. 14 | 15 | Cluster names and client names must comply to erlang atom syntax. 16 | 17 | kafka_clusters: 18 | kafka_cluster_1: 19 | - "localhost:9092" 20 | kafka_cluster_2: 21 | - "kafka-1:9092" 22 | - "kafka-2:9092" 23 | brod_clients: 24 | - client: brod_client_1 25 | cluster: kafka_cluster_1 26 | config: [] # optional 27 | - client: brod_client_2 # example for SSL connection 28 | cluster: kafka_cluster_1 29 | config: 30 | ssl: true 31 | - client: brod_client_3 # example for SASL_SSL and custom certificates 32 | cluster: kafka_cluster_1 33 | config: 34 | ssl: 35 | # start with "priv/" or provide full path 36 | cacertfile: priv/ssl/ca.crt 37 | certfile: priv/ssl/client.crt 38 | keyfile: priv/ssl/client.key 39 | sasl: 40 | # or 'plain' or 'scram_sha_256' 41 | mechanism: scram_sha_512 42 | username: brucke 43 | password: secret 44 | routes: 45 | - upstream_client: brod_client_1 46 | downstream_client: brod_client_1 47 | upstream_topics: 48 | - "topic_1" 49 | downstream_topic: "topic_2" 50 | repartitioning_strategy: strict_p2p 51 | default_begin_offset: earliest # optional 52 | filter_module: brucke_filter # optional 53 | filter_init_arg: "" # optional 54 | upstream_cg_id: example-consumer-group-1 # optional, build from cluster name otherwise 55 | 56 | ## Client Configs 57 | 58 | - `ssl` Either set to `true` or be specific about the cert/key files to use 59 | `cacertfile`, `certfile` and `keyfile`. In case the file paths start with `priv/` 60 | `brucke` will look in application `priv` dir (i.e. `code:priv_dir(brucke)`). 61 | - `sasl` Configure SASL auth `mechanism`, `username` and `password` for SASL PLAIN authentication. 62 | - `query_api_versions` Set to `false` when connecting to kafka 0.9.x or earlier 63 | 64 | See more configs in `brod:start_client/4` API doc. 65 | 66 | ## Routing Configs 67 | 68 | ### Upstream Consumer Group ID 69 | 70 | `upstream_cg_id` identifies the consumer group which route workers (maybe distributed across different Erlang nodes) should join in, and commit offsets to. 71 | Group IDs don't have to be unique, two or more routes may share the same group ID. 72 | However, the same topic may not appear in two routes share the same group ID. 73 | 74 | ### Default Begin Offset 75 | 76 | The `default_begin_offset` route config is to tell upstream consumer from where to start streaming messsages. 77 | Supported values are `earliest`, `latest` (default) or a kafka offset (integer) value. 78 | This config is to tell brucke route worker from which offset it should start streaming messages for the first run. 79 | In case of restart, brucke should continue from committed offset. 80 | 81 | NOTE: Offsets committed to kafka may expire, in that case, brucke will fallback to this default being offset. 82 | NOTE: Currently this option is used for ALL partitions in upstream topic(s). 83 | 84 | In case there is a need to discard committed offsets, pick a new group ID. 85 | 86 | ### Offset Commit Policy 87 | The `Offset_commit_policy` specify how upstream consumer manages the offset per topic-partition. 88 | Two values are available: `commit_to_kafka_v2` or `consumer_managed`. 89 | when `consumer_managed` is used, topic-partition offsets will be stored in dets (set). see "PATH to offsets DETS file". 90 | 91 | default: `commit_to_kafka_v2` 92 | 93 | ### Repartitioning strategy 94 | 95 | NOTE: For compacted topics, strict_p2p is the only choice. 96 | 97 | - key_hash: hash the message key to downstream partition number 98 | - strict_p2p: strictly map the upstream partition number to downstream partition number, worker will refuse to start if 99 | upstream and downstream topic has different number of partitions 100 | - random: randomly distribute upstream messages to downstream partitions 101 | 102 | ### Message Transformation 103 | 104 | Depending on message format version in kafka broker, possible message formats from kafka are 105 | 106 | - Version 0: key-value without timestamp (kafka prior to `0.10`) 107 | - Version 1: key-value with timestamp (since kafka `0.10`) 108 | - Version 2: key-value with headers and timestamp (since kafka `0.11`) 109 | 110 | When downstream kafka supports lower version than upstream, unsupported fields are dropped. 111 | Otherwise messages are upgraded using default values as below: 112 | - Local host's OS time is used as default timestamp if upstream message message has no `create` timestamp 113 | - Empty list `[]` is used as default `headers` field for downstream message if upstream message has no `headers` 114 | 115 | ### Customized Message Filtering and or Transformation 116 | 117 | Implement `brucke_filter` behaviour to have messages filtered and or transformed before produced to downstream topic. 118 | 119 | If `brucke` is packed standalone (i.e. not used as a dependency in a parent project), and the beam files for 120 | `brucke_filter` modules are installed elsewhere, configure `filter_ebin_dirs` app env (sys.config), 121 | or set system OS env variables `BRUCKE_FILTER_EBIN_PATHS`. 122 | 123 | ### Other Routing Options 124 | 125 | - `compression`: `no_compression` (defulat), `gzip` or `snappy` 126 | - `max_partitions_per_group_member`: default = 12, Number of partitions one group member should work on. 127 | - `required_acks`: `all` (default), `leader` or `none` 128 | 129 | 130 | ## Offsets DETS file 131 | You can config brucke where to start consuming by providing the none empty 'Offsets DETS file' when brucke starts. 132 | 133 | It should be set table and the record should be in following spec: 134 | 135 | ``` 136 | { {GroupID :: binary() , Topic :: binary(), Partition :: non_neg_integer() }, Offset :: -1 | non_neg_integer() }. 137 | ``` 138 | 139 | `offsets_dets_path` specify the PATH to the offsets dets file which is used by 'Offset Commit Policy'. 140 | If file does not exists, brucke will create empty one and use it. 141 | default: "/tmp/brucke_offsets.DETS" 142 | 143 | ## ratelimit 144 | Ratelimit the number of messages produced to downstream. 145 | 146 | `ratelimit_interval` defines the interval in milliseconds, default: 0 means disabled. 147 | `ratelimit_threshold` defines the threshold, set to 0 to pause. 148 | 149 | You can change these two settings in realtime via restapi. 150 | 151 | example: 152 | 153 | Limit the msg rate of consumer group: `group1` of cluster: `clusterA` to 10 msgs/s: 154 | 155 | ``` 156 | POST /plugins/ratelimiter/clusterA/group1 157 | {"interval": "100", "threshold", "1"} 158 | 159 | ``` 160 | 161 | # Graphite reporting 162 | 163 | If the following app config variables are set, brucke will send metrics to a configured graphite endpoint: 164 | 165 | - graphite_root_path: a prefix for metrics, e.g "myservice" 166 | - graphite_host: e.g. "localhost" 167 | - graphite_port: e.g. 2003 168 | 169 | Alternatively, you can use corresponding OS env variables: 170 | 171 | - BRUCKE_GRAPHITE_ROOT_PATH 172 | - BRUCKE_GRAPHITE_HOST 173 | - BRUCKE_GRAPHITE_PORT 174 | 175 | # RPM packaging 176 | 177 | Generate a release and package it into an rpm package: 178 | 179 | make rpm 180 | 181 | Brucke package installs release and creates corresponding systemd service. Config files are in /etc/brucke, OS env can 182 | be set at /etc/sysconfig/brucke, logs are in /var/log/brucke. 183 | 184 | Operating via systemctl: 185 | 186 | systemctl start brucke 187 | systemctl enable brucke 188 | systemctl status brucke 189 | 190 | # Script for config validation 191 | 192 | Build: 193 | ``` 194 | make escript 195 | ``` 196 | 197 | Usage: 198 | ``` 199 | brucke [filter-module-ebin-dir ...] 200 | ``` 201 | 202 | Set `BRUCKE_FILTER_MODULE_BEAM_DIRS` for extra filter module beam lookup. 203 | For valid configs the validation command should silently exit with code 0. 204 | 205 | # Http Endpoint for Healthcheck 206 | 207 | Default port is 8080, customize via `http_port` config option or via `BRUCKE_HTTP_PORT` OS env variable. 208 | 209 | GET /ping 210 | Returns `pong` if the application is up and running. 211 | 212 | GET /healthcheck 213 | Responds with status 200 if everything is OK, and 500 if something is not OK. 214 | Also returns healthy and unhealthy routes in response body in JSON format. 215 | 216 | Example response: 217 | 218 | { 219 | "discarded": [ 220 | { 221 | "downstream": { 222 | "endpoints": [ 223 | { 224 | "host": "localhost", 225 | "port": 9192 226 | } 227 | ], 228 | "topic": "brucke-filter-test-downstream" 229 | }, 230 | "options": { 231 | "default_begin_offset": "earliest", 232 | "filter_init_arg": [], 233 | "filter_module": "brucke_test_filter", 234 | "repartitioning_strategy": "strict_p2p" 235 | }, 236 | "reason": [ 237 | "filter module brucke_test_filter is not found\nreason:embedded\n" 238 | ], 239 | "upstream": { 240 | "endpoints": [ 241 | { 242 | "host": "localhost", 243 | "port": 9092 244 | } 245 | ], 246 | "topics": [ 247 | "brucke-filter-test-upstream" 248 | ] 249 | } 250 | } 251 | ], 252 | "healthy": [ 253 | { 254 | "downstream": { 255 | "endpoints": [ 256 | { 257 | "host": "localhost", 258 | "port": 9192 259 | } 260 | ], 261 | "topic": "brucke-basic-test-downstream" 262 | }, 263 | "options": { 264 | "consumer_config": { 265 | "begin_offset": "earliest" 266 | }, 267 | "filter_init_arg": [], 268 | "filter_module": "brucke_filter", 269 | "max_partitions_per_group_member": 12, 270 | "producer_config": { 271 | "compression": "no_compression" 272 | }, 273 | "repartitioning_strategy": "strict_p2p" 274 | }, 275 | "upstream": { 276 | "endpoints": [ 277 | { 278 | "host": "localhost", 279 | "port": 9092 280 | } 281 | ], 282 | "topics": "brucke-basic-test-upstream" 283 | } 284 | }, 285 | { 286 | "downstream": { 287 | "endpoints": [ 288 | { 289 | "host": "localhost", 290 | "port": 9092 291 | } 292 | ], 293 | "topic": "brucke-test-topic-2" 294 | }, 295 | "options": { 296 | "consumer_config": { 297 | "begin_offset": "earliest" 298 | }, 299 | "filter_init_arg": [], 300 | "filter_module": "brucke_filter", 301 | "max_partitions_per_group_member": 12, 302 | "producer_config": { 303 | "compression": "no_compression" 304 | }, 305 | "repartitioning_strategy": "strict_p2p" 306 | }, 307 | "upstream": { 308 | "endpoints": [ 309 | { 310 | "host": "localhost", 311 | "port": 9092 312 | } 313 | ], 314 | "topics": "brucke-test-topic-1" 315 | } 316 | }, 317 | { 318 | "downstream": { 319 | "endpoints": [ 320 | { 321 | "host": "localhost", 322 | "port": 9092 323 | } 324 | ], 325 | "topic": "brucke-test-topic-3" 326 | }, 327 | "options": { 328 | "consumer_config": { 329 | "begin_offset": "latest" 330 | }, 331 | "filter_init_arg": [], 332 | "filter_module": "brucke_filter", 333 | "max_partitions_per_group_member": 12, 334 | "producer_config": { 335 | "compression": "no_compression" 336 | }, 337 | "repartitioning_strategy": "key_hash" 338 | }, 339 | "upstream": { 340 | "endpoints": [ 341 | { 342 | "host": "localhost", 343 | "port": 9092 344 | } 345 | ], 346 | "topics": "brucke-test-topic-2" 347 | } 348 | }, 349 | { 350 | "downstream": { 351 | "endpoints": [ 352 | { 353 | "host": "localhost", 354 | "port": 9192 355 | } 356 | ], 357 | "topic": "brucke-test-topic-1" 358 | }, 359 | "options": { 360 | "consumer_config": { 361 | "begin_offset": "latest" 362 | }, 363 | "filter_init_arg": [], 364 | "filter_module": "brucke_filter", 365 | "max_partitions_per_group_member": 12, 366 | "producer_config": { 367 | "compression": "no_compression" 368 | }, 369 | "repartitioning_strategy": "random" 370 | }, 371 | "upstream": { 372 | "endpoints": [ 373 | { 374 | "host": "localhost", 375 | "port": 9192 376 | } 377 | ], 378 | "topics": "brucke-test-topic-3" 379 | } 380 | } 381 | ], 382 | "status": "failing", 383 | "unhealthy": [] 384 | } 385 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | - 1.12.0 2 | Make upstream consumer group ID configurable 3 | - 1.12.1/2 4 | Fix Route config validation 5 | - 1.13.0 6 | * Update README to document configs 7 | * Upgrade brod from 2.5.0 to 3.2.0 8 | - 1.13.1 9 | * Fix discarded route formatting for healthcheck 10 | - 1.14.0 11 | * Support `required_acks` producer config 12 | - 1.14.1 13 | * Upgrade brod from 3.2.0 to 3.3.0 to include the fix of heartbeat timeout issue for brod_group_coordinator 14 | * Add discarded routes to discarded ets after initial validation 15 | - 1.14.3 16 | * Handle deleted topic 17 | * Upgrade to brod 3.3.4 --- 1.14.1 will not respect default beging_offset in yml config 18 | - 1.14.4 19 | * Ensure string (instead of integer array) in healthcheck JSON report 20 | - 1.14.5 21 | * Fix duplicate upstream topic validation 22 | - 1.15.0 23 | * Upgrade brod to `3.6.1` 24 | * Add scram sasl support 25 | * Support magic-v2 message format (message timestamp and headers). 26 | - 1.15.1 27 | * Allow single new format message as filter result 28 | - 1.16.0 29 | * Add reate limit plugin 30 | - 1.17.0 31 | * Add escript for config validation 32 | * Logger replace lager 33 | * Upgrade brod from 3.7.0 to 3.7.2 34 | - 1.17.1 35 | * Upgrade brod from 3.7.2 to 3.7.5 36 | - 1.17.2 37 | * Support the 'healthcheck_port' config key name 38 | -------------------------------------------------------------------------------- /include/brucke_int.hrl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2017 Klarna AB 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -ifndef(BRUCKE_INT_HRL). 18 | -define(BRUCKE_INT_HRL, true). 19 | 20 | -include_lib("brod/include/brod.hrl"). 21 | 22 | -define(APPLICATION, brucke). 23 | 24 | -define(undef, undefined). 25 | 26 | -type filename() :: string(). 27 | 28 | -type route_option_key() :: repartitioning_strategy 29 | | producer_config 30 | | consumer_config 31 | | max_partitions_per_group_member 32 | | filter_module 33 | | filter_init_arg 34 | | offset_commit_policy 35 | | upstream_cg_id 36 | | ratelimit_threshold 37 | | ratelimit_interval. 38 | 39 | %% Message repartitioning strategy. 40 | %% NOTE: For compacted topics, strict_p2p is the only choice. 41 | %% key_hash: 42 | %% Hash the message key to downstream partition number 43 | %% strict_p2p: 44 | %% Strictly map the upstream partition number to downstream partition 45 | %% number, worker will refuse to start if upstream and downstream 46 | %% topic has different number of partitions 47 | %% random: 48 | %% Randomly distribute upstream messages to downstream partitions 49 | -type repartitioning_strategy() :: key_hash 50 | | strict_p2p 51 | | random. 52 | 53 | -define(DEFAULT_REPARTITIONING_STRATEGY, key_hash). 54 | 55 | -define(IS_VALID_REPARTITIONING_STRATEGY(X), 56 | (X =:= key_hash orelse 57 | X =:= strict_p2p orelse 58 | X =:= random)). 59 | 60 | -define(RATELIMIT_DISABLED, 0). 61 | -define(DEFAULT_FILTER_MODULE, brucke_filter). 62 | -define(DEFAULT_FILTER_INIT_ARG, []). 63 | -define(MAX_PARTITIONS_PER_GROUP_MEMBER, 12). 64 | -define(DEFAULT_DEFAULT_BEGIN_OFFSET, latest). 65 | -define(DEFAULT_COMPRESSION, no_compression). 66 | -define(DEFAULT_REQUIRED_ACKS, -1). 67 | -define(DEFAULT_OFFSET_COMMIT_POLICY, commit_to_kafka_v2). 68 | -define(DEFAULT_OFFSETS_DETS_PATH, "/tmp/brucke_offsets.DETS"). 69 | -define(DEFAULT_RATELIMIT_INTEVAL, ?RATELIMIT_DISABLED). 70 | -define(DEFAULT_RATELIMIT_THRESHOLD, 0). 71 | 72 | -type consumer_group_id() :: binary(). 73 | -type hostname() :: string(). 74 | -type portnum() :: pos_integer(). 75 | -type endpoint() :: {hostname(), portnum()}. 76 | -type cluster_name() :: binary(). 77 | -type cluster() :: {cluster_name(), [endpoint()]}. 78 | -type client() :: {brod:client_id(), [endpoint()], brod:client_config()}. 79 | -type route_options() :: [{route_option_key(), term()}] 80 | | #{route_option_key() => term()}. 81 | -type topic_name() :: atom() | string() | binary(). 82 | 83 | -type upstream() :: {brod:client_id(), brod:topic()}. 84 | -type downstream() :: {brod:client_id(), brod:topic()}. 85 | 86 | -record(route, { upstream :: upstream() 87 | , downstream :: downstream() 88 | , options :: route_options() 89 | , reason :: undefined | binary() 90 | }). 91 | 92 | -type route() :: #route{}. 93 | -type raw_route() :: proplists:proplist(). 94 | 95 | -ifndef(APPLICATION). 96 | -define(APPLICATION, brucke). 97 | -endif. 98 | 99 | -define(I2B(I), list_to_binary(integer_to_list(I))). 100 | 101 | -define(OFFSETS_TAB, brucke_offsets). 102 | -define(RATELIMITER_TAB, brucke_ratelimiter). 103 | 104 | %% counter and gauge 105 | -define(INC(Name, Value), brucke_metrics:inc(Name, Value)). 106 | -define(SET(Name, Value), brucke_metrics:set(Name, Value)). 107 | -define(TOPIC(Topic), brucke_metrics:format_topic(Topic)). 108 | 109 | %% Metric names 110 | -define(MX_TOTAL_VOLUME(Cluster, Topic, Partition, Bytes), 111 | ?INC([Cluster, ?TOPIC(Topic), ?I2B(Partition), <<"bytes">>], Bytes)). 112 | 113 | -define(MX_HIGH_WM_OFFSET(Cluster, Topic, Partition, Offset), 114 | ?SET([Cluster, ?TOPIC(Topic), ?I2B(Partition), <<"high-wm">>], Offset)). 115 | -define(MX_CURRENT_OFFSET(Cluster, Topic, Partition, Offset), 116 | ?SET([Cluster, ?TOPIC(Topic), ?I2B(Partition), <<"current">>], Offset)). 117 | -define(MX_LAGGING_OFFSET(Cluster, Topic, Partition, Offset), 118 | ?SET([Cluster, ?TOPIC(Topic), ?I2B(Partition), <<"lagging">>], Offset)). 119 | 120 | -ifdef(OTP_RELEASE). 121 | -define(BIND_STACKTRACE(Var), :Var). 122 | -define(GET_STACKTRACE(Var), ok). 123 | -else. 124 | -define(BIND_STACKTRACE(Var),). 125 | -define(GET_STACKTRACE(Var), Var = erlang:get_stacktrace()). 126 | -endif. 127 | 128 | 129 | -endif. 130 | 131 | %%%_* Emacs ==================================================================== 132 | %%% Local Variables: 133 | %%% allout-layout: t 134 | %%% erlang-indent-level: 2 135 | %%% End: 136 | -------------------------------------------------------------------------------- /priv/brucke.yml: -------------------------------------------------------------------------------- 1 | offsets_dets_path: "/tmp/brucke_offsets_ct.DETS" 2 | kafka_clusters: 3 | local_cluster: 4 | - localhost:9092 5 | local_cluster_ssl: 6 | - localhost:9093 7 | local_cluster_ssl_sasl: 8 | - localhost:9094 9 | brod_clients: 10 | - client: client_1 11 | cluster: local_cluster 12 | - client: client_2 13 | cluster: local_cluster_ssl_sasl 14 | config: 15 | ssl: 16 | # start with "priv/" or provide full path 17 | cacertfile: priv/ssl/ca.crt 18 | certfile: priv/ssl/client.crt 19 | keyfile: priv/ssl/client.key 20 | sasl: 21 | mechanism: scram_sha_256 22 | username: alice 23 | password: ecila 24 | - client: client_3 25 | cluster: local_cluster_ssl 26 | config: 27 | ssl: 28 | # start with "priv/" or provide full path 29 | cacertfile: priv/ssl/ca.crt 30 | certfile: priv/ssl/client.crt 31 | keyfile: priv/ssl/client.key 32 | routes: 33 | - upstream_client: client_1 34 | downstream_client: client_1 35 | upstream_topics: 36 | - brucke-test-topic-1 37 | downstream_topic: brucke-test-topic-2 38 | repartitioning_strategy: strict_p2p 39 | default_begin_offset: latest 40 | upstream_cg_id: route-1 41 | 42 | - upstream_client: client_1 43 | downstream_client: client_1 44 | upstream_topics: 45 | - brucke-test-topic-2 46 | downstream_topic: brucke-test-topic-3 47 | repartitioning_strategy: key_hash 48 | upstream_cg_id: route-2 49 | 50 | - upstream_client: client_2 51 | downstream_client: client_2 52 | upstream_topics: 53 | - brucke-test-topic-3 54 | downstream_topic: brucke-test-topic-1 55 | repartitioning_strategy: random 56 | upstream_cg_id: route-3 57 | 58 | - upstream_client: client_1 59 | downstream_client: client_2 60 | upstream_topics: 61 | - brucke-basic-test-upstream 62 | downstream_topic: brucke-basic-test-downstream 63 | repartitioning_strategy: strict_p2p 64 | default_begin_offset: latest 65 | upstream_cg_id: basic-test 66 | 67 | - upstream_client: client_3 68 | downstream_client: client_2 69 | upstream_topics: 70 | - brucke-filter-test-upstream 71 | downstream_topic: brucke-filter-test-downstream 72 | repartitioning_strategy: random 73 | filter_module: brucke_test_filter 74 | filter_init_arg: "" 75 | default_begin_offset: latest 76 | upstream_cg_id: brucke-filter-test-random 77 | 78 | - upstream_client: client_1 79 | downstream_client: client_2 80 | upstream_topics: 81 | - brucke-filter-consumer-managed-offsets-test-upstream 82 | downstream_topic: brucke-filter-consumer-managed-offsets-test-downstream 83 | repartitioning_strategy: strict_p2p 84 | default_begin_offset: latest 85 | upstream_cg_id: brucke-filter-test-consumer-managed-offsets 86 | offset_commit_policy: consumer_managed 87 | ratelimit_interval: 0 88 | 89 | - upstream_client: client_3 90 | downstream_client: client_2 91 | upstream_topics: 92 | - brucke-ratelimiter-test-upstream 93 | downstream_topic: brucke-ratelimiter-test-downstream 94 | repartitioning_strategy: strict_p2p 95 | default_begin_offset: latest 96 | upstream_cg_id: brucke-ratelimiter-test 97 | max_partitions_per_group_member: 1 98 | ratelimit_interval: 1000 99 | ratelimit_threshold: 0 100 | -------------------------------------------------------------------------------- /priv/ssl/ca.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDkzCCAnugAwIBAgIJAPjeRT8z4mElMA0GCSqGSIb3DQEBCwUAMGAxCzAJBgNV 3 | BAYTAlNFMRIwEAYDVQQIDAlTdG9ja2hvbG0xEjAQBgNVBAcMCVN0b2NraG9sbTEN 4 | MAsGA1UECgwEYnJvZDENMAsGA1UECwwEdGVzdDELMAkGA1UEAwwCKi4wHhcNMTYx 5 | MTA0MTYxNDM2WhcNMjYxMTAyMTYxNDM2WjBgMQswCQYDVQQGEwJTRTESMBAGA1UE 6 | CAwJU3RvY2tob2xtMRIwEAYDVQQHDAlTdG9ja2hvbG0xDTALBgNVBAoMBGJyb2Qx 7 | DTALBgNVBAsMBHRlc3QxCzAJBgNVBAMMAiouMIIBIjANBgkqhkiG9w0BAQEFAAOC 8 | AQ8AMIIBCgKCAQEAyIbBpX2DvhIbcXx1uho3Vm+hOLXrZJwNgVL3yDx/anGPvD2a 9 | ZkUjdrJNh8jy5ZFA7jBQGLYIyMQYY8UMyAPIQbCsi0wvFhcWfv+/VTSOfgcK04D+ 10 | QQRni8lkWI66oBcM02Wtwo3K5W7KWJ+LOAaV5hmSvLhcyIsSQC6MRBGRGJ89Oyza 11 | 7s1FrCY0HCa6BicY48sLTHTT8MScK5kOMO5KqMK8rY/dLRYynhC2K8/stzqN27HI 12 | MoktDEzzCAfRaNfXE8o1NekJcpFLQNi9/nab7vcbWo/QmUCCF0Ny5BGWEx+GpEp9 13 | HjVM5KYAYlDqpMm3wttMs7dtU9lEXZk69uCejwIDAQABo1AwTjAdBgNVHQ4EFgQU 14 | I1wMy5ObzZNi7qh3W9VSYKJRctYwHwYDVR0jBBgwFoAUI1wMy5ObzZNi7qh3W9VS 15 | YKJRctYwDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAGmyomSHF4sZh 16 | yNIdDmeUp+nOFE6dXuK1vo8PlKrf+tyajNdic5ZMCF9BzbKSxFnejwHA2fzBlu6P 17 | 27kmmMPWuAhvcyxNciLZ/gGCn5gMPutKaweuaD6G93jkIngdXtbz+c6icpwsO9cK 18 | Z0mdVuesJnmLQYLn9pHDzGUGYPFZpHVXwQzyAVw4m9T+aqKwwe/0dL1Z/8b/iuwN 19 | K0S4/c7gLH8rB1jQisHomgHano43TzJq8ZFX7wF1E2tnHDdGk+uEZr5C7VPRgrF8 20 | /DhGGJnw3AoQgD5g1YqFGA5pA0AXr4RF27Y7bKYnzvbktOkfcNhw/4P2rKXWWs1Q 21 | x2xsU3VaTQ== 22 | -----END CERTIFICATE----- 23 | -------------------------------------------------------------------------------- /priv/ssl/client.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIDPDCCAiQCCQDU7hj/emVKfDANBgkqhkiG9w0BAQsFADBgMQswCQYDVQQGEwJT 3 | RTESMBAGA1UECAwJU3RvY2tob2xtMRIwEAYDVQQHDAlTdG9ja2hvbG0xDTALBgNV 4 | BAoMBGJyb2QxDTALBgNVBAsMBHRlc3QxCzAJBgNVBAMMAiouMB4XDTE2MTEwNDE2 5 | MTQzNloXDTI2MTEwMjE2MTQzNlowYDELMAkGA1UEBhMCU0UxEjAQBgNVBAgMCVN0 6 | b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMQ0wCwYDVQQKDARicm9kMQ0wCwYD 7 | VQQLDAR0ZXN0MQswCQYDVQQDDAIqLjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC 8 | AQoCggEBAKtN+UzF8g9JRhKXn/rnm4jCr26+HpboDQYxG1HSCwwWskdOMK1b/8w4 9 | ipNzpoV16teRW5AVdq5Z6DDzBE5X43rrJZ9+x6pd25mVyktmwAIxEYscLtxN1UoL 10 | a5EF13D8UPWCyzylThhUwi67bHvbLeWzAKoccKqdV/5ZNjFnqt9Q9seFOxyXNcFE 11 | /qfUQTfkcL4rei2dgkFPFOXbF2rKRgMaiseyVAJP0G8AcsCkQvaYnkQrJ8nAZBtI 12 | vZmq2og9PW7Z8rEbm9TVLnLNtEE5Lx2S1SQS9QPccYJDAyQJLCOw2ikGQPgtDfbs 13 | KILEp+MChTWgEeb/LBlN/qa+zDraDm0CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA 14 | EdsizFjP+hWSa5A0UFRIETvAztpTd+pjWWVv3DwCCRc2aMys+GYnR5fkHtnwKr7u 15 | diZ8SSMZQFhlxA9MRNe8++wKeKeCzqrwIV1+mQcGqrJLl6sxW5TcMs/bRy5BPwZJ 16 | RGlcz6HdLY8UBZzY2Qy2A4VecqwNe07Vg+7Yebui4w09pt5045S/q33/arb/LKP+ 17 | 1CbCjNyF3QC0aww+YgML+PgjnNtqbO/85qV424/dMX+aNAotQ/zBdEfEXyFaCoAE 18 | yCHA99FnhHsQ9gwv9vhMLAX+yiBIEoh3e18EtmZdsvsTpDd1KI4nrh44TJfEY65+ 19 | fNeAXYygkzsN14bbk9PgMw== 20 | -----END CERTIFICATE----- 21 | -------------------------------------------------------------------------------- /priv/ssl/client.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCrTflMxfIPSUYS 3 | l5/655uIwq9uvh6W6A0GMRtR0gsMFrJHTjCtW//MOIqTc6aFderXkVuQFXauWegw 4 | 8wROV+N66yWffseqXduZlcpLZsACMRGLHC7cTdVKC2uRBddw/FD1gss8pU4YVMIu 5 | u2x72y3lswCqHHCqnVf+WTYxZ6rfUPbHhTsclzXBRP6n1EE35HC+K3otnYJBTxTl 6 | 2xdqykYDGorHslQCT9BvAHLApEL2mJ5EKyfJwGQbSL2ZqtqIPT1u2fKxG5vU1S5y 7 | zbRBOS8dktUkEvUD3HGCQwMkCSwjsNopBkD4LQ327CiCxKfjAoU1oBHm/ywZTf6m 8 | vsw62g5tAgMBAAECggEADFm50Jww4INC5xJBeYB7STfoGA7i+7RNRBYERzjijQOR 9 | 5OwxPD52yc2FyC29Yr/mp5YWSOQTQ2y9/dF3jQJvJyyO8NneIV1U+NTA2gDVdRL+ 10 | lc35Xu7JouYB4lnOd5npaFn+tyef4scxnNbscl2SCI6ITLtyMAraDj92VceInUMF 11 | 28srCTMdjbhVLpeq80qdeDVnyzlmua1W8pjR1lNXY2IECS9gTp6+JLiMQ0FJlC9V 12 | r+U5iAoqLCNh+QpdM+2Z8kbkKA5PqsWcAhx+dTTkbRPp59r7Qd2xtxde5PGlm6zp 13 | cqXgbWaXCMlbL5C7eOyPfLty3+KPrR6LGW6jGEqioQKBgQDcK2LGx/1PE2Y27p+O 14 | RImN5SYERiRnYen7bm1CBAoH1J5LDxWWcf8Bz8/y4bNvEZJVosvPDRoNilI4RTYD 15 | JiJw/qXio6FG78yIzvCK0WLIPgq6stufdbd/+UsNrDbGTuhk/qti8TSckEEgrUWg 16 | U0NgEc/zyIMQK/4mZSgqeUpuxQKBgQDHLsxRT3Ile4sT2arxv0/KzSmnEL9hCAa9 17 | Cf+N0mWPrt6rzJUTD0+FBToXGP3k4auKETRb3XHvSHCwwl6Bi+NTPpVYqBT53jEv 18 | fSb6bMjSlZyja+miVh/7TI2on7keus19XtZyks7PKoHa+i4w61zy5jbBdBC/kU1y 19 | 8O3HXF4biQKBgQCI6/5o6vTQmZrmrK3TtzHoacqju89l79GoyPrvpD1ss0CiI0Zk 20 | oo5ZXRjQzqZde4sK8MxY/qfmJdCOKBS4Dp46sVMOyH5C9Fy59CBJ5H/PUi4v/41v 21 | 9LBiyPFxFlmWKHqEXJDPXnw+pcOrA7caRs3O0CUIUfmYNBPBYwWArJ+qlQKBgFpO 22 | 25BaJvTbqNkdLaZiCTl3/9ShgUPrMbLwH5AbvrSAorDeFxEHNhSnpAjo6eSmdPIq 23 | jsTACHJnM8DQv6yY0j7h9zC1NJ19omtXoR6VyA/CibyGpu1VgzabJPc5Q+Os6pJX 24 | N3/HFEFVkn7IQ70mWYQ/4L+hch6JMMZWeliTho+RAoGADcqzTMLtp7kRH8LQcq1n 25 | oCE2FYJPvpd8PWlMCZ0ewSk6CbIgLvwJ+Hw0040m11HlCG7xVNdJV0rAU68Z7txm 26 | pYIXL3D9MlJWCWMjZ7k11fuN1EtPLYYhMgS7FhADdUfFhnRGDkF2LnbvZIh3UtN6 27 | H5khVwyCU9LwQoxKfTmuxnY= 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {deps, [ {brod, "3.8.0"} 2 | , {yamerl, "0.4.0"} 3 | , {graphiter, "1.0.5"} 4 | , {jsone, "1.4.6"} 5 | , {cowboy, "1.1.2"} 6 | ]}. 7 | 8 | {edoc_opts, [{preprocess, true}]}. 9 | 10 | {profiles, [ {test, [{deps, [meck]}]} 11 | , {dev, [{relx, [ {dev_mode, true} 12 | , {include_erts, false}]}]} 13 | , {prod, [{relx, [ {dev_mode, false} 14 | , {include_erts, true}]}]}]}. 15 | 16 | {erl_opts, [ error 17 | , warn_unused_vars 18 | , warn_shadow_vars 19 | , warn_unused_import 20 | , warn_obsolete_guard 21 | , debug_info 22 | ]}. 23 | 24 | {ct_use_short_names, true}. 25 | {cover_enabled, true}. 26 | {cover_export_enabled, true}. 27 | {cover_opts, [verbose]}. 28 | {eunit_opts, [verbose]}. 29 | 30 | {relx, [ {release, {brucke, "version is taken from VSN file"}, 31 | [sasl, brucke]} 32 | , {extended_start_script, true} 33 | , {sys_config, "rel/sys.config"} 34 | , {vm_args, "rel/vm.args"} 35 | ] 36 | }. 37 | 38 | {xref_checks, [undefined_function_calls, undefined_functions, 39 | locals_not_used, deprecated_function_calls, 40 | deprecated_functions]}. 41 | 42 | {xref_ignores, [ {brucke_filter, behaviour_info, 1} 43 | , {brucke_filter, filter, 6} 44 | , {brucke_filter, init, 3} 45 | , {brucke_http_healthcheck_handler, content_types_provided, 2} 46 | , {brucke_http_healthcheck_handler, handle_request, 2} 47 | , {brucke_http_healthcheck_handler, init, 2} 48 | , {brucke_http_healthcheck_handler, init, 3} 49 | , {brucke_http_ping_handler, content_types_provided, 2} 50 | , {brucke_http_ping_handler, handle_request, 2} 51 | , {brucke_http_ping_handler, init, 2} 52 | , {brucke_http_ping_handler, init, 3} 53 | , {brucke_lib, get_producer_config, 1} 54 | , {brucke_member, start_link, 1} 55 | ]}. 56 | 57 | -------------------------------------------------------------------------------- /rebar.config.script: -------------------------------------------------------------------------------- 1 | CONFIG1 = 2 | case file:consult("src/brucke.app.src") of 3 | {ok, [{_, _, L}]} -> 4 | {vsn, Version} = lists:keyfind(vsn, 1, L), 5 | {relx, RelxOptions0} = lists:keyfind(relx, 1, CONFIG), 6 | {release, {brucke, _}, Apps} = lists:keyfind(release, 1, RelxOptions0), 7 | RelxOptions = lists:keystore(release, 1, RelxOptions0, {release, {brucke, Version}, Apps}), 8 | lists:keystore(relx, 1, CONFIG, {relx, RelxOptions}); 9 | {error, enoent} -> 10 | %% evaluated as a dependency of other projects 11 | %% can't be a release, do nothing 12 | CONFIG 13 | end, 14 | case os:getenv("TRAVIS") of 15 | "true" -> 16 | JobId = os:getenv("TRAVIS_JOB_ID"), 17 | [{coveralls_service_job_id, JobId}, 18 | {plugins, [coveralls]}, 19 | {coveralls_coverdata, "_build/test/cover/*.coverdata"}, 20 | {coveralls_service_name , "travis-ci"} | CONFIG1]; 21 | _ -> 22 | CONFIG1 23 | end. 24 | -------------------------------------------------------------------------------- /rel/sys.config: -------------------------------------------------------------------------------- 1 | %% -*- mode: Erlang; fill-column: 80; -*- 2 | [ { brucke 3 | , [ {config_file, {priv, "brucke.yml"}} %% or abs path 4 | , {graphite_host, undefined} %% undefined to disable, "localhost" to test 5 | , {graphite_port, 2003} 6 | , {healthcheck_port, 8080} 7 | , {filter_ebin_dirs, []} 8 | ] 9 | } 10 | ]. 11 | -------------------------------------------------------------------------------- /rel/vm.args: -------------------------------------------------------------------------------- 1 | -sname brucke 2 | -setcookie brucke 3 | -heart 4 | +sbwt none 5 | -------------------------------------------------------------------------------- /rpm/brucke.spec: -------------------------------------------------------------------------------- 1 | %define debug_package %{nil} 2 | %define _service %{_name} 3 | %define _user %{_name} 4 | %define _group %{_name} 5 | %define _conf_dir %{_sysconfdir}/%{_service} 6 | %define _log_dir %{_var}/log/%{_service} 7 | 8 | Summary: %{_description} 9 | Name: %{_name} 10 | Version: %{_version} 11 | Release: 1%{?dist} 12 | License: Apache License, Version 2.0 13 | URL: https://github.com/klarna/brucke.git 14 | BuildRoot: %{_tmppath}/%{_name}-%{_version}-root 15 | Vendor: Klarna AB 16 | Packager: Ivan Dyachkov 17 | Provides: %{_service} 18 | BuildRequires: systemd 19 | %systemd_requires 20 | 21 | %description 22 | %{_description} 23 | 24 | %prep 25 | 26 | %build 27 | 28 | %install 29 | mkdir -p %{buildroot}%{_libdir} 30 | mkdir -p %{buildroot}%{_log_dir} 31 | mkdir -p %{buildroot}%{_unitdir} 32 | mkdir -p %{buildroot}%{_conf_dir} 33 | mkdir -p %{buildroot}%{_sysconfdir}/sysconfig 34 | mkdir -p %{buildroot}%{_bindir} 35 | mkdir -p %{buildroot}%{_sharedstatedir}/%{_service} 36 | cp -r _build/prod/rel/%{_name} %{buildroot}%{_libdir}/ 37 | mv %{buildroot}%{_libdir}/%{_name}/releases/%{_version}/sys.config %{buildroot}%{_conf_dir}/ 38 | mv %{buildroot}%{_libdir}/%{_name}/releases/%{_version}/vm.args %{buildroot}%{_conf_dir}/ 39 | 40 | cat > %{buildroot}%{_unitdir}/%{_service}.service < %{buildroot}%{_sysconfdir}/sysconfig/%{_service} < %{buildroot}/%{_bindir}/%{_service} </dev/null || /usr/sbin/groupadd -r %{_group} 85 | if ! /usr/bin/getent passwd %{_user} >/dev/null ; then 86 | /usr/sbin/useradd -r -g %{_group} -m -d %{_sharedstatedir}/%{_service} -c "%{_service}" %{_user} 87 | fi 88 | fi 89 | 90 | %post 91 | %systemd_post %{_service}.service 92 | 93 | %preun 94 | %systemd_preun %{_service}.service 95 | 96 | %postun 97 | %systemd_postun 98 | 99 | %files 100 | %defattr(-,root,root) 101 | %{_libdir}/%{_name} 102 | %attr(0755,root,root) %{_bindir}/%{_service} 103 | %{_unitdir}/%{_service}.service 104 | %config(noreplace) %{_sysconfdir}/sysconfig/%{_service} 105 | %attr(0755,%{_user},%{_group}) %config(noreplace) %{_conf_dir}/* 106 | %attr(0700,%{_user},%{_group}) %dir %{_sharedstatedir}/%{_service} 107 | %attr(0755,%{_user},%{_group}) %dir %{_log_dir} 108 | %attr(0755,%{_user},%{_group}) %dir %{_conf_dir} 109 | -------------------------------------------------------------------------------- /scripts/cover-print-not-covered-lines.escript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %%! -smp enable -sname notcoveredlinessummary -pa _build/default/lib/brucke/ebin 4 | 5 | %%% 6 | %%% Copyright (c) 2015-2018, Klarna Bank AB (publ) 7 | %%% 8 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 9 | %%% you may not use this file except in compliance with the License. 10 | %%% You may obtain a copy of the License at 11 | %%% 12 | %%% http://www.apache.org/licenses/LICENSE-2.0 13 | %%% 14 | %%% Unless required by applicable law or agreed to in writing, software 15 | %%% distributed under the License is distributed on an "AS IS" BASIS, 16 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | %%% See the License for the specific language governing permissions and 18 | %%% limitations under the License. 19 | %%% 20 | 21 | -mode(compile). 22 | 23 | main([_|_] = Files) -> 24 | ok = import_cover_data(Files), 25 | Modules = get_imported_modules(), 26 | Result = [{Mod, analyse_module(Mod)} || Mod <- Modules], 27 | lists:foreach(fun({Module, NotCoveredLines}) -> 28 | print_mod_summary(Module, lists:sort(NotCoveredLines)) 29 | end, Result); 30 | main(_) -> 31 | io:format(user, "expecting at least one over data file", []), 32 | halt(1). 33 | 34 | import_cover_data([]) -> ok; 35 | import_cover_data([File | Rest]) -> 36 | io:format(user, "using coverdata file: ~s\n", [File]), 37 | Parent = self(), 38 | Ref = make_ref(), 39 | erlang:spawn_link( 40 | fun() -> 41 | %% shutup the chatty prints from cover:xxx calls 42 | {ok, F} = file:open("/dev/null", [write]), 43 | group_leader(F, self()), 44 | ok = cover:import(File), 45 | Parent ! {Ref, ok}, 46 | receive 47 | stop -> 48 | %% keep it alive 49 | exit(normal) 50 | end 51 | end), 52 | receive 53 | {Ref, ok} -> 54 | import_cover_data(Rest) 55 | end. 56 | 57 | get_imported_modules() -> 58 | All = cover:imported_modules(), 59 | Filtered = 60 | lists:filter( 61 | fun(Mod) -> 62 | case lists:reverse(atom_to_list(Mod)) of 63 | "ETIUS_" ++ _ -> false; %% ignore coverage for xxx_SUITE 64 | _ -> true 65 | end 66 | end, All), 67 | lists:sort(Filtered). 68 | 69 | analyse_module(Module) -> 70 | {ok, Lines} = cover:analyse(Module, coverage, line), 71 | lists:foldr( 72 | fun({{_Mod, 0}, _}, Acc) -> Acc; 73 | ({{_Mod, _Line}, {1, 0}}, Acc) -> Acc; 74 | ({{_Mod, Line}, {0, 1}}, Acc) -> [Line | Acc] 75 | end, [], Lines). 76 | 77 | print_mod_summary(_Module, []) -> ok; 78 | print_mod_summary(Module, NotCoveredLines) -> 79 | io:format(user, "================ ~p ================\n", [Module]), 80 | case whicherl(Module) of 81 | Filename when is_list(Filename) -> 82 | print_lines(Filename, NotCoveredLines); 83 | _ -> 84 | erlang:error({erl_file_not_found, Module}) 85 | end. 86 | 87 | print_lines(_Filename, []) -> 88 | ok; 89 | print_lines(Filename, Lines) -> 90 | {ok, Fd} = file:open(Filename, [read]), 91 | try 92 | print_lines(Fd, 1, Lines) 93 | after 94 | file:close(Fd) 95 | end. 96 | 97 | print_lines(_Fd, _N, []) -> 98 | ok; 99 | print_lines(Fd, N, [M | Rest] = Lines) -> 100 | Continue = 101 | case io:get_line(Fd, "") of 102 | eof -> 103 | erlang:error({eof, N, Lines}); 104 | Line when N =:= M -> 105 | io:format(user, "~5p: ~s", [N, Line]), 106 | Rest; 107 | _ -> 108 | Lines 109 | end, 110 | print_lines(Fd, N+1, Continue). 111 | 112 | whicherl(Module) when is_atom(Module) -> 113 | {ok, {Module, [{compile_info, Props}]}} = 114 | beam_lib:chunks(code:which(Module), [compile_info]), 115 | proplists:get_value(source, Props). 116 | 117 | %%%_* Emacs ==================================================================== 118 | %%% Local Variables: 119 | %%% allout-layout: t 120 | %%% erlang-indent-level: 2 121 | %%% End: 122 | -------------------------------------------------------------------------------- /scripts/start-test-brokers.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | 3 | KAFKA_VERSION=1.1 4 | THIS_DIR="$(cd "$(dirname "$0")" && pwd)" 5 | 6 | BROD_VSN=$(erl -noinput -eval " 7 | {ok, Configs} = file:consult(\"$THIS_DIR/../rebar.config\"), 8 | {deps, Deps} = lists:keyfind(deps, 1, Configs), 9 | {brod, Version} = lists:keyfind(brod, 1, Deps), 10 | io:format(Version), 11 | erlang:halt(0).") 12 | 13 | if [ ! -d "./brod-$BROD_VSN" ]; then 14 | curl -L -o brod.tar.gz "https://github.com/klarna/brod/archive/${BROD_VSN}.tar.gz" 15 | tar -zxf brod.tar.gz 16 | fi 17 | 18 | cd "./brod-$BROD_VSN/scripts" 19 | 20 | sudo KAFKA_VERSION=${KAFKA_VERSION} docker-compose -f docker-compose.yml down || true 21 | sudo KAFKA_VERSION=${KAFKA_VERSION} docker-compose -f docker-compose.yml up -d 22 | 23 | ## wait 4 secons for kafka to be ready 24 | n=0 25 | while [ "$(sudo docker exec kafka-1 bash -c '/opt/kafka/bin/kafka-topics.sh --zookeeper localhost --list')" != '' ]; do 26 | if [ $n -gt 4 ]; then 27 | echo "timedout waiting for kafka" 28 | exit 1 29 | fi 30 | n=$(( n + 1 )) 31 | sleep 1 32 | done 33 | 34 | function create_topic { 35 | TOPIC_NAME="$1" 36 | PARTITIONS="${2:-1}" 37 | REPLICAS="${3:-1}" 38 | CMD="/opt/kafka/bin/kafka-topics.sh --zookeeper localhost --create --partitions $PARTITIONS --replication-factor $REPLICAS --topic $TOPIC_NAME" 39 | sudo docker exec kafka-1 bash -c "$CMD" 40 | } 41 | 42 | ## loop 43 | create_topic brucke-test-topic-1 3 2 44 | create_topic brucke-test-topic-2 3 2 45 | create_topic brucke-test-topic-3 2 2 46 | create_topic brucke-test-topic-4 3 2 47 | create_topic brucke-test-topic-5 25 2 48 | create_topic brucke-test-topic-6 13 2 49 | 50 | ## basic test 51 | create_topic brucke-basic-test-upstream 1 2 52 | create_topic brucke-basic-test-downstream 1 2 53 | 54 | ## filter test 55 | create_topic brucke-filter-test-upstream 1 2 56 | create_topic brucke-filter-test-downstream 1 2 57 | 58 | ## ratelimiter filter test 59 | create_topic brucke-ratelimiter-test-upstream 3 2 60 | create_topic brucke-ratelimiter-test-downstream 3 2 61 | 62 | ## consumer managed offsets test 63 | create_topic brucke-filter-consumer-managed-offsets-test-upstream 3 2 64 | create_topic brucke-filter-consumer-managed-offsets-test-downstream 3 2 65 | 66 | # this is to warm-up kafka group coordinator for deterministic in tests 67 | sudo docker exec kafka-1 /opt/kafka/bin/kafka-consumer-groups.sh --bootstrap-server localhost:9092 --new-consumer --group test-group --describe > /dev/null 2>&1 68 | 69 | # for kafka 0.11 or later, add sasl-scram test credentials 70 | sudo docker exec kafka-1 /opt/kafka/bin/kafka-configs.sh --zookeeper localhost:2181 --alter --add-config 'SCRAM-SHA-256=[iterations=8192,password=ecila],SCRAM-SHA-512=[password=ecila]' --entity-type users --entity-name alice 71 | -------------------------------------------------------------------------------- /scripts/vsn-check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | THIS_DIR="$(cd "$(dirname "$0")" && pwd)" 4 | MAKEFILE="$THIS_DIR/../Makefile" 5 | APP_SRC="$THIS_DIR/../src/brucke.app.src" 6 | 7 | PROJECT_VERSION=$1 8 | 9 | ESCRIPT=$(cat < 23 | ok = code:add_paths(FilterModuleEbinPaths), 24 | ok = brucke_config:init(ConfigFile), 25 | Discarded = brucke_routes:get_discarded(), 26 | Discarded =/= [] andalso error({discarded, Discarded}). 27 | 28 | %%%_* Emacs ==================================================================== 29 | %%% Local Variables: 30 | %%% allout-layout: t 31 | %%% erlang-indent-level: 2 32 | %%% End: 33 | -------------------------------------------------------------------------------- /src/brucke_app.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module(brucke_app). 18 | -behaviour(application). 19 | 20 | -export([ start/2 21 | , stop/1 22 | ]). 23 | 24 | -export([ graphite_root_path/0 25 | , graphite_host/0 26 | , graphite_port/0 27 | , http_port/0 28 | , config_file/0 29 | ]). 30 | 31 | -include("brucke_int.hrl"). 32 | 33 | %% App env getters 34 | graphite_root_path() -> app_env(graphite_root_path). 35 | 36 | graphite_host() -> app_env(graphite_host). 37 | 38 | graphite_port() -> app_env(graphite_port). 39 | 40 | http_port() -> app_env(healthcheck_port, app_env(http_port, 8080)). 41 | 42 | config_file() -> app_env(config_file, {priv, "brucke.yml"}). 43 | 44 | %% @private Application callback. 45 | start(_Type, _Args) -> 46 | ok = maybe_update_env(), 47 | ok = add_filter_ebin_dirs(), 48 | brucke_sup:start_link(). 49 | 50 | %% @private Application callback. 51 | stop(_State) -> 52 | ok. 53 | 54 | maybe_update_env() -> 55 | VarSpecs = 56 | [ {"BRUCKE_GRAPHITE_ROOT_PATH", graphite_root_path, binary} 57 | , {"BRUCKE_GRAPHITE_HOST", graphite_host, string} 58 | , {"BRUCKE_GRAPHITE_PORT", graphite_port, integer} 59 | , {"BRUCKE_HTTP_PORT", http_port, integer} 60 | , {"BRUCKE_FILTER_EBIN_PATHS", filter_ebin_dirs, fun parse_paths/1} 61 | , {"BRUCKE_CONFIG_FILE", config_file, string} 62 | ], 63 | maybe_update_env(VarSpecs). 64 | 65 | maybe_update_env([]) -> ok; 66 | maybe_update_env([{EnvVarName, AppVarName, Type} | VarSpecs]) -> 67 | ok = maybe_set_app_env(EnvVarName, AppVarName, Type), 68 | maybe_update_env(VarSpecs). 69 | 70 | maybe_set_app_env(EnvVarName, AppVarName, Type) -> 71 | EnvVar = os:getenv(EnvVarName), 72 | case EnvVar of 73 | false -> ok; 74 | [] -> ok; 75 | X -> 76 | Value = transform_env_var_value(X, Type), 77 | logger:info("Setting app-env ~p from os-env ~s, value=~p", 78 | [AppVarName, EnvVarName, Value]), 79 | application:set_env(?APPLICATION, AppVarName, Value) 80 | end, 81 | ok. 82 | 83 | transform_env_var_value(S, string) -> S; 84 | transform_env_var_value(S, binary) -> list_to_binary(S); 85 | transform_env_var_value(I, integer) -> list_to_integer(I); 86 | transform_env_var_value(I, Fun) -> Fun(I). 87 | 88 | %% Parse comma or colon separated paths. 89 | parse_paths(Paths) -> string:tokens(Paths, ":,"). 90 | 91 | %% @private Add extra ebin paths to code path. 92 | %% There is usually no need to set `filter_ebin_dirs` 93 | %% if brucke is used as a lib application for another project. 94 | %% @end 95 | -spec add_filter_ebin_dirs() -> ok. 96 | add_filter_ebin_dirs() -> 97 | Dirs = application:get_env(?APPLICATION, filter_ebin_dirs, []), 98 | ok = code:add_pathsa(Dirs). 99 | 100 | app_env(Key) -> app_env(Key, undefined). 101 | 102 | app_env(Key, Def) -> 103 | application:get_env(?APPLICATION, Key, Def). 104 | 105 | %%%_* Emacs ==================================================================== 106 | %%% Local Variables: 107 | %%% allout-layout: t 108 | %%% erlang-indent-level: 2 109 | %%% End: 110 | -------------------------------------------------------------------------------- /src/brucke_backlog.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | %% This module manages an opaque structure for callers to: 18 | %% add new items to backlog 19 | %% acknowledge finished items 20 | %% prune finished items (sucessive header items) 21 | %% implemented as gb_tree because we need to 22 | %% 1. lookup using key (kafka offset) 23 | %% 2. scan items starting from smallest 24 | -module(brucke_backlog). 25 | 26 | -export([ new/0 27 | , add/3 28 | , ack/3 29 | , prune/1 30 | , get_producers/1 31 | , to_list/1 32 | ]). 33 | 34 | -export_type([backlog/0]). 35 | 36 | -opaque backlog() :: gb_trees:tree(). 37 | -type offset() :: brod:offset(). 38 | -type ref() :: {pid(), reference()}. 39 | -define(REF_TUPLE_POS, 2). 40 | 41 | -spec new() -> backlog(). 42 | new() -> gb_trees:empty(). 43 | 44 | %% @doc Add new pending acks for the given upstream offset to backlog. 45 | -spec add(offset(), [ref()], backlog()) -> backlog(). 46 | add(Offset, Refs, Backlog) -> 47 | gb_trees:enter(Offset, Refs, Backlog). 48 | 49 | %% @doc Delete pending reference associated to the given upstream offset. 50 | -spec ack(offset(), reference(), backlog()) -> backlog(). 51 | ack(Offset, Ref, Backlog) -> 52 | case gb_trees:lookup(Offset, Backlog) of 53 | none -> Backlog; 54 | {value, Refs0} -> 55 | Refs = lists:keydelete(Ref, ?REF_TUPLE_POS, Refs0), 56 | gb_trees:enter(Offset, Refs, Backlog) 57 | end. 58 | 59 | %% @doc Delete sucessive header items which are completed. 60 | -spec prune(backlog()) -> {false | offset(), backlog()}. 61 | prune(Backlog) -> 62 | prune_loop(Backlog, false). 63 | 64 | %% @doc Get all producer pids. 65 | -spec get_producers(backlog()) -> [pid()]. 66 | get_producers(Backlog) -> 67 | [Pid || Refs <- gb_trees:values(Backlog), {Pid, _Ref} <- Refs]. 68 | 69 | %% @hidden For test 70 | -spec to_list(backlog()) -> [{offset(), [{pid(), reference()}]}]. 71 | to_list(Backlog) -> gb_trees:to_list(Backlog). 72 | 73 | %%%_* internals ================================================================ 74 | 75 | prune_loop(Backlog, ResultOffset) -> 76 | case gb_trees:is_empty(Backlog) of 77 | true -> 78 | {ResultOffset, Backlog}; 79 | false -> 80 | case gb_trees:take_smallest(Backlog) of 81 | {Offset, [], NewBacklog} -> 82 | %% all downstream messages from this upstream offset are finished 83 | prune_loop(NewBacklog, Offset); 84 | _ -> 85 | {ResultOffset, Backlog} 86 | end 87 | end. 88 | 89 | %%%_* Emacs ==================================================================== 90 | %%% Local Variables: 91 | %%% allout-layout: t 92 | %%% erlang-indent-level: 2 93 | %%% End: 94 | -------------------------------------------------------------------------------- /src/brucke_config.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | %% A brucke config file is a YAML file. 18 | %% Cluster names and client names must comply to erlang atom syntax. 19 | %% 20 | %% kafka_clusters: 21 | %% kafka_cluster_1: 22 | %% - localhost:9092 23 | %% kafka_cluster_2: 24 | %% - kafka-1:9092 25 | %% - kafka-2:9092 26 | %% brod_clients: 27 | %% - client: brod_client_1 28 | %% cluster: kafka_cluster_1 29 | %% config: 30 | %% ssl: 31 | %% # start with "priv/" or provide full path 32 | %% cacertfile: priv/ssl/ca.crt 33 | %% certfile: priv/ssl/client.crt 34 | %% keyfile: priv/ssl/client.key 35 | %% routes: 36 | %% - upstream_client: brod_client_1 37 | %% downstream_client: brod_client_1 38 | %% upstream_topics: 39 | %% - "topic_1" 40 | %% downstream_topic: "topic_2" 41 | %% repartitioning_strategy: strict_p2p 42 | %% default_begin_offset: earliest # optional 43 | %% compression: no_compression # optional 44 | %% required_acks: -1 # optional 45 | %% ratelimit_interval: 1000 #(ms) optional 46 | %% ratelimit_threshold: 0 #optional 47 | %% 48 | -module(brucke_config). 49 | 50 | -export([ init/0 51 | , init/1 52 | , is_configured_client_id/1 53 | , get_cluster_name/1 54 | , all_clients/0 55 | , get_client_endpoints/1 56 | ]). 57 | 58 | %% Exported for test 59 | -export([ validate_client_config/2 60 | ]). 61 | 62 | -include("brucke_int.hrl"). 63 | 64 | -define(ETS, ?MODULE). 65 | 66 | -type config_tag() :: atom() | string() | binary(). 67 | -type config_value() :: atom() | string() | integer(). 68 | -type config_entry() :: {config_tag(), config_value() | config()}. 69 | -type config() :: [config_entry()]. 70 | -type client_id() :: brod:client_id(). 71 | 72 | %%%_* APIs ===================================================================== 73 | 74 | -spec init() -> ok | no_return(). 75 | init() -> 76 | File = assert_file(brucke_app:config_file()), 77 | init(File). 78 | 79 | init(File) -> 80 | yamerl_app:set_param(node_mods, [yamerl_node_erlang_atom]), 81 | try 82 | [Configs] = yamerl_constr:file(File, [{erlang_atom_autodetection, true}]), 83 | do_init(Configs) 84 | catch C : E ?BIND_STACKTRACE(Stack) -> 85 | ?GET_STACKTRACE(Stack), 86 | logger:emergency("failed to load brucke config file ~s: ~p:~p\n~p", 87 | [File, C, E, Stack]), 88 | exit({bad_brucke_config, File, Stack}) 89 | end. 90 | 91 | -spec is_configured_client_id(brod:client_id()) -> boolean(). 92 | is_configured_client_id(ClientId) when is_atom(ClientId) -> 93 | case lookup(ClientId) of 94 | false -> false; 95 | {ClientId, _, _} -> true 96 | end. 97 | 98 | -spec get_cluster_name(brod:client_id()) -> cluster_name(). 99 | get_cluster_name(ClientId) when is_atom(ClientId) -> 100 | case lookup(ClientId) of 101 | {ClientId, ClusterName, _Config} -> 102 | ClusterName; 103 | false -> 104 | erlang:error({bad_client_id, ClientId}) 105 | end. 106 | 107 | -spec all_clients() -> [client()]. 108 | all_clients() -> 109 | [{ClientId, 110 | begin 111 | {ClusterName, Endpoints} = lookup(ClusterName), 112 | Endpoints 113 | end, 114 | ClientConfig 115 | } || {ClientId, ClusterName, ClientConfig} <- ets:tab2list(?ETS)]. 116 | 117 | -spec get_client_endpoints(brod:client_id()) -> [{string(), integer()}]. 118 | get_client_endpoints(ClientId) -> 119 | case lookup(ClientId) of 120 | false -> 121 | []; 122 | {ClientId, ClusterName, _} -> 123 | case lookup(ClusterName) of 124 | false -> 125 | []; 126 | {ClusterName, Endpoints} -> 127 | Endpoints 128 | end 129 | end. 130 | 131 | %%%_* Internal functions ======================================================= 132 | 133 | -spec assert_file(filename() | {priv, filename()}) -> filename() | no_return(). 134 | assert_file({priv, Path}) -> 135 | assert_file(filename:join(code:priv_dir(?APPLICATION), Path)); 136 | assert_file(Path) -> 137 | case filelib:is_regular(Path) of 138 | true -> 139 | Path; 140 | false -> 141 | logger:emergency("~s is not a regular file", [Path]), 142 | exit({bad_brucke_config_file, Path}) 143 | end. 144 | 145 | -spec do_init([config()]) -> ok | no_return(). 146 | do_init(Configs) -> 147 | Kf = fun(K) -> 148 | case lists:keyfind(K, 1, Configs) of 149 | {K, V} -> 150 | V; 151 | false -> 152 | logger:emergency("~p is not found in config", [K]), 153 | exit({mandatory_config_entry_not_found, K}) 154 | end 155 | end, 156 | Clusters = Kf(kafka_clusters), 157 | Clients = Kf(brod_clients), 158 | Routes = Kf(routes), 159 | case ets:info(?ETS) of 160 | ?undef -> 161 | ok; 162 | _ -> 163 | logger:emergency("config already loaded"), 164 | exit({?ETS, already_created}) 165 | end, 166 | ?ETS = ets:new(?ETS, [named_table, protected, set]), 167 | 168 | OffsetsDets = proplists:get_value(offsets_dets_path, Configs, ?DEFAULT_OFFSETS_DETS_PATH), 169 | 170 | {ok, ?OFFSETS_TAB} = dets:open_file(?OFFSETS_TAB, [{file, OffsetsDets}, {ram_file, true}]), 171 | 172 | try 173 | init(Clusters, Clients, Routes) 174 | catch 175 | error : Reason ?BIND_STACKTRACE(Stack) -> 176 | ?GET_STACKTRACE(Stack), 177 | ok = destroy(), 178 | erlang:exit({error, Reason, Stack}) 179 | end. 180 | 181 | -spec destroy() -> ok. 182 | destroy() -> 183 | try 184 | ets:delete(?ETS), 185 | ok 186 | catch error : badarg -> 187 | ok 188 | end. 189 | 190 | lookup(Key) -> 191 | case ets:lookup(?ETS, Key) of 192 | [] -> false; 193 | [R] -> R 194 | end. 195 | 196 | -spec init(config(), config(), config()) -> ok | no_return(). 197 | init(Clusters, _, _) when not is_list(Clusters) orelse Clusters == [] -> 198 | logger:emergency("Expecting list of kafka clusters " 199 | "Got ~P\n", [Clusters, 9]), 200 | exit(bad_cluster_list); 201 | init(_, Clients, _) when not is_list(Clients) orelse Clients == [] -> 202 | logger:emergency("Expecting list of brod clients " 203 | "Got ~P\n", [Clients, 9]), 204 | exit(bad_client_list); 205 | init(_, _, Routes) when not is_list(Routes) orelse Routes == [] -> 206 | logger:emergency("Expecting list of brucke routes " 207 | "Got ~P\n", [Routes, 9]), 208 | exit(bad_route_list); 209 | init(Clusters, Clients, Routes) -> 210 | lists:foreach( 211 | fun(Cluster) -> 212 | {ClusterName, Endpoints} = validate_cluster(Cluster), 213 | case lookup(ClusterName) of 214 | false -> 215 | ok; 216 | {ClusterName, _} -> 217 | logger:emergency("Duplicated cluster name ~p", [ClusterName]), 218 | exit({duplicated_cluster_name, ClusterName}) 219 | end, 220 | ets:insert(?ETS, {ClusterName, Endpoints}) 221 | end, Clusters), 222 | lists:foreach( 223 | fun(Client) -> 224 | {ClientId, ClusterName, ClientConfig} = validate_client(Client), 225 | case lookup(ClientId) of 226 | false -> ok; 227 | _ -> 228 | logger:emergency("Duplicated brod client id ~p", [ClientId]), 229 | exit({duplicated_brod_client_id, ClientId}) 230 | end, 231 | case lookup(ClusterName) of 232 | false -> 233 | logger:emergency("Cluster name ~s for client ~p is not found", 234 | [ClusterName, ClientId]), 235 | exit({cluster_not_found_for_client, ClusterName, ClientId}); 236 | _ -> 237 | ok 238 | end, 239 | ets:insert(?ETS, {ClientId, ClusterName, ClientConfig}) 240 | end, Clients), 241 | ok = brucke_routes:init(Routes). 242 | 243 | validate_cluster({ClusterId, [_|_] = Endpoints}) -> 244 | {ensure_binary(ClusterId), 245 | [validate_endpoint(Endpoint) || Endpoint <- Endpoints]}; 246 | validate_cluster(Other) -> 247 | logger:emergency("Expecing cluster config with cluster id " 248 | "and a list of hostname:port endpoints"), 249 | exit({bad_cluster_config, Other}). 250 | 251 | validate_client(Client) -> 252 | try 253 | {_, ClientId} = lists:keyfind(client, 1, Client), 254 | {_, ClusterName} = lists:keyfind(cluster, 1, Client), 255 | Config0 = proplists:get_value(config, Client, []), 256 | Config1 = validate_client_config(ClientId, Config0), 257 | %% Enable api version query by default 258 | Config = case proplists:get_value(query_api_versions, Config1) of 259 | undefined -> [{query_api_versions, true} | Config1]; 260 | _ -> Config1 261 | end, 262 | {ensure_atom(ClientId), 263 | ensure_binary(ClusterName), 264 | Config} 265 | catch 266 | error : Reason ?BIND_STACKTRACE(Stack) -> 267 | ?GET_STACKTRACE(Stack), 268 | logger:emergency("Bad brod client config: ~P.\nreason=~p\nstack=~p", 269 | [Client, 9, Reason, Stack]), 270 | exit(bad_client_config) 271 | end. 272 | 273 | ensure_atom(A) when is_atom(A) -> A. 274 | 275 | ensure_binary(A) when is_atom(A) -> 276 | ensure_binary(atom_to_list(A)); 277 | ensure_binary(L) when is_list(L) -> 278 | list_to_binary(L); 279 | ensure_binary(B) when is_binary(B) -> 280 | B. 281 | 282 | validate_endpoint(HostPort) when is_list(HostPort) -> 283 | case string:tokens(HostPort, ":") of 284 | [Host, Port] -> 285 | try 286 | {Host, list_to_integer(Port)} 287 | catch 288 | _ : _ -> 289 | exit_on_bad_endpoint(HostPort) 290 | end; 291 | _Other -> 292 | exit_on_bad_endpoint(HostPort) 293 | end; 294 | validate_endpoint(Other) -> 295 | exit_on_bad_endpoint(Other). 296 | 297 | exit_on_bad_endpoint(Bad) -> 298 | logger:emergency("Expecting endpoints string of patern Host:Port\n" 299 | "Got ~P", [Bad, 9]), 300 | exit(bad_endpoint). 301 | 302 | %% @hidden Exported for test 303 | validate_client_config(ClientId, Config) when is_list(Config) -> 304 | lists:map(fun(ConfigEntry) -> 305 | do_validate_client_config(ClientId, ConfigEntry) 306 | end, Config); 307 | validate_client_config(ClientId, Config) -> 308 | logger:emergency("Expecing client config to be a list for client ~p.\nGot:~p", 309 | [ClientId, Config]), 310 | exit(bad_client_config). 311 | 312 | do_validate_client_config(ClientId, {ssl, Options}) -> 313 | {ssl, validate_ssl_option(ClientId, Options)}; 314 | do_validate_client_config(ClientId, {sasl, Options}) -> 315 | {sasl, validate_sasl_option(ClientId, Options)}; 316 | do_validate_client_config(_ClientId, {_, _} = ConfigEntry) -> 317 | ConfigEntry; 318 | do_validate_client_config(ClientId, Other) -> 319 | logger:emergency("Unknown client config entry for client ~p," 320 | "expecting kv-pair\nGot:~p", [ClientId, Other]), 321 | exit(bad_client_config_entry). 322 | 323 | -spec validate_ssl_option(client_id(), true | list()) -> 324 | boolean() | list() | none(). 325 | validate_ssl_option(_ClientId, true) -> 326 | true; 327 | validate_ssl_option(ClientId, SslOptions) -> 328 | Options = 329 | lists:foldl( 330 | fun(OptName, OptIn) -> 331 | validate_ssl_option(ClientId, OptIn, OptName) 332 | end, SslOptions, [ cacertfile 333 | , certfile 334 | , keyfile 335 | ]), 336 | case Options =:= [] of 337 | true -> true; 338 | false -> Options 339 | end. 340 | 341 | -spec validate_ssl_option(client_id(), list(), 342 | cacertfile | certfile | keyfile) -> list() | none(). 343 | validate_ssl_option(ClientId, SslOptions, OptName) -> 344 | case lists:keyfind(OptName, 1, SslOptions) of 345 | {_, Filename0} -> 346 | Filename = validate_ssl_file(ClientId, Filename0), 347 | lists:keyreplace(OptName, 1, SslOptions, {OptName, Filename}); 348 | false -> 349 | SslOptions 350 | end. 351 | 352 | -spec validate_ssl_file(client_id(), filename()) -> filename() | none(). 353 | validate_ssl_file(ClientId, Filename) -> 354 | Path = 355 | case filename:split(Filename) of 356 | ["priv" | PrivPath] -> 357 | filename:join([code:priv_dir(?APPLICATION) | PrivPath]); 358 | _ -> 359 | Filename 360 | end, 361 | case filelib:is_regular(Path) of 362 | true -> 363 | Path; 364 | false -> 365 | logger:emergency("ssl file ~p not found for client ~p", [Path, ClientId]), 366 | exit(bad_ssl_file) 367 | end. 368 | 369 | -spec validate_sasl_option(client_id(), list()) -> 370 | {plain | scram_sha_256 | scram_sha_512, binary(), binary()} | none(). 371 | validate_sasl_option(ClientId, SaslOptions) -> 372 | Default = {mechanism, plain}, 373 | Mechanism = do_validate_sasl_option(ClientId, mechanism, SaslOptions ++ [Default]), 374 | Username = do_validate_sasl_option(ClientId, username, SaslOptions), 375 | Password = do_validate_sasl_option(ClientId, password, SaslOptions), 376 | validate_sasl_mechanism(ClientId, Mechanism) andalso 377 | {list_to_atom(Mechanism), list_to_binary(Username), list_to_binary(Password)}. 378 | 379 | validate_sasl_mechanism(_ClientId, "plain") -> true; 380 | validate_sasl_mechanism(_ClientId, "scram_sha_256") -> true; 381 | validate_sasl_mechanism(_ClientId, "scram_sha_512") -> true; 382 | validate_sasl_mechanism(ClientId, Other) -> 383 | logger:emergency("Unknown sasl mechanism ~p is for client ~p", [Other, ClientId]), 384 | exit(bad_sasl_mechanism). 385 | 386 | do_validate_sasl_option(ClientId, Option, SaslOptions) -> 387 | case lists:keyfind(Option, 1, SaslOptions) of 388 | {_, Value} when is_list(Value) -> Value; 389 | {_, Value} when is_atom(Value) -> atom_to_list(Value); 390 | false -> 391 | logger:emergency("SASL ~p is not specified or in wrong format for client ~p", [Option, ClientId]), 392 | exit(bad_sasl_credentials) 393 | end. 394 | 395 | %%%_* Emacs ==================================================================== 396 | %%% Local Variables: 397 | %%% allout-layout: t 398 | %%% erlang-indent-level: 2 399 | %%% End: 400 | -------------------------------------------------------------------------------- /src/brucke_filter.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module(brucke_filter). 18 | 19 | -export([ init/3 20 | , init/4 21 | , filter/7 22 | , filter/8 23 | ]). 24 | 25 | -export_type([ cb_state/0 26 | , filter_result/0 27 | , filter_return/0 28 | ]). 29 | 30 | -include("brucke_int.hrl"). 31 | 32 | -type cb_state() :: term(). 33 | -type headers() :: kpro:headers(). 34 | -type filter_result() :: boolean() 35 | | {brod:key(), brod:value()} 36 | | {brod:msg_ts(), brod:key(), brod:value()} 37 | | #{key => iodata(), 38 | value => iodata(), 39 | ts => brod:msg_ts(), 40 | headers => [{binary(), binary()}] 41 | } 42 | | [filter_result()]. 43 | -type filter_return() :: {filter_result(), cb_state()}. 44 | 45 | -define(DEFAULT_STATE, []). 46 | 47 | %% Called when route worker (`brucke_subscriber') start/restart. 48 | -callback init(UpstreamTopic :: brod:topic(), 49 | UpstreamPartition :: brod:partition(), 50 | InitArg :: term()) -> {ok, cb_state()}. 51 | 52 | %% Called by assignment worker (`brucke_subscriber') for each message. 53 | %% Return value implications: 54 | %% true: No change, forward the message as-is to downstream 55 | %% false: Discard the message 56 | %% {NewKey, NewValue}: Produce the transformed new Key and Value to downstream. 57 | -callback filter(Topic :: brod:topic(), 58 | Partition :: brod:partition(), 59 | Offset :: brod:offset(), 60 | Key :: brod:key(), 61 | Value :: brod:value(), 62 | Headers :: headers(), 63 | CbState :: cb_state()) -> filter_return(). 64 | 65 | %% @doc Call callback module's init API 66 | -spec init(module(), brod:topic(), brod:partition(), term()) -> 67 | {ok, cb_state()}. 68 | init(Module, UpstreamTopic, UpstreamPartition, InitArg) -> 69 | Module:init(UpstreamTopic, UpstreamPartition, InitArg). 70 | 71 | %% @doc The default filter does not do anything special. 72 | -spec init(brod:topic(), brod:partition(), term()) -> {ok, _}. 73 | init(_UpstreamTopic, _UpstreamPartition, _InitArg) -> 74 | {ok, ?DEFAULT_STATE}. 75 | 76 | %% @doc Filter message set. 77 | -spec filter(module(), brod:topic(), brod:partition(), brod:offset(), 78 | brod:key(), brod:value(), headers(), cb_state()) -> filter_return(). 79 | filter(Module, Topic, Partition, Offset, Key, Value, Headers, CbState) -> 80 | Module:filter(Topic, Partition, Offset, Key, Value, Headers, CbState). 81 | 82 | %% @doc The default filter does nothing. 83 | -spec filter(brod:topic(), brod:partition(), brod:offset(), 84 | brod:key(), brod:value(), headers(), cb_state()) -> filter_return(). 85 | filter(_Topic, _Partition, _Offset, _Key, _Value, _Headers, ?DEFAULT_STATE) -> 86 | {true, ?DEFAULT_STATE}. 87 | 88 | %%%_* Emacs ==================================================================== 89 | %%% Local Variables: 90 | %%% allout-layout: t 91 | %%% erlang-indent-level: 2 92 | %%% End: 93 | -------------------------------------------------------------------------------- /src/brucke_http.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2017 Klarna AB 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | -module(brucke_http). 17 | 18 | %% API 19 | -export([ init/0 20 | , register_plugin_handler/3] 21 | ). 22 | 23 | -define(APP, brucke). 24 | -define(HTTP_LISTENER, http). 25 | 26 | -define(DEF_PATHS, [ {"/ping", brucke_http_ping_handler, []} 27 | , {"/", brucke_http_healthcheck_handler, []} 28 | , {"/healthcheck", brucke_http_healthcheck_handler, []} 29 | ]). 30 | 31 | init() -> 32 | case brucke_app:http_port() of 33 | undefined -> 34 | %% not configured, do not start anything 35 | ok; 36 | Port -> 37 | Paths = get_route_path(), 38 | set_route_path(Paths), 39 | Dispatch = cowboy_router:compile(routes(Paths)), 40 | 41 | logger:info("Starting http listener on port ~p", [Port]), 42 | 43 | case cowboy:start_http(?HTTP_LISTENER, 8, [{port, Port}], 44 | [ {env, [{dispatch, Dispatch}]} 45 | , {onresponse, fun error_hook/4}]) of 46 | {ok, _Pid} -> 47 | ok; 48 | {error, {already_started, _Pid}} -> 49 | ok; 50 | Other -> 51 | {error, Other} 52 | end 53 | end. 54 | 55 | error_hook(Code, _Headers, _Body, Req) -> 56 | {Method, _} = cowboy_req:method(Req), 57 | {Version, _} = cowboy_req:version(Req), 58 | {Path, _} = cowboy_req:path(Req), 59 | {{Ip, _Port}, _} = cowboy_req:peer(Req), 60 | Level = log_level(Code), 61 | logger:Level(self(), "~s ~s ~b ~p ~s", [Method, Path, Code, Version, inet:ntoa(Ip)]), 62 | Req. 63 | 64 | log_level(Code) when Code < 400 -> info; 65 | log_level(_Code) -> error. 66 | 67 | -spec register_plugin_handler(PluginPath::string(), Handler::module(), Opts :: any()) -> ok. 68 | register_plugin_handler(PluginPath, Handler, Opts) -> 69 | Path = {"/plugins/" ++ PluginPath, Handler, Opts}, 70 | NewPaths = [ Path | get_route_path() ], 71 | Dispatch = cowboy_router:compile(routes(NewPaths)), 72 | cowboy:set_env(?HTTP_LISTENER, dispatch, Dispatch), 73 | ok. 74 | 75 | -spec get_route_path() -> [cowboy_router:route_path()]. 76 | get_route_path()-> 77 | application:get_env(?APP, route_paths, ?DEF_PATHS). 78 | 79 | -spec set_route_path([cowboy_router:route_path()]) -> ok. 80 | set_route_path(Paths)-> 81 | application:set_env(?APP, route_paths, Paths). 82 | 83 | -spec routes([cowboy_router:route_path()]) -> cowboy_router:routes(). 84 | routes(PathLists) -> 85 | [{'_', PathLists}]. 86 | 87 | %%%_* Emacs ==================================================================== 88 | %%% Local Variables: 89 | %%% allout-layout: t 90 | %%% erlang-indent-level: 2 91 | %%% End: 92 | -------------------------------------------------------------------------------- /src/brucke_http_healthcheck_handler.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2017-2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | -module(brucke_http_healthcheck_handler). 17 | 18 | -export([ init/3 19 | , handle_request/2 20 | , content_types_provided/2 21 | ]). 22 | 23 | 24 | -include("brucke_int.hrl"). 25 | 26 | init(_Transport, _Req, []) -> 27 | {upgrade, protocol, cowboy_rest}. 28 | 29 | content_types_provided(Req, State) -> 30 | {[{<<"application/json">>, handle_request}], Req, State}. 31 | 32 | handle_request(Req, State) -> 33 | HealthStatus0 = brucke_routes:health_status(), 34 | Status = get_status_string(HealthStatus0), 35 | HealthStatus = maps:put(status, Status, HealthStatus0), 36 | {jsone:encode(HealthStatus), Req, State}. 37 | 38 | %% @private 39 | get_status_string(#{unhealthy := [], discarded := []}) -> <<"ok">>; 40 | get_status_string(_HealthStatus) -> <<"failing">>. 41 | 42 | %%%_* Emacs ==================================================================== 43 | %%% Local Variables: 44 | %%% allout-layout: t 45 | %%% erlang-indent-level: 2 46 | %%% End: 47 | -------------------------------------------------------------------------------- /src/brucke_http_ping_handler.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2017 Klarna AB 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | -module(brucke_http_ping_handler). 17 | 18 | -export([ init/3 19 | , handle_request/2 20 | , content_types_provided/2 21 | ]). 22 | 23 | init(_Transport, _Req, []) -> 24 | {upgrade, protocol, cowboy_rest}. 25 | 26 | content_types_provided(Req, State) -> 27 | {[{<<"text/plain">>, handle_request}], Req, State}. 28 | 29 | handle_request(Req, State) -> 30 | {<<"pong">>, Req, State}. 31 | 32 | %%%_* Emacs ==================================================================== 33 | %%% Local Variables: 34 | %%% allout-layout: t 35 | %%% erlang-indent-level: 2 36 | %%% End: 37 | -------------------------------------------------------------------------------- /src/brucke_lib.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2017 Klarna AB 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module(brucke_lib). 18 | 19 | -export([ log_skipped_route_alert/2 20 | , get_consumer_config/1 21 | , get_repartitioning_strategy/1 22 | , get_producer_config/1 23 | , fmt_route/1 24 | ]). 25 | 26 | -include("brucke_int.hrl"). 27 | 28 | %%%_* APIs ===================================================================== 29 | 30 | -spec fmt_route(route()) -> iodata(). 31 | fmt_route(#route{upstream = Upstream, downstream = Downstream}) -> 32 | io_lib:format("~p -> ~p", [Upstream, Downstream]). 33 | 34 | -spec log_skipped_route_alert(route() | raw_route(), iodata()) -> ok. 35 | log_skipped_route_alert(#route{} = Route, Reasons) -> 36 | logger:alert("SKIPPING bad route: ~s\nREASON(s):~s", 37 | [fmt_route(Route), Reasons]), 38 | ok; 39 | log_skipped_route_alert(Route, Reasons) -> 40 | logger:alert("SKIPPING bad route: ~p\nREASON(s):~s", 41 | [Route, Reasons]), 42 | ok. 43 | 44 | -spec get_repartitioning_strategy(route_options()) -> repartitioning_strategy(). 45 | get_repartitioning_strategy(Options) -> 46 | maps:get(repartitioning_strategy, Options, ?DEFAULT_REPARTITIONING_STRATEGY). 47 | 48 | -spec get_consumer_config(route_options()) -> brod:consumer_config(). 49 | get_consumer_config(Options) -> 50 | maybe_use_brucke_defaults( 51 | maps:get(consumer_config, Options, []), 52 | default_consumer_config()). 53 | 54 | -spec get_producer_config(route_options()) -> brod:producer_config(). 55 | get_producer_config(Options) -> 56 | maybe_use_brucke_defaults( 57 | maps:get(producer_config, Options, []), 58 | default_producer_config()). 59 | 60 | %%%_* Internal Functions ======================================================= 61 | 62 | 63 | %% @private use hard-coded defaults if not found in config 64 | maybe_use_brucke_defaults(Config, []) -> 65 | Config; 66 | maybe_use_brucke_defaults(Config, [{K, V} | Rest]) -> 67 | NewConfig = 68 | case lists:keyfind(K, 1, Config) of 69 | {K, _} -> Config; 70 | false -> [{K, V} | Config] 71 | end, 72 | maybe_use_brucke_defaults(NewConfig, Rest). 73 | 74 | %% @private The default values for brucke. 75 | default_consumer_config() -> 76 | [ {prefetch_count, 12} 77 | , {begin_offset, latest} 78 | ]. 79 | 80 | %% @private Default producer config 81 | default_producer_config() -> 82 | [ {max_linger_ms, 2000} %% 2 seconds 83 | , {max_linger_count, 100} 84 | , {max_batch_size, 800000} %% 800K 85 | , {ack_timeout, 10000} %% 10 seconds 86 | , {retry_backoff_ms, 1000} 87 | ]. 88 | 89 | %%%_* Emacs ==================================================================== 90 | %%% Local Variables: 91 | %%% allout-layout: t 92 | %%% erlang-indent-level: 2 93 | %%% End: 94 | -------------------------------------------------------------------------------- /src/brucke_member.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2017 Klarna AB 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | %% The upstream topic consumer group member 18 | -module(brucke_member). 19 | 20 | -behaviour(gen_server). 21 | -behaviour(brod_group_member). 22 | 23 | -export([ start_link/1 24 | , is_healthy/1 25 | ]). 26 | 27 | %% gen_server callbacks 28 | -export([ code_change/3 29 | , handle_call/3 30 | , handle_cast/2 31 | , handle_info/2 32 | , init/1 33 | , terminate/2 34 | ]). 35 | 36 | %% group coordinator callbacks 37 | -export([ get_committed_offsets/2 38 | , assign_partitions/3 39 | , assignments_received/4 40 | , assignments_revoked/1 41 | ]). 42 | 43 | -include("brucke_int.hrl"). 44 | 45 | -type topic() :: brod:topic(). 46 | -type partition() :: brod:partition(). 47 | -type offset() :: brod:offset(). 48 | 49 | -define(SUBSCRIBER_RESTART_DELAY_SECONDS, 10). 50 | -define(DEAD(Ts), {dead_since, Ts}). 51 | -define(IGNORED(Ts), {ignored_since, Ts}). 52 | -record(subscriber, { partition :: partition() 53 | , begin_offset :: offset() 54 | , pid :: pid() 55 | | ?DEAD(erlang:timestamp()) 56 | | ?IGNORED(erlang:timestamp()) 57 | }). 58 | 59 | %%%_* APIs ===================================================================== 60 | 61 | -spec start_link(route()) -> {ok, pid()} | {error, any()}. 62 | start_link(Route) -> gen_server:start_link(?MODULE, Route, []). 63 | 64 | %% @doc returns true if all subscribers are up and running 65 | -spec is_healthy(pid()) -> boolean(). 66 | is_healthy(Pid) -> 67 | gen_server:call(Pid, is_healthy). 68 | 69 | -spec get_cg_id(pid()) -> brod:group_id(). 70 | get_cg_id(Pid) -> 71 | gen_server:call(Pid, get_cg_id). 72 | 73 | %%%_* brod_group_member callbacks ========================================== 74 | 75 | -spec get_committed_offsets(pid(), [{topic(), partition()}]) -> 76 | {ok, [{{topic(), partition()}, offset()}]}. 77 | get_committed_offsets(GroupMemberPid, TopicPartitions) -> 78 | GroupId = get_cg_id(GroupMemberPid), 79 | Res = lists:map(fun({Topic, Partition}) -> 80 | case dets:lookup(?OFFSETS_TAB, {GroupId, Topic, Partition}) of 81 | [{_K, V}] -> 82 | [{{Topic, Partition}, V}]; 83 | _ -> 84 | [] 85 | end 86 | end, TopicPartitions), 87 | {ok, lists:append(Res)}. 88 | 89 | -spec assign_partitions(pid(), [brod:group_member()], 90 | [{topic(), partition()}]) -> 91 | [{brod:group_member_id(), 92 | [brod:partition_assignment()]}]. 93 | assign_partitions(_Pid, _Members, _TopicPartitions) -> 94 | erlang:exit({no_impl, assign_partitions}). 95 | 96 | -spec assignments_received(pid(), brod:group_member_id(), 97 | brod:group_generation_id(), 98 | brod:received_assignments()) -> ok. 99 | assignments_received(MemberPid, MemberId, GenerationId, Assignments) -> 100 | Msg = {assignments_received, MemberId, GenerationId, Assignments}, 101 | gen_server:cast(MemberPid, Msg). 102 | 103 | -spec assignments_revoked(pid()) -> ok. 104 | assignments_revoked(GroupMemberPid) -> 105 | gen_server:cast(GroupMemberPid, assignments_revoked). 106 | 107 | %%%_* gen_server callbacks ===================================================== 108 | 109 | init(Route) -> 110 | erlang:process_flag(trap_exit, true), 111 | self() ! {post_init, Route}, 112 | {ok, #{}}. 113 | 114 | handle_info({post_init, #route{options = Options} = Route}, State) -> 115 | {UpstreamClientId, UpstreamTopic} = Route#route.upstream, 116 | {DownstreamClientId, DownstreamTopic} = Route#route.downstream, 117 | OffsetCommitPolicy = maps:get(offset_commit_policy, Options), 118 | GroupConfig = [{offset_commit_policy, OffsetCommitPolicy} 119 | ,{offset_commit_interval_seconds, 10} 120 | ], 121 | ConsumerConfig = brucke_lib:get_consumer_config(Options), 122 | ProducerConfig = maps:get(producer_config, Options, []), 123 | ok = brod:start_consumer(UpstreamClientId, UpstreamTopic, ConsumerConfig), 124 | ok = brod:start_producer(DownstreamClientId, DownstreamTopic, ProducerConfig), 125 | ConsumerGroupId = brucke_routes:get_cg_id(Options), 126 | {ok, Pid} = 127 | brod_group_coordinator:start_link(UpstreamClientId, ConsumerGroupId, [UpstreamTopic], 128 | GroupConfig, ?MODULE, self()), 129 | _ = send_loopback_msg(1, restart_dead_subscribers), 130 | {noreply, State#{ coordinator => Pid 131 | , route => Route 132 | , cg_id => ConsumerGroupId 133 | , subscribers => [] 134 | , upstream_client_mref => erlang:monitor(process, UpstreamClientId) 135 | , downstream_client_mref => erlang:monitor(process, DownstreamClientId) 136 | }}; 137 | handle_info(restart_dead_subscribers, State) -> 138 | {noreply, restart_dead_subscribers(State)}; 139 | handle_info({ack, Partition, Offset}, State) -> 140 | {noreply, handle_ack(State, Partition, Offset)}; 141 | handle_info({'EXIT', Pid, _Reason}, #{coordinator := Pid} = State) -> 142 | {stop, coordinator_down, State}; 143 | handle_info({'EXIT', Pid, Reason}, State) -> 144 | NewState = handle_subscriber_down(State, Pid, Reason), 145 | {noreply, NewState}; 146 | handle_info({'DOWN', Ref, process, _Pid, _Reason}, 147 | #{upstream_client_mref := Ref} = State) -> 148 | {stop, upstream_client_down, State}; 149 | handle_info({'DOWN', Ref, process, _Pid, _Reason}, 150 | #{downstream_client_mref := Ref} = State) -> 151 | {stop, downstream_client_down, State}; 152 | handle_info(Info, State) -> 153 | logger:error("Unknown info: ~p", [Info]), 154 | {noreply, State}. 155 | 156 | handle_call(get_cg_id, _From, #{cg_id := GroupId} = State) -> 157 | {reply, GroupId, State}; 158 | handle_call(is_healthy, _From, State) -> 159 | #{subscribers := Subscribers} = State, 160 | Res = lists:all(fun(#subscriber{pid = Pid}) -> is_pid(Pid) end, Subscribers), 161 | {reply, Res, State}; 162 | handle_call(Call, _From, State) -> 163 | logger:error("Unknown call: ~p", [Call]), 164 | {reply, {error, unknown_call}, State}. 165 | 166 | handle_cast({assignments_received, MemberId, GenerationId, Assignments}, 167 | #{ route := Route 168 | , subscribers := Subscribers0 169 | } = State) -> 170 | ok = stop_subscribers(Subscribers0), 171 | Subscribers = start_subscribers(Route, Assignments), 172 | {noreply, State#{ member_id => MemberId 173 | , generation_id => GenerationId 174 | , subscribers := Subscribers 175 | }}; 176 | handle_cast(assignments_revoked, #{subscribers := Subscribers} = State) -> 177 | ok = stop_subscribers(Subscribers), 178 | {noreply, State#{subscribers := []}}; 179 | handle_cast(Cast, State) -> 180 | logger:error("Unknown cast: ~p", [Cast]), 181 | {noreply, State}. 182 | 183 | terminate(_Reason, #{coordinator := Coordinator} = _State) -> 184 | _ = brod_group_coordinator:commit_offsets(Coordinator), 185 | ok. 186 | 187 | code_change(_OldVsn, State, _Extra) -> 188 | {ok, State}. 189 | 190 | -spec stop_subscribers([#subscriber{}]) -> ok. 191 | stop_subscribers([]) -> ok; 192 | stop_subscribers([#subscriber{pid = Pid} | Rest]) -> 193 | case is_pid(Pid) of 194 | true -> ok = brucke_subscriber:stop(Pid); 195 | false -> ok 196 | end, 197 | stop_subscribers(Rest). 198 | 199 | start_subscribers(#route{} = Route, Assignments) -> 200 | lists:map( 201 | fun(#brod_received_assignment{ partition = UpstreamPartition 202 | , begin_offset = Offset 203 | }) -> 204 | {ok, Pid} = brucke_subscriber:start_link(Route, UpstreamPartition, Offset), 205 | #subscriber{ partition = UpstreamPartition 206 | , begin_offset = Offset 207 | , pid = Pid 208 | } 209 | end, Assignments). 210 | 211 | handle_ack(#{ subscribers := Subscribers 212 | , coordinator := Coordinator 213 | , generation_id := GenerationId 214 | , cg_id := GroupId 215 | , route := Route 216 | } = State, Partition, Offset) -> 217 | case lists:keyfind(Partition, #subscriber.partition, Subscribers) of 218 | #subscriber{} = Subscriber -> 219 | NewSubscriber = Subscriber#subscriber{begin_offset = Offset + 1}, 220 | NewSubscribers = lists:keyreplace(Partition, #subscriber.partition, 221 | Subscribers, NewSubscriber), 222 | #route{ upstream = {_UpstreamClientId, UpstreamTopic} 223 | , options = Options 224 | } = Route, 225 | 226 | consumer_managed =:= maps:get(offset_commit_policy, Options) 227 | andalso dets:insert(?OFFSETS_TAB, {{GroupId, UpstreamTopic, Partition}, Offset}), 228 | 229 | ok = brod_group_coordinator:ack(Coordinator, GenerationId, UpstreamTopic, 230 | Partition, Offset), 231 | State#{subscribers := NewSubscribers}; 232 | false -> 233 | logger:info("discarded ack ~s:~w:~w", 234 | [fmt_route(Route), Partition, Offset]), 235 | State 236 | end. 237 | 238 | handle_subscriber_down(#{ subscribers := Subscribers 239 | , route := Route 240 | } = State, Pid, Reason) -> 241 | case lists:keyfind(Pid, #subscriber.pid, Subscribers) of 242 | #subscriber{partition = Partition} = Subscriber -> 243 | logger:error("subscriber ~s:~p down, reason:\n~p", 244 | [fmt_route(Route), Partition, Reason]), 245 | NotPid = 246 | case Reason of 247 | normal -> 248 | %% I don't do anything even when all subscribers are IGNORED. 249 | %% The best thing to do here to to keep silent, 250 | %% because if I exit here (even with 'normal' reason) 251 | %% I will cause a group re-balance, then the deleted topic 252 | %% will get assigned to other members in the group, 253 | %% and cause a reassignment of ALL topic-partitions. 254 | %% Then the very next member gets (a part of) this route assigned 255 | %% will repeat this. i.e. the group will never be able to 256 | %% be balanced and probably re-producing a lot of duplicated 257 | %% messages due to a delayed commit. 258 | ?IGNORED(os:timestamp()); 259 | _ -> 260 | ?DEAD(os:timestamp()) 261 | end, 262 | NewSubscriber = Subscriber#subscriber{pid = NotPid}, 263 | NewSubscribers = lists:keyreplace(Partition, #subscriber.partition, 264 | Subscribers, NewSubscriber), 265 | State#{subscribers := NewSubscribers}; 266 | false -> 267 | %% stale down message 268 | State 269 | end. 270 | 271 | send_loopback_msg(TimeoutSeconds, Msg) -> 272 | erlang:send_after(timer:seconds(TimeoutSeconds), self(), Msg). 273 | 274 | restart_dead_subscribers(#{ route := Route 275 | , subscribers := Subscribers 276 | } = State) -> 277 | _ = send_loopback_msg(1, restart_dead_subscribers), 278 | State#{subscribers := restart_dead_subscribers(Route, Subscribers)}. 279 | 280 | restart_dead_subscribers(_Route, []) -> []; 281 | restart_dead_subscribers(Route, [#subscriber{ pid = ?DEAD(Ts) 282 | , partition = Partition 283 | , begin_offset = BeginOffset 284 | } = S | Rest]) -> 285 | Elapsed = timer:now_diff(os:timestamp(), Ts), 286 | case Elapsed > ?SUBSCRIBER_RESTART_DELAY_SECONDS * 1000000 of 287 | true -> 288 | {ok, Pid} = brucke_subscriber:start_link(Route, Partition, BeginOffset), 289 | [S#subscriber{pid = Pid} | restart_dead_subscribers(Route, Rest)]; 290 | false -> 291 | [S | restart_dead_subscribers(Route, Rest)] 292 | end; 293 | restart_dead_subscribers(Route, [#subscriber{} = S | Rest]) -> 294 | [S | restart_dead_subscribers(Route, Rest)]. 295 | 296 | -spec fmt_route(route()) -> iodata(). 297 | fmt_route(Route) -> brucke_lib:fmt_route(Route). 298 | 299 | %%%_* Emacs ==================================================================== 300 | %%% Local Variables: 301 | %%% allout-layout: t 302 | %%% erlang-indent-level: 2 303 | %%% End: 304 | -------------------------------------------------------------------------------- /src/brucke_metrics.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2017 Klarna AB 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | -module(brucke_metrics). 17 | 18 | -export([ init/0 19 | , inc/2 20 | , set/2 21 | , format_topic/1 22 | ]). 23 | 24 | -include("brucke_int.hrl"). 25 | 26 | -compile({no_auto_import,[set/1]}). 27 | 28 | -define(WRITER, ?MODULE). %% the registered name of graphiter_writer 29 | 30 | %% @doc Initialize metrics writer. 31 | -spec init() -> ok | {error, any()}. 32 | init() -> 33 | Prefix0 = atom_to_list(?APPLICATION), 34 | Prefix = case brucke_app:graphite_root_path() of 35 | undefined -> Prefix0; 36 | Root -> Root 37 | end, 38 | case brucke_app:graphite_host() of 39 | undefined -> 40 | %% not configured, do not start anything 41 | ok; 42 | Host -> 43 | Opts0 = [{prefix, iolist_to_binary(Prefix)}, {host, Host}], 44 | Opts = case brucke_app:graphite_port() of 45 | undefined -> Opts0; 46 | Port -> [{port, Port} | Opts0] 47 | end, 48 | case graphiter:start(?WRITER, Opts) of 49 | {ok, _Pid} -> 50 | ok; 51 | {error, {already_started, _Pid}} -> 52 | ok; 53 | Other -> 54 | {error, Other} 55 | end 56 | end. 57 | 58 | %% @doc Increment counter. 59 | -spec inc(graphiter:path(), integer()) -> ok. 60 | inc(_Path, 0) -> ok; 61 | inc(Path, Inc) when is_integer(Inc) -> 62 | graphiter:incr_cast(?WRITER, Path, Inc). 63 | 64 | %% @doc Set gauge value. 65 | -spec set(graphiter:path(), number()) -> ok. 66 | set(Path, Val) when is_number(Val) -> 67 | graphiter:cast(?WRITER, Path, Val). 68 | 69 | %% @doc Replace the dots in topic names with hyphens. 70 | -spec format_topic(brod:topic()) -> binary(). 71 | format_topic(Topic) -> 72 | binary:replace(Topic, <<".">>, <<"-">>, [global]). 73 | 74 | %%%_* Emacs ==================================================================== 75 | %%% Local Variables: 76 | %%% allout-layout: t 77 | %%% erlang-indent-level: 2 78 | %%% End: 79 | -------------------------------------------------------------------------------- /src/brucke_ratelimiter.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | -module(brucke_ratelimiter). 17 | 18 | -behaviour(gen_server). 19 | 20 | %% API 21 | -export([start_link/2]). 22 | 23 | %% gen_server callbacks 24 | -export([ init/1 25 | , handle_call/3 26 | , handle_cast/2 27 | , handle_info/2 28 | , terminate/2 29 | , code_change/3 30 | , format_status/2]). 31 | 32 | %% called by brucke_sup 33 | -export([init/0]). 34 | 35 | -export([ ensure_started/2 36 | , name_it/1 37 | , set_rate/2 38 | , get_rate/1 39 | , acquire/1]). 40 | 41 | -include("brucke_int.hrl"). 42 | 43 | -define(TAB, ?RATELIMITER_TAB). 44 | 45 | -record(state, { id 46 | , ticker 47 | , interval 48 | , threshold 49 | , blocked_pids = [] 50 | }). 51 | 52 | -type rate() :: {Interval :: non_neg_integer(), Threshold :: non_neg_integer()}. 53 | -type rid() :: atom() | {Cluster :: list() | binary() | atom(), 54 | CgId :: list() | binary() | atom() }. 55 | %%%=================================================================== 56 | %%% API 57 | %%%=================================================================== 58 | 59 | %%-------------------------------------------------------------------- 60 | %% @doc 61 | %% Starts the server 62 | %% @end 63 | %%-------------------------------------------------------------------- 64 | -spec start_link(Name::atom(), Rate :: any()) 65 | -> {ok, Pid :: pid()} | 66 | {error, Error :: {already_started, pid()}} | 67 | {error, Error :: term()} | 68 | ignore. 69 | start_link(Name, Rate) -> 70 | gen_server:start_link({local, Name}, ?MODULE, [Name, Rate], []). 71 | 72 | -spec ensure_started(Name :: atom(), Rate :: rate()) -> ok. 73 | ensure_started(Name, {Interval, Threshold}) -> 74 | case brucke_ratelimiter:start_link(Name, {Interval, Threshold}) of 75 | {ok, _RPid} -> 76 | ok; 77 | {error, {already_started, _RPid}} -> 78 | ok 79 | end. 80 | 81 | -spec name_it([string() | binary() | atom()]) -> atom(). 82 | name_it(Names)-> 83 | NamesStr = lists:map(fun to_list/1, Names), 84 | list_to_atom(string:join(["rtl" | NamesStr ], "_")). 85 | 86 | -spec set_rate(Rid :: rid(), RateSettings :: proplist:proplist()) -> ok | noargs. 87 | set_rate({Cluster, Cgid}, RateSettings) -> 88 | set_rate(name_it([Cluster, Cgid]), RateSettings); 89 | set_rate(Rid, RateSettings) when is_list(RateSettings)-> 90 | gen_server:call(Rid, {set_rate, RateSettings}). 91 | 92 | -spec get_rate(rid()) -> rate(). 93 | get_rate({Cluster, Cgid}) -> 94 | get_rate(name_it([Cluster, Cgid])); 95 | get_rate(Rid) when is_atom(Rid)-> 96 | gen_server:call(Rid, get_rate). 97 | 98 | %% acquire blocks if necessary until new window. 99 | -spec acquire(RateLimiter :: atom()) -> ok. 100 | acquire(Server)-> 101 | case ets:update_counter(?TAB, Server, {2, -1, 0, 0}) of 102 | 0 -> 103 | unblock = gen_server:call(Server, blocked, infinity), 104 | acquire(Server); 105 | _ -> 106 | ok 107 | end. 108 | 109 | init() -> 110 | init_http(), 111 | init_tab(). 112 | 113 | 114 | %%%=================================================================== 115 | %%% gen_server callbacks 116 | %%%=================================================================== 117 | 118 | %%-------------------------------------------------------------------- 119 | %% @private 120 | %% @doc 121 | %% Initializes the server 122 | %% @end 123 | %%-------------------------------------------------------------------- 124 | -spec init(Args :: list()) 125 | -> {ok, State :: term()} | 126 | {ok, State :: term(), Timeout :: timeout()} | 127 | {ok, State :: term(), hibernate} | 128 | {stop, Reason :: term()} | 129 | ignore. 130 | init([Id, {Interval, Threshold}]) -> 131 | process_flag(trap_exit, true), 132 | %% init counter 133 | reset_counter(Id, Threshold), 134 | {ok, Tref} = timer:send_interval(Interval, self(), reset), 135 | {ok, #state{id = Id, ticker = Tref, interval = Interval, threshold = Threshold}}. 136 | 137 | %%-------------------------------------------------------------------- 138 | %% @private 139 | %% @doc 140 | %% Handling call messages 141 | %% @end 142 | %%-------------------------------------------------------------------- 143 | -spec handle_call(Request :: term(), From :: {pid(), term()}, State :: term()) -> 144 | {reply, Reply :: term(), NewState :: term()} | 145 | {reply, Reply :: term(), NewState :: term(), Timeout :: timeout()} | 146 | {reply, Reply :: term(), NewState :: term(), hibernate} | 147 | {noreply, NewState :: term()} | 148 | {noreply, NewState :: term(), Timeout :: timeout()} | 149 | {noreply, NewState :: term(), hibernate} | 150 | {stop, Reason :: term(), Reply :: term(), NewState :: term()} | 151 | {stop, Reason :: term(), NewState :: term()}. 152 | handle_call(blocked, From, #state{blocked_pids = BlockedPids} = State) -> 153 | {noreply, State#state{blocked_pids = [From | BlockedPids]}}; 154 | 155 | 156 | handle_call(get_rate, _From, #state{interval = Interval, threshold = Threshold} = State) -> 157 | {reply, {Interval, Threshold}, State}; 158 | 159 | handle_call({set_rate, Rateprop}, _From, #state{ ticker = OldTref 160 | , id = Rid 161 | , interval = OldInterval 162 | , threshold = OldThreshold} = State) -> 163 | NewInterval = proplists:get_value(interval, Rateprop, undefined), 164 | NewThreshold = proplists:get_value(threshold, Rateprop, undefined), 165 | 166 | %%% fine control what to reset to reduce unnecessary jitter 167 | case {NewInterval, NewThreshold} of 168 | {undefined, undefined} -> 169 | {reply, noargs, State}; 170 | {undefined, OldThreshold} -> 171 | {reply, ok, State}; 172 | {undefined, _} when is_integer(NewThreshold) -> 173 | reset_counter(Rid, NewThreshold), 174 | {reply, ok, State#state{threshold = NewThreshold}}; 175 | {OldInterval, OldThreshold} -> 176 | {reply, ok, State}; 177 | {OldInterval, _} when is_integer(NewThreshold) -> 178 | reset_counter(Rid, NewThreshold), 179 | {reply, ok, State#state{threshold = NewThreshold}}; 180 | {NewInterval, _} -> 181 | %% interval is updated, reset ticker 182 | NewTref = reset_ticker(OldTref, NewInterval), 183 | reset_counter(Rid, NewThreshold), 184 | {reply, ok, State#state{ticker = NewTref, interval = NewInterval, threshold = NewThreshold}} 185 | end. 186 | 187 | %%-------------------------------------------------------------------- 188 | %% @private 189 | %% @doc 190 | %% Handling cast messages 191 | %% @end 192 | %%-------------------------------------------------------------------- 193 | -spec handle_cast(Request :: term(), State :: term()) -> 194 | {noreply, NewState :: term()} | 195 | {noreply, NewState :: term(), Timeout :: timeout()} | 196 | {noreply, NewState :: term(), hibernate} | 197 | {stop, Reason :: term(), NewState :: term()}. 198 | handle_cast(_, #state{} = State) -> 199 | {noreply, State}. 200 | 201 | %%-------------------------------------------------------------------- 202 | %% @private 203 | %% @doc 204 | %% Handling all non call/cast messages 205 | %% @end 206 | %%-------------------------------------------------------------------- 207 | -spec handle_info(Info :: timeout() | term(), State :: term()) -> 208 | {noreply, NewState :: term()} | 209 | {noreply, NewState :: term(), Timeout :: timeout()} | 210 | {noreply, NewState :: term(), hibernate} | 211 | {stop, Reason :: normal | term(), NewState :: term()}. 212 | handle_info(reset, #state{id = Rid, blocked_pids = Pids, 213 | threshold = Threshold} = State) -> 214 | %%% new tick, reset cnt and unblock all blocked process 215 | reset_counter(Rid, Threshold), 216 | [gen_server:reply(P, unblock) || P <- Pids], 217 | {noreply, State#state{blocked_pids = []}}. 218 | 219 | %%-------------------------------------------------------------------- 220 | %% @private 221 | %% @doc 222 | %% This function is called by a gen_server when it is about to 223 | %% terminate. It should be the opposite of Module:init/1 and do any 224 | %% necessary cleaning up. When it returns, the gen_server terminates 225 | %% with Reason. The return value is ignored. 226 | %% @end 227 | %%-------------------------------------------------------------------- 228 | -spec terminate(Reason :: normal | shutdown | {shutdown, term()} | term(), 229 | State :: term()) -> any(). 230 | terminate(_Reason, _State) -> 231 | ok. 232 | 233 | %%-------------------------------------------------------------------- 234 | %% @private 235 | %% @doc 236 | %% Convert process state when code is changed 237 | %% @end 238 | %%-------------------------------------------------------------------- 239 | -spec code_change(OldVsn :: term() | {down, term()}, 240 | State :: term(), 241 | Extra :: term()) -> {ok, NewState :: term()} | 242 | {error, Reason :: term()}. 243 | code_change(_OldVsn, State, _Extra) -> 244 | {ok, State}. 245 | 246 | %%-------------------------------------------------------------------- 247 | %% @private 248 | %% @doc 249 | %% This function is called for changing the form and appearance 250 | %% of gen_server status when it is returned from sys:get_status/1,2 251 | %% or when it appears in termination error logs. 252 | %% @end 253 | %%-------------------------------------------------------------------- 254 | -spec format_status(Opt :: normal | terminate, 255 | Status :: list()) -> Status :: term(). 256 | format_status(_Opt, Status) -> 257 | Status. 258 | 259 | %%%=================================================================== 260 | %%% Internal functions 261 | %%%=================================================================== 262 | init_http() -> 263 | brucke_http:register_plugin_handler("ratelimiter/:cluster/:cgid", 264 | brucke_ratelimiter_http_handler, []). 265 | init_tab()-> 266 | ets:new(?TAB, [ named_table 267 | , public 268 | , set 269 | , {read_concurrency, true} 270 | ]), 271 | ok. 272 | 273 | reset_ticker(OldTref, NewInterval) -> 274 | timer:cancel(OldTref), 275 | {ok, Tref} = timer:send_interval(NewInterval, self(), reset), 276 | Tref. 277 | 278 | reset_counter(Id, NewCnt) -> 279 | ets:insert(?TAB, {Id, NewCnt}). 280 | 281 | to_list(N) when is_list(N) -> N; 282 | to_list(N) when is_binary(N) -> binary_to_list(N); 283 | to_list(N) when is_atom(N) -> atom_to_list(N). 284 | 285 | 286 | %%%_* Emacs ==================================================================== 287 | %%% Local Variables: 288 | %%% allout-layout: t 289 | %%% erlang-indent-level: 2 290 | %%% End: 291 | -------------------------------------------------------------------------------- /src/brucke_ratelimiter_http_handler.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | -module(brucke_ratelimiter_http_handler). 17 | 18 | -export([ init/3 19 | , allowed_methods/2 20 | , content_types_accepted/2 21 | , handle_request/2 22 | , content_types_provided/2 23 | ]). 24 | 25 | -include("brucke_int.hrl"). 26 | 27 | init(_Transport, _Req, []) -> 28 | {upgrade, protocol, cowboy_rest}. 29 | 30 | allowed_methods(Req, State) -> 31 | {[<< "POST" >>, << "PUT" >>], Req, State}. 32 | 33 | content_types_accepted(Req, State) -> 34 | {[{{<<"application">>, <<"json">>, []}, handle_request}], Req, State}. 35 | 36 | content_types_provided(Req, State) -> 37 | {[{{<<"application">>, <<"json">>, []}, handle_request}], Req, State}. 38 | 39 | handle_request(Req, State) -> 40 | {Cluster, Req1} = cowboy_req:binding(cluster, Req), 41 | {Cgid, Req1} = cowboy_req:binding(cgid, Req), 42 | {ok, Body, Req2} = cowboy_req:body(Req1), 43 | Args = lists:map(fun({<< "interval" >>, V}) -> 44 | {interval, to_int(V)}; 45 | ({<< "threshold" >>, V}) -> 46 | {threshold, to_int(V)} 47 | end, jsone:decode(Body, [{object_format,proplist}])), 48 | Res = brucke_ratelimiter:set_rate({Cluster, Cgid}, Args), 49 | {true, cowboy_req:set_resp_body(jsone:encode(Res), Req2), State}. 50 | 51 | to_int(V) when is_binary(V) -> 52 | binary_to_integer(V); 53 | to_int(V) when is_list(V)-> 54 | list_to_integer(V); 55 | to_int(V) when is_integer(V)-> 56 | V. 57 | 58 | %%%_* Emacs ==================================================================== 59 | %%% Local Variables: 60 | %%% allout-layout: t 61 | %%% erlang-indent-level: 2 62 | %%% End: 63 | -------------------------------------------------------------------------------- /src/brucke_routes.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | -module(brucke_routes). 17 | 18 | -export([ all/0 19 | , init/1 20 | , health_status/0 21 | , get_cg_id/1 22 | , add_skipped_route/2 23 | , get_discarded/0 24 | ]). 25 | 26 | -include("brucke_int.hrl"). 27 | 28 | -define(T_ROUTES, brucke_routes). 29 | -define(T_DISCARDED_ROUTES, brucke_discarded_routes). 30 | 31 | -define(IS_PRINTABLE(C), (C >= 32 andalso C < 127)). 32 | 33 | -define(IS_VALID_TOPIC_NAME(N), 34 | (is_atom(N) orelse 35 | is_binary(N) orelse 36 | is_list(N) andalso N =/= [] andalso ?IS_PRINTABLE(hd(N)))). 37 | 38 | -define(NO_CG_ID_OPTION, ?undef). 39 | 40 | -type raw_cg_id() :: ?undef | atom() | string() | binary(). 41 | -type cg_id() :: binary(). 42 | 43 | %%%_* APIs ===================================================================== 44 | 45 | -spec init([raw_route()]) -> ok | no_return(). 46 | init(Routes) when is_list(Routes) -> 47 | ets:info(?T_ROUTES) =/= ?undef andalso exit({?T_ROUTES, already_created}), 48 | ets:info(?T_DISCARDED_ROUTES) =/= ?undef andalso exit({?T_DISCARDED_ROUTES, already_created}), 49 | %% set table, allow no entry duplication 50 | ets:new(?T_ROUTES, [named_table, public, set, 51 | {keypos, #route.upstream}]), 52 | ets:new(?T_DISCARDED_ROUTES, [named_table, public, bag]), 53 | try 54 | ok = do_init_loop(Routes) 55 | catch C : E ?BIND_STACKTRACE(Stack) -> 56 | ?GET_STACKTRACE(Stack), 57 | ok = destroy(), 58 | erlang:C({E, Stack}) 59 | end; 60 | init(Other) -> 61 | logger:emergency("Expecting list of routes, got ~P", [Other, 9]), 62 | erlang:exit(bad_routes_config). 63 | 64 | %% @doc Delete Ets. 65 | -spec destroy() -> ok. 66 | destroy() -> 67 | try 68 | ets:delete(?T_ROUTES), 69 | ets:delete(?T_DISCARDED_ROUTES), 70 | ok 71 | catch error : badarg -> 72 | ok 73 | end. 74 | 75 | %% @doc Get all routes from cache. 76 | -spec all() -> [route()]. 77 | all() -> 78 | ets:tab2list(?T_ROUTES). 79 | 80 | %% @doc Routes health status, returns lists of healthy, unhealthy and discarded routes 81 | -spec health_status() -> maps:map(). 82 | health_status() -> 83 | {Healthy0, Unhealthy0} = lists:partition(fun is_healthy/1, all()), 84 | Healthy = lists:map(fun format_route/1, Healthy0), 85 | Unhealthy = lists:map(fun format_route/1, Unhealthy0), 86 | #{healthy => Healthy, unhealthy => Unhealthy, 87 | discarded => get_discarded()}. 88 | 89 | %% @doc Get all discarded routes. 90 | get_discarded() -> 91 | lists:map(fun format_skipped_route/1, 92 | ets:tab2list(?T_DISCARDED_ROUTES)). 93 | 94 | %% @doc Get upstream consumer group ID from route options. 95 | %% If no `upstream_cg_id' configured, build it from cluster name. 96 | %% @end 97 | -spec get_cg_id(route_options()) -> cg_id(). 98 | get_cg_id(Options) -> 99 | maps:get(upstream_cg_id, Options). 100 | 101 | %% @doc Add skipped routes found during or after initialization phase. 102 | %% e.g. Bad topic name found during init validation, 103 | %% or when the route worker finds out the upstream or downstream topic does not exist. 104 | %% @end 105 | -spec add_skipped_route(route() | raw_route(), iolist()) -> ok. 106 | add_skipped_route(Route, Reason) -> 107 | case Route of 108 | #route{} -> 109 | %% delete from healthy routes table 110 | _ = ets:delete(?T_ROUTES, Route#route.upstream); 111 | _ -> 112 | ok 113 | end, 114 | %% insert to discarded table 115 | ets:insert(?T_DISCARDED_ROUTES, {Route, iolist_to_binary(Reason)}), 116 | brucke_lib:log_skipped_route_alert(Route, Reason), 117 | ok. 118 | 119 | %%%_* Internal functions ======================================================= 120 | 121 | %% @private 122 | format_route(#route{} = R) -> 123 | {UpClient, UpTopics} = R#route.upstream, 124 | {DnClient, DnTopic} = R#route.downstream, 125 | #{upstream => #{endpoints => endpoints_to_maps(brucke_config:get_client_endpoints(UpClient)), 126 | topics => UpTopics}, 127 | downstream => #{endpoints => endpoints_to_maps(brucke_config:get_client_endpoints(DnClient)), 128 | topic => DnTopic}, 129 | options => R#route.options}. 130 | 131 | %% @private 132 | format_skipped_route({R, Reason}) when is_list(R) -> 133 | UpClient = proplists:get_value(upstream_client, R), 134 | UpTopics = lists:map(fun topic/1, proplists:get_value(upstream_topics, R, [])), 135 | DnClient = proplists:get_value(downstream_client, R), 136 | DnTopic = topic(proplists:get_value(downstream_topic, R, "")), 137 | ExceptOptsKeys = [upstream_client, upstream_topics, downstream_client, downstream_topic], 138 | #{upstream => #{endpoints => endpoints_to_maps(brucke_config:get_client_endpoints(UpClient)), 139 | topics => UpTopics}, 140 | downstream => #{endpoints => endpoints_to_maps(brucke_config:get_client_endpoints(DnClient)), 141 | topic => DnTopic}, 142 | reason => Reason, 143 | options => format_raw_options(R, ExceptOptsKeys) 144 | }; 145 | format_skipped_route({#route{} = Route, Reason}) -> 146 | Map = format_route(Route), 147 | Map#{reason => Reason}; 148 | format_skipped_route({X, Reason}) -> 149 | fmt("invalid route specification ~p\nreason:~s", [X, Reason]). 150 | 151 | %% @private 152 | format_raw_options([], _ExceptOptsKeys) -> []; 153 | format_raw_options([{K, V} | Rest], ExceptOptsKeys) -> 154 | case lists:member(K, ExceptOptsKeys) of 155 | true -> format_raw_options(Rest, ExceptOptsKeys); 156 | false -> [{K, bin_str(V)} | format_raw_options(Rest, ExceptOptsKeys)] 157 | end. 158 | 159 | %% @private 160 | bin_str(L) when is_list(L) -> erlang:iolist_to_binary(L); 161 | bin_str(X) -> X. 162 | 163 | %% @private 164 | endpoints_to_maps(Endpoints) -> 165 | lists:map(fun({Host, Port}) -> #{host => list_to_binary(Host), port => Port} end, Endpoints). 166 | 167 | %% @private 168 | -spec is_healthy(route()) -> boolean(). 169 | is_healthy(#route{upstream = U}) -> 170 | Members = brucke_sup:get_group_member_children(U), 171 | lists:all( 172 | fun({_Id, Pid}) -> 173 | is_pid(Pid) andalso brucke_member:is_healthy(Pid) 174 | end, Members). 175 | 176 | -spec do_init_loop([raw_route()]) -> ok. 177 | do_init_loop([]) -> ok; 178 | do_init_loop([RawRoute | Rest]) -> 179 | try 180 | case validate_route(RawRoute) of 181 | {ok, Routes} -> 182 | ets:insert(?T_ROUTES, Routes); 183 | {error, Reasons} -> 184 | Rs = [[Reason, "\n"] || Reason <- Reasons], 185 | add_skipped_route(RawRoute, Rs) 186 | end 187 | catch throw : Reason -> 188 | ReasonTxt = io_lib:format("~p", [Reason]), 189 | add_skipped_route(RawRoute, ReasonTxt) 190 | end, 191 | do_init_loop(Rest). 192 | 193 | -spec validate_route(raw_route()) -> {ok, [route()]} | {error, [binary()]}. 194 | validate_route(RawRoute0) -> 195 | %% use maps to ensure 196 | %% 1. key-value list 197 | %% 2. later value should overwrite earlier in case of key duplication 198 | RawRouteMap = 199 | try 200 | maps:from_list(RawRoute0) 201 | catch error : badarg -> 202 | throw(bad_route) 203 | end, 204 | RawRoute = maps:to_list(RawRouteMap), 205 | case apply_route_schema(RawRoute, schema(), defaults(), #{}, []) of 206 | {ok, #{ upstream_client := UpstreamClient 207 | , downstream_client := DownstreamClient 208 | , upstream_topics := UpstreamTopics 209 | , downstream_topic := DownstreamTopic 210 | } = RouteAsMap} -> 211 | case UpstreamClient =:= DownstreamClient of 212 | true -> ok = ensure_no_loopback(UpstreamTopics, DownstreamTopic); 213 | false -> ok 214 | end, 215 | {ok, convert_to_route_record(RouteAsMap)}; 216 | {error, Reasons} -> 217 | {error, Reasons} 218 | end. 219 | 220 | convert_to_route_record(Route) -> 221 | #{ upstream_client := UpstreamClientId 222 | , upstream_topics := UpstreamTopics 223 | , downstream_client := DownstreamClientId 224 | , downstream_topic := DownstreamTopic 225 | , repartitioning_strategy := RepartitioningStrategy 226 | , max_partitions_per_group_member := MaxPartitionsPerGroupMember 227 | , filter_module := FilterModule 228 | , filter_init_arg := FilterInitArg 229 | , default_begin_offset := BeginOffset 230 | , compression := Compression 231 | , required_acks := RequiredAcks 232 | , upstream_cg_id := RawCgId 233 | , offset_commit_policy := OffsetCommitPolicy 234 | , ratelimit_interval := RatelimitInteval 235 | , ratelimit_threshold := RatelimitThreshold 236 | } = Route, 237 | ProducerConfig = [{compression, Compression}, 238 | {required_acks, required_acks(RequiredAcks)}], 239 | ConsumerConfig = [{begin_offset, BeginOffset}], 240 | Options = 241 | #{ repartitioning_strategy => RepartitioningStrategy 242 | , max_partitions_per_group_member => MaxPartitionsPerGroupMember 243 | , filter_module => FilterModule 244 | , filter_init_arg => FilterInitArg 245 | , producer_config => ProducerConfig 246 | , consumer_config => ConsumerConfig 247 | , upstream_cg_id => mk_cg_id(UpstreamClientId, RawCgId) 248 | , offset_commit_policy => OffsetCommitPolicy 249 | , ratelimit_interval => RatelimitInteval 250 | , ratelimit_threshold => RatelimitThreshold 251 | }, 252 | %% flatten out the upstream topics 253 | %% to simplify the config as if it's all 254 | %% one upstream topic to one downstream topic mapping 255 | MapF = 256 | fun(Topic) -> 257 | #route{ upstream = {UpstreamClientId, topic(Topic)} 258 | , downstream = {DownstreamClientId, topic(DownstreamTopic)} 259 | , options = Options} 260 | end, 261 | lists:map(MapF, topics(UpstreamTopics)). 262 | 263 | %% @private 264 | required_acks(all) -> -1; 265 | required_acks(leader) -> 1; 266 | required_acks(none) -> 0; 267 | required_acks(Number) -> Number. 268 | 269 | %% @private 270 | defaults() -> 271 | #{ repartitioning_strategy => ?DEFAULT_REPARTITIONING_STRATEGY 272 | , max_partitions_per_group_member => ?MAX_PARTITIONS_PER_GROUP_MEMBER 273 | , default_begin_offset => ?DEFAULT_DEFAULT_BEGIN_OFFSET 274 | , compression => ?DEFAULT_COMPRESSION 275 | , required_acks => ?DEFAULT_REQUIRED_ACKS 276 | , filter_module => ?DEFAULT_FILTER_MODULE 277 | , filter_init_arg => ?DEFAULT_FILTER_INIT_ARG 278 | , upstream_cg_id => ?NO_CG_ID_OPTION 279 | , offset_commit_policy => ?DEFAULT_OFFSET_COMMIT_POLICY 280 | , ratelimit_interval => ?DEFAULT_RATELIMIT_INTEVAL 281 | , ratelimit_threshold => ?DEFAULT_RATELIMIT_THRESHOLD 282 | }. 283 | 284 | schema() -> 285 | #{ upstream_client => 286 | fun(_, Id) -> 287 | is_configured_client_id(Id) orelse 288 | <<"unknown upstream client id">> 289 | end 290 | , downstream_client => 291 | fun(_, Id) -> 292 | is_configured_client_id(Id) orelse 293 | <<"unknown downstream client id">> 294 | end 295 | , downstream_topic => 296 | fun(_, Topic) -> 297 | ?IS_VALID_TOPIC_NAME(Topic) orelse 298 | invalid_topic_name(downstream, Topic) 299 | end 300 | , upstream_topics => 301 | fun(#{upstream_client := UpstreamClientId} = RawRoute, Topic) -> 302 | CgId = maps:get(upstream_cg_id, RawRoute, ?NO_CG_ID_OPTION), 303 | validate_upstream_topics(UpstreamClientId, CgId, Topic) 304 | end 305 | , repartitioning_strategy => 306 | fun(_, S) -> 307 | ?IS_VALID_REPARTITIONING_STRATEGY(S) orelse 308 | fmt("unknown repartitioning strategy ~p", [S]) 309 | end 310 | , max_partitions_per_group_member => 311 | fun(_, M) -> 312 | (is_integer(M) andalso M > 0) orelse 313 | fmt("max_partitions_per_group_member " 314 | "should be a positive integer\nGto~p", [M]) 315 | end 316 | , default_begin_offset => 317 | fun(_, B) -> 318 | (B =:= latest orelse 319 | B =:= earliest orelse 320 | is_integer(B)) orelse 321 | fmt("default_begin_offset should be either " 322 | "'latest', 'earliest' or an integer\nGot~p", [B]) 323 | end 324 | , compression => 325 | fun(_, C) -> 326 | C =:= no_compression orelse 327 | C =:= gzip orelse 328 | C =:= snappy orelse 329 | fmt("compression should be one of " 330 | "[no_compression, gzip, snappy]\nGot~p", [C]) 331 | end 332 | , required_acks => 333 | fun(_, A) -> 334 | A =:= all orelse 335 | A =:= leader orelse 336 | A =:= none orelse 337 | A =:= -1 orelse 338 | A =:= 1 orelse 339 | A =:= 0 orelse 340 | fmt("required_acks should be one of " 341 | "[all, leader, none, -1, 1 0]\nGot~p", [A]) 342 | end 343 | , filter_module => 344 | fun(_, Module) -> 345 | case code:ensure_loaded(Module) of 346 | {module, Module} -> 347 | true; 348 | {error, What} -> 349 | fmt("filter module ~p is not found\nreason:~p\n", 350 | [Module, What]) 351 | end 352 | end 353 | , filter_init_arg => fun(_, _Arg) -> true end 354 | , upstream_cg_id => fun(_, _Name) -> true end 355 | , offset_commit_policy => fun(_ , A) -> 356 | A =:= commit_to_kafka_v2 orelse 357 | (A =:= consumer_managed andalso undefined =/= dets:info(?OFFSETS_TAB)) orelse 358 | fmt("offset_commit_policy is set to ~p, it should be 'commit_to_kafka_v2'" 359 | "or 'consumer_managed'. ", [A]) 360 | end 361 | , ratelimit_interval => fun(_, ?RATELIMIT_DISABLED) -> true; 362 | (_, A) when is_integer(A) andalso A > 0 -> true; 363 | (_,_) -> fmt("ratelimit_interval should be >= 0", []) 364 | end 365 | , ratelimit_threshold => fun(_, A) when is_integer(A) andalso A >= 0 -> true; 366 | (_, _) -> fmt("ratelimit_threshold should be >= 0", []) 367 | end 368 | }. 369 | 370 | -spec apply_route_schema(raw_route(), map(), map(), map(), [binary()]) -> 371 | {ok, map()} | {error, [binary()]}. 372 | apply_route_schema([], Schema, Defaults, Result, Errors0) -> 373 | Errors1 = 374 | case maps:to_list(maps:without(maps:keys(Defaults), Schema)) of 375 | [] -> 376 | Errors0; 377 | Missing -> 378 | MissingAttrs = [K || {K, _V} <- Missing], 379 | [fmt("missing mandatory attributes ~p", [MissingAttrs]) | Errors0] 380 | end, 381 | Errors = [E || E <- lists:flatten(Errors1), E =/= true], 382 | case [] =:= Errors of 383 | true -> 384 | %% merge (overwrite) parsed values to defaults 385 | {ok, maps:merge(Defaults, Result)}; 386 | false -> 387 | {error, Errors} 388 | end; 389 | apply_route_schema([{K, V} | Rest], Schema, Defaults, Result, Errors) -> 390 | case maps:find(K, Schema) of 391 | {ok, Fun} -> 392 | NewSchema = maps:remove(K, Schema), 393 | NewResult = Result#{K => V}, 394 | try Fun(NewResult, V) of 395 | true -> 396 | apply_route_schema(Rest, NewSchema, Defaults, NewResult, Errors); 397 | Error -> 398 | NewErrors = [Error | Errors], 399 | apply_route_schema(Rest, NewSchema, Defaults, NewResult, NewErrors) 400 | catch 401 | C : E : Stack -> 402 | NewErrors = [fmt("crashed valiating ~p: ~p:~p\n~p", 403 | [K, C, E, Stack]) | Errors], 404 | apply_route_schema(Rest, NewSchema, Defaults, NewResult, NewErrors) 405 | end; 406 | error -> 407 | Error = 408 | case is_atom(K) of 409 | true -> fmt("unknown attribute ~p", [K]); 410 | false -> fmt("unknown attribute ~p, expecting atom", [K]) 411 | end, 412 | apply_route_schema(Rest, Schema, Defaults, Result, [Error | Errors]) 413 | end. 414 | 415 | %% This is to ensure there is no direct loopback due to typo for example. 416 | %% indirect loopback would be fun for testing, so not trying to build a graph 417 | %% {upstream, topic_1} -> {downstream, topic_2} 418 | %% {downstream, topic_2} -> {upstream, topic_1} 419 | %% you get a perfect data generator for load testing. 420 | -spec ensure_no_loopback(topic_name() | [topic_name()], topic_name()) -> ok. 421 | ensure_no_loopback(UpstreamTopics, DownstreamTopic) -> 422 | case lists:member(topic(DownstreamTopic), topics(UpstreamTopics)) of 423 | true -> throw(direct_loopback); 424 | false -> ok 425 | end. 426 | 427 | -spec is_configured_client_id(brod:client_id()) -> boolean(). 428 | is_configured_client_id(ClientId) -> 429 | brucke_config:is_configured_client_id(ClientId). 430 | 431 | -spec validate_upstream_topics(brod:client_id(), raw_cg_id(), 432 | topic_name() | [topic_name()]) -> binary(). 433 | validate_upstream_topics(_ClientId, _CgId, []) -> 434 | invalid_topic_name(upstream, []); 435 | validate_upstream_topics(ClientId, CgId, Topic) when ?IS_VALID_TOPIC_NAME(Topic) -> 436 | validate_upstream_topic(ClientId, CgId, Topic); 437 | validate_upstream_topics(ClientId, CgId, Topics0) when is_list(Topics0) -> 438 | case lists:partition(fun(T) -> ?IS_VALID_TOPIC_NAME(T) end, Topics0) of 439 | {Topics, []} -> 440 | [validate_upstream_topic(ClientId, CgId, T) || T <- Topics]; 441 | {_, InvalidTopics} -> 442 | invalid_topic_name(upstream, InvalidTopics) 443 | end. 444 | 445 | -spec validate_upstream_topic(brod:client_id(), raw_cg_id(), topic_name()) -> 446 | true | binary(). 447 | validate_upstream_topic(ClientId, RawCgId, Topic) -> 448 | ClusterName = brucke_config:get_cluster_name(ClientId), 449 | CgId = mk_cg_id(ClientId, RawCgId), 450 | case is_cg_duplication(ClusterName, CgId, Topic) of 451 | false -> 452 | validate_upstream_client(ClientId, Topic); 453 | true -> 454 | fmt("Duplicated routes for upstream topic ~s " 455 | "in the same consumer group ~s in cluster ~s.", 456 | [Topic, CgId, ClusterName]) 457 | end. 458 | 459 | %% Duplicated upstream topic is not allowed for the same client. 460 | %% Because one `brod_consumer' allows only one subscriber. 461 | -spec validate_upstream_client(brod:client_id(), topic_name()) -> 462 | true | binary(). 463 | validate_upstream_client(ClientId, Topic) -> 464 | case ets:lookup(?T_ROUTES, {ClientId, topic(Topic)}) of 465 | [] -> 466 | true; 467 | _ -> 468 | fmt("Upstream topic ~p is used more than once for client ~p", 469 | [Topic, ClientId]) 470 | end. 471 | 472 | %% Make upstream consumer group ID. 473 | %% If upstream_cg_id is not found in route option, 474 | %% build the ID from upstream cluster name (for backward compatibility). 475 | -spec mk_cg_id(brod:client_id(), raw_cg_id()) -> cg_id(). 476 | mk_cg_id(ClientId, ?NO_CG_ID_OPTION) -> 477 | ClusterName = brucke_config:get_cluster_name(ClientId), 478 | erlang:iolist_to_binary([ClusterName, "-brucke-cg"]); 479 | mk_cg_id(_ClientId, A) when is_atom(A) -> 480 | erlang:atom_to_binary(A, utf8); 481 | mk_cg_id(_ClientId, Str) -> 482 | erlang:iolist_to_binary(Str). 483 | 484 | %% @private Scan all routes (the ones already added to ETS) for duplicated 485 | %% route. Two routes are considered duplication when they have the same upstream 486 | %% cluster name + consumer group ID + topic name 487 | %% @end 488 | -spec is_cg_duplication(cluster_name(), cg_id(), topic_name()) -> boolean(). 489 | is_cg_duplication(ClusterName, CgId, Topic) -> 490 | lists:any( 491 | fun(#route{upstream = {ClientId, Topic_}, options = Options}) -> 492 | ClusterName_ = brucke_config:get_cluster_name(ClientId), 493 | CgId_ = get_cg_id(Options), 494 | {ClusterName_, CgId_, Topic_} =:= {ClusterName, CgId, topic(Topic)} 495 | end, all()). 496 | 497 | %% @private Find the given top 498 | 499 | -spec invalid_topic_name(upstream | downstream, any()) -> binary(). 500 | invalid_topic_name(UpOrDown_stream, NameOrList) -> 501 | fmt("expecting ~p topic(s) to be (a list of) atom() | string() | binary()\n" 502 | "got: ~p", [UpOrDown_stream, NameOrList]). 503 | 504 | %% @private Accept atom(), string(), or binary() as topic name, 505 | %% unified to binary(). 506 | %% @end 507 | -spec topic(topic_name()) -> brod:topic(). 508 | topic(Topic) when is_atom(Topic) -> topic(atom_to_list(Topic)); 509 | topic(Topic) when is_binary(Topic) -> Topic; 510 | topic(Topic) when is_list(Topic) -> list_to_binary(Topic). 511 | 512 | -spec topics(topic_name() | [topic_name()]) -> [brod:topic()]. 513 | topics(TopicName) when ?IS_VALID_TOPIC_NAME(TopicName) -> 514 | [topic(TopicName)]; 515 | topics(TopicNames) when is_list(TopicNames) -> 516 | [topic(T) || T <- TopicNames]. 517 | 518 | -spec fmt(string(), [term()]) -> binary(). 519 | fmt(Fmt, Args) -> iolist_to_binary(io_lib:format(Fmt, Args)). 520 | 521 | %%%_* Tests ==================================================================== 522 | 523 | -ifdef(TEST). 524 | 525 | -include_lib("eunit/include/eunit.hrl"). 526 | 527 | no_ets_leak_test() -> 528 | clean_setup(), 529 | ?assertNot(lists:member(?T_ROUTES, ets:all())), 530 | ?assertNot(lists:member(?T_DISCARDED_ROUTES, ets:all())), 531 | try 532 | init([a|b]) 533 | catch _ : _ -> 534 | ?assertNot(lists:member(?T_ROUTES, ets:all())), 535 | ?assertNot(lists:member(?T_DISCARDED_ROUTES, ets:all())) 536 | end. 537 | 538 | client_not_configured_test() -> 539 | clean_setup(false), 540 | R0 = 541 | [ {upstream_client, client_2} 542 | , {upstream_topics, topic_1} 543 | , {downstream_client, client_3} 544 | , {downstream_topic, topic_2} 545 | ], 546 | ok = init([R0]), 547 | ?assertEqual([], all()), 548 | ok = destroy(). 549 | 550 | bad_topic_name_test() -> 551 | clean_setup(), 552 | Base = 553 | [ {upstream_client, client_1} 554 | , {downstream_client, client_1} 555 | ], 556 | Route1 = [{upstream_topics, [topic_1]}, {downstream_topic, []} | Base], 557 | Route2 = [{upstream_topics, topic_1}, {downstream_topic, ["topic_x"]} | Base], 558 | Route3 = [{upstream_topics, []}, {downstream_topic, []} | Base], 559 | Route4 = [{upstream_topics, [[]]}, {downstream_topic, []} | Base], 560 | ok = init([Route1, Route2, Route3, Route4]), 561 | ?assertEqual([], all()), 562 | ok = destroy(). 563 | 564 | bad_routing_options_test() -> 565 | clean_setup(), 566 | R0 = 567 | [ {upstream_client, client_1} 568 | , {upstream_topics, topic_1} 569 | , {downstream_client, client_1} 570 | , {downstream_topic, topic_2} 571 | ], 572 | Routes = [ [{default_begin_offset, x} | R0] 573 | , [{repartitioning_strategy, x} | R0] 574 | , [{max_partitions_per_group_member, x} | R0] 575 | , [{"unknown_string", x} | R0] 576 | , [{unknown, x} | R0] 577 | , [{compression, x} | R0] 578 | , [{required_acks, 2} | R0] 579 | , [{required_acks, x} | R0] 580 | , [{filter_module, x} | R0] 581 | , [{offset_commit_policy, x} | R0] 582 | , [{ratelimit_threshold, x} | R0] 583 | , [{ratelimit_threshold, -1} | R0] 584 | , [{ratelimit_interval, -1} | R0] 585 | ], 586 | ok = init(Routes), 587 | ?assertEqual([], all()), 588 | ok = destroy(). 589 | 590 | mandatory_attribute_missing_test() -> 591 | clean_setup(), 592 | R = [ {upstream_client, client_1} 593 | , {upstream_topics, topic_1} 594 | , {downstream_client, client_1} 595 | ], 596 | ok = init([R]), 597 | ?assertEqual([], all()), 598 | ok = destroy(). 599 | 600 | duplicated_source_test() -> 601 | clean_setup(), 602 | ValidRoute1 = [ {upstream_client, client_1} 603 | , {upstream_topics, [<<"topic_1">>, "topic_2"]} 604 | , {downstream_client, client_1} 605 | , {downstream_topic, <<"topic_3">>} 606 | ], 607 | ValidRoute2 = [ {upstream_client, client_1} 608 | , {upstream_topics, <<"topic_4">>} 609 | , {downstream_client, client_1} 610 | , {downstream_topic, <<"topic_3">>} 611 | ], 612 | DupeRoute1 = [ {upstream_client, client_1} 613 | , {upstream_topics, "topic_1"} 614 | , {downstream_client, client_1} 615 | , {downstream_topic, topic_4} 616 | , {upstream_cg_id, "different-than-ValidRoute1"} 617 | ], 618 | ValidRoute3 = [ {upstream_client, client_2} 619 | , {upstream_topics, <<"topic_1">>} 620 | , {downstream_client, client_2} 621 | , {downstream_topic, <<"topic_5">>} 622 | , {upstream_cg_id, <<"the-id">>} 623 | ], 624 | ValidRoute4 = [ {upstream_client, client_3} 625 | , {upstream_topics, <<"topic_1">>} 626 | , {downstream_client, client_2} 627 | , {downstream_topic, <<"topic_6">>} 628 | , {upstream_cg_id, <<"the-id-2">>} 629 | ], 630 | DupeRoute2 = ValidRoute4, 631 | 632 | ok = init([ValidRoute1, ValidRoute2, DupeRoute1, 633 | ValidRoute3, ValidRoute4, DupeRoute2]), 634 | ?assertMatch([ #route{upstream = {client_1, <<"topic_1">>}, 635 | downstream = {client_1, <<"topic_3">>}} 636 | , #route{upstream = {client_1, <<"topic_2">>}} 637 | , #route{upstream = {client_1, <<"topic_4">>}} 638 | , #route{upstream = {client_2, <<"topic_1">>}, 639 | downstream = {client_2, <<"topic_5">>}} 640 | , #route{upstream = {client_3, <<"topic_1">>}, 641 | downstream = {client_2, <<"topic_6">>}} 642 | ], all_sorted()), 643 | ?assertEqual([], ets:lookup(?T_ROUTES, {client_1, <<"unknown_topic">>})), 644 | ok = destroy(). 645 | 646 | direct_loopback_test() -> 647 | clean_setup(), 648 | Routes = [ [ {upstream_client, client_1} 649 | , {upstream_topics, topic_1} 650 | , {downstream_client, client_1} 651 | , {downstream_topic, topic_1} 652 | ] 653 | , [ {upstream_client, client_1} 654 | , {upstream_topics, topic_1} 655 | , {downstream_client, client_2} 656 | , {downstream_topic, topic_2} 657 | ] 658 | ], 659 | ok = init(Routes), 660 | ?assertMatch([#route{ upstream = {client_1, <<"topic_1">>} 661 | , downstream = {client_2, <<"topic_2">>} 662 | , options = #{}}], 663 | all()), 664 | ok = destroy(). 665 | 666 | bad_config_test() -> 667 | ?assertException(exit, bad_routes_config, init(<<"not a list">>)). 668 | 669 | clean_setup() -> clean_setup(true). 670 | 671 | clean_setup(IsConfiguredClientId) -> 672 | ok = destroy(), 673 | try 674 | meck:unload(brucke_config) 675 | catch _:_ -> 676 | ok 677 | end, 678 | meck:new(brucke_config, [no_passthrough_cover]), 679 | case IsConfiguredClientId of 680 | true -> 681 | meck:expect(brucke_config, is_configured_client_id, 1, true), 682 | meck:expect(brucke_config, get_cluster_name, 1, <<"group-id">>); 683 | false -> 684 | meck:expect(brucke_config, is_configured_client_id, 1, false), 685 | meck:expect(brucke_config, get_cluster_name, 1, 686 | fun(Id) -> erlang:error({bad_client_id, Id}) end) 687 | end, 688 | ok. 689 | 690 | all_sorted() -> lists:keysort(#route.upstream, all()). 691 | 692 | topics_test() -> 693 | ?assertEqual([<<"topic_1">>], topics(topic_1)). 694 | 695 | -endif. 696 | 697 | %%%_* Emacs ==================================================================== 698 | %%% Local Variables: 699 | %%% allout-layout: t 700 | %%% erlang-indent-level: 2 701 | %%% End: 702 | -------------------------------------------------------------------------------- /src/brucke_subscriber.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module(brucke_subscriber). 18 | 19 | -export([ start_link/3 20 | , loop/1 21 | , stop/1 22 | ]). 23 | 24 | -include("brucke_int.hrl"). 25 | 26 | -type partition() :: brod:partition(). 27 | -type offset() :: brod:offset(). 28 | -type state() :: map(). 29 | -type pending_acks() :: brucke_backlog:backlog(). 30 | -type cb_state() :: brucke_filter:cb_state(). 31 | 32 | -define(SUBSCRIBE_RETRY_LIMIT, 3). 33 | -define(SUBSCRIBE_RETRY_SECONDS, 2). 34 | 35 | %% Because the upstream messages might be dispatched to 36 | %% different downstream partitions, there is no single downstream 37 | %% producer process to monitor. 38 | %% Instead, we periodically send a loop-back message to check if 39 | %% the producer pids are still alive 40 | -define(CHECK_PRODUCER_DELAY, timer:seconds(30)). 41 | -define(CHECK_PRODUCER_MSG, check_producer). 42 | 43 | -define(kafka_ack(UpstreamOffset, Ref), {kafka_ack, UpstreamOffset, Ref}). 44 | 45 | %%%_* APIs ===================================================================== 46 | 47 | -spec start_link(route(), partition(), ?undef | offset()) -> {ok, pid()}. 48 | start_link(Route, UpstreamPartition, BeginOffset) -> 49 | Parent = self(), 50 | #route{ upstream = {UpstreamClientId, UpstreamTopic} 51 | , options = Options 52 | } = Route, 53 | #{ filter_module := FilterModule 54 | , filter_init_arg := InitArg 55 | , upstream_cg_id := CgId 56 | , ratelimit_interval := RatelimitInteval 57 | , ratelimit_threshold := RatelimitThreshold 58 | } = Options, 59 | 60 | UpstreamClusterName = brucke_config:get_cluster_name(UpstreamClientId), 61 | 62 | RatelimiterName = brucke_ratelimiter:name_it([binary_to_list(UpstreamClusterName), binary_to_list(CgId)]), 63 | Ratelimiter = case RatelimitInteval == ?RATELIMIT_DISABLED of 64 | true -> 65 | unlimited; 66 | false -> 67 | brucke_ratelimiter:ensure_started(RatelimiterName, {RatelimitInteval, RatelimitThreshold}), 68 | RatelimiterName 69 | end, 70 | 71 | State = #{ route => Route 72 | , upstream_partition => UpstreamPartition 73 | , parent => Parent 74 | , consumer => subscribing 75 | , pending_acks => brucke_backlog:new() 76 | , upstream_cluster => UpstreamClusterName 77 | }, 78 | Pid = proc_lib:spawn_link( 79 | fun() -> 80 | {ok, CbState} = brucke_filter:init(FilterModule, UpstreamTopic, 81 | UpstreamPartition, InitArg), 82 | loop(State#{ filter_cb_state => CbState 83 | , ratelimiter => Ratelimiter 84 | }) 85 | end), 86 | Pid ! {subscribe, BeginOffset, 0}, 87 | _ = erlang:send_after(?CHECK_PRODUCER_DELAY, Pid, ?CHECK_PRODUCER_MSG), 88 | {ok, Pid}. 89 | 90 | stop(Pid) when is_pid(Pid) -> 91 | erlang:monitor(process, Pid), 92 | _ = exit(Pid, shutdown), 93 | receive 94 | {'DOWN', _Ref, process, Pid, _reason} -> 95 | ok 96 | end. 97 | 98 | %%%_* Internal Functions ======================================================= 99 | 100 | -spec loop(state()) -> no_return(). 101 | loop(State) -> 102 | receive 103 | Msg -> 104 | ?MODULE:loop(handle_msg(State, Msg)) 105 | end. 106 | 107 | -spec handle_msg(state(), term()) -> state() | no_return(). 108 | handle_msg(State, {subscribe, BeginOffset, Count}) -> 109 | subscribe(State, BeginOffset, Count); 110 | handle_msg(State, ?CHECK_PRODUCER_MSG) -> 111 | check_producer(State); 112 | handle_msg(State, {Pid, #kafka_message_set{} = MsgSet}) -> 113 | handle_message_set(State, Pid, MsgSet); 114 | handle_msg(State, ?kafka_ack(UpstreamOffset, Ref)) -> 115 | handle_produce_reply(State, UpstreamOffset, Ref); 116 | handle_msg(State, {'DOWN', _Ref, process, Pid, _Reason}) -> 117 | handle_consumer_down(State, Pid); 118 | handle_msg(_State, Unknown) -> 119 | erlang:exit({unknown_message, Unknown}). 120 | 121 | -spec subscribe(state(), offset(), non_neg_integer()) -> state() | no_return(). 122 | subscribe(#{ route := Route 123 | , upstream_partition := UpstreamPartition 124 | , consumer := subscribing 125 | } = State, BeginOffset, RetryCount) -> 126 | #route{upstream = {UpstreamClientId, UpstreamTopic}} = Route, 127 | SubscribeOptions = 128 | case is_integer(BeginOffset) of 129 | true -> 130 | true = (BeginOffset >= 0), %% assert 131 | [{begin_offset, BeginOffset}]; 132 | false -> 133 | [] %% use the default begin offset in consumer config 134 | end, 135 | case brod:subscribe(UpstreamClientId, self(), UpstreamTopic, 136 | UpstreamPartition, SubscribeOptions) of 137 | {ok, Pid} -> 138 | _ = erlang:monitor(process, Pid), 139 | State#{consumer := Pid}; 140 | {error, _Reason} when RetryCount < ?SUBSCRIBE_RETRY_LIMIT -> 141 | Msg = {subscribe, BeginOffset, RetryCount+1}, 142 | _ = erlang:send_after(timer:seconds(?SUBSCRIBE_RETRY_SECONDS), self(), Msg), 143 | State; 144 | {error, Reason} -> 145 | exit({failed_to_subscribe, Reason}) 146 | end; 147 | subscribe(State, _BeginOffset, _UnknownRef) -> 148 | State. 149 | 150 | -spec handle_message_set(state(), pid(), #kafka_message_set{}) -> state(). 151 | handle_message_set(#{ route := Route 152 | , upstream_cluster := Cluster 153 | , upstream_partition := Partition 154 | } = State, Pid, MsgSet) -> 155 | #route{ upstream = {_UpstreamClientId, Topic} 156 | , options = RouteOptions 157 | } = Route, 158 | #{consumer := Pid} = State, %% assert 159 | #kafka_message_set{high_wm_offset = HighWmOffset} = MsgSet, 160 | ?MX_HIGH_WM_OFFSET(Cluster, Topic, Partition, HighWmOffset), 161 | ?MX_TOTAL_VOLUME(Cluster, Topic, Partition, msg_set_bytes(MsgSet)), 162 | NewState = State#{high_wm_offset => HighWmOffset}, 163 | do_handle_message_set(NewState, MsgSet, RouteOptions). 164 | 165 | do_handle_message_set(#{ route := Route 166 | , pending_acks := PendingAcks 167 | , filter_cb_state := CbState 168 | , ratelimiter := Ratelimiter 169 | } = State, MsgSet, RouteOptions) -> 170 | RepartStrategy = brucke_lib:get_repartitioning_strategy(RouteOptions), 171 | #{ filter_module := FilterModule 172 | } = RouteOptions, 173 | #kafka_message_set{ topic = Topic 174 | , partition = Partition 175 | , messages = Messages 176 | } = MsgSet, 177 | #route{ upstream = {_UpstreamClientId, Topic} 178 | , downstream = {DownstreamClientId, DownstreamTopic} 179 | } = Route, 180 | {ok, PartCnt} = brod_client:get_partitions_count(DownstreamClientId, DownstreamTopic), 181 | FilterFun = 182 | fun(#kafka_message{ offset = Offset 183 | , key = Key 184 | , value = Value 185 | , headers = Headers 186 | } = Msg, CbStateIn) -> 187 | {FilterResult, NewCbState} = 188 | brucke_filter:filter(FilterModule, Topic, Partition, Offset, 189 | Key, Value, Headers, CbStateIn), 190 | {make_batch(Msg, FilterResult), NewCbState} 191 | end, 192 | ProduceFun = 193 | fun(Msg, Cb) -> 194 | Key = maps:get(key, Msg, <<>>), 195 | DownstreamPartition = partition(PartCnt, Partition, RepartStrategy, Key), 196 | {ok, ProducerPid} = brod:get_producer(DownstreamClientId, DownstreamTopic, DownstreamPartition), 197 | Ratelimiter =/= unlimited andalso ratelimit(Ratelimiter), 198 | case brod:produce_cb(ProducerPid, Key, Msg, Cb) of 199 | ok -> {ok, ProducerPid}; 200 | {error, Reason} -> erlang:exit(Reason) 201 | end 202 | end, 203 | {NewPendingAcks, NewCbState} = 204 | produce(FilterFun, ProduceFun, Messages, PendingAcks, CbState), 205 | handle_acked(State#{ pending_acks := NewPendingAcks 206 | , filter_cb_state := NewCbState 207 | }). 208 | 209 | %% Make a batch input for downstream producer 210 | make_batch(_Message, false) -> 211 | %% discard message 212 | []; 213 | make_batch(#kafka_message{ ts_type = TsType 214 | , ts = Ts 215 | , key = Key 216 | , value = Value 217 | , headers = Headers 218 | }, true) -> 219 | %% to downstream as-is 220 | [mk_msg(Key, Value, resolve_ts(TsType, Ts), Headers)]; 221 | make_batch(#kafka_message{ ts_type = TsType 222 | , ts = Ts 223 | }, {K, V}) -> 224 | %% old version filter return format k-v (without timestamp) 225 | [mk_msg(K, V, resolve_ts(TsType, Ts), [])]; 226 | make_batch(#kafka_message{}, {T, K, V}) -> 227 | %% old version filter return format t-k-v 228 | [mk_msg(K, V, T, [])]; 229 | make_batch(#kafka_message{ ts_type = TsType 230 | , ts = Ts 231 | }, M) when is_map(M) -> 232 | K = maps:get(key, M, <<>>), 233 | V = maps:get(value, M, <<>>), 234 | T = maps:get(ts, M, resolve_ts(TsType, Ts)), 235 | H = maps:get(headers, M, []), 236 | [mk_msg(K, V, T, H)]; 237 | make_batch(#kafka_message{}, L) when is_list(L) -> 238 | %% filter retruned a batch 239 | F = fun({K, V}) -> mk_msg(K, V, now_ts(), []); 240 | ({T, K, V}) -> mk_msg(K, V, T, []); 241 | (M) when is_map(M) -> M 242 | end, 243 | lists:map(F, L). 244 | 245 | mk_msg(K, V, T, Headers) -> 246 | #{key => K, value => V, ts => T, headers => Headers}. 247 | 248 | now_ts() -> kpro_lib:now_ts(). 249 | 250 | resolve_ts(create, Ts) when Ts > 0 -> Ts; 251 | resolve_ts(_, _) -> now_ts(). 252 | 253 | -spec produce(fun((brod:message(), cb_state()) -> brod:batch_input()), 254 | fun((brod:batch_input(), brod:produce_ack_cb()) -> {ok, pid()}), 255 | [#kafka_message{}], pending_acks(), 256 | cb_state()) -> {pending_acks(), cb_state()}. 257 | produce(_FilterFun, _ProduceFun, [], PendingAcks, CbState) -> 258 | {PendingAcks, CbState}; 259 | produce(FilterFun, ProduceFun, 260 | [#kafka_message{offset = Offset} = Msg | Rest], 261 | PendingAcks0, CbState0) -> 262 | {FilterResult, CbState} = FilterFun(Msg, CbState0), 263 | Caller = self(), 264 | PendingRefs = 265 | case FilterResult of 266 | [] -> []; %% discard this message 267 | Batch -> 268 | lists:map( 269 | fun(OneMsg) -> 270 | Ref = make_ref(), 271 | Cb = fun(_, _) -> Caller ! ?kafka_ack(Offset, Ref) end, 272 | {ok, Pid} = ProduceFun(OneMsg, Cb), 273 | {Pid, Ref} 274 | end, Batch) 275 | end, 276 | PendingAcks = brucke_backlog:add(Offset, PendingRefs, PendingAcks0), 277 | produce(FilterFun, ProduceFun, Rest, PendingAcks, CbState). 278 | 279 | -spec handle_produce_reply(state(), offset(), reference()) -> state(). 280 | handle_produce_reply(#{pending_acks := PendingAcks0} = State, UpstreamOffset, Ref) -> 281 | PendingAcks = brucke_backlog:ack(UpstreamOffset, Ref, PendingAcks0), 282 | handle_acked(State#{pending_acks := PendingAcks}). 283 | 284 | -spec handle_acked(state()) -> state(). 285 | handle_acked(#{ pending_acks := PendingAcks 286 | , parent := Parent 287 | , upstream_cluster := UpstreamCluster 288 | , upstream_partition := UpstreamPartition 289 | , consumer := ConsumerPid 290 | , route := Route 291 | , high_wm_offset := HighWmOffset 292 | } = State) -> 293 | #route{upstream = {_UpstreamClientId, UpstreamTopic}} = Route, 294 | {OffsetToAck, NewPendingAcks} = brucke_backlog:prune(PendingAcks), 295 | case is_integer(OffsetToAck) of 296 | true -> 297 | %% tell upstream consumer to fetch more 298 | ok = brod:consume_ack(ConsumerPid, OffsetToAck), 299 | ?MX_CURRENT_OFFSET(UpstreamCluster, UpstreamTopic, 300 | UpstreamPartition, OffsetToAck), 301 | ?MX_LAGGING_OFFSET(UpstreamCluster, UpstreamTopic, 302 | UpstreamPartition, HighWmOffset - OffsetToAck), 303 | %% tell parent to update my next begin_offset in case i crash 304 | %% parent should also report it to coordinator and (later) commit to kafka 305 | Parent ! {ack, UpstreamPartition, OffsetToAck}; 306 | false -> 307 | ok 308 | end, 309 | State#{pending_acks := NewPendingAcks}. 310 | 311 | %% Return 'true' if ALL producers of unacked requests are still alive. 312 | -spec check_producer(state()) -> state() | no_return(). 313 | check_producer(#{pending_acks := Pendings} = State) -> 314 | Pids = brucke_backlog:get_producers(Pendings), 315 | NewState = 316 | case lists:all(fun erlang:is_process_alive/1, Pids) of 317 | true -> State; 318 | false -> erlang:exit(producer_down) 319 | end, 320 | _ = erlang:send_after(?CHECK_PRODUCER_DELAY, self(), ?CHECK_PRODUCER_MSG), 321 | NewState. 322 | 323 | -spec handle_consumer_down(state(), pid()) -> state(). 324 | handle_consumer_down(#{consumer := Pid} = _State, Pid) -> 325 | erlang:exit(consumer_down); 326 | handle_consumer_down(State, _UnknownPid) -> 327 | State. 328 | 329 | -spec partition(integer(), partition(), repartitioning_strategy(), binary()) -> 330 | brod_partition_fun(). 331 | partition(_PartitionCount, UpstreamPartition, strict_p2p, _Key) -> 332 | UpstreamPartition; 333 | partition(PartitionCount, _UpstreamPartition, key_hash, Key) -> 334 | erlang:phash2(Key, PartitionCount); 335 | partition(PartitionCount, _UpstreamPartition, random, _Key) -> 336 | rand:uniform(PartitionCount) - 1. 337 | 338 | msg_set_bytes(#kafka_message_set{messages = Messages}) -> 339 | msg_set_bytes(Messages, 0). 340 | 341 | msg_set_bytes([], Bytes) -> Bytes; 342 | msg_set_bytes([#kafka_message{key = K, value = V, 343 | headers = Headers} | Rest], Bytes) -> 344 | msg_set_bytes(Rest, Bytes + size(K) + size(V) + header_bytes(Headers)). 345 | 346 | header_bytes([]) -> 0; 347 | header_bytes([{K, V} | Rest]) -> size(K) + size(V) + header_bytes(Rest). 348 | 349 | ratelimit(Ratelimiter) -> 350 | brucke_ratelimiter:acquire(Ratelimiter). 351 | 352 | %%%_* Emacs ==================================================================== 353 | %%% Local Variables: 354 | %%% allout-layout: t 355 | %%% erlang-indent-level: 2 356 | %%% End: 357 | -------------------------------------------------------------------------------- /src/brucke_sup.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2016-2017 Klarna AB 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module(brucke_sup). 18 | -behaviour(supervisor3). 19 | -dialyzer(no_undefined_callbacks). 20 | 21 | -export([ start_link/0 22 | , init/1 23 | , post_init/1 24 | , get_group_member_children/1 25 | ]). 26 | 27 | -include("brucke_int.hrl"). 28 | 29 | -define(ROOT_SUP, brucke_sup). 30 | -define(GROUP_MEMBER_SUP, brucke_member_sup). 31 | 32 | start_link() -> 33 | supervisor3:start_link({local, ?ROOT_SUP}, ?MODULE, ?ROOT_SUP). 34 | 35 | -spec get_group_member_children(term()) -> [{ID::term(), pid() | atom()}]. 36 | get_group_member_children(Upstream) -> 37 | case supervisor3:find_child(?ROOT_SUP, Upstream) of 38 | Pid when is_pid(Pid) -> get_children(Pid); 39 | _ -> [] 40 | end. 41 | 42 | init(?ROOT_SUP) -> 43 | ok = brucke_http:init(), 44 | ok = brucke_config:init(), 45 | ok = brucke_metrics:init(), 46 | ok = brucke_ratelimiter:init(), 47 | AllClients = brucke_config:all_clients(), 48 | lists:foreach( 49 | fun({ClientId, Endpoints, ClientConfig}) -> 50 | ok = brod:start_client(Endpoints, ClientId, ClientConfig) 51 | end, AllClients), 52 | AllRoutes = brucke_routes:all(), 53 | RouteSups = [route_sup_spec(Route) || Route <- AllRoutes], 54 | {ok, {{one_for_one, 0, 1}, RouteSups}}; 55 | init({?GROUP_MEMBER_SUP, _Route}) -> 56 | post_init. 57 | 58 | post_init({?GROUP_MEMBER_SUP, #route{} = Route}) -> 59 | #route{ upstream = Upstream 60 | , downstream = Downstream 61 | , options = Options} = Route, 62 | case {get_partition_count(Upstream), get_partition_count(Downstream)} of 63 | {none, _} -> 64 | Reason = "upstream topic not found in kafka", 65 | ok = brucke_routes:add_skipped_route(Route, Reason), 66 | exit(normal); 67 | {_, none} -> 68 | Reason = "downstream topic not found in kafka", 69 | ok = brucke_routes:add_skipped_route(Route, Reason), 70 | exit(normal); 71 | {UpstreamPartitionsCount, DownstreamPartitionCount} -> 72 | case validate_repartitioning_strategy(UpstreamPartitionsCount, 73 | DownstreamPartitionCount, 74 | Options) of 75 | ok -> 76 | Workers = route_worker_specs(Route, UpstreamPartitionsCount), 77 | {ok, {{one_for_one, 0, 1}, Workers}}; 78 | {error, Reason} -> 79 | ok = brucke_routes:add_skipped_route(Route, Reason), 80 | exit(normal) 81 | end 82 | end. 83 | 84 | route_sup_spec(#route{upstream = Upstream} = Route) -> 85 | { _ID = Upstream 86 | , _Start = {supervisor3, start_link, [?MODULE, {?GROUP_MEMBER_SUP, Route}]} 87 | , _Restart = {transient, _DelaySeconds = 20} 88 | , _Shutdown = infinity 89 | , _Type = supervisor 90 | , _Module = [?MODULE] 91 | }. 92 | 93 | route_worker_specs(#route{options = Options} = Route, PartitionsCount) -> 94 | PartitionCountLimit = max_partitions_per_group_member(Options), 95 | route_worker_specs(Route, PartitionCountLimit, _Seqno = 1, PartitionsCount). 96 | 97 | route_worker_specs( _, _, _, Count) when Count =< 0 -> []; 98 | route_worker_specs(Route, Limit, Seqno, Count) -> 99 | [{ _ID = Seqno 100 | , _Start = {brucke_member, start_link, [Route]} 101 | , _Restart = {permanent, _DelaySeconds = 60} 102 | , _Shutdown = 5000 103 | , _Type = worker 104 | , _Module = [brucke_member] 105 | } | route_worker_specs(Route, Limit, Seqno+1, Count - Limit)]. 106 | 107 | get_partition_count({ClientId, Topic}) -> 108 | case brod_client:get_partitions_count(ClientId, Topic) of 109 | {ok, Count} -> 110 | Count; 111 | {error, Reason} -> 112 | exit({failed_to_get_partition_count, ClientId, Topic, Reason}) 113 | end. 114 | 115 | -spec max_partitions_per_group_member(route_options()) -> pos_integer(). 116 | max_partitions_per_group_member(Options) -> 117 | maps:get(max_partitions_per_group_member, Options, 118 | ?MAX_PARTITIONS_PER_GROUP_MEMBER). 119 | 120 | -spec validate_repartitioning_strategy(pos_integer(), pos_integer(), 121 | route_options()) -> 122 | ok | {error, iodata()}. 123 | validate_repartitioning_strategy(UpstreamPartitionCount, 124 | DownstreamPartitionCount, Options) -> 125 | case brucke_lib:get_repartitioning_strategy(Options) of 126 | strict_p2p when UpstreamPartitionCount =/= DownstreamPartitionCount -> 127 | {error, "The number of partitions in upstream/downstream topics should " 128 | "be the same for 'strict_p2p' repartitioning strategy"}; 129 | _Other -> 130 | ok 131 | end. 132 | 133 | -spec get_children(atom() | pid()) -> [{ID::term(), pid() | atom()}]. 134 | get_children(Pid) -> 135 | lists:map(fun({ID, ChildPid, _Worker, _Modules}) -> 136 | {ID, ChildPid} 137 | end, supervisor3:which_children(Pid)). 138 | 139 | %%%_* Emacs ==================================================================== 140 | %%% Local Variables: 141 | %%% allout-layout: t 142 | %%% erlang-indent-level: 2 143 | %%% End: 144 | -------------------------------------------------------------------------------- /test/brucke_SUITE.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2017-2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | %% @private 18 | -module(brucke_SUITE). 19 | 20 | %% Test framework 21 | -export([ init_per_suite/1 22 | , end_per_suite/1 23 | , init_per_testcase/2 24 | , end_per_testcase/2 25 | , all/0 26 | , suite/0 27 | ]). 28 | 29 | %% Test cases 30 | -export([ t_basic/1 31 | , t_consumer_managed_offset/1 32 | , t_filter/1 33 | , t_filter_with_ts/1 34 | , t_random_dispatch/1 35 | , t_split_message/1 36 | , t_route_ratelimiter/1 37 | , t_route_ratelimiter_change_rate/1 38 | , t_route_ratelimiter_topic_rate/1 39 | ]). 40 | 41 | -include_lib("common_test/include/ct.hrl"). 42 | -include_lib("eunit/include/eunit.hrl"). 43 | -include_lib("brod/include/brod.hrl"). 44 | 45 | -define(HOST, "localhost"). 46 | -define(HOSTS, [{?HOST, 9092}]). 47 | -define(OFFSETS_TAB, brucke_offsets). 48 | 49 | %%%_* ct callbacks ============================================================= 50 | 51 | suite() -> [{timetrap, {seconds, 30}}]. 52 | 53 | init_per_suite(Config) -> 54 | NewConfig = prepare_data_t_consumer_managed_offset(Config), 55 | _ = application:load(brucke), 56 | application:set_env(brucke, config_file, {priv, "brucke.yml"}), 57 | {ok, _} = application:ensure_all_started(brucke), 58 | NewConfig. 59 | 60 | end_per_suite(_Config) -> 61 | application:stop(brucke), 62 | application:stop(brod), 63 | ok. 64 | 65 | init_per_testcase(Case, Config) -> 66 | try 67 | ?MODULE:Case({init, Config}) 68 | catch 69 | error : function_clause -> 70 | Config 71 | end. 72 | 73 | end_per_testcase(Case, Config) -> 74 | try 75 | ?MODULE:Case({'end', Config}) 76 | catch 77 | error : function_clause -> 78 | ok 79 | end, 80 | ok. 81 | 82 | all() -> [F || {F, _A} <- module_info(exports), 83 | case atom_to_list(F) of 84 | "t_" ++ _ -> true; 85 | _ -> false 86 | end]. 87 | 88 | %%%_* Test functions =========================================================== 89 | 90 | t_basic(Config) when is_list(Config) -> 91 | UPSTREAM = <<"brucke-basic-test-upstream">>, 92 | DOWNSTREAM = <<"brucke-basic-test-downstream">>, 93 | Client = client_1, %% configured in priv/brucke.yml 94 | ok = wait_for_subscriber(Client, UPSTREAM), 95 | ok = brod:start_producer(Client, UPSTREAM, []), 96 | {ok, Offset} = brod:resolve_offset(?HOSTS, DOWNSTREAM, 0, latest), 97 | Headers = [{<<"foo">>, <<"bar">>}], 98 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<>>, <<"v0">>), 99 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<>>, <<"v1">>), 100 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<>>, 101 | #{value => <<"v2">>, headers => Headers}), 102 | FetchFun = fun(Of) -> fetch(DOWNSTREAM, 0, Of) end, 103 | Messages = fetch_loop(FetchFun, Offset, _TryMax = 20, [], _Count = 3), 104 | ?assertMatch([{_, <<"v0">>}, 105 | {_, <<"v1">>}, 106 | {_, <<"v2">>, Headers}], Messages). 107 | 108 | t_consumer_managed_offset(Config) when is_list(Config) -> 109 | %%% preconditions are set in prepare_data_t_consumer_managed_offset/1 110 | Client = client_1, 111 | UPSTREAM = <<"brucke-filter-consumer-managed-offsets-test-upstream">>, 112 | DOWNSTREAM = <<"brucke-filter-consumer-managed-offsets-test-downstream">>, 113 | DSOffsets = ?config({DOWNSTREAM, offsets}, Config), 114 | [ok = wait_for_subscriber(Client, UPSTREAM, P) || {P, _} <- DSOffsets], 115 | Result = [{P, Msg#kafka_message.value} || {P,O} <- DSOffsets, Msg <- fetch(DOWNSTREAM, P, O)], 116 | Expected = [ {0, <<"4">>} % partition 0, 117 | , {0, <<"5">>} 118 | , {0, <<"6">>} 119 | , {0, <<"7">>} 120 | , {0, <<"8">>} 121 | , {1, <<"15">>} % partition 1 122 | , {1, <<"16">>} 123 | , {1, <<"17">>} 124 | , {1, <<"18">>} 125 | , {2, <<"26">>} % partition 2 126 | , {2, <<"27">>} 127 | , {2, <<"28">>} 128 | ], 129 | ?assertEqual(Expected, Result). 130 | 131 | 132 | %% Send 3 messages to upstream topic 133 | %% Expect them to be mirrored to downstream toicp with filter/transformation 134 | %% logic implemented in `brucke_test_filter' module. see config `priv/brucke.yml' 135 | t_filter(Config) when is_list(Config) -> 136 | UPSTREAM = <<"brucke-filter-test-upstream">>, 137 | DOWNSTREAM = <<"brucke-filter-test-downstream">>, 138 | Client = client_1, %% configured in priv/brucke.yml 139 | ok = wait_for_subscriber(client_3, UPSTREAM), 140 | ok = brod:start_producer(Client, UPSTREAM, []), 141 | {ok, Offset} = brod:resolve_offset(?HOSTS, DOWNSTREAM, 0, latest), 142 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<"as_is">>, <<"0">>), 143 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<"discard">>, <<"1">>), 144 | V = uniq_int(), 145 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<"increment">>, bin(V)), 146 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<"append_state">>, <<"foo">>), 147 | FetchFun = fun(Of) -> fetch(DOWNSTREAM, 0, Of) end, 148 | Messages = fetch_loop(FetchFun, Offset, _TryMax = 20, [], 2), 149 | NewV = bin(V + 1), 150 | ?assertMatch([{_T0, <<"0">>}, 151 | {_T2, NewV}, 152 | {_T3, <<"foo 3">>} 153 | ], Messages). 154 | 155 | t_filter_with_ts(Config) when is_list(Config) -> 156 | UPSTREAM = <<"brucke-filter-test-upstream">>, 157 | DOWNSTREAM = <<"brucke-filter-test-downstream">>, 158 | Client = client_1, %% configured in priv/brucke.yml 159 | ok = wait_for_subscriber(client_3, UPSTREAM), 160 | ok = brod:start_producer(Client, UPSTREAM, []), 161 | {ok, Offset} = brod:resolve_offset(?HOSTS, DOWNSTREAM, 0, latest), 162 | T0 = ts(), 163 | I0 = uniq_int(), 164 | V0 = #{ts => T0, value => bin(I0)}, 165 | I1 = uniq_int(), 166 | V1 = #{value => bin(I1)}, 167 | T2 = T0 + 1, 168 | I2 = uniq_int(), 169 | V2 = #{ts => T2, value => bin(I2)}, 170 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<"as_is">>, V0), 171 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<"discard">>, V1), 172 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<"increment">>, V2), 173 | FetchFun = fun(Of) -> fetch(DOWNSTREAM, 0, Of) end, 174 | Messages = fetch_loop(FetchFun, Offset, _TryMax = 20, [], 2), 175 | ?assertEqual([{T0, bin(I0)}, {T2, bin(I2 + 1)}], Messages). 176 | 177 | t_route_ratelimiter(Config) when is_list(Config) -> 178 | %% Test rate 0 means pause 179 | UPSTREAM = <<"brucke-ratelimiter-test-upstream">>, 180 | DOWNSTREAM = <<"brucke-ratelimiter-test-downstream">>, 181 | Client = client_1, 182 | ok = wait_for_subscriber(client_3, UPSTREAM), 183 | ok = brod:start_producer(Client, UPSTREAM, []), 184 | [brod:produce_sync(Client, UPSTREAM, 0, << "foo" >>, << "bar" >>) || _V <- lists:seq(1,20)], 185 | {ok, Offset} = brod:resolve_offset(?HOSTS, DOWNSTREAM, 0, latest), 186 | ct:sleep(3000), 187 | {ok, Offset2} = brod:resolve_offset(?HOSTS, DOWNSTREAM, 0, latest), 188 | %%% in priv/brucke.yml rate is set to 0 msgs/s so no message should be delivered to downstream. 189 | ?assertEqual(Offset2, Offset), 190 | ok. 191 | 192 | t_route_ratelimiter_change_rate(Config) when is_list(Config) -> 193 | %% Test we can change the rate via filter restapi 194 | %% configured in priv/brucke.yml 195 | UPSTREAM = <<"brucke-ratelimiter-test-upstream">>, 196 | DOWNSTREAM = <<"brucke-ratelimiter-test-downstream">>, 197 | Client = client_1, 198 | Rid = {local_cluster_ssl, 'brucke-ratelimiter-test'}, 199 | ok = wait_for_subscriber(client_3, UPSTREAM), 200 | ok = brod:start_producer(Client, UPSTREAM, []), 201 | {ok, Offset} = brod:resolve_offset(?HOSTS, DOWNSTREAM, 0, latest), 202 | ?assertEqual({1000, 0}, brucke_ratelimiter:get_rate(Rid)), 203 | set_rate_limiter(Rid, 10), 204 | ?assertEqual({1000, 10}, brucke_ratelimiter:get_rate(Rid)), 205 | [brod:produce_sync(Client, UPSTREAM, 0, << "foo" >>, << "bar" >>) || _V <- lists:seq(1,100)], 206 | timer:sleep(3000), 207 | {ok, Offset2} = brod:resolve_offset(?HOSTS, DOWNSTREAM, 0, latest), 208 | Diff = Offset2 - Offset, 209 | ?assert(Diff < 40 orelse Diff > 20), 210 | ok. 211 | 212 | t_route_ratelimiter_topic_rate(Config) when is_list(Config) -> 213 | %% Test ratelimit is on topic, not per topic-partition. 214 | %% configured in priv/brucke.yml 215 | UPSTREAM = <<"brucke-ratelimiter-test-upstream">>, 216 | DOWNSTREAM = <<"brucke-ratelimiter-test-downstream">>, 217 | Client = client_1, 218 | Rid = {local_cluster_ssl, 'brucke-ratelimiter-test'}, 219 | Partitions = [0, 1, 2], 220 | GetLatestPartitionOffsets = fun() -> 221 | lists:map(fun(P) -> 222 | {ok, Offset} = brod:resolve_offset(?HOSTS, DOWNSTREAM, P, latest), 223 | Offset 224 | end, Partitions) 225 | end, 226 | 227 | ok = wait_for_subscriber(client_3, UPSTREAM), 228 | ok = brod:start_producer(Client, UPSTREAM, []), 229 | 230 | %%% pause 231 | set_rate_limiter(Rid, 0), 232 | ct:sleep(1000), 233 | 234 | ?assertEqual({1000, 0}, brucke_ratelimiter:get_rate(Rid)), 235 | 236 | Offsets0 = GetLatestPartitionOffsets(), 237 | 238 | [brod:produce_sync(Client, UPSTREAM, P, << "foo2" >>, << "bar" >>) || _V <- lists:seq(1, 100), P <- Partitions], 239 | 240 | %% here we also change interval 241 | set_rate_limiter(Rid, {100,10}), 242 | 243 | ?assertEqual({100, 10}, brucke_ratelimiter:get_rate(Rid)), 244 | 245 | ct:sleep(3000), 246 | 247 | Offsets1 = GetLatestPartitionOffsets(), 248 | 249 | Total = lists:foldl(fun({New, Old}, Acc)-> 250 | Diff = New - Old, 251 | ?assert(Diff > 0), 252 | Acc + Diff 253 | end, 0, lists:zip(Offsets1, Offsets0)), 254 | ?assert(Total > 200 andalso Total < 400), 255 | ok. 256 | 257 | t_random_dispatch(Config) when is_list(Config) -> 258 | UPSTREAM = <<"brucke-filter-test-upstream">>, 259 | DOWNSTREAM = <<"brucke-filter-test-downstream">>, 260 | Client = client_1, %% configured in priv/brucke.yml 261 | ok = wait_for_subscriber(client_3, UPSTREAM), 262 | ok = brod:start_producer(Client, UPSTREAM, []), 263 | {ok, Offset} = brod:resolve_offset(?HOSTS, DOWNSTREAM, 0, latest), 264 | T = ts(), 265 | Msg = #{ts => T, key => <<"as_is">>, value => <<"51">>}, 266 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<>>, [Msg]), 267 | FetchFun = fun(Of) -> fetch(DOWNSTREAM, 0, Of) end, 268 | Messages = fetch_loop(FetchFun, Offset, _TryMax = 20, [], 1), 269 | ?assertMatch([{T, <<"51">>}], Messages). 270 | 271 | t_split_message(Config) when is_list(Config) -> 272 | UPSTREAM = <<"brucke-filter-test-upstream">>, 273 | DOWNSTREAM = <<"brucke-filter-test-downstream">>, 274 | Client = client_1, %% configured in priv/brucke.yml 275 | ok = wait_for_subscriber(client_3, UPSTREAM), 276 | ok = brod:start_producer(Client, UPSTREAM, []), 277 | {ok, Offset} = brod:resolve_offset(?HOSTS, DOWNSTREAM, 0, latest), 278 | T = ts(), 279 | Msg = #{ts => T, key => <<"split_value">>, value => <<"a,b,c">>}, 280 | ok = brod:produce_sync(Client, UPSTREAM, 0, <<>>, [Msg]), 281 | FetchFun = fun(Of) -> fetch(DOWNSTREAM, 0, Of) end, 282 | Messages = fetch_loop(FetchFun, Offset, _TryMax = 20, [], 3), 283 | ?assertMatch([{_, <<"a">>}, 284 | {_, <<"b">>}, 285 | {_, <<"c">>}], Messages). 286 | 287 | %%%_* Help functions =========================================================== 288 | 289 | %% wait for subsceriber of the upstream topic 290 | wait_for_subscriber(Client, Topic) -> 291 | wait_for_subscriber(Client, Topic, 0). 292 | wait_for_subscriber(Client, Topic, Partition) -> 293 | F = fun() -> 294 | {ok, Pid} = brod:get_consumer(Client, Topic, Partition), 295 | {error, {already_subscribed_by, _}} = 296 | brod_consumer:subscribe(Pid, fake_subscriber, []), 297 | exit(normal) 298 | end, 299 | {Pid, Mref} = erlang:spawn_monitor(fun() -> wait_for_subscriber(F) end), 300 | receive 301 | {'DOWN', Mref, process, Pid, normal} -> 302 | ok; 303 | {'DOWN', Mref, process, Pid, Reason} -> 304 | ct:fail(Reason) 305 | after 306 | 20000 -> 307 | exit(Pid, kill), 308 | error(timeout) 309 | end. 310 | 311 | wait_for_subscriber(F) -> 312 | try 313 | F() 314 | catch 315 | error : _Reason -> 316 | timer:sleep(1000), 317 | wait_for_subscriber(F) 318 | end. 319 | 320 | fetch(Topic, Partition, Offset) -> 321 | {ok, {_HwOffset, Messages}} = 322 | brod:fetch({?HOSTS, []}, Topic, Partition, Offset), 323 | Messages. 324 | 325 | ts() -> os:system_time() div 1000000. 326 | 327 | fetch_loop(_F, _Offset, N, Acc, C) when N =< 0 orelse C =< 0 -> Acc; 328 | fetch_loop(F, Offset, N, Acc, Count) -> 329 | case F(Offset) of 330 | [] -> fetch_loop(F, Offset, N - 1, Acc, Count); 331 | Msgs0 -> 332 | Msgs = 333 | lists:map( 334 | fun(#kafka_message{ ts = Ts 335 | , value = Value 336 | , headers = Headers 337 | }) -> 338 | case Headers =:= [] of 339 | true -> {Ts, Value}; 340 | false -> {Ts, Value, Headers} 341 | end 342 | end, Msgs0), 343 | C = length(Msgs), 344 | fetch_loop(F, Offset + C, N - 1, Acc ++ Msgs, Count - C) 345 | end. 346 | 347 | uniq_int() -> os:system_time(). 348 | 349 | bin(X) -> integer_to_binary(X). 350 | 351 | prepare_data_t_consumer_managed_offset(Config) -> 352 | Client = client_prepare_data, 353 | UPSTREAM = <<"brucke-filter-consumer-managed-offsets-test-upstream">>, 354 | DOWNSTREAM = <<"brucke-filter-consumer-managed-offsets-test-downstream">>, 355 | ConsumerGroup = <<"brucke-filter-test-consumer-managed-offsets">>, 356 | {ok,_} = application:ensure_all_started(brod), 357 | Partitions = [0, 1, 2], 358 | Messages = [4, 5, 6, 7, 8], 359 | DSOffsets = resolve_offsets(DOWNSTREAM, Partitions), 360 | USOffsets = resolve_offsets(UPSTREAM, Partitions), 361 | ct:pal("PartitionOffsets for ~p are ~p", [UPSTREAM, USOffsets]), 362 | case USOffsets of 363 | [{0, 0}, {1, 0}, {2, 0}] -> % kafka is empty. this is new test env, produce test data 364 | ct:pal("insert test msgs to topic ~p", [UPSTREAM]), 365 | ok = brod:start_client(?HOSTS, Client), 366 | ok = brod:start_producer(Client, UPSTREAM, _ProducerConfig = []), 367 | [ ok = brod:produce_sync(Client, UPSTREAM, P, <<>>, integer_to_binary(P*10 + M)) || 368 | P <- Partitions, 369 | M <- Messages], 370 | ok = brod:stop_client(Client); 371 | _ -> 372 | skip 373 | end, 374 | ok = prepare_brucke_offsets_dets(ConsumerGroup, UPSTREAM, USOffsets), 375 | [{{DOWNSTREAM, offsets}, DSOffsets} | Config]. 376 | 377 | prepare_brucke_offsets_dets(GroupId, Topic, PartitionOffsets) -> 378 | {ok, ?OFFSETS_TAB} = dets:open_file(?OFFSETS_TAB, 379 | [{file, "/tmp/brucke_offsets_ct.DETS"}, 380 | {ram_file, true}]), 381 | {Partitions, _Offsets} = lists:unzip(PartitionOffsets), 382 | TestOffsets = [-1, 0, 1], %% because brod coordinator will do offset+1 383 | lists:foreach(fun({Partition, Offset}) -> 384 | ok = dets:insert(?OFFSETS_TAB, {{GroupId, Topic, Partition}, Offset}) 385 | end, lists:zip(Partitions, TestOffsets)), 386 | ok = dets:close(?OFFSETS_TAB). 387 | 388 | resolve_offsets(Topic, Partitions) -> 389 | lists:map(fun(P) -> 390 | {ok, Offset} = brod:resolve_offset(?HOSTS, Topic, P, latest), 391 | {P, Offset} 392 | end, Partitions). 393 | 394 | set_rate_limiter(Rid, MsgPerSec) when is_integer(MsgPerSec)-> 395 | set_rate_limiter(Rid, {1000, MsgPerSec}); 396 | set_rate_limiter({Cluster, Cgid}, {I, T}) -> 397 | RatelimiterApiUrl = lists:flatten(io_lib:format("http://localhost:~p/plugins/ratelimiter/~s/~s", 398 | [brucke_app:http_port(), Cluster, Cgid])), 399 | Body = jsone:encode([ {interval, I} 400 | , {threshold, T}]), 401 | {ok, {{"HTTP/1.1",200,"OK"}, _, "\"ok\"" }} = 402 | httpc:request(post, {RatelimiterApiUrl, [{"Accept", "application/json"}], 403 | "application/json", Body}, [], []). 404 | 405 | %%%_* Emacs ==================================================================== 406 | %%% Local Variables: 407 | %%% allout-layout: t 408 | %%% erlang-indent-level: 2 409 | %%% End: 410 | -------------------------------------------------------------------------------- /test/brucke_backlog_tests.erl: -------------------------------------------------------------------------------- 1 | -module(brucke_backlog_tests). 2 | 3 | -include_lib("eunit/include/eunit.hrl"). 4 | 5 | full_flow_test() -> 6 | Funs = 7 | [ fun(B) -> add(0, [{pid1, ref1}], B) end 8 | , fun(B) -> add(1, [{pid2, ref2}, {pid2, ref3}], B) end 9 | , fun(B) -> ack(1, ref3, B) end 10 | , fun(B) -> {false, NewB} = prune(B), NewB end 11 | , fun(B) -> ?assertEqual([{0, [{pid1, ref1}]}, {1, [{pid2, ref2}]}], to_list(B)), B end 12 | , fun(B) -> ?assertEqual([pid1, pid2], get_producers(B)), B end 13 | , fun(B) -> ack(1, ref4, B) end 14 | , fun(B) -> ack(2, ref4, B) end 15 | , fun(B) -> ack(0, ref1, B) end 16 | , fun(B) -> {0, NewB} = prune(B), NewB end 17 | , fun(B) -> ?assertEqual([{1, [{pid2, ref2}]}], to_list(B)), B end 18 | , fun(B) -> ack(1, ref2, B) end 19 | , fun(B) -> {1, NewB} = prune(B), NewB end 20 | , fun(B) -> ?assertEqual([], to_list(B)) end 21 | ], 22 | ok = lists:foldl(fun(F, B) -> F(B) end, new(), Funs). 23 | 24 | new() -> brucke_backlog:new(). 25 | add(Offset, Refs, Backlog) -> brucke_backlog:add(Offset, Refs, Backlog). 26 | ack(Offset, Ref, Backlog) -> brucke_backlog:ack(Offset, Ref, Backlog). 27 | prune(Backlog) -> brucke_backlog:prune(Backlog). 28 | to_list(Backlog) -> brucke_backlog:to_list(Backlog). 29 | get_producers(Backlog) -> brucke_backlog:get_producers(Backlog). 30 | 31 | %%%_* Emacs ==================================================================== 32 | %%% Local Variables: 33 | %%% allout-layout: t 34 | %%% erlang-indent-level: 2 35 | %%% End: 36 | -------------------------------------------------------------------------------- /test/brucke_config_tests.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2018 Klarna Bank AB (publ) 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | -module(brucke_config_tests). 17 | 18 | -include_lib("eunit/include/eunit.hrl"). 19 | 20 | validate_client_config_test_() -> 21 | [ %% cacertfile is mandatory, and bad file should trigger exception 22 | ?_assertException(exit, bad_ssl_file, 23 | validate_client_config([{ssl, [{cacertfile, "no-such-file"}]}])), 24 | %% certfile is optional but providing a bad file should still 25 | %% raise an exception 26 | ?_assertException(exit, bad_ssl_file, 27 | validate_client_config([{ssl, [{cacertfile, "priv/ssl/ca.crt"}, 28 | {certfile, "no-such-file"}]}])), 29 | %% OK case 30 | ?_assertMatch([{ssl, [{cacertfile, _}]}], 31 | validate_client_config([{ssl, [{cacertfile, "priv/ssl/ca.crt"}]}])), 32 | 33 | ?_assertMatch([{ssl, [{cacertfile, _}, {keyfile, _}, {certfile, _}]}], 34 | validate_client_config([{ssl, [{cacertfile, "priv/ssl/ca.crt"}, 35 | {keyfile, "priv/ssl/client.key"}, 36 | {certfile, "priv/ssl/client.crt"} 37 | ]}])), 38 | ?_assertEqual([], validate_client_config([])), 39 | ?_assertEqual([{ssl, true}, {sasl, {plain, <<"foo">>, <<"bar">>}}], 40 | validate_client_config([{ssl, true}, 41 | {sasl, [{username, foo}, 42 | {password, bar}]}])), 43 | ?_assertEqual([{ssl, true}, {sasl, {scram_sha_256, <<"foo">>, <<"bar">>}}], 44 | validate_client_config([{ssl, true}, 45 | {sasl, [{username, foo}, 46 | {password, bar}, 47 | {mechanism, scram_sha_256}]}])), 48 | ?_assertEqual([{sasl, {scram_sha_512, <<"foo">>, <<"bar">>}}], 49 | validate_client_config([{sasl, [{username, foo}, 50 | {password, "bar"}, 51 | {mechanism, scram_sha_512}]}])) 52 | ]. 53 | 54 | validate_client_config(Config) -> 55 | brucke_config:validate_client_config(client_id, Config). 56 | 57 | %%%_* Emacs ==================================================================== 58 | %%% Local Variables: 59 | %%% allout-layout: t 60 | %%% erlang-indent-level: 2 61 | %%% End: 62 | -------------------------------------------------------------------------------- /test/brucke_http_SUITE.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2017 Klarna AB 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | %% @private 18 | -module(brucke_http_SUITE). 19 | 20 | %% Test framework 21 | -export([ init_per_suite/1 22 | , end_per_suite/1 23 | , init_per_testcase/2 24 | , end_per_testcase/2 25 | , all/0 26 | , suite/0 27 | ]). 28 | 29 | %% HTTP endpoint cases 30 | -export([ t_ping/1 31 | , t_healthcheck/1 32 | , t_ratelimter_restapi/1 33 | ]). 34 | 35 | -include_lib("common_test/include/ct.hrl"). 36 | -include_lib("eunit/include/eunit.hrl"). 37 | 38 | -define(assertHttpOK(Code, Msg), (case Code >= 200 andalso Code < 300 of 39 | true -> ok; 40 | false -> ct:fail("~p: ~s", [Code, Msg]) 41 | end)). 42 | 43 | %%%_* ct callbacks ============================================================= 44 | 45 | suite() -> [{timetrap, {seconds, 30}}]. 46 | 47 | init_per_suite(Config) -> 48 | _ = application:load(brucke), 49 | application:set_env(brucke, config_file, {priv, "brucke.yml"}), 50 | _ = application:ensure_all_started(brucke), 51 | Config. 52 | 53 | end_per_suite(_Config) -> 54 | application:stop(brucke), 55 | application:stop(brod), 56 | ok. 57 | 58 | init_per_testcase(Case, Config) -> 59 | try 60 | ?MODULE:Case({init, Config}) 61 | catch 62 | error : function_clause -> 63 | Config 64 | end. 65 | 66 | end_per_testcase(Case, Config) -> 67 | try 68 | ?MODULE:Case({'end', Config}) 69 | catch 70 | error : function_clause -> 71 | ok 72 | end, 73 | ok. 74 | 75 | all() -> [F || {F, _A} <- module_info(exports), 76 | case atom_to_list(F) of 77 | "t_" ++ _ -> true; 78 | _ -> false 79 | end]. 80 | 81 | %%%_* Test functions =========================================================== 82 | 83 | t_ping(Config) when is_list(Config) -> 84 | URL = lists:flatten(io_lib:format("http://localhost:~p/ping", [brucke_app:http_port()])), 85 | {ok, {{"HTTP/1.1", ReturnCode, _State}, _Head, ReplyBody}} = httpc:request(URL), 86 | ?assertHttpOK(ReturnCode, ReplyBody), 87 | ok. 88 | 89 | t_healthcheck(Config) when is_list(Config) -> 90 | URL = lists:flatten(io_lib:format("http://localhost:~p/healthcheck", [brucke_app:http_port()])), 91 | {ok, {{"HTTP/1.1", ReturnCode, _State}, _Head, ReplyBody}} = httpc:request(URL), 92 | ?assertHttpOK(ReturnCode, ReplyBody), 93 | ok. 94 | 95 | t_ratelimter_restapi(Config) when is_list(Config) -> 96 | Rid = 'local_cluster_ssl/brucke-ratelimiter-test', 97 | wait_for_ratelimiter('rtl_local_cluster_ssl_brucke-ratelimiter-test'), %%this is internal name 98 | RatelimiterApiUrl = lists:flatten(io_lib:format("http://localhost:~p/plugins/ratelimiter/~s", 99 | [brucke_app:http_port(), atom_to_list(Rid)])), 100 | ct:pal("RatelimiterApiUrl is ~p",[RatelimiterApiUrl]), 101 | [?assertMatch({ok, {{"HTTP/1.1",200,"OK"}, _, "\"ok\"" }}, 102 | httpc:request(post, {RatelimiterApiUrl, [{"Accept", "application/json"}], 103 | "application/json", jsone:encode(Body)}, [], [])) 104 | || Body <- [ [ {interval, "1000"}, {threshold, "1"} ] 105 | , [ {interval, "1000"}, {threshold, "1"} ] 106 | , [ {interval, "200"} ] 107 | , [ {threshold, "2"} ] 108 | , [ {threshold, 2} ] 109 | , [ {interval, << "200" >> } ] 110 | , [ {threshold, 0} ] 111 | ] 112 | ], 113 | %%% set nothing 114 | ?assertMatch({ok, {{"HTTP/1.1",200,"OK"}, _, "\"noargs\"" }}, 115 | httpc:request(post, {RatelimiterApiUrl, [{"Accept", "application/json"}], 116 | "application/json", jsone:encode([])}, [], [])). 117 | 118 | 119 | wait_for_ratelimiter(Rid) -> 120 | case whereis(Rid) of 121 | Pid when is_pid(Pid) -> 122 | ok; 123 | _ -> 124 | ct:sleep(500), 125 | wait_for_ratelimiter(Rid) 126 | end. 127 | 128 | %%%_* Emacs ==================================================================== 129 | %%% Local Variables: 130 | %%% allout-layout: t 131 | %%% erlang-indent-level: 2 132 | %%% End: 133 | -------------------------------------------------------------------------------- /test/brucke_test_filter.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright (c) 2017 Klarna AB 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | -module(brucke_test_filter). 17 | 18 | -export([ init/3 19 | , filter/7 20 | ]). 21 | 22 | -behaviour(brucke_filter). 23 | 24 | -record(state, {seqno = 0}). 25 | 26 | init(_UpstreamTopic, _DownstreamTopic, _InitArg) -> 27 | {ok, #state{}}. 28 | 29 | filter(_Topic, _Partition, _Offset, Key, Value, Headers, 30 | #state{seqno = Seqno} = State) -> 31 | Res = case Key of 32 | <<"as_is">> -> 33 | true; 34 | <<"discard">> -> 35 | false; 36 | <<"increment">> -> 37 | {Key, bin(int(Value) + 1)}; 38 | <<"append_state">> -> 39 | #{ key => Key 40 | , value => [Value, " ", integer_to_list(State#state.seqno)] 41 | , headers => Headers 42 | }; 43 | <<"split_value">> -> 44 | Values = binary:split(Value, <<",">>, [global]), 45 | [#{value => V} || V <- Values] 46 | end, 47 | {Res, State#state{seqno = Seqno + 1}}. 48 | 49 | bin(X) -> integer_to_binary(X). 50 | int(B) -> binary_to_integer(B). 51 | 52 | %%%_* Emacs ==================================================================== 53 | %%% Local Variables: 54 | %%% allout-layout: t 55 | %%% erlang-indent-level: 2 56 | %%% End: 57 | --------------------------------------------------------------------------------