├── .gitignore ├── .rebar3 └── rebar_compiler_erl │ └── source.dag ├── LICENSE ├── Makefile ├── README.md ├── TODO ├── ebin ├── emqx_cli_demo_kafka.beam ├── emqx_plugin_kafka.app ├── emqx_plugin_kafka.beam ├── emqx_plugin_kafka_app.beam └── emqx_plugin_kafka_sup.beam ├── etc └── emqx_plugin_kafka.config ├── rebar.config ├── rebar.config.script ├── src ├── emqx_cli_demo_kafka.erl ├── emqx_plugin_kafka.app.src ├── emqx_plugin_kafka.app.src.script ├── emqx_plugin_kafka.erl ├── emqx_plugin_kafka_app.erl └── emqx_plugin_kafka_sup.erl └── test └── emqx_plugin_kafka_SUITE.erl /.gitignore: -------------------------------------------------------------------------------- 1 | *.sublime-project 2 | *.sublime-workspace 3 | *.lock 4 | _build/ 5 | -------------------------------------------------------------------------------- /.rebar3/rebar_compiler_erl/source.dag: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ULTRAKID/emqx_plugin_kafka/e23cc37bc926cf7bf8f9fc8f89082fafecba8bc8/.rebar3/rebar_compiler_erl/source.dag -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ## shallow clone for speed 2 | 3 | REBAR_GIT_CLONE_OPTIONS += --depth 1 4 | export REBAR_GIT_CLONE_OPTIONS 5 | 6 | REBAR = rebar3 7 | all: compile 8 | 9 | compile: 10 | $(REBAR) compile 11 | 12 | ct: compile 13 | $(REBAR) as test ct -v 14 | 15 | eunit: compile 16 | $(REBAR) as test eunit 17 | 18 | xref: 19 | $(REBAR) xref 20 | 21 | cover: 22 | $(REBAR) cover 23 | 24 | clean: distclean 25 | 26 | distclean: 27 | @rm -rf _build 28 | @rm -f data/app.*.config data/vm.*.args rebar.lock 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 适配emqx v4.3版本 2 | 3 | ## 更新说明 4 | 5 | *v4.3.12.7*: [Release emqx-4.3.12.7-with-kafka · ULTRAKID/emqx_plugin_kafka (github.com)](https://github.com/ULTRAKID/emqx_plugin_kafka/releases/tag/emqx-4.3.12.7-with-kafka) 6 | 7 | `2022.08.02` 新增了是否对`publish`消息进行`base64`编码后再`kafka`转发的配置项`publish_base64`，发送数据为字节流的时候建议启用该配置。 8 | 9 | *v4.3.12.6*: [Release emqx-4.3.12.6-with-kafka · ULTRAKID/emqx_plugin_kafka (github.com)](https://github.com/ULTRAKID/emqx_plugin_kafka/releases/tag/emqx-4.3.12.6-with-kafka) 10 | 11 | `2022.06.16` 将原来的`kafka`依赖从`ekaf`替换成[kafka4beam](https://github.com/kafka4beam)/**[brod](https://github.com/kafka4beam/brod)**。 12 | 13 | emqx-plugin-template 14 | ==================== 15 | 16 | [emqx/emqx-plugin-template at emqx-v4 (github.com)](https://github.com/emqx/emqx-plugin-template/tree/emqx-v4) This is a template plugin for the EMQ X broker. 17 | 18 | Plugin Config 19 | ------------- 20 | 21 | 配置文件模板及说明见`etc/emqx_plugin_kafka.config` 22 | 23 | Authentication and ACL 24 | ---------------------- 25 | 26 | ``` 27 | emqx:hook('client.authenticate', fun ?MODULE:on_client_authenticate/3, [Env]). 28 | emqx:hook('client.check_acl', fun ?MODULE:on_client_check_acl/5, [Env]). 29 | ``` 30 | 31 | Build the EMQX broker 32 | ----------------- 33 | ###### 1. 基于CentOS7.5环境下编译，先安装相关插件 34 | 35 | ``` 36 | yum -y install make gcc gcc-c++ glibc-devel glibc-headers kernel-devel kernel-headers m4 ncurses ncurses-devel openssl openssl-devel openssl-libs zlib zlib-devel libselinux-devel xmlto perl git wget 37 | 38 | 注意：openssl的版本不是1.1.1k，则需要通过源码openssl-1.1.1k.tar.gz来安装openssl 39 | ``` 40 | ###### 2. 准备Erlang/OTP 22及以上环境 41 | 42 | 参照[Erlang and Elixir Packages Download - Erlang Solutions (erlang-solutions.com)](https://www.erlang-solutions.com/downloads/) 官网安装方式。 43 | 44 | ###### 3. 下载EMQX源码 45 | 46 | 官方源码仓库地址为[emqx/emqx: An Open-Source, Cloud-Native, Distributed MQTT Message Broker for IoT. (github.com)](https://github.com/emqx/emqx) ，分支为`main-v4.3` 47 | 48 | 本人修改了官方的编译脚本，并且在插件目录里添加了该kafka插件的信息，仓库地址为[ULTRAKID/emqx at main-v4.3 (github.com)](https://github.com/ULTRAKID/emqx/tree/main-v4.3) ，分支为`main-v4.3`。 49 | 50 | ###### 4. 修改EMQX文件，增加kafka插件 51 | 参照[emqx/README.md at main-v4.3 · ULTRAKID/emqx (github.com)](https://github.com/ULTRAKID/emqx/blob/main-v4.3/lib-extra/README.md) 。 52 | 53 | 注：[ULTRAKID/emqx at main-v4.3 (github.com)](https://github.com/ULTRAKID/emqx/tree/main-v4.3) 仓库内已进行此项修改。 54 | 55 | ###### 5. 编译EMQX，并且启动EMQX 56 | 57 | 进入emqx目录，执行make命令，需要保持外网通畅，有条件建议科学上网。 58 | 59 | 二进制编译命令：`make` 60 | 61 | docker镜像打包：`make emqx-docker` 62 | 63 | 64 | License 65 | ------- 66 | 67 | Apache License Version 2.0 68 | 69 | Author 70 | ------ 71 | 72 | Ultrakid. 73 | 74 | EMQ X Team. 75 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | 1. Add a script to generate plugin project 2 | 2. Upgrade the README.md 3 | 3. Add the plugin development guide 4 | -------------------------------------------------------------------------------- /ebin/emqx_cli_demo_kafka.beam: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ULTRAKID/emqx_plugin_kafka/e23cc37bc926cf7bf8f9fc8f89082fafecba8bc8/ebin/emqx_cli_demo_kafka.beam -------------------------------------------------------------------------------- /ebin/emqx_plugin_kafka.app: -------------------------------------------------------------------------------- 1 | {application,emqx_plugin_kafka, 2 | [{description,"EMQ X plugin kafka bridge, Edit by yzs-lyg"}, 3 | {vsn,"master"}, 4 | {modules,[emqx_cli_demo_kafka,emqx_plugin_kafka, 5 | emqx_plugin_kafka_app,emqx_plugin_kafka_sup]}, 6 | {registered,[emqx_plugin_kafka_sup]}, 7 | {applications,[kernel,stdlib,ekaf]}, 8 | {mod,{emqx_plugin_kafka_app,[]}}, 9 | {env,[]}, 10 | {licenses,["Apache-2.0"]}, 11 | {maintainers,["lyg <3613840847@qq.com>"]}, 12 | {links,[{"Github", 13 | "https://github.com/yzs/emqx-plugin-kafka"}]}]}. 14 | -------------------------------------------------------------------------------- /ebin/emqx_plugin_kafka.beam: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ULTRAKID/emqx_plugin_kafka/e23cc37bc926cf7bf8f9fc8f89082fafecba8bc8/ebin/emqx_plugin_kafka.beam -------------------------------------------------------------------------------- /ebin/emqx_plugin_kafka_app.beam: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ULTRAKID/emqx_plugin_kafka/e23cc37bc926cf7bf8f9fc8f89082fafecba8bc8/ebin/emqx_plugin_kafka_app.beam -------------------------------------------------------------------------------- /ebin/emqx_plugin_kafka_sup.beam: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ULTRAKID/emqx_plugin_kafka/e23cc37bc926cf7bf8f9fc8f89082fafecba8bc8/ebin/emqx_plugin_kafka_sup.beam -------------------------------------------------------------------------------- /etc/emqx_plugin_kafka.config: -------------------------------------------------------------------------------- 1 | [ 2 | {emqx_plugin_kafka, [ 3 | {kafka_address_list, [{"localhost", 9092}]}, %% kafka地址，可配置多个 4 | {kafka_config, [ %% 这部分是https://github.com/kafka4beam/brod库需要的配置 5 | {reconnect_cool_down_seconds, 10}, %% socket error recovery 6 | {query_api_versions, true} %% Kafka 0.9.x or Earlier请设置为false 7 | ]}, 8 | {topic, <<"emqx-topic">>}, %% 转发的目标topic 9 | {publish_base64, false} %% 对于publish到mqtt的消息内容，是否进行base64编码之后再转发（为了兼容发送字节流的情况） 10 | ]} 11 | ]. 12 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {deps, [ 2 | {cuttlefish, {git, "https://github.com/emqx/cuttlefish.git", {branch, "main"}}}, 3 | {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.3"}}}, 4 | {emqx, {git_subdir, "https://github.com/ULTRAKID/emqx", {tag, "v4.3.12-with-kafka"}, "apps/emqx"}} 5 | ]}. 6 | 7 | {erl_opts, [debug_info]}. 8 | -------------------------------------------------------------------------------- /rebar.config.script: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | 3 | DEPS = case lists:keyfind(deps, 1, CONFIG) of 4 | {_, Deps} -> Deps; 5 | _ -> [] 6 | end, 7 | 8 | ComparingFun = fun 9 | _Fun([C1|R1], [C2|R2]) when is_list(C1), is_list(C2); 10 | is_integer(C1), is_integer(C2) -> C1 < C2 orelse _Fun(R1, R2); 11 | _Fun([C1|R1], [C2|R2]) when is_integer(C1), is_list(C2) -> _Fun(R1, R2); 12 | _Fun([C1|R1], [C2|R2]) when is_list(C1), is_integer(C2) -> true; 13 | _Fun(_, _) -> false 14 | end, 15 | 16 | SortFun = fun(T1, T2) -> 17 | C = fun(T) -> 18 | [case catch list_to_integer(E) of 19 | I when is_integer(I) -> I; 20 | _ -> E 21 | end || E <- re:split(string:sub_string(T, 2), "[.-]", [{return, list}])] 22 | end, 23 | ComparingFun(C(T1), C(T2)) 24 | end, 25 | 26 | VTags = string:tokens(os:cmd("git tag -l \"v*\" --points-at $(git rev-parse $(git describe --abbrev=0 --tags))"), "\n"), 27 | 28 | Tags = case VTags of 29 | [] -> string:tokens(os:cmd("git tag -l \"e*\" --points-at $(git rev-parse $(git describe --abbrev=0 --tags))"), "\n"); 30 | _ -> VTags 31 | end, 32 | 33 | LatestTag = lists:last(lists:sort(SortFun, Tags)), 34 | 35 | Branch = case os:getenv("GITHUB_RUN_ID") of 36 | false -> os:cmd("git branch | grep -e '^*' | cut -d' ' -f 2") -- "\n"; 37 | _ -> re:replace(os:getenv("GITHUB_REF"), "^refs/heads/|^refs/tags/", "", [global, {return ,list}]) 38 | end, 39 | 40 | GitDescribe = case re:run(Branch, "master|^dev/|^hotfix/", [{capture, none}]) of 41 | match -> {branch, Branch}; 42 | _ -> {tag, LatestTag} 43 | end, 44 | 45 | UrlPrefix = "https://github.com/emqx/", 46 | 47 | %% EMQX_DEP = {emqx, {git, UrlPrefix ++ "emqx", GitDescribe}}, 48 | EMQX_MGMT_DEP = {emqx_management, {git, UrlPrefix ++ "emqx-management", GitDescribe}}, 49 | 50 | %% NewDeps = [EMQX_DEP, EMQX_MGMT_DEP | DEPS], 51 | NewDeps = [EMQX_MGMT_DEP | DEPS], 52 | 53 | CONFIG1 = lists:keystore(deps, 1, CONFIG, {deps, NewDeps}), 54 | 55 | CONFIG1. 56 | -------------------------------------------------------------------------------- /src/emqx_cli_demo_kafka.erl: -------------------------------------------------------------------------------- 1 | %%-------------------------------------------------------------------- 2 | %% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%-------------------------------------------------------------------- 16 | 17 | -module(emqx_cli_demo_kafka). 18 | 19 | -export([cmd/1]). 20 | 21 | cmd(["arg1", "arg2"]) -> 22 | emqx_ctl:print("ok"); 23 | 24 | cmd(_) -> 25 | emqx_ctl:usage([{"cmd arg1 arg2", "cmd demo"}]). 26 | 27 | -------------------------------------------------------------------------------- /src/emqx_plugin_kafka.app.src: -------------------------------------------------------------------------------- 1 | {application, emqx_plugin_kafka, 2 | [{description, "EMQ X plugin kafka bridge, Edit by ultrakid, original coder: yzs-lyg"}, 3 | {vsn, "git"}, 4 | {modules, []}, 5 | {registered, [emqx_plugin_kafka_sup]}, 6 | {applications, [kernel,stdlib, brod]}, 7 | {mod, {emqx_plugin_kafka_app,[]}}, 8 | {env, []}, 9 | {licenses, ["Apache-2.0"]}, 10 | {maintainers, ["ultrakid ", "lyg <3613840847@qq.com>"]}, 11 | {links,[{"Github","https://github.com/ULTRAKID/emqx_plugin_kafka"}, {"Github","https://github.com/yzs/emqx-plugin-kafka"}]} 12 | ] 13 | }. 14 | 15 | -------------------------------------------------------------------------------- /src/emqx_plugin_kafka.app.src.script: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% .app.src.script 3 | 4 | RemoveLeadingV = 5 | fun(Tag) -> 6 | case re:run(Tag, "v\[0-9\]+\.\[0-9\]+\.*") of 7 | nomatch -> 8 | Tag; 9 | {match, _} -> 10 | %% if it is a version number prefixed by 'v' then remove the 'v' 11 | "v" ++ Vsn = Tag, 12 | Vsn 13 | end 14 | end, 15 | 16 | case os:getenv("EMQX_DEPS_DEFAULT_VSN") of 17 | false -> CONFIG; % env var not defined 18 | [] -> CONFIG; % env var set to empty string 19 | Tag -> 20 | [begin 21 | AppConf0 = lists:keystore(vsn, 1, AppConf, {vsn, RemoveLeadingV(Tag)}), 22 | {application, App, AppConf0} 23 | end || Conf = {application, App, AppConf} <- CONFIG] 24 | end. 25 | 26 | -------------------------------------------------------------------------------- /src/emqx_plugin_kafka.erl: -------------------------------------------------------------------------------- 1 | %%-------------------------------------------------------------------- 2 | %% Copyright (c) 2015-2017 Feng Lee . 3 | %% 4 | %% Modified by Ramez Hanna 5 | %% 6 | %% Licensed under the Apache License, Version 2.0 (the "License"); 7 | %% you may not use this file except in compliance with the License. 8 | %% You may obtain a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, software 13 | %% distributed under the License is distributed on an "AS IS" BASIS, 14 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | %% See the License for the specific language governing permissions and 16 | %% limitations under the License. 17 | %%-------------------------------------------------------------------- 18 | 19 | -module(emqx_plugin_kafka). 20 | 21 | % -include("emqx_plugin_kafka.hrl"). 22 | 23 | % -include_lib("emqx/include/emqx.hrl"). 24 | 25 | -include("emqx.hrl"). 26 | -include_lib("kernel/include/logger.hrl"). 27 | 28 | -export([load/1, unload/0]). 29 | 30 | %% Client Lifecircle Hooks 31 | -export([on_client_connect/3 32 | , on_client_connack/4 33 | , on_client_connected/3 34 | , on_client_disconnected/4 35 | , on_client_authenticate/3 36 | , on_client_check_acl/5 37 | , on_client_subscribe/4 38 | , on_client_unsubscribe/4 39 | ]). 40 | 41 | %% Session Lifecircle Hooks 42 | -export([on_session_created/3 43 | , on_session_subscribed/4 44 | , on_session_unsubscribed/4 45 | , on_session_resumed/3 46 | , on_session_discarded/3 47 | , on_session_takeovered/3 48 | , on_session_terminated/4 49 | ]). 50 | 51 | %% Message Pubsub Hooks 52 | -export([on_message_publish/2 53 | , on_message_delivered/3 54 | , on_message_acked/3 55 | , on_message_dropped/4 56 | ]). 57 | 58 | 59 | %% Called when the plugin application start 60 | load(Env) -> 61 | kafka_init([Env]), 62 | emqx:hook('client.connect', {?MODULE, on_client_connect, [Env]}), 63 | emqx:hook('client.connack', {?MODULE, on_client_connack, [Env]}), 64 | emqx:hook('client.connected', {?MODULE, on_client_connected, [Env]}), 65 | emqx:hook('client.disconnected', {?MODULE, on_client_disconnected, [Env]}), 66 | emqx:hook('client.authenticate', {?MODULE, on_client_authenticate, [Env]}), 67 | emqx:hook('client.check_acl', {?MODULE, on_client_check_acl, [Env]}), 68 | emqx:hook('client.subscribe', {?MODULE, on_client_subscribe, [Env]}), 69 | emqx:hook('client.unsubscribe', {?MODULE, on_client_unsubscribe, [Env]}), 70 | emqx:hook('session.created', {?MODULE, on_session_created, [Env]}), 71 | emqx:hook('session.subscribed', {?MODULE, on_session_subscribed, [Env]}), 72 | emqx:hook('session.unsubscribed', {?MODULE, on_session_unsubscribed, [Env]}), 73 | emqx:hook('session.resumed', {?MODULE, on_session_resumed, [Env]}), 74 | emqx:hook('session.discarded', {?MODULE, on_session_discarded, [Env]}), 75 | emqx:hook('session.takeovered', {?MODULE, on_session_takeovered, [Env]}), 76 | emqx:hook('session.terminated', {?MODULE, on_session_terminated, [Env]}), 77 | emqx:hook('message.publish', {?MODULE, on_message_publish, [Env]}), 78 | emqx:hook('message.delivered', {?MODULE, on_message_delivered, [Env]}), 79 | emqx:hook('message.acked', {?MODULE, on_message_acked, [Env]}), 80 | emqx:hook('message.dropped', {?MODULE, on_message_dropped, [Env]}). 81 | 82 | on_client_connect(ConnInfo = #{clientid := ClientId}, Props, _Env) -> 83 | ?LOG_INFO("[KAFKA PLUGIN]Client(~s) connect, ConnInfo: ~p, Props: ~p~n", 84 | [ClientId, ConnInfo, Props]), 85 | ok. 86 | 87 | on_client_connack(ConnInfo = #{clientid := ClientId}, Rc, Props, _Env) -> 88 | ?LOG_INFO("[KAFKA PLUGIN]Client(~s) connack, ConnInfo: ~p, Rc: ~p, Props: ~p~n", 89 | [ClientId, ConnInfo, Rc, Props]), 90 | ok. 91 | 92 | on_client_connected(ClientInfo = #{clientid := ClientId}, ConnInfo, _Env) -> 93 | ?LOG_INFO("[KAFKA PLUGIN]Client(~s) connected, ClientInfo:~n~p~n, ConnInfo:~n~p~n", 94 | [ClientId, ClientInfo, ConnInfo]), 95 | {IpAddr, _Port} = maps:get(peername, ConnInfo), 96 | Action = <<"connected">>, 97 | Now = now_mill_secs(), 98 | Online = 1, 99 | Payload = [ 100 | {action, Action}, 101 | {device_id, ClientId}, 102 | {username, maps:get(username, ClientInfo)}, 103 | {keepalive, maps:get(keepalive, ConnInfo)}, 104 | {ipaddress, iolist_to_binary(ntoa(IpAddr))}, 105 | {proto_name, maps:get(proto_name, ConnInfo)}, 106 | {proto_ver, maps:get(proto_ver, ConnInfo)}, 107 | {ts, Now}, 108 | {online, Online} 109 | ], 110 | produce_kafka_payload(ClientId, Payload), 111 | ok. 112 | 113 | on_client_disconnected(ClientInfo = #{clientid := ClientId}, ReasonCode, ConnInfo, _Env) -> 114 | ?LOG_INFO("[KAFKA PLUGIN]Client(~s) disconnected due to ~p, ClientInfo:~n~p~n, ConnInfo:~n~p~n", 115 | [ClientId, ReasonCode, ClientInfo, ConnInfo]), 116 | Action = <<"disconnected">>, 117 | Now = now_mill_secs(), 118 | Online = 0, 119 | Payload = [ 120 | {action, Action}, 121 | {device_id, ClientId}, 122 | {username, maps:get(username, ClientInfo)}, 123 | {reason, ReasonCode}, 124 | {ts, Now}, 125 | {online, Online} 126 | ], 127 | produce_kafka_payload(ClientId, Payload), 128 | ok. 129 | 130 | on_client_authenticate(_ClientInfo = #{clientid := ClientId}, Result, _Env) -> 131 | ?LOG_INFO("[KAFKA PLUGIN]Client(~s) authenticate, Result:~n~p~n", [ClientId, Result]), 132 | ok. 133 | 134 | on_client_check_acl(_ClientInfo = #{clientid := ClientId}, Topic, PubSub, Result, _Env) -> 135 | ?LOG_INFO("[KAFKA PLUGIN]Client(~s) check_acl, PubSub:~p, Topic:~p, Result:~p~n", 136 | [ClientId, PubSub, Topic, Result]), 137 | ok. 138 | 139 | %%---------------------------client subscribe start--------------------------%% 140 | on_client_subscribe(#{clientid := ClientId}, _Properties, TopicFilters, _Env) -> 141 | ?LOG_INFO("[KAFKA PLUGIN]Client(~s) will subscribe: ~p~n", [ClientId, TopicFilters]), 142 | Topic = erlang:element(1, erlang:hd(TopicFilters)), 143 | Qos = erlang:element(2, lists:last(TopicFilters)), 144 | Action = <<"subscribe">>, 145 | Now = now_mill_secs(), 146 | Payload = [ 147 | {device_id, ClientId}, 148 | {action, Action}, 149 | {topic, Topic}, 150 | {qos, maps:get(qos, Qos)}, 151 | {ts, Now} 152 | ], 153 | produce_kafka_payload(ClientId, Payload), 154 | ok. 155 | %%---------------------client subscribe stop----------------------%% 156 | on_client_unsubscribe(#{clientid := ClientId}, _Properties, TopicFilters, _Env) -> 157 | ?LOG_INFO("[KAFKA PLUGIN]Client(~s) will unsubscribe ~p~n", [ClientId, TopicFilters]), 158 | Topic = erlang:element(1, erlang:hd(TopicFilters)), 159 | Action = <<"unsubscribe">>, 160 | Now = now_mill_secs(), 161 | Payload = [ 162 | {device_id, ClientId}, 163 | {action, Action}, 164 | {topic, Topic}, 165 | {ts, Now} 166 | ], 167 | produce_kafka_payload(ClientId, Payload), 168 | ok. 169 | 170 | on_message_dropped(#message{topic = <<"$SYS/", _/binary>>}, _By, _Reason, _Env) -> 171 | ok; 172 | on_message_dropped(Message, _By = #{node := Node}, Reason, _Env) -> 173 | ?LOG_INFO("[KAFKA PLUGIN]Message dropped by node ~s due to ~s: ~s~n", 174 | [Node, Reason, emqx_message:format(Message)]), 175 | ok. 176 | 177 | 178 | %%---------------------------message publish start--------------------------%% 179 | on_message_publish(Message = #message{topic = <<"$SYS/", _/binary>>}, _Env) -> 180 | ok; 181 | on_message_publish(Message, _Env) -> 182 | {ok, ClientId, Payload} = format_payload(Message), 183 | produce_kafka_payload(ClientId, Payload), 184 | ok. 185 | %%---------------------message publish stop----------------------%% 186 | 187 | on_message_delivered(_ClientInfo = #{clientid := ClientId}, Message, _Env) -> 188 | ?LOG_INFO("[KAFKA PLUGIN]Message delivered to client(~s): ~s~n", 189 | [ClientId, emqx_message:format(Message)]), 190 | Topic = Message#message.topic, 191 | Payload = transform_payload(Message#message.payload), 192 | Qos = Message#message.qos, 193 | From = Message#message.from, 194 | Timestamp = Message#message.timestamp, 195 | Content = [ 196 | {action, <<"message_delivered">>}, 197 | {from, From}, 198 | {to, ClientId}, 199 | {topic, Topic}, 200 | {payload, Payload}, 201 | {qos, Qos}, 202 | {cluster_node, node()}, 203 | {ts, Timestamp} 204 | ], 205 | produce_kafka_payload(ClientId, Content), 206 | ok. 207 | 208 | on_message_acked(_ClientInfo = #{clientid := ClientId}, Message, _Env) -> 209 | ?LOG_INFO("[KAFKA PLUGIN]Message acked by client(~s): ~s~n", 210 | [ClientId, emqx_message:format(Message)]), 211 | Topic = Message#message.topic, 212 | Payload = transform_payload(Message#message.payload), 213 | Qos = Message#message.qos, 214 | From = Message#message.from, 215 | Timestamp = Message#message.timestamp, 216 | Content = [ 217 | {action, <<"message_acked">>}, 218 | {from, From}, 219 | {to, ClientId}, 220 | {topic, Topic}, 221 | {payload, Payload}, 222 | {qos, Qos}, 223 | {cluster_node, node()}, 224 | {ts, Timestamp} 225 | ], 226 | produce_kafka_payload(ClientId, Content), 227 | ok. 228 | 229 | %%-------------------------------------------------------------------- 230 | %% Session Lifecircle Hooks 231 | %%-------------------------------------------------------------------- 232 | 233 | on_session_created(#{clientid := ClientId}, SessInfo, _Env) -> 234 | ?LOG_INFO("[KAFKA PLUGIN]Session(~s) created, Session Info:~n~p~n", [ClientId, SessInfo]), 235 | ok. 236 | 237 | 238 | on_session_subscribed(#{clientid := ClientId}, Topic, SubOpts, _Env) -> 239 | ?LOG_INFO("[KAFKA PLUGIN]Session(~s) subscribed ~s with subopts: ~p~n", [ClientId, Topic, SubOpts]), 240 | ok. 241 | 242 | on_session_unsubscribed(#{clientid := ClientId}, Topic, Opts, _Env) -> 243 | ?LOG_INFO("[KAFKA PLUGIN]Session(~s) unsubscribed ~s with opts: ~p~n", [ClientId, Topic, Opts]), 244 | ok. 245 | 246 | on_session_resumed(#{clientid := ClientId}, SessInfo, _Env) -> 247 | ?LOG_INFO("[KAFKA PLUGIN]Session(~s) resumed, Session Info:~n~p~n", [ClientId, SessInfo]), 248 | ok. 249 | 250 | on_session_discarded(_ClientInfo = #{clientid := ClientId}, SessInfo, _Env) -> 251 | ?LOG_INFO("[KAFKA PLUGIN]Session(~s) is discarded. Session Info: ~p~n", [ClientId, SessInfo]), 252 | ok. 253 | 254 | on_session_takeovered(_ClientInfo = #{clientid := ClientId}, SessInfo, _Env) -> 255 | ?LOG_INFO("[KAFKA PLUGIN]Session(~s) is takeovered. Session Info: ~p~n", [ClientId, SessInfo]), 256 | ok. 257 | 258 | on_session_terminated(_ClientInfo = #{clientid := ClientId}, Reason, SessInfo, _Env) -> 259 | ?LOG_INFO("[KAFKA PLUGIN]Session(~s) is terminated due to ~p~nSession Info: ~p~n", 260 | [ClientId, Reason, SessInfo]), 261 | ok. 262 | 263 | kafka_init(_Env) -> 264 | ?LOG_INFO("Start to init emqx plugin kafka..... ~n"), 265 | {ok, AddressList} = application:get_env(emqx_plugin_kafka, kafka_address_list), 266 | ?LOG_INFO("[KAFKA PLUGIN]KafkaAddressList = ~p~n", [AddressList]), 267 | {ok, KafkaConfig} = application:get_env(emqx_plugin_kafka, kafka_config), 268 | ?LOG_INFO("[KAFKA PLUGIN]KafkaConfig = ~p~n", [KafkaConfig]), 269 | {ok, KafkaTopic} = application:get_env(emqx_plugin_kafka, topic), 270 | ?LOG_INFO("[KAFKA PLUGIN]KafkaTopic = ~s~n", [KafkaTopic]), 271 | {ok, _} = application:ensure_all_started(brod), 272 | ok = brod:start_client(AddressList, emqx_repost_worker, KafkaConfig), 273 | ok = brod:start_producer(emqx_repost_worker, KafkaTopic, []), 274 | ?LOG_INFO("Init emqx plugin kafka successfully.....~n"), 275 | ok. 276 | 277 | get_kafka_topic() -> 278 | {ok, Topic} = application:get_env(emqx_plugin_kafka, topic), 279 | Topic. 280 | 281 | need_base64() -> 282 | {ok, NeedBase64} = application:get_env(emqx_plugin_kafka, publish_base64), 283 | NeedBase64. 284 | 285 | transform_payload(Payload) -> 286 | NeedBase64 = need_base64(), 287 | if 288 | NeedBase64 == true -> 289 | Content = list_to_binary(base64:encode_to_string(Payload)); 290 | NeedBase64 == false -> 291 | Content = Payload 292 | end, 293 | Content. 294 | 295 | 296 | format_payload(Message) -> 297 | Username = emqx_message:get_header(username, Message), 298 | Topic = Message#message.topic, 299 | % ?LOG_INFO("[KAFKA PLUGIN]Tail= ~s , RawType= ~s~n",[Tail,RawType]), 300 | ClientId = Message#message.from, 301 | Content = transform_payload(Message#message.payload), 302 | Payload = [{action, message_publish}, 303 | {device_id, ClientId}, 304 | {username, Username}, 305 | {topic, Topic}, 306 | {payload, Content}, 307 | {ts, Message#message.timestamp}], 308 | 309 | {ok, ClientId, Payload}. 310 | 311 | 312 | %% Called when the plugin application stop 313 | unload() -> 314 | emqx:unhook('client.connect', {?MODULE, on_client_connect}), 315 | emqx:unhook('client.connack', {?MODULE, on_client_connack}), 316 | emqx:unhook('client.connected', {?MODULE, on_client_connected}), 317 | emqx:unhook('client.disconnected', {?MODULE, on_client_disconnected}), 318 | emqx:unhook('client.authenticate', {?MODULE, on_client_authenticate}), 319 | emqx:unhook('client.check_acl', {?MODULE, on_client_check_acl}), 320 | emqx:unhook('client.subscribe', {?MODULE, on_client_subscribe}), 321 | emqx:unhook('client.unsubscribe', {?MODULE, on_client_unsubscribe}), 322 | emqx:unhook('session.created', {?MODULE, on_session_created}), 323 | emqx:unhook('session.subscribed', {?MODULE, on_session_subscribed}), 324 | emqx:unhook('session.unsubscribed', {?MODULE, on_session_unsubscribed}), 325 | emqx:unhook('session.resumed', {?MODULE, on_session_resumed}), 326 | emqx:unhook('session.discarded', {?MODULE, on_session_discarded}), 327 | emqx:unhook('session.takeovered', {?MODULE, on_session_takeovered}), 328 | emqx:unhook('session.terminated', {?MODULE, on_session_terminated}), 329 | emqx:unhook('message.publish', {?MODULE, on_message_publish}), 330 | emqx:unhook('message.delivered', {?MODULE, on_message_delivered}), 331 | emqx:unhook('message.acked', {?MODULE, on_message_acked}), 332 | emqx:unhook('message.dropped', {?MODULE, on_message_dropped}). 333 | 334 | produce_kafka_payload(Key, Message) -> 335 | Topic = get_kafka_topic(), 336 | {ok, MessageBody} = emqx_json:safe_encode(Message), 337 | % ?LOG_INFO("[KAFKA PLUGIN]Message = ~s~n",[MessageBody]), 338 | Payload = iolist_to_binary(MessageBody), 339 | brod:produce_cb(emqx_repost_worker, Topic, hash, Key, Payload, fun(_,_) -> ok end), 340 | ok. 341 | 342 | ntoa({0, 0, 0, 0, 0, 16#ffff, AB, CD}) -> 343 | inet_parse:ntoa({AB bsr 8, AB rem 256, CD bsr 8, CD rem 256}); 344 | ntoa(IP) -> 345 | inet_parse:ntoa(IP). 346 | now_mill_secs() -> 347 | erlang:system_time(millisecond). 348 | -------------------------------------------------------------------------------- /src/emqx_plugin_kafka_app.erl: -------------------------------------------------------------------------------- 1 | %%-------------------------------------------------------------------- 2 | %% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%-------------------------------------------------------------------- 16 | 17 | -module(emqx_plugin_kafka_app). 18 | 19 | -behaviour(application). 20 | 21 | -emqx_plugin(?MODULE). 22 | 23 | -export([ start/2 24 | , stop/1 25 | ]). 26 | 27 | start(_StartType, _StartArgs) -> 28 | {ok, Sup} = emqx_plugin_kafka_sup:start_link(), 29 | emqx_plugin_kafka:load(application:get_all_env()), 30 | {ok, Sup}. 31 | 32 | stop(_State) -> 33 | emqx_plugin_kafka:unload(). 34 | 35 | -------------------------------------------------------------------------------- /src/emqx_plugin_kafka_sup.erl: -------------------------------------------------------------------------------- 1 | %%-------------------------------------------------------------------- 2 | %% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%-------------------------------------------------------------------- 16 | 17 | -module(emqx_plugin_kafka_sup). 18 | 19 | -behaviour(supervisor). 20 | 21 | -export([start_link/0]). 22 | 23 | -export([init/1]). 24 | 25 | start_link() -> 26 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 27 | 28 | init([]) -> 29 | {ok, { {one_for_all, 0, 1}, []} }. 30 | 31 | -------------------------------------------------------------------------------- /test/emqx_plugin_kafka_SUITE.erl: -------------------------------------------------------------------------------- 1 | %%-------------------------------------------------------------------- 2 | %% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved. 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%-------------------------------------------------------------------- 16 | 17 | -module(emqx_plugin_kafka_SUITE). 18 | 19 | -compile(export_all). 20 | 21 | all() -> []. 22 | 23 | groups() -> []. 24 | --------------------------------------------------------------------------------