├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── erlang环境配置.md ├── etc └── emqx_backend_mysql.conf ├── include └── emqx_backend_mysql.hrl ├── priv └── emqx_backend_mysql.schema ├── rebar.config ├── rebar.config.script ├── rebar3.crashdump ├── sql ├── mqtt_acked.sql ├── mqtt_client.sql ├── mqtt_msg.sql ├── mqtt_retain.sql └── mqtt_sub.sql ├── src ├── emqx_backend_mysql.app ├── emqx_backend_mysql.app.src ├── emqx_backend_mysql.app.src.script ├── emqx_backend_mysql.erl ├── emqx_backend_mysql_actions.erl ├── emqx_backend_mysql_app.erl ├── emqx_backend_mysql_batcher.erl ├── emqx_backend_mysql_cli.erl └── emqx_backend_mysql_sup.erl └── test └── emqx_backend_mysql_SUITE.erl /.gitignore: -------------------------------------------------------------------------------- 1 | # eclipse ignore 2 | .settings/ 3 | .project 4 | .classpath 5 | 6 | # idea ignore 7 | .idea/ 8 | *.ipr 9 | *.iml 10 | *.iws 11 | 12 | # system ignore 13 | .DS_Store 14 | Thumbs.db 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ## shallow clone for speed 2 | 3 | REBAR_GIT_CLONE_OPTIONS += --depth 1 4 | export REBAR_GIT_CLONE_OPTIONS 5 | 6 | REBAR = rebar3 7 | all: compile 8 | 9 | compile: 10 | $(REBAR) compile 11 | 12 | ct: compile 13 | $(REBAR) as test ct -v 14 | 15 | eunit: compile 16 | $(REBAR) as test eunit 17 | 18 | xref: 19 | $(REBAR) xref 20 | 21 | cover: 22 | $(REBAR) cover 23 | 24 | clean: distclean 25 | 26 | distclean: 27 | @rm -rf _build 28 | @rm -f data/app.*.config data/vm.*.args rebar.lock 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # buildrun-emqx-backend-mysql 2 | EMQX client connection and message save to MySQL 3 | 4 | emqx 客户端连接状态和消息持久化到MySQL 插件 5 | 6 | ### erlang 环境配置 7 | 8 | 若没配置erlang环境，请参考[erlang 环境配置](./erlang环境配置.md). 9 | 10 | ### 编译发布插件 11 | 12 | 1、clone emqx-rel 项目, 切换到改 tag v4.1.2 13 | 14 | ``` bash 15 | git clone https://github.com/emqx/emqx-rel.git 16 | cd emqx-rel 17 | git checkout v4.1.2 18 | make 19 | ``` 20 | 21 | 正常构建成功后，运行 `_build/emqx/rel/emqx/bin/emqx console` 启动 emqx 22 | 23 | ### 添加自定义插件 24 | 25 | 2.rebar.config 添加依赖 26 | 27 | ```erl 28 | {deps, 29 | [ {buildrun_emqx_backend_mysql, {git, "https://github.com/goBuildRun/buildrun-emqx-backend-mysql.git", {branch, "master"}}} 30 | , .... 31 | .... 32 | ] 33 | } 34 | 35 | ``` 36 | 37 | 3.rebar.config 中 relx 段落添加 38 | 39 | ```erl 40 | {relx, 41 | [... 42 | , ... 43 | , {release, {emqx, git_describe}, 44 | [ 45 | {buildrun_emqx_backend_mysql, load}, 46 | ] 47 | } 48 | ] 49 | } 50 | ``` 51 | 4.编译 52 | 53 | > make 54 | 55 | 打开 `localhost:18083` 的插件栏可以看到 `buildrun_emqx_backend_mysql` 插件。 56 | 57 | ### config配置 58 | 59 | File: etc/buildrun_emqx_backend_mysql.conf 60 | 61 | ``` 62 | # mysql 服务器 63 | mysql.server = 127.0.0.1:3306 64 | 65 | # 连接池数量 66 | mysql.pool_size = 8 67 | 68 | # mysql 用户名 69 | mysql.username = buildrun 70 | 71 | # mysql密码 72 | mysql.password = buildrun 73 | 74 | # 数据库名 75 | mysql.database = mqtt 76 | 77 | # 超时时间（秒） 78 | mysql.query_timeout = 10s 79 | 80 | ``` 81 | 82 | ### mqtt_client.sql, mqtt_msg.sql 83 | 84 | 到你的数据库中执行[mqtt_client.sql](./sql/mqtt_client.sql), [mqtt_msg.sql](./sql/mqtt_msg.sql)。 85 | 86 | ### 加载插件 87 | 88 | > ./bin/emqx_ctl plugins load buildrun_emqx_backend_mysql 89 | 90 | 或者编辑 91 | 92 | data/loaded_plugins 93 | 94 | > 添加 {buildrun_emqx_backend_mysql, true}. 95 | 96 | 设置插件自动启动 97 | 98 | 注意：这种方式适用emqx未启动之前 99 | 100 | 或者 101 | 102 | 可以在emqx启动后，在 dashboard 插件栏点击启用 `buildrun_emqx_backend_mysql` 即可启用成功 103 | 104 | ### 使用 105 | 106 | 此插件会把public发布的消息保存到mysql中，但并不是全部。也可以在业务规则中添加存储类型的资源。 107 | 108 | 需要在发布消息的参数中 retain 值设置为 true。 这样这条消息才会被保存在mysql中 109 | 110 | eg： 111 | 112 | ```json 113 | { 114 | "topic": "buildruntopic", 115 | "payload": "hello,Buidrun", 116 | "qos": 1, 117 | "retain": true, 118 | "client_id": "mqttjs_8b3b4182ae" 119 | } 120 | ``` 121 | 122 | ### 构建镜像 123 | 124 | `emqx-rel` 项目自带构建镜像的功能 125 | 126 | ``` bash 127 | make emqx-docker-build 128 | ``` 129 | 130 | ### 最后 131 | 132 | 有什么问题和功能需求都可以给我提issue，欢迎关注。 133 | 134 | ### License 135 | 136 | Apache License Version 2.0 137 | -------------------------------------------------------------------------------- /erlang环境配置.md: -------------------------------------------------------------------------------- 1 | # erlang环境配置 2 | 3 | erlang 推荐使用 22 版本，经过测试用 23 版本编译出来运行有问题。各平台安装 erlang 详情请参考官方文档。 linux 系统可以使用 kerl 脚本安装，构建最新版本的 emqx 时注意编译的 gcc 版本不能用 10，编译通不过，推荐用 gcc-9 4 | 5 | 下载 kerl 6 | 7 | ``` bash 8 | curl -O https://raw.githubusercontent.com/kerl/kerl/master/kerl 9 | chmod +x kerl 10 | ``` 11 | 12 | 安装 erlang 13 | 14 | ``` bash 15 | kerl update releases 16 | # 如果直接构建报错，修改 ~/.kerl/builds/22.3/otp_src_22.3/Makefile 17 | # 将 gcc 改为 gcc9 18 | kerl build 22.3 22.3 19 | kerl install 22.3 /opt/erlang/22.3 20 | # 启用环境 21 | source /opt/erlang/22.3/activate 22 | # 设置环境变量，这样不用每次手动启用 23 | echo 'source /opt/erlang/22.3/activate' >> ~/.bashrc 24 | ``` 25 | 26 | 安装 rebar3 27 | 28 | ``` bash 29 | curl -OL https://s3.amazonaws.com/rebar3/rebar3 30 | ./rebar3 local install 31 | # 然后根据输出操作 32 | ``` 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /etc/emqx_backend_mysql.conf: -------------------------------------------------------------------------------- 1 | ##==================================================================== 2 | ## Configuration for EMQ X MySQL Backend 3 | ##==================================================================== 4 | 5 | ## MySQL Server 6 | backend.mysql.pool1.server = 127.0.0.1:3306 7 | 8 | ## MySQL Pool Size 9 | backend.mysql.pool1.pool_size = 8 10 | ## MySQL Username 11 | backend.mysql.pool1.user = root 12 | ## MySQL Password 13 | backend.mysql.pool1.password = public 14 | ## MySQL Database 15 | backend.mysql.pool1.database = mqtt 16 | 17 | ## Client Connected Record 18 | backend.mysql.hook.client.connected.1 = {"action": {"function": "on_client_connected"}, "pool": "pool1"} 19 | 20 | ## Session Created Record 21 | backend.mysql.hook.client.connected.2 = {"action": {"function": "on_subscribe_lookup"}, "pool": "pool1"} 22 | 23 | ## Client DisConnected Record 24 | backend.mysql.hook.client.disconnected.1 = {"action": {"function": "on_client_disconnected"}, "pool": "pool1"} 25 | 26 | ## Lookup Unread Message QOS > 0 27 | backend.mysql.hook.session.subscribed.1 = {"topic": "#", "action": {"function": "on_message_fetch"}, "offline_opts": {"max_returned_count": 500, "time_range": "2h"}, "pool": "pool1"} 28 | 29 | ## Lookup Retain Message 30 | backend.mysql.hook.session.subscribed.2 = {"topic": "#", "action": {"function": "on_retain_lookup"}, "pool": "pool1"} 31 | 32 | ## Delete Acked Record 33 | backend.mysql.hook.session.unsubscribed.1= {"topic": "#", "action": {"sql": ["delete from mqtt_acked where clientid = ${clientid} and topic = ${topic}"]}, "pool": "pool1"} 34 | 35 | ## Store Publish Message QOS > 0 36 | backend.mysql.hook.message.publish.1 = {"topic": "#", "action": {"function": "on_message_publish"}, "pool": "pool1"} 37 | 38 | ## Store Retain Message 39 | backend.mysql.hook.message.publish.2 = {"topic": "#", "action": {"function": "on_message_retain"}, "pool": "pool1"} 40 | 41 | ## Delete Retain Message 42 | backend.mysql.hook.message.publish.3 = {"topic": "#", "action": {"function": "on_retain_delete"}, "pool": "pool1"} 43 | 44 | ## Store Ack 45 | backend.mysql.hook.message.acked.1 = {"topic": "#", "action": {"function": "on_message_acked"}, "pool": "pool1"} 46 | 47 | ## Max number of fetch offline messages 48 | ## max_returned_count = 500 49 | 50 | ## Time Range 51 | ## d - day 52 | ## h - hour 53 | ## m - minute 54 | ## s - second 55 | ## time_range = 2h 56 | ## backend.mysql.hook.session.subscribed.1 = {"topic": "#", "action": {"function": "on_message_fetch"}, "offline_opts": {"max_returned_count": 500, "time_range": "2h"}, "pool": "pool1"} -------------------------------------------------------------------------------- /include/emqx_backend_mysql.hrl: -------------------------------------------------------------------------------- 1 | -define(APP, emqx_backend_mysql). 2 | -------------------------------------------------------------------------------- /priv/emqx_backend_mysql.schema: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% emqx_backend_mysql config mapping 3 | 4 | {mapping, "backend.mysql.$name.server", "emqx_backend_mysql.pools", [ 5 | {default, {"127.0.0.1", 3306}}, 6 | {datatype, [integer, ip, string]} 7 | ]}. 8 | 9 | {mapping, "backend.mysql.$name.pool_size", "emqx_backend_mysql.pools", [ 10 | {default, 8}, 11 | {datatype, integer} 12 | ]}. 13 | 14 | {mapping, "backend.mysql.$name.user", "emqx_backend_mysql.pools", [ 15 | {default, ""}, 16 | {datatype, string}, 17 | hidden 18 | ]}. 19 | 20 | {mapping, "backend.mysql.$name.password", "emqx_backend_mysql.pools", [ 21 | {default, ""}, 22 | {datatype, string} 23 | ]}. 24 | 25 | {mapping, "backend.mysql.$name.database", "emqx_backend_mysql.pools", [ 26 | {default, ""}, 27 | {datatype, string} 28 | ]}. 29 | 30 | {translation, "emqx_backend_mysql.pools", fun(Conf) -> 31 | Vars = cuttlefish_variable:fuzzy_matches(["backend", "mysql", "$name", "server"], Conf), 32 | Key = fun(Name, Attr) -> string:join(["backend.mysql", Name, Attr], ".") end, 33 | lists:map(fun({_, Name}) -> 34 | {MyHost, MyPort} = 35 | case cuttlefish:conf_get(Key(Name, "server"), Conf) of 36 | {Ip, Port} -> {Ip, Port}; 37 | S -> case string:tokens(S, ":") of 38 | [Domain] -> {Domain, 3306}; 39 | [Domain, Port] -> {Domain, list_to_integer(Port)} 40 | end 41 | end, 42 | {list_to_atom(Name), [{host, MyHost}, {port, MyPort}, 43 | {pool_size, cuttlefish:conf_get(Key(Name, "pool_size"), Conf)}, 44 | {user, cuttlefish:conf_get(Key(Name, "user"), Conf)}, 45 | {password, cuttlefish:conf_get(Key(Name, "password"), Conf)}, 46 | {database, cuttlefish:conf_get(Key(Name, "database"), Conf)}, 47 | {auto_reconnect, 1}, 48 | {encoding, utf8}, 49 | {keep_alive, true}]} 50 | end, Vars) 51 | end}. 52 | 53 | {mapping, "backend.mysql.hook.client.connected.$name", "emqx_backend_mysql.hooks", [ 54 | {datatype, string} 55 | ]}. 56 | 57 | {mapping, "backend.mysql.hook.client.disconnected.$name", "emqx_backend_mysql.hooks", [ 58 | {datatype, string} 59 | ]}. 60 | 61 | {mapping, "backend.mysql.hook.session.subscribed.$name", "emqx_backend_mysql.hooks", [ 62 | {datatype, string} 63 | ]}. 64 | 65 | {mapping, "backend.mysql.hook.session.unsubscribed.$name", "emqx_backend_mysql.hooks", [ 66 | {datatype, string} 67 | ]}. 68 | 69 | {mapping, "backend.mysql.hook.message.publish.$name", "emqx_backend_mysql.hooks", [ 70 | {datatype, string} 71 | ]}. 72 | 73 | {mapping, "backend.mysql.hook.message.acked.$name", "emqx_backend_mysql.hooks", [ 74 | {datatype, string} 75 | ]}. 76 | 77 | {mapping, "backend.mysql.hook.message.delivered.$name", "emqx_backend_mysql.hooks", [ 78 | {datatype, string} 79 | ]}. 80 | 81 | 82 | {translation, "emqx_backend_mysql.hooks", fun(Conf) -> 83 | Hooks = cuttlefish_variable:filter_by_prefix("backend.mysql.hook", Conf), 84 | lists:map( 85 | fun({[_, _, _,Name1,Name2, _], Val}) -> 86 | {lists:concat([Name1,".",Name2]), list_to_binary(Val)} 87 | end, Hooks) 88 | end}. 89 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {deps, []}. 2 | 3 | {erl_opts, [debug_info]}. 4 | -------------------------------------------------------------------------------- /rebar.config.script: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | 3 | DEPS = case lists:keyfind(deps, 1, CONFIG) of 4 | {_, Deps} -> Deps; 5 | _ -> [] 6 | end, 7 | 8 | ComparingFun = fun 9 | _Fun([C1|R1], [C2|R2]) when is_list(C1), is_list(C2); 10 | is_integer(C1), is_integer(C2) -> C1 < C2 orelse _Fun(R1, R2); 11 | _Fun([C1|R1], [C2|R2]) when is_integer(C1), is_list(C2) -> _Fun(R1, R2); 12 | _Fun([C1|R1], [C2|R2]) when is_list(C1), is_integer(C2) -> true; 13 | _Fun(_, _) -> false 14 | end, 15 | 16 | SortFun = fun(T1, T2) -> 17 | C = fun(T) -> 18 | [case catch list_to_integer(E) of 19 | I when is_integer(I) -> I; 20 | _ -> E 21 | end || E <- re:split(string:sub_string(T, 2), "[.-]", [{return, list}])] 22 | end, 23 | ComparingFun(C(T1), C(T2)) 24 | end, 25 | 26 | VTags = string:tokens(os:cmd("git tag -l \"v*\" --points-at $(git rev-parse $(git describe --abbrev=0 --tags))"), "\n"), 27 | 28 | Tags = case VTags of 29 | [] -> string:tokens(os:cmd("git tag -l \"e*\" --points-at $(git rev-parse $(git describe --abbrev=0 --tags))"), "\n"); 30 | _ -> VTags 31 | end, 32 | 33 | LatestTag = lists:last(lists:sort(SortFun, Tags)), 34 | 35 | Branch = case os:getenv("GITHUB_RUN_ID") of 36 | false -> os:cmd("git branch | grep -e '^*' | cut -d' ' -f 2") -- "\n"; 37 | _ -> re:replace(os:getenv("GITHUB_REF"), "^refs/heads/|^refs/tags/", "", [global, {return ,list}]) 38 | end, 39 | 40 | GitDescribe = case re:run(Branch, "master|^dev/|^hotfix/", [{capture, none}]) of 41 | match -> {branch, Branch}; 42 | _ -> {tag, LatestTag} 43 | end, 44 | 45 | UrlPrefix = "https://github.com/emqx/", 46 | 47 | EMQX_DEP = {emqx, {git, UrlPrefix ++ "emqx", GitDescribe}}, 48 | EMQX_MGMT_DEP = {emqx_management, {git, UrlPrefix ++ "emqx-management", GitDescribe}}, 49 | 50 | NewDeps = [EMQX_DEP, EMQX_MGMT_DEP | DEPS], 51 | 52 | CONFIG1 = lists:keystore(deps, 1, CONFIG, {deps, NewDeps}), 53 | 54 | CONFIG1. 55 | -------------------------------------------------------------------------------- /rebar3.crashdump: -------------------------------------------------------------------------------- 1 | Error: {badmatch,{error,{1,file,undefined_script}}} 2 | [{rebar_config,consult_file_,1, 3 | [{file,"/private/tmp/rebar3-20191223-90221-8wanow/rebar3-3.13.0/src/rebar_config.erl"}, 4 | {line,230}]}, 5 | {rebar_app_discover,create_app_info,3, 6 | [{file,"/private/tmp/rebar3-20191223-90221-8wanow/rebar3-3.13.0/src/rebar_app_discover.erl"}, 7 | {line,356}]}, 8 | {rebar_app_discover,try_handle_app_src_file,5, 9 | [{file,"/private/tmp/rebar3-20191223-90221-8wanow/rebar3-3.13.0/src/rebar_app_discover.erl"}, 10 | {line,457}]}, 11 | {rebar_utils,filtermap,2, 12 | [{file,"/private/tmp/rebar3-20191223-90221-8wanow/rebar3-3.13.0/src/rebar_utils.erl"}, 13 | {line,113}]}, 14 | {rebar_app_discover,do,2, 15 | [{file,"/private/tmp/rebar3-20191223-90221-8wanow/rebar3-3.13.0/src/rebar_app_discover.erl"}, 16 | {line,26}]}, 17 | {rebar_prv_app_discovery,do,1, 18 | [{file,"/private/tmp/rebar3-20191223-90221-8wanow/rebar3-3.13.0/src/rebar_prv_app_discovery.erl"}, 19 | {line,38}]}, 20 | {rebar_core,do,2, 21 | [{file,"/private/tmp/rebar3-20191223-90221-8wanow/rebar3-3.13.0/src/rebar_core.erl"}, 22 | {line,154}]}, 23 | {rebar3,run_aux,2, 24 | [{file,"/private/tmp/rebar3-20191223-90221-8wanow/rebar3-3.13.0/src/rebar3.erl"}, 25 | {line,182}]}] 26 | 27 | -------------------------------------------------------------------------------- /sql/mqtt_acked.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS `mqtt_acked`; 2 | CREATE TABLE `mqtt_acked` ( 3 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 4 | `clientid` varchar(200) DEFAULT NULL, 5 | `topic` varchar(200) DEFAULT NULL, 6 | `mid` int(200) DEFAULT NULL, 7 | `created` timestamp NULL DEFAULT NULL, 8 | PRIMARY KEY (`id`), 9 | UNIQUE KEY `mqtt_acked_key` (`clientid`,`topic`) 10 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 11 | 12 | -------------------------------------------------------------------------------- /sql/mqtt_client.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS `mqtt_client`; 2 | CREATE TABLE `mqtt_client` ( 3 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 4 | `clientid` varchar(64) DEFAULT NULL, 5 | `state` varchar(3) DEFAULT NULL, 6 | `node` varchar(100) DEFAULT NULL, 7 | `online_at` datetime DEFAULT NULL, 8 | `offline_at` datetime DEFAULT NULL, 9 | `created` timestamp NULL DEFAULT CURRENT_TIMESTAMP, 10 | PRIMARY KEY (`id`), 11 | KEY `mqtt_client_idx` (`clientid`), 12 | UNIQUE KEY `mqtt_client_key` (`clientid`) 13 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 14 | 15 | -------------------------------------------------------------------------------- /sql/mqtt_msg.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS `mqtt_msg`; 2 | CREATE TABLE `mqtt_msg` ( 3 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 4 | `msgid` varchar(100) DEFAULT NULL, 5 | `topic` varchar(1024) DEFAULT NULL, 6 | `sender` varchar(1024) DEFAULT NULL, 7 | `node` varchar(60) DEFAULT NULL, 8 | `qos` int(11) DEFAULT '0', 9 | `retain` tinyint(2) DEFAULT NULL, 10 | `payload` blob, 11 | `arrived` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, 12 | PRIMARY KEY (`id`) 13 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 14 | 15 | -------------------------------------------------------------------------------- /sql/mqtt_retain.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS `mqtt_retain`; 2 | CREATE TABLE `mqtt_retain` ( 3 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 4 | `topic` varchar(200) DEFAULT NULL, 5 | `msgid` varchar(60) DEFAULT NULL, 6 | `sender` varchar(100) DEFAULT NULL, 7 | `node` varchar(100) DEFAULT NULL, 8 | `qos` int(2) DEFAULT NULL, 9 | `payload` blob, 10 | `arrived` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, 11 | PRIMARY KEY (`id`), 12 | UNIQUE KEY `mqtt_retain_key` (`topic`) 13 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 14 | 15 | -------------------------------------------------------------------------------- /sql/mqtt_sub.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS `mqtt_sub`; 2 | CREATE TABLE `mqtt_sub` ( 3 | `id` int(11) unsigned NOT NULL AUTO_INCREMENT, 4 | `clientid` varchar(64) DEFAULT NULL, 5 | `topic` varchar(255) DEFAULT NULL, 6 | `qos` int(3) DEFAULT NULL, 7 | `created` timestamp NULL DEFAULT CURRENT_TIMESTAMP, 8 | PRIMARY KEY (`id`), 9 | KEY `mqtt_sub_idx` (`clientid`,`topic`(255),`qos`), 10 | UNIQUE KEY `mqtt_sub_key` (`clientid`,`topic`) 11 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 12 | 13 | -------------------------------------------------------------------------------- /src/emqx_backend_mysql.app: -------------------------------------------------------------------------------- 1 | {application,emqx_backend_mysql, 2 | [{description,"EMQ X MySQL Backend"}, 3 | {vsn,"4.1.1"}, 4 | {modules,[emqx_backend_mysql,emqx_backend_mysql_actions, 5 | emqx_backend_mysql_app,emqx_backend_mysql_batcher, 6 | emqx_backend_mysql_cli,emqx_backend_mysql_sup]}, 7 | {registered,[emqx_backend_mysql_sup]}, 8 | {applications,[kernel,stdlib,mysql,ecpool]}, 9 | {mod,{emqx_backend_mysql_app,[]}}]}. 10 | -------------------------------------------------------------------------------- /src/emqx_backend_mysql.app.src: -------------------------------------------------------------------------------- 1 | {application, emqx_backend_mysql, 2 | [{description, "EMQ X Backend Mysql"}, 3 | {vsn, "git"}, 4 | {modules, []}, 5 | {registered, [emqx_backend_mysql_sup]}, 6 | {applications, [kernel,stdlib]}, 7 | {mod, {emqx_backend_mysql_app,[]}}, 8 | {env, []}, 9 | {licenses, ["Apache-2.0"]}, 10 | {maintainers, ["EMQ X Team "]}, 11 | {links, [{"Homepage", "https://emqx.io/"}, 12 | {"Github", "https://github.com/emqx/emqx-plugin-template"} 13 | ]} 14 | ]}. 15 | -------------------------------------------------------------------------------- /src/emqx_backend_mysql.app.src.script: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | %% .app.src.script 3 | 4 | RemoveLeadingV = 5 | fun(Tag) -> 6 | case re:run(Tag, "^[v|e]?[0-9]\.[0-9]\.([0-9]|(rc|beta|alpha)\.[0-9])", [{capture, none}]) of 7 | nomatch -> 8 | re:replace(Tag, "/", "-", [{return ,list}]); 9 | _ -> 10 | %% if it is a version number prefixed by 'v' or 'e', then remove it 11 | re:replace(Tag, "[v|e]", "", [{return ,list}]) 12 | end 13 | end, 14 | 15 | case os:getenv("EMQX_DEPS_DEFAULT_VSN") of 16 | false -> CONFIG; % env var not defined 17 | [] -> CONFIG; % env var set to empty string 18 | Tag -> 19 | [begin 20 | AppConf0 = lists:keystore(vsn, 1, AppConf, {vsn, RemoveLeadingV(Tag)}), 21 | {application, App, AppConf0} 22 | end || Conf = {application, App, AppConf} <- CONFIG] 23 | end. 24 | 25 | -------------------------------------------------------------------------------- /src/emqx_backend_mysql.erl: -------------------------------------------------------------------------------- 1 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 2 | "mqx_backend_mysql.erl", 3 | 1). 4 | 5 | -module(emqx_backend_mysql). 6 | 7 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/inclu" 8 | "de/emqx_backend_mysql.hrl", 9 | 1). 10 | 11 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 12 | "mqx_backend_mysql.erl", 13 | 10). 14 | 15 | -file("/emqx_rel/_checkouts/emqx/include/emqx.hrl", 1). 16 | 17 | -record(subscription, {topic, subid, subopts}). 18 | 19 | -record(message, 20 | {id :: binary(), qos = 0, from :: atom() | binary(), 21 | flags :: #{atom() => boolean()}, headers :: map(), 22 | topic :: binary(), payload :: binary(), 23 | timestamp :: integer()}). 24 | 25 | -record(delivery, 26 | {sender :: pid(), message :: #message{}}). 27 | 28 | -record(route, 29 | {topic :: binary(), 30 | dest :: node() | {binary(), node()}}). 31 | 32 | -type trie_node_id() :: binary() | atom(). 33 | 34 | -record(trie_node, 35 | {node_id :: trie_node_id(), 36 | edge_count = 0 :: non_neg_integer(), 37 | topic :: binary() | undefined, flags :: [atom()]}). 38 | 39 | -record(trie_edge, 40 | {node_id :: trie_node_id(), 41 | word :: binary() | atom()}). 42 | 43 | -record(trie, 44 | {edge :: #trie_edge{}, node_id :: trie_node_id()}). 45 | 46 | -record(alarm, 47 | {id :: binary(), 48 | severity :: notice | warning | error | critical, 49 | title :: iolist(), summary :: iolist(), 50 | timestamp :: erlang:timestamp()}). 51 | 52 | -record(plugin, 53 | {name :: atom(), dir :: string(), descr :: string(), 54 | vendor :: string(), active = false :: boolean(), 55 | info :: map(), type :: atom()}). 56 | 57 | -record(command, 58 | {name :: atom(), action :: atom(), 59 | args = [] :: list(), opts = [] :: list(), 60 | usage :: string(), descr :: string()}). 61 | 62 | -record(banned, 63 | {who :: 64 | {clientid, binary()} | {username, binary()} | 65 | {ip_address, inet:ip_address()}, 66 | by :: binary(), reason :: binary(), at :: integer(), 67 | until :: integer()}). 68 | 69 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 70 | "mqx_backend_mysql.erl", 71 | 12). 72 | 73 | -export([pool_name/1]). 74 | 75 | -export([register_metrics/0, load/0, unload/0]). 76 | 77 | -export([on_client_connected/3, on_subscribe_lookup/3, 78 | on_client_disconnected/4, on_message_fetch/4, 79 | on_retain_lookup/4, on_message_publish/2, 80 | on_message_store/2, on_message_retain/2, 81 | on_retain_delete/2, on_message_delivered/3, 82 | on_message_acked/3, run_mysql_sql/2, run_mysql_sql/3, 83 | run_mysql_sql/4]). 84 | 85 | pool_name(Pool) -> 86 | list_to_atom(lists:concat([emqx_backend_mysql, '_', 87 | Pool])). 88 | 89 | register_metrics() -> 90 | [emqx_metrics:new(MetricName) 91 | || MetricName 92 | <- ['backend.mysql.client_connected', 93 | 'backend.mysql.subscribe_lookup', 94 | 'backend.mysql.client_disconnected', 95 | 'backend.mysql.on_message_fetch', 96 | 'backend.mysql.on_retain_lookup', 97 | 'backend.mysql.on_message_publish', 98 | 'backend.mysql.on_message_store', 99 | 'backend.mysql.on_message_retain', 100 | 'backend.mysql.on_retain_delete', 101 | 'backend.mysql.on_message_acked', 102 | 'backend.mysql.run_mysql_sql.publish', 103 | 'backend.mysql.run_mysql_sql.acked_delivered', 104 | 'backend.mysql.run_mysql_sql.sub_unsub', 105 | 'backend.mysql.run_mysql_sql.connected', 106 | 'backend.mysql.run_mysql_sql.disconnected']]. 107 | 108 | load() -> 109 | HookList = 110 | parse_hook(application:get_env(emqx_backend_mysql, 111 | hooks, [])), 112 | lists:foreach(fun ({Hook, Action, Pool, Filter, 113 | OfflineOpts}) -> 114 | case proplists:get_value(<<"function">>, Action) of 115 | undefined -> 116 | SqlList = [compile_sql(SQL) 117 | || SQL 118 | <- proplists:get_value(<<"sql">>, 119 | Action, 120 | [])], 121 | load_(Hook, run_mysql_sql, OfflineOpts, 122 | {Filter, Pool, SqlList}); 123 | Fun -> 124 | load_(Hook, b2a(Fun), OfflineOpts, 125 | {Filter, Pool}) 126 | end 127 | end, 128 | HookList), 129 | io:format("~s is loaded.~n", [emqx_backend_mysql]), 130 | ok. 131 | 132 | load_(Hook, Fun, OfflineOpts, Params) -> 133 | case Hook of 134 | 'client.connected' -> 135 | emqx:hook(Hook, fun emqx_backend_mysql:Fun/3, [Params]); 136 | 'client.disconnected' -> 137 | emqx:hook(Hook, fun emqx_backend_mysql:Fun/4, [Params]); 138 | 'session.subscribed' -> 139 | emqx:hook(Hook, fun emqx_backend_mysql:Fun/4, 140 | [erlang:append_element(Params, OfflineOpts)]); 141 | 'session.unsubscribed' -> 142 | emqx:hook(Hook, fun emqx_backend_mysql:Fun/4, [Params]); 143 | 'message.publish' -> 144 | emqx:hook(Hook, fun emqx_backend_mysql:Fun/2, [Params]); 145 | 'message.acked' -> 146 | emqx:hook(Hook, fun emqx_backend_mysql:Fun/3, [Params]); 147 | 'message.delivered' -> 148 | emqx:hook(Hook, fun emqx_backend_mysql:Fun/3, [Params]) 149 | end. 150 | 151 | unload() -> 152 | HookList = 153 | parse_hook(application:get_env(emqx_backend_mysql, 154 | hooks, [])), 155 | lists:foreach(fun ({Hook, Action, _Pool, _Filter, 156 | _OfflineOpts}) -> 157 | case proplists:get_value(<<"function">>, Action) of 158 | undefined -> unload_(Hook, run_mysql_sql); 159 | Fun -> unload_(Hook, b2a(Fun)) 160 | end 161 | end, 162 | HookList), 163 | io:format("~s is unloaded.~n", [emqx_backend_mysql]), 164 | ok. 165 | 166 | unload_(Hook, Fun) -> 167 | case Hook of 168 | 'client.connected' -> 169 | emqx:unhook(Hook, fun emqx_backend_mysql:Fun/3); 170 | 'client.disconnected' -> 171 | emqx:unhook(Hook, fun emqx_backend_mysql:Fun/4); 172 | 'session.subscribed' -> 173 | emqx:unhook(Hook, fun emqx_backend_mysql:Fun/4); 174 | 'session.unsubscribed' -> 175 | emqx:unhook(Hook, fun emqx_backend_mysql:Fun/4); 176 | 'message.publish' -> 177 | emqx:unhook(Hook, fun emqx_backend_mysql:Fun/2); 178 | 'message.acked' -> 179 | emqx:unhook(Hook, fun emqx_backend_mysql:Fun/3); 180 | 'message.delivered' -> 181 | emqx:unhook(Hook, fun emqx_backend_mysql:Fun/3) 182 | end. 183 | 184 | on_client_connected(#{clientid := ClientId}, _ConnInfo, 185 | {Filter, Pool}) -> 186 | with_filter(fun () -> 187 | emqx_metrics:inc('backend.mysql.client_connected'), 188 | emqx_backend_mysql_cli:client_connected(Pool, 189 | [{clientid, 190 | ClientId}]) 191 | end, 192 | undefined, Filter). 193 | 194 | on_subscribe_lookup(#{clientid := ClientId}, _ConnInfo, 195 | {Filter, Pool}) -> 196 | with_filter(fun () -> 197 | emqx_metrics:inc('backend.mysql.subscribe_lookup'), 198 | case emqx_backend_mysql_cli:subscribe_lookup(Pool, 199 | [{clientid, 200 | ClientId}]) 201 | of 202 | [] -> ok; 203 | TopicTable -> self() ! {subscribe, TopicTable}, ok 204 | end 205 | end, 206 | undefined, Filter). 207 | 208 | on_client_disconnected(#{clientid := ClientId}, _Reason, 209 | _ConnInfo, {Filter, Pool}) -> 210 | with_filter(fun () -> 211 | emqx_metrics:inc('backend.mysql.client_disconnected'), 212 | emqx_backend_mysql_cli:client_disconnected(Pool, 213 | [{clientid, 214 | ClientId}]) 215 | end, 216 | undefined, Filter). 217 | 218 | on_message_fetch(#{clientid := ClientId}, Topic, Opts, 219 | {Filter, Pool, OfflineOpts}) -> 220 | with_filter(fun () -> 221 | emqx_metrics:inc('backend.mysql.on_message_fetch'), 222 | case maps:get(qos, Opts, 0) > 0 andalso 223 | maps:get(first, Opts, true) 224 | of 225 | true -> 226 | MsgList = 227 | emqx_backend_mysql_cli:message_fetch(Pool, 228 | [{clientid, 229 | ClientId}, 230 | {topic, 231 | Topic}], 232 | OfflineOpts), 233 | [self() ! {deliver, Topic, Msg} 234 | || Msg <- MsgList]; 235 | false -> ok 236 | end 237 | end, 238 | Topic, Filter). 239 | 240 | on_retain_lookup(_Client, Topic, _Opts, 241 | {Filter, Pool, _OfflineOpts}) -> 242 | with_filter(fun () -> 243 | emqx_metrics:inc('backend.mysql.on_retain_lookup'), 244 | MsgList = emqx_backend_mysql_cli:lookup_retain(Pool, 245 | [{topic, 246 | Topic}]), 247 | [self() ! {deliver, Topic, set_retain(true, Msg)} 248 | || Msg <- MsgList] 249 | end, 250 | Topic, Filter). 251 | 252 | set_retain(Value, Msg) -> 253 | Msg1 = emqx_message:set_flags(#{retained => Value}, 254 | Msg), 255 | emqx_message:set_header(retained, Value, Msg1). 256 | 257 | on_message_publish(Msg = #message{flags = 258 | #{retain := true}, 259 | payload = <<>>}, 260 | _Rule) -> 261 | {ok, Msg}; 262 | on_message_publish(Msg = #message{qos = Qos}, 263 | {_Filter, _Pool}) 264 | when Qos =:= 0 -> 265 | {ok, Msg}; 266 | on_message_publish(Msg0 = #message{topic = Topic}, 267 | {Filter, Pool}) -> 268 | with_filter(fun () -> 269 | emqx_metrics:inc('backend.mysql.on_message_publish'), 270 | Msg = emqx_backend_mysql_cli:message_publish(Pool, 271 | Msg0), 272 | {ok, Msg} 273 | end, 274 | Msg0, Topic, Filter). 275 | 276 | on_message_store(Msg = #message{flags = 277 | #{retain := true}, 278 | payload = <<>>}, 279 | _Rule) -> 280 | {ok, Msg}; 281 | on_message_store(Msg0 = #message{topic = Topic}, 282 | {Filter, Pool}) -> 283 | with_filter(fun () -> 284 | emqx_metrics:inc('backend.mysql.on_message_store'), 285 | Msg = emqx_backend_mysql_cli:message_store(Pool, Msg0), 286 | {ok, Msg} 287 | end, 288 | Msg0, Topic, Filter). 289 | 290 | on_message_retain(Msg = #message{flags = 291 | #{retain := false}}, 292 | _Rule) -> 293 | {ok, Msg}; 294 | on_message_retain(Msg = #message{flags = 295 | #{retain := true}, 296 | payload = <<>>}, 297 | _Rule) -> 298 | {ok, Msg}; 299 | on_message_retain(Msg0 = #message{flags = 300 | #{retain := true}, 301 | topic = Topic, headers = Headers0}, 302 | {Filter, Pool}) -> 303 | Headers = case erlang:is_map(Headers0) of 304 | true -> Headers0; 305 | false -> #{} 306 | end, 307 | case maps:find(retained, Headers) of 308 | {ok, true} -> {ok, Msg0}; 309 | _ -> 310 | with_filter(fun () -> 311 | emqx_metrics:inc('backend.mysql.on_message_retain'), 312 | Msg = emqx_backend_mysql_cli:message_retain(Pool, 313 | Msg0), 314 | {ok, Msg} 315 | end, 316 | Msg0, Topic, Filter) 317 | end; 318 | on_message_retain(Msg, _Rule) -> {ok, Msg}. 319 | 320 | on_retain_delete(Msg0 = #message{flags = 321 | #{retain := true}, 322 | topic = Topic, payload = <<>>}, 323 | {Filter, Pool}) -> 324 | with_filter(fun () -> 325 | emqx_metrics:inc('backend.mysql.on_retain_delete'), 326 | Msg = emqx_backend_mysql_cli:delete_retain(Pool, Msg0), 327 | {ok, Msg} 328 | end, 329 | Msg0, Topic, Filter); 330 | on_retain_delete(Msg, _Rule) -> {ok, Msg}. 331 | 332 | on_message_delivered(_Client, _Msg, _Rule) -> ok. 333 | 334 | on_message_acked(#{clientid := ClientId}, 335 | #message{topic = Topic, headers = Headers}, 336 | {Filter, Pool}) -> 337 | case maps:get(mysql_id, Headers, undefined) of 338 | undefined -> ok; 339 | Id -> 340 | with_filter(fun () -> 341 | emqx_metrics:inc('backend.mysql.on_message_acked'), 342 | emqx_backend_mysql_cli:message_acked(Pool, 343 | [{clientid, 344 | ClientId}, 345 | {topic, 346 | Topic}, 347 | {mysql_id, 348 | Id}]) 349 | end, 350 | Topic, Filter) 351 | end. 352 | 353 | run_mysql_sql(Msg0 = #message{topic = Topic}, 354 | {Filter, Pool, SqlList}) -> 355 | with_filter(fun () -> 356 | emqx_metrics:inc('backend.mysql.run_mysql_sql.publish'), 357 | Msg = emqx_backend_mysql_cli:run_mysql_sql(Pool, Msg0, 358 | SqlList), 359 | {ok, Msg} 360 | end, 361 | Msg0, Topic, Filter). 362 | 363 | run_mysql_sql(#{clientid := ClientId}, 364 | #message{topic = Topic, id = MsgId}, 365 | {Filter, Pool, SqlList}) -> 366 | with_filter(fun () -> 367 | emqx_metrics:inc('backend.mysql.run_mysql_sql.acked_delivered'), 368 | emqx_backend_mysql_cli:run_mysql_sql(Pool, 369 | [{clientid, 370 | ClientId}, 371 | {topic, Topic}, 372 | {msgid, MsgId}], 373 | SqlList) 374 | end, 375 | Topic, Filter); 376 | run_mysql_sql(#{clientid := ClientId}, _ConnAttrs, 377 | {Filter, Pool, SqlList}) -> 378 | with_filter(fun () -> 379 | emqx_metrics:inc('backend.mysql.run_mysql_sql.connected'), 380 | emqx_backend_mysql_cli:run_mysql_sql(Pool, 381 | [{clientid, 382 | ClientId}], 383 | SqlList) 384 | end, 385 | undefined, Filter); 386 | run_mysql_sql(_, _, _) -> ok. 387 | 388 | run_mysql_sql(#{clientid := ClientId}, Topic, Opts, 389 | {Filter, Pool, SqlList}) 390 | when is_binary(Topic) -> 391 | with_filter(fun () -> 392 | emqx_metrics:inc('backend.mysql.run_mysql_sql.sub_unsub'), 393 | QoS = maps:get(qos, Opts, 0), 394 | emqx_backend_mysql_cli:run_mysql_sql(Pool, 395 | [{clientid, 396 | ClientId}, 397 | {topic, Topic}, 398 | {qos, QoS}], 399 | SqlList) 400 | end, 401 | Topic, Filter); 402 | run_mysql_sql(Client, Topic, Opts, 403 | {Filter, Pool, SqlList, _}) 404 | when is_binary(Topic) -> 405 | run_mysql_sql(Client, Topic, Opts, 406 | {Filter, Pool, SqlList}); 407 | run_mysql_sql(#{clientid := ClientId}, _Reason, 408 | _ConnInfo, {_Filter, Pool, SqlList}) -> 409 | emqx_metrics:inc('backend.mysql.run_mysql_sql.disconnected'), 410 | emqx_backend_mysql_cli:run_mysql_sql(Pool, 411 | [{clientid, ClientId}], SqlList); 412 | run_mysql_sql(_, _, _, _) -> ok. 413 | 414 | parse_hook(Hooks) -> parse_hook(Hooks, []). 415 | 416 | parse_hook([], Acc) -> Acc; 417 | parse_hook([{Hook, Item} | Hooks], Acc) -> 418 | Params = emqx_json:decode(Item), 419 | Action = proplists:get_value(<<"action">>, Params), 420 | Pool = proplists:get_value(<<"pool">>, Params), 421 | Filter = proplists:get_value(<<"topic">>, Params), 422 | OfflineOpts = 423 | parse_offline_opts(proplists:get_value(<<"offline_opts">>, 424 | Params, [])), 425 | parse_hook(Hooks, 426 | [{l2a(Hook), Action, pool_name(b2a(Pool)), Filter, 427 | OfflineOpts} 428 | | Acc]). 429 | 430 | compile_sql(SQL) -> 431 | case re:run(SQL, <<"\\$\\{[^}]+\\}">>, 432 | [global, {capture, all, binary}]) 433 | of 434 | nomatch -> {SQL, []}; 435 | {match, Vars} -> 436 | {re:replace(SQL, <<"\\$\\{[^}]+\\}">>, <<"?">>, 437 | [global, {return, binary}]), 438 | [Var || [Var] <- Vars]} 439 | end. 440 | 441 | with_filter(Fun, _, undefined) -> Fun(), ok; 442 | with_filter(Fun, Topic, Filter) -> 443 | case emqx_topic:match(Topic, Filter) of 444 | true -> Fun(), ok; 445 | false -> ok 446 | end. 447 | 448 | with_filter(Fun, _, _, undefined) -> Fun(); 449 | with_filter(Fun, Msg, Topic, Filter) -> 450 | case emqx_topic:match(Topic, Filter) of 451 | true -> Fun(); 452 | false -> {ok, Msg} 453 | end. 454 | 455 | l2a(L) -> erlang:list_to_atom(L). 456 | 457 | b2a(B) -> erlang:binary_to_atom(B, utf8). 458 | 459 | b2l(B) -> erlang:binary_to_list(B). 460 | 461 | parse_offline_opts(OfflineOpts) -> 462 | parse_offline_opts(OfflineOpts, []). 463 | 464 | parse_offline_opts([], Acc) -> Acc; 465 | parse_offline_opts([{<<"max_returned_count">>, 466 | MaxReturnedCount} 467 | | OfflineOpts], 468 | Acc) 469 | when is_integer(MaxReturnedCount) -> 470 | parse_offline_opts(OfflineOpts, 471 | [{max_returned_count, MaxReturnedCount} | Acc]); 472 | parse_offline_opts([{<<"time_range">>, TimeRange} 473 | | OfflineOpts], 474 | Acc) 475 | when is_binary(TimeRange) -> 476 | parse_offline_opts(OfflineOpts, 477 | [{time_range, 478 | cuttlefish_duration:parse(b2l(TimeRange), s)} 479 | | Acc]); 480 | parse_offline_opts([_ | OfflineOpts], Acc) -> 481 | parse_offline_opts(OfflineOpts, Acc). 482 | 483 | 484 | -------------------------------------------------------------------------------- /src/emqx_backend_mysql_actions.erl: -------------------------------------------------------------------------------- 1 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 2 | "mqx_backend_mysql_actions.erl", 3 | 1). 4 | 5 | -module(emqx_backend_mysql_actions). 6 | 7 | -export(['$logger_header'/0]). 8 | 9 | -behaviour(ecpool_worker). 10 | 11 | -file("/emqx_rel/_checkouts/emqx/include/emqx.hrl", 1). 12 | 13 | -record(subscription, {topic, subid, subopts}). 14 | 15 | -record(message, 16 | {id :: binary(), qos = 0, from :: atom() | binary(), 17 | flags :: #{atom() => boolean()}, headers :: map(), 18 | topic :: binary(), payload :: binary(), 19 | timestamp :: integer()}). 20 | 21 | -record(delivery, 22 | {sender :: pid(), message :: #message{}}). 23 | 24 | -record(route, 25 | {topic :: binary(), 26 | dest :: node() | {binary(), node()}}). 27 | 28 | -type trie_node_id() :: binary() | atom(). 29 | 30 | -record(trie_node, 31 | {node_id :: trie_node_id(), 32 | edge_count = 0 :: non_neg_integer(), 33 | topic :: binary() | undefined, flags :: [atom()]}). 34 | 35 | -record(trie_edge, 36 | {node_id :: trie_node_id(), 37 | word :: binary() | atom()}). 38 | 39 | -record(trie, 40 | {edge :: #trie_edge{}, node_id :: trie_node_id()}). 41 | 42 | -record(alarm, 43 | {id :: binary(), 44 | severity :: notice | warning | error | critical, 45 | title :: iolist(), summary :: iolist(), 46 | timestamp :: erlang:timestamp()}). 47 | 48 | -record(plugin, 49 | {name :: atom(), dir :: string(), descr :: string(), 50 | vendor :: string(), active = false :: boolean(), 51 | info :: map(), type :: atom()}). 52 | 53 | -record(command, 54 | {name :: atom(), action :: atom(), 55 | args = [] :: list(), opts = [] :: list(), 56 | usage :: string(), descr :: string()}). 57 | 58 | -record(banned, 59 | {who :: 60 | {clientid, binary()} | {username, binary()} | 61 | {ip_address, inet:ip_address()}, 62 | by :: binary(), reason :: binary(), at :: integer(), 63 | until :: integer()}). 64 | 65 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 66 | "mqx_backend_mysql_actions.erl", 67 | 12). 68 | 69 | -file("/emqx_rel/_checkouts/emqx/include/logger.hrl", 70 | 1). 71 | 72 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 73 | "mqx_backend_mysql_actions.erl", 74 | 13). 75 | 76 | -export([connect/1, send_batch/2]). 77 | 78 | -import(emqx_rule_utils, [str/1, unsafe_atom_key/1]). 79 | 80 | -export([on_resource_create/2, on_resource_destroy/2, 81 | on_get_resource_status/2]). 82 | 83 | -export([on_action_create_data_to_mysql/2]). 84 | 85 | -export([on_action_create_offline_msg/2]). 86 | 87 | -export([on_action_create_lookup_sub/2]). 88 | 89 | -resource_type(#{create => on_resource_create, 90 | description => 91 | #{en => 92 | <<77, 121, 83, 81, 76, 32, 68, 97, 116, 97, 98, 97, 115, 101>>, 93 | zh => 94 | <<77, 121, 83, 81, 76, 32, 230, 149, 176, 230, 141, 174, 229, 186, 147>>}, 95 | destroy => on_resource_destroy, name => backend_mysql, 96 | params => 97 | #{auto_reconnect => 98 | #{default => true, 99 | description => 100 | #{en => 101 | <<73, 102, 32, 114, 101, 45, 116, 114, 102 | 121, 32, 119, 104, 101, 110, 32, 116, 103 | 104, 101, 32, 99, 111, 110, 110, 101, 104 | 99, 116, 105, 111, 110, 32, 108, 111, 105 | 115, 116>>, 106 | zh => 107 | <<77, 121, 83, 81, 76, 32, 232, 191, 158, 108 | 230, 142, 165, 230, 150, 173, 229, 188, 109 | 128, 230, 151, 182, 230, 152, 175, 229, 110 | 144, 166, 233, 135, 141, 232, 191, 111 | 158>>}, 112 | order => 9, 113 | title => 114 | #{en => 115 | <<69, 110, 97, 98, 108, 101, 32, 82, 101, 116 | 99, 111, 110, 110, 101, 99, 116>>, 117 | zh => 118 | <<230, 152, 175, 229, 144, 166, 233, 135, 119 | 141, 232, 191, 158>>}, 120 | type => boolean}, 121 | batch_size => 122 | #{default => 100, 123 | description => 124 | #{en => 125 | <<84, 104, 101, 32, 115, 105, 122, 101, 126 | 32, 111, 102, 32, 77, 121, 83, 81, 76, 127 | 32, 98, 97, 116, 99, 104, 32, 105, 110, 128 | 115, 101, 114, 116, 32, 83, 81, 76>>, 129 | zh => 130 | <<77, 121, 83, 81, 76, 32, 230, 137, 185, 131 | 233, 135, 143, 229, 134, 153, 229, 133, 132 | 165, 229, 164, 167, 229, 176, 143>>}, 133 | order => 7, 134 | title => 135 | #{en => 136 | <<66, 97, 116, 99, 104, 32, 83, 105, 122, 137 | 101>>, 138 | zh => 139 | <<230, 137, 185, 233, 135, 143, 229, 134, 140 | 153, 229, 133, 165, 229, 164, 167, 229, 141 | 176, 143>>}, 142 | type => number}, 143 | batch_time => 144 | #{default => 10, 145 | description => 146 | #{en => 147 | <<84, 105, 109, 101, 32, 111, 102, 32, 148 | 77, 121, 83, 81, 76, 32, 98, 97, 116, 149 | 99, 104, 32, 105, 110, 115, 101, 114, 150 | 116, 32, 83, 81, 76, 32, 105, 110, 116, 151 | 101, 114, 118, 97, 108, 40, 109, 105, 152 | 108, 108, 105, 115, 101, 99, 111, 110, 153 | 100, 41>>, 154 | zh => 155 | <<77, 121, 83, 81, 76, 32, 230, 137, 185, 156 | 233, 135, 143, 229, 134, 153, 229, 133, 157 | 165, 233, 151, 180, 233, 154, 148, 40, 158 | 230, 175, 171, 231, 167, 146, 41>>}, 159 | order => 8, 160 | title => 161 | #{en => 162 | <<66, 97, 116, 99, 104, 32, 84, 105, 109, 163 | 101>>, 164 | zh => 165 | <<230, 137, 185, 233, 135, 143, 229, 134, 166 | 153, 229, 133, 165, 233, 151, 180, 233, 167 | 154, 148, 40, 230, 175, 171, 231, 167, 168 | 146, 41>>}, 169 | type => number}, 170 | database => 171 | #{description => 172 | #{en => 173 | <<68, 97, 116, 97, 98, 97, 115, 101, 32, 174 | 110, 97, 109, 101, 32, 102, 111, 114, 175 | 32, 99, 111, 110, 110, 101, 99, 116, 176 | 105, 110, 103, 32, 116, 111, 32, 77, 177 | 121, 83, 81, 76>>, 178 | zh => 179 | <<77, 121, 83, 81, 76, 32, 230, 149, 176, 180 | 230, 141, 174, 229, 186, 147, 229, 144, 181 | 141>>}, 182 | order => 3, required => true, 183 | title => 184 | #{en => 185 | <<77, 121, 83, 81, 76, 32, 68, 97, 116, 186 | 97, 98, 97, 115, 101>>, 187 | zh => 188 | <<77, 121, 83, 81, 76, 32, 230, 149, 176, 189 | 230, 141, 174, 229, 186, 147, 229, 144, 190 | 141>>}, 191 | type => string}, 192 | password => 193 | #{default => <<>>, 194 | description => 195 | #{en => 196 | <<80, 97, 115, 115, 119, 111, 114, 100, 197 | 32, 102, 111, 114, 32, 99, 111, 110, 198 | 110, 101, 99, 116, 105, 110, 103, 32, 199 | 116, 111, 32, 77, 121, 83, 81, 76>>, 200 | zh => 201 | <<77, 121, 83, 81, 76, 32, 229, 175, 134, 202 | 231, 160, 129>>}, 203 | order => 6, 204 | title => 205 | #{en => 206 | <<77, 121, 83, 81, 76, 32, 80, 97, 115, 207 | 115, 119, 111, 114, 100>>, 208 | zh => 209 | <<77, 121, 83, 81, 76, 32, 229, 175, 134, 210 | 231, 160, 129>>}, 211 | type => string}, 212 | pool_size => 213 | #{default => 8, 214 | description => 215 | #{en => 216 | <<84, 104, 101, 32, 115, 105, 122, 101, 217 | 32, 111, 102, 32, 99, 111, 110, 110, 218 | 101, 99, 116, 105, 111, 110, 32, 112, 219 | 111, 111, 108, 32, 102, 111, 114, 32, 220 | 77, 121, 83, 81, 76>>, 221 | zh => 222 | <<77, 121, 83, 81, 76, 32, 232, 191, 158, 223 | 230, 142, 165, 230, 177, 160, 229, 164, 224 | 167, 229, 176, 143>>}, 225 | order => 4, 226 | title => 227 | #{en => 228 | <<80, 111, 111, 108, 32, 83, 105, 122, 229 | 101>>, 230 | zh => 231 | <<232, 191, 158, 230, 142, 165, 230, 177, 232 | 160, 229, 164, 167, 229, 176, 143>>}, 233 | type => number}, 234 | server => 235 | #{default => 236 | <<49, 50, 55, 46, 48, 46, 48, 46, 49, 58, 51, 237 | 51, 48, 54>>, 238 | description => 239 | #{en => 240 | <<77, 121, 83, 81, 76, 32, 73, 80, 32, 241 | 97, 100, 100, 114, 101, 115, 115, 32, 242 | 111, 114, 32, 104, 111, 115, 116, 110, 243 | 97, 109, 101, 32, 97, 110, 100, 32, 244 | 112, 111, 114, 116>>, 245 | zh => 246 | <<77, 121, 83, 81, 76, 32, 230, 156, 141, 247 | 229, 138, 161, 229, 153, 168, 229, 156, 248 | 176, 229, 157, 128>>}, 249 | order => 1, required => true, 250 | title => 251 | #{en => 252 | <<77, 121, 83, 81, 76, 32, 83, 101, 114, 253 | 118, 101, 114>>, 254 | zh => 255 | <<77, 121, 83, 81, 76, 32, 230, 156, 141, 256 | 229, 138, 161, 229, 153, 168>>}, 257 | type => string}, 258 | user => 259 | #{description => 260 | #{en => 261 | <<85, 115, 101, 114, 110, 97, 109, 101, 262 | 32, 102, 111, 114, 32, 99, 111, 110, 263 | 110, 101, 99, 116, 105, 110, 103, 32, 264 | 116, 111, 32, 77, 121, 83, 81, 76>>, 265 | zh => 266 | <<77, 121, 83, 81, 76, 32, 231, 148, 168, 267 | 230, 136, 183, 229, 144, 141>>}, 268 | order => 5, required => true, 269 | title => 270 | #{en => 271 | <<77, 121, 83, 81, 76, 32, 85, 115, 101, 272 | 114, 32, 78, 97, 109, 101>>, 273 | zh => 274 | <<77, 121, 83, 81, 76, 32, 231, 148, 168, 275 | 230, 136, 183, 229, 144, 141>>}, 276 | type => string}}, 277 | status => on_get_resource_status, 278 | title => 279 | #{en => <<77, 121, 83, 81, 76>>, 280 | zh => <<77, 121, 83, 81, 76>>}}). 281 | 282 | -rule_action(#{create => on_action_create_data_to_mysql, 283 | description => 284 | #{en => 285 | <<68, 97, 116, 97, 32, 116, 111, 32, 77, 121, 83, 81, 286 | 76>>, 287 | zh => 288 | <<228, 191, 157, 229, 173, 152, 230, 149, 176, 230, 289 | 141, 174, 229, 136, 176, 32, 77, 121, 83, 81, 76, 32, 290 | 230, 149, 176, 230, 141, 174, 229, 186, 147>>}, 291 | for => '$any', name => data_to_mysql, 292 | params => 293 | #{'$resource' => 294 | #{description => 295 | #{en => 296 | <<66, 105, 110, 100, 32, 97, 32, 114, 101, 297 | 115, 111, 117, 114, 99, 101, 32, 116, 298 | 111, 32, 116, 104, 105, 115, 32, 97, 99, 299 | 116, 105, 111, 110>>, 300 | zh => 301 | <<231, 187, 153, 229, 138, 168, 228, 189, 302 | 156, 231, 187, 145, 229, 174, 154, 228, 303 | 184, 128, 228, 184, 170, 232, 181, 132, 304 | 230, 186, 144>>}, 305 | required => true, 306 | title => 307 | #{en => 308 | <<82, 101, 115, 111, 117, 114, 99, 101, 32, 309 | 73, 68>>, 310 | zh => 311 | <<232, 181, 132, 230, 186, 144, 32, 73, 312 | 68>>}, 313 | type => string}, 314 | sql => 315 | #{description => 316 | #{en => 317 | <<83, 81, 76, 32, 116, 101, 109, 112, 108, 318 | 97, 116, 101, 32, 119, 105, 116, 104, 32, 319 | 112, 108, 97, 99, 101, 104, 111, 108, 320 | 100, 101, 114, 115, 32, 102, 111, 114, 321 | 32, 105, 110, 115, 101, 114, 116, 105, 322 | 110, 103, 47, 117, 112, 100, 97, 116, 323 | 105, 110, 103, 32, 100, 97, 116, 97, 32, 324 | 116, 111, 32, 77, 121, 83, 81, 76>>, 325 | zh => 326 | <<229, 140, 133, 229, 144, 171, 228, 186, 327 | 134, 229, 141, 160, 228, 189, 141, 231, 328 | 172, 166, 231, 154, 132, 32, 83, 81, 76, 329 | 32, 230, 168, 161, 230, 157, 191, 239, 330 | 188, 140, 231, 148, 168, 228, 187, 165, 331 | 230, 143, 146, 229, 133, 165, 230, 136, 332 | 150, 230, 155, 180, 230, 150, 176, 230, 333 | 149, 176, 230, 141, 174, 229, 136, 176, 334 | 32, 77, 121, 83, 81, 76, 32, 230, 149, 335 | 176, 230, 141, 174, 229, 186, 147>>}, 336 | input => textarea, required => true, 337 | title => 338 | #{en => 339 | <<83, 81, 76, 32, 84, 101, 109, 112, 108, 340 | 97, 116, 101>>, 341 | zh => 342 | <<83, 81, 76, 32, 230, 168, 161, 230, 157, 343 | 191>>}, 344 | type => string}}, 345 | title => 346 | #{en => 347 | <<68, 97, 116, 97, 32, 116, 111, 32, 77, 121, 83, 81, 348 | 76>>, 349 | zh => 350 | <<228, 191, 157, 229, 173, 152, 230, 149, 176, 230, 351 | 141, 174, 229, 136, 176, 32, 77, 121, 83, 81, 76>>}, 352 | types => [backend_mysql]}). 353 | 354 | -rule_action(#{create => on_action_create_offline_msg, 355 | description => 356 | #{en => 357 | <<79, 102, 102, 108, 105, 110, 101, 32, 77, 115, 103, 358 | 32, 116, 111, 32, 77, 121, 83, 81, 76>>, 359 | zh => 360 | <<231, 166, 187, 231, 186, 191, 230, 182, 136, 230, 361 | 129, 175, 228, 191, 157, 229, 173, 152, 229, 136, 362 | 176, 32, 77, 121, 83, 81, 76>>}, 363 | for => '$any', name => offline_msg_to_mysql, 364 | params => 365 | #{'$resource' => 366 | #{description => 367 | #{en => 368 | <<66, 105, 110, 100, 32, 97, 32, 114, 101, 369 | 115, 111, 117, 114, 99, 101, 32, 116, 370 | 111, 32, 116, 104, 105, 115, 32, 97, 99, 371 | 116, 105, 111, 110>>, 372 | zh => 373 | <<231, 187, 153, 229, 138, 168, 228, 189, 374 | 156, 231, 187, 145, 229, 174, 154, 228, 375 | 184, 128, 228, 184, 170, 232, 181, 132, 376 | 230, 186, 144>>}, 377 | required => true, 378 | title => 379 | #{en => 380 | <<82, 101, 115, 111, 117, 114, 99, 101, 32, 381 | 73, 68>>, 382 | zh => 383 | <<232, 181, 132, 230, 186, 144, 32, 73, 384 | 68>>}, 385 | type => string}, 386 | max_returned_count => 387 | #{default => 0, 388 | description => 389 | #{en => 390 | <<77, 97, 120, 32, 110, 117, 109, 98, 101, 391 | 114, 32, 111, 102, 32, 102, 101, 116, 99, 392 | 104, 32, 111, 102, 102, 108, 105, 110, 393 | 101, 32, 109, 101, 115, 115, 97, 103, 394 | 101, 115>>, 395 | zh => 396 | <<232, 142, 183, 229, 143, 150, 231, 166, 397 | 187, 231, 186, 191, 230, 182, 136, 230, 398 | 129, 175, 231, 154, 132, 230, 156, 128, 399 | 229, 164, 167, 230, 157, 161, 230, 149, 400 | 176>>}, 401 | required => false, 402 | title => 403 | #{en => 404 | <<77, 97, 120, 32, 82, 101, 116, 117, 114, 405 | 110, 101, 100, 32, 67, 111, 117, 110, 406 | 116>>, 407 | zh => 408 | <<77, 97, 120, 32, 82, 101, 116, 117, 114, 409 | 110, 101, 100, 32, 67, 111, 117, 110, 410 | 116>>}, 411 | type => number}, 412 | time_range => 413 | #{default => <<>>, 414 | description => 415 | #{en => 416 | <<84, 105, 109, 101, 32, 82, 97, 110, 103, 417 | 101, 32, 111, 102, 32, 102, 101, 116, 99, 418 | 104, 32, 111, 102, 102, 108, 105, 110, 419 | 101, 32, 109, 101, 115, 115, 97, 103, 420 | 101, 115>>, 421 | zh => 422 | <<232, 142, 183, 229, 143, 150, 230, 156, 423 | 128, 232, 191, 145, 230, 151, 182, 233, 424 | 151, 180, 232, 140, 131, 229, 155, 180, 425 | 229, 134, 133, 231, 154, 132, 231, 166, 426 | 187, 231, 186, 191, 230, 182, 136, 230, 427 | 129, 175>>}, 428 | required => false, 429 | title => 430 | #{en => 431 | <<84, 105, 109, 101, 32, 82, 97, 110, 103, 432 | 101>>, 433 | zh => 434 | <<84, 105, 109, 101, 32, 82, 97, 110, 103, 435 | 101>>}, 436 | type => string}}, 437 | title => 438 | #{en => 439 | <<79, 102, 102, 108, 105, 110, 101, 32, 77, 115, 103, 440 | 32, 116, 111, 32, 77, 121, 83, 81, 76>>, 441 | zh => 442 | <<231, 166, 187, 231, 186, 191, 230, 182, 136, 230, 443 | 129, 175, 228, 191, 157, 229, 173, 152, 229, 136, 444 | 176, 32, 77, 121, 83, 81, 76>>}, 445 | types => [backend_mysql]}). 446 | 447 | -rule_action(#{create => on_action_create_lookup_sub, 448 | description => 449 | #{en => 450 | <<71, 101, 116, 32, 83, 117, 98, 115, 99, 114, 105, 451 | 112, 116, 105, 111, 110, 32, 76, 105, 115, 116, 32, 452 | 70, 114, 111, 109, 32, 77, 121, 83, 81, 76>>, 453 | zh => 454 | <<228, 187, 142, 32, 77, 121, 83, 81, 76, 32, 228, 184, 455 | 173, 232, 142, 183, 229, 143, 150, 232, 174, 162, 456 | 233, 152, 133, 229, 136, 151, 232, 161, 168>>}, 457 | for => '$any', name => lookup_sub_to_mysql, 458 | params => 459 | #{'$resource' => 460 | #{description => 461 | #{en => 462 | <<66, 105, 110, 100, 32, 97, 32, 114, 101, 463 | 115, 111, 117, 114, 99, 101, 32, 116, 464 | 111, 32, 116, 104, 105, 115, 32, 97, 99, 465 | 116, 105, 111, 110>>, 466 | zh => 467 | <<231, 187, 153, 229, 138, 168, 228, 189, 468 | 156, 231, 187, 145, 229, 174, 154, 228, 469 | 184, 128, 228, 184, 170, 232, 181, 132, 470 | 230, 186, 144>>}, 471 | required => true, 472 | title => 473 | #{en => 474 | <<82, 101, 115, 111, 117, 114, 99, 101, 32, 475 | 73, 68>>, 476 | zh => 477 | <<232, 181, 132, 230, 186, 144, 32, 73, 478 | 68>>}, 479 | type => string}}, 480 | title => 481 | #{en => 482 | <<83, 117, 98, 115, 99, 114, 105, 112, 116, 105, 111, 483 | 110, 32, 76, 105, 115, 116, 32, 70, 114, 111, 109, 484 | 32, 77, 121, 83, 81, 76>>, 485 | zh => 486 | <<228, 187, 142, 32, 77, 121, 83, 81, 76, 32, 228, 184, 487 | 173, 232, 142, 183, 229, 143, 150, 232, 174, 162, 488 | 233, 152, 133, 229, 136, 151, 232, 161, 168>>}, 489 | types => [backend_mysql]}). 490 | 491 | on_resource_create(ResId, 492 | Config = #{<<"server">> := Server, <<"user">> := User, 493 | <<"database">> := DB}) -> 494 | begin 495 | logger:log(info, #{}, 496 | #{report_cb => 497 | fun (_) -> 498 | {'$logger_header'() ++ 499 | "Initiating Resource ~p, ResId: ~p", 500 | [backend_mysql, ResId]} 501 | end, 502 | mfa => 503 | {emqx_backend_mysql_actions, on_resource_create, 2}, 504 | line => 229}) 505 | end, 506 | {ok, _} = application:ensure_all_started(ecpool), 507 | {Host, Port} = host(Server), 508 | Options = [{host, str(Host)}, {port, Port}, 509 | {user, str(User)}, 510 | {password, str(maps:get(<<"password">>, Config, ""))}, 511 | {database, str(DB)}, 512 | {auto_reconnect, 513 | case maps:get(<<"auto_reconnect">>, Config, true) of 514 | true -> 15; 515 | false -> false 516 | end}, 517 | {pool_size, maps:get(<<"pool_size">>, Config, 8)}, 518 | {batch_size, maps:get(<<"batch_size">>, Config, 100)}, 519 | {batch_time, maps:get(<<"batch_time">>, Config, 10)}], 520 | PoolName = pool_name(ResId), 521 | start_resource(ResId, PoolName, Options), 522 | #{<<"pool">> => PoolName}. 523 | 524 | start_resource(ResId, PoolName, Options) -> 525 | case ecpool:start_sup_pool(PoolName, 526 | emqx_backend_mysql_actions, Options) 527 | of 528 | {ok, _} -> 529 | begin 530 | logger:log(info, #{}, 531 | #{report_cb => 532 | fun (_) -> 533 | {'$logger_header'() ++ 534 | "Initiated Resource ~p Successfully, " 535 | "ResId: ~p", 536 | [backend_mysql, ResId]} 537 | end, 538 | mfa => {emqx_backend_mysql_actions, start_resource, 3}, 539 | line => 251}) 540 | end; 541 | {error, {already_started, _Pid}} -> 542 | on_resource_destroy(ResId, #{<<"pool">> => PoolName}), 543 | start_resource(ResId, PoolName, Options); 544 | {error, Reason} -> 545 | begin 546 | logger:log(error, #{}, 547 | #{report_cb => 548 | fun (_) -> 549 | {'$logger_header'() ++ 550 | "Initiate Resource ~p failed, ResId: " 551 | "~p, ~0p", 552 | [backend_mysql, ResId, Reason]} 553 | end, 554 | mfa => {emqx_backend_mysql_actions, start_resource, 3}, 555 | line => 256}) 556 | end, 557 | error({{backend_mysql, ResId}, create_failed}) 558 | end. 559 | 560 | -spec on_get_resource_status(ResId :: binary(), 561 | Params :: map()) -> Status :: map(). 562 | 563 | on_get_resource_status(_ResId, 564 | #{<<"pool">> := PoolName}) -> 565 | Status = [ecpool_worker:is_connected(Worker) 566 | || {_WorkerName, Worker} <- ecpool:workers(PoolName)], 567 | #{is_alive => 568 | length(Status) > 0 andalso 569 | lists:all(fun (St) -> St =:= true end, Status)}. 570 | 571 | on_resource_destroy(ResId, #{<<"pool">> := PoolName}) -> 572 | begin 573 | logger:log(info, #{}, 574 | #{report_cb => 575 | fun (_) -> 576 | {'$logger_header'() ++ 577 | "Destroying Resource ~p, ResId: ~p", 578 | [backend_mysql, ResId]} 579 | end, 580 | mfa => 581 | {emqx_backend_mysql_actions, on_resource_destroy, 2}, 582 | line => 267}) 583 | end, 584 | case ecpool:stop_sup_pool(PoolName) of 585 | ok -> 586 | begin 587 | logger:log(info, #{}, 588 | #{report_cb => 589 | fun (_) -> 590 | {'$logger_header'() ++ 591 | "Destroyed Resource ~p Successfully, " 592 | "ResId: ~p", 593 | [backend_mysql, ResId]} 594 | end, 595 | mfa => 596 | {emqx_backend_mysql_actions, on_resource_destroy, 597 | 2}, 598 | line => 270}) 599 | end; 600 | {error, Reason} -> 601 | begin 602 | logger:log(error, #{}, 603 | #{report_cb => 604 | fun (_) -> 605 | {'$logger_header'() ++ 606 | "Destroy Resource ~p failed, ResId: ~p, ~p", 607 | [backend_mysql, ResId, Reason]} 608 | end, 609 | mfa => 610 | {emqx_backend_mysql_actions, on_resource_destroy, 611 | 2}, 612 | line => 272}) 613 | end, 614 | error({{backend_mysql, ResId}, destroy_failed}) 615 | end. 616 | 617 | -spec on_action_create_data_to_mysql(ActionInstId :: 618 | binary(), 619 | #{}) -> fun((Msg :: map()) -> any()). 620 | 621 | on_action_create_data_to_mysql(ActionInstId, 622 | #{<<"pool">> := PoolName, 623 | <<"sql">> := SqlTemplate}) -> 624 | PrepareSqlKey = unsafe_atom_key(ActionInstId), 625 | begin 626 | logger:log(info, #{}, 627 | #{report_cb => 628 | fun (_) -> 629 | {'$logger_header'() ++ 630 | "Initiating Action ~p, SqlTemplate: ~p, " 631 | "PrepareSqlKey: ~p", 632 | [on_action_create_data_to_mysql, SqlTemplate, 633 | PrepareSqlKey]} 634 | end, 635 | mfa => 636 | {emqx_backend_mysql_actions, 637 | on_action_create_data_to_mysql, 2}, 638 | line => 285}) 639 | end, 640 | {PrepareStatement, GetPrepareParams} = 641 | emqx_rule_utils:preproc_sql(SqlTemplate, '?'), 642 | prepare_sql(PrepareSqlKey, PrepareStatement, PoolName), 643 | ecpool:add_reconnect_callback(PoolName, 644 | fun (Batch) -> 645 | prepare_sql_to_conn(PrepareSqlKey, 646 | PrepareStatement, 647 | emqx_backend_mysql_batcher:get_cbst(Batch)) 648 | end), 649 | fun (Msg, _Envs) -> 650 | case mysql_query(PoolName, PrepareSqlKey, 651 | PrepareStatement, GetPrepareParams(Msg)) 652 | of 653 | {error, Reason} -> error({data_to_mysql, Reason}); 654 | _ -> ok 655 | end 656 | end. 657 | 658 | on_action_create_offline_msg(ActionInstId, 659 | #{<<"pool">> := PoolName, 660 | <<"time_range">> := TimeRange0, 661 | <<"max_returned_count">> := 662 | MaxReturnedCount0}) -> 663 | InsertSql = <<"insert into mqtt_msg(msgid, sender, " 664 | "topic, qos, retain, payload, arrived) " 665 | "values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?) " 666 | ")">>, 667 | DeleteSql = <<"delete from mqtt_msg where msgid = ? " 668 | "and topic = ?">>, 669 | SelectSql = <<"select id, topic, msgid, sender, qos, " 670 | "payload, retain, UNIX_TIMESTAMP(arrived) " 671 | "from mqtt_msg where topic = ? order " 672 | "by id DESC">>, 673 | InsertKey = unsafe_atom_key(<<"insert_", 674 | ActionInstId/binary>>), 675 | DeleteKey = unsafe_atom_key(<<"delete_", 676 | ActionInstId/binary>>), 677 | SelectKey = unsafe_atom_key(<<"select_", 678 | ActionInstId/binary>>), 679 | TimeRange = case to_undefined(TimeRange0) of 680 | undefined -> undefined; 681 | TimeRange0 -> cuttlefish_duration:parse(TimeRange0, s) 682 | end, 683 | MaxReturnedCount = to_undefined(MaxReturnedCount0), 684 | {SelectSql1, Params} = case {TimeRange, 685 | MaxReturnedCount} 686 | of 687 | {undefined, undefined} -> {SelectSql, []}; 688 | {TimeRange, undefined} -> 689 | {<= FROM_UNIXTIME(?) ">>, 691 | [TimeRange div 1000]}; 692 | {undefined, MaxReturnedCount} -> 693 | {<>, 694 | [MaxReturnedCount]}; 695 | {TimeRange, MaxReturnedCount} -> 696 | {<= FROM_UNIXTIME(?) limit ? ">>, 698 | [TimeRange div 1000, MaxReturnedCount]} 699 | end, 700 | prepare_sql(InsertKey, InsertSql, PoolName), 701 | prepare_sql(DeleteKey, DeleteSql, PoolName), 702 | prepare_sql(SelectKey, SelectSql1, PoolName), 703 | ecpool:add_reconnect_callback(PoolName, 704 | fun (Conn) -> 705 | prepare_sql_to_conn(InsertKey, 706 | InsertSql, Conn), 707 | prepare_sql_to_conn(SelectKey, 708 | SelectSql1, Conn), 709 | prepare_sql_to_conn(DeleteKey, 710 | DeleteSql, Conn) 711 | end), 712 | fun (Msg = #{event := Event, topic := Topic}, _Envs) -> 713 | case Event of 714 | 'message.publish' -> 715 | #{id := MsgId, qos := Qos, flags := Flags, 716 | payload := Payload, publish_received_at := Ts, 717 | clientid := From} = 718 | Msg, 719 | Retain = maps:get(retain, Flags, true), 720 | insert_message(PoolName, InsertKey, InsertSql, 721 | [MsgId, From, Topic, Qos, i(Retain), Payload, 722 | Ts div 1000]); 723 | 'session.subscribed' -> 724 | MsgList = lookup_message(PoolName, SelectKey, SelectSql, 725 | [Topic] ++ Params), 726 | [self() ! {deliver, Topic, M} || M <- MsgList]; 727 | 'message.acked' -> 728 | #{id := MsgId} = Msg, 729 | delete_message(PoolName, DeleteKey, DeleteSql, 730 | [MsgId, Topic]) 731 | end 732 | end. 733 | 734 | on_action_create_lookup_sub(ActionInstId, 735 | #{<<"pool">> := PoolName}) -> 736 | SelectPrepare = 737 | <<"select topic, qos from mqtt_sub where " 738 | "clientid = ?">>, 739 | InsertPrepare = 740 | <<"insert into mqtt_sub(clientid, topic, " 741 | "qos) values(?, ?, ?)">>, 742 | SelectKey = unsafe_atom_key(<<"select_", 743 | ActionInstId/binary>>), 744 | InsertKey = unsafe_atom_key(<<"insert_", 745 | ActionInstId/binary>>), 746 | prepare_sql(SelectKey, SelectPrepare, PoolName), 747 | prepare_sql(InsertKey, InsertPrepare, PoolName), 748 | ecpool:add_reconnect_callback(PoolName, 749 | fun (Batch) -> 750 | prepare_sql_to_conn(SelectKey, 751 | SelectPrepare, 752 | emqx_backend_mysql_batcher:get_cbst(Batch)), 753 | prepare_sql_to_conn(InsertKey, 754 | InsertPrepare, 755 | emqx_backend_mysql_batcher:get_cbst(Batch)) 756 | end), 757 | fun (Msg = #{event := Event, clientid := ClientId}, 758 | _Envs) -> 759 | case Event of 760 | 'client.connected' -> 761 | case lookup_subscribe(PoolName, SelectKey, 762 | SelectPrepare, [ClientId]) 763 | of 764 | [] -> ok; 765 | TopicTable -> self() ! {subscribe, TopicTable} 766 | end; 767 | 'session.subscribed' -> 768 | #{topic := Topic, qos := QoS} = Msg, 769 | insert_subscribe(PoolName, InsertKey, InsertPrepare, 770 | [ClientId, Topic, QoS]) 771 | end 772 | end. 773 | 774 | prepare_sql(PrepareSqlKey, PrepareStatement, 775 | PoolName) -> 776 | [begin 777 | {ok, Batch} = ecpool_worker:client(Worker), 778 | prepare_sql_to_conn(PrepareSqlKey, PrepareStatement, 779 | emqx_backend_mysql_batcher:get_cbst(Batch)), 780 | case binary:split(PrepareStatement, <<"values ">>) of 781 | [Insert, Params] -> 782 | emqx_backend_mysql_batcher:set_value(Batch, 783 | {insert_sql, PrepareSqlKey}, 784 | {binary_to_list(Insert), 785 | binary_to_list(Params)}); 786 | _ -> ok 787 | end 788 | end 789 | || {_WorkerName, Worker} <- ecpool:workers(PoolName)]. 790 | 791 | prepare_sql_to_conn(PrepareSqlKey, PrepareStatement, 792 | Conn) -> 793 | case mysql:prepare(Conn, PrepareSqlKey, 794 | PrepareStatement) 795 | of 796 | {ok, Name} -> 797 | begin 798 | logger:log(info, #{}, 799 | #{report_cb => 800 | fun (_) -> 801 | {'$logger_header'() ++ 802 | "Prepare Statement: ~p, PreparedSQL Key: ~p", 803 | [PrepareStatement, Name]} 804 | end, 805 | mfa => 806 | {emqx_backend_mysql_actions, prepare_sql_to_conn, 807 | 3}, 808 | line => 398}) 809 | end; 810 | {error, Reason} -> 811 | begin 812 | logger:log(error, #{}, 813 | #{report_cb => 814 | fun (_) -> 815 | {'$logger_header'() ++ 816 | "Prepare Statement: ~p, ~0p", 817 | [PrepareStatement, Reason]} 818 | end, 819 | mfa => 820 | {emqx_backend_mysql_actions, prepare_sql_to_conn, 821 | 3}, 822 | line => 400}) 823 | end, 824 | error(prepare_sql_failed) 825 | end. 826 | 827 | connect(Options) -> 828 | InitFun = fun () -> 829 | {ok, Conn} = mysql:start_link(Options), 830 | put(backslash_escapes_enabled, 831 | gen_server:call(Conn, backslash_escapes_enabled)), 832 | {ok, Conn} 833 | end, 834 | Handler = fun (Conn, Batch) -> 835 | {emqx_backend_mysql_actions:send_batch(Conn, Batch), 836 | Conn} 837 | end, 838 | emqx_backend_mysql_batcher:start_link(InitFun, Handler, 839 | #{batch_size => 840 | proplists:get_value(batch_size, 841 | Options), 842 | batch_time => 843 | proplists:get_value(batch_time, 844 | Options)}). 845 | 846 | mysql_query(Pool, Ref, PrepareStatement, Params) -> 847 | begin 848 | logger:log(debug, #{}, 849 | #{report_cb => 850 | fun (_) -> 851 | {'$logger_header'() ++ 852 | "Send to mysql, pool: ~p, prepared_key: " 853 | "~p, ~0p", 854 | [Pool, Ref, Params]} 855 | end, 856 | mfa => {emqx_backend_mysql_actions, mysql_query, 4}, 857 | line => 420}) 858 | end, 859 | ecpool:with_client(Pool, 860 | fun (Conn) -> 861 | Fun = fun () -> 862 | get({insert_sql, Ref}) =/= 863 | undefined 864 | end, 865 | case emqx_backend_mysql_batcher:try_batch(Conn, 866 | {Ref, 867 | Params}, 868 | Fun) 869 | of 870 | {error, disconnected} -> exit(Conn, restart); 871 | {error, not_prepared} -> 872 | prepare_sql_to_conn(Ref, PrepareStatement, 873 | emqx_backend_mysql_batcher:get_cbst(Conn)), 874 | emqx_backend_mysql_batcher:try_batch(Conn, 875 | {Ref, 876 | Params}, 877 | Fun); 878 | Res -> Res 879 | end 880 | end). 881 | 882 | mysql_query_not_batch(Pool, Ref, PrepareStatement, 883 | Params) -> 884 | begin 885 | logger:log(debug, #{}, 886 | #{report_cb => 887 | fun (_) -> 888 | {'$logger_header'() ++ 889 | "Send to mysql, pool: ~p, prepared_key: " 890 | "~p, ~0p", 891 | [Pool, Ref, Params]} 892 | end, 893 | mfa => 894 | {emqx_backend_mysql_actions, mysql_query_not_batch, 4}, 895 | line => 434}) 896 | end, 897 | ecpool:with_client(Pool, 898 | fun (Conn) -> 899 | MConn = 900 | emqx_backend_mysql_batcher:get_cbst(Conn), 901 | case mysql:execute(MConn, Ref, Params) of 902 | {error, disconnected} -> exit(Conn, restart); 903 | {error, not_prepared} -> 904 | prepare_sql_to_conn(Ref, PrepareStatement, 905 | MConn), 906 | mysql:execute(MConn, Ref, Params); 907 | Res -> Res 908 | end 909 | end). 910 | 911 | send_batch(Conn, {Ref, Params}) -> 912 | mysql:execute(Conn, Ref, Params); 913 | send_batch(Conn, Batch) -> send_inserts(Conn, Batch). 914 | 915 | send_inserts(Conn, Msgs) -> 916 | Msgs1 = maps:to_list(lists:foldl(fun ({_, K, V}, Acc) -> 917 | L = maps:get(K, Acc, []), 918 | Acc#{K => [V | L]} 919 | end, 920 | #{}, Msgs)), 921 | lists:map(fun ({Ref, Msgs2}) -> 922 | SQL = insert_batch_sql(Ref, Msgs), 923 | case mysql:query(Conn, SQL, lists:flatten(Msgs2)) of 924 | {error, Reason} -> error(insert_error, Reason); 925 | _ -> lists:map(fun (_) -> ok end, Msgs) 926 | end 927 | end, 928 | Msgs1). 929 | 930 | insert_batch_sql(Ref, Msgs) -> 931 | {InsertSQL, ValuesSQL} = get({insert_sql, Ref}), 932 | [InsertSQL, "values ", values(Msgs, ValuesSQL)]. 933 | 934 | values(Msgs, ValuesSQL) -> values(Msgs, ValuesSQL, []). 935 | 936 | values([], _, Acc) -> Acc; 937 | values([_ | Rest], ValuesSQL, Acc) -> 938 | NewAcc = case Rest == [] of 939 | true -> Acc ++ ValuesSQL; 940 | false -> Acc ++ ValuesSQL ++ ", " 941 | end, 942 | values(Rest, ValuesSQL, NewAcc). 943 | 944 | pool_name(ResId) -> 945 | list_to_atom("backend_mysql:" ++ str(ResId)). 946 | 947 | host(Server) when is_binary(Server) -> 948 | case string:split(Server, ":") of 949 | [Host, Port] -> {Host, binary_to_integer(Port)}; 950 | [Host] -> {Host, 3306} 951 | end. 952 | 953 | insert_message(Pool, PrepareSqlKey, PrepareStatement, 954 | PrepareParams) -> 955 | case mysql_query_not_batch(Pool, PrepareSqlKey, 956 | PrepareStatement, PrepareParams) 957 | of 958 | {error, Error} -> 959 | begin 960 | logger:log(error, #{}, 961 | #{report_cb => 962 | fun (_) -> 963 | {'$logger_header'() ++ 964 | "Failed to store message: ~p", 965 | [Error]} 966 | end, 967 | mfa => {emqx_backend_mysql_actions, insert_message, 4}, 968 | line => 492}) 969 | end; 970 | _ -> ok 971 | end. 972 | 973 | lookup_message(Pool, PrepareSqlKey, PrepareStatement, 974 | PrepareParams) -> 975 | case mysql_query_not_batch(Pool, PrepareSqlKey, 976 | PrepareStatement, PrepareParams) 977 | of 978 | {ok, _Cols, []} -> []; 979 | {ok, Cols, Rows} -> 980 | [begin 981 | Msg = 982 | emqx_backend_mysql_cli:record_to_msg(lists:zip(Cols, 983 | Row)), 984 | Msg#message{id = emqx_guid:from_hexstr(Msg#message.id)} 985 | end 986 | || Row <- lists:reverse(Rows)]; 987 | {error, Error} -> 988 | begin 989 | logger:log(error, #{}, 990 | #{report_cb => 991 | fun (_) -> 992 | {'$logger_header'() ++ 993 | "Failed to lookup message error: ~p", 994 | [Error]} 995 | end, 996 | mfa => {emqx_backend_mysql_actions, lookup_message, 4}, 997 | line => 503}) 998 | end, 999 | [] 1000 | end. 1001 | 1002 | delete_message(Pool, PrepareSqlKey, PrepareStatement, 1003 | PrepareParams) -> 1004 | case mysql_query_not_batch(Pool, PrepareSqlKey, 1005 | PrepareStatement, PrepareParams) 1006 | of 1007 | {error, Error} -> 1008 | begin 1009 | logger:log(error, #{}, 1010 | #{report_cb => 1011 | fun (_) -> 1012 | {'$logger_header'() ++ 1013 | "Failed to delete message: ~p", 1014 | [Error]} 1015 | end, 1016 | mfa => {emqx_backend_mysql_actions, delete_message, 4}, 1017 | line => 508}) 1018 | end; 1019 | _ -> ok 1020 | end. 1021 | 1022 | insert_subscribe(Pool, PrepareSqlKey, PrepareStatement, 1023 | PrepareParams) -> 1024 | case mysql_query_not_batch(Pool, PrepareSqlKey, 1025 | PrepareStatement, PrepareParams) 1026 | of 1027 | {error, Error} -> 1028 | begin 1029 | logger:log(error, #{}, 1030 | #{report_cb => 1031 | fun (_) -> 1032 | {'$logger_header'() ++ 1033 | "Failed to store subscribe: ~p", 1034 | [Error]} 1035 | end, 1036 | mfa => 1037 | {emqx_backend_mysql_actions, insert_subscribe, 4}, 1038 | line => 514}) 1039 | end; 1040 | _ -> ok 1041 | end. 1042 | 1043 | lookup_subscribe(Pool, PrepareSqlKey, PrepareStatement, 1044 | PrepareParams) -> 1045 | case mysql_query_not_batch(Pool, PrepareSqlKey, 1046 | PrepareStatement, PrepareParams) 1047 | of 1048 | {ok, _Cols, []} -> []; 1049 | {error, Error} -> 1050 | begin 1051 | logger:log(error, #{}, 1052 | #{report_cb => 1053 | fun (_) -> 1054 | {'$logger_header'() ++ 1055 | "Lookup subscription error: ~p", 1056 | [Error]} 1057 | end, 1058 | mfa => 1059 | {emqx_backend_mysql_actions, lookup_subscribe, 4}, 1060 | line => 521}) 1061 | end, 1062 | []; 1063 | {ok, _Cols, Rows} -> 1064 | [{Topic, #{qos => Qos}} || [Topic, Qos] <- Rows] 1065 | end. 1066 | 1067 | i(true) -> 1; 1068 | i(false) -> 0. 1069 | 1070 | to_undefined(<<>>) -> undefined; 1071 | to_undefined(0) -> undefined; 1072 | to_undefined(V) -> V. 1073 | 1074 | '$logger_header'() -> "[RuleEngine MySql] ". 1075 | 1076 | 1077 | -------------------------------------------------------------------------------- /src/emqx_backend_mysql_app.erl: -------------------------------------------------------------------------------- 1 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 2 | "mqx_backend_mysql_app.erl", 3 | 1). 4 | 5 | -module(emqx_backend_mysql_app). 6 | 7 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/inclu" 8 | "de/emqx_backend_mysql.hrl", 9 | 1). 10 | 11 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 12 | "mqx_backend_mysql_app.erl", 13 | 10). 14 | 15 | -behaviour(application). 16 | 17 | -emqx_plugin(backend). 18 | 19 | -export([start/2, stop/1]). 20 | 21 | start(_Type, _Args) -> 22 | Pools = application:get_env(emqx_backend_mysql, pools, 23 | []), 24 | {ok, Sup} = emqx_backend_mysql_sup:start_link(Pools), 25 | emqx_backend_mysql:register_metrics(), 26 | emqx_backend_mysql:load(), 27 | {ok, Sup}. 28 | 29 | stop(_State) -> emqx_backend_mysql:unload(). 30 | 31 | 32 | -------------------------------------------------------------------------------- /src/emqx_backend_mysql_batcher.erl: -------------------------------------------------------------------------------- 1 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 2 | "mqx_backend_mysql_batcher.erl", 3 | 1). 4 | 5 | -module(emqx_backend_mysql_batcher). 6 | 7 | -behaviour(gen_statem). 8 | 9 | -export([start_link/3, call/2, call/3, get_cbst/1, 10 | try_batch/3, set_value/3]). 11 | 12 | -export([callback_mode/0, init/1]). 13 | 14 | -export([do/3]). 15 | 16 | callback_mode() -> [state_functions, state_enter]. 17 | 18 | start_link(InitFun, CallHandler, Opts) -> 19 | gen_statem:start_link(emqx_backend_mysql_batcher, 20 | {InitFun, CallHandler, Opts}, []). 21 | 22 | call(Pid, Data) -> gen_statem:call(Pid, {call, Data}). 23 | 24 | call(Pid, Data, Opts) -> 25 | gen_statem:call(Pid, {call, Data, Opts}). 26 | 27 | try_batch(Pid, Data, Fun) -> 28 | gen_statem:call(Pid, {try_batch, Data, Fun}). 29 | 30 | set_value(Pid, Key, Value) -> 31 | gen_statem:cast(Pid, {set_value, Key, Value}). 32 | 33 | get_cbst(Pid) -> gen_statem:call(Pid, get_cbst). 34 | 35 | init({InitFun, CallHandler, Opts}) -> 36 | {ok, CbSt} = InitFun(), 37 | BatchSize = maps:get(batch_size, Opts, 100), 38 | BatchTime = maps:get(batch_time, Opts, 10), 39 | St = #{batch_size => BatchSize, batch_time => BatchTime, 40 | handler => CallHandler, cb_st => CbSt, acc => [], 41 | acc_left => BatchSize}, 42 | {ok, do, St}. 43 | 44 | do(enter, _, #{batch_time := T}) -> 45 | Action = {timeout, T, flush}, 46 | {keep_state_and_data, [Action]}; 47 | do({call, From}, {call, Data, flush}, St) -> 48 | {repeat_state, flush(From, Data, St)}; 49 | do({call, From}, {call, Data}, 50 | #{acc := Acc, acc_left := Left} = St0) -> 51 | St = St0#{acc := [{From, Data} | Acc], 52 | acc_left := Left - 1}, 53 | case Left =< 1 of 54 | true -> {repeat_state, flush(St)}; 55 | false -> {repeat_state, St} 56 | end; 57 | do({call, From}, {try_batch, {Ref, Data}, Fun}, St) -> 58 | case Fun() of 59 | true -> do({call, From}, {call, {i, Ref, Data}}, St); 60 | false -> 61 | do({call, From}, {call, {Ref, Data}, flush}, St) 62 | end; 63 | do(cast, {set_value, Key, Value}, St) -> 64 | put(Key, Value), {repeat_state, St}; 65 | do({call, From}, get_cbst, #{cb_st := CbSt}) -> 66 | {keep_state_and_data, [{reply, From, CbSt}]}; 67 | do(info, {timeout, _Ref, flush}, St) -> 68 | {repeat_state, flush(St)}; 69 | do(timeout, flush, St) -> {repeat_state, flush(St)}. 70 | 71 | flush(#{acc := []} = St) -> St; 72 | flush(#{acc := Acc, batch_size := Size, cb_st := CbSt, 73 | handler := Handler} = 74 | St) -> 75 | {Froms, Batch} = lists:unzip(lists:reverse(Acc)), 76 | {Results, NewCbSt} = Handler(CbSt, Batch), 77 | lists:foreach(fun ({From, Result}) -> 78 | gen_statem:reply(From, Result) 79 | end, 80 | lists:zip(Froms, Results)), 81 | St#{cb_st := NewCbSt, acc_left := Size, acc := []}. 82 | 83 | flush(From, Data, 84 | #{cb_st := CbSt, handler := Handler} = St) -> 85 | {Result, NewCbSt} = Handler(CbSt, Data), 86 | gen_statem:reply(From, Result), 87 | St#{cb_st := NewCbSt}. 88 | 89 | 90 | -------------------------------------------------------------------------------- /src/emqx_backend_mysql_cli.erl: -------------------------------------------------------------------------------- 1 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 2 | "mqx_backend_mysql_cli.erl", 3 | 1). 4 | 5 | -module(emqx_backend_mysql_cli). 6 | 7 | -file("/emqx_rel/_checkouts/emqx/include/emqx.hrl", 1). 8 | 9 | -record(subscription, {topic, subid, subopts}). 10 | 11 | -record(message, 12 | {id :: binary(), qos = 0, from :: atom() | binary(), 13 | flags :: #{atom() => boolean()}, headers :: map(), 14 | topic :: binary(), payload :: binary(), 15 | timestamp :: integer()}). 16 | 17 | -record(delivery, 18 | {sender :: pid(), message :: #message{}}). 19 | 20 | -record(route, 21 | {topic :: binary(), 22 | dest :: node() | {binary(), node()}}). 23 | 24 | -type trie_node_id() :: binary() | atom(). 25 | 26 | -record(trie_node, 27 | {node_id :: trie_node_id(), 28 | edge_count = 0 :: non_neg_integer(), 29 | topic :: binary() | undefined, flags :: [atom()]}). 30 | 31 | -record(trie_edge, 32 | {node_id :: trie_node_id(), 33 | word :: binary() | atom()}). 34 | 35 | -record(trie, 36 | {edge :: #trie_edge{}, node_id :: trie_node_id()}). 37 | 38 | -record(alarm, 39 | {id :: binary(), 40 | severity :: notice | warning | error | critical, 41 | title :: iolist(), summary :: iolist(), 42 | timestamp :: erlang:timestamp()}). 43 | 44 | -record(plugin, 45 | {name :: atom(), dir :: string(), descr :: string(), 46 | vendor :: string(), active = false :: boolean(), 47 | info :: map(), type :: atom()}). 48 | 49 | -record(command, 50 | {name :: atom(), action :: atom(), 51 | args = [] :: list(), opts = [] :: list(), 52 | usage :: string(), descr :: string()}). 53 | 54 | -record(banned, 55 | {who :: 56 | {clientid, binary()} | {username, binary()} | 57 | {ip_address, inet:ip_address()}, 58 | by :: binary(), reason :: binary(), at :: integer(), 59 | until :: integer()}). 60 | 61 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 62 | "mqx_backend_mysql_cli.erl", 63 | 10). 64 | 65 | -export([client_connected/2, subscribe_lookup/2, 66 | client_disconnected/2, message_fetch/3, lookup_retain/2, 67 | message_publish/2, message_store/2, message_retain/2, 68 | delete_retain/2, message_acked/2, run_mysql_sql/3]). 69 | 70 | -export([record_to_msg/1]). 71 | 72 | -export([connect/1, send_batch/2]). 73 | 74 | client_connected(Pool, Msg) -> 75 | mysql_execute(Pool, client_connected_query, 76 | [proplists:get_value(clientid, Msg), 1, 77 | atom_to_list(node()), 1, atom_to_list(node())]), 78 | ok. 79 | 80 | subscribe_lookup(Pool, Msg) -> 81 | case mysql_execute(Pool, subscribe_lookup_query, 82 | feed_var([<<"${clientid}">>], Msg)) 83 | of 84 | {ok, _Cols, []} -> []; 85 | {error, Error} -> 86 | logger:error("Lookup retain error: ~p", [Error]), []; 87 | {ok, _Cols, Rows} -> 88 | [{Topic, #{qos => Qos}} || [Topic, Qos] <- Rows] 89 | end. 90 | 91 | client_disconnected(Pool, Msg) -> 92 | mysql_execute(Pool, client_disconnected_query, 93 | [0, proplists:get_value(clientid, Msg)]), 94 | ok. 95 | 96 | message_fetch(Pool, Msg, Opts) -> 97 | Topic = proplists:get_value(topic, Msg), 98 | ClientId = proplists:get_value(clientid, Msg), 99 | case mysql_execute(Pool, message_fetch_query, 100 | feed_var([<<"${topic}">>, <<"${clientid}">>], Msg)) 101 | of 102 | {ok, [_], []} -> 103 | MsgId = case mysql_execute(Pool, message_lastid_query, 104 | [Topic]) 105 | of 106 | {ok, [_], []} -> 0; 107 | {ok, [_], [[MId]]} -> MId; 108 | {error, Error2} -> 109 | logger:error("Lookup msg error: ~p", [Error2]), 0 110 | end, 111 | mysql_execute(Pool, insert_acked_query, 112 | [ClientId, Topic, MsgId]), 113 | []; 114 | {ok, [_], [[AckId]]} -> 115 | Sql = <<"select id, topic, msgid, sender, qos, " 116 | "payload, retain, UNIX_TIMESTAMP(arrived) " 117 | "as arrived from mqtt_msg where id > " 118 | "? and topic = ? ">>, 119 | {Sql1, Params1} = case proplists:get_value(time_range, 120 | Opts) 121 | of 122 | undefined -> 123 | {<<" order by id DESC ">>, [AckId, Topic]}; 124 | TimeRange -> 125 | Time = erlang:system_time(seconds) - 126 | TimeRange, 127 | {<<" and arrived >= FROM_UNIXTIME(?) order " 128 | "by id DESC ">>, 129 | [AckId, Topic, Time]} 130 | end, 131 | {Sql2, Params2} = case 132 | proplists:get_value(max_returned_count, Opts) 133 | of 134 | undefined -> {<<"">>, Params1}; 135 | MaxReturnedCount -> 136 | {<<" limit ?">>, 137 | Params1 ++ [MaxReturnedCount]} 138 | end, 139 | case mysql_query(Pool, 140 | <>, Params2) 141 | of 142 | {ok, _Cols, []} -> []; 143 | {ok, Cols, Rows} -> 144 | [begin 145 | M = record_to_msg(lists:zip(Cols, Row)), 146 | M#message{id = emqx_guid:from_base62(M#message.id)} 147 | end 148 | || Row <- lists:reverse(Rows)]; 149 | {error, Error1} -> 150 | logger:error("Lookup msg error: ~p", [Error1]), [] 151 | end; 152 | {error, Error} -> 153 | logger:error("Lookup msg error: ~p", [Error]), [] 154 | end. 155 | 156 | lookup_retain(Pool, Msg0) -> 157 | case mysql_execute(Pool, lookup_retain_query, 158 | [proplists:get_value(topic, Msg0)]) 159 | of 160 | {ok, Cols, [Row | _]} -> 161 | M = record_to_msg(lists:zip(Cols, Row)), 162 | [M#message{id = emqx_guid:from_base62(M#message.id)}]; 163 | {ok, _Cols, []} -> []; 164 | {error, Error} -> 165 | logger:error("Lookup retain error: ~p", [Error]), [] 166 | end. 167 | 168 | message_publish(Pool, Msg) -> 169 | ParamsKey = [<<"${msgid}">>, <<"${clientid}">>, 170 | <<"${topic}">>, <<"${qos}">>, <<"${retain}">>, 171 | <<"${payload}">>, <<"${timestamp}">>], 172 | case mysql_insert(Pool, message_publish_query, 173 | feed_var(ParamsKey, Msg)) 174 | of 175 | {ok, Id} -> emqx_message:set_header(mysql_id, Id, Msg); 176 | {error, Error} -> 177 | logger:error("Failed to store message: ~p", [Error]), 178 | Msg 179 | end. 180 | 181 | message_store(Pool, Msg) -> 182 | case mysql_insert_batch(Pool, Msg) of 183 | {error, Error} -> 184 | logger:error("Failed to store message: ~p", [Error]), 185 | Msg; 186 | _ -> Msg 187 | end. 188 | 189 | message_retain(Pool, Msg) -> 190 | ParamsKey = [<<"${topic}">>, <<"${msgid}">>, 191 | <<"${clientid}">>, <<"${qos}">>, <<"${payload}">>, 192 | <<"${timestamp}">>, <<"${msgid}">>, <<"${clientid}">>, 193 | <<"${qos}">>, <<"${payload}">>, <<"${timestamp}">>], 194 | case mysql_execute(Pool, message_retain_query, 195 | feed_var(ParamsKey, Msg)) 196 | of 197 | ok -> Msg; 198 | {error, Error} -> 199 | logger:error("Failed to retain message: ~p", [Error]), 200 | Msg 201 | end. 202 | 203 | delete_retain(Pool, Msg) -> 204 | case mysql_execute(Pool, delete_retain_query, 205 | feed_var([<<"${topic}">>], Msg)) 206 | of 207 | ok -> Msg; 208 | {error, Error} -> 209 | logger:error("Failed to delete retain: ~p", [Error]), 210 | Msg 211 | end. 212 | 213 | message_acked(Pool, Msg) -> 214 | ParamsKey = [<<"${clientid}">>, <<"${topic}">>, 215 | <<"${mysql_id}">>, <<"${mysql_id}">>], 216 | case mysql_execute(Pool, message_acked_query, 217 | feed_var(ParamsKey, Msg)) 218 | of 219 | ok -> ok; 220 | {error, Error} -> 221 | logger:error("Failed to ack message: ~p", [Error]) 222 | end. 223 | 224 | run_mysql_sql(Pool, Msg, SqlList) -> 225 | lists:foreach(fun ({Sql, ParamsKey}) -> 226 | case mysql_query(Pool, Sql, feed_var(ParamsKey, Msg)) 227 | of 228 | ok -> ok; 229 | {error, Error} -> 230 | logger:error("Sql:~p~n, params:~p, error: ~p", 231 | [Sql, ParamsKey, Error]) 232 | end 233 | end, 234 | SqlList), 235 | Msg. 236 | 237 | feed_var(Params, Msg) -> feed_var(Params, Msg, []). 238 | 239 | feed_var([], _Msg, Acc) -> lists:reverse(Acc); 240 | feed_var([<<"${topic}">> | Params], 241 | Msg = #message{topic = Topic}, Acc) -> 242 | feed_var(Params, Msg, [Topic | Acc]); 243 | feed_var([<<"${topic}">> | Params], Vals, Acc) 244 | when is_list(Vals) -> 245 | feed_var(Params, Vals, 246 | [proplists:get_value(topic, Vals, null) | Acc]); 247 | feed_var([<<"${msgid}">> | Params], 248 | Msg = #message{id = MsgId}, Acc) -> 249 | feed_var(Params, Msg, 250 | [emqx_guid:to_base62(MsgId) | Acc]); 251 | feed_var([<<"${msgid}">> | Params], Vals, Acc) 252 | when is_list(Vals) -> 253 | feed_var(Params, Vals, 254 | [emqx_guid:to_base62(proplists:get_value(msgid, Vals, 255 | null)) 256 | | Acc]); 257 | feed_var([<<"${mysql_id}">> | Params], Vals, Acc) 258 | when is_list(Vals) -> 259 | feed_var(Params, Vals, 260 | [proplists:get_value(mysql_id, Vals, null) | Acc]); 261 | feed_var([<<"${clientid}">> | Params], 262 | Msg = #message{from = ClientId}, Acc) 263 | when is_atom(ClientId) -> 264 | feed_var(Params, Msg, 265 | [atom_to_binary(ClientId, utf8) | Acc]); 266 | feed_var([<<"${clientid}">> | Params], 267 | Msg = #message{from = ClientId}, Acc) 268 | when is_binary(ClientId) -> 269 | feed_var(Params, Msg, [ClientId | Acc]); 270 | feed_var([<<"${clientid}">> | Params], Vals, Acc) 271 | when is_list(Vals) -> 272 | feed_var(Params, Vals, 273 | [proplists:get_value(clientid, Vals, null) | Acc]); 274 | feed_var([<<"${qos}">> | Params], 275 | Msg = #message{qos = Qos}, Acc) -> 276 | feed_var(Params, Msg, [Qos | Acc]); 277 | feed_var([<<"${qos}">> | Params], Vals, Acc) 278 | when is_list(Vals) -> 279 | feed_var(Params, Vals, 280 | [proplists:get_value(qos, Vals, null) | Acc]); 281 | feed_var([<<"${retain}">> | Params], 282 | Msg = #message{flags = #{retain := Retain}}, Acc) -> 283 | feed_var(Params, Msg, [i(Retain) | Acc]); 284 | feed_var([<<"${payload}">> | Params], 285 | Msg = #message{payload = Payload}, Acc) -> 286 | feed_var(Params, Msg, [Payload | Acc]); 287 | feed_var([<<"${timestamp}">> | Params], 288 | Msg = #message{timestamp = Ts}, Acc) -> 289 | feed_var(Params, Msg, [round(Ts / 1000) | Acc]); 290 | feed_var([<<"${timestamp_str}">> | Params], 291 | Msg = #message{timestamp = Ts}, Acc) -> 292 | {{Y, M, D}, {H, Mm, S}} = 293 | calendar:system_time_to_local_time(Ts, millisecond), 294 | TsStr = 295 | iolist_to_binary(io_lib:format("~w-~2.2.0w-~2.2.0w ~2.2.0w:~2.2.0w:~2.2.0w", 296 | [Y, M, D, H, Mm, S])), 297 | feed_var(Params, Msg, [TsStr | Acc]); 298 | feed_var([_ | Params], Msg, Acc) -> 299 | feed_var(Params, Msg, [null | Acc]). 300 | 301 | i(true) -> 1; 302 | i(false) -> 0. 303 | 304 | b(0) -> false; 305 | b(1) -> true. 306 | 307 | record_to_msg(Record) -> 308 | record_to_msg(Record, #message{headers = #{}}). 309 | 310 | record_to_msg([], Msg) -> Msg; 311 | record_to_msg([{<<"id">>, Id} | Record], Msg) -> 312 | record_to_msg(Record, 313 | emqx_message:set_header(mysql_id, Id, Msg)); 314 | record_to_msg([{<<"msgid">>, MsgId} | Record], Msg) -> 315 | record_to_msg(Record, Msg#message{id = MsgId}); 316 | record_to_msg([{<<"topic">>, Topic} | Record], Msg) -> 317 | record_to_msg(Record, Msg#message{topic = Topic}); 318 | record_to_msg([{<<"sender">>, Sender} | Record], Msg) -> 319 | record_to_msg(Record, Msg#message{from = Sender}); 320 | record_to_msg([{<<"qos">>, Qos} | Record], Msg) -> 321 | record_to_msg(Record, Msg#message{qos = Qos}); 322 | record_to_msg([{<<"retain">>, R} | Record], Msg) -> 323 | record_to_msg(Record, 324 | Msg#message{flags = #{retain => b(R)}}); 325 | record_to_msg([{<<"payload">>, Payload} | Record], 326 | Msg) -> 327 | record_to_msg(Record, Msg#message{payload = Payload}); 328 | record_to_msg([{<<"arrived">>, Arrived} | Record], 329 | Msg) -> 330 | record_to_msg(Record, Msg#message{timestamp = Arrived}); 331 | record_to_msg([_ | Record], Msg) -> 332 | record_to_msg(Record, Msg). 333 | 334 | connect(Options) -> 335 | Prepares = [{client_connected_query, 336 | <<"insert into mqtt_client(clientid, state, " 337 | "node, online_at, offline_at) values(?, " 338 | "?, ?, now(), null) on duplicate key " 339 | "update state = ?, node = ?, online_at " 340 | "= now(), offline_at = null">>}, 341 | {subscribe_lookup_query, 342 | <<"select topic, qos from mqtt_sub where " 343 | "clientid = ?">>}, 344 | {client_disconnected_query, 345 | <<"update mqtt_client set state = ?, offline_at " 346 | "= now() where clientid = ?">>}, 347 | {message_fetch_query, 348 | <<"select mid from mqtt_acked where topic " 349 | "= ? and clientid = ?">>}, 350 | {message_lastid_query, 351 | <<"select id from mqtt_msg where topic " 352 | "= ? order by id DESC limit 1">>}, 353 | {insert_acked_query, 354 | <<"insert into mqtt_acked(clientid, topic, " 355 | "mid) values(?, ?, ?)">>}, 356 | {lookup_retain_query, 357 | <<"select topic, msgid, sender, qos, payload, " 358 | "UNIX_TIMESTAMP(arrived) as arrived from " 359 | "mqtt_retain where topic = ?">>}, 360 | {message_publish_query, 361 | <<"insert into mqtt_msg(msgid, sender, " 362 | "topic, qos, retain, payload, arrived) " 363 | "values (?, ?, ?, ?, ?, ?, FROM_UNIXTIME(?))">>}, 364 | {message_retain_query, 365 | <<"insert into mqtt_retain(topic, msgid, " 366 | "sender, qos, payload, arrived) values " 367 | "(?, ?, ?, ?, ?, FROM_UNIXTIME(?))on " 368 | "duplicate key update msgid = ?, sender " 369 | "= ?, qos = ?, payload = ?, arrived = " 370 | "FROM_UNIXTIME(?)">>}, 371 | {delete_retain_query, 372 | <<"delete from mqtt_retain where topic " 373 | "= ?">>}, 374 | {message_acked_query, 375 | <<"insert into mqtt_acked(clientid, topic, " 376 | "mid) values(?, ?, ?) on duplicate key " 377 | "update mid = ?">>}], 378 | InitFun = fun () -> 379 | {ok, Conn} = mysql:start_link([{prepare, Prepares} 380 | | Options]), 381 | put(backslash_escapes_enabled, 382 | gen_server:call(Conn, backslash_escapes_enabled)), 383 | {ok, Conn} 384 | end, 385 | Handler = fun (Conn, Batch) -> 386 | {emqx_backend_mysql_cli:send_batch(Conn, Batch), Conn} 387 | end, 388 | emqx_backend_mysql_batcher:start_link(InitFun, Handler, 389 | #{}). 390 | 391 | mysql_insert(Pool, Ref, Params) -> 392 | logger:debug("Pool:~p, Statement: ~p, Params:~p", 393 | [Pool, Ref, Params]), 394 | ecpool:with_client(Pool, 395 | fun (C) -> 396 | emqx_backend_mysql_batcher:call(C, 397 | {insert_returning_id, 398 | Ref, Params}, 399 | flush) 400 | end). 401 | 402 | mysql_query(Pool, Sql, Params) -> 403 | logger:debug("Pool:~p, SQL: ~p, Params:~p", 404 | [Pool, Sql, Params]), 405 | ecpool:with_client(Pool, 406 | fun (C) -> 407 | emqx_backend_mysql_batcher:call(C, 408 | {query, Sql, 409 | Params}, 410 | flush) 411 | end). 412 | 413 | mysql_execute(Pool, Ref, Params) -> 414 | logger:debug("Pool:~p, Statement: ~p, Params:~p", 415 | [Pool, Ref, Params]), 416 | ecpool:with_client(Pool, 417 | fun (C) -> 418 | emqx_backend_mysql_batcher:call(C, 419 | {execute, Ref, 420 | Params}, 421 | flush) 422 | end). 423 | 424 | mysql_insert_batch(Pool, Msg) -> 425 | logger:debug("Pool:~p, batch insert Msg: ~p", 426 | [Pool, Msg]), 427 | ecpool:with_client(Pool, 428 | fun (C) -> emqx_backend_mysql_batcher:call(C, Msg) end). 429 | 430 | send_batch(Conn, {What, Sql, Params}) -> 431 | send_one(What, Conn, Sql, Params); 432 | send_batch(Conn, Batch) -> send_inserts(Conn, Batch). 433 | 434 | send_one(insert_returning_id, Conn, Sql, Params) -> 435 | case mysql:execute(Conn, Sql, Params) of 436 | ok -> {ok, mysql:insert_id(Conn)}; 437 | {error, Error} -> {error, Error} 438 | end; 439 | send_one(query, Conn, Sql, Params) -> 440 | mysql:query(Conn, Sql, Params); 441 | send_one(execute, Conn, Sql, Params) -> 442 | mysql:execute(Conn, Sql, Params). 443 | 444 | send_inserts(_Conn, []) -> []; 445 | send_inserts(Conn, Msgs) -> 446 | SQL = insert_msgs_sql(Msgs), 447 | mysql:query(Conn, SQL), 448 | lists:map(fun (_) -> ok end, Msgs). 449 | 450 | insert_msgs_sql(Msgs) -> 451 | ParamsKey = [<<"${msgid}">>, <<"${clientid}">>, 452 | <<"${topic}">>, <<"${qos}">>, <<"${retain}">>, 453 | <<"${payload}">>, <<"${timestamp_str}">>], 454 | Values = lists:map(fun (Msg) -> 455 | lists:map(fun quote/1, feed_var(ParamsKey, Msg)) 456 | end, 457 | Msgs), 458 | ["insert into mqtt_msg(msgid, sender, " 459 | "topic, qos, retain, payload, arrived) ", 460 | "values ", values(Values)]. 461 | 462 | values([]) -> []; 463 | values([Columns | Rest]) -> 464 | ["(", infix(Columns, ", "), ")", 465 | case Rest of 466 | [] -> ""; 467 | _ -> ", " 468 | end 469 | | values(Rest)]. 470 | 471 | infix([X], _) -> [X]; 472 | infix([H | T], In) -> [H, In | infix(T, In)]. 473 | 474 | quote(V) when is_integer(V) -> integer_to_list(V); 475 | quote(V) when is_atom(V) -> atom_to_binary(V, utf8); 476 | quote(V) when is_list(V) -> 477 | quote(unicode:characters_to_binary(V)); 478 | quote(V) when is_binary(V) -> quote_str(V). 479 | 480 | quote_str(Bin0) when is_binary(Bin0) -> 481 | Bin = case get(backslash_escapes_enabled) of 482 | true -> 483 | binary:replace(Bin0, <<"\\">>, <<"\\\\">>, [global]); 484 | false -> Bin0 485 | end, 486 | Escaped = binary:replace(Bin, <<"'">>, <<"''">>, 487 | [global]), 488 | [$', Escaped, $']. 489 | 490 | 491 | -------------------------------------------------------------------------------- /src/emqx_backend_mysql_sup.erl: -------------------------------------------------------------------------------- 1 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 2 | "mqx_backend_mysql_sup.erl", 3 | 1). 4 | 5 | -module(emqx_backend_mysql_sup). 6 | 7 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/inclu" 8 | "de/emqx_backend_mysql.hrl", 9 | 1). 10 | 11 | -file("/emqx_rel/_checkouts/emqx_backend_mysql/src/e" 12 | "mqx_backend_mysql_sup.erl", 13 | 10). 14 | 15 | -behaviour(supervisor). 16 | 17 | -export([start_link/1]). 18 | 19 | -export([init/1]). 20 | 21 | start_link(Pools) -> 22 | supervisor:start_link({local, emqx_backend_mysql_sup}, 23 | emqx_backend_mysql_sup, [Pools]). 24 | 25 | init([Pools]) -> 26 | {ok, 27 | {{one_for_one, 10, 100}, 28 | [pool_spec(Pool, Env) || {Pool, Env} <- Pools]}}. 29 | 30 | pool_spec(Pool, Env) -> 31 | ecpool:pool_spec({emqx_backend_mysql, Pool}, 32 | emqx_backend_mysql:pool_name(Pool), 33 | emqx_backend_mysql_cli, Env). 34 | 35 | 36 | -------------------------------------------------------------------------------- /test/emqx_backend_mysql_SUITE.erl: -------------------------------------------------------------------------------- 1 | %%-------------------------------------------------------------------- 2 | %% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. 3 | %% 4 | %% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %% you may not use this file except in compliance with the License. 6 | %% You may obtain a copy of the License at 7 | %% 8 | %% http://www.apache.org/licenses/LICENSE-2.0 9 | %% 10 | %% Unless required by applicable law or agreed to in writing, software 11 | %% distributed under the License is distributed on an "AS IS" BASIS, 12 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %% See the License for the specific language governing permissions and 14 | %% limitations under the License. 15 | %%-------------------------------------------------------------------- 16 | 17 | -module(emqx_backend_mysql_SUITE). 18 | 19 | -compile(export_all). 20 | 21 | all() -> []. 22 | 23 | groups() -> []. 24 | --------------------------------------------------------------------------------