├── .drone.yml
├── .erlang
├── .gitignore
├── CHANGES.md
├── Gihufile
├── LICENSE
├── Makefile
├── README.md
├── bump-version
├── include
├── lmq.hrl
└── lmq_test.hrl
├── lmq.sublime-project
├── rebar
├── rebar.config
├── rel
├── files
│ ├── app.config
│ ├── erl
│ ├── install_upgrade.escript
│ ├── lmq
│ ├── lmq-admin
│ ├── lmq.cmd
│ ├── nodetool
│ ├── start_erl.cmd
│ └── vm.args
└── reltool.config
├── src
├── influxdb_client.erl
├── lmq.app.src
├── lmq.erl
├── lmq_api.erl
├── lmq_app.erl
├── lmq_console.erl
├── lmq_cow_msg.erl
├── lmq_cow_prop.erl
├── lmq_cow_queue.erl
├── lmq_cow_reply.erl
├── lmq_cow_stats.erl
├── lmq_event.erl
├── lmq_handler.erl
├── lmq_handler_dist.erl
├── lmq_hook.erl
├── lmq_hook_preserve_header.erl
├── lmq_lib.erl
├── lmq_metrics.erl
├── lmq_misc.erl
├── lmq_mpull.erl
├── lmq_mpull_sup.erl
├── lmq_queue.erl
├── lmq_queue_mgr.erl
├── lmq_queue_sup.erl
├── lmq_queue_supersup.erl
└── lmq_sup.erl
└── test
├── lmq_SUITE.erl
├── lmq_api_http_SUITE.erl
├── lmq_event_SUITE.erl
├── lmq_hook_crash.erl
├── lmq_hook_invalid.erl
├── lmq_hook_sample1.erl
├── lmq_hook_sample2.erl
├── lmq_hook_test.erl
├── lmq_lib_SUITE.erl
├── lmq_mpull_SUITE.erl
├── lmq_msgpack_SUITE.erl
├── lmq_queue_SUITE.erl
├── lmq_queue_mgr_SUITE.erl
└── lmq_test_handler.erl
/.drone.yml:
--------------------------------------------------------------------------------
1 | ---
2 | image: erlangR16B02
3 | script:
4 | - make test
5 |
--------------------------------------------------------------------------------
/.erlang:
--------------------------------------------------------------------------------
1 | lists:foreach(fun(Path) ->
2 | code:add_patha(filename:join(Path, "ebin"))
3 | end, filelib:wildcard("./deps/*")).
4 | code:add_patha("./ebin").
5 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | deps
2 | ebin
3 | log
4 | logs
5 | *.beam
6 | Mnesia.*
7 | .eunit
8 | rel/lmq*
9 |
--------------------------------------------------------------------------------
/CHANGES.md:
--------------------------------------------------------------------------------
1 | ## 更新履歴
2 | ### 0.7.0 (Unreleased)
3 | * 残りリトライ数を API レスポンスに追加
4 | * プロパティに差分が無い時は更新処理をスキップ
5 | * 大きなデータ (>7MB) を push した時のエラーを修正
6 |
7 | ### 0.6.4 (2014-04-22)
8 | * StatsD をサポート
9 |
10 | ### 0.6.3 (2014-02-18)
11 | * 複数キューが対象の API (``/messages?qre=``) で ``t`` が0以外だとクラッシュするバグを修正
12 |
13 | ### 0.6.2 (2014-01-28)
14 | * HTTP API を変更
15 |
16 | ### 0.6.1 (2013-12-26)
17 | * msgpack の str サポートを有効化
18 |
19 | ### 0.6.0 (2013-12-02)
20 | * REST ベースの HTTP API を実装
21 | * MessagePack-RPC API と同等の機能を実装
22 | * 今後は HTTP API が標準になる予定
23 | * nack/release 時もリトライ回数を消費するように変更
24 | * 安定性の向上とバグ修正
25 | * pull 待ちクライアント切断時の挙動を改善
26 | * キューのプロパティを初期化できないバグを修正
27 |
28 | ### 0.5.0 (2013-10-10)
29 | * クラスタリング機能を実装
30 | * マスターレス方式を採用
31 | * キューは全てのノードにミラーされる
32 | * 統計情報を収集
33 | * `lmq-admin` コマンドを追加
34 | * クラスタ操作
35 | * LMQ の状態と統計情報の表示
36 | * `pull_any` を再実装
37 | * リクエストを受けてから作成されたキューもチェック対象にするように変更
38 | * 安定性の向上とバグ修正
39 | * システムテーブルをオンメモリに変更
40 | * 多数のクライアントが `pull` した時に、一部に異常なレスポンスが返るバグを修正
41 | * リトライ超過で破棄するメッセージが大量にある時に、応答が無くなるバグを修正
42 |
43 | ### 0.4.0 (2013-09-10)
44 | * 明示的に `create` をしなくても、`push`/`pull` をした時に自動的にキューを作成するように変更
45 | * プロパティの実装を強化
46 | * 稼働中のキューのプロパティを更新する機能を追加
47 | * 正規表現でデフォルト値を登録する機能を追加
48 | * API の変更
49 | * `create` を廃止
50 | * `update_props` を追加
51 | * `set_default_props`/`get_default_props` を追加
52 | * `push` のレスポンスを変更
53 | * `pull`/`pull_any` のレスポンスを変更
54 | * 安定性の向上とバグ修正
55 | * `release` でリトライ回数が減るのを修正
56 | * `pull_any` を timeout: 0 で呼ぶとメッセージを取得できないことがあるのを修正
57 | * その他の修正
58 |
59 | ### 0.3.0 (2013-08-05)
60 | * 大幅なパフォーマンスの向上
61 | * push: 6.1 倍
62 | * pull: 1.9 倍
63 | * pull+done: 47倍
64 | * 起動時に自動的に DB を初期化するように変更
65 | * 設定ファイルのパスを変更
66 | * その他バグ修正
67 |
68 | ### 0.2.0 (2013-07-24)
69 | * キューのプロパティ機能を実装
70 | * 再送までの時間を設定可能
71 | * 再送回数を設定可能
72 | * パックする時間を設定可能
73 | * パック機能を実装
74 | * メッセージを push してから一定期間内に push されたメッセージをまとめる
75 | * メッセージがまとめられるまでは pull できない
76 | * 一度まとめられたメッセージは、その単位で再送される
77 | * 新しい API を追加
78 | * delete()
79 | * push_all()
80 | * pull_any()
81 | * 既存の API にパラメータを追加
82 | * create() に property を追加
83 | * pull() に timeout を追加
84 | * 安定性の向上とバグ修正
85 | * 起動時に既存のキューを読み込むように変更
86 | * 複数のクライアントから同時にアクセスした際の安定性を向上
87 | * ロギングを追加
88 | * その他バグ修正
89 |
90 | ### 0.1.0 (2013-06-24)
91 | * 初期リリース
92 |
--------------------------------------------------------------------------------
/Gihufile:
--------------------------------------------------------------------------------
1 | FROM ubuntu
2 | RUN ln -sf /usr/share/zoneinfo/Asia/Tokyo /etc/localtime
3 | RUN sed -i -e 's,archive.ubuntu.com,ftp.jaist.ac.jp/pub/Linux,' /etc/apt/sources.list
4 | RUN apt-get update
5 | RUN apt-get install -y build-essential curl
6 | RUN apt-get install -y libncurses5-dev openssl libssl-dev fop xsltproc unixodbc-dev
7 | RUN apt-get install -y git
8 | RUN curl -s -O http://erlang.org/download/otp_src_R16B01.tar.gz
9 | RUN tar xzf otp_src_R16B01.tar.gz
10 | RUN cd otp_src_R16B01 && ./configure && make && make install
11 | WORKDIR /gihuci
12 | CMD make test
13 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | The MIT License (MIT)
2 |
3 | Copyright (c) 2013-2014 Internet Initiative Japan Inc.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in
13 | all copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21 | THE SOFTWARE.
22 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | REBAR = $(shell pwd)/rebar
2 |
3 | .PHONY: deps test rel eunit
4 |
5 | all: deps compile
6 |
7 | compile:
8 | $(REBAR) compile
9 |
10 | deps:
11 | $(REBAR) get-deps
12 |
13 | clean:
14 | $(REBAR) clean
15 | -rm test/*.beam
16 |
17 | distclean: clean
18 | $(REBAR) delete-deps
19 |
20 | eunit: all
21 | $(REBAR) skip_deps=true eunit
22 |
23 | test: all
24 | $(REBAR) skip_deps=true eunit ct
25 |
26 | testclean:
27 | -rm -r .eunit
28 | -rm -r logs
29 |
30 | generate:
31 | $(REBAR) generate
32 |
33 | rel: deps compile generate
34 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # LMQ - Lightweight Message Queue
2 | [](https://drone.io/github.com/iij/lmq/latest)
3 |
4 | LMQ は Erlang 製のメッセージキューです。
5 | HTTP API を使ってキューに任意のデータを投入し、任意のタイミングで取り出すことができます。
6 |
7 | LMQ は使いやすく、運用しやすく、十分に早い MQ を目指して開発されました。
8 | その上で、流れ込むメッセージを処理するための特徴的な機能をいくつか有しています。
9 |
10 | LMQ は次のような特徴を持ちます。
11 |
12 | * REST ベースの HTTP API で簡単に利用できる
13 | * 1台で動作し、2台をつなげば冗長化できる
14 | * in memory で十分に早く動作する
15 | * 名前を付けて複数のキューを作成できる
16 | * タイムアウトに基づくメッセージの再送
17 | * キュー毎にタイムアウト時間等のプロパティを個別に設定できる
18 | * キューの自動作成
19 | * 投入されたメッセージは常にキューに蓄積される
20 | * コンシューマがいなくてもロストしない
21 | * LMQ 内で複数のキューへメッセージを複製
22 | * 投入されたメッセージの時間単位での集約
23 | * 統計情報の出力
24 |
25 | **※仕様は開発中のもので、将来のバージョンで非互換の変更が加わる可能性があります。**
26 |
27 | ## インストール
28 | Erlang R16B01 以上が必要になりますので、あらかじめインストールしておいてください。
29 |
30 | リポジトリを取得し、以下のコマンドを実行します。
31 |
32 | $ cd $REPO_ROOT
33 | $ make rel
34 |
35 | これにより、`$REPO_ROOT/rel/lmq` ディレクトリに必要なファイルが全てコピーされます。
36 |
37 | ## 使い方
38 |
39 | 起動するには以下のコマンドを実行します。
40 |
41 | $ bin/lmq start
42 |
43 | 起動しているか確認するには以下のようにします。
44 |
45 | $ bin/lmq ping
46 | pong
47 |
48 | これでシングル構成で動作するようになります。
49 |
50 | メッセージの投入は HTTP 経由で行います。以下では重要でないヘッダは省略しています。
51 |
52 | $ curl -i -XPOST localhost:9980/messages/myqueue -H 'content-type: text/plain' -d 'hello world!'
53 | HTTP/1.1 200 OK
54 | content-length: 15
55 | content-type: application/json
56 |
57 | {"accum":"no"}
58 |
59 | `{"accum":"no"}` は、キューがメッセージをどのように処理したかを示しています。詳細は[メッセージの集約](#accumlate)を参照してください。
60 |
61 | キューに投入したメッセージを取り出します。
62 |
63 | $ curl -i localhost:9980/messages/myqueue
64 | HTTP/1.1 200 OK
65 | content-length: 12
66 | content-type: text/plain
67 | x-lmq-queue-name: myqueue
68 | x-lmq-message-id: f0eca12e-19f2-4922-bcc9-6e42bd585937
69 | x-lmq-message-type: normal
70 |
71 | hello world!
72 |
73 | レスポンスボディは投入したメッセージがそのまま返ってきます。`content-type` も同様です。
74 |
75 | その他、メッセージを処理するために重要な情報が HTTP ヘッダに含まれます。`x-lmq-message-id` は各メッセージに割り当てられたユニークな ID です。LMQ はメッセージの再送処理のために、処理が完了したら ack を返すことになっています。この時にメッセージ ID を用います。
76 |
77 | $ curl -i -XPOST 'localhost:9980/messages/myqueue/f0eca12e-19f2-4922-bcc9-6e42bd585937?reply=ack'
78 | HTTP/1.1 204 No Content
79 |
80 | `404 Not Found` が返ってきた場合は、既にメッセージがタイムアウトして再送待ちになっている可能性があります。メッセージを取り出すところからやり直してみてください。デフォルトでは、再送までの時間は30秒に設定されています。
81 |
82 | LMQ には他にも API が用意されています。詳細は [HTTP API](#http_api) を参照してください。
83 |
84 | ## キューのプロパティ
85 | キューにはプロパティを設定することができ、その値により各キューの動作をカスタマイズすることができます。利用可能なプロパティと、デフォルト値は以下の通りです。
86 |
87 | name | type | default | description
88 | --- | --- | ---: | ---
89 | accum (pack) | float | 0 | メッセージを集約する期間（秒）、0 で無効
90 | retry | integer | 2 | メッセージの再送回数
91 | timeout | float | 30 | メッセージが再送されるまでの時間（秒）
92 |
93 | LMQ はキューのプロパティを設定するための複数の方法を用意しています。プロパティを設定する全ての API は、変更したいプロパティを投入することが可能です。設定されなかったプロパティにはデフォルト値が使われます。
94 |
95 | プロパティは以下の順に優先されます。
96 |
97 | 1. キューに設定されたプロパティ
98 | 2. デフォルトプロパティリストの最初にマッチしたもの
99 | 3. システムのデフォルト値（変更不可）
100 |
101 | 例えば、キュー `foo` に `{"accum": 15}` というプロパティを設定し、デフォルトプロパティに `[[".*", {"timeout": 60}]]` を設定した場合、キュー `foo` の実際のプロパティは以下のようになります。
102 |
103 | {"accum": 15, "retry": 2, "timeout": 60}
104 |
105 | プロパティの変更はいつでも可能ですが、新しいプロパティが反映されるのは、変更後に Put したメッセージだけです。
106 |
107 | ## タイムアウトと再送
108 | LMQ はメッセージが Get されてからの経過時間を管理しており、これが `timeout` 値を超えると、そのメッセージは正しく処理されなかったと見なしてメッセージをキューに戻します。これにより、他のクライアントがメッセージを Get できるようになります。これが LMQ における再送処理です。
109 |
110 | LMQ はメッセージを再送する際にメッセージ ID を変更します。そのため、メッセージ ID を指定する必要のある Reply は、再送された元のメッセージに対して呼び出しても失敗します。
111 |
112 | メッセージは `retry` 回数だけ再送されます。そして、この回数を超過したメッセージは破棄されます。`retry` を 0 に設定したキューでは、メッセージは決して自動的には再送されませんが、タイムアウトの発生によって破棄される点は同一なので注意してください。メッセージ ID を指定する必要のある Reply は、破棄されたメッセージに対して呼び出しても失敗します。
113 |
114 | タイムアウトは `ext` を Reply することでリセットできます。時間のかかる処理をする前に `ext` を Reply すれば、処理がタイムアウトする可能性を低くすることができます。
115 |
116 | また、`nack` を Reply する事でメッセージを意図的にキューに戻すことが可能です。結果として、メッセージは即座に再送されます。この場合でもリトライ回数としてカウントされるので注意してください。クライアントがメッセージを処理できない状態になった時は、明示的に `nack` を Reply することで、LMQ による再送を待たずに他のクライアントにメッセージを渡すことができます。
117 |
118 | ## メッセージの集約
119 | LMQ には、一定時間内に特定のキューに Put された全てのメッセージを集約して、時間経過後に一つのメッセージとしてキューに入れる集約機能があります。この集約されたメッセージのことを複合メッセージと呼びます。複合メッセージは通常のメッセージと同様に処理されます。すなわち、再送処理や API による操作は複合メッセージ単位になります。
120 |
121 | この機能を有効にするには、キューのプロパティの `accum` を 0 より大きくしてください。なお、集約が有効なキューでは、Put されたメッセージは `accum` 時間が経過するまで取り出せないことに注意してください。
122 |
123 | ## クラスタリング
124 | 耐障害性を向上させるために、2つ以上の LMQ ノードを使ってクラスタを組むことができます。LMQ クラスタは P2P 方式で実装され、マスターレスモデルになっています。キューのデータはクラスタ内の全ノードで同期されるため、クラスタを構成するノードが落ちたとしても影響はほとんどありません。また、クライアントはクラスタ内のどのノードに対してもリクエストを送ることができます。
125 |
126 | クラスタを組むには以下のようにします。説明上は *lmq1.example.com* と *lmq2.example.com* でクラスタを組むことにします。
127 |
128 | 1. `etc/vm.args` ファイルを編集します。
129 |
130 | `-name lmq@127.0.0.1` という行の *127.0.0.1* を他のノードからアクセスできる IP アドレス、もしくはホスト名に変更します。例として、それぞれ *lmq1.example.com* と *lmq2.example.com* にします。
131 |
132 | 2. それぞれのホストで LMQ を起動します。
133 |
134 | lmq1.example.com$ bin/lmq start
135 | lmq2.example.com$ bin/lmq start
136 |
137 | 3. クラスタを組みます。
138 |
139 | lmq2.example.com$ bin/lmq-admin join lmq@lmq1.example.com
140 |
141 | *lmq2.example.com* をクラスタから抜くには以下のようにします。
142 |
143 | lmq2.example.com$ bin/lmq-admin leave
144 |
145 | ## ステータス表示
146 | LMQ の現在のステータスを表示するには、以下のコマンドを実行します。
147 |
148 | $ bin/lmq-admin status
149 |
150 | ## StatsD / Graphite
151 | StatsD 経由で Graphite に metrics を送信することができます。デフォルトで localhost:8125 に送るので、別のサーバに送るには `app.config` の以下の項目を変更してください。
152 |
153 | ```erlang
154 | {statsderl, [
155 | {hostname, "localhost"},
156 | {port, 8125}
157 | ]},
158 | ```
159 |
160 | また、`stats_interval` を1以上にすると、指定した間隔で各キューのメッセージ数を送信します。単位は ms です。パフォーマンスに影響するため、設定するなら 30,000 程度を推奨します。
161 |
162 | ```erlang
163 | {lmq, [
164 | {stats_interval, 0}
165 | ]},
166 | ```
167 |
168 | ## パフォーマンスチューニング
169 | 高いパフォーマンスが必要な環境では、API のアクセスログを無効化することで 15-30% ほど多くの API リクエストを処理できるようになります。
170 |
171 | API のアクセスログは `info` レベルで吐かれるので、lager の全てのハンドラのログレベルを `notice` 以上にすればアクセスログによるオーバーヘッドが無くなります。
172 |
173 | 例えば、以下のように `app.config` を設定できます。
174 |
175 | ```erlang
176 | {lager, [
177 | {handlers, [
178 | {lager_file_backend, [
179 | [{file, "./log/console.log"}, {level, notice}, {size, 0}, {date, "$D0"}, {count, 14}]
180 | ]}
181 | ]},
182 | ```
183 |
184 | ## HTTP API
185 | LMQ は HTTP インタフェースを 9980 番ポートで提供しています（ポートは変更可能）。URL に渡す値は全て URL エンコードしてください。
186 |
187 | リクエスト/レスポンスは JSON 形式です。
188 |
189 | ### メッセージ操作
190 | #### GET /messages/:name
191 | * 概要: メッセージをキューから1つ取得する
192 | * パラメータ:
193 | * name: キュー名
194 | * クエリパラメータ（オプション）
195 | * t: Timeout, メッセージを取得できるまで待つ時間（秒）
196 | * cf: Compound Format, 複合メッセージのフォーマット
197 | * multipart: `multipart/mixed` 形式。デフォルト
198 | * msgpack: `msgpack` 形式
199 | * レスポンスコード: `200 OK`
200 | * レスポンスヘッダ:
201 | * content-type: POST 時の content-type
202 | * x-lmq-queue-name: メッセージを取り出したキュー名
203 | * x-lmq-message-id: メッセージの ID
204 | * x-lmq-message-type: メッセージの種類
205 | * normal: 通常のメッセージ
206 | * compound: 複合メッセージ
207 | * レスポンスボディ: POST されたデータ
208 |
209 | #### POST /messages/:name
210 | * 概要: メッセージをキューに追加する
211 | * パラメータ:
212 | * name: キュー名
213 | * リクエストボディ: メッセージの内容となるデータ
214 | * レスポンスコード: `200 OK`
215 | * レスポンスボディ:
216 |
217 | ```json
218 | {"accum": "FLAG"}
219 | ```
220 |
221 | * FLAG: 集約処理が行われたかどうか
222 | * new: 新たに集約が開始された
223 | * yes: 既存のメッセージと集約された
224 | * no: 集約されなかった
225 |
226 | #### GET /messages?qre=:regexp
227 | * 概要: メッセージを regexp にマッチするキューの**いずれか**から1つ取得する
228 | * パラメータ:
229 | * qre: 対象のキューを絞り込む正規表現
230 | * クエリパラメータ（オプション）: `GET /messages/:name` と同様
231 | * レスポンス: `GET /messages/:name` と同様
232 |
233 | #### POST /messages?qre=:regexp
234 | * 概要: メッセージを regexp にマッチする**全ての**キューに追加する
235 | * パラメータ:
236 | * qre: 対象のキューを絞り込む正規表現
237 | * リクエストボディ: メッセージの内容となるデータ
238 | * レスポンスコード: `200 OK`
239 | * レスポンスボディ:
240 |
241 | ```json
242 | {
243 | "QUEUE NAME 1": {"accum": "FLAG 1"},
244 | "QUEUE NAME 2": {"accum": "FLAG 2"},
245 | ...
246 | }
247 | ```
248 |
249 | * QUEUE NAME N: メッセージを追加したキュー名
250 | * FLAG N: 集約処理が行われたかどうか
251 | * new: 新たに集約が開始された
252 | * yes: 既存のメッセージと集約された
253 | * no: 集約されなかった
254 |
255 | #### POST /messages/:name/:msgid?reply=:type
256 | * 概要: 取得したメッセージの処理結果を通知する
257 | * パラメータ:
258 | * name: キュー名
259 | * msgid: 対象メッセージの ID
260 | * type: 処理結果
261 | * ack: 処理が正常に終了 -> メッセージをキューから削除
262 | * nack: 処理が継続できなくなった -> メッセージをキューに戻す
263 | * ext: 処理に時間がかかっている -> メッセージの処理可能時間を延長
264 | * レスポンスコード: `204 No Content`
265 |
266 | ### キュー操作
267 | #### DELETE /queues/:name
268 | * 概要: キューを削除する。キュー内のメッセージは全て破棄される
269 | * パラメータ:
270 | * name: キュー名
271 | * レスポンスコード: `204 No Content`
272 |
273 | ### プロパティ操作
274 | #### GET /properties
275 | * 概要: デフォルトプロパティを取得する
276 | * レスポンスコード: `200 OK`
277 | * レスポンスボディ:
278 |
279 | ```json
280 | [[REGEXP, PROPERTIES], ...]
281 | ```
282 |
283 | * REGEXP: 対象のキューを絞り込む正規表現
284 | * PROPERTIES: REGEXP にマッチしたキューに設定するプロパティ
285 |
286 | #### PUT /properties
287 | * 概要: デフォルトプロパティを設定する。既存の設定があれば上書きされる
288 | * リクエストボディ: `GET /properties` のレスポンスと同様
289 | * レスポンスコード: `204 No Content`
290 |
291 | #### DELETE /properties
292 | * 概要: デフォルトプロパティを初期化する
293 | * レスポンスコード: `204 No Content`
294 |
295 | #### GET /properties/:name
296 | * 概要: キューのプロパティを取得する
297 | * パラメータ:
298 | * name: キュー名
299 | * レスポンスコード: `200 OK`
300 | * レスポンスボディ:
301 |
302 | ```json
303 | {
304 | "accum": ACCUM,
305 | "retry": RETRY,
306 | "timeout": TIMEOUT
307 | }
308 | ```
309 |
310 | 詳細は[キューのプロパティ](#properties)を参照
311 |
312 | #### PATCH /properties/:name
313 | * 概要: キューのプロパティを部分的に更新する。指定しなかったプロパティは既存の値を踏襲する
314 | * パラメータ:
315 | * name: キュー名
316 | * リクエストボディ:
317 | プロパティの内、変更したいものだけを記したもの
318 |
319 | 例:
320 |
321 | ```json
322 | {"accum": 30}
323 | {"accum": 30, "retry": 5}
324 | ```
325 |
326 | * レスポンスコード: `204 No Content`
327 |
328 | #### DELETE /properties/:name
329 | * 概要: キューのプロパティを初期化する
330 | * パラメータ:
331 | * name: キュー名
332 | * レスポンスコード: `204 No Content`
333 |
334 | ## MessagePack-RPC API
335 |
336 | LMQ は MessagePack-RPC インタフェースを 18800 番ポートで提供しています。ここでは、各 API について `method(args) -> return_value` の形式で説明します。
337 |
338 | 戻り値は正常に処理された時のものについて記載します。各 API はエラーを返す可能性がある点に留意してください。
339 |
340 | *MessagePack-RPC API は、後方互換性のために 0.6 でのメッセージ集約プロパティの変更の影響を受けていません。そのため、HTTP API とは一部異なります。*
341 |
342 | ### Request Message
343 |
344 | #### create(name[, property]) -> "ok"
345 | **この API は 0.4.0 で削除されました。代わりに `update_props` を使用してください。**
346 |
347 | #### delete(name) -> "ok"
348 |
349 | 指定したキューを削除します。
350 |
351 |
352 | - name (string)
- 削除するキューの名前
353 |
354 |
355 | #### push(name, content) -> "ok"
356 | 指定したキューにメッセージを投入します。キューがなければ作成します。
357 |
358 |
359 | - name (string)
- キューの名前
360 | - content (object)
- 投入するデータで、形式は問わない
361 |
362 |
363 | #### pull(name[, timeout]) -> {"queue": name, "id": id, "type": type, "content": content} | "empty"
364 | 指定したキューからメッセージを取り出します。キューがなければ作成します。
365 |
366 | timeout を指定し、タイムアウトした時は `empty` 文字列が返ります。
367 |
368 | `type` が "package" の場合、`content` は `push` されたデータの配列になります。
369 |
370 |
371 | - name (string)
- キューの名前
372 | - timeout (float)
- タイムアウトまでの秒数
373 | - id (string)
- メッセージの ID
374 | - type (string)
- メッセージのタイプ。normal: 通常、package: パッケージ
375 | - content
- push() により投入されたデータ
376 |
377 |
378 | #### push_all(regexp, content) -> "ok"
379 |
380 | 正規表現にマッチする全てのキューにメッセージを投入します。
381 |
382 |
383 | - regexp (string)
- 正規表現
384 | - content (object)
- 投入するデータで、形式は問わない
385 |
386 |
387 | #### pull_any(regexp[, timeout]) -> {"queue": name, "id": id, "type": type, "content": content} | "empty"
388 |
389 | 正規表現にマッチするキューの中から、最も早く取り出せたメッセージを取得します。
390 |
391 | timeout を指定し、タイムアウトした時は `empty` 文字列が返ります。
392 |
393 | `type` が "package" の場合、`content` は `push` されたデータの配列になります。
394 |
395 |
396 | - regexp (string)
- 正規表現
397 | - timeout (float)
- タイムアウトまでの秒数
398 | - name (string)
- キューの名前
399 | - id (string)
- メッセージの ID
400 | - type (string)
- メッセージのタイプ。normal: 通常、package: パッケージ
401 | - content
- push() により投入されたデータ
402 |
403 |
404 | #### done(name, id) -> "ok"
405 |
406 | メッセージの完了報告をし、キューからメッセージを取り除きます。pull() により取り出されたメッセージが一定時間内に完了しなかった場合、LMQ は自動的にメッセージを再送します。
407 |
408 | メッセージが再送されると、古い ID は無効になり、完了報告が失敗します。
409 |
410 |
411 | - name (string)
- キューの名前
412 | - id (string)
- 完了報告するメッセージの ID
413 |
414 |
415 | #### retain(name, id) -> "ok"
416 |
417 | メッセージの再送タイマーをリセットします。
418 | これにより、再送までの時間を延長することができます。
419 |
420 | メッセージが既に再送されていた場合、呼び出しは失敗します。
421 |
422 |
423 | - name (string)
- キューの名前
424 | - id (string)
- 再送をリセットするメッセージの ID
425 |
426 |
427 | #### release(name, id) -> "ok"
428 |
429 | メッセージをキューに戻します。メッセージは即座に再送されます。
430 |
431 |
432 | - name (string)
- キューの名前
433 | - id (string)
- キューに戻すメッセージの ID
434 |
435 |
436 | #### update_props(name[, property]) -> "ok"
437 | キューのプロパティを更新します。キューがなければ作成します。
438 |
439 | 変更したいプロパティだけ指定すれば、残りはデフォルト値が使用されます。
440 | また、`property` 自体を省略すると、全てデフォルト値に設定されます。
441 |
442 |
443 | - name (string)
- キューの名前
444 | - property (dict)
- キューの動作に関わるプロパティ
445 |
446 |
447 | ### set_default_props(props_list) -> "ok"
448 | デフォルトプロパティを設定します。既存の内容は完全に上書きされます。
449 |
450 | props_list は [[regexp, property], ...] の構造を持つリストです。
451 | キューの作成時に、新しいキューの名前が regexp にマッチするかを先頭から評価していき、初めてマッチした property を使用します。どのルールにもマッチしなければ、システムのデフォルトを使用します。
452 |
453 | ### get_default_props() -> props_list
454 | 設定されているデフォルトプロパティを取得する。
455 |
--------------------------------------------------------------------------------
/bump-version:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -e
4 |
5 | if [ $# -ne 1 ]; then
6 | echo "usage: $0 VERSION"
7 | exit 1
8 | fi
9 |
10 | sed -i .bak -e "s/{vsn, \".*\"}/{vsn, \"$1\"}/" src/lmq.app.src
11 | rm src/lmq.app.src.bak
12 |
13 | sed -i .bak -e "s/{rel, \"lmq\", \".*\"/{rel, \"lmq\", \"$1\"/" rel/reltool.config
14 | rm rel/reltool.config.bak
15 |
16 | echo Files updated successfully, version bumped to $1
17 | exit 0
18 |
--------------------------------------------------------------------------------
/include/lmq.hrl:
--------------------------------------------------------------------------------
1 | -record(message, {id={lmq_misc:unixtime(), uuid:get_v4()},
2 | state=available, type=normal, retry=0, content}).
3 | -record(queue_info, {name, props}).
4 | -record(lmq_info, {key, value}).
5 |
6 | -define(DEFAULT_QUEUE_PROPS, [{accum, 0}, {retry, 2}, {timeout, 30}]).
7 | -define(LMQ_INFO_TABLE, '__lmq_info__').
8 | -define(LMQ_INFO_TABLE_DEFS, [{type, set},
9 | {attributes, record_info(fields, lmq_info)},
10 | {record_name, lmq_info}]).
11 | -define(QUEUE_INFO_TABLE, '__lmq_queue_info__').
12 | -define(QUEUE_INFO_TABLE_DEFS, [{type, set},
13 | {attributes, record_info(fields, queue_info)},
14 | {record_name, queue_info}]).
15 |
16 | -define(LMQ_ALL_METRICS, all).
17 | -define(STATSD_SAMPLERATE, 0.5).
18 |
--------------------------------------------------------------------------------
/include/lmq_test.hrl:
--------------------------------------------------------------------------------
1 | -define(LMQ_EVENT, lmq_event).
2 |
3 | -define(EVENT_OR_FAIL(Event), receive {test_handler, Event} -> ok
4 | after 50 -> ct:fail(no_response)
5 | end).
6 | -define(EVENT_AND_FAIL(Event), receive {test_handler, Event} ->
7 | ct:fail(invalid_event)
8 | after 50 -> ok
9 | end).
10 |
--------------------------------------------------------------------------------
/lmq.sublime-project:
--------------------------------------------------------------------------------
1 | {
2 | "folders": [
3 | {"path": ".",
4 | "folder_exclude_patterns": [".eunit", "logs", "log", "Mnesia.*", "deps"],
5 | "file_exclude_patterns": ["*.beam"]}
6 | ],
7 | "settings": {
8 | "tab_size": 4
9 | },
10 | "build_systems": [
11 | ]
12 | }
13 |
--------------------------------------------------------------------------------
/rebar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/iij/lmq/3f01c555af973a07a3f2b22ff95a2bc1c7930bc2/rebar
--------------------------------------------------------------------------------
/rebar.config:
--------------------------------------------------------------------------------
1 | {sub_dirs, ["rel"]}.
2 | {erl_opts, [warnings_as_errors,
3 | {parse_transform, lager_transform},
4 | debug_info]}.
5 | {cover_enabled, true}.
6 | {eunit_opts, [verbose]}.
7 |
8 | {deps, [
9 | {lager, "2.0.0",
10 | {git, "git://github.com/basho/lager", {tag, "2.0.0"}}},
11 | {uuid, ".*",
12 | {git, "git://github.com/okeuday/uuid.git", {branch, "master"}}},
13 | {msgpack_rpc, "0.6.3",
14 | {git, "git://github.com/msgpack-rpc/msgpack-rpc-erlang.git", {branch, "master"}}},
15 | {folsom, "0.7.4",
16 | {git, "git://github.com/boundary/folsom.git", {tag, "0.7.4"}}},
17 | {cowboy, ".*",
18 | {git, "git://github.com/ninenines/cowboy.git", {tag, "1.0.0"}}},
19 | {ibrowse, ".*",
20 | {git, "git://github.com/cmullaparthi/ibrowse.git", {branch, "master"}}},
21 | {jsonx, ".*",
22 | {git, "git://github.com/iskra/jsonx.git", {branch, "master"}}},
23 | {statsderl, ".*",
24 | {git, "git://github.com/lpgauth/statsderl.git", {branch, "master"}}},
25 | {eper, ".*",
26 | {git, "git://github.com/massemanet/eper.git", {branch, "master"}}}
27 | ]}.
28 |
--------------------------------------------------------------------------------
/rel/files/app.config:
--------------------------------------------------------------------------------
1 | [
2 | %% LMQ config
3 | {lmq, [
4 | {port, 18800},
5 | {stats_interval, 20000},
6 | {influxdb, [
7 | {host, "localhost"},
8 | {port, 4444}
9 | ]}
10 | ]},
11 |
12 | %% Lager config
13 | {lager, [
14 | {handlers, [
15 | {lager_file_backend, [
16 | [{file, "./log/error.log"}, {level, error},
17 | {size, 0}, %% do not rotate by size
18 | {date, "$D0"}, %% rotate at midnight
19 | {count, 14}], %% save recent 2 weeks log
20 | [{file, "./log/console.log"}, {level, info},
21 | {size, 0}, {date, "$D0"}, {count, 14}]
22 | ]}
23 | ]},
24 | {crash_log, "./log/crash.log"},
25 | {crash_log_msg_size, 65536},
26 | {crash_log_size, 0},
27 | {crash_log_date, "$D0"},
28 | {crash_log_count, 14},
29 | {error_logger_redirect, true}
30 | ]},
31 |
32 | %% StatsD config
33 | {statsderl, [
34 | {hostname, "localhost"},
35 | {port, 8125}
36 | ]},
37 |
38 | %% SASL config
39 | {sasl, [
40 | {sasl_error_logger, false}
41 | ]}
42 | ].
43 |
--------------------------------------------------------------------------------
/rel/files/erl:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 |
3 | ## This script replaces the default "erl" in erts-VSN/bin. This is necessary
4 | ## as escript depends on erl and in turn, erl depends on having access to a
5 | ## bootscript (start.boot). Note that this script is ONLY invoked as a side-effect
6 | ## of running escript -- the embedded node bypasses erl and uses erlexec directly
7 | ## (as it should).
8 | ##
9 | ## Note that this script makes the assumption that there is a start_clean.boot
10 | ## file available in $ROOTDIR/release/VSN.
11 |
12 | # Determine the abspath of where this script is executing from.
13 | ERTS_BIN_DIR=$(cd ${0%/*} && pwd)
14 |
15 | # Now determine the root directory -- this script runs from erts-VSN/bin,
16 | # so we simply need to strip off two dirs from the end of the ERTS_BIN_DIR
17 | # path.
18 | ROOTDIR=${ERTS_BIN_DIR%/*/*}
19 |
20 | # Parse out release and erts info
21 | START_ERL=`cat $ROOTDIR/releases/start_erl.data`
22 | ERTS_VSN=${START_ERL% *}
23 | APP_VSN=${START_ERL#* }
24 |
25 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin
26 | EMU=beam
27 | PROGNAME=`echo $0 | sed 's/.*\\///'`
28 | CMD="$BINDIR/erlexec"
29 | export EMU
30 | export ROOTDIR
31 | export BINDIR
32 | export PROGNAME
33 |
34 | exec $CMD -boot $ROOTDIR/releases/$APP_VSN/start_clean ${1+"$@"}
35 |
--------------------------------------------------------------------------------
/rel/files/install_upgrade.escript:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env escript
2 | %%! -noshell -noinput
3 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
4 | %% ex: ft=erlang ts=4 sw=4 et
5 |
6 | -define(TIMEOUT, 60000).
7 | -define(INFO(Fmt,Args), io:format(Fmt,Args)).
8 |
9 | main([NodeName, Cookie, ReleasePackage]) ->
10 | TargetNode = start_distribution(NodeName, Cookie),
11 | {ok, Vsn} = rpc:call(TargetNode, release_handler, unpack_release,
12 | [ReleasePackage], ?TIMEOUT),
13 | ?INFO("Unpacked Release ~p~n", [Vsn]),
14 | {ok, OtherVsn, Desc} = rpc:call(TargetNode, release_handler,
15 | check_install_release, [Vsn], ?TIMEOUT),
16 | {ok, OtherVsn, Desc} = rpc:call(TargetNode, release_handler,
17 | install_release, [Vsn], ?TIMEOUT),
18 | ?INFO("Installed Release ~p~n", [Vsn]),
19 | ok = rpc:call(TargetNode, release_handler, make_permanent, [Vsn], ?TIMEOUT),
20 | ?INFO("Made Release ~p Permanent~n", [Vsn]);
21 | main(_) ->
22 | init:stop(1).
23 |
24 | start_distribution(NodeName, Cookie) ->
25 | MyNode = make_script_node(NodeName),
26 | {ok, _Pid} = net_kernel:start([MyNode, shortnames]),
27 | erlang:set_cookie(node(), list_to_atom(Cookie)),
28 | TargetNode = make_target_node(NodeName),
29 | case {net_kernel:hidden_connect_node(TargetNode),
30 | net_adm:ping(TargetNode)} of
31 | {true, pong} ->
32 | ok;
33 | {_, pang} ->
34 | io:format("Node ~p not responding to pings.\n", [TargetNode]),
35 | init:stop(1)
36 | end,
37 | TargetNode.
38 |
39 | make_target_node(Node) ->
40 | [_, Host] = string:tokens(atom_to_list(node()), "@"),
41 | list_to_atom(lists:concat([Node, "@", Host])).
42 |
43 | make_script_node(Node) ->
44 | list_to_atom(lists:concat([Node, "_upgrader_", os:getpid()])).
45 |
--------------------------------------------------------------------------------
/rel/files/lmq:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | # -*- tab-width:4;indent-tabs-mode:nil -*-
3 | # ex: ts=4 sw=4 et
4 |
5 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd)
6 |
7 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}
8 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc
9 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log
10 | # Note the trailing slash on $PIPE_DIR/
11 | PIPE_DIR=/tmp/$RUNNER_BASE_DIR/
12 | RUNNER_USER=
13 |
14 | # Make sure this script is running as the appropriate user
15 | if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then
16 | exec sudo -u $RUNNER_USER -i $0 $@
17 | fi
18 |
19 | # Make sure CWD is set to runner base dir
20 | cd $RUNNER_BASE_DIR
21 |
22 | # Make sure log directory exists
23 | mkdir -p $RUNNER_LOG_DIR
24 | # Identify the script name
25 | SCRIPT=`basename $0`
26 |
27 | # Parse out release and erts info
28 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data`
29 | ERTS_VSN=${START_ERL% *}
30 | APP_VSN=${START_ERL#* }
31 |
32 | # Use releases/VSN/vm.args if it exists otherwise use etc/vm.args
33 | if [ -e "$RUNNER_BASE_DIR/releases/$APP_VSN/vm.args" ]; then
34 | VMARGS_PATH="$RUNNER_BASE_DIR/releases/$APP_VSN/vm.args"
35 | else
36 | VMARGS_PATH="$RUNNER_ETC_DIR/vm.args"
37 | fi
38 |
39 | # Use releases/VSN/sys.config if it exists otherwise use etc/app.config
40 | if [ -e "$RUNNER_BASE_DIR/releases/$APP_VSN/sys.config" ]; then
41 | CONFIG_PATH="$RUNNER_BASE_DIR/releases/$APP_VSN/sys.config"
42 | else
43 | CONFIG_PATH="$RUNNER_ETC_DIR/app.config"
44 | fi
45 |
46 | # Extract the target node name from node.args
47 | NAME_ARG=`egrep '^-s?name' $VMARGS_PATH`
48 | if [ -z "$NAME_ARG" ]; then
49 | echo "vm.args needs to have either -name or -sname parameter."
50 | exit 1
51 | fi
52 |
53 | # Extract the name type and name from the NAME_ARG for REMSH
54 | REMSH_TYPE=`echo $NAME_ARG | awk '{print $1}'`
55 | REMSH_NAME=`echo $NAME_ARG | awk '{print $2}'`
56 |
57 | # Note the `date +%s`, used to allow multiple remsh to the same node transparently
58 | REMSH_NAME_ARG="$REMSH_TYPE remsh`date +%s`@`echo $REMSH_NAME | awk -F@ '{print $2}'`"
59 | REMSH_REMSH_ARG="-remsh $REMSH_NAME"
60 |
61 | # Extract the target cookie
62 | COOKIE_ARG=`grep '^-setcookie' $VMARGS_PATH`
63 | if [ -z "$COOKIE_ARG" ]; then
64 | echo "vm.args needs to have a -setcookie parameter."
65 | exit 1
66 | fi
67 |
68 | # Add ERTS bin dir to our path
69 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin
70 |
71 | # Setup command to control the node
72 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG"
73 |
74 | # Setup remote shell command to control node
75 | REMSH="$ERTS_PATH/erl $REMSH_NAME_ARG $REMSH_REMSH_ARG $COOKIE_ARG"
76 |
77 | # Check the first argument for instructions
78 | case "$1" in
79 | start)
80 | # Make sure there is not already a node running
81 | RES=`$NODETOOL ping`
82 | if [ "$RES" = "pong" ]; then
83 | echo "Node is already running!"
84 | exit 1
85 | fi
86 | shift # remove $1
87 | RUN_PARAM=$(printf "\'%s\' " "$@")
88 | HEART_COMMAND="$RUNNER_BASE_DIR/bin/$SCRIPT start $RUN_PARAM"
89 | export HEART_COMMAND
90 | mkdir -p $PIPE_DIR
91 | $ERTS_PATH/run_erl -daemon $PIPE_DIR $RUNNER_LOG_DIR "exec $RUNNER_BASE_DIR/bin/$SCRIPT console $RUN_PARAM" 2>&1
92 | ;;
93 |
94 | stop)
95 | # Wait for the node to completely stop...
96 | case `uname -s` in
97 | Linux|Darwin|FreeBSD|DragonFly|NetBSD|OpenBSD)
98 | # PID COMMAND
99 | PID=`ps ax -o pid= -o command=|\
100 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'`
101 | ;;
102 | SunOS)
103 | # PID COMMAND
104 | PID=`ps -ef -o pid= -o args=|\
105 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'`
106 | ;;
107 | CYGWIN*)
108 | # UID PID PPID TTY STIME COMMAND
109 | PID=`ps -efW|grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $2}'`
110 | ;;
111 | esac
112 | $NODETOOL stop
113 | ES=$?
114 | if [ "$ES" -ne 0 ]; then
115 | exit $ES
116 | fi
117 | while `kill -0 $PID 2>/dev/null`;
118 | do
119 | sleep 1
120 | done
121 | ;;
122 |
123 | restart)
124 | ## Restart the VM without exiting the process
125 | $NODETOOL restart
126 | ES=$?
127 | if [ "$ES" -ne 0 ]; then
128 | exit $ES
129 | fi
130 | ;;
131 |
132 | reboot)
133 | ## Restart the VM completely (uses heart to restart it)
134 | $NODETOOL reboot
135 | ES=$?
136 | if [ "$ES" -ne 0 ]; then
137 | exit $ES
138 | fi
139 | ;;
140 |
141 | ping)
142 | ## See if the VM is alive
143 | $NODETOOL ping
144 | ES=$?
145 | if [ "$ES" -ne 0 ]; then
146 | exit $ES
147 | fi
148 | ;;
149 |
150 | attach)
151 | # Make sure a node IS running
152 | RES=`$NODETOOL ping`
153 | ES=$?
154 | if [ "$ES" -ne 0 ]; then
155 | echo "Node is not running!"
156 | exit $ES
157 | fi
158 |
159 | shift
160 | exec $ERTS_PATH/to_erl $PIPE_DIR
161 | ;;
162 |
163 | remote_console)
164 | # Make sure a node IS running
165 | RES=`$NODETOOL ping`
166 | ES=$?
167 | if [ "$ES" -ne 0 ]; then
168 | echo "Node is not running!"
169 | exit $ES
170 | fi
171 |
172 | shift
173 | exec $REMSH
174 | ;;
175 |
176 | upgrade)
177 | if [ -z "$2" ]; then
178 | echo "Missing upgrade package argument"
179 | echo "Usage: $SCRIPT upgrade {package base name}"
180 | echo "NOTE {package base name} MUST NOT include the .tar.gz suffix"
181 | exit 1
182 | fi
183 |
184 | # Make sure a node IS running
185 | RES=`$NODETOOL ping`
186 | ES=$?
187 | if [ "$ES" -ne 0 ]; then
188 | echo "Node is not running!"
189 | exit $ES
190 | fi
191 |
192 | node_name=`echo $NAME_ARG | awk '{print $2}'`
193 | erlang_cookie=`echo $COOKIE_ARG | awk '{print $2}'`
194 |
195 | $ERTS_PATH/escript $RUNNER_BASE_DIR/bin/install_upgrade.escript $node_name $erlang_cookie $2
196 | ;;
197 |
198 | console|console_clean)
199 | # .boot file typically just $SCRIPT (ie, the app name)
200 | # however, for debugging, sometimes start_clean.boot is useful:
201 | case "$1" in
202 | console) BOOTFILE=$SCRIPT ;;
203 | console_clean) BOOTFILE=start_clean ;;
204 | esac
205 | # Setup beam-required vars
206 | ROOTDIR=$RUNNER_BASE_DIR
207 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin
208 | EMU=beam
209 | PROGNAME=`echo $0 | sed 's/.*\\///'`
210 | CMD="$BINDIR/erlexec -boot $RUNNER_BASE_DIR/releases/$APP_VSN/$BOOTFILE -mode embedded -config $CONFIG_PATH -args_file $VMARGS_PATH"
211 | export EMU
212 | export ROOTDIR
213 | export BINDIR
214 | export PROGNAME
215 |
216 | # Dump environment info for logging purposes
217 | echo "Exec: $CMD" -- ${1+"$@"}
218 | echo "Root: $ROOTDIR"
219 |
220 | # Log the startup
221 | logger -t "$SCRIPT[$$]" "Starting up"
222 |
223 | # Start the VM
224 | exec $CMD -- ${1+"$@"}
225 | ;;
226 |
227 | foreground)
228 | # start up the release in the foreground for use by runit
229 | # or other supervision services
230 |
231 | BOOTFILE=$SCRIPT
232 | FOREGROUNDOPTIONS="-noinput +Bd"
233 |
234 | # Setup beam-required vars
235 | ROOTDIR=$RUNNER_BASE_DIR
236 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin
237 | EMU=beam
238 | PROGNAME=`echo $0 | sed 's/.*\///'`
239 | CMD="$BINDIR/erlexec $FOREGROUNDOPTIONS -boot $RUNNER_BASE_DIR/releases/$APP_VSN/$BOOTFILE -config $CONFIG_PATH -args_file $VMARGS_PATH"
240 | export EMU
241 | export ROOTDIR
242 | export BINDIR
243 | export PROGNAME
244 |
245 | # Dump environment info for logging purposes
246 | echo "Exec: $CMD" -- ${1+"$@"}
247 | echo "Root: $ROOTDIR"
248 |
249 | # Start the VM
250 | exec $CMD -- ${1+"$@"}
251 | ;;
252 | *)
253 | echo "Usage: $SCRIPT {start|foreground|stop|restart|reboot|ping|console|console_clean|attach|remote_console|upgrade}"
254 | exit 1
255 | ;;
256 | esac
257 |
258 | exit 0
259 |
--------------------------------------------------------------------------------
/rel/files/lmq-admin:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | # -*- tab-width:4;indent-tabs-mode:nil -*-
3 | # ex: ts=4 sw=4 et
4 |
5 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd)
6 |
7 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*}
8 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc
9 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log
10 | # Note the trailing slash on $PIPE_DIR/
11 | PIPE_DIR=/tmp/$RUNNER_BASE_DIR/
12 | RUNNER_USER=
13 |
14 | # Make sure this script is running as the appropriate user
15 | if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then
16 | exec sudo -u $RUNNER_USER -i $0 $@
17 | fi
18 |
19 | # Make sure CWD is set to runner base dir
20 | cd $RUNNER_BASE_DIR
21 |
22 | # Make sure log directory exists
23 | mkdir -p $RUNNER_LOG_DIR
24 | # Identify the script name
25 | SCRIPT=`basename $0`
26 |
27 | # Parse out release and erts info
28 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data`
29 | ERTS_VSN=${START_ERL% *}
30 | APP_VSN=${START_ERL#* }
31 |
32 | # Use releases/VSN/vm.args if it exists otherwise use etc/vm.args
33 | if [ -e "$RUNNER_BASE_DIR/releases/$APP_VSN/vm.args" ]; then
34 | VMARGS_PATH="$RUNNER_BASE_DIR/releases/$APP_VSN/vm.args"
35 | else
36 | VMARGS_PATH="$RUNNER_ETC_DIR/vm.args"
37 | fi
38 |
39 | # Use releases/VSN/sys.config if it exists otherwise use etc/app.config
40 | if [ -e "$RUNNER_BASE_DIR/releases/$APP_VSN/sys.config" ]; then
41 | CONFIG_PATH="$RUNNER_BASE_DIR/releases/$APP_VSN/sys.config"
42 | else
43 | CONFIG_PATH="$RUNNER_ETC_DIR/app.config"
44 | fi
45 |
46 | # Extract the target node name from node.args
47 | NAME_ARG=`egrep '^-s?name' $VMARGS_PATH`
48 | if [ -z "$NAME_ARG" ]; then
49 | echo "vm.args needs to have either -name or -sname parameter."
50 | exit 1
51 | fi
52 |
53 | # Extract the name type and name from the NAME_ARG for REMSH
54 | REMSH_TYPE=`echo $NAME_ARG | awk '{print $1}'`
55 | REMSH_NAME=`echo $NAME_ARG | awk '{print $2}'`
56 |
57 | # Note the `date +%s`, used to allow multiple remsh to the same node transparently
58 | REMSH_NAME_ARG="$REMSH_TYPE remsh`date +%s`@`echo $REMSH_NAME | awk -F@ '{print $2}'`"
59 | REMSH_REMSH_ARG="-remsh $REMSH_NAME"
60 |
61 | # Extract the target cookie
62 | COOKIE_ARG=`grep '^-setcookie' $VMARGS_PATH`
63 | if [ -z "$COOKIE_ARG" ]; then
64 | echo "vm.args needs to have a -setcookie parameter."
65 | exit 1
66 | fi
67 |
68 | # Add ERTS bin dir to our path
69 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin
70 |
71 | # Setup command to control the node
72 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG"
73 |
74 | # Setup remote shell command to control node
75 | REMSH="$ERTS_PATH/erl $REMSH_NAME_ARG $REMSH_REMSH_ARG $COOKIE_ARG"
76 |
77 | ensure_node_is_up() {
78 | if ! $NODETOOL ping > /dev/null; then
79 | echo "Node is not running!"
80 | exit 1
81 | fi
82 | }
83 |
84 | # Check the first argument for instructions
85 | case "$1" in
86 | join)
87 | if [ -z $2 ]; then
88 | echo "Usage: $SCRIPT join "
89 | exit 1
90 | fi
91 | ensure_node_is_up
92 | $NODETOOL rpc lmq_console join "$2"
93 | ;;
94 | leave)
95 | ensure_node_is_up
96 | $NODETOOL rpc lmq_console leave
97 | ;;
98 | status)
99 | ensure_node_is_up
100 | $NODETOOL rpc lmq_console status
101 | ;;
102 | stats)
103 | ensure_node_is_up
104 | $NODETOOL rpc lmq_console stats
105 | ;;
106 | *)
107 | echo "Usage: $SCRIPT {join|leave|status|stats}"
108 | exit 1
109 | ;;
110 | esac
111 |
112 | exit 0
113 |
--------------------------------------------------------------------------------
/rel/files/lmq.cmd:
--------------------------------------------------------------------------------
1 | @setlocal
2 |
3 | @set node_name=lmq
4 |
5 | @rem Get the absolute path to the parent directory,
6 | @rem which is assumed to be the node root.
7 | @for /F "delims=" %%I in ("%~dp0..") do @set node_root=%%~fI
8 |
9 | @set releases_dir=%node_root%\releases
10 |
11 | @rem Parse ERTS version and release version from start_erl.data
12 | @for /F "tokens=1,2" %%I in (%releases_dir%\start_erl.data) do @(
13 | @call :set_trim erts_version %%I
14 | @call :set_trim release_version %%J
15 | )
16 |
17 | @rem extract erlang cookie from vm.args
18 | @set vm_args=%releases_dir%\%release_version%\vm.args
19 | @for /f "usebackq tokens=1-2" %%I in (`findstr /b \-setcookie %vm_args%`) do @set erlang_cookie=%%J
20 |
21 | @set erts_bin=%node_root%\erts-%erts_version%\bin
22 |
23 | @set service_name=%node_name%_%release_version%
24 |
25 | @if "%1"=="usage" @goto usage
26 | @if "%1"=="install" @goto install
27 | @if "%1"=="uninstall" @goto uninstall
28 | @if "%1"=="start" @goto start
29 | @if "%1"=="stop" @goto stop
30 | @if "%1"=="restart" @call :stop && @goto start
31 | @if "%1"=="console" @goto console
32 | @if "%1"=="query" @goto query
33 | @if "%1"=="attach" @goto attach
34 | @if "%1"=="upgrade" @goto upgrade
35 | @echo Unknown command: "%1"
36 |
37 | :usage
38 | @echo Usage: %~n0 [install^|uninstall^|start^|stop^|restart^|console^|query^|attach^|upgrade]
39 | @goto :EOF
40 |
41 | :install
42 | @%erts_bin%\erlsrv.exe add %service_name% -c "Erlang node %node_name% in %node_root%" -sname %node_name% -w %node_root% -m %node_root%\bin\start_erl.cmd -args " ++ %node_name% ++ %node_root%" -stopaction "init:stop()."
43 | @goto :EOF
44 |
45 | :uninstall
46 | @%erts_bin%\erlsrv.exe remove %service_name%
47 | @%erts_bin%\epmd.exe -kill
48 | @goto :EOF
49 |
50 | :start
51 | @%erts_bin%\erlsrv.exe start %service_name%
52 | @goto :EOF
53 |
54 | :stop
55 | @%erts_bin%\erlsrv.exe stop %service_name%
56 | @goto :EOF
57 |
58 | :console
59 | @start %erts_bin%\werl.exe -boot %releases_dir%\%release_version%\%node_name% -config %releases_dir%\%release_version%\sys.config -args_file %vm_args% -sname %node_name%
60 | @goto :EOF
61 |
62 | :query
63 | @%erts_bin%\erlsrv.exe list %service_name%
64 | @exit /b %ERRORLEVEL%
65 | @goto :EOF
66 |
67 | :attach
68 | @for /f "usebackq" %%I in (`hostname`) do @set hostname=%%I
69 | start %erts_bin%\werl.exe -boot %releases_dir%\%release_version%\start_clean -remsh %node_name%@%hostname% -sname console -setcookie %erlang_cookie%
70 | @goto :EOF
71 |
72 | :upgrade
73 | @if "%2"=="" (
74 | @echo Missing upgrade package argument
75 | @echo Usage: %~n0 upgrade {package base name}
76 | @echo NOTE {package base name} MUST NOT include the .tar.gz suffix
77 | @goto :EOF
78 | )
79 | @%erts_bin%\escript.exe %node_root%\bin\install_upgrade.escript %node_name% %erlang_cookie% %2
80 | @goto :EOF
81 |
82 | :set_trim
83 | @set %1=%2
84 | @goto :EOF
85 |
--------------------------------------------------------------------------------
/rel/files/nodetool:
--------------------------------------------------------------------------------
1 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
2 | %% ex: ft=erlang ts=4 sw=4 et
3 | %% -------------------------------------------------------------------
4 | %%
5 | %% nodetool: Helper Script for interacting with live nodes
6 | %%
7 | %% -------------------------------------------------------------------
8 |
9 | main(Args) ->
10 | ok = start_epmd(),
11 | %% Extract the args
12 | {RestArgs, TargetNode} = process_args(Args, [], undefined),
13 |
14 | %% See if the node is currently running -- if it's not, we'll bail
15 | case {net_kernel:hidden_connect_node(TargetNode), net_adm:ping(TargetNode)} of
16 | {true, pong} ->
17 | ok;
18 | {_, pang} ->
19 | io:format("Node ~p not responding to pings.\n", [TargetNode]),
20 | halt(1)
21 | end,
22 |
23 | case RestArgs of
24 | ["ping"] ->
25 | %% If we got this far, the node already responsed to a ping, so just dump
26 | %% a "pong"
27 | io:format("pong\n");
28 | ["stop"] ->
29 | io:format("~p\n", [rpc:call(TargetNode, init, stop, [], 60000)]);
30 | ["restart"] ->
31 | io:format("~p\n", [rpc:call(TargetNode, init, restart, [], 60000)]);
32 | ["reboot"] ->
33 | io:format("~p\n", [rpc:call(TargetNode, init, reboot, [], 60000)]);
34 | ["rpc", Module, Function | RpcArgs] ->
35 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function),
36 | [RpcArgs], 60000) of
37 | ok ->
38 | ok;
39 | {badrpc, Reason} ->
40 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]),
41 | halt(1);
42 | _ ->
43 | halt(1)
44 | end;
45 | ["rpcterms", Module, Function, ArgsAsString] ->
46 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function),
47 | consult(ArgsAsString), 60000) of
48 | {badrpc, Reason} ->
49 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]),
50 | halt(1);
51 | Other ->
52 | io:format("~p\n", [Other])
53 | end;
54 | Other ->
55 | io:format("Other: ~p\n", [Other]),
56 | io:format("Usage: nodetool {ping|stop|restart|reboot}\n")
57 | end,
58 | net_kernel:stop().
59 |
60 | process_args([], Acc, TargetNode) ->
61 | {lists:reverse(Acc), TargetNode};
62 | process_args(["-setcookie", Cookie | Rest], Acc, TargetNode) ->
63 | erlang:set_cookie(node(), list_to_atom(Cookie)),
64 | process_args(Rest, Acc, TargetNode);
65 | process_args(["-name", TargetName | Rest], Acc, _) ->
66 | ThisNode = append_node_suffix(TargetName, "_maint_"),
67 | {ok, _} = net_kernel:start([ThisNode, longnames]),
68 | process_args(Rest, Acc, nodename(TargetName));
69 | process_args(["-sname", TargetName | Rest], Acc, _) ->
70 | ThisNode = append_node_suffix(TargetName, "_maint_"),
71 | {ok, _} = net_kernel:start([ThisNode, shortnames]),
72 | process_args(Rest, Acc, nodename(TargetName));
73 | process_args([Arg | Rest], Acc, Opts) ->
74 | process_args(Rest, [Arg | Acc], Opts).
75 |
76 |
77 | start_epmd() ->
78 | [] = os:cmd(epmd_path() ++ " -daemon"),
79 | ok.
80 |
81 | epmd_path() ->
82 | ErtsBinDir = filename:dirname(escript:script_name()),
83 | Name = "epmd",
84 | case os:find_executable(Name, ErtsBinDir) of
85 | false ->
86 | case os:find_executable(Name) of
87 | false ->
88 | io:format("Could not find epmd.~n"),
89 | halt(1);
90 | GlobalEpmd ->
91 | GlobalEpmd
92 | end;
93 | Epmd ->
94 | Epmd
95 | end.
96 |
97 |
98 | nodename(Name) ->
99 | case string:tokens(Name, "@") of
100 | [_Node, _Host] ->
101 | list_to_atom(Name);
102 | [Node] ->
103 | [_, Host] = string:tokens(atom_to_list(node()), "@"),
104 | list_to_atom(lists:concat([Node, "@", Host]))
105 | end.
106 |
107 | append_node_suffix(Name, Suffix) ->
108 | case string:tokens(Name, "@") of
109 | [Node, Host] ->
110 | list_to_atom(lists:concat([Node, Suffix, os:getpid(), "@", Host]));
111 | [Node] ->
112 | list_to_atom(lists:concat([Node, Suffix, os:getpid()]))
113 | end.
114 |
115 |
116 | %%
117 | %% Given a string or binary, parse it into a list of terms, ala file:consult/0
118 | %%
119 | consult(Str) when is_list(Str) ->
120 | consult([], Str, []);
121 | consult(Bin) when is_binary(Bin)->
122 | consult([], binary_to_list(Bin), []).
123 |
124 | consult(Cont, Str, Acc) ->
125 | case erl_scan:tokens(Cont, Str, 0) of
126 | {done, Result, Remaining} ->
127 | case Result of
128 | {ok, Tokens, _} ->
129 | {ok, Term} = erl_parse:parse_term(Tokens),
130 | consult([], Remaining, [Term | Acc]);
131 | {eof, _Other} ->
132 | lists:reverse(Acc);
133 | {error, Info, _} ->
134 | {error, Info}
135 | end;
136 | {more, Cont1} ->
137 | consult(Cont1, eof, Acc)
138 | end.
139 |
--------------------------------------------------------------------------------
/rel/files/start_erl.cmd:
--------------------------------------------------------------------------------
1 | @setlocal
2 |
3 | @rem Parse arguments. erlsrv.exe prepends erl arguments prior to first ++.
4 | @rem Other args are position dependent.
5 | @set args="%*"
6 | @for /F "delims=++ tokens=1,2,3" %%I in (%args%) do @(
7 | @set erl_args=%%I
8 | @call :set_trim node_name %%J
9 | @call :set_trim node_root %%K
10 | )
11 |
12 | @set releases_dir=%node_root%\releases
13 |
14 | @rem parse ERTS version and release version from start_erl.dat
15 | @for /F "tokens=1,2" %%I in (%releases_dir%\start_erl.data) do @(
16 | @call :set_trim erts_version %%I
17 | @call :set_trim release_version %%J
18 | )
19 |
20 | @set erl_exe=%node_root%\erts-%erts_version%\bin\erl.exe
21 | @set boot_file=%releases_dir%\%release_version%\%node_name%
22 |
23 | @if exist %releases_dir%\%release_version%\sys.config (
24 | @set app_config=%releases_dir%\%release_version%\sys.config
25 | ) else (
26 | @set app_config=%node_root%\etc\app.config
27 | )
28 |
29 | @if exist %releases_dir%\%release_version%\vm.args (
30 | @set vm_args=%releases_dir%\%release_version%\vm.args
31 | ) else (
32 | @set vm_args=%node_root%\etc\vm.args
33 | )
34 |
35 | @%erl_exe% %erl_args% -boot %boot_file% -config %app_config% -args_file %vm_args%
36 |
37 | :set_trim
38 | @set %1=%2
39 | @goto :EOF
40 |
--------------------------------------------------------------------------------
/rel/files/vm.args:
--------------------------------------------------------------------------------
1 | ## Name of the node
2 | -name lmq@127.0.0.1
3 |
4 | ## Cookie for distributed erlang
5 | -setcookie lmq
6 |
7 | ## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive
8 | ## (Disabled by default..use with caution!)
9 | ##-heart
10 |
11 | ## Enable kernel poll and a few async threads
12 | ##+K true
13 | ##+A 5
14 |
15 | ## Increase number of concurrent ports/sockets
16 | ##-env ERL_MAX_PORTS 4096
17 |
18 | ## Tweak GC to run more often
19 | ##-env ERL_FULLSWEEP_AFTER 10
20 |
--------------------------------------------------------------------------------
/rel/reltool.config:
--------------------------------------------------------------------------------
1 | {sys, [
2 | {lib_dirs, ["../deps"]},
3 | {erts, [{mod_cond, derived}, {app_file, strip}]},
4 | {app_file, strip},
5 | {rel, "lmq", "0.6.4",
6 | [
7 | kernel,
8 | stdlib,
9 | sasl,
10 | mnesia,
11 | lager,
12 | ranch,
13 | msgpack,
14 | msgpack_rpc,
15 | uuid,
16 | lmq
17 | ]},
18 | {rel, "start_clean", "",
19 | [
20 | kernel,
21 | stdlib
22 | ]},
23 | {boot_rel, "lmq"},
24 | {profile, embedded},
25 | {incl_cond, exclude},
26 | {excl_archive_filters, [".*"]}, %% Do not archive built libs
27 | {excl_sys_filters, ["^bin/.*", "^erts.*/bin/(dialyzer|typer)",
28 | "^erts.*/(doc|info|include|lib|man|src)"]},
29 | {excl_app_filters, ["\.gitignore"]},
30 | {app, sasl, [{incl_cond, include}]},
31 | {app, stdlib, [{incl_cond, include}]},
32 | {app, kernel, [{incl_cond, include}]},
33 | {app, crypto, [{incl_cond, include}]},
34 | {app, compiler, [{incl_cond, include}]},
35 | {app, goldrush, [{incl_cond, include}]},
36 | {app, syntax_tools, [{incl_cond, include}]},
37 | {app, quickrand, [{incl_cond, include}]},
38 | {app, uuid, [{incl_cond, include}]},
39 | {app, lager, [{incl_cond, include}]},
40 | {app, mnesia, [{incl_cond, include}]},
41 | {app, ranch, [{incl_cond, include}]},
42 | {app, cowlib, [{incl_cond, include}]},
43 | {app, cowboy, [{incl_cond, include}]},
44 | {app, jsonx, [{incl_cond, include}]},
45 | {app, msgpack, [{incl_cond, include}]},
46 | {app, msgpack_rpc, [{incl_cond, include}]},
47 | {app, bear, [{incl_cond, include}]},
48 | {app, folsom, [{incl_cond, include}]},
49 | {app, statsderl, [{incl_cond, include}]},
50 | {app, eper, [{incl_cond, include}]},
51 | {app, lmq, [{incl_cond, include}, {lib_dir, ".."}]}
52 | ]}.
53 |
54 | {target_dir, "lmq"}.
55 |
56 | {overlay, [
57 | {mkdir, "log/sasl"},
58 | {copy, "files/erl", "\{\{erts_vsn\}\}/bin/erl"},
59 | {copy, "files/nodetool", "\{\{erts_vsn\}\}/bin/nodetool"},
60 | {copy, "files/lmq", "bin/lmq"},
61 | {copy, "files/lmq-admin", "bin/lmq-admin"},
62 | {copy, "files/lmq.cmd", "bin/lmq.cmd"},
63 | {copy, "files/start_erl.cmd", "bin/start_erl.cmd"},
64 | {copy, "files/install_upgrade.escript", "bin/install_upgrade.escript"},
65 | {copy, "files/app.config", "etc/app.config"},
66 | {copy, "files/vm.args", "etc/vm.args"}
67 | ]}.
68 |
--------------------------------------------------------------------------------
/src/influxdb_client.erl:
--------------------------------------------------------------------------------
1 | -module(influxdb_client).
2 |
3 | -behaviour(gen_server).
4 |
5 | -export([start_link/2, write/1]).
6 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
7 | code_change/3, terminate/2]).
8 |
9 | -record(state, {enabled, socket, host, port}).
10 |
11 | %% ==================================================================
12 | %% Public API
13 | %% ==================================================================
14 | start_link(Host, Port) ->
15 | gen_server:start_link({local, ?MODULE}, ?MODULE, {Host, Port}, []).
16 |
17 | write(Series) ->
18 | gen_server:cast(?MODULE, {write, Series}).
19 |
20 | %% ==================================================================
21 | %% gen_server callbacks
22 | %% ==================================================================
23 | init({Host, Port}) ->
24 | case resolve_hostname(Host) of
25 | {ok, Addr} ->
26 | {ok, Socket} = gen_udp:open(0, [{active, false}]),
27 | {ok, #state{enabled=true, socket=Socket, host=Addr, port=Port}};
28 | _ ->
29 | {ok, #state{enabled=false}}
30 | end.
31 |
32 | handle_call(_, _, State) ->
33 | {reply, ok, State}.
34 |
35 | handle_cast(_, #state{enabled=false}=State) ->
36 | {noreply, State};
37 | handle_cast({write, Series}, #state{socket=Socket, host=Host, port=Port}=State) ->
38 | Data = jsonx:encode(Series),
39 | gen_udp:send(Socket, Host, Port, Data),
40 | {noreply, State}.
41 |
42 | handle_info(_, State) ->
43 | {noreply, State}.
44 |
45 | code_change(_OldVsn, State, _Extra) ->
46 | {ok, State}.
47 |
48 | terminate(_Reason, _State) ->
49 | ok.
50 |
51 | %% ==================================================================
52 | %% Private functions
53 | %% ==================================================================
54 | resolve_hostname(Addr) when is_tuple(Addr) ->
55 | Addr;
56 | resolve_hostname(Hostname) ->
57 | case inet:gethostbyname(Hostname) of
58 | {ok, {_, _, _, _, _, [Addr|_]}} -> {ok, Addr};
59 | _ -> {error, invalid}
60 | end.
61 |
--------------------------------------------------------------------------------
/src/lmq.app.src:
--------------------------------------------------------------------------------
1 | {application, lmq,
2 | [
3 | {description, ""},
4 | {vsn, "0.6.4"},
5 | {registered, []},
6 | {applications, [
7 | kernel,
8 | stdlib,
9 | lager,
10 | crypto,
11 | uuid,
12 | mnesia,
13 | ranch,
14 | cowlib,
15 | cowboy,
16 | jsonx,
17 | msgpack,
18 | msgpack_rpc,
19 | folsom,
20 | statsderl
21 | ]},
22 | {mod, { lmq_app, []}},
23 | {env, [
24 | {port, 18800},
25 | {http, {"0.0.0.0", 9980}},
26 | {stats_interval, 20000},
27 | {influxdb, [
28 | {host, "localhost"},
29 | {port, 4444}
30 | ]}
31 | ]}
32 | ]}.
33 |
--------------------------------------------------------------------------------
/src/lmq.erl:
--------------------------------------------------------------------------------
1 | -module(lmq).
2 |
3 | -include("lmq.hrl").
4 | -export([start/0, stop/0]).
5 | -export([push/2, push/3, pull/1, pull/2, pull/3, ack/2, abort/2, keep/2,
6 | push_all/2, push_all/3, pull_any/1, pull_any/2, pull_any/3, delete/1,
7 | get_props/1, update_props/1, update_props/2,
8 | set_default_props/1, get_default_props/0,
9 | status/0, queue_status/1, stats/0, stats/1]).
10 |
11 | -define(DEPS, [lager, crypto, quickrand, uuid, msgpack, msgpack_rpc,
12 | mnesia, ranch, cowlib, cowboy, jsonx, folsom, statsderl, lmq]).
13 |
14 | %% ==================================================================
15 | %% Public API
16 | %% ==================================================================
17 |
18 | start() ->
19 | [ensure_started(Dep) || Dep <- ?DEPS],
20 | lager:set_loglevel(lager_console_backend, debug).
21 |
22 | stop() ->
23 | [application:stop(Dep) || Dep <- lists:reverse(?DEPS)],
24 | ok.
25 |
26 | push(Name, Content) ->
27 | push(Name, [], Content).
28 |
29 | push(Name, MD, Content) when is_binary(Name) ->
30 | push(binary_to_atom(Name, latin1), MD, Content);
31 | push(Name, MD, Content) when is_atom(Name) ->
32 | Pid = lmq_queue_mgr:get(Name, [create]),
33 | lmq_queue:push(Pid, {MD, Content}).
34 |
35 | pull(Name) when is_binary(Name) ->
36 | pull(binary_to_atom(Name, latin1));
37 | pull(Name) when is_atom(Name) ->
38 | Pid = lmq_queue_mgr:get(Name, [create]),
39 | Msg = lmq_queue:pull(Pid),
40 | [{queue, Name} | lmq_lib:export_message(Msg)].
41 |
42 | pull(Name, Timeout) when is_binary(Name) ->
43 | pull(binary_to_atom(Name, latin1), Timeout);
44 | pull(Name, Timeout) when is_atom(Name) ->
45 | Pid = lmq_queue_mgr:get(Name, [create]),
46 | case lmq_queue:pull(Pid, Timeout) of
47 | empty -> empty;
48 | Msg -> [{queue, Name} | lmq_lib:export_message(Msg)]
49 | end.
50 |
51 | pull(Name, Timeout, Monitor) when is_binary(Name) ->
52 | pull(binary_to_atom(Name, latin1), Timeout, Monitor);
53 | pull(Name, Timeout, Monitor) when is_atom(Name) ->
54 | Pid = lmq_queue_mgr:get(Name, [create]),
55 | Id = lmq_queue:pull_async(Pid, Timeout),
56 | Wait = case Timeout of
57 | infinity -> infinity;
58 | 0 -> infinity;
59 | N -> round(N * 1000)
60 | end,
61 | MonitorRef = erlang:monitor(process, Monitor),
62 | R = receive
63 | {Id, {error, timeout}} -> empty;
64 | {Id, Msg} -> [{queue, Name} | lmq_lib:export_message(Msg)];
65 | {'DOWN', MonitorRef, process, Monitor, _} ->
66 | lmq_queue:pull_cancel(Pid, Id),
67 | receive
68 | {Id, #message{id={_, UUID}}} -> lmq_queue:put_back(Pid, UUID)
69 | after 0 -> ok
70 | end,
71 | {error, down}
72 | after Wait ->
73 | empty
74 | end,
75 | erlang:demonitor(MonitorRef, [flush]),
76 | R.
77 |
78 | ack(Name, UUID) ->
79 | process_message(done, Name, UUID).
80 |
81 | abort(Name, UUID) ->
82 | process_message(release, Name, UUID).
83 |
84 | keep(Name, UUID) ->
85 | process_message(retain, Name, UUID).
86 |
87 | push_all(Regexp, Content) ->
88 | push_all(Regexp, [], Content).
89 |
90 | push_all(Regexp, MD, Content) when is_binary(Regexp) ->
91 | case lmq_queue_mgr:match(Regexp) of
92 | {error, _}=R ->
93 | R;
94 | Queues ->
95 | {ok, [{Name, lmq_queue:push(Pid, {MD, Content})} || {Name, Pid} <- Queues]}
96 | end.
97 |
98 | pull_any(Regexp) ->
99 | pull_any(Regexp, inifinity).
100 |
101 | pull_any(Regexp, Timeout) when is_binary(Regexp) ->
102 | {ok, Pid} = lmq_mpull:start(),
103 | lmq_mpull:pull(Pid, Regexp, Timeout).
104 |
105 | pull_any(Regexp, Timeout, Monitor) when is_binary(Regexp) ->
106 | {ok, Pid} = lmq_mpull:start(),
107 | {ok, Ref} = lmq_mpull:pull_async(Pid, Regexp, Timeout),
108 | MonitorRef = erlang:monitor(process, Monitor),
109 | receive
110 | {Ref, Msg} ->
111 | erlang:demonitor(MonitorRef, [flush]),
112 | Msg;
113 | {'DOWN', MonitorRef, process, Monitor, _} ->
114 | lmq_mpull:pull_cancel(Pid),
115 | receive
116 | {Ref, [{queue, Name}, {id, UUID}, _, _]} ->
117 | Q = lmq_queue_mgr:get(Name, [create]),
118 | lmq_queue:put_back(Q, UUID)
119 | after 0 -> ok
120 | end,
121 | {error, down}
122 | end.
123 |
124 | delete(Name) when is_binary(Name) ->
125 | delete(binary_to_atom(Name, latin1));
126 | delete(Name) when is_atom(Name) ->
127 | lmq_queue_mgr:delete(Name).
128 |
129 | get_props(Name) when is_binary(Name) ->
130 | get_props(binary_to_atom(Name, latin1));
131 | get_props(Name) when is_atom(Name) ->
132 | lmq_lib:get_properties(Name).
133 |
134 | update_props(Name) ->
135 | update_props(Name, []).
136 |
137 | update_props(Name, Props) when is_binary(Name) ->
138 | update_props(binary_to_atom(Name, latin1), Props);
139 | update_props(Name, Props) when is_atom(Name) ->
140 | lmq_queue_mgr:get(Name, [create, update, {props, Props}]).
141 |
142 | set_default_props(Props) ->
143 | lmq_queue_mgr:set_default_props(Props).
144 |
145 | get_default_props() ->
146 | lmq_queue_mgr:get_default_props().
147 |
148 | status() ->
149 | [{active_nodes, lists:sort(mnesia:system_info(running_db_nodes))},
150 | {all_nodes, lists:sort(mnesia:system_info(db_nodes))},
151 | {queues, [{N, queue_status(N)} || N <- lists:sort(lmq_lib:all_queue_names())]}
152 | ].
153 |
154 | queue_status(Name) ->
155 | [{size, mnesia:table_info(Name, size)},
156 | {memory, mnesia:table_info(Name, memory) * erlang:system_info(wordsize)},
157 | {nodes, mnesia:table_info(Name, where_to_write)},
158 | {props, lmq_lib:get_properties(Name)}
159 | ].
160 |
161 | stats() ->
162 | [stats(N) || N <- lists:sort(lmq_lib:all_queue_names())].
163 |
164 | stats(Name) when is_atom(Name) ->
165 | {Name, [{push, lmq_metrics:get_metric(Name, push)},
166 | {pull, lmq_metrics:get_metric(Name, pull)},
167 | {retention, lmq_metrics:get_metric(Name, retention)}
168 | ]}.
169 |
170 | %% ==================================================================
171 | %% Private functions
172 | %% ==================================================================
173 |
174 | ensure_started(App) ->
175 | case application:start(App) of
176 | ok -> ok;
177 | {error, {already_started, App}} -> ok
178 | end.
179 |
180 | process_message(Fun, Name, UUID) when is_atom(Fun), is_binary(Name) ->
181 | process_message(Fun, binary_to_atom(Name, latin1), UUID);
182 | process_message(Fun, Name, UUID) when is_atom(Fun), is_atom(Name) ->
183 | case lmq_queue_mgr:get(Name) of
184 | not_found ->
185 | {error, queue_not_found};
186 | Pid ->
187 | try parse_uuid(UUID) of
188 | MsgId ->
189 | case lmq_queue:Fun(Pid, MsgId) of
190 | ok -> ok;
191 | not_found -> {error, not_found}
192 | end
193 | catch exit:badarg ->
194 | {error, not_found}
195 | end
196 | end.
197 |
198 | parse_uuid(UUID) when is_binary(UUID) ->
199 | parse_uuid(binary_to_list(UUID));
200 | parse_uuid(UUID) when is_list(UUID) ->
201 | uuid:string_to_uuid(UUID);
202 | parse_uuid(UUID) ->
203 | UUID.
204 |
--------------------------------------------------------------------------------
/src/lmq_api.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_api).
2 |
3 | -export([delete/1, push/2, pull/1, pull/2, push_all/2,
4 | pull_any/1, pull_any/2, done/2, retain/2, release/2,
5 | update_props/1, update_props/2, set_default_props/1, get_default_props/0,
6 | normalize_props/1, export_props/1,
7 | normalize_default_props/1, export_default_props/1]).
8 | -include("lmq.hrl").
9 |
10 | delete(Name) when is_binary(Name) ->
11 | lager:info("lmq_api:delete(~s)", [Name]),
12 | ok = lmq:delete(Name),
13 | <<"ok">>.
14 |
15 | push(Name, Content) when is_binary(Name) ->
16 | lager:info("lmq_api:push(~s, ...)", [Name]),
17 | Bin = msgpack:pack(Content),
18 | case lmq:push(Name, [{<<"content-type">>,
19 | <<"application/x-msgpack">>}], Bin) of
20 | ok -> <<"ok">>;
21 | {accum, new} -> <<"packing started">>;
22 | {accum, yes} -> <<"packed">>
23 | end.
24 |
25 | pull(Name) when is_binary(Name) ->
26 | lager:info("lmq_api:pull(~s)", [Name]),
27 | Response = lmq:pull(Name),
28 | export_message(extract_content_value(Response)).
29 |
30 | pull(Name, Timeout) when is_binary(Name) ->
31 | lager:info("lmq_api:pull(~s, ~p)", [Name, Timeout]),
32 | {monitors, [{process, Conn}]} = erlang:process_info(self(), monitors),
33 | case lmq:pull(Name, Timeout, Conn) of
34 | {error, down} -> ok;
35 | empty -> <<"empty">>;
36 | Msg -> export_message(extract_content_value(Msg))
37 | end.
38 |
39 | push_all(Regexp, Content) when is_binary(Regexp) ->
40 | lager:info("lmq_api:push_all(~s, ...)", [Regexp]),
41 | case lmq:push_all(Regexp, Content) of
42 | {ok, _} -> <<"ok">>;
43 | {error, Reason} -> throw(Reason)
44 | end.
45 |
46 | pull_any(Regexp) ->
47 | pull_any(Regexp, inifinity).
48 |
49 | pull_any(Regexp, Timeout) when is_binary(Regexp) ->
50 | lager:info("lmq_api:pull_any(~s, ~p)", [Regexp, Timeout]),
51 | Timeout2 = if
52 | is_number(Timeout) -> round(Timeout * 1000);
53 | true -> Timeout
54 | end,
55 | {monitors, [{process, Conn}]} = erlang:process_info(self(), monitors),
56 | case lmq:pull_any(Regexp, Timeout2, Conn) of
57 | {error, down} -> ok;
58 | empty -> <<"empty">>;
59 | Msg -> export_message(extract_content_value(Msg))
60 | end.
61 |
62 | done(Name, UUID) when is_binary(Name), is_binary(UUID) ->
63 | lager:info("lmq_api:done(~s, ~s)", [Name, UUID]),
64 | process_message(ack, Name, UUID).
65 |
66 | retain(Name, UUID) when is_binary(Name), is_binary(UUID) ->
67 | lager:info("lmq_api:retain(~s, ~s)", [Name, UUID]),
68 | process_message(keep, Name, UUID).
69 |
70 | release(Name, UUID) when is_binary(Name), is_binary(UUID) ->
71 | lager:info("lmq_api:release(~s, ~s)", [Name, UUID]),
72 | process_message(abort, Name, UUID).
73 |
74 | update_props(Name) when is_binary(Name) ->
75 | lager:info("lmq_api:update_props(~s)", [Name]),
76 | lmq:update_props(Name),
77 | <<"ok">>.
78 |
79 | update_props(Name, Props) when is_binary(Name) ->
80 | lager:info("lmq_api:update_props(~s, ~p)", [Name, Props]),
81 | case normalize_props(Props) of
82 | {ok, Props2} ->
83 | lmq:update_props(Name, to_new_props(Props2)),
84 | <<"ok">>;
85 | {error, Reason} ->
86 | throw(Reason)
87 | end.
88 |
89 | set_default_props(PropsList) ->
90 | lager:info("lmq_api:set_default_props(~p)", [PropsList]),
91 | case normalize_default_props(PropsList) of
92 | {ok, PropsList2} ->
93 | PropsList3 = [{K, to_new_props(V)} || {K, V} <- PropsList2],
94 | case lmq:set_default_props(PropsList3) of
95 | ok -> <<"ok">>;
96 | Reason -> throw(Reason)
97 | end;
98 | {error, Reason} ->
99 | throw(Reason)
100 | end.
101 |
102 | get_default_props() ->
103 | lager:info("lmq_api:get_default_props()"),
104 | Props = [{K, to_old_props(V)} || {K, V} <- lmq:get_default_props()],
105 | export_default_props(Props).
106 |
107 | normalize_props({Props}) ->
108 | %% jiffy style to proplists
109 | normalize_props(Props, []).
110 |
111 | export_props(Props) ->
112 | export_props(Props, []).
113 |
114 | normalize_default_props(DefaultProps) ->
115 | normalize_default_props(DefaultProps, []).
116 |
117 | export_default_props(DefaultProps) ->
118 | export_default_props(DefaultProps, []).
119 |
120 | %% ==================================================================
121 | %% Private functions
122 | %% ==================================================================
123 |
124 | process_message(Fun, Name, UUID) when is_atom(Fun) ->
125 | case lmq:Fun(Name, UUID) of
126 | ok -> <<"ok">>;
127 | {error, Reason} -> throw(Reason)
128 | end.
129 |
130 | export_message(Msg) ->
131 | export_message(Msg, []).
132 |
133 | export_message([{id, V} | Tail], Acc) ->
134 | export_message(Tail, [{<<"id">>, V} | Acc]);
135 | export_message([{type, compound} | Tail], Acc) ->
136 | export_message(Tail, [{<<"type">>, <<"package">>} | Acc]);
137 | export_message([{K, V} | Tail], Acc) when is_atom(V) ->
138 | export_message(Tail, [{atom_to_binary(K, latin1),
139 | atom_to_binary(V, latin1)} | Acc]);
140 | export_message([{K, V} | Tail], Acc) when is_atom(K) ->
141 | export_message(Tail, [{atom_to_binary(K, latin1), V} | Acc]);
142 | export_message([], Acc) ->
143 | {lists:reverse(Acc)}.
144 |
145 | normalize_props([{<<"pack">>, Duration} | T], Acc) when is_number(Duration) ->
146 | normalize_props(T, [{pack, round(Duration * 1000)} | Acc]);
147 | normalize_props([{<<"accum">>, Duration} | T], Acc) when is_number(Duration) ->
148 | normalize_props(T, [{accum, round(Duration * 1000)} | Acc]);
149 | normalize_props([{<<"retry">>, N} | T], Acc) when is_integer(N) ->
150 | normalize_props(T, [{retry, N} | Acc]);
151 | normalize_props([{<<"timeout">>, N} | T], Acc) when is_number(N) ->
152 | normalize_props(T, [{timeout, N} | Acc]);
153 | normalize_props([{<<"hooks">>, {L}} | T], Acc) when is_list(L) ->
154 | normalize_props(T, [{hooks, normalize_hooks(L, [])} | Acc]);
155 | normalize_props([], Acc) ->
156 | {ok, lists:reverse(Acc)};
157 | normalize_props(_, _) ->
158 | {error, invalid}.
159 |
160 | normalize_hooks([], Acc) ->
161 | lists:reverse(Acc);
162 | normalize_hooks([{Name, L}|T], Acc) when is_binary(Name), is_list(L) ->
163 | Name2 = binary_to_atom(Name, latin1),
164 | normalize_hooks(T, [{Name2, normalize_hooks2(L, [])}|Acc]).
165 |
166 | normalize_hooks2([], Acc) ->
167 | lists:reverse(Acc);
168 | normalize_hooks2([[Name, L]|T], Acc) when is_binary(Name), is_list(L) ->
169 | Name2 = binary_to_existing_atom(Name, latin1),
170 | normalize_hooks2(T, [{Name2, L}|Acc]).
171 |
172 | export_props([{pack, Duration} | T], Acc) ->
173 | export_props(T, [{<<"pack">>, Duration / 1000} | Acc]);
174 | export_props([{accum, Duration} | T], Acc) ->
175 | export_props(T, [{<<"accum">>, Duration / 1000} | Acc]);
176 | export_props([{hooks, L} | T], Acc) when is_list(L) ->
177 | export_props(T, [{<<"hooks">>, export_hooks(L, [])} | Acc]);
178 | export_props([{K, V} | T], Acc) ->
179 | export_props(T, [{atom_to_binary(K, latin1), V} | Acc]);
180 | export_props([], Acc) ->
181 | {lists:reverse(Acc)}.
182 |
183 | export_hooks([], Acc) ->
184 | {lists:reverse(Acc)};
185 | export_hooks([{Name, L}|T], Acc) when is_atom(Name), is_list(L) ->
186 | export_hooks(T, [{atom_to_binary(Name, latin1), export_hooks2(L, [])}|Acc]).
187 |
188 | export_hooks2([], Acc) ->
189 | lists:reverse(Acc);
190 | export_hooks2([{Name, L}|T], Acc) when is_atom(Name), is_list(L) ->
191 | export_hooks2(T, [[atom_to_binary(Name, latin1), L]|Acc]).
192 |
193 | normalize_default_props([[Regexp, Props]|T], Acc) when is_binary(Regexp) ->
194 | case normalize_props(Props) of
195 | {ok, Props2} -> normalize_default_props(T, [{Regexp, Props2} | Acc]);
196 | {error, _}=R -> R
197 | end;
198 | normalize_default_props([], Acc) ->
199 | {ok, lists:reverse(Acc)};
200 | normalize_default_props(_, _) ->
201 | {error, invalid}.
202 |
203 | export_default_props([{Regexp, Props} | T], Acc) when is_list(Regexp); is_binary(Regexp) ->
204 | export_default_props(T, [[Regexp, export_props(Props)] | Acc]);
205 | export_default_props([], Acc) ->
206 | lists:reverse(Acc).
207 |
208 | extract_content_value([QUEUE, ID, {type, normal}=TYPE, RETRY, {content, {MD, V}}]) ->
209 | [QUEUE, ID, TYPE, RETRY, {content, maybe_decode(MD, V)}];
210 | extract_content_value([QUEUE, ID, {type, compound}=TYPE, RETRY, {content, Content}]) ->
211 | [QUEUE, ID, TYPE, RETRY, {content, [maybe_decode(MD, V) || {MD, V} <- Content]}].
212 |
213 | maybe_decode(MD, V) ->
214 | case proplists:get_value(<<"content-type">>, MD) of
215 | <<"application/x-msgpack">> ->
216 | {ok, Term} = msgpack:unpack(V),
217 | Term;
218 | _ ->
219 | V
220 | end.
221 |
222 | to_new_props(Props) ->
223 | lists:keymap(fun(pack) -> accum; (Other) -> Other end, 1, Props).
224 |
225 | to_old_props(Props) ->
226 | lists:keymap(fun(accum) -> pack; (Other) -> Other end, 1, Props).
227 |
228 | %% ==================================================================
229 | %% EUnit test
230 | %% ==================================================================
231 |
232 | -ifdef(TEST).
233 | -include_lib("eunit/include/eunit.hrl").
234 |
235 | normalize_props_test() ->
236 | ?assertEqual({ok, [{retry, 1}]}, normalize_props({[{<<"retry">>, 1}]})),
237 | ?assertEqual({ok, [{retry, 2}, {timeout, 5.0}, {pack, 500}]},
238 | normalize_props({[{<<"retry">>, 2}, {<<"timeout">>, 5.0},
239 | {<<"pack">>, 0.5}]})),
240 | ?assertEqual({ok, [{hooks, [{hook_point, [{hook_module, [<<"args1">>, <<"args2">>]}]}]}]},
241 | normalize_props({[{<<"hooks">>,
242 | {[{<<"hook_point">>,
243 | [[<<"hook_module">>, [<<"args1">>, <<"args2">>]]]}]}}]})),
244 | ?assertEqual({error, invalid},
245 | normalize_props({[{<<"retry">>, <<"3">>}, {<<"timeout">>, <<"5.0">>},
246 | {<<"pack">>, <<"1">>}]})),
247 | ?assertEqual({error, invalid}, normalize_props({[{<<"not supported">>, 4}]})).
248 |
249 | export_props_test() ->
250 | ?assertEqual({[{<<"retry">>, 3}, {<<"timeout">>, 5.0}, {<<"pack">>, 0.5}]},
251 | export_props([{retry, 3}, {timeout, 5.0}, {pack, 500}])),
252 | ?assertEqual({[{<<"hooks">>,
253 | {[{<<"hook_point">>,
254 | [[<<"hook_module">>, [<<"args1">>, <<"args2">>]]]}]}}]},
255 | export_props([{hooks, [{hook_point, [{hook_module, [<<"args1">>, <<"args2">>]}]}]}])).
256 |
257 | normalize_default_props_test() ->
258 | ?assertEqual({ok, [{<<"lmq">>, [{pack, 1000}]}, {<<"def">>, [{retry, 0}]}]},
259 | normalize_default_props([[<<"lmq">>, {[{<<"pack">>, 1}]}],
260 | [<<"def">>, {[{<<"retry">>, 0}]}]])),
261 | ?assertEqual({ok, [{<<"lmq">>, [{hooks, [{hook_point, [{hook_module, [<<"args1">>, <<"args2">>]}]}]}]}]},
262 | normalize_default_props([[<<"lmq">>,
263 | {[{<<"hooks">>,
264 | {[{<<"hook_point">>,
265 | [[<<"hook_module">>, [<<"args1">>, <<"args2">>]]]}]}}]}]])),
266 | ?assertEqual({error, invalid},
267 | normalize_default_props([[<<"lmq">>, {[{<<"pack">>, <<"1">>}]}]])),
268 | ?assertEqual({error, invalid}, normalize_default_props([<<"lmq">>])).
269 |
270 | export_default_props_test() ->
271 | ?assertEqual([[<<"lmq">>, {[{<<"pack">>, 1.0}]}],
272 | [<<"def">>, {[{<<"retry">>, 0}]}]],
273 | export_default_props([{<<"lmq">>, [{pack, 1000}]},
274 | {<<"def">>, [{retry, 0}]}])),
275 | ?assertEqual([[<<"lmq">>,
276 | {[{<<"hooks">>,
277 | {[{<<"hook_point">>,
278 | [[<<"hook_module">>, [<<"args1">>, <<"args2">>]]]}]}}]}]],
279 | export_default_props([{<<"lmq">>,
280 | [{hooks, [{hook_point, [{hook_module, [<<"args1">>, <<"args2">>]}]}]}]}])).
281 |
282 | export_message_test() ->
283 | Ref = make_ref(),
284 | M = #message{content=Ref},
285 | UUID = list_to_binary(uuid:uuid_to_string(element(2, M#message.id))),
286 | ?assertEqual({[{<<"id">>, UUID}, {<<"type">>, <<"normal">>},
287 | {<<"retry">>, 0}, {<<"content">>, Ref}]},
288 | export_message(lmq_lib:export_message(M))),
289 |
290 | M2 = M#message{type=compound},
291 | ?assertEqual({[{<<"queue">>, <<"test">>}, {<<"id">>, UUID},
292 | {<<"type">>, <<"package">>}, {<<"retry">>, 0}, {<<"content">>, Ref}]},
293 | export_message([{queue, test} | lmq_lib:export_message(M2)])).
294 |
295 | props_compatibility_test() ->
296 | ?assertEqual([{accum, 0}, {retry, 2}, {timeout, 30}],
297 | to_new_props([{pack, 0}, {retry, 2}, {timeout, 30}])),
298 | ?assertEqual([{pack, 0}, {retry, 2}, {timeout, 30}],
299 | to_old_props([{accum, 0}, {retry, 2}, {timeout, 30}])).
300 |
301 | -endif.
302 |
--------------------------------------------------------------------------------
/src/lmq_app.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_app).
2 |
3 | -behaviour(application).
4 |
5 | %% Application callbacks
6 | -export([start/2, stop/1]).
7 |
8 | -define(MSGPACK_SERV, msgpack_serv).
9 |
10 | %% ===================================================================
11 | %% Application callbacks
12 | %% ===================================================================
13 |
14 | start(_StartType, _StartArgs) ->
15 | {ok, Port} = application:get_env(port),
16 | {ok, Http} = application:get_env(http),
17 | maybe_join(application:get_env(join)),
18 | R = lmq_sup:start_link(),
19 | start_cowboy(Http),
20 | {ok, _} = msgpack_rpc_server:start(?MSGPACK_SERV, tcp, lmq_api, [{port, Port}]),
21 | R.
22 |
23 | stop(_State) ->
24 | msgpack_rpc_server:stop(?MSGPACK_SERV),
25 | ok.
26 |
27 | %% ==================================================================
28 | %% Private functions
29 | %% ==================================================================
30 |
31 | start_cowboy({Ip, Port}) ->
32 | Dispatch = cowboy_router:compile([
33 | {'_', [{"/messages", lmq_cow_msg, [multi]},
34 | {"/messages/:name", lmq_cow_msg, []},
35 | {"/messages/:name/:id", lmq_cow_reply, []},
36 | {"/queues/:name", lmq_cow_queue, []},
37 | {"/properties", lmq_cow_prop, []},
38 | {"/properties/:name", lmq_cow_prop, []},
39 | {"/stats", lmq_cow_stats, []}
40 | ]}
41 | ]),
42 |
43 | {ok, Ip2} = inet_parse:address(Ip),
44 | cowboy:start_http(lmq_http_listener, 100,
45 | [{ip, Ip2}, {port, Port}],
46 | [{env, [{dispatch, Dispatch}]},
47 | {max_keepalive, 10000}]
48 | ).
49 |
50 | maybe_join(undefined) ->
51 | ok = lmq_lib:init_mnesia();
52 | maybe_join({ok, Node}) ->
53 | case {net_kernel:connect_node(Node), net_adm:ping(Node)} of
54 | {true, pong} ->
55 | rpc:call(Node, lmq_console, add_new_node, [node()]);
56 | {_, pang} ->
57 | ok = lmq_lib:init_mnesia()
58 | end.
59 |
--------------------------------------------------------------------------------
/src/lmq_console.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_console).
2 |
3 | -export([join/1, leave/1, add_new_node/1, status/1, stats/1]).
4 |
5 | join([NodeStr]) when is_list(NodeStr) ->
6 | Node = list_to_atom(NodeStr),
7 | join(Node);
8 |
9 | join(Node) when is_atom(Node) ->
10 | case {net_kernel:connect_node(Node), net_adm:ping(Node)} of
11 | {true, pong} ->
12 | ok = application:stop(lmq),
13 | delete_local_schema(),
14 | R = rpc:call(Node, lmq_console, add_new_node, [node()]),
15 | ok = application:start(lmq),
16 | R;
17 | {_, pang} ->
18 | {error, not_reachable}
19 | end.
20 |
21 | leave([]) ->
22 | RunningNodes = [Node || Node <- mnesia:system_info(running_db_nodes),
23 | Node =/= node()],
24 | ok = application:stop(lmq),
25 | ok = application:stop(mnesia),
26 | R = case leave_cluster(RunningNodes) of
27 | ok -> ok = mnesia:delete_schema([node()]);
28 | Other -> Other
29 | end,
30 | ok = application:start(mnesia),
31 | ok = application:start(lmq),
32 | R.
33 |
34 | delete_local_schema() ->
35 | ok = application:stop(mnesia),
36 | ok = mnesia:delete_schema([node()]),
37 | ok = application:start(mnesia).
38 |
39 | add_new_node(Node) ->
40 | case mnesia:change_config(extra_db_nodes, [Node]) of
41 | {ok, _} -> copy_all_tables(Node);
42 | {error, _}=E -> E
43 | end.
44 |
45 | status([]) ->
46 | Status = lmq:status(),
47 | io:format(" All nodes: ~s~n", [string:join(
48 | [atom_to_list(N) || N <- proplists:get_value(all_nodes, Status)],
49 | ", ")]),
50 | io:format("Active nodes: ~s~n", [string:join(
51 | [atom_to_list(N) || N <- proplists:get_value(active_nodes, Status)],
52 | ", ")]),
53 | io:format("~n"),
54 | lists:foreach(fun({Name, QStatus}) ->
55 | io:format("~s ~7.B messages ~15.B bytes~n", [
56 | string:left(atom_to_list(Name), 40),
57 | proplists:get_value(size, QStatus),
58 | proplists:get_value(memory, QStatus)
59 | ]),
60 | Props = proplists:get_value(props, QStatus),
61 | io:format(" accum: ~p, retry: ~B, timeout: ~p~n", [
62 | proplists:get_value(accum, Props) / 1000,
63 | proplists:get_value(retry, Props),
64 | proplists:get_value(timeout, Props) / 1
65 | ])
66 | end, proplists:get_value(queues, Status)),
67 | ok.
68 |
69 | stats([]) ->
70 | lists:foreach(fun({Name, Info}) ->
71 | Push = proplists:get_value(push, Info),
72 | Pull = proplists:get_value(pull, Info),
73 | Retention = proplists:get_value(retention, Info),
74 | io:format("~s~n", [Name]),
75 | io:format(" push rate: 1min ~.2f, 5min ~.2f, 15min ~.2f, 1day ~.2f~n", [
76 | proplists:get_value(K, Push) || K <- [one, five, fifteen, day]]),
77 | io:format(" pull rate: 1min ~.2f, 5min ~.2f, 15min ~.2f, 1day ~.2f~n", [
78 | proplists:get_value(K, Pull) || K <- [one, five, fifteen, day]]),
79 | io:format(" retention time: min ~.3f, max ~.3f, mean ~.3f, median ~.3f~n", [
80 | proplists:get_value(K, Retention) || K <- [min, max, arithmetic_mean, median]])
81 | end, lmq:stats()).
82 |
83 | copy_all_tables(Node) ->
84 | Tables = mnesia:system_info(tables),
85 | Errors = lists:foldl(fun(Tab, Failed) ->
86 | case mnesia:add_table_copy(Tab, Node, ram_copies) of
87 | {atomic, ok} -> Failed;
88 | {aborted, {already_exists, _, _}} -> Failed;
89 | {aborted, Reason} -> [{Tab, Reason} | Failed]
90 | end
91 | end, [], Tables),
92 | case Errors of
93 | [] -> ok;
94 | Other -> {error, {copy_failed, Other}}
95 | end.
96 |
97 | leave_cluster([Node | Rest]) when is_atom(Node) ->
98 | case rpc:call(Node, mnesia, del_table_copy, [schema, node()]) of
99 | {atomic, ok} ->
100 | ok;
101 | {aborted, Reason} ->
102 | case Rest of
103 | [] -> {error, Reason};
104 | _ -> leave_cluster(Rest)
105 | end
106 | end.
107 |
--------------------------------------------------------------------------------
/src/lmq_cow_msg.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_cow_msg).
2 |
3 | -export([init/3, handle/2, info/3, terminate/3]).
4 |
5 | -record(state, {queue, pull, push, ref, cf}).
6 |
7 | init(_Transport, Req, []) ->
8 | {Name, Req2} = cowboy_req:binding(name, Req),
9 | init(cowboy_req:method(Req2), #state{queue=Name, pull=pull, push=push});
10 | init(_Transport, Req, [multi]) ->
11 | case validate_multi_request(Req) of
12 | {{ok, Regexp}, Req2} ->
13 | init(cowboy_req:method(Req2), #state{queue=Regexp, pull=pull_any, push=push_all});
14 | {error, Req2} ->
15 | {ok, Req3} = cowboy_req:reply(400, Req2),
16 | {shutdown, Req3, #state{}}
17 | end.
18 |
19 | init({<<"GET">>, Req}, #state{queue=Queue, pull=Pull}=State) ->
20 | case validate([fun validate_timeout/1,
21 | fun validate_compound_format/1], Req) of
22 | {ok, [Timeout, CF], Req2} ->
23 | Self = self(),
24 | Ref = make_ref(),
25 | spawn_link(fun() -> Self ! {Ref, lmq:Pull(Queue, Timeout, Self)} end),
26 | {loop, Req2, State#state{ref=Ref, cf=CF}};
27 | {error, Req2} ->
28 | {ok, Req3} = cowboy_req:reply(400, Req2),
29 | {shutdown, Req3, State}
30 | end;
31 | init({<<"POST">>, Req}, State) ->
32 | {ok, Req, State};
33 | init({_, Req}, State) ->
34 | {ok, Req2} = cowboy_req:reply(405, Req),
35 | {shutdown, Req2, State}.
36 |
37 | handle(Req, #state{queue=Queue, push=Push}=State) ->
38 | {ok, Content, Req2} = fetch_body(Req, <<>>),
39 | {CT, Req3} = cowboy_req:header(<<"content-type">>, Req2),
40 | MD = case CT of
41 | undefined -> [];
42 | _ -> [{<<"content-type">>, CT}]
43 | end,
44 | Res = export_push_resp(do_push(Push, Queue, MD, Content, Req3)),
45 | Res2 = jsonx:encode(Res),
46 | {ok, Req4} = cowboy_req:reply(200,
47 | [{<<"content-type">>, <<"application/json">>}], Res2, Req3),
48 | {ok, Req4, State}.
49 |
50 | info({Ref, empty}, Req, #state{ref=Ref}=State) ->
51 | {ok, Req2} = cowboy_req:reply(204, Req),
52 | {ok, Req2, State};
53 | info({Ref, Msg}, Req, #state{ref=Ref, cf=CF}=State) ->
54 | {Hdrs, V} = encode_body(proplists:get_value(type, Msg), CF, Msg),
55 | Hdrs2 = Hdrs ++ [{<<"x-lmq-queue-name">>,
56 | atom_to_binary(proplists:get_value(queue, Msg), latin1)},
57 | {<<"x-lmq-message-id">>, proplists:get_value(id, Msg)},
58 | {<<"x-lmq-message-type">>,
59 | atom_to_binary(proplists:get_value(type, Msg), latin1)},
60 | {<<"x-lmq-retry-remaining">>, integer_to_binary(proplists:get_value(retry, Msg))}],
61 | {ok, Req2} = cowboy_req:reply(200, Hdrs2, V, Req),
62 | {ok, Req2, State};
63 | info(_Msg, Req, State) ->
64 | {loop, Req, State}.
65 |
66 | terminate({normal, _}, _Req, _State) ->
67 | %% maybe reuse connection
68 | ok;
69 | terminate({error, _}, _Req, _State) ->
70 | %% close connection and halt this process
71 | error.
72 |
73 | %% ==================================================================
74 | %% Private functions
75 | %% ==================================================================
76 | fetch_body(Req, Acc) ->
77 | case cowboy_req:body(Req) of
78 | {ok, Data, Req2} ->
79 | {ok, <>, Req2};
80 | {more, Data, Req2} ->
81 | fetch_body(Req2, <>);
82 | Error ->
83 | Error
84 | end.
85 |
86 | do_push(push, Queue, MD, Content, Req) when is_binary(Queue) ->
87 | do_push(push, binary_to_atom(Queue, latin1), MD, Content, Req);
88 | do_push(push, Queue, MD, Content, Req) ->
89 | {MD2, Content2, _} = lmq_hook:call(Queue, pre_http_push, {MD, Content, Req}),
90 | lmq:push(Queue, MD2, Content2);
91 | do_push(push_all, Regexp, MD, Content, Req) ->
92 | case lmq_queue_mgr:match(Regexp) of
93 | {error, _}=R ->
94 | R;
95 | Queues ->
96 | {ok, [{Name, do_push(push, Name, MD, Content, Req)} || {Name, _} <- Queues]}
97 | end.
98 |
99 | export_push_resp({ok, L}) when is_list(L) ->
100 | L2 = lists:filtermap(fun({N, R}) ->
101 | case R of
102 | {error, _} -> false;
103 | _ -> {true, {N, export_push_resp(R)}}
104 | end
105 | end, L),
106 | {L2};
107 | export_push_resp(ok) ->
108 | {[{accum, no}]};
109 | export_push_resp({accum, _}=R) ->
110 | {[R]}.
111 |
112 | validate(L, Req) when is_list(L) ->
113 | lists:foldl(fun(F, {ok, Acc, Req2}) ->
114 | case F(Req2) of
115 | {ok, V, Req3} -> {ok, Acc ++ [V], Req3};
116 | {error, Req3} -> {error, Req3}
117 | end;
118 | (_, {error, Req2}) ->
119 | {error, Req2}
120 | end, {ok, [], Req}, L).
121 |
122 | validate_multi_request(Req) ->
123 | case cowboy_req:qs_val(<<"qre">>, Req) of
124 | {undefined, Req2} -> {error, Req2};
125 | {Regexp, Req2} ->
126 | case re:compile(Regexp) of
127 | {ok, _} -> {{ok, Regexp}, Req2};
128 | _ -> {error, Req2}
129 | end
130 | end.
131 |
132 | validate_timeout(Req) ->
133 | case cowboy_req:qs_val(<<"t">>, Req) of
134 | {undefined, Req2} ->
135 | {ok, infinity, Req2};
136 | {V, Req2} ->
137 | case lmq_misc:btof(V) of
138 | {ok, 0.0} ->
139 | {ok, 0, Req2};
140 | {ok, T} ->
141 | case cowboy_req:qs_val(<<"qre">>, Req2) of
142 | {undefined, Req3} -> {ok, T, Req3};
143 | {_, Req3} -> {ok, round(T * 1000), Req3}
144 | end;
145 | {error, _} ->
146 | {error, Req2}
147 | end
148 | end.
149 |
150 | validate_compound_format(Req) ->
151 | case cowboy_req:qs_val(<<"cf">>, Req) of
152 | {undefined, Req2} ->
153 | {ok, multipart, Req2};
154 | {<<"multipart">>, Req2} ->
155 | {ok, multipart, Req2};
156 | {<<"msgpack">>, Req2} ->
157 | {ok, msgpack, Req2};
158 | {_, Req2} ->
159 | {error, Req2}
160 | end.
161 |
162 | encode_body(normal, _, Msg) ->
163 | {MD, V} = proplists:get_value(content, Msg),
164 | {make_headers(MD), V};
165 | encode_body(compound, multipart, Msg) ->
166 | Boundary = proplists:get_value(id, Msg),
167 | Hdrs = [{<<"content-type">>, <<"multipart/mixed; boundary=", Boundary/binary>>}],
168 | V = to_multipart(Boundary, proplists:get_value(content, Msg)),
169 | {Hdrs, V};
170 | encode_body(compound, msgpack, Msg) ->
171 | Hdrs = [{<<"content-type">>, <<"application/x-msgpack">>}],
172 | V = [[{stringify_metadata(make_headers(MD))}, V] || {MD, V} <- proplists:get_value(content, Msg)],
173 | {Hdrs, msgpack:pack(V, [{enable_str, true}])}.
174 |
175 | make_headers(MD) ->
176 | case proplists:is_defined(<<"content-type">>, MD) of
177 | true -> MD;
178 | false -> [{<<"content-type">>, <<"application/octet-stream">>}|MD]
179 | end.
180 |
181 | to_multipart(Boundary, Contents) when is_list(Contents) ->
182 | [[[<<"\r\n--">>, Boundary, <<"\r\n">>, encode_multipart_item(C)]
183 | || C <- Contents],
184 | <<"\r\n--">>, Boundary, <<"--\r\n">>].
185 |
186 | encode_multipart_item({MD, V}) ->
187 | Hdrs = make_headers(MD),
188 | [[[Key, <<": ">>, Value, <<"\r\n">>] || {Key, Value} <- Hdrs],
189 | <<"content-transfer-encoding: binary\r\n">>,
190 | <<"\r\n">>, V].
191 |
192 | stringify_metadata(MD) ->
193 | stringify_metadata(MD, []).
194 |
195 | stringify_metadata([{K, V}|Tail], Acc) when is_binary(K), is_binary(V) ->
196 | stringify_metadata(Tail, [{binary_to_list(K), binary_to_list(V)}|Acc]);
197 | stringify_metadata([], Acc) ->
198 | lists:reverse(Acc).
199 |
200 | %% ==================================================================
201 | %% EUnit test
202 | %% ==================================================================
203 | -ifdef(TEST).
204 | -include_lib("eunit/include/eunit.hrl").
205 |
206 | export_push_resp_test_() ->
207 | [?_assertEqual({[{accum, no}]}, export_push_resp(ok)),
208 | ?_assertEqual({[{foo, {[{accum, no}]}}, {bar, {[{accum, no}]}}]},
209 | export_push_resp({ok, [{foo, ok}, {bar, ok}]})),
210 | ?_assertEqual({[{bar, {[{accum, no}]}}]},
211 | export_push_resp({ok, [{foo, {error, no_queue_exists}}, {bar, ok}]}))].
212 |
213 | encode_body_normal_test_() ->
214 | Msg1 = [{id, <<"id1">>}, {content, {[{<<"content-type">>, <<"text/plain">>}], <<"msg1">>}}],
215 | Msg2 = [{id, <<"id2">>}, {content, {[], <<"msg2">>}}],
216 | Msg3 = [{id, <<"id3">>}, {content, {[{<<"x-lmq-sequence">>, <<"1">>}], <<"msg3">>}}],
217 | [?_assertEqual({[{<<"content-type">>, <<"text/plain">>}], <<"msg1">>},
218 | encode_body(normal, multipart, Msg1)),
219 | ?_assertEqual({[{<<"content-type">>, <<"application/octet-stream">>}], <<"msg2">>},
220 | encode_body(normal, multipart, Msg2)),
221 | ?_assertEqual({[{<<"content-type">>, <<"application/octet-stream">>},
222 | {<<"x-lmq-sequence">>, <<"1">>}], <<"msg3">>},
223 | encode_body(normal, multipart, Msg3))].
224 |
225 | encode_body_multipart_test_() ->
226 | Msg1 = [{id, <<"id1">>}, {content, [{[{<<"content-type">>, <<"text/plain">>}], <<"msg1">>},
227 | {[], <<"msg2">>},
228 | {[{<<"x-lmq-sequence">>, <<"1">>}], <<"msg3">>}]}],
229 | [?_assertEqual([{<<"content-type">>, <<"multipart/mixed; boundary=id1">>}],
230 | element(1, encode_body(compound, multipart, Msg1))),
231 | ?_assertEqual(<<"\r\n--id1\r\n",
232 | "content-type: text/plain\r\n",
233 | "content-transfer-encoding: binary\r\n",
234 | "\r\n",
235 | "msg1"
236 | "\r\n--id1\r\n",
237 | "content-type: application/octet-stream\r\n",
238 | "content-transfer-encoding: binary\r\n",
239 | "\r\n",
240 | "msg2",
241 | "\r\n--id1\r\n",
242 | "content-type: application/octet-stream\r\n",
243 | "x-lmq-sequence: 1\r\n",
244 | "content-transfer-encoding: binary\r\n",
245 | "\r\n",
246 | "msg3",
247 | "\r\n--id1--\r\n">>,
248 | iolist_to_binary(element(2, encode_body(compound, multipart, Msg1))))
249 | ].
250 |
251 | encode_body_msgpack_test_() ->
252 | Msg1 = [{id, <<"id1">>}, {content, [{[{<<"content-type">>, <<"text/plain">>}], <<"msg1">>},
253 | {[], <<"msg2">>},
254 | {[{<<"x-lmq-sequence">>, <<"1">>}], <<"msg3">>}]}],
255 | Bin1 = msgpack:pack([[{[{"content-type", "text/plain"}]}, <<"msg1">>],
256 | [{[{"content-type", "application/octet-stream"}]}, <<"msg2">>],
257 | [{[{"content-type", "application/octet-stream"},
258 | {"x-lmq-sequence", "1"}]}, <<"msg3">>]],
259 | [{enable_str, true}]),
260 | [?_assertEqual({[{<<"content-type">>, <<"application/x-msgpack">>}], Bin1},
261 | encode_body(compound, msgpack, Msg1))].
262 |
263 | -endif.
264 |
--------------------------------------------------------------------------------
/src/lmq_cow_prop.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_cow_prop).
2 |
3 | -export([init/3, rest_init/2, allowed_methods/2, delete_resource/2,
4 | content_types_provided/2, content_types_accepted/2]).
5 | -export([to_json/2, update_props/2]).
6 |
7 | -record(state, {method, name}).
8 |
9 | init(_Transport, _Req, _Opts) ->
10 | {upgrade, protocol, cowboy_rest}.
11 |
12 | rest_init(Req, []) ->
13 | {Name, Req2} = cowboy_req:binding(name, Req),
14 | {Method, Req3} = cowboy_req:method(Req2),
15 | {ok, Req3, #state{method=Method, name=Name}}.
16 |
17 | allowed_methods(Req, #state{name=undefined}=State) ->
18 | {[<<"GET">>, <<"PUT">>, <<"DELETE">>], Req, State};
19 | allowed_methods(Req, State) ->
20 | {[<<"GET">>, <<"PATCH">>, <<"DELETE">>], Req, State}.
21 |
22 | delete_resource(Req, #state{name=undefined}=State) ->
23 | lmq:set_default_props([]),
24 | {true, Req, State};
25 | delete_resource(Req, #state{name=Name}=State) ->
26 | lmq:update_props(Name),
27 | {true, Req, State}.
28 |
29 | content_types_provided(Req, State) ->
30 | {[{{<<"application">>, <<"json">>, '*'}, to_json}
31 | ], Req, State}.
32 |
33 | content_types_accepted(Req, State) ->
34 | {[{{<<"application">>, <<"json">>, '*'}, update_props}
35 | ], Req, State}.
36 |
37 | to_json(Req, #state{name=undefined}=State) ->
38 | {jsonx:encode(lmq_api:export_default_props(lmq:get_default_props())), Req, State};
39 | to_json(Req, #state{name=Name}=State) ->
40 | {jsonx:encode(lmq_api:export_props(lmq:get_props(Name))), Req, State}.
41 |
42 | update_props(Req, State) ->
43 | {ok, Content, Req2} = cowboy_req:body(Req),
44 | update_props(jsonx:decode(Content), Req2, State).
45 |
46 | update_props(Body, Req, #state{name=undefined}=State) ->
47 | case lmq_api:normalize_default_props(Body) of
48 | {ok, Props} ->
49 | lmq:set_default_props(Props),
50 | {true, Req, State};
51 | {error, _} ->
52 | {false, Req, State}
53 | end;
54 | update_props(Body, Req, #state{name=Name}=State) ->
55 | case lmq_api:normalize_props(Body) of
56 | {ok, Props} ->
57 | lmq:update_props(Name, Props),
58 | {true, Req, State};
59 | {error, _} ->
60 | {false, Req, State}
61 | end.
62 |
--------------------------------------------------------------------------------
/src/lmq_cow_queue.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_cow_queue).
2 |
3 | -export([init/3, rest_init/2, allowed_methods/2, resource_exists/2, delete_resource/2]).
4 |
5 | -record(state, {name}).
6 |
7 | init(_Transport, _Req, _Opts) ->
8 | {upgrade, protocol, cowboy_rest}.
9 |
10 | rest_init(Req, []) ->
11 | {Name, Req2} = cowboy_req:binding(name, Req),
12 | {ok, Req2, #state{name=binary_to_atom(Name, latin1)}}.
13 |
14 | allowed_methods(Req, State) ->
15 | {[<<"DELETE">>], Req, State}.
16 |
17 | resource_exists(Req, #state{name=Name}=State) ->
18 | {lmq_queue_mgr:get(Name) =/= not_found, Req, State}.
19 |
20 | delete_resource(Req, #state{name=Name}=State) ->
21 | ok = lmq:delete(Name),
22 | {true, Req, State}.
23 |
--------------------------------------------------------------------------------
/src/lmq_cow_reply.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_cow_reply).
2 |
3 | -export([init/3, rest_init/2, malformed_request/2, allowed_methods/2,
4 | allow_missing_post/2, resource_exists/2, content_types_accepted/2]).
5 | -export([process_post/2]).
6 |
7 | -record(state, {reply, action}).
8 |
9 | init(_Transport, _Req, []) ->
10 | {upgrade, protocol, cowboy_rest}.
11 |
12 | rest_init(Req, _Opts) ->
13 | {Reply, Req2} = cowboy_req:qs_val(<<"reply">>, Req),
14 | {ok, Req2, #state{reply=Reply}}.
15 |
16 | malformed_request(Req, #state{reply=Reply}=State) ->
17 | Invalid = false =:= lists:member(Reply, [<<"ack">>, <<"nack">>, <<"ext">>]),
18 | {Invalid, Req, State}.
19 |
20 | allowed_methods(Req, State) ->
21 | {[<<"POST">>], Req, State}.
22 |
23 | allow_missing_post(Req, State) ->
24 | {false, Req, State}.
25 |
26 | resource_exists(Req, State) ->
27 | %% Since no efficient way exists to know whether the message exists
28 | %% or not, try to process request here.
29 | action(Req, State).
30 |
31 | content_types_accepted(Req, State) ->
32 | {[{'*', process_post}], Req, State}.
33 |
34 | process_post(Req, #state{action=ok}=State) ->
35 | {true, Req, State};
36 | process_post(Req, State) ->
37 | {false, Req, State}.
38 |
39 | %% ==================================================================
40 | %% Private functions
41 | %% ==================================================================
42 |
43 | action(Req, #state{reply= <<"ack">>}=State) ->
44 | action(ack, Req, State);
45 | action(Req, #state{reply= <<"nack">>}=State) ->
46 | action(abort, Req, State);
47 | action(Req, #state{reply= <<"ext">>}=State) ->
48 | action(keep, Req, State).
49 |
50 | action(Fun, Req, State) ->
51 | {Queue, Req2} = cowboy_req:binding(name, Req),
52 | {MsgId, Req3} = cowboy_req:binding(id, Req2),
53 | case lmq:Fun(Queue, MsgId) of
54 | ok -> {true, Req3, State#state{action=ok}};
55 | {error, Reason} -> {false, Req3, State#state{action=Reason}}
56 | end.
57 |
--------------------------------------------------------------------------------
/src/lmq_cow_stats.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_cow_stats).
2 |
3 | -export([init/3, rest_init/2, allowed_methods/2, content_types_provided/2]).
4 | -export([stats_to_json/2]).
5 |
6 | init(_Transport, _Req, _Opts) ->
7 | {upgrade, protocol, cowboy_rest}.
8 |
9 | rest_init(Req, []) ->
10 | {ok, Req, []}.
11 |
12 | allowed_methods(Req, State) ->
13 | {[<<"GET">>], Req, State}.
14 |
15 | content_types_provided(Req, State) ->
16 | {[{{<<"application">>, <<"json">>, '*'}, stats_to_json}
17 | ], Req, State}.
18 |
19 | stats_to_json(Req, State) ->
20 | Status = lmq:status(),
21 | Stats = lmq:stats(),
22 | Queues = lists:foldr(fun({Name, Info}, Acc) ->
23 | S = jsonify_stats(proplists:get_value(Name, Stats)),
24 | Info2 = [{stats, S}|Info],
25 | [{Name, Info2}|Acc]
26 | end, [], proplists:get_value(queues, Status)),
27 | R = lists:keyreplace(queues, 1, Status, {queues, Queues}),
28 | {jsonx:encode(R), Req, State}.
29 |
30 | jsonify_stats(Stats) ->
31 | R = lists:foldr(fun({K, V}, Acc) when K =:= percentile ->
32 | [{K, jsonify_proplist_keys(V)}|Acc];
33 | ({K, V}, Acc) when K =:= histogram ->
34 | [{K, jsonify_histogram(V)}|Acc];
35 | (E, Acc) ->
36 | [E|Acc]
37 | end, [], proplists:get_value(retention, Stats)),
38 | lists:keyreplace(retention, 1, Stats, {retention, R}).
39 |
40 | jsonify_proplist_keys(L) ->
41 | lists:reverse(jsonify_proplist_keys(L, [])).
42 |
43 | jsonify_proplist_keys([], Acc) ->
44 | Acc;
45 | jsonify_proplist_keys([{K, V}|T], Acc) when is_integer(K) ->
46 | jsonify_proplist_keys(T, [{integer_to_binary(K), V}|Acc]);
47 | jsonify_proplist_keys([E|T], Acc) ->
48 | jsonify_proplist_keys(T, [E|Acc]).
49 |
50 | jsonify_histogram(L) ->
51 | lists:foldr(fun({Bin, Count}, Acc) -> [[{bin, Bin}, {count, Count}]|Acc] end, [], L).
52 |
53 | %% ==================================================================
54 | %% EUnit test
55 | %% ==================================================================
56 | -ifdef(TEST).
57 | -include_lib("eunit/include/eunit.hrl").
58 |
59 | jsonify_stats_test() ->
60 | Value = [{push,
61 | [{count,24},
62 | {one,0.264930464299551},
63 | {five,0.07182249856427589},
64 | {fifteen,0.025679351706894803},
65 | {day,2.7766597276742793e-4},
66 | {mean,9.008392962075889e-8},
67 | {acceleration,
68 | [{one_to_five,6.436932191175838e-4},
69 | {five_to_fifteen,7.690524476230181e-5},
70 | {one_to_fifteen,2.658345695473958e-4}]}]},
71 | {pull,
72 | [{count,24},
73 | {one,0.264930464299551},
74 | {five,0.07182249856427589},
75 | {fifteen,0.025679351706894803},
76 | {day,2.7766597276742793e-4},
77 | {mean,9.008394821789985e-8},
78 | {acceleration,
79 | [{one_to_five,6.436932191175838e-4},
80 | {five_to_fifteen,7.690524476230181e-5},
81 | {one_to_fifteen,2.658345695473958e-4}]}]},
82 | {retention,
83 | [{min,5.290508270263672e-4},
84 | {max,0.0031280517578125},
85 | {arithmetic_mean,0.0015397212084601907},
86 | {geometric_mean,0.00127043818306925},
87 | {harmonic_mean,0.0010519247023452565},
88 | {median,0.0016698837280273438},
89 | {variance,8.87850095456516e-7},
90 | {standard_deviation,9.422579771254345e-4},
91 | {skewness,0.43425944017223067},
92 | {kurtosis,-1.3902812493309589},
93 | {percentile,
94 | [{50,0.0016698837280273438},
95 | {75,0.002238035202026367},
96 | {90,0.0029451847076416016},
97 | {95,0.0031011104583740234},
98 | {99,0.0031280517578125},
99 | {999,0.0031280517578125}]},
100 | {histogram,[{0.0031280517578125,17}]},
101 | {n,17}]}],
102 | Expected = [{push,
103 | [{count,24},
104 | {one,0.264930464299551},
105 | {five,0.07182249856427589},
106 | {fifteen,0.025679351706894803},
107 | {day,2.7766597276742793e-4},
108 | {mean,9.008392962075889e-8},
109 | {acceleration,
110 | [{one_to_five,6.436932191175838e-4},
111 | {five_to_fifteen,7.690524476230181e-5},
112 | {one_to_fifteen,2.658345695473958e-4}]}]},
113 | {pull,
114 | [{count,24},
115 | {one,0.264930464299551},
116 | {five,0.07182249856427589},
117 | {fifteen,0.025679351706894803},
118 | {day,2.7766597276742793e-4},
119 | {mean,9.008394821789985e-8},
120 | {acceleration,
121 | [{one_to_five,6.436932191175838e-4},
122 | {five_to_fifteen,7.690524476230181e-5},
123 | {one_to_fifteen,2.658345695473958e-4}]}]},
124 | {retention,
125 | [{min,5.290508270263672e-4},
126 | {max,0.0031280517578125},
127 | {arithmetic_mean,0.0015397212084601907},
128 | {geometric_mean,0.00127043818306925},
129 | {harmonic_mean,0.0010519247023452565},
130 | {median,0.0016698837280273438},
131 | {variance,8.87850095456516e-7},
132 | {standard_deviation,9.422579771254345e-4},
133 | {skewness,0.43425944017223067},
134 | {kurtosis,-1.3902812493309589},
135 | {percentile,
136 | [{<<"50">>,0.0016698837280273438},
137 | {<<"75">>,0.002238035202026367},
138 | {<<"90">>,0.0029451847076416016},
139 | {<<"95">>,0.0031011104583740234},
140 | {<<"99">>,0.0031280517578125},
141 | {<<"999">>,0.0031280517578125}]},
142 | {histogram,[[{bin, 0.0031280517578125},{count,17}]]},
143 | {n,17}]}],
144 | ?assertEqual(Expected, jsonify_stats(Value)).
145 |
146 | -endif.
147 |
--------------------------------------------------------------------------------
/src/lmq_event.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_event).
2 |
3 | -export([start_link/0, start_link/1, add_handler/1, add_handler/2,
4 | notify/1, notify_remote/2]).
5 | -export([queue_created/1, queue_deleted/1, new_message/1]).
6 |
7 | %% ==================================================================
8 | %% Public API
9 | %% ==================================================================
10 |
11 | start_link() ->
12 | Nodes = [N || N <- mnesia:system_info(running_db_nodes),
13 | N =/= node()],
14 | start_link(Nodes).
15 |
16 | start_link(Nodes) ->
17 | {ok, Pid} = gen_event:start_link({local, ?MODULE}),
18 | add_handler(lmq_handler_dist, [Nodes]),
19 | add_handler(lmq_handler),
20 | {ok, Pid}.
21 |
22 | add_handler(Module) ->
23 | add_handler(Module, []).
24 |
25 | add_handler(Module, Args) ->
26 | gen_event:add_handler(?MODULE, Module, Args).
27 |
28 | notify(Event) ->
29 | gen_event:notify(?MODULE, {local, Event}).
30 |
31 | notify_remote(Node, Event) ->
32 | gen_event:notify({?MODULE, Node}, {remote, Event}).
33 |
34 | queue_created(Name) ->
35 | notify({queue_created, Name}).
36 |
37 | queue_deleted(Name) ->
38 | notify({queue_deleted, Name}).
39 |
40 | new_message(QName) ->
41 | notify({new_message, QName}).
42 |
--------------------------------------------------------------------------------
/src/lmq_handler.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_handler).
2 |
3 | -behaviour(gen_event).
4 |
5 | -export([init/1, handle_event/2, handle_call/2, handle_info/2,
6 | code_change/3, terminate/2]).
7 |
8 | %% ==================================================================
9 | %% gen_event callbacks
10 | %% ==================================================================
11 |
12 | init([]) ->
13 | {ok, []}.
14 |
15 | handle_event({local, {new_message, _}}, State) ->
16 | {ok, State};
17 |
18 | handle_event({remote, {new_message, QName}}, State) ->
19 | lager:debug("new message arrived at remote queue ~s", [QName]),
20 | case lmq_queue_mgr:get(QName) of
21 | not_found -> ok;
22 | Pid -> lmq_queue:notify(Pid)
23 | end,
24 | {ok, State};
25 |
26 | handle_event({local, {queue_created, Name}}, State) ->
27 | lager:debug("local queue_created ~s", [Name]),
28 | [lmq_mpull:maybe_pull(Pid, Name) || Pid <- lmq_mpull:list_active()],
29 | {ok, State};
30 |
31 | handle_event({remote, {queue_created, Name}=E}, State) ->
32 | lager:debug("remote queue_created ~s", [Name]),
33 | case lmq_queue_mgr:get(Name) of
34 | not_found ->
35 | lmq_queue_mgr:get(Name, [create]),
36 | handle_event({local, E}, State);
37 | Pid ->
38 | lmq_queue:reload_properties(Pid),
39 | {ok, State}
40 | end;
41 |
42 | handle_event({local, {queue_deleted, _}}, State) ->
43 | {ok, State};
44 |
45 | handle_event({remote, {queue_deleted, Name}}, State) ->
46 | lager:debug("remote queue deleted ~s", [Name]),
47 | lmq_queue_mgr:delete(Name),
48 | {ok, State};
49 |
50 | handle_event(Event, State) ->
51 | lager:warning("Unknown event received: ~p", [Event]),
52 | {ok, State}.
53 |
54 | handle_call(_, State) ->
55 | {ok, ok, State}.
56 |
57 | handle_info(_, State) ->
58 | {ok, State}.
59 |
60 | code_change(_OldVsn, State, _Extra) ->
61 | {ok, State}.
62 |
63 | terminate(_Reason, _State) ->
64 | ok.
65 |
--------------------------------------------------------------------------------
/src/lmq_handler_dist.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_handler_dist).
2 |
3 | -behaviour(gen_event).
4 |
5 | -export([init/1, handle_event/2, handle_call/2, handle_info/2,
6 | code_change/3, terminate/2]).
7 |
8 | -record(state, {nodes}).
9 |
10 | %% ==================================================================
11 | %% gen_event callbacks
12 | %% ==================================================================
13 |
14 | init([Nodes]) ->
15 | broadcast(Nodes, {join_node, node()}),
16 | {ok, #state{nodes=Nodes}}.
17 |
18 | handle_event({remote, {join_node, Node}}, #state{nodes=Nodes}=S) ->
19 | case lists:member(Node, Nodes) of
20 | true -> {ok, S};
21 | false -> {ok, S#state{nodes=[Node | Nodes]}}
22 | end;
23 |
24 | handle_event({remote, _Event}, State) ->
25 | {ok, State};
26 |
27 | handle_event({local, Event}, State) ->
28 | broadcast(State#state.nodes, Event),
29 | {ok, State};
30 |
31 | handle_event(_Event, State) ->
32 | {ok, State}.
33 |
34 | handle_call(_, State) ->
35 | {ok, ok, State}.
36 |
37 | handle_info(_, State) ->
38 | {ok, State}.
39 |
40 | code_change(_OldVsn, State, _Extra) ->
41 | {ok, State}.
42 |
43 | terminate(_Reason, _State) ->
44 | ok.
45 |
46 | %% ==================================================================
47 | %% Private functions
48 | %% ==================================================================
49 |
50 | broadcast(Nodes, Event) ->
51 | [lmq_event:notify_remote(Node, Event) || Node <- Nodes],
52 | ok.
53 |
--------------------------------------------------------------------------------
/src/lmq_hook.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_hook).
2 |
3 | %% ==================================================================
4 | %% behaviour definition
5 | %% ==================================================================
6 | -callback init() ->
7 | {ok | stop, Reason :: term()}.
8 |
9 | -callback hooks() ->
10 | [atom()].
11 |
12 | -callback activate([term()]) ->
13 | {ok, term()}.
14 |
15 | -callback deactivate(term()) ->
16 | ok.
17 |
18 | %% ==================================================================
19 | %% hook implementation
20 | %% ==================================================================
21 | %% hook configuration for each queue:
22 | %% [{hook_name, [{lmq_hook_sample1, [term(), ...]},
23 | %% {lmq_hook_sample2, [term(), ...]}]}]
24 | %% hook state for each queue:
25 | %% [{hook_name, [{lmq_hook_sample1, State},
26 | %% {lmq_hook_sample2, State}]}]
27 | -behaviour(gen_server).
28 |
29 | -export([start_link/0, stop/0, register/2, call/3]).
30 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
31 | code_change/3, terminate/2]).
32 |
33 | -record(state, {loaded_hooks :: list(module()),
34 | queue_hooks :: dict()
35 | }).
36 |
37 | %% ==================================================================
38 | %% Public API
39 | %% ==================================================================
40 | start_link() ->
41 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
42 |
43 | stop() ->
44 | gen_server:call(?MODULE, stop).
45 |
46 | register(Queue, Config) when is_atom(Queue) ->
47 | gen_server:call(?MODULE, {register, Queue, Config}).
48 |
49 | call(Queue, Hook, Value) when is_atom(Queue), is_atom(Hook) ->
50 | gen_server:call(?MODULE, {call, Queue, Hook, Value}).
51 |
52 | %% ==================================================================
53 | %% gen_server callbacks
54 | %% ==================================================================
55 | init([]) ->
56 | {ok, #state{loaded_hooks=[], queue_hooks=dict:new()}}.
57 |
58 | handle_call({register, Queue, Config}, _From, #state{loaded_hooks=LoadedHooks,
59 | queue_hooks=QueueHooks}=State) ->
60 | try lists:foldl(fun maybe_init_hook/2, LoadedHooks, get_modules(Config)) of
61 | LoadedHooks2 ->
62 | Hooks = case dict:find(Queue, QueueHooks) of
63 | {ok, V} -> V;
64 | error -> []
65 | end,
66 | try update_hooks(Hooks, Config) of
67 | Hooks2 ->
68 | {reply, ok, State#state{loaded_hooks=LoadedHooks2,
69 | queue_hooks=dict:store(Queue, Hooks2, QueueHooks)}}
70 | catch
71 | error:function_clause ->
72 | {reply, {error, bad_config}, State#state{loaded_hooks=LoadedHooks2}};
73 | error:not_hookable ->
74 | {reply, {error, bad_config}, State#state{loaded_hooks=LoadedHooks2}}
75 | end
76 | catch
77 | error:undef ->
78 | {reply, {error, bad_hook}, State}
79 | end;
80 | handle_call({call, Queue, HookName, Value}, From, #state{queue_hooks=QueueHooks}=State) ->
81 | case dict:find(Queue, QueueHooks) of
82 | {ok, Hooks} ->
83 | F = fun() ->
84 | R = lists:foldl(fun({Module, S}, Acc) ->
85 | try Module:HookName(Acc, S)
86 | catch _:_ -> Acc
87 | end
88 | end, Value, proplists:get_value(HookName, Hooks, [])),
89 | gen_server:reply(From, R)
90 | end,
91 | spawn_link(F),
92 | {noreply, State};
93 | error ->
94 | {reply, Value, State}
95 | end;
96 | handle_call(stop, _From, State) ->
97 | {stop, normal, ok, State}.
98 |
99 | handle_cast(_, State) ->
100 | {noreply, State}.
101 |
102 | handle_info(_, State) ->
103 | {noreply, State}.
104 |
105 | code_change(_OldVsn, State, _Extra) ->
106 | {ok, State}.
107 |
108 | terminate(_Reason, _State) ->
109 | ok.
110 |
111 | %% ==================================================================
112 | %% Private functions
113 | %% ==================================================================
114 | get_modules(Config) ->
115 | lists:foldl(fun({_, Hooks}, Acc) ->
116 | Names = proplists:get_keys(Hooks),
117 | lists:merge(Acc, lists:sort(Names))
118 | end, [], Config).
119 |
120 | maybe_init_hook(Module, Hooks) ->
121 | case proplists:is_defined(Module, Hooks) of
122 | true ->
123 | Hooks;
124 | false ->
125 | ok = Module:init(),
126 | [Module|Hooks]
127 | end.
128 |
129 | update_hooks(OldHooks, Config) ->
130 | update_hooks(OldHooks, Config, []).
131 |
132 | update_hooks([], [], Acc) ->
133 | lists:reverse(Acc);
134 | update_hooks([{_, Modules}|T], [], Acc) ->
135 | lists:foreach(fun({Module, State}) ->
136 | catch Module:deactivate(State)
137 | end, Modules),
138 | update_hooks(T, [], Acc);
139 | update_hooks(OldHooks, [{HookName, Modules}|T], Acc) ->
140 | Creater = fun({Module, Args}) ->
141 | case lists:member(HookName, Module:hooks()) of
142 | true ->
143 | {ok, State} = Module:activate(Args),
144 | {Module, State};
145 | false ->
146 | erlang:error(not_hookable)
147 | end
148 | end,
149 | Remover = fun({Module, State}) -> catch Module:deactivate(State) end,
150 | Hooks = proplists:get_value(HookName, OldHooks, []),
151 | Hooks2 = sort_same_order(Hooks, Modules, Creater, Remover),
152 | OldHooks2 = proplists:delete(HookName, OldHooks),
153 | update_hooks(OldHooks2, T, [{HookName, Hooks2}|Acc]).
154 |
155 | sort_same_order(List, Guide, Creater, Remover) ->
156 | sort_same_order(List, Guide, [], Creater, Remover).
157 |
158 | sort_same_order([], [], Acc, _, _) ->
159 | lists:reverse(Acc);
160 | sort_same_order(List, [], Acc, Creater, Remover) ->
161 | lists:foreach(Remover, List),
162 | sort_same_order([], [], Acc, Creater, Remover);
163 | sort_same_order([], Guide, Acc, Creater, Remover) ->
164 | Acc2 =lists:foldl(fun(E, A) -> [Creater(E)|A] end, Acc, Guide),
165 | sort_same_order([], [], Acc2, Creater, Remover);
166 | sort_same_order([{Key, _}=Old|List], [{Key, _}=New|Guide], Acc, Creater, Remover) ->
167 | Remover(Old),
168 | sort_same_order(List, Guide, [Creater(New)|Acc], Creater, Remover);
169 | sort_same_order(List, [{Key, _}=E|Guide], Acc, Creater, Remover) ->
170 | case proplists:get_value(Key, List) of
171 | undefined ->
172 | sort_same_order(List, Guide, [Creater(E)|Acc], Creater, Remover);
173 | Other ->
174 | List2 = proplists:delete(Key, List),
175 | sort_same_order(List2, Guide, [{Key, Other}|Acc], Creater, Remover)
176 | end.
177 |
178 | %% ==================================================================
179 | %% EUnit test
180 | %% ==================================================================
181 | -ifdef(TEST).
182 | -include_lib("eunit/include/eunit.hrl").
183 |
184 | sort_same_order_test_() ->
185 | C = fun({K, [E]}) -> {K, E} end,
186 | R = fun(_) -> ok end,
187 | [?_assertEqual([{a, 1}, {b, 2}], sort_same_order([], [{a, [1]}, {b, [2]}], C, R)),
188 | ?_assertEqual([{b, 2}], sort_same_order([{a, 1}, {b, 2}, {c, 3}], [{b, [2]}], C, R)),
189 | ?_assertEqual([{b, 3}], sort_same_order([{b, 2}], [{b, [3]}], C, R)),
190 | ?_assertEqual([{c, 3}, {d, 4}, {a, 1}],
191 | sort_same_order([{a, 1}, {b, 2}, {c, 3}], [{c, [3]}, {d, [4]}, {a, [1]}], C, R))].
192 |
193 | -endif.
194 |
--------------------------------------------------------------------------------
/src/lmq_hook_preserve_header.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_hook_preserve_header).
2 | -behaviour(lmq_hook).
3 |
4 | -export([init/0, hooks/0, activate/1, deactivate/1]).
5 | -export([pre_http_push/2]).
6 |
7 | init() ->
8 | ok.
9 |
10 | hooks() ->
11 | [pre_http_push].
12 |
13 | activate(Preserve) ->
14 | {ok, Preserve}.
15 |
16 | deactivate(_) ->
17 | ok.
18 |
19 | pre_http_push({MD, Content, Req}, Preserve) ->
20 | {Hdrs, Req2} = cowboy_req:headers(Req),
21 | MD2 = lists:foldl(fun(K, Acc) ->
22 | case proplists:get_value(K, Hdrs) of
23 | undefined -> Acc;
24 | V -> lists:keystore(K, 1, Acc, {K, V})
25 | end
26 | end, MD, Preserve),
27 | {MD2, Content, Req2}.
28 |
29 | %% ==================================================================
30 | %% EUnit test
31 | %% ==================================================================
32 | -ifdef(TEST).
33 | -include_lib("eunit/include/eunit.hrl").
34 |
35 | hooks_test_() ->
36 | [?_assertEqual([pre_http_push], hooks())].
37 |
38 | activate_test_() ->
39 | [?_assertEqual({ok, [<<"header1">>]}, activate([<<"header1">>]))].
40 |
41 | deactivate_test_() ->
42 | [?_assertEqual(ok, deactivate([<<"header1">>]))].
43 |
44 | pre_http_push_test_() ->
45 | Hdrs = [{<<"content-type">>, <<"text/plain">>},
46 | {<<"x-custom-header">>, <<"foo">>},
47 | {<<"x-dummy-header">>, <<"dummy">>}],
48 | Req = cowboy_req:new([], [], undefined, <<"POST">>, <<"/messages/test">>, <<"">>,
49 | <<"HTTP/1.1">>, Hdrs, undefined, undefined, <<"">>, false, false, undefined),
50 | MD = [{<<"content-type">>, <<"application/octet-stream">>}],
51 | Body = <<"body">>,
52 | [?_assertEqual({[{<<"content-type">>, <<"application/octet-stream">>},
53 | {<<"x-custom-header">>, <<"foo">>}],
54 | Body, Req},
55 | pre_http_push({MD, Body, Req}, [<<"x-custom-header">>])),
56 | ?_assertEqual({[{<<"content-type">>, <<"text/plain">>},
57 | {<<"x-custom-header">>, <<"foo">>}],
58 | Body, Req},
59 | pre_http_push({MD, Body, Req}, [<<"content-type">>, <<"x-custom-header">>]))].
60 |
61 | -endif.
62 |
--------------------------------------------------------------------------------
/src/lmq_lib.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_lib).
2 |
3 | -include("lmq.hrl").
4 | -include_lib("stdlib/include/qlc.hrl").
5 | -export([init_mnesia/0, create_admin_table/0,
6 | get_lmq_info/1, get_lmq_info/2, set_lmq_info/2,
7 | queue_info/1, update_queue_props/2, all_queue_names/0, create/1,
8 | create/2, delete/1, enqueue/2, enqueue/3, dequeue/2, done/2, retain/3,
9 | release/2, put_back/2, first/1, rfind/2, waittime/1, export_message/1,
10 | get_properties/1, get_properties/2]).
11 |
12 | init_mnesia() ->
13 | case mnesia:system_info(db_nodes) =:= [node()] of
14 | true -> create_admin_table();
15 | false -> ok
16 | end.
17 |
18 | create_admin_table() ->
19 | case mnesia:create_table(?LMQ_INFO_TABLE, ?LMQ_INFO_TABLE_DEFS) of
20 | {atomic, ok} -> ok;
21 | {aborted, {already_exists, ?LMQ_INFO_TABLE}} -> ok;
22 | Other1 ->
23 | lager:error("Failed to create admin table: ~p", [Other1])
24 | end,
25 | case mnesia:create_table(?QUEUE_INFO_TABLE, ?QUEUE_INFO_TABLE_DEFS) of
26 | {atomic, ok} -> ok;
27 | {aborted, {already_exists, ?QUEUE_INFO_TABLE}} -> ok;
28 | Other2 ->
29 | lager:error("Failed to create admin table: ~p", [Other2])
30 | end.
31 |
32 | get_lmq_info(Key) ->
33 | transaction(fun() ->
34 | case mnesia:read(?LMQ_INFO_TABLE, Key) of
35 | [Info] -> {ok, Info#lmq_info.value};
36 | _ -> {error, not_found}
37 | end
38 | end).
39 |
40 | get_lmq_info(Key, Default) ->
41 | case get_lmq_info(Key) of
42 | {ok, _}=R -> R;
43 | {error, _} -> {ok, Default}
44 | end.
45 |
46 | set_lmq_info(Key, Value) ->
47 | Info = #lmq_info{key=Key, value=Value},
48 | transaction(fun() ->
49 | ok = mnesia:write(?LMQ_INFO_TABLE, Info, write)
50 | end).
51 |
52 | queue_info(Name) when is_atom(Name) ->
53 | F = fun() ->
54 | case qlc:e(qlc:q([P || #queue_info{name=N, props=P}
55 | <- mnesia:table(?QUEUE_INFO_TABLE),
56 | N =:= Name])) of
57 | [P] -> P;
58 | [] -> not_found
59 | end
60 | end,
61 | transaction(F).
62 |
63 | update_queue_props(Name, Props) when is_atom(Name) ->
64 | Info = #queue_info{name=Name, props=Props},
65 | transaction(fun() ->
66 | mnesia:write(?QUEUE_INFO_TABLE, Info, write)
67 | end).
68 |
69 | all_queue_names() ->
70 | transaction(fun() ->
71 | qlc:e(qlc:q([N || #queue_info{name=N}
72 | <- mnesia:table(?QUEUE_INFO_TABLE)]))
73 | end).
74 |
75 | create(Name) when is_atom(Name) ->
76 | create(Name, []).
77 |
78 | create(Name, Props) when is_atom(Name) ->
79 | Def = [
80 | {type, ordered_set},
81 | {attributes, record_info(fields, message)},
82 | {record_name, message},
83 | {ram_copies, mnesia:system_info(db_nodes)}
84 | ],
85 | Info = #queue_info{name=Name, props=Props},
86 | F = fun() -> mnesia:write(?QUEUE_INFO_TABLE, Info, write) end,
87 |
88 | case mnesia:create_table(Name, Def) of
89 | {atomic, ok} ->
90 | ok = transaction(F),
91 | lmq_event:queue_created(Name);
92 | {aborted, {already_exists, Name}} ->
93 | case queue_info(Name) of
94 | Props ->
95 | ok;
96 | _ ->
97 | ok = transaction(F),
98 | lmq_event:queue_created(Name)
99 | end;
100 | Other ->
101 | lager:error("Failed to create table '~p': ~p", [Name, Other])
102 | end.
103 |
104 | delete(Name) when is_atom(Name) ->
105 | F = fun() -> mnesia:delete(?QUEUE_INFO_TABLE, Name, write) end,
106 | transaction(F),
107 | case mnesia:delete_table(Name) of
108 | {atomic, ok} ->
109 | lmq_event:queue_deleted(Name),
110 | ok;
111 | {aborted, {no_exists, Name}} ->
112 | ok;
113 | Other ->
114 | lager:error("Failed to delete table '~p': ~p", [Name, Other])
115 | end.
116 |
117 | enqueue(Name, Content) ->
118 | enqueue(Name, Content, []).
119 |
120 | enqueue(Name, Content, Opts) ->
121 | case proplists:get_value(accum, Opts, 0) == 0 of
122 | true ->
123 | Retry = increment(proplists:get_value(retry, Opts, infinity)),
124 | Msg = #message{content=Content, retry=Retry},
125 | transaction(fun() -> mnesia:write(Name, Msg, write) end);
126 | false -> %% accumulate duration in milliseconds
127 | accumulate_message(Name, Content, Opts)
128 | end.
129 |
130 | accumulate_message(Name, Content, Opts) ->
131 | transaction(fun() ->
132 | QC = qlc:cursor(qlc:q([M || M=#message{id={TS, _}, state=accum}
133 | <- mnesia:table(Name),
134 | TS >= lmq_misc:unixtime()])),
135 | {Ret, Msg} = case qlc:next_answers(QC, 1) of
136 | [M] -> %% accumulating process already started
137 | Content1 = M#message.content ++ [Content],
138 | {{accum, yes}, M#message{content=Content1}};
139 | [] -> %% add new message for accumulating
140 | Retry = increment(proplists:get_value(retry, Opts, infinity)),
141 | Duration = proplists:get_value(accum, Opts),
142 | Id={lmq_misc:unixtime() + Duration / 1000, uuid:get_v4()},
143 | {{accum, new}, #message{id=Id, state=accum, type=compound,
144 | retry=Retry, content=[Content]}}
145 | end,
146 | mnesia:write(Name, Msg, write),
147 | ok = qlc:delete_cursor(QC),
148 | Ret
149 | end).
150 |
151 | dequeue(Name, Timeout) ->
152 | case transaction(fun() -> get_first_message(Name, Timeout) end) of
153 | {ok, Msg, Retention} ->
154 | lmq_metrics:update_metric(Name, retention, Retention),
155 | Msg;
156 | continue ->
157 | dequeue(Name, Timeout);
158 | Other ->
159 | Other
160 | end.
161 |
162 | get_first_message(Name, Timeout) ->
163 | case mnesia:first(Name) of
164 | '$end_of_table' ->
165 | empty;
166 | Key ->
167 | [M] = mnesia:read(Name, Key, read),
168 | {TS, _} = M#message.id,
169 | Now = lmq_misc:unixtime(),
170 | case TS > Now of
171 | true ->
172 | empty;
173 | false ->
174 | mnesia:delete(Name, Key, write),
175 | NewId = {Now + Timeout, uuid:get_v4()},
176 | case M#message.retry of
177 | infinity ->
178 | NewMsg = M#message{id=NewId, state=processing},
179 | mnesia:write(Name, NewMsg, write),
180 | {ok, NewMsg, Now - TS};
181 | N when N > 0 ->
182 | NewMsg = M#message{id=NewId, state=processing, retry=N-1},
183 | mnesia:write(Name, NewMsg, write),
184 | {ok, NewMsg, Now - TS};
185 | _ ->
186 | %% quit transaction to avoid infinite loop caused by
187 | %% large amount of waste messages
188 | continue
189 | end
190 | end
191 | end.
192 |
193 | increment(infinity) ->
194 | infinity;
195 |
196 | increment(Number) when is_integer(Number) ->
197 | Number + 1.
198 |
199 | done(Name, UUID) ->
200 | Now = lmq_misc:unixtime(),
201 | F = fun() ->
202 | case rfind(Name, UUID) of
203 | '$end_of_table' ->
204 | not_found;
205 | #message{id={TS, UUID}} when TS < Now ->
206 | not_found;
207 | #message{id=Key, state=processing} ->
208 | mnesia:delete(Name, Key, write);
209 | _ ->
210 | not_found
211 | end
212 | end,
213 | transaction(F).
214 |
215 | release(Name, UUID) ->
216 | put_back(Name, UUID, consume).
217 |
218 | put_back(Name, UUID) ->
219 | put_back(Name, UUID, keep).
220 |
221 | put_back(Name, UUID, Retry) ->
222 | Now = lmq_misc:unixtime(),
223 | F = fun() ->
224 | case rfind(Name, UUID) of
225 | '$end_of_table' ->
226 | not_found;
227 | #message{id={TS, UUID}} when TS < Now ->
228 | not_found;
229 | #message{state=processing, retry=R}=M ->
230 | M1 = if Retry =:= consume ->
231 | M#message{id={Now, UUID}, state=available, retry=R};
232 | true ->
233 | M#message{id={Now, UUID}, state=available, retry=increment(R)}
234 | end,
235 | mnesia:write(Name, M1, write),
236 | mnesia:delete(Name, M#message.id, write);
237 | _ ->
238 | not_found
239 | end
240 | end,
241 | transaction(F).
242 |
243 | retain(Name, UUID, Timeout) ->
244 | Now = lmq_misc:unixtime(),
245 | F = fun() ->
246 | case rfind(Name, UUID) of
247 | '$end_of_table' ->
248 | not_found;
249 | #message{id={TS, UUID}} when TS < Now ->
250 | not_found;
251 | #message{state=processing}=M ->
252 | M1 = M#message{id={Now + Timeout, UUID}},
253 | mnesia:write(Name, M1, write),
254 | mnesia:delete(Name, M#message.id, write);
255 | _ ->
256 | not_found
257 | end
258 | end,
259 | transaction(F).
260 |
261 | waittime(Name) ->
262 | case first(Name) of
263 | {error, _}=E -> E;
264 | empty -> infinity;
265 | Message ->
266 | {TS, _} = Message#message.id,
267 | Timeout = round((TS - lmq_misc:unixtime()) * 1000),
268 | lists:max([Timeout, 0])
269 | end.
270 |
271 | first(Name) ->
272 | F = fun() ->
273 | case mnesia:first(Name) of
274 | '$end_of_table' -> empty;
275 | Key ->
276 | [Item] = mnesia:read(Name, Key, read),
277 | Item
278 | end
279 | end,
280 | transaction(F).
281 |
282 | rfind(Tab, Id) ->
283 | F = fun(M, _Acc) when element(2, M#message.id) =:= Id ->
284 | %% break loop, throw will be handled by foldr
285 | throw(M);
286 | (_, Acc) -> Acc
287 | end,
288 | mnesia:foldr(F, '$end_of_table', Tab).
289 |
290 | transaction(F) ->
291 | case mnesia:transaction(F) of
292 | {atomic, Val} -> Val;
293 | {aborted, {no_exists, _}} -> {error, no_queue_exists};
294 | {aborted, Reason} -> {error, Reason}
295 | end.
296 |
297 | get_properties(Name) ->
298 | Base = get_props(Name),
299 | case lmq_lib:queue_info(Name) of
300 | not_found -> Base;
301 | Props -> lmq_misc:extend(Props, Base)
302 | end.
303 |
304 | get_properties(Name, Override) ->
305 | Props = get_properties(Name),
306 | lmq_misc:extend(Override, Props).
307 |
308 | get_props(Name) ->
309 | {ok, DefaultProps} = get_lmq_info(default_props, []),
310 | get_props(Name, DefaultProps).
311 |
312 | get_props(_Name, []) ->
313 | ?DEFAULT_QUEUE_PROPS;
314 |
315 | get_props(Name, PropsList) when is_atom(Name) ->
316 | get_props(atom_to_list(Name), PropsList);
317 |
318 | get_props(Name, [{Regexp, Props} | T]) when is_list(Name) ->
319 | {ok, MP} = re:compile(Regexp),
320 | case re:run(Name, MP) of
321 | {match, _} -> lmq_misc:extend(Props, ?DEFAULT_QUEUE_PROPS);
322 | _ -> get_props(Name, T)
323 | end.
324 |
325 | export_message(M=#message{}) ->
326 | UUID = list_to_binary(uuid:uuid_to_string(element(2, M#message.id))),
327 | [{id, UUID}, {type, M#message.type}, {retry, M#message.retry}, {content, M#message.content}].
328 |
329 | %% ==================================================================
330 | %% EUnit tests
331 | %% ==================================================================
332 |
333 | -ifdef(TEST).
334 | -include_lib("eunit/include/eunit.hrl").
335 |
336 | get_props_test() ->
337 | DefaultProps = [{"lmq", [{retry, 0}]}],
338 | Props = lmq_misc:extend([{retry, 0}], ?DEFAULT_QUEUE_PROPS),
339 | ?assertEqual(Props, get_props(lmq, DefaultProps)),
340 | ?assertEqual(Props, get_props("lmq", DefaultProps)),
341 | ?assertEqual(?DEFAULT_QUEUE_PROPS, get_props("foo", DefaultProps)),
342 | ?assertEqual(?DEFAULT_QUEUE_PROPS, get_props("lmq", [])).
343 |
344 | export_message_test() ->
345 | Ref = make_ref(),
346 | M = #message{content=Ref},
347 | UUID = list_to_binary(uuid:uuid_to_string(element(2, M#message.id))),
348 | ?assertEqual([{id, UUID}, {type, normal}, {retry, 0}, {content, Ref}],
349 | export_message(M)),
350 |
351 | M2 = M#message{type=compound, retry=1},
352 | ?assertEqual([{id, UUID}, {type, compound}, {retry, 1}, {content, Ref}],
353 | export_message(M2)).
354 |
355 | -endif.
356 |
--------------------------------------------------------------------------------
/src/lmq_metrics.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_metrics).
2 |
3 | -export([create_queue_metrics/1, update_metric/2, update_metric/3,
4 | get_metric/2, get_metric_name/2]).
5 |
6 | -include("lmq.hrl").
7 |
8 | %% ==================================================================
9 | %% Public API
10 | %% ==================================================================
11 |
12 | create_queue_metrics(Name) when is_atom(Name) ->
13 | folsom_metrics:new_meter(get_metric_name(Name, push)),
14 | folsom_metrics:new_meter(get_metric_name(Name, pull)),
15 | folsom_metrics:new_histogram(get_metric_name(Name, retention),
16 | slide_uniform, {60, 100}),
17 |
18 | Metrics = [get_metric_name(Name, push),
19 | get_metric_name(Name, pull),
20 | get_metric_name(Name, retention)],
21 | [folsom_metrics:tag_metric(M, Name) || M <- Metrics],
22 | [folsom_metrics:tag_metric(M, ?LMQ_ALL_METRICS) || M <- Metrics],
23 | ok.
24 |
25 | update_metric(Name, push) when is_atom(Name) ->
26 | folsom_metrics:notify({get_metric_name(Name, push), 1}),
27 | influxdb_client:write(influxdb_data(rate, [queue, action, count], [[Name, push, 1]])),
28 | statsderl:increment(statsd_name(Name, push), 1, ?STATSD_SAMPLERATE);
29 |
30 | update_metric(Name, pull) when is_atom(Name) ->
31 | folsom_metrics:notify({get_metric_name(Name, pull), 1}),
32 | influxdb_client:write(influxdb_data(rate, [queue, action, count], [[Name, pull, 1]])),
33 | statsderl:increment(statsd_name(Name, pull), 1, ?STATSD_SAMPLERATE);
34 | update_metric(Name, List) when is_atom(Name), is_list(List) ->
35 | {Columns, Points} = lists:foldl(fun({Type, Value}, {Columns, Points}) ->
36 | update_metric(Name, Type, Value),
37 | {[Type|Columns], [Value|Points]}
38 | end, {[queue], [Name]}, List),
39 | influxdb_client:write(influxdb_data(stats, Columns, [Points])).
40 |
41 | update_metric(Name, retention, Time) when is_atom(Name) ->
42 | folsom_metrics:notify({get_metric_name(Name, retention), Time}),
43 | influxdb_client:write(influxdb_data(stats, [queue, retention], [[Name, Time]])),
44 | statsderl:timing(statsd_name(Name, retention), Time, ?STATSD_SAMPLERATE);
45 | update_metric(Name, Type, Value) ->
46 | statsderl:gauge(statsd_name(Name, Type), Value, 1).
47 |
48 | get_metric(Name, push) when is_atom(Name) ->
49 | folsom_metrics:get_metric_value(get_metric_name(Name, push));
50 |
51 | get_metric(Name, pull) when is_atom(Name) ->
52 | folsom_metrics:get_metric_value(get_metric_name(Name, pull));
53 |
54 | get_metric(Name, retention) when is_atom(Name) ->
55 | folsom_metrics:get_histogram_statistics(get_metric_name(Name, retention)).
56 |
57 | get_metric_name(Name, Type) when is_atom(Name), is_atom(Type) ->
58 | get_metric_name(atom_to_binary(Name, latin1), Type);
59 |
60 | get_metric_name(Name, push) when is_binary(Name) ->
61 | get_metric_name(Name, <<"push">>);
62 |
63 | get_metric_name(Name, pull) when is_binary(Name) ->
64 | get_metric_name(Name, <<"pull">>);
65 |
66 | get_metric_name(Name, retention) when is_binary(Name) ->
67 | get_metric_name(Name, <<"retention">>);
68 |
69 | get_metric_name(Name, Type) when is_binary(Name), is_binary(Type) ->
70 | <>/binary, Type/binary>>.
71 |
72 | %% ==================================================================
73 | %% EUnit tests
74 | %% ==================================================================
75 |
76 | statsd_name(Name, Action) ->
77 | lists:flatten(io_lib:format("lmq.~s.~s", [Name, Action])).
78 |
79 | influxdb_data(Name, Columns, Points) ->
80 | [[{name, Name}, {columns, Columns}, {points, Points}]].
81 |
82 | -ifdef(TEST).
83 | -include_lib("eunit/include/eunit.hrl").
84 |
85 | get_metric_name_test() ->
86 | ?assertEqual(<<"foo_push">>, get_metric_name(foo, push)),
87 | ?assertEqual(<<"foo_pull">>, get_metric_name(foo, pull)),
88 | ?assertEqual(<<"foo_retention">>, get_metric_name(foo, retention)).
89 |
90 | statsd_name_test() ->
91 | ?assertEqual("lmq.queue.push", statsd_name(queue, push)),
92 | ?assertEqual("lmq.Q/N.push", statsd_name('Q/N', push)).
93 |
94 | -endif.
95 |
--------------------------------------------------------------------------------
/src/lmq_misc.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_misc).
2 | -export([unixtime/0, extend/2, btof/1]).
3 |
4 | unixtime() ->
5 | {MegaSecs, Secs, MicroSecs} = os:timestamp(),
6 | MegaSecs * 1000000 + Secs + MicroSecs / 1000000.
7 |
8 | extend(Override, Base) ->
9 | Props = lists:foldl(fun({K, _}=T, Acc) ->
10 | lists:keystore(K, 1, Acc, T)
11 | end, Base, Override),
12 | lists:keysort(1, Props).
13 |
14 | btof(B) ->
15 | B2 = <>,
16 | case string:to_float(binary_to_list(B2)) of
17 | {error, _} -> {error, badarg};
18 | {F, _} -> {ok, F}
19 | end.
20 |
21 | -ifdef(TEST).
22 | -include_lib("eunit/include/eunit.hrl").
23 |
24 | -define(assertEq(S, Expected),
25 | try
26 | ?assertEqual(S, Expected)
27 | catch _:_ ->
28 | ?debugFmt("~nexpected: ~p~n", [Expected]),
29 | ?debugFmt("~ngot: ~p~n", [S]),
30 | throw(assertion_error)
31 | end).
32 |
33 | unixtime_test() ->
34 | T = lmq_misc:unixtime(),
35 | ?assert(is_float(T)),
36 | ?assertEq(round(T) div 1000000000, 1).
37 |
38 | extend_test() ->
39 | ?assertEq(lmq_misc:extend([{retry, infinity}, {timeout, 5}],
40 | [{timeout, 30}, {type, normal}]),
41 | [{retry, infinity}, {timeout, 5}, {type, normal}]),
42 | ?assertEq(lmq_misc:extend([], [{timeout, 30}, {type, normal}]),
43 | [{timeout, 30}, {type, normal}]).
44 |
45 | binary_to_float_test() ->
46 | ?assertEq({ok, 10.0}, btof(<<"10">>)),
47 | ?assertEq({ok, 10.0}, btof(<<"10.0">>)),
48 | ?assertEq({ok, 12.34}, btof(<<"12.34">>)),
49 | ?assertEq({ok, 12.34}, btof(<<"12.34abc">>)),
50 | ?assertEq({error, badarg}, btof(<<"abc">>)).
51 |
52 | -endif.
53 |
--------------------------------------------------------------------------------
/src/lmq_mpull.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_mpull).
2 |
3 | -behaviour(gen_fsm).
4 |
5 | -include("lmq.hrl").
6 |
7 | -export([init/1, handle_event/3, handle_sync_event/4, handle_info/3,
8 | terminate/3, code_change/4,
9 | idle/2, idle/3, waiting/2, finalize/2]).
10 | -export([start/0, start_link/0, pull/2, pull/3,
11 | pull_async/2, pull_async/3, pull_cancel/1,
12 | maybe_pull/2, list_active/0]).
13 |
14 | -define(UNEXPECTED(Event, State),
15 | lager:warning("~p received unknown event ~p while in state ~p",
16 | [self(), Event, State])).
17 | -define(CLOSE_WAIT, 10).
18 |
19 | -record(state, {from, monitor, regexp, timeout, mapping}).
20 |
21 | %% ==================================================================
22 | %% Public API
23 | %% ==================================================================
24 |
25 | start() ->
26 | supervisor:start_child(lmq_mpull_sup, []).
27 |
28 | start_link() ->
29 | gen_fsm:start_link(?MODULE, [], []).
30 |
31 | pull(Pid, Regexp) ->
32 | gen_fsm:sync_send_event(Pid, {pull, Regexp, infinity}, infinity).
33 |
34 | pull(Pid, Regexp, Timeout) ->
35 | gen_fsm:sync_send_event(Pid, {pull, Regexp, Timeout}, infinity).
36 |
37 | pull_async(Pid, Regexp) ->
38 | pull_async(Pid, Regexp, infinity).
39 |
40 | pull_async(Pid, Regexp, Timeout) ->
41 | gen_fsm:sync_send_event(Pid, {pull_async, Regexp, Timeout}).
42 |
43 | pull_cancel(Pid) ->
44 | gen_fsm:send_event(Pid, cancel).
45 |
46 | maybe_pull(Pid, QName) when is_atom(QName) ->
47 | gen_fsm:send_event(Pid, {maybe_pull, QName}).
48 |
49 | list_active() ->
50 | [Pid || {_, Pid, _, _} <- supervisor:which_children(lmq_mpull_sup)].
51 |
52 | %% ==================================================================
53 | %% gen_fsm callbacks
54 | %% ==================================================================
55 |
56 | init([]) ->
57 | {ok, idle, #state{}}.
58 |
59 | idle(Event, State) ->
60 | ?UNEXPECTED(Event, idle),
61 | {next_state, idle, State}.
62 |
63 | idle({pull, Regexp, Timeout}, {Pid, _}=From, #state{}=S) ->
64 | Monitor = erlang:monitor(process, Pid),
65 | case lmq_queue_mgr:match(Regexp) of
66 | {error, _}=R ->
67 | {stop, error, R, S#state{from=From, monitor=Monitor}};
68 | Queues ->
69 | Mapping = lists:foldl(fun({_, QPid}=Q, Acc) ->
70 | Id = lmq_queue:pull_async(QPid, Timeout),
71 | dict:store(Id, Q, Acc)
72 | end, dict:new(), Queues),
73 | if is_number(Timeout), Timeout > 0 ->
74 | gen_fsm:send_event_after(Timeout, cancel);
75 | true -> ok
76 | end,
77 | State = S#state{from=From, monitor=Monitor, regexp=Regexp,
78 | timeout=Timeout, mapping=Mapping},
79 | {next_state, waiting, State}
80 | end;
81 |
82 | idle({pull_async, Regexp, Timeout}, {Pid, _}=From, #state{}=S) ->
83 | case idle({pull, Regexp, Timeout}, From, S) of
84 | {next_state, Next, State} ->
85 | Ref = make_ref(),
86 | {reply, {ok, Ref}, Next, State#state{from={async, Pid, Ref}}};
87 | Other ->
88 | Other
89 | end;
90 |
91 | idle(Event, _From, State) ->
92 | ?UNEXPECTED(Event, idle),
93 | {next_state, idle, State}.
94 |
95 | waiting({maybe_pull, QName}, #state{}=S) ->
96 | State = case re:compile(S#state.regexp) of
97 | {ok, MP} ->
98 | case re:run(atom_to_list(QName), MP) of
99 | {match, _} ->
100 | case lmq_queue_mgr:get(QName) of
101 | not_found -> S;
102 | Pid ->
103 | Id = lmq_queue:pull_async(Pid, S#state.timeout),
104 | Mapping = dict:store(Id, {QName, Pid}, S#state.mapping),
105 | S#state{mapping=Mapping}
106 | end;
107 | _ -> S
108 | end;
109 | {error, _} -> S
110 | end,
111 | {next_state, waiting, State};
112 |
113 | waiting(cancel, #state{}=S) ->
114 | waiting(timeout, S);
115 |
116 | waiting(timeout, #state{}=S) ->
117 | cancel_pull(S#state.mapping),
118 | reply(S#state.from, empty),
119 | {next_state, finalize, S, ?CLOSE_WAIT};
120 |
121 | waiting(Event, State) ->
122 | ?UNEXPECTED(Event, waiting),
123 | {next_state, waiting, State}.
124 |
125 | finalize(timeout, State) ->
126 | {stop, normal, State};
127 |
128 | finalize(Event, State) ->
129 | ?UNEXPECTED(Event, finalize),
130 | {next_state, finalize, State}.
131 |
132 | handle_info({Id, #message{}=M}, waiting, #state{mapping=Mapping}=S) ->
133 | {Name, _} = dict:fetch(Id, Mapping),
134 | cancel_pull(dict:erase(Id, Mapping)),
135 | Response = lmq_lib:export_message(M),
136 | reply(S#state.from, [{queue, Name} | Response]),
137 | {next_state, finalize, S, ?CLOSE_WAIT};
138 |
139 | handle_info({Id, {error, Reason}}, waiting, #state{mapping=Mapping}=S) ->
140 | Mapping1 = dict:erase(Id, Mapping),
141 | lager:debug("pull_any for ~p: ~p, rest ~p",
142 | [element(1, dict:fetch(Id, Mapping)), Reason, dict:size(Mapping1)]),
143 | case dict:size(Mapping1) of
144 | 0 ->
145 | reply(S#state.from, empty),
146 | %% it is safe to shutdown because all responses are received.
147 | {stop, normal, S};
148 | _ ->
149 | %% short period for procces messages that already in the mailbox.
150 | {next_state, waiting, S#state{mapping=Mapping1}, ?CLOSE_WAIT}
151 | end;
152 |
153 | handle_info({Id, #message{id={_, UUID}}}, finalize, #state{}=S) ->
154 | {_, Pid} = dict:fetch(Id, S#state.mapping),
155 | lmq_queue:put_back(Pid, UUID),
156 | {next_state, finalize, S, ?CLOSE_WAIT};
157 |
158 | handle_info({_Id, {error, _Reason}}, finalize, State) ->
159 | {next_state, finalize, State, ?CLOSE_WAIT};
160 |
161 | handle_info({'DOWN', Monitor, process, _, _}, _, #state{monitor=Monitor}=S) ->
162 | cancel_pull(S#state.mapping),
163 | {next_state, finalize, S, ?CLOSE_WAIT};
164 |
165 | handle_info(Event, StateName, State) ->
166 | ?UNEXPECTED(Event, StateName),
167 | {next_state, StateName, State}.
168 |
169 | handle_event(Event, StateName, State) ->
170 | ?UNEXPECTED(Event, StateName),
171 | {next_state, StateName, State}.
172 |
173 | handle_sync_event(Event, _From, StateName, State) ->
174 | ?UNEXPECTED(Event, StateName),
175 | {reply, error, StateName, State}.
176 |
177 | terminate(normal, _StateName, #state{monitor=Monitor}) ->
178 | erlang:demonitor(Monitor, [flush]),
179 | ok;
180 |
181 | terminate(error, _StateName, #state{monitor=Monitor}) ->
182 | erlang:demonitor(Monitor, [flush]),
183 | ok.
184 |
185 | code_change(_OldVsn, StateName, State, _Extra) ->
186 | {ok, StateName, State}.
187 |
188 | %% ==================================================================
189 | %% Private functions
190 | %% ==================================================================
191 |
192 | cancel_pull(Mapping) ->
193 | %% after calling this function, queues never sent a new message.
194 | dict:map(fun(Id, {_, Pid}) -> lmq_queue:pull_cancel(Pid, Id) end, Mapping).
195 |
196 | reply({async, Pid, Ref}, Msg) ->
197 | Pid ! {Ref, Msg};
198 | reply(From, Msg) ->
199 | gen_fsm:reply(From, Msg).
200 |
--------------------------------------------------------------------------------
/src/lmq_mpull_sup.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_mpull_sup).
2 |
3 | -behaviour(supervisor).
4 |
5 | -export([init/1]).
6 | -export([start_link/0]).
7 |
8 | start_link() ->
9 | supervisor:start_link({local, ?MODULE}, ?MODULE, []).
10 |
11 | init([]) ->
12 | MaxRestart = 5,
13 | MaxTime = 3600,
14 | {ok, {{simple_one_for_one, MaxRestart, MaxTime},
15 | [{lmq_mpull,
16 | {lmq_mpull, start_link, []},
17 | temporary,
18 | 5000,
19 | worker,
20 | [lmq_mpull]}]}}.
21 |
--------------------------------------------------------------------------------
/src/lmq_queue.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_queue).
2 | -behaviour(gen_server).
3 | -export([start/1, start/2, start_link/1, start_link/2, stop/1, notify/1,
4 | push/2, pull/1, pull/2, pull_async/1, pull_async/2, pull_cancel/2,
5 | done/2, retain/2, release/2, put_back/2, props/2, get_properties/1,
6 | reload_properties/1]).
7 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
8 | code_change/3, terminate/2]).
9 |
10 | -include("lmq.hrl").
11 | -record(state, {name, props, raw_props, waiting=queue:new(), monitors=gb_sets:empty()}).
12 | -record(waiting, {from, ref, timeout, start_time=lmq_misc:unixtime()}).
13 |
14 | start(Name) ->
15 | supervisor:start_child(lmq_queue_sup, [Name]).
16 |
17 | start(Name, Props) ->
18 | supervisor:start_child(lmq_queue_sup, [Name, Props]).
19 |
20 | start_link(Name) when is_atom(Name) ->
21 | case lmq_lib:queue_info(Name) of
22 | not_found -> ok = lmq_lib:create(Name);
23 | _ -> ok
24 | end,
25 | gen_server:start_link(?MODULE, Name, []).
26 |
27 | start_link(Name, Props) when is_atom(Name) ->
28 | ok = lmq_lib:create(Name, Props),
29 | gen_server:start_link(?MODULE, Name, []).
30 |
31 | push(Pid, Content) ->
32 | gen_server:call(Pid, {push, Content}).
33 |
34 | pull(Pid) ->
35 | gen_server:call(Pid, {pull, infinity}, infinity).
36 |
37 | pull(Pid, 0) ->
38 | %% in this case, cannot use gen_server's timeout
39 | case gen_server:call(Pid, {pull, 0}) of
40 | {error, timeout} -> empty;
41 | R -> R
42 | end;
43 |
44 | pull(Pid, Timeout) ->
45 | try
46 | case gen_server:call(Pid, {pull, Timeout}, round(Timeout * 1000)) of
47 | {error, timeout} -> empty;
48 | R -> R
49 | end
50 | catch
51 | exit:{timeout, _} -> empty
52 | end.
53 |
54 | pull_async(Pid) ->
55 | pull_async(Pid, infinity).
56 |
57 | pull_async(Pid, Timeout) ->
58 | gen_server:call(Pid, {pull_async, Timeout}).
59 |
60 | pull_cancel(Pid, Ref) ->
61 | gen_server:call(Pid, {pull_cancel, Ref}).
62 |
63 | done(Pid, UUID) ->
64 | gen_server:call(Pid, {done, UUID}).
65 |
66 | retain(Pid, UUID) ->
67 | gen_server:call(Pid, {retain, UUID}).
68 |
69 | release(Pid, UUID) ->
70 | gen_server:call(Pid, {release, UUID}).
71 |
72 | put_back(Pid, UUID) ->
73 | gen_server:call(Pid, {put_back, UUID}).
74 |
75 | props(Pid, Props) ->
76 | gen_server:call(Pid, {props, Props}).
77 |
78 | get_properties(Pid) ->
79 | gen_server:call(Pid, get_properties).
80 |
81 | reload_properties(Pid) ->
82 | gen_server:cast(Pid, reload_properties).
83 |
84 | notify(Pid) ->
85 | gen_server:cast(Pid, notify).
86 |
87 | stop(Pid) ->
88 | gen_server:call(Pid, stop).
89 |
90 | %% ==================================================================
91 | %% gen_server callbacks
92 | %% ==================================================================
93 |
94 | init(Name) ->
95 | lager:info("Starting the queue: ~s ~p", [Name, self()]),
96 | Props = lmq_lib:get_properties(Name),
97 | RawProps = lmq_lib:queue_info(Name),
98 | lmq_hook:register(Name, proplists:get_value(hooks, Props, [])),
99 | %% this is necessary when a queue restarted by supervisor
100 | lmq_queue_mgr:queue_started(Name, self()),
101 | ok = lmq_metrics:create_queue_metrics(Name),
102 | {ok, #state{name=Name, props=Props, raw_props=RawProps}}.
103 |
104 | handle_call(stop, _From, State) ->
105 | lager:info("Stopping the queue: ~s ~p", [State#state.name, self()]),
106 | {stop, normal, ok, State};
107 |
108 | handle_call(Msg, From, State) ->
109 | case handle_queue_call(Msg, From, State) of
110 | {reply, Reply, State1} ->
111 | {State2, Sleep} = prepare_sleep(State1),
112 | {reply, Reply, State2, Sleep};
113 | {noreply, State1} ->
114 | {State2, Sleep} = prepare_sleep(State1),
115 | {noreply, State2, Sleep}
116 | end.
117 |
118 | handle_cast(reload_properties, S) ->
119 | Props = lmq_lib:get_properties(S#state.name),
120 | lmq_hook:register(S#state.name, proplists:get_value(hooks, Props, [])),
121 | lager:info("Reload queue properties: ~s ~p", [S#state.name, Props]),
122 | State = S#state{props=Props},
123 | {State1, Sleep} = prepare_sleep(State),
124 | {noreply, State1, Sleep};
125 |
126 | handle_cast(notify, State) ->
127 | {State1, Sleep} = prepare_sleep(State),
128 | {noreply, State1, Sleep};
129 |
130 | handle_cast(Msg, State) ->
131 | lager:warning("Unknown message received: ~p", [Msg]),
132 | {noreply, State}.
133 |
134 | handle_info(timeout, S=#state{}) ->
135 | NewState = maybe_push_message(S),
136 | lager:debug("number of waitings in ~p: ~p", [S#state.name, queue:len(NewState#state.waiting)]),
137 | {State, Sleep} = prepare_sleep(NewState),
138 | {noreply, State, Sleep};
139 |
140 | handle_info({'DOWN', Ref, process, _Pid, _}, S=#state{}) ->
141 | State = remove_waiting(Ref, S),
142 | {State1, Sleep} = prepare_sleep(State),
143 | {noreply, State1, Sleep};
144 |
145 | handle_info(Msg, State) ->
146 | lager:warning("Unknown message received: ~p", [Msg]),
147 | {noreply, State}.
148 |
149 | code_change(_OldVsn, State, _Extra) ->
150 | {ok, State}.
151 |
152 | terminate(_Reason, _State) ->
153 | ok.
154 |
155 | %% ==================================================================
156 | %% Private functions
157 | %% ==================================================================
158 |
159 | handle_queue_call({push, Content}, _From, S=#state{}) ->
160 | R = lmq_lib:enqueue(S#state.name, Content, S#state.props),
161 | lmq_event:new_message(S#state.name),
162 | lmq_metrics:update_metric(S#state.name, push),
163 | {reply, R, S};
164 |
165 | handle_queue_call({pull, Timeout}, From={Pid, _}, S=#state{}) ->
166 | State = add_waiting(From, Pid, Timeout, S),
167 | {noreply, State};
168 |
169 | handle_queue_call({pull_async, Timeout}, {Pid, _}, S=#state{}) ->
170 | State = add_waiting(Pid, Timeout, S),
171 | Ref = (queue:get_r(State#state.waiting))#waiting.ref,
172 | {reply, Ref, State};
173 |
174 | handle_queue_call({pull_cancel, Ref}, _From, S=#state{}) ->
175 | State = remove_waiting(Ref, S),
176 | {reply, ok, State};
177 |
178 | handle_queue_call({done, UUID}, _From, S=#state{}) ->
179 | R = lmq_lib:done(S#state.name, UUID),
180 | {reply, R, S};
181 |
182 | handle_queue_call({retain, UUID}, _From, S=#state{props=Props}) ->
183 | R = lmq_lib:retain(S#state.name, UUID, proplists:get_value(timeout, Props)),
184 | {reply, R, S};
185 |
186 | handle_queue_call({release, UUID}, _From, S=#state{}) ->
187 | R = lmq_lib:release(S#state.name, UUID),
188 | {reply, R, S};
189 |
190 | handle_queue_call({put_back, UUID}, _From, S=#state{}) ->
191 | R = lmq_lib:put_back(S#state.name, UUID),
192 | {reply, R, S};
193 |
194 | handle_queue_call({props, Props}, _From, S=#state{raw_props=Props}) ->
195 | lager:info("Queue property of ~s already set to ~p", [S#state.name, Props]),
196 | {reply, ok, S};
197 | handle_queue_call({props, Props}, _From, S=#state{}) ->
198 | lmq_lib:update_queue_props(S#state.name, Props),
199 | Props1 = lmq_lib:get_properties(S#state.name),
200 | lmq_hook:register(S#state.name, proplists:get_value(hooks, Props1, [])),
201 | lager:info("Queue property of ~s updated to ~p", [S#state.name, Props1]),
202 | State = S#state{props=Props1, raw_props=Props},
203 | {reply, ok, State};
204 |
205 | handle_queue_call(get_properties, _From, S) ->
206 | Props = S#state.props,
207 | {reply, Props, S}.
208 |
209 | maybe_push_message(S=#state{props=Props, waiting=Waiting}) ->
210 | case queue:out(Waiting) of
211 | {{value, W=#waiting{ref=Ref}}, NewWaiting} ->
212 | Timeout = proplists:get_value(timeout, Props),
213 | %% timeout = 0 is special case and it is safe to use it,
214 | %% because invalid waitings are removed before sleeping.
215 | %% thus, only waitings added in this tick are remained.
216 | case (W#waiting.timeout =:= 0 orelse wait_valid(W)) andalso
217 | lmq_lib:dequeue(S#state.name, Timeout) of
218 | false -> %% client timeout
219 | maybe_push_message(S#state{waiting=NewWaiting});
220 | empty ->
221 | S;
222 | Msg ->
223 | erlang:demonitor(Ref, [flush]),
224 | Monitors = gb_sets:delete(Ref, S#state.monitors),
225 | case W#waiting.from of
226 | {_, _}=From -> gen_server:reply(From, Msg);
227 | P when is_pid(P) -> P ! {Ref, Msg}
228 | end,
229 | lmq_metrics:update_metric(S#state.name, pull),
230 | S#state{waiting=NewWaiting, monitors=Monitors}
231 | end;
232 | {empty, Waiting} ->
233 | S
234 | end.
235 |
236 | add_waiting(Pid, Timeout, S=#state{}) ->
237 | add_waiting(Pid, Pid, Timeout, S).
238 |
239 | add_waiting(From, MonitorPid, Timeout, S=#state{}) ->
240 | Ref = erlang:monitor(process, MonitorPid),
241 | Waiting = queue:in(#waiting{from=From, ref=Ref, timeout=Timeout},
242 | S#state.waiting),
243 | Monitors = gb_sets:add(Ref, S#state.monitors),
244 | S#state{waiting=Waiting, monitors=Monitors}.
245 |
246 | remove_waiting(Ref, S=#state{monitors=M}) ->
247 | case gb_sets:is_member(Ref, M) of
248 | true ->
249 | erlang:demonitor(Ref, [flush]),
250 | Waiting = queue:filter(
251 | fun(#waiting{ref=V}) when V =:= Ref -> false;
252 | (_) -> true
253 | end, S#state.waiting),
254 | S#state{waiting=Waiting, monitors=gb_sets:delete(Ref, M)};
255 | false ->
256 | S
257 | end.
258 |
259 | wait_valid(#waiting{timeout=infinity}) ->
260 | true;
261 | wait_valid(#waiting{start_time=StartTime, timeout=Timeout}) ->
262 | StartTime + Timeout > lmq_misc:unixtime().
263 |
264 | prepare_sleep(S=#state{}) ->
265 | case queue:is_empty(S#state.waiting) of
266 | true -> {S, infinity};
267 | false ->
268 | case lmq_lib:waittime(S#state.name) of
269 | 0 -> {S, 0};
270 | T ->
271 | %% remove invalid waitings before sleeping
272 | Waitings = queue:filter(fun(W=#waiting{}) ->
273 | case wait_valid(W) of
274 | true -> true;
275 | false ->
276 | case W#waiting.from of
277 | {_, _}=From -> gen_server:reply(From, {error, timeout});
278 | P when is_pid(P) -> P ! {W#waiting.ref, {error, timeout}}
279 | end,
280 | false
281 | end
282 | end, S#state.waiting),
283 | case queue:is_empty(Waitings) of
284 | true -> {S#state{waiting=Waitings}, infinity};
285 | false -> {S#state{waiting=Waitings}, T}
286 | end
287 | end
288 | end.
289 |
--------------------------------------------------------------------------------
/src/lmq_queue_mgr.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_queue_mgr).
2 |
3 | -behaviour(gen_server).
4 | -export([start_link/0, start_link/1, queue_started/2, delete/1, get/1, get/2, match/1,
5 | set_default_props/1, get_default_props/0]).
6 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
7 | code_change/3, terminate/2]).
8 |
9 | -include("lmq.hrl").
10 |
11 | -record(state, {sup, qmap=dict:new(), stats_interval}).
12 |
13 | %% ==================================================================
14 | %% Public API
15 | %% ==================================================================
16 |
17 | start_link() ->
18 | start_link([]).
19 |
20 | start_link(Opts) ->
21 | gen_server:start_link({local, ?MODULE}, ?MODULE, Opts, []).
22 |
23 | queue_started(Name, QPid) when is_atom(Name) ->
24 | gen_server:cast(?MODULE, {queue_started, Name, QPid}).
25 |
26 | get(Name) when is_atom(Name) ->
27 | gen_server:call(?MODULE, {get, Name, []}).
28 |
29 | get(Name, Opts) when is_atom(Name) ->
30 | gen_server:call(?MODULE, {get, Name, Opts}).
31 |
32 | match(Regexp) when is_list(Regexp); is_binary(Regexp) ->
33 | gen_server:call(?MODULE, {match, Regexp}).
34 |
35 | delete(Name) when is_atom(Name) ->
36 | gen_server:call(?MODULE, {delete, Name}).
37 |
38 | set_default_props(PropsList) ->
39 | gen_server:call(?MODULE, {set_default_props, PropsList}).
40 |
41 | get_default_props() ->
42 | gen_server:call(?MODULE, {get_default_props}).
43 |
44 | %% ==================================================================
45 | %% gen_server callbacks
46 | %% ==================================================================
47 |
48 | init(Opts) ->
49 | lager:info("Starting the queue manager: ~p ~p", [self(), Opts]),
50 | lists:foreach(fun(Name) ->
51 | lmq_queue:start(Name)
52 | end, lmq_lib:all_queue_names()),
53 |
54 | StatsInterval = proplists:get_value(stats_interval, Opts),
55 | maybe_send_after(StatsInterval, emit_stats),
56 | {ok, #state{stats_interval=StatsInterval}}.
57 |
58 | handle_call({delete, Name}, _From, S=#state{}) when is_atom(Name) ->
59 | State = case dict:find(Name, S#state.qmap) of
60 | {ok, {Pid, _}} ->
61 | lmq_queue:stop(Pid),
62 | S#state{qmap=dict:erase(Name, S#state.qmap)};
63 | error ->
64 | S
65 | end,
66 | ok = lmq_lib:delete(Name),
67 | {reply, ok, State};
68 |
69 | handle_call({get, Name, Opts}, _From, S=#state{}) when is_atom(Name) ->
70 | case dict:find(Name, S#state.qmap) of
71 | {ok, {Pid, _}} ->
72 | case proplists:get_value(update, Opts) of
73 | true ->
74 | Props1 = case proplists:get_value(props, Opts, []) of
75 | [] -> [];
76 | Props -> lmq_misc:extend(Props, lmq_lib:queue_info(Name))
77 | end,
78 | case lmq_queue:props(Pid, Props1) of
79 | ok -> {reply, Pid, S};
80 | _ -> {reply, error, S}
81 | end;
82 | undefined ->
83 | {reply, Pid, S}
84 | end;
85 | error ->
86 | case proplists:get_value(create, Opts) of
87 | true ->
88 | {ok, Pid} = case proplists:get_value(props, Opts) of
89 | undefined -> lmq_queue:start(Name);
90 | Props -> lmq_queue:start(Name, lists:keysort(1, Props))
91 | end,
92 | lager:info("The new queue created: ~s ~p", [Name, Pid]),
93 | {reply, Pid, update_qmap(Name, Pid, S)};
94 | undefined ->
95 | {reply, not_found, S}
96 | end
97 | end;
98 |
99 | handle_call({match, Regexp}, _From, S=#state{}) ->
100 | R = case re:compile(Regexp) of
101 | {ok, MP} ->
102 | dict:fold(fun(Name, {Pid, _}, Acc) ->
103 | case re:run(atom_to_list(Name), MP) of
104 | {match, _} -> [{Name, Pid} | Acc];
105 | _ -> Acc
106 | end
107 | end, [], S#state.qmap);
108 | {error, _} ->
109 | {error, invalid_regexp}
110 | end,
111 | {reply, R, S};
112 |
113 | handle_call({set_default_props, PropsList}, _From, S=#state{}) ->
114 | case validate_props_list(PropsList) of
115 | {ok, _PropsList} ->
116 | lmq_lib:set_lmq_info(default_props, PropsList),
117 | dict:fold(fun(_, {Pid, _}, _) ->
118 | lmq_queue:reload_properties(Pid)
119 | end, ok, S#state.qmap),
120 | {reply, ok, S};
121 | {error, Reason} ->
122 | {reply, Reason, S}
123 | end;
124 |
125 | handle_call({get_default_props}, _From, S=#state{}) ->
126 | PropsList = case lmq_lib:get_lmq_info(default_props) of
127 | {ok, Value} -> Value;
128 | _ -> []
129 | end,
130 | {reply, PropsList, S};
131 |
132 | handle_call(Msg, _From, State) ->
133 | lager:warning("Unknown message: ~p", [Msg]),
134 | {noreply, State}.
135 |
136 | handle_cast({queue_started, Name, Pid}, S) when is_atom(Name) ->
137 | {noreply, update_qmap(Name, Pid, S)};
138 |
139 | handle_cast(_, State) ->
140 | {noreply, State}.
141 |
142 | handle_info({'DOWN', Ref, process, _Pid, _}, S=#state{qmap=QMap}) ->
143 | NewQMap = dict:filter(fun(_, {_, R}) ->
144 | R =/= Ref
145 | end, QMap),
146 | {noreply, S#state{qmap=NewQMap}};
147 | handle_info(emit_stats, S=#state{}) ->
148 | maybe_send_after(S#state.stats_interval, emit_stats),
149 | WordSize = erlang:system_info(wordsize),
150 | Queues = dict:fetch_keys(S#state.qmap),
151 | lists:foreach(fun(Queue) ->
152 | Size = mnesia:table_info(Queue, size),
153 | Memory = mnesia:table_info(Queue, memory) * WordSize,
154 | lmq_metrics:update_metric(Queue, [{size, Size}, {memory, Memory}])
155 | end, Queues),
156 | {noreply, S};
157 | handle_info(Msg, State) ->
158 | lager:warning("Unknown message: ~p", [Msg]),
159 | {noreply, State}.
160 |
161 | terminate(_Reason, _State) ->
162 | ok.
163 |
164 | code_change(_OldVsn, State, _Extra) ->
165 | {ok, State}.
166 |
167 | %% ==================================================================
168 | %% Private functions
169 | %% ==================================================================
170 |
171 | update_qmap(Name, Pid, #state{qmap=QMap}=S) ->
172 | Ref = erlang:monitor(process, Pid),
173 | S#state{qmap=dict:store(Name, {Pid, Ref}, QMap)}.
174 |
175 | validate_props_list(PropsList) ->
176 | try
177 | {ok, validate_props_list(PropsList, [])}
178 | catch
179 | error:function_clause -> {error, invalid_syntax}
180 | end.
181 |
182 | validate_props_list([], Acc) ->
183 | lists:reverse(Acc);
184 |
185 | validate_props_list([{Regexp, Props}|T], Acc) when is_list(Regexp); is_binary(Regexp), is_list(Props) ->
186 | {ok, MP} = re:compile(Regexp),
187 | Props1 = lmq_misc:extend(Props, ?DEFAULT_QUEUE_PROPS),
188 | validate_props_list(T, [{MP, Props1} | Acc]).
189 |
190 | maybe_send_after(Time, Message) when is_integer(Time) ->
191 | erlang:send_after(Time, ?MODULE, Message);
192 | maybe_send_after(_, _) ->
193 | ignore.
194 |
195 | %% ==================================================================
196 | %% EUnit tests
197 | %% ==================================================================
198 |
199 | -ifdef(TEST).
200 | -include_lib("eunit/include/eunit.hrl").
201 |
202 | validate_props_test() ->
203 | ?assertEqual(
204 | {ok, [{element(2, re:compile("lmq/a")), [{accum, 0}, {retry, 1}, {timeout, 30}]},
205 | {element(2, re:compile("lmq/.*")), [{accum, 0}, {retry, 2}, {timeout, 60}]}]},
206 | validate_props_list([{"lmq/a", [{retry, 1}]},
207 | {"lmq/.*", [{timeout, 60}]}])),
208 | ?assertEqual(
209 | {ok, [{element(2, re:compile(<<"lmq/.*">>)), [{accum, 0}, {retry, 1}, {timeout, 30}]}]},
210 | validate_props_list([{<<"lmq/.*">>, [{retry, 1}]}])),
211 | ?assertEqual(
212 | {error, invalid_syntax},
213 | validate_props_list([{"lmq/a", {retry, 1}}])).
214 |
215 | -endif.
216 |
--------------------------------------------------------------------------------
/src/lmq_queue_sup.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_queue_sup).
2 |
3 | -behaviour(supervisor).
4 | -export([start_link/0]).
5 | -export([init/1]).
6 |
7 | start_link() ->
8 | supervisor:start_link({local, ?MODULE}, ?MODULE, []).
9 |
10 | init([]) ->
11 | MaxRestart = 5,
12 | MaxTime = 3600,
13 | {ok, {{simple_one_for_one, MaxRestart, MaxTime},
14 | [{lmq_queue,
15 | {lmq_queue, start_link, []},
16 | transient,
17 | 5000,
18 | worker,
19 | [lmq_queue]}]}}.
20 |
--------------------------------------------------------------------------------
/src/lmq_queue_supersup.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_queue_supersup).
2 |
3 | -behaviour(supervisor).
4 | -export([start_link/0]).
5 | -export([init/1]).
6 |
7 | start_link() ->
8 | supervisor:start_link(?MODULE, []).
9 |
10 | init([]) ->
11 | MgrArgs = case application:get_env(stats_interval) of
12 | {ok, T} when is_integer(T), T > 0 ->
13 | [[{stats_interval, T}]];
14 | _ ->
15 | []
16 | end,
17 |
18 | QueueSup = {lmq_queue_sup,
19 | {lmq_queue_sup, start_link, []},
20 | permanent, infinity, supervisor, [lmq_queue_sup]},
21 |
22 | QueueMgr = {lmq_queue_mgr,
23 | {lmq_queue_mgr, start_link, MgrArgs},
24 | permanent, 5000, worker, [lmq_queue_mgr]},
25 |
26 | Children = [QueueSup, QueueMgr],
27 | {ok, {{one_for_all, 5, 10}, Children}}.
28 |
--------------------------------------------------------------------------------
/src/lmq_sup.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_sup).
2 |
3 | -behaviour(supervisor).
4 |
5 | %% API
6 | -export([start_link/0]).
7 |
8 | %% Supervisor callbacks
9 | -export([init/1]).
10 |
11 | %% Helper macro for declaring children of supervisor
12 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
13 | -define(CHILD(I, Type, Args), {I, {I, start_link, Args}, permanent, 5000, Type, [I]}).
14 |
15 | %% ===================================================================
16 | %% API functions
17 | %% ===================================================================
18 |
19 | start_link() ->
20 | supervisor:start_link({local, ?MODULE}, ?MODULE, []).
21 |
22 | %% ===================================================================
23 | %% Supervisor callbacks
24 | %% ===================================================================
25 |
26 | init([]) ->
27 | {ok, InfluxDB} = application:get_env(influxdb),
28 | Host = proplists:get_value(host, InfluxDB),
29 | Port = proplists:get_value(port, InfluxDB),
30 |
31 | {ok, {{one_for_all, 5, 10},
32 | [?CHILD(lmq_event, worker),
33 | ?CHILD(lmq_hook, worker),
34 | ?CHILD(lmq_queue_supersup, supervisor),
35 | ?CHILD(lmq_mpull_sup, supervisor),
36 | ?CHILD(influxdb_client, worker, [Host, Port])]}}.
37 |
--------------------------------------------------------------------------------
/test/lmq_SUITE.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_SUITE).
2 |
3 | -include("lmq.hrl").
4 | -include_lib("common_test/include/ct.hrl").
5 | -export([init_per_suite/1, end_per_suite/1,
6 | init_per_testcase/2, end_per_testcase/2, all/0]).
7 | -export([push/1, pull/1, update_props/1, properties/1, status/1, stats/1]).
8 |
9 | all() ->
10 | [push, pull, update_props, properties, status, stats].
11 |
12 | init_per_suite(Config) ->
13 | Priv = ?config(priv_dir, Config),
14 | application:start(mnesia),
15 | application:set_env(mnesia, dir, Priv),
16 | lmq:start(),
17 | Config.
18 |
19 | end_per_suite(_Config) ->
20 | lmq:stop(),
21 | mnesia:delete_schema([node()]).
22 |
23 | init_per_testcase(_, Config) ->
24 | [{qname, lmq_test} | Config].
25 |
26 | end_per_testcase(_, Config) ->
27 | Name = ?config(qname, Config),
28 | lmq_queue_mgr:delete(Name).
29 |
30 | push(Config) ->
31 | Name = ?config(qname, Config),
32 | lmq:push(atom_to_binary(Name, latin1), <<"test 1">>),
33 | lmq:push(atom_to_binary(Name, latin1),
34 | [{<<"content-type">>, <<"text/plain">>}], <<"test 2">>),
35 | [{queue, Name}, {id, _}, {type, normal}, {retry, 2},
36 | {content, {[], <<"test 1">>}}] = lmq:pull(Name, 0),
37 | [{queue, Name}, {id, _}, {type, normal}, {retry, 2},
38 | {content, {[{<<"content-type">>, <<"text/plain">>}], <<"test 2">>}}]
39 | = lmq:pull(Name, 0).
40 |
41 | pull(Config) ->
42 | Name = ?config(qname, Config),
43 | Parent = self(),
44 | spawn_link(fun() -> Parent ! lmq:pull(Name) end),
45 | timer:sleep(100),
46 | lmq:push(Name, <<"test_data">>),
47 | receive
48 | [{queue, Name}, {id, _}, {type, normal}, {retry, 2}, {content, {[], <<"test_data">>}}] -> ok
49 | after 100 ->
50 | ct:fail(no_response)
51 | end.
52 |
53 | update_props(Config) ->
54 | Name = ?config(qname, Config),
55 | true = is_pid(lmq:update_props(Name, [{retry, 1}, {timeout, 0}])),
56 | ok = lmq:push(Name, 1),
57 | [{queue, Name}, {id, _}, {type, normal}, {retry, 1}, {content, {[], 1}}] = lmq:pull(Name),
58 | [{queue, Name}, {id, _}, {type, normal}, {retry, 0}, {content, {[], 1}}] = lmq:pull(Name),
59 | empty = lmq:pull(Name, 0),
60 |
61 | %% change retry count
62 | true = is_pid(lmq:update_props(Name, [{retry, 0}])),
63 | ok = lmq:push(Name, 2),
64 | [{queue, Name}, {id, _}, {type, normal}, {retry, 0}, {content, {[], 2}}] = lmq:pull(Name),
65 | empty = lmq:pull(Name, 0).
66 |
67 | properties(_Config) ->
68 | N1 = lmq_properies_test_q1,
69 | N2 = lmq_properies_test_q2,
70 | N3 = lmq_properies_test_q3,
71 | P1 = lmq_misc:extend([{retry, 0}], ?DEFAULT_QUEUE_PROPS),
72 | P2 = lmq_misc:extend([{retry, 1}, {timeout, 0}], ?DEFAULT_QUEUE_PROPS),
73 | P3 = lmq_misc:extend([{retry, 0}, {timeout, 0}], ?DEFAULT_QUEUE_PROPS),
74 |
75 | ok = lmq:push(N1, 1),
76 | Q1 = lmq_queue_mgr:get(N1),
77 | Q2 = lmq:update_props(N2),
78 | Q3 = lmq:update_props(N3, [{retry, 0}]),
79 | ?DEFAULT_QUEUE_PROPS = lmq_queue:get_properties(Q1),
80 | ?DEFAULT_QUEUE_PROPS = lmq_queue:get_properties(Q2),
81 | P1 = lmq_queue:get_properties(Q3),
82 |
83 | lmq:set_default_props([{".*", [{retry, 1}, {timeout, 0}]}]),
84 | timer:sleep(50),
85 | P2 = lmq_queue:get_properties(Q1),
86 | P2 = lmq_queue:get_properties(Q2),
87 | P3 = lmq_queue:get_properties(Q3),
88 | lmq:set_default_props([]).
89 |
90 | status(Config) ->
91 | Name = ?config(qname, Config),
92 | Node = node(),
93 | Q = lmq_queue_mgr:get(Name, [create]),
94 | lmq_queue:push(Q, 1),
95 | Status = lmq:status(),
96 | [Node] = proplists:get_value(all_nodes, Status),
97 | [Node] = proplists:get_value(active_nodes, Status),
98 | QStatus = proplists:get_value(Name, proplists:get_value(queues, Status)),
99 | 1 = proplists:get_value(size, QStatus),
100 | true = proplists:get_value(memory, QStatus) > 0,
101 | [Node] = proplists:get_value(nodes, QStatus),
102 | ?DEFAULT_QUEUE_PROPS = proplists:get_value(props, QStatus).
103 |
104 | stats(Config) ->
105 | Name = ?config(qname, Config),
106 | Q = lmq_queue_mgr:get(Name, [create]),
107 | lmq_queue:push(Q, 1),
108 | Stats = proplists:get_value(Name, lmq:stats()),
109 | [push, pull, retention] = proplists:get_keys(Stats).
110 |
--------------------------------------------------------------------------------
/test/lmq_event_SUITE.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_event_SUITE).
2 |
3 | -include_lib("common_test/include/ct.hrl").
4 | -include("lmq.hrl").
5 | -include("lmq_test.hrl").
6 |
7 | -export([init_per_suite/1, end_per_suite/1,
8 | init_per_testcase/2, end_per_testcase/2,
9 | all/0]).
10 | -export([emit_new_message/1, handle_new_message/1, handle_local_queue_created/1,
11 | handle_remote_queue_created/1]).
12 |
13 | all() ->
14 | [emit_new_message, handle_new_message, handle_local_queue_created,
15 | handle_remote_queue_created].
16 |
17 | init_per_suite(Config) ->
18 | Priv = ?config(priv_dir, Config),
19 | application:start(mnesia),
20 | application:set_env(mnesia, dir, Priv),
21 | lmq:start(),
22 | Config.
23 |
24 | end_per_suite(_Config) ->
25 | lmq:stop(),
26 | mnesia:delete_schema([node()]).
27 |
28 | init_per_testcase(_, Config) ->
29 | lmq_event:add_handler(lmq_test_handler, self()),
30 | [{qname, lmq_event_test} | Config].
31 |
32 | end_per_testcase(_, Config) ->
33 | Name = ?config(qname, Config),
34 | lmq_queue_mgr:delete(Name).
35 |
36 | emit_new_message(Config) ->
37 | Name = ?config(qname, Config),
38 | Q = lmq_queue_mgr:get(Name, [create]),
39 | lmq_queue:push(Q, 1),
40 | ?EVENT_OR_FAIL({local, {new_message, Name}}).
41 |
42 | handle_new_message(Config) ->
43 | Name = ?config(qname, Config),
44 | send_remote_event({new_message, Name}),
45 | timer:sleep(50),
46 | not_found = lmq_queue_mgr:get(Name),
47 |
48 | Q = lmq_queue_mgr:get(Name, [create]),
49 | Parent = self(),
50 | spawn(fun() -> Parent ! {Q, lmq_queue:pull(Q)} end),
51 | timer:sleep(50),
52 | lmq_lib:enqueue(Name, 1),
53 | send_remote_event({new_message, Name}),
54 | receive {Q, M} when M#message.content =:= 1 -> ok
55 | after 50 -> ct:fail(no_response)
56 | end.
57 |
58 | handle_local_queue_created(_Config) ->
59 | lmq_queue_mgr:get('lmq/mpull/a', [create]),
60 | lmq_queue_mgr:get('lmq/mpull/b', [create]),
61 | {ok, Pid} = lmq_mpull:start(),
62 | Parent = self(), Ref = make_ref(),
63 | spawn(fun() -> Parent ! {Ref, lmq_mpull:pull(Pid, <<"lmq/mpull/.*">>, 100)} end),
64 | timer:sleep(10),
65 | Q3 = lmq_queue_mgr:get('lmq/mpull/c', [create]),
66 | lmq_queue:push(Q3, <<"push after pull">>),
67 |
68 | receive {Ref, [{queue, 'lmq/mpull/c'}, _, _, _,
69 | {content, <<"push after pull">>}]} -> ok
70 | after 50 -> ct:fail(no_response)
71 | end.
72 |
73 | handle_remote_queue_created(_Config) ->
74 | %% start the queue if not exists
75 | not_found = lmq_queue_mgr:get('lmq/remote/a'),
76 | lmq_lib:create('lmq/remote/a', [{retry, 100}]),
77 | send_remote_event({queue_created, 'lmq/remote/a'}),
78 | timer:sleep(10),
79 | Q1 = lmq_queue_mgr:get('lmq/remote/a'),
80 | 100 = proplists:get_value(retry, lmq_queue:get_properties(Q1)),
81 |
82 | %% update the queue if exists
83 | Q2 = lmq_queue_mgr:get('lmq/remote/b', [create]),
84 | 2 = proplists:get_value(retry, lmq_queue:get_properties(Q2)),
85 | lmq_lib:create('lmq/remote/b', [{retry, 100}]),
86 | send_remote_event({queue_created, 'lmq/remote/b'}),
87 | timer:sleep(10),
88 | 100 = proplists:get_value(retry, lmq_queue:get_properties(Q2)),
89 |
90 | %% ensure mpull also works
91 | lmq_lib:create('lmq/remote/c'),
92 | lmq_lib:enqueue('lmq/remote/c', 1),
93 | {ok, Pid} = lmq_mpull:start(),
94 | Parent = self(), Ref = make_ref(),
95 | spawn(fun() -> Parent ! {Ref, lmq_mpull:pull(Pid, <<"lmq/remote/.*">>, 100)} end),
96 | not_found = lmq_queue_mgr:get('lmq/remote/c'),
97 | send_remote_event({queue_created, 'lmq/remote/c'}),
98 | receive {Ref, [{queue, 'lmq/remote/c'}, _, _, _, {content, 1}]} -> ok
99 | after 50 -> ct:fail(no_response)
100 | end.
101 |
102 | send_remote_event(Event) ->
103 | gen_event:notify(?LMQ_EVENT, {remote, Event}).
104 |
--------------------------------------------------------------------------------
/test/lmq_hook_crash.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_hook_crash).
2 |
3 | -export([init/0, hooks/0, activate/1, deactivate/1]).
4 | -export([hook1/2]).
5 |
6 | init() ->
7 | ok.
8 |
9 | hooks() ->
10 | [hook1, hook2].
11 |
12 | activate([N]) ->
13 | {ok, N}.
14 |
15 | deactivate(N) ->
16 | N / 0.
17 |
18 | hook1(N, State) ->
19 | State / N.
20 |
--------------------------------------------------------------------------------
/test/lmq_hook_invalid.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_hook_invalid).
2 |
3 | -export([init/1, hooks/0, activate/1, deactivate/1]).
4 | -export([custom_hook/2]).
5 |
6 | init([]) ->
7 | ok.
8 |
9 | hooks() ->
10 | [custom_hook].
11 |
12 | activate([N]) ->
13 | {ok, N}.
14 |
15 | deactivate(N) ->
16 | true = is_integer(N),
17 | ok.
18 |
19 | custom_hook(N, State) ->
20 | N + State.
21 |
--------------------------------------------------------------------------------
/test/lmq_hook_sample1.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_hook_sample1).
2 | -behaviour(lmq_hook).
3 |
4 | -export([init/0, hooks/0, activate/1, deactivate/1]).
5 | -export([custom_hook/2]).
6 |
7 | init() ->
8 | ok.
9 |
10 | hooks() ->
11 | [custom_hook].
12 |
13 | activate([N]) ->
14 | {ok, N}.
15 |
16 | deactivate(N) ->
17 | true = is_integer(N),
18 | ok.
19 |
20 | custom_hook(N, State) ->
21 | N + State.
22 |
--------------------------------------------------------------------------------
/test/lmq_hook_sample2.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_hook_sample2).
2 | -behaviour(lmq_hook).
3 |
4 | -export([init/0, hooks/0, activate/1, deactivate/1]).
5 | -export([custom_hook/2, custom_hook2/2]).
6 |
7 | init() ->
8 | ok.
9 |
10 | hooks() ->
11 | [custom_hook, custom_hook2].
12 |
13 | activate([N, M]) ->
14 | {ok, N * M}.
15 |
16 | deactivate(N) ->
17 | true = is_integer(N),
18 | ok.
19 |
20 | custom_hook(N, State) ->
21 | N * State.
22 |
23 | custom_hook2(N, State) ->
24 | custom_hook(N, State).
25 |
--------------------------------------------------------------------------------
/test/lmq_hook_test.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_hook_test).
2 | -include_lib("eunit/include/eunit.hrl").
3 |
4 | -define(setup(F), {setup, fun start/0, fun stop/1, F}).
5 |
6 | %% ==================================================================
7 | %% test descriptions
8 | %% ==================================================================
9 | start_stop_test_() ->
10 | {"The hook manager can be started, stopped and has a registered name",
11 | ?setup(fun is_registered/1)}.
12 |
13 | register_test_() ->
14 | [{"A hook can be registered and be called",
15 | ?setup(fun register_hook/1)},
16 | {"Multiple hooks can be registered and be called sequentially",
17 | ?setup(fun register_multiple_hooks/1)},
18 | {"Hooks can be updated",
19 | ?setup(fun update_hooks/1)},
20 | {"Hooks cannot be affected other queues",
21 | ?setup(fun other_queue/1)},
22 | {"Hooks can be configured separately per queue",
23 | ?setup(fun configured_per_queue/1)},
24 | {"Registering never crashes and simply aborts",
25 | ?setup(fun abort_registering/1)},
26 | {"Unregistering never crashes",
27 | ?setup(fun unregistering_never_crash/1)},
28 | {"Hooks not included in new config are unregistered",
29 | ?setup(fun automatic_unregister/1)}
30 | ].
31 |
32 | call_test_() ->
33 | [{"Calling never crashes",
34 | ?setup(fun call_never_crash/1)}].
35 |
36 | %% ==================================================================
37 | %% setup functions
38 | %% ==================================================================
39 | start() ->
40 | {ok, Pid} = lmq_hook:start_link(),
41 | Pid.
42 |
43 | stop(_) ->
44 | lmq_hook:stop().
45 |
46 | %% ==================================================================
47 | %% actual tests
48 | %% ==================================================================
49 | is_registered(Pid) ->
50 | [?_assert(erlang:is_process_alive(Pid)),
51 | ?_assertEqual(Pid, whereis(lmq_hook))].
52 |
53 | register_hook(_) ->
54 | Config = [{custom_hook, [{lmq_hook_sample1, [1]}]}],
55 | Res = lmq_hook:register(hook_test, Config),
56 | [?_assertEqual(ok, Res),
57 | ?_assertEqual(3, lmq_hook:call(hook_test, custom_hook, 2))].
58 |
59 | register_multiple_hooks(_) ->
60 | Config = [{custom_hook, [{lmq_hook_sample1, [1]}, {lmq_hook_sample2, [1, 2]}]}],
61 | Res = lmq_hook:register(hook_test, Config),
62 | [?_assertEqual(ok, Res),
63 | ?_assertEqual(6, lmq_hook:call(hook_test, custom_hook, 2))].
64 |
65 | update_hooks(_) ->
66 | Config1 = [{custom_hook, [{lmq_hook_sample1, [1]}]}],
67 | Config2 = [{custom_hook, [{lmq_hook_sample1, [1]}, {lmq_hook_sample2, [1, 2]}]}],
68 | Config3 = [{custom_hook, [{lmq_hook_sample2, [1, 2]}, {lmq_hook_sample1, [1]}]}],
69 | Config4 = [{custom_hook, [{lmq_hook_sample2, [1, 2]}]}],
70 | Res = [lmq_hook:register(hook_test, Config1),
71 | lmq_hook:call(hook_test, custom_hook, 2),
72 | lmq_hook:register(hook_test, Config2),
73 | lmq_hook:call(hook_test, custom_hook, 2),
74 | lmq_hook:register(hook_test, Config3),
75 | lmq_hook:call(hook_test, custom_hook, 2),
76 | lmq_hook:register(hook_test, Config4),
77 | lmq_hook:call(hook_test, custom_hook, 2)],
78 | [?_assertEqual([ok, 3, ok, 6, ok, 5, ok, 4], Res)].
79 |
80 | other_queue(_) ->
81 | Config = [{custom_hook, [{lmq_hook_sample1, [1]}]}],
82 | ok = lmq_hook:register(hook_test, Config),
83 | [?_assertEqual(2, lmq_hook:call(not_hooked, custom_hook, 2))].
84 |
85 | configured_per_queue(_) ->
86 | Config1 = [{custom_hook, [{lmq_hook_sample1, [1]}]}],
87 | Config2 = [{custom_hook, [{lmq_hook_sample1, [2]}]}],
88 | Config3 = [{custom_hook, [{lmq_hook_sample2, [2, 3]}]}],
89 | Res = [lmq_hook:register(hook_test1, Config1),
90 | lmq_hook:register(hook_test2, Config2),
91 | lmq_hook:register(hook_test3, Config3)],
92 | [?_assertEqual([ok, ok, ok], Res),
93 | ?_assertEqual(2, lmq_hook:call(hook_test1, custom_hook, 1)),
94 | ?_assertEqual(3, lmq_hook:call(hook_test2, custom_hook, 1)),
95 | ?_assertEqual(6, lmq_hook:call(hook_test3, custom_hook, 1))].
96 |
97 | abort_registering(_) ->
98 | Config1 = [{custom_hook, [{lmq_hook_sample1, [1]}]}],
99 | Config2 = [{custom_hook, [{lmq_hook_sample2, [1]}]}],
100 | Config3 = [{custom_hook, [{lmq_hook_no_exist, [1]}]}],
101 | Config4 = [{custom_hook, [{lmq_hook_invalid, [1]}]}],
102 | Config5 = [{invalid_hook, [{lmq_hook_sample1, [1]}]}],
103 | Res = [lmq_hook:register(hook_test, Config1),
104 | lmq_hook:call(hook_test, custom_hook, 2),
105 | lmq_hook:register(hook_test, Config2),
106 | lmq_hook:call(hook_test, custom_hook, 2),
107 | lmq_hook:register(hook_test, Config3),
108 | lmq_hook:call(hook_test, custom_hook, 2),
109 | lmq_hook:register(hook_test, Config4),
110 | lmq_hook:call(hook_test, custom_hook, 2),
111 | lmq_hook:register(hook_test, Config5),
112 | lmq_hook:call(hook_test, custom_hook, 2),
113 | lmq_hook:call(hook_test, invalid_hook, 2)],
114 | [?_assertEqual([ok, 3,
115 | {error, bad_config}, 3,
116 | {error, bad_hook}, 3,
117 | {error, bad_hook}, 3,
118 | {error, bad_config}, 3, 2], Res)].
119 |
120 | unregistering_never_crash(_) ->
121 | Config1 = [{hook1, [{lmq_hook_crash, [2]}]}],
122 | Config2 = [{hook1, []}],
123 | Res = [lmq_hook:register(hook_test, Config1),
124 | lmq_hook:register(hook_test, Config2)],
125 | [?_assertEqual([ok, ok], Res)].
126 |
127 | automatic_unregister(_) ->
128 | Config1 = [{custom_hook, [{lmq_hook_sample1, [1]}]}],
129 | Config2 = [{custom_hook2, [{lmq_hook_sample2, [1, 2]}]}],
130 | Res = [lmq_hook:register(hook_test, Config1),
131 | lmq_hook:call(hook_test, custom_hook, 2),
132 | lmq_hook:call(hook_test, custom_hook2, 2),
133 | lmq_hook:register(hook_test, Config2),
134 | lmq_hook:call(hook_test, custom_hook, 2),
135 | lmq_hook:call(hook_test, custom_hook2, 2)],
136 | [?_assertEqual([ok, 3, 2, ok, 2, 4], Res)].
137 |
138 | call_never_crash(_) ->
139 | Config = [{hook1, [{lmq_hook_crash, [2]}]}],
140 | ok = lmq_hook:register(hook_test, Config),
141 | [?_assertEqual(1.0, lmq_hook:call(hook_test, hook1, 2)),
142 | ?_assertEqual(0, lmq_hook:call(hook_test, hook1, 0)),
143 | ?_assertEqual(1, lmq_hook:call(hook_test, hook2, 1))].
144 |
--------------------------------------------------------------------------------
/test/lmq_lib_SUITE.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_lib_SUITE).
2 |
3 | -include("lmq.hrl").
4 | -include("lmq_test.hrl").
5 | -include_lib("common_test/include/ct.hrl").
6 | -export([init_per_suite/1, end_per_suite/1, init_per_testcase/2, end_per_testcase/2,
7 | all/0]).
8 | -export([lmq_info/1, create_delete/1, queue_names/1, done/1, release/1, retain/1, waittime/1,
9 | limit_retry/1, error_case/1, accumlate/1, property/1, many_waste_messages/1]).
10 | -export([enqueue_many/2]).
11 |
12 | all() ->
13 | [lmq_info, create_delete, queue_names, done, release, retain, waittime, limit_retry,
14 | error_case, accumlate, property, many_waste_messages].
15 |
16 | init_per_suite(Config) ->
17 | Priv = ?config(priv_dir, Config),
18 | application:start(mnesia),
19 | application:set_env(mnesia, dir, Priv),
20 | application:start(folsom),
21 | ok = lmq_lib:init_mnesia(),
22 | Config.
23 |
24 | end_per_suite(_Config) ->
25 | application:stop(mnesia),
26 | mnesia:delete_schema([node()]),
27 | ok.
28 |
29 | init_per_testcase(create_delete, Config) ->
30 | lmq_event:start_link(),
31 | lmq_event:add_handler(lmq_test_handler, self()),
32 | Config;
33 |
34 | init_per_testcase(_, Config) ->
35 | lmq_event:start_link(),
36 | lmq_event:add_handler(lmq_test_handler, self()),
37 | Name = test,
38 | lmq_lib:create(Name),
39 | [{qname, Name} | Config].
40 |
41 | end_per_testcase(_, Config) ->
42 | ok = lmq_lib:delete(?config(qname, Config)).
43 |
44 | lmq_info(_Config) ->
45 | ok = lmq_lib:set_lmq_info(name, lmq),
46 | {ok, lmq} = lmq_lib:get_lmq_info(name),
47 | {error, not_found} = lmq_lib:get_lmq_info(non_exists),
48 | {ok, []} = lmq_lib:get_lmq_info(non_exists, []).
49 |
50 | create_delete(_Config) ->
51 | %% create new queue
52 | ok = lmq_lib:create(test),
53 | message = mnesia:table_info(test, record_name),
54 | [] = lmq_lib:queue_info(test),
55 | ?EVENT_OR_FAIL({local, {queue_created, test}}),
56 |
57 | %% update properties
58 | ok = lmq_lib:create(test, [{timeout, 10}]),
59 | [{timeout, 10}] = lmq_lib:queue_info(test),
60 | ?EVENT_OR_FAIL({local, {queue_created, test}}),
61 |
62 | %% no event occurred when properties are not changed
63 | ok = lmq_lib:create(test, [{timeout, 10}]),
64 | [{timeout, 10}] = lmq_lib:queue_info(test),
65 | ?EVENT_AND_FAIL({local, {queue_created, test}}),
66 |
67 | %% delete
68 | ok = lmq_lib:delete(test),
69 | {aborted, {no_exists, _}} = mnesia:delete_table(test),
70 | not_found = lmq_lib:queue_info(test),
71 | ok = lmq_lib:delete(test).
72 |
73 | queue_names(Config) ->
74 | Name = fooooo,
75 | lmq_lib:create(Name),
76 | true = lists:sort([?config(qname, Config), Name]) =:=
77 | lists:sort(lmq_lib:all_queue_names()),
78 | lmq_lib:delete(Name).
79 |
80 | done(Config) ->
81 | Name = ?config(qname, Config),
82 | ok = lmq_lib:enqueue(Name, make_ref()),
83 | Now = lmq_misc:unixtime(),
84 | M = lmq_lib:dequeue(Name, 10),
85 | {TS, UUID} = M#message.id,
86 | true = Now < TS andalso TS - Now - 10 < 1,
87 | ok = lmq_lib:done(Name, UUID),
88 | not_found = lmq_lib:done(Name, UUID).
89 |
90 | release(Config) ->
91 | Name = ?config(qname, Config),
92 | ok = lmq_lib:enqueue(Name, make_ref(), [{retry, 2}]),
93 | M = lmq_lib:dequeue(Name, 30),
94 | 2 = M#message.retry,
95 | {_, UUID} = M#message.id,
96 | ok = lmq_lib:release(Name, UUID),
97 | not_found = lmq_lib:release(Name, UUID),
98 |
99 | M1 = lmq_lib:dequeue(Name, 30),
100 | 1 = M1#message.retry,
101 | ok = lmq_lib:put_back(Name, element(2, M1#message.id)),
102 | 1 = (lmq_lib:dequeue(Name, 30))#message.retry,
103 |
104 | ok = lmq_lib:enqueue(Name, make_ref()),
105 | M2 = lmq_lib:dequeue(Name, 30),
106 | infinity = M2#message.retry,
107 | ok = lmq_lib:release(Name, element(2, M2#message.id)),
108 | infinity = (lmq_lib:dequeue(Name, 30))#message.retry.
109 |
110 | retain(Config) ->
111 | Name = ?config(qname, Config),
112 | ok = lmq_lib:enqueue(Name, make_ref()),
113 | Now = lmq_misc:unixtime(),
114 | M = lmq_lib:dequeue(Name, 15),
115 | {TS, UUID} = M#message.id,
116 | true = Now < TS andalso TS - Now - 15 < 1,
117 | ok = lmq_lib:retain(Name, UUID, 30),
118 | not_found = lmq_lib:retain(Name, "AAA", 30).
119 |
120 | waittime(Config) ->
121 | Name = ?config(qname, Config),
122 | empty = lmq_lib:first(Name),
123 | infinity = lmq_lib:waittime(Name),
124 | ok = lmq_lib:enqueue(Name, make_ref()),
125 | 0 = lmq_lib:waittime(Name),
126 | lmq_lib:dequeue(Name, 30),
127 | true = 29500 < lmq_lib:waittime(Name).
128 |
129 | limit_retry(Config) ->
130 | Name = ?config(qname, Config),
131 | Timeout = 0,
132 | Ref = make_ref(),
133 | %% retry 5 times, that means dequeue succeed 6 times
134 | ok = lmq_lib:enqueue(Name, Ref, [{retry, 5}]),
135 | lists:all(fun(M) -> Ref =:= M#message.content end,
136 | [lmq_lib:dequeue(Name, Timeout) || _ <- lists:seq(1, 6)]),
137 | empty = lmq_lib:dequeue(Name, Timeout),
138 | %% retry infinity
139 | ok = lmq_lib:enqueue(Name, Ref),
140 | lists:all(fun(M) -> Ref =:= M#message.content end,
141 | [lmq_lib:dequeue(Name, Timeout) || _ <- lists:seq(1, 10)]).
142 |
143 | error_case(_Config) ->
144 | Name = '__abcdefg__',
145 | {error, no_queue_exists} = lmq_lib:enqueue(Name, make_ref()),
146 | {error, no_queue_exists} = lmq_lib:dequeue(Name, 30),
147 | {error, no_queue_exists} = lmq_lib:done(Name, "AAA"),
148 | {error, no_queue_exists} = lmq_lib:release(Name, "AAA"),
149 | {error, no_queue_exists} = lmq_lib:retain(Name, "AAA", 30),
150 | {error, no_queue_exists} = lmq_lib:waittime(Name).
151 |
152 | accumlate(Config) ->
153 | Name = ?config(qname, Config),
154 | Timeout = 30,
155 | R1 = make_ref(), R2 = make_ref(), R3 = make_ref(),
156 | %% 0 means not accumulate
157 | ok = lmq_lib:enqueue(Name, R1, [{accum, 0}]),
158 | R1 = (lmq_lib:dequeue(Name, Timeout))#message.content,
159 |
160 | %% get 2 compound messages, each compound message contains 1 message
161 | {accum, new} = lmq_lib:enqueue(Name, R1, [{accum, 1}]), timer:sleep(1),
162 | {accum, new} = lmq_lib:enqueue(Name, R2, [{accum, 1}]), timer:sleep(1),
163 | [R1] = (lmq_lib:dequeue(Name, Timeout))#message.content,
164 | [R2] = (lmq_lib:dequeue(Name, Timeout))#message.content,
165 |
166 | %% accumulate and no accumulate message
167 | {accum, new} = lmq_lib:enqueue(Name, R1, [{accum, 100}]),
168 | {accum, yes} = lmq_lib:enqueue(Name, R2, [{accum, 100}]),
169 | ok = lmq_lib:enqueue(Name, R3),
170 | R3 = (lmq_lib:dequeue(Name, Timeout))#message.content,
171 | empty = lmq_lib:dequeue(Name, Timeout),
172 | timer:sleep(100),
173 | [R1, R2] = (lmq_lib:dequeue(Name, Timeout))#message.content.
174 |
175 | property(_Config) ->
176 | N1 = property_default,
177 | N2 = property_override1,
178 | N3 = property_override2,
179 | P1 = ?DEFAULT_QUEUE_PROPS,
180 | P2 = lmq_misc:extend([{retry, infinity}, {timeout, 0}], ?DEFAULT_QUEUE_PROPS),
181 | P3 = lmq_misc:extend([{retry, 0}, {timeout, 0}], ?DEFAULT_QUEUE_PROPS),
182 |
183 | lmq_lib:create(N1),
184 | lmq_lib:create(N2),
185 | lmq_lib:create(N3, [{retry, 0}]),
186 | lmq_lib:set_lmq_info(default_props,
187 | [{"override", [{retry, infinity}, {timeout, 0}]}]),
188 |
189 | P1 = lmq_lib:get_properties(N1),
190 | P2 = lmq_lib:get_properties(N2),
191 | P3 = lmq_lib:get_properties(N3),
192 | P3 = lmq_lib:get_properties(N2, [{retry, 0}]),
193 |
194 | P4 = lmq_misc:extend([{accum, 1}], ?DEFAULT_QUEUE_PROPS),
195 | P5 = lmq_misc:extend([{accum, 1}, {retry, 0}], ?DEFAULT_QUEUE_PROPS),
196 | lmq_lib:set_lmq_info(default_props, [{"override", [{accum, 1}]}]),
197 | P1 = lmq_lib:get_properties(N1),
198 | P4 = lmq_lib:get_properties(N2),
199 | P5 = lmq_lib:get_properties(N3).
200 |
201 | many_waste_messages(Config) ->
202 | Name = ?config(qname, Config),
203 | rpc:pmap({?MODULE, enqueue_many}, [Name], lists:seq(1, 10)),
204 | ct:timetrap(10000),
205 | empty = lmq_lib:dequeue(Name, 0).
206 |
207 | enqueue_many(I, Name) ->
208 | [lmq_lib:enqueue(Name, I*N, [{retry, -1}]) || N <- lists:seq(1, 1000)].
209 |
--------------------------------------------------------------------------------
/test/lmq_mpull_SUITE.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_mpull_SUITE).
2 |
3 | -include("lmq.hrl").
4 | -include_lib("common_test/include/ct.hrl").
5 | -export([init_per_suite/1, end_per_suite/1,
6 | init_per_testcase/2, end_per_testcase/2, all/0]).
7 | -export([pull/1, pull_async/1, client_closed/1]).
8 |
9 | all() ->
10 | [pull, pull_async, client_closed].
11 |
12 | init_per_suite(Config) ->
13 | Priv = ?config(priv_dir, Config),
14 | application:start(mnesia),
15 | application:set_env(mnesia, dir, Priv),
16 | lmq:start(),
17 | Config.
18 |
19 | end_per_suite(_Config) ->
20 | lmq:stop(),
21 | mnesia:delete_schema([node()]).
22 |
23 | init_per_testcase(_, Config) ->
24 | Config.
25 |
26 | end_per_testcase(_, _Config) ->
27 | Queues = ['mpull/a', 'mpull/b'],
28 | [lmq:delete(Q) || Q <- Queues],
29 | ok.
30 |
31 | pull(_Config) ->
32 | Queues = ['mpull/a', 'mpull/b'],
33 | [lmq:update_props(Q, [{retry, 0}]) || Q <- Queues],
34 | [lmq:push(Q, Q) || Q <- Queues],
35 |
36 | Pids = [Pid || {ok, Pid} <- [lmq_mpull:start() || _ <- lists:seq(1, 4)]],
37 | [{queue, R1}, _, _, _, _] = lmq_mpull:pull(lists:nth(1, Pids), <<"mpull/.*">>),
38 | [{queue, R2}, _, _, _, _] = lmq_mpull:pull(lists:nth(2, Pids), <<"mpull/.*">>, 0),
39 | true = lists:sort([R1, R2]) =:= Queues,
40 |
41 | empty = lmq_mpull:pull(lists:nth(3, Pids), <<"mpull/.*">>, 0),
42 | empty = lmq_mpull:pull(lists:nth(4, Pids), <<"mpull/.*">>, 200).
43 |
44 | pull_async(_Config) ->
45 | lmq:push('mpull/a', 1),
46 | {ok, MP1} = lmq_mpull:start(),
47 | {ok, Ref1} = lmq_mpull:pull_async(MP1, <<"mpull/a">>),
48 | receive {Ref1, [{queue, 'mpull/a'}, _, _, _, {content, {[], 1}}]} -> ok
49 | after 50 -> ct:fail(no_response)
50 | end,
51 |
52 | {ok, MP2} = lmq_mpull:start(),
53 | {ok, _} = lmq_mpull:pull_async(MP2, <<"mpull/a">>),
54 | ok = lmq_mpull:pull_cancel(MP2),
55 |
56 | lmq:push('mpull/a', 3),
57 | {ok, MP3} = lmq_mpull:start(),
58 | {ok, Ref3} = lmq_mpull:pull_async(MP3, <<"mpull/a">>),
59 | receive {Ref3, [{queue, 'mpull/a'}, _, _, _, {content, {[], 3}}]} -> ok
60 | after 50 -> ct:fail(no_response)
61 | end.
62 |
63 | client_closed(_Config) ->
64 | lmq:push('mpull/a', 1),
65 | lmq:pull('mpull/a', 0),
66 | Pid = spawn(fun() ->
67 | {ok, MP1} = lmq_mpull:start(),
68 | lmq_mpull:pull(MP1, <<"mpull/.*">>, 1000)
69 | end),
70 | timer:sleep(10),
71 | exit(Pid, kill),
72 | timer:sleep(10),
73 |
74 | lmq:push('mpull/a', 2),
75 | {ok, MP2} = lmq_mpull:start(),
76 | [{queue, 'mpull/a'}, _, _, _, {content, {[], 2}}] = lmq_mpull:pull(MP2, <<"mpull/.*">>, 1000).
77 |
--------------------------------------------------------------------------------
/test/lmq_msgpack_SUITE.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_msgpack_SUITE).
2 |
3 | -include_lib("common_test/include/ct.hrl").
4 | -export([init_per_suite/1, end_per_suite/1,
5 | init_per_testcase/2, end_per_testcase/2, all/0]).
6 | -export([push_pull_done/1, release/1, props_and_timeout/1, packed_queue/1,
7 | push_all/1, pull_any/1, default_props/1]).
8 |
9 | -define(RPC(Client, Method, Args), msgpack_rpc_client:call(Client, Method, Args)).
10 |
11 | all() ->
12 | [push_pull_done, release, props_and_timeout, packed_queue, push_all, pull_any,
13 | default_props].
14 |
15 | init_per_suite(Config) ->
16 | Priv = ?config(priv_dir, Config),
17 | application:start(mnesia),
18 | application:set_env(mnesia, dir, Priv),
19 | lmq:start(),
20 | Config.
21 |
22 | end_per_suite(_Config) ->
23 | lmq:stop(),
24 | mnesia:delete_schema([node()]).
25 |
26 | init_per_testcase(_, Config) ->
27 | {ok, Pid} = msgpack_rpc_client:connect(tcp, "localhost", 18800, []),
28 | [{client, Pid}, {qname, <<"msgpack_test">>} | Config].
29 |
30 | end_per_testcase(_, Config) ->
31 | Client = ?config(client, Config),
32 | Name = ?config(qname, Config),
33 | msgpack_rpc_client:close(Client),
34 | lmq_queue_mgr:delete(binary_to_atom(Name, latin1)).
35 |
36 | push_pull_done(Config) ->
37 | Client = ?config(client, Config),
38 | Name = ?config(qname, Config),
39 | Content = <<"test data">>,
40 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, push, [Name, Content]),
41 | {ok, Res} = msgpack_rpc_client:call(Client, pull, [Name]),
42 | {[{<<"queue">>, Name}, {<<"id">>, UUID}, {<<"type">>, <<"normal">>}, {<<"retry">>, 2}, {<<"content">>, Content}]} = Res,
43 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, retain, [Name, UUID]),
44 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, done, [Name, UUID]).
45 |
46 | release(Config) ->
47 | Client = ?config(client, Config),
48 | Name = ?config(qname, Config),
49 | Content = <<"test data 2">>,
50 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, push, [Name, Content]),
51 | {ok, Res} = msgpack_rpc_client:call(Client, pull, [Name]),
52 | {[{<<"queue">>, Name}, {<<"id">>, UUID}, {<<"type">>, <<"normal">>}, {<<"retry">>, 2}, {<<"content">>, Content}]} = Res,
53 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, release, [Name, UUID]),
54 | {error, _} = msgpack_rpc_client:call(Client, done, [Name, UUID]),
55 | {ok, Res1} = msgpack_rpc_client:call(Client, pull, [Name]),
56 | {[{<<"queue">>, Name}, {<<"id">>, UUID1}, {<<"type">>, <<"normal">>}, {<<"retry">>, 1}, {<<"content">>, Content}]} = Res1,
57 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, done, [Name, UUID1]).
58 |
59 | props_and_timeout(Config) ->
60 | Client = ?config(client, Config),
61 | Name = ?config(qname, Config),
62 | Props = {[{<<"retry">>, 1}, {<<"timeout">>, 0}]},
63 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, update_props, [Name, Props]),
64 | {error, _} = msgpack_rpc_client:call(Client, update_props, [Name, {[{<<"pack">>, <<"10">>}]}]),
65 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, push, [Name, <<"test">>]),
66 | {ok, Res} = msgpack_rpc_client:call(Client, pull, [Name, 0.2]),
67 | {[{<<"queue">>, Name}, {<<"id">>, _UUID}, {<<"type">>, <<"normal">>}, {<<"retry">>, 1}, {<<"content">>, <<"test">>}]} = Res,
68 | {ok, {[_, _, _, {<<"retry">>, 0}, {<<"content">>, <<"test">>}]}} = ?RPC(Client, pull, [Name, 0]),
69 | {ok, <<"empty">>} = ?RPC(Client, pull, [Name, 0.1]),
70 |
71 | {ok, <<"ok">>} = ?RPC(Client, update_props, [Name]),
72 | timer:sleep(10),
73 | {ok, <<"ok">>} = ?RPC(Client, push, [Name, <<"test2">>]),
74 | {ok, {[_, _, _, {<<"retry">>, 2}, _]}} = ?RPC(Client, pull, [Name, 0]),
75 | {ok, <<"empty">>} = ?RPC(Client, pull, [Name, 0]).
76 |
77 | packed_queue(Config) ->
78 | Client = ?config(client, Config),
79 | Name = ?config(qname, Config),
80 | Props = {[{<<"timeout">>, 0.4}, {<<"pack">>, 0.2}]},
81 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, update_props, [Name, Props]),
82 | {ok, <<"packing started">>} = msgpack_rpc_client:call(Client, push, [Name, 1]),
83 | {ok, <<"packed">>} = msgpack_rpc_client:call(Client, push, [Name, 2]),
84 | {ok, <<"empty">>} = msgpack_rpc_client:call(Client, pull, [Name, 0]),
85 | timer:sleep(200),
86 | {ok, {[{<<"queue">>, Name}, {<<"id">>, _}, {<<"type">>, <<"package">>}, {<<"retry">>, 2}, {<<"content">>, [1, 2]}]}} =
87 | msgpack_rpc_client:call(Client, pull, [Name, 0]),
88 | {ok, <<"packing started">>} = msgpack_rpc_client:call(Client, push, [Name, 3]),
89 | timer:sleep(400),
90 | {ok, {[{<<"queue">>, Name}, {<<"id">>, _}, {<<"type">>, <<"package">>}, {<<"retry">>, 2}, {<<"content">>, [3]}]}} =
91 | msgpack_rpc_client:call(Client, pull, [Name, 0]),
92 | {ok, {[{<<"queue">>, Name}, {<<"id">>, _}, {<<"type">>, <<"package">>}, {<<"retry">>, 1}, {<<"content">>, [1, 2]}]}} =
93 | msgpack_rpc_client:call(Client, pull, [Name, 0]).
94 |
95 | push_all(Config) ->
96 | Client = ?config(client, Config),
97 | Names = [<<"lmq/foo">>, <<"lmq/bar">>],
98 | [msgpack_rpc_client:call(Client, update_props, [Name]) || Name <- Names],
99 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, push_all, [<<"lmq/.*">>, <<"data">>]),
100 | {ok, {[{<<"queue">>, <<"lmq/foo">>}, {<<"id">>, _}, {<<"type">>, <<"normal">>}, {<<"retry">>, 2}, {<<"content">>, <<"data">>}]}} =
101 | msgpack_rpc_client:call(Client, pull, [<<"lmq/foo">>, 0]),
102 | {ok, {[{<<"queue">>, <<"lmq/bar">>}, {<<"id">>, _}, {<<"type">>, <<"normal">>}, {<<"retry">>, 2}, {<<"content">>, <<"data">>}]}} =
103 | msgpack_rpc_client:call(Client, pull, [<<"lmq/bar">>, 0]),
104 | [msgpack_rpc_client:call(Client, delete, [N]) || N <- Names].
105 |
106 | pull_any(Config) ->
107 | Client = ?config(client, Config),
108 | Names = [<<"lmq/foo">>, <<"lmq/bar">>],
109 | Props = {[{<<"retry">>, 0}]},
110 | [msgpack_rpc_client:call(Client, update_props, [Name, Props]) || Name <- Names],
111 | [msgpack_rpc_client:call(Client, push, [Name, Name]) || Name <- Names],
112 | {ok, {[{<<"queue">>, R1}, {<<"id">>, _}, {<<"type">>, <<"normal">>}, {<<"retry">>, 0}, {<<"content">>, R1}]}} =
113 | msgpack_rpc_client:call(Client, pull_any, [<<"lmq/.*">>, 0]),
114 | {ok, {[{<<"queue">>, R2}, {<<"id">>, _}, {<<"type">>, <<"normal">>}, {<<"retry">>, 0}, {<<"content">>, R2}]}} =
115 | msgpack_rpc_client:call(Client, pull_any, [<<"lmq/.*">>, 0]),
116 | true = lists:sort([R1, R2]) =:= lists:sort(Names),
117 | {ok, <<"empty">>} =
118 | msgpack_rpc_client:call(Client, pull_any, [<<"lmq/.*">>, 0]),
119 | {ok, <<"empty">>} =
120 | msgpack_rpc_client:call(Client, pull_any, [<<"lmq/.*">>, 0.2]),
121 | [msgpack_rpc_client:call(Client, delete, [N]) || N <- Names].
122 |
123 | default_props(Config) ->
124 | Client = ?config(client, Config),
125 | DefaultProps = [[<<"def/">>, {[{<<"retry">>, 0}, {<<"timeout">>, 0}]}],
126 | [<<"lmq/">>, {[{<<"timeout">>, 0}]}]],
127 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, set_default_props, [DefaultProps]),
128 | {error, _} = msgpack_rpc_client:call(Client, set_default_props,
129 | lists:nth(1, DefaultProps)),
130 | {ok, DefaultProps} = msgpack_rpc_client:call(Client, get_default_props, []),
131 | Name = <<"def/a">>,
132 | {ok, <<"ok">>} = msgpack_rpc_client:call(Client, push, [Name, 1]),
133 | {ok, {[{<<"queue">>, Name}, {<<"id">>, _}, {<<"type">>, <<"normal">>}, {<<"retry">>, 0}, {<<"content">>, 1}]}} =
134 | msgpack_rpc_client:call(Client, pull, [Name, 0]),
135 | {ok, <<"empty">>} = msgpack_rpc_client:call(Client, pull, [Name, 0]).
136 |
--------------------------------------------------------------------------------
/test/lmq_queue_SUITE.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_queue_SUITE).
2 |
3 | -include("lmq.hrl").
4 | -include_lib("common_test/include/ct.hrl").
5 | -export([init_per_suite/1, end_per_suite/1,
6 | init_per_testcase/2, end_per_testcase/2,
7 | all/0, groups/0]).
8 | -export([init/1, push_pull_done/1, release/1, release_multi/1, multi_queue/1,
9 | pull_timeout/1, async_request/1, pull_async/1, pull_and_timeout/1,
10 | update_properties/1, reload_properties/1]).
11 |
12 | all() ->
13 | [init, push_pull_done, release, release_multi, multi_queue, pull_timeout,
14 | async_request, pull_async, update_properties, reload_properties, {group, timing}].
15 |
16 | groups() ->
17 | [{timing, [{repeat_until_any_fail, 10}], [pull_and_timeout]}].
18 |
19 | init_per_suite(Config) ->
20 | Priv = ?config(priv_dir, Config),
21 | application:start(mnesia),
22 | application:set_env(mnesia, dir, Priv),
23 | application:start(lager),
24 | application:start(folsom),
25 | ok = lmq_lib:init_mnesia(),
26 | Config.
27 |
28 | end_per_suite(_Config) ->
29 | application:stop(mnesia),
30 | mnesia:delete_schema([node()]).
31 |
32 | init_per_testcase(init, Config) ->
33 | {ok, _} = lmq_event:start_link(),
34 | {ok, _} = lmq_hook:start_link(),
35 | Config;
36 |
37 | init_per_testcase(_, Config) ->
38 | {ok, _} = lmq_event:start_link(),
39 | {ok, _} = lmq_hook:start_link(),
40 | {ok, Pid} = lmq_queue:start_link(message),
41 | [{queue, Pid} | Config].
42 |
43 | end_per_testcase(init, _Config) ->
44 | lmq_hook:stop();
45 | end_per_testcase(_, Config) ->
46 | lmq_queue:stop(?config(queue, Config)),
47 | lmq_hook:stop(),
48 | lmq_lib:delete(message).
49 |
50 | init(_Config) ->
51 | not_found = lmq_lib:queue_info(queue_test_1),
52 | {ok, Q1} = lmq_queue:start_link(queue_test_1),
53 | [] = lmq_lib:queue_info(queue_test_1),
54 | ?DEFAULT_QUEUE_PROPS = lmq_queue:get_properties(Q1),
55 | lmq_queue:stop(Q1),
56 |
57 | P2 = lmq_misc:extend([{timeout, 10}], ?DEFAULT_QUEUE_PROPS),
58 | {ok, Q2} = lmq_queue:start_link(queue_test_1, [{timeout, 10}]),
59 | [{timeout, 10}] = lmq_lib:queue_info(queue_test_1),
60 | P2 = lmq_queue:get_properties(Q2),
61 | lmq_queue:stop(Q2),
62 |
63 | {ok, Q3} = lmq_queue:start_link(queue_test_1),
64 | [{timeout, 10}] = lmq_lib:queue_info(queue_test_1),
65 | P2 = lmq_queue:get_properties(Q3),
66 | lmq_queue:stop(Q3),
67 |
68 | Props = [{hooks, [{custom_hook, [{lmq_hook_sample1, [1]}]}]}],
69 | 1 = lmq_hook:call(queue_test_1, custom_hook, 1),
70 | {ok, Q4} = lmq_queue:start_link(queue_test_1, Props),
71 | Props = lmq_lib:queue_info(queue_test_1),
72 | P3 = lmq_misc:extend(Props, ?DEFAULT_QUEUE_PROPS),
73 | P3 = lmq_queue:get_properties(Q4),
74 | 2 = lmq_hook:call(queue_test_1, custom_hook, 1),
75 | lmq_queue:stop(Q4).
76 |
77 | push_pull_done(Config) ->
78 | Pid = ?config(queue, Config),
79 | Ref = make_ref(),
80 | ok = lmq_queue:push(Pid, Ref),
81 | M = lmq_queue:pull(Pid),
82 | {TS, UUID} = M#message.id,
83 | true = is_float(TS),
84 | true = is_binary(UUID),
85 | uuid:uuid_to_string(UUID),
86 | Ref = M#message.content,
87 | ok = lmq_queue:done(Pid, UUID).
88 |
89 | release(Config) ->
90 | Pid = ?config(queue, Config),
91 | Ref = make_ref(),
92 | ok = lmq_queue:push(Pid, Ref),
93 | M1 = lmq_queue:pull(Pid),
94 | Ref = M1#message.content,
95 | {_, UUID1} = M1#message.id,
96 | 2 = M1#message.retry,
97 |
98 | ok = lmq_queue:release(Pid, UUID1),
99 | not_found = lmq_queue:release(Pid, UUID1),
100 | not_found = lmq_queue:done(Pid, UUID1),
101 | M2 = lmq_queue:pull(Pid),
102 | Ref = M2#message.content,
103 | {_, UUID2} = M2#message.id,
104 | true = UUID1 =/= UUID2,
105 | 1 = M2#message.retry,
106 |
107 | ok = lmq_queue:put_back(Pid, UUID2),
108 | not_found = lmq_queue:put_back(Pid, UUID2),
109 | M3 = lmq_queue:pull(Pid),
110 | Ref = M3#message.content,
111 | {_, UUID3} = M3#message.id,
112 | 1 = M2#message.retry,
113 | ok = lmq_queue:done(Pid, UUID3).
114 |
115 | release_multi(Config) ->
116 | Pid = ?config(queue, Config),
117 | R = make_ref(),
118 | Parent = self(),
119 | ok = lmq_queue:push(Pid, R),
120 | M1 = lmq_queue:pull(Pid),
121 | spawn(fun() -> Parent ! lmq_queue:pull(Pid) end),
122 | R = M1#message.content,
123 | {_, UUID1} = M1#message.id,
124 | timer:sleep(10), %% waiting for starting process
125 | ok = lmq_queue:release(Pid, UUID1),
126 | receive
127 | M2 ->
128 | R = M2#message.content,
129 | {_, UUID2} = M2#message.id,
130 | true = UUID1 =/= UUID2,
131 | ok = lmq_queue:done(Pid, UUID2)
132 | after 100 ->
133 | ct:fail(no_response)
134 | end.
135 |
136 | multi_queue(Config) ->
137 | Q1 = ?config(queue, Config),
138 | {ok, Q2} = lmq_queue:start_link(for_test),
139 | Ref1 = make_ref(),
140 | Ref2 = make_ref(),
141 | ok = lmq_queue:push(Q1, Ref1),
142 | ok = lmq_queue:push(Q2, Ref2),
143 | M2 = lmq_queue:pull(Q2), Ref2 = M2#message.content,
144 | M1 = lmq_queue:pull(Q1), Ref1 = M1#message.content,
145 | ok = lmq_queue:done(Q1, element(2, M1#message.id)),
146 | ok = lmq_queue:retain(Q2, element(2, M2#message.id)),
147 | ok = lmq_queue:release(Q2, element(2, M2#message.id)),
148 | M3 = lmq_queue:pull(Q2),
149 | ok = lmq_queue:done(Q2, element(2, M3#message.id)),
150 | ok = lmq_queue:stop(Q2).
151 |
152 | pull_timeout(_Config) ->
153 | {ok, Q} = lmq_queue:start_link(pull_timeout, [{timeout, 0.3}]),
154 | Ref = make_ref(),
155 | empty = lmq_queue:pull(Q, 0),
156 | ok = lmq_queue:push(Q, Ref),
157 | M1 = lmq_queue:pull(Q, 0), Ref = M1#message.content,
158 | M2 = lmq_queue:pull(Q), Ref = M2#message.content,
159 | true = M1 =/= M2,
160 | empty = lmq_queue:pull(Q, 0.2),
161 | M3 = lmq_queue:pull(Q, 0.2), Ref = M3#message.content.
162 |
163 | async_request(_Config) ->
164 | {ok, Q} = lmq_queue:start_link(async_request, [{timeout, 0.4}]),
165 | R1 = make_ref(),
166 | R2 = make_ref(),
167 | ok = lmq_queue:push(Q, R1),
168 | ok = lmq_queue:push(Q, R2),
169 | Parent = self(),
170 | spawn(fun() ->
171 | Parent ! {1, lmq_queue:pull(Q)},
172 | Parent ! {2, lmq_queue:pull(Q)},
173 | Parent ! {3, lmq_queue:pull(Q, 0.5)}
174 | end),
175 | lists:foreach(fun(_) ->
176 | receive
177 | {1, M1} -> ct:pal("~p", [M1]), R1 = M1#message.content;
178 | {2, M2} ->
179 | ct:pal("~p", [M2]), {_, UUID2} = M2#message.id,
180 | ok = lmq_queue:done(Q, UUID2);
181 | {3, M3} -> ct:pal("~p", [M3]), R1 = M3#message.content
182 | end
183 | end, lists:seq(1, 3)).
184 |
185 | pull_async(_Config) ->
186 | {ok, Q} = lmq_queue:start_link(message, [{timeout, 0.1}]),
187 | R = make_ref(),
188 | lmq_queue:push(Q, R),
189 | Id1 = lmq_queue:pull_async(Q),
190 | receive {Id1, M1} when R =:= M1#message.content -> ok
191 | after 100 -> ct:fail(no_response)
192 | end,
193 |
194 | Id2 = lmq_queue:pull_async(Q),
195 | receive {Id2, M2} when R =:= M2#message.content -> ok
196 | after 150 -> ct:fail(no_response)
197 | end,
198 |
199 | Id3 = lmq_queue:pull_async(Q),
200 | ok = lmq_queue:pull_cancel(Q, Id3),
201 | receive {Id3, _} -> ct:fail(cancel_failed)
202 | after 150 -> ok
203 | end.
204 |
205 | pull_and_timeout(Config) ->
206 | Pid = ?config(queue, Config),
207 | Parent = self(),
208 | Ref = make_ref(),
209 | F = fun() -> Parent ! {Ref, lmq_queue:pull(Pid, 0.01)} end,
210 | [spawn(F) || _ <- lists:seq(1, 100)],
211 | wait_pull_and_timeout(Ref, 100).
212 |
213 | wait_pull_and_timeout(_, 0) ->
214 | ok;
215 | wait_pull_and_timeout(Ref, N) ->
216 | receive {Ref, empty} -> wait_pull_and_timeout(Ref, N-1)
217 | after 200 -> ct:fail(no_response)
218 | end.
219 |
220 | update_properties(Config) ->
221 | Q = ?config(queue, Config),
222 | Props1 = [{timeout, 1}],
223 | Props2 = [{hooks, [{custom_hook, [{lmq_hook_sample1, [1]}]}]}],
224 | [] = lmq_lib:queue_info(message),
225 | ?DEFAULT_QUEUE_PROPS = lmq_queue:get_properties(Q),
226 | lmq_queue:props(Q, Props1),
227 | true = lmq_misc:extend(Props1, ?DEFAULT_QUEUE_PROPS) =:= lmq_queue:get_properties(Q),
228 | Props1 = lmq_lib:queue_info(message),
229 | 1 = lmq_hook:call(message, custom_hook, 1),
230 | lmq_queue:props(Q, Props2),
231 | true = lmq_misc:extend(Props2, ?DEFAULT_QUEUE_PROPS) =:= lmq_queue:get_properties(Q),
232 | Props2 = lmq_lib:queue_info(message),
233 | 2 = lmq_hook:call(message, custom_hook, 1).
234 |
235 | reload_properties(Config) ->
236 | Q = ?config(queue, Config),
237 | Props1 = [{timeout, 1}],
238 | Props2 = [{hooks, [{custom_hook, [{lmq_hook_sample1, [1]}]}]}],
239 | [] = lmq_lib:queue_info(message),
240 | ?DEFAULT_QUEUE_PROPS = lmq_queue:get_properties(Q),
241 | lmq_lib:update_queue_props(message, Props1),
242 | lmq_queue:reload_properties(Q),
243 | true = lmq_misc:extend(Props1, ?DEFAULT_QUEUE_PROPS) =:= lmq_queue:get_properties(Q),
244 | Props1 = lmq_lib:queue_info(message),
245 | 1 = lmq_hook:call(message, custom_hook, 1),
246 | lmq_lib:update_queue_props(message, Props2),
247 | lmq_queue:reload_properties(Q),
248 | true = lmq_misc:extend(Props2, ?DEFAULT_QUEUE_PROPS) =:= lmq_queue:get_properties(Q),
249 | Props2 = lmq_lib:queue_info(message),
250 | 2 = lmq_hook:call(message, custom_hook, 1).
251 |
--------------------------------------------------------------------------------
/test/lmq_queue_mgr_SUITE.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_queue_mgr_SUITE).
2 |
3 | -include("lmq.hrl").
4 | -include_lib("common_test/include/ct.hrl").
5 | -export([init_per_suite/1, end_per_suite/1,
6 | init_per_testcase/2, end_per_testcase/2,
7 | all/0]).
8 | -export([multi_queue/1, get/1, match/1, restart_queue/1, auto_load/1,
9 | default_props/1]).
10 |
11 | all() ->
12 | [multi_queue, get, match, restart_queue, auto_load, default_props].
13 |
14 | init_per_suite(Config) ->
15 | Priv = ?config(priv_dir, Config),
16 | application:start(mnesia),
17 | application:set_env(mnesia, dir, Priv),
18 | lmq_lib:init_mnesia(),
19 | Config.
20 |
21 | end_per_suite(_Config) ->
22 | application:stop(mnesia),
23 | mnesia:delete_schema([node()]).
24 |
25 | init_per_testcase(auto_load, Config) ->
26 | {ok, _} = lmq_event:start_link(),
27 | {ok, _} = lmq_hook:start_link(),
28 | Config;
29 |
30 | init_per_testcase(_, Config) ->
31 | {ok, _} = lmq_event:start_link(),
32 | {ok, _} = lmq_hook:start_link(),
33 | {ok, _} = lmq_queue_supersup:start_link(),
34 | Config.
35 |
36 | end_per_testcase(auto_load, _Config) ->
37 | lmq_hook:stop(),
38 | mnesia:transaction(fun() -> mnesia:delete({?LMQ_INFO_TABLE, default_props}) end);
39 |
40 | end_per_testcase(_, _Config) ->
41 | lmq_hook:stop().
42 |
43 | multi_queue(_Config) ->
44 | Q1 = lmq_queue_mgr:get(q1, [create]),
45 | Q2 = lmq_queue_mgr:get(q2, [create]),
46 | R1 = make_ref(), R2 = make_ref(),
47 | lmq_queue:push(Q1, R1),
48 | lmq_queue:push(Q2, R2),
49 | M2 = lmq_queue:pull(Q2),
50 | M1 = lmq_queue:pull(Q1),
51 | R1 = M1#message.content,
52 | R2 = M2#message.content.
53 |
54 | get(_Config) ->
55 | not_found = lmq_queue_mgr:get('get/a'),
56 | true = is_pid(lmq_queue_mgr:get('get/a', [create])),
57 | [] = lmq_lib:queue_info('get/a'),
58 |
59 | %% ensure do not override props if exists
60 | true = is_pid(lmq_queue_mgr:get('get/a', [create, {props, [{retry, infinity}]}])),
61 | [] = lmq_lib:queue_info('get/a'),
62 |
63 | %% ensure override props if setting update flag
64 | true = is_pid(lmq_queue_mgr:get('get/a', [update, {props, [{retry, infinity}]}])),
65 | [{retry, infinity}] = lmq_lib:queue_info('get/a'),
66 |
67 | %% ensure custom props can be set
68 | true = is_pid(lmq_queue_mgr:get('get/b', [create, {props, [{retry, infinity}]}])),
69 | [{retry, infinity}] = lmq_lib:queue_info('get/b'),
70 |
71 | %% ensure old props are considered when updating
72 | true = is_pid(lmq_queue_mgr:get('get/b', [update, {props, [{pack, 10}]}])),
73 | [{pack, 10}, {retry, infinity}] = lmq_lib:queue_info('get/b'),
74 |
75 | %% ensure do not override props if exists in mnesia
76 | lmq_lib:create('get/c', [{pack, 10}]),
77 | true = is_pid(lmq_queue_mgr:get('get/c', [create])),
78 | [{pack, 10}] = lmq_lib:queue_info('get/c').
79 |
80 | match(_Config) ->
81 | lmq_queue_mgr:get(foo, [create]),
82 | Q1 = lmq_queue_mgr:get('foo/bar', [create]),
83 | Q2 = lmq_queue_mgr:get('foo/baz', [create]),
84 | R = lmq_queue_mgr:match("^foo/.*"),
85 | R = lmq_queue_mgr:match(<<"^foo/.*">>),
86 | true = lists:sort([{'foo/bar', Q1}, {'foo/baz', Q2}]) =:= lists:sort(R),
87 | %% error case
88 | [] = lmq_queue_mgr:match("AAA"),
89 | {error, invalid_regexp} = lmq_queue_mgr:match("a[1-").
90 |
91 | restart_queue(_Config) ->
92 | Q1 = lmq_queue_mgr:get(test, [create]),
93 | exit(Q1, kill),
94 | timer:sleep(50), % sleep until DOWN message handled
95 | Q2 = lmq_queue_mgr:get(test),
96 | true = Q1 =/= Q2.
97 |
98 | auto_load(_Config) ->
99 | DefaultProps = [{"lmq", [{retry, 0}]}],
100 | lmq_lib:set_lmq_info(default_props, DefaultProps),
101 | lmq_lib:create(auto_loaded_1),
102 | lmq_lib:create(auto_loaded_2),
103 | {ok, _} = lmq_queue_supersup:start_link(),
104 | timer:sleep(100), % wait until queue will be started
105 | DefaultProps = lmq_queue_mgr:get_default_props(),
106 | true = is_pid(lmq_queue_mgr:get('auto_loaded_1')),
107 | true = is_pid(lmq_queue_mgr:get('auto_loaded_2')),
108 | not_found = lmq_queue_mgr:get('auto_loaded_3').
109 |
110 | default_props(_Config) ->
111 | PropsList = [{"lmq", [{retry, 0}]}],
112 | [] = lmq_queue_mgr:get_default_props(),
113 | ok = lmq_queue_mgr:set_default_props(PropsList),
114 | {ok, PropsList} = lmq_lib:get_lmq_info(default_props),
115 | PropsList = lmq_queue_mgr:get_default_props(),
116 | invalid_syntax = lmq_queue_mgr:set_default_props({}).
117 |
--------------------------------------------------------------------------------
/test/lmq_test_handler.erl:
--------------------------------------------------------------------------------
1 | -module(lmq_test_handler).
2 |
3 | -behaviour(gen_event).
4 |
5 | -export([init/1, handle_event/2, handle_call/2, handle_info/2,
6 | code_change/3, terminate/2]).
7 |
8 | init(Pid) ->
9 | {ok, Pid}.
10 |
11 | handle_event(Event, Pid) ->
12 | ct:pal("Event received: ~p", [Event]),
13 | Pid ! {test_handler, Event},
14 | {ok, Pid}.
15 |
16 | handle_call(_, State) ->
17 | {ok, ok, State}.
18 |
19 | handle_info(_, State) ->
20 | {ok, State}.
21 |
22 | code_change(_OldVsn, State, _Extra) ->
23 | {ok, State}.
24 |
25 | terminate(_Reason, _State) ->
26 | ok.
27 |
--------------------------------------------------------------------------------