├── test ├── .gitignore └── queuesk_SUITE.erl ├── rebar ├── .gitignore ├── .travis.yml ├── TODO.md ├── src ├── queuesk_utils.erl ├── queuesk.app.src ├── queuesk_sup.erl ├── queuesk_pool_sup.erl ├── queuesk_app.erl ├── queuesk_manager.erl ├── queuesk_pool_scheduler.erl └── queuesk.erl ├── Makefile ├── include └── queuesk.hrl └── README.md /test/.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/hamidreza-s/Queuesk/HEAD/rebar -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | ebin 3 | Mnesia.nonode@nohost 4 | .rebar 5 | *~ 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: erlang 3 | otp_release: 4 | - R15B03 5 | - R16B03 6 | - 17.5 7 | - 18.2 8 | script: make test 9 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | * add shaper to pool_scheduler 2 | * add 'active' and 'passive' feature to queues 3 | * implement unit and common tests 4 | * write documentation 5 | -------------------------------------------------------------------------------- /src/queuesk_utils.erl: -------------------------------------------------------------------------------- 1 | -module(queuesk_utils). 2 | 3 | -export([get_config/1]). 4 | 5 | %%=================================================================== 6 | %% API 7 | %%=================================================================== 8 | 9 | %%-------------------------------------------------------------------- 10 | %% get_config 11 | %%-------------------------------------------------------------------- 12 | get_config(Key) -> 13 | application:get_env(queuesk, Key). 14 | -------------------------------------------------------------------------------- /src/queuesk.app.src: -------------------------------------------------------------------------------- 1 | {application, queuesk, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib 9 | ]}, 10 | {mod, {queuesk_app, []}}, 11 | {env, [ 12 | {mode, development}, %% development | debug | production 13 | {queue_id_prefix, qsk_queue}, 14 | {default_queue_type, priority}, 15 | {default_queue_persist, false}, 16 | {default_queue_schedulers, 1}, 17 | {default_task_priority, 1}, 18 | {default_task_retry, 1}, 19 | {default_task_timeout, 3000} 20 | ]} 21 | ]}. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: build 2 | 3 | build: deps 4 | ./rebar compile 5 | 6 | deps: 7 | ./rebar get-deps 8 | 9 | clean: 10 | rm -rf ./Mnesia.nonode@nohost 11 | rm -rf ./ebin/* 12 | 13 | test: build 14 | @mkdir -p ./test/logs 15 | @exec ct_run \ 16 | -dir ./test \ 17 | -include ./include \ 18 | -pa ./ebin \ 19 | -logdir ./test/logs 20 | 21 | debug: build 22 | erl -pa ebin deps/*/ebin -s queuesk -boot start_sasl 23 | 24 | live: build 25 | erl -pa ebin deps/*/ebin -s queuesk 26 | 27 | todo: 28 | @echo "[+] TODOs in general:" 29 | @echo "---------------------" 30 | @cat ./TODO.md 31 | @echo 32 | @echo "[+] TODOs in source:" 33 | @echo "--------------------" 34 | @grep -ir "@todo" ./src 35 | -------------------------------------------------------------------------------- /src/queuesk_sup.erl: -------------------------------------------------------------------------------- 1 | -module(queuesk_sup). 2 | -behaviour(supervisor). 3 | 4 | -export([start_link/0]). 5 | -export([init/1]). 6 | 7 | %%=================================================================== 8 | %% API functions 9 | %%=================================================================== 10 | 11 | %%-------------------------------------------------------------------- 12 | %% start_link 13 | %%-------------------------------------------------------------------- 14 | start_link() -> 15 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 16 | 17 | %%=================================================================== 18 | %% Supervisor callbacks 19 | %%=================================================================== 20 | 21 | %%-------------------------------------------------------------------- 22 | %% init 23 | %%-------------------------------------------------------------------- 24 | init([]) -> 25 | QueueskPoolSup = {queuesk_pool_sup, 26 | {queuesk_pool_sup, start_link, []}, 27 | permanent, 28 | 3000, 29 | supervisor, 30 | [queuesk_pool_sup]}, 31 | 32 | QueueskManager = {queuesk_manager, 33 | {queuesk_manager, start_link, []}, 34 | permanent, 35 | 3000, 36 | worker, 37 | [queuesk_manager]}, 38 | 39 | {ok, {{one_for_all, 5, 10}, 40 | [QueueskPoolSup, QueueskManager]}}. 41 | 42 | -------------------------------------------------------------------------------- /src/queuesk_pool_sup.erl: -------------------------------------------------------------------------------- 1 | -module(queuesk_pool_sup). 2 | -behaviour(supervisor). 3 | 4 | -export([start_link/0, 5 | add_scheduler/1, 6 | init/1]). 7 | 8 | %%=================================================================== 9 | %% Supervisor API 10 | %%=================================================================== 11 | 12 | %%-------------------------------------------------------------------- 13 | %% start_link 14 | %%-------------------------------------------------------------------- 15 | start_link() -> 16 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 17 | 18 | %%-------------------------------------------------------------------- 19 | %% add_scheduler 20 | %%-------------------------------------------------------------------- 21 | add_scheduler(Queue) -> 22 | supervisor:start_child(?MODULE, [Queue]). 23 | 24 | %%=================================================================== 25 | %% Supervisor Callbacks 26 | %%=================================================================== 27 | 28 | %%-------------------------------------------------------------------- 29 | %% init 30 | %%-------------------------------------------------------------------- 31 | init([]) -> 32 | {ok, {{simple_one_for_one, 5, 10}, 33 | [ 34 | {queuesk_pool_scheduler, 35 | {queuesk_pool_scheduler, start_link, []}, 36 | permanent, 37 | 3000, 38 | worker, 39 | [queuesk_pool_scheduler]} 40 | ]}}. 41 | -------------------------------------------------------------------------------- /src/queuesk_app.erl: -------------------------------------------------------------------------------- 1 | -module(queuesk_app). 2 | -behaviour(application). 3 | 4 | -export([start/2, 5 | stop/1]). 6 | 7 | -include("queuesk.hrl"). 8 | 9 | %%=================================================================== 10 | %% Application callbacks 11 | %%=================================================================== 12 | 13 | %%-------------------------------------------------------------------- 14 | %% start 15 | %%-------------------------------------------------------------------- 16 | start(_StartType, _StartArgs) -> 17 | 18 | ok = init_database(), 19 | 20 | queuesk_sup:start_link(). 21 | 22 | %%-------------------------------------------------------------------- 23 | %% stop 24 | %%-------------------------------------------------------------------- 25 | stop(_State) -> 26 | ok. 27 | 28 | %%=================================================================== 29 | %% Internals 30 | %%=================================================================== 31 | 32 | %%-------------------------------------------------------------------- 33 | %% init_database 34 | %%-------------------------------------------------------------------- 35 | init_database() -> 36 | case mnesia:create_schema([node()]) of 37 | ok -> 38 | ok; 39 | {error, {_, {already_exists, _}}} -> 40 | ok 41 | end, 42 | 43 | mnesia:start(), 44 | 45 | mnesia:create_table(qsk_queue_registery, 46 | [{type, ordered_set}, 47 | {disc_copies, [node()]}, 48 | {attributes, record_info(fields, 49 | qsk_queue_registery)}]), 50 | 51 | mnesia:create_table(qsk_queue_failed_record, 52 | [{type, bag}, 53 | {disc_only_copies, [node()]}, 54 | {attributes, record_info(fields, 55 | qsk_queue_failed_record)}]), 56 | 57 | mnesia:wait_for_tables([qsk_queue_registery, 58 | qsk_queue_failed_record], 5000), 59 | 60 | ok. 61 | -------------------------------------------------------------------------------- /include/queuesk.hrl: -------------------------------------------------------------------------------- 1 | %%=================================================================== 2 | %% Records 3 | %%=================================================================== 4 | 5 | %%-------------------------------------------------------------------- 6 | %% qsk_queue_registery 7 | %%-------------------------------------------------------------------- 8 | -record(qsk_queue_registery, {queue_name, queue_id, type, persist, schedulers}). 9 | 10 | %%-------------------------------------------------------------------- 11 | %% qsk_queue_schema 12 | %%-------------------------------------------------------------------- 13 | -record(qsk_queue_schema, {priority, task, retry, timeout}). 14 | 15 | %%-------------------------------------------------------------------- 16 | %% qsk_queue_record 17 | %%-------------------------------------------------------------------- 18 | -record(qsk_queue_record, {priority, task, retry, timeout, queue_id}). 19 | 20 | %%-------------------------------------------------------------------- 21 | %% qsk_queue_failed_record 22 | %%-------------------------------------------------------------------- 23 | -record(qsk_queue_failed_record, {queue_id, qsk_queue_record, reason, time}). 24 | 25 | %%=================================================================== 26 | %% Macros 27 | %%=================================================================== 28 | 29 | %%-------------------------------------------------------------------- 30 | %% specific queue record 31 | %%-------------------------------------------------------------------- 32 | -define(SPECIFIC_QUEUE_REC(Rec), {Rec#qsk_queue_record.queue_id, 33 | Rec#qsk_queue_record.priority, 34 | Rec#qsk_queue_record.task, 35 | Rec#qsk_queue_record.retry, 36 | Rec#qsk_queue_record.timeout}). 37 | 38 | %%-------------------------------------------------------------------- 39 | %% generic queue record 40 | %%-------------------------------------------------------------------- 41 | -define(GENERIC_QUEUE_REC(Rec), 42 | begin 43 | {QueueID, Priority, Task, Retry, Timeout} = Rec, 44 | #qsk_queue_record{priority = Priority, 45 | task = Task, 46 | retry = Retry, 47 | timeout = Timeout, 48 | queue_id = QueueID} 49 | end). 50 | 51 | %%-------------------------------------------------------------------- 52 | %% supervisor child 53 | %%-------------------------------------------------------------------- 54 | -define(SUPERVISOR_CHILD(I, Type), {I, {I, start_link, []}, permanent, 55 | 5000, Type, [I]}). 56 | 57 | %%-------------------------------------------------------------------- 58 | %% now timestamp 59 | %%-------------------------------------------------------------------- 60 | -define(NOW_TIMESTAMP, now()). 61 | 62 | %%-------------------------------------------------------------------- 63 | %% debug 64 | %%-------------------------------------------------------------------- 65 | -define(DEBUG(String, Args), begin 66 | case queuesk_utils:get_config(mode) of 67 | {ok, debug} -> 68 | error_logger:info_msg(String ++ "~n", Args); 69 | _ -> 70 | dont 71 | end 72 | end). 73 | -------------------------------------------------------------------------------- /src/queuesk_manager.erl: -------------------------------------------------------------------------------- 1 | -module(queuesk_manager). 2 | -behaviour(gen_server). 3 | 4 | -export([start_link/0]). 5 | 6 | -export([init/1, 7 | handle_call/3, 8 | handle_cast/2, 9 | handle_info/2, 10 | terminate/2, 11 | code_change/3]). 12 | 13 | -include("queuesk.hrl"). 14 | 15 | %%=================================================================== 16 | %% API Functions 17 | %%=================================================================== 18 | 19 | %%-------------------------------------------------------------------- 20 | %% start_link 21 | %%-------------------------------------------------------------------- 22 | start_link() -> 23 | gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). 24 | 25 | %%==================================================================== 26 | %% Generic Server Callback 27 | %%==================================================================== 28 | 29 | %%-------------------------------------------------------------------- 30 | %% init 31 | %-------------------------------------------------------------------- 32 | init([]) -> 33 | 34 | ok = init_schedulers_pool(), 35 | ok = run_remaining_tasks(), 36 | 37 | {ok, undefined}. 38 | 39 | %%-------------------------------------------------------------------- 40 | %% handle_call 41 | %%-------------------------------------------------------------------- 42 | handle_call(_Msg, _From, State) -> 43 | {reply, ok, State}. 44 | 45 | %%-------------------------------------------------------------------- 46 | %% handle_cast 47 | %%-------------------------------------------------------------------- 48 | handle_cast(_Msg, State) -> 49 | {noreply, State}. 50 | 51 | %%-------------------------------------------------------------------- 52 | %% handle_info 53 | %%-------------------------------------------------------------------- 54 | handle_info(_Msg, State) -> 55 | {noreply, State}. 56 | 57 | %%-------------------------------------------------------------------- 58 | %% terminate 59 | %%-------------------------------------------------------------------- 60 | terminate(_Reason, _State) -> 61 | ok. 62 | %%-------------------------------------------------------------------- 63 | %% code_change 64 | %%-------------------------------------------------------------------- 65 | code_change(_OldVsn, State, _Extra) -> 66 | {ok, State}. 67 | 68 | %%==================================================================== 69 | %% Internal Functions 70 | %%==================================================================== 71 | 72 | %%-------------------------------------------------------------------- 73 | %% init_schedulers_pool 74 | %%-------------------------------------------------------------------- 75 | init_schedulers_pool() -> 76 | 77 | Queues = queuesk:queue_list(), 78 | [begin 79 | queuesk_pool_sup:add_scheduler(Queue) 80 | end || Queue <- Queues], 81 | 82 | ok. 83 | 84 | %%-------------------------------------------------------------------- 85 | %% run_remaining_tasks 86 | %%-------------------------------------------------------------------- 87 | run_remaining_tasks() -> 88 | 89 | Queues = queuesk:queue_list(), 90 | [begin 91 | 92 | QueueID = Queue#qsk_queue_registery.queue_id, 93 | Acc = 0, 94 | mnesia:activity( 95 | transaction, 96 | fun() -> 97 | mnesia:foldl( 98 | fun(TaskRec, _) -> 99 | queuesk_pool_scheduler:submit_task( 100 | QueueID, ?GENERIC_QUEUE_REC(TaskRec)) 101 | end, 102 | Acc, 103 | QueueID) 104 | end) 105 | 106 | end || Queue <- Queues], 107 | 108 | ok. 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Queuesk 2 | ====== 3 | 4 | Queuesk (pronounce it like kiosk /ˈkiːɑːsk/) is a lightweight priority task queue which can be used inside your Erlang application as a dependency. The order of executing tasks is based on their priority levels. Also you can set timeout and a number for retrying the tasks which would fail or crash when executing. Each queue's tasks can be durable or not durable after restart. 5 | 6 | Quick Start 7 | ---- 8 | 9 | **Installation** 10 | 11 | On a machine with an installed Erlang/OTP R15 or newer, just clone this repo and go through following steps: 12 | 13 | ```bash 14 | $ git clone https://github.com/bisphone/Queuesk.git 15 | $ cd Queuesk 16 | $ make 17 | ``` 18 | 19 | Also for using it as a dependency in your project using `rebar`, add the following code snippet to your `rebar.config` file: 20 | 21 | ```erlang 22 | {deps, [ 23 | %% ... 24 | {queuesk, ".*", {git, "git://github.com/bisphone/queuesk.git", {branch, "master"}}} 25 | ]}. 26 | ``` 27 | 28 | **Usage** 29 | 30 | Let's imagine we need a task queue for newly registered users in our website for doing some tasks after their registration. First we must create the queue, and because the tasks are not trivial to us, we make the queue persistent. 31 | 32 | ```erlang 33 | {ok, QueueID} = queuesk:queue_add(new_users, [{persist, true}]). 34 | ``` 35 | 36 | In this scenario each user must receive a welcome email and also her friends must be notified about her registration. Because sending welcome email has more priority, we set its priority higher. 37 | 38 | ```erlang 39 | ok = queuesk:task_push(QueueID, 40 | fun() -> send_welcome_email() end, 41 | [{priority, 1}, {retry, 3}, {timeout, 3000}]), 42 | ok = queuesk:task_push(QueueID, 43 | fun() -> send_notification() end, 44 | [{priority, 2}, {retry, 2}, {timeout, 3000}]) 45 | ``` 46 | 47 | The retry and timeout parameters help to tailor each task to the needs and restrictions of our usecases. 48 | 49 | Configuration 50 | ---- 51 | 52 | There are some default configuration which you can change based on your needs as follows: 53 | 54 | ```erlang 55 | {mode, development}, 56 | {queue_id_prefix, qsk_queue}, 57 | {default_queue_type, priority}, 58 | {default_queue_persist, false}, 59 | {default_queue_schedulers, 1}, 60 | {default_task_priority, 1}, 61 | {default_task_retry, 1}, 62 | {default_task_timeout, 3000} 63 | ``` 64 | 65 | These config parameters resides in `src/queuesk.app.src` file. 66 | 67 | API 68 | ---- 69 | 70 | The main APIs are exported via `queuesk` module as follows: 71 | 72 | **Types** 73 | 74 | ```erlang 75 | -type queue_opts() :: {persist, true | false}. 76 | -type task_id() :: {integer(), erlang:timestamp()}. 77 | -type task_opts() :: {priority, integer()} | {retry, integer()} | {timeout, integer()}. 78 | -type task_func() :: (fun() -> nok | ok). 79 | ``` 80 | 81 | **Queue API** 82 | 83 | ```erlang 84 | -spec queue_add(QueueName :: atom(), Opts :: [queue_opts()]) -> ok. 85 | -spec queue_remove(QueueName :: atom()) -> ok. 86 | -spec queue_get(QueueName :: atom()) -> {ok, QueueRecord :: #qsk_queue_registery{}} | not_exist. 87 | -spec queue_get_id(QueueName :: atom()) -> {ok, QueueID :: atom()} | not_exit. 88 | -spec queue_list() -> [#qsk_queue_registery{}]. 89 | ``` 90 | 91 | **Task API** 92 | 93 | ```erlang 94 | -spec task_push(QueueID :: atom(), Func :: task_func(), Opts :: [task_opts()]) -> ok. 95 | -spec task_pop(QueueID :: atom()) -> #qsk_queue_record{} | empty. 96 | -spec task_peek(QueueID :: atom()) -> #qsk_queue_record{} | empty. 97 | -spec task_remove(QueueID :: atom(), Key :: task_id()) -> ok. 98 | ``` 99 | 100 | Contribution 101 | ----- 102 | 103 | Comments, contributions and patches are greatly appreciated. 104 | 105 | License 106 | ----- 107 | The MIT License (MIT). 108 | -------------------------------------------------------------------------------- /src/queuesk_pool_scheduler.erl: -------------------------------------------------------------------------------- 1 | -module(queuesk_pool_scheduler). 2 | 3 | -export([start_link/1, 4 | init/2]). 5 | 6 | -export([submit_task/2]). 7 | 8 | -include("queuesk.hrl"). 9 | 10 | -record(state, {scheduler_ref, parent_pid, queue_rec}). 11 | 12 | %%=================================================================== 13 | %% API 14 | %%=================================================================== 15 | 16 | %%-------------------------------------------------------------------- 17 | %% start_link 18 | %%-------------------------------------------------------------------- 19 | start_link(Queue) -> 20 | proc_lib:start_link(?MODULE, init, [self(), Queue]). 21 | 22 | %%-------------------------------------------------------------------- 23 | %% init 24 | %%-------------------------------------------------------------------- 25 | init(Parent, Queue) -> 26 | SchedulerRef = Queue#qsk_queue_registery.queue_id, 27 | register(SchedulerRef, self()), 28 | proc_lib:init_ack({ok, Parent}), 29 | loop(#state{scheduler_ref = SchedulerRef, 30 | parent_pid = Parent, 31 | queue_rec = Queue}). 32 | 33 | 34 | %%-------------------------------------------------------------------- 35 | %% loop 36 | %%-------------------------------------------------------------------- 37 | loop(State) -> 38 | receive 39 | {task, high_priority, Task} -> 40 | ok = run_task(Task, State), 41 | loop(State) 42 | after 0 -> 43 | receive 44 | {task, med_priority, Task} -> 45 | ok = run_task(Task, State), 46 | loop(State) 47 | after 0 -> 48 | receive 49 | {task, low_priority, Task} -> 50 | ok = run_task(Task, State), 51 | loop(State) 52 | end 53 | end 54 | end. 55 | 56 | %%-------------------------------------------------------------------- 57 | %% submit_task 58 | %%-------------------------------------------------------------------- 59 | submit_task(SchedulerPid, #qsk_queue_record{priority = PriorityNum} = Task) -> 60 | 61 | %% @TODO: implement a shaper not to be got overflow 62 | 63 | PriorityAtom = 64 | case PriorityNum of 65 | 1 -> high_priority; 66 | 2 -> med_priority; 67 | _ -> low_priority 68 | end, 69 | SchedulerPid ! {task, PriorityAtom, Task}, 70 | ok. 71 | 72 | %%=================================================================== 73 | %% Internal Functions 74 | %%=================================================================== 75 | 76 | %%-------------------------------------------------------------------- 77 | %% done_task 78 | %%-------------------------------------------------------------------- 79 | done_task(QueueID, #qsk_queue_record{priority = TaskKey}) -> 80 | ok = queuesk:task_remove(QueueID, TaskKey). 81 | 82 | %%-------------------------------------------------------------------- 83 | %% failed_task 84 | %%-------------------------------------------------------------------- 85 | failed_task(QueueID, Task, Reason) -> 86 | ok = mnesia:dirty_write(#qsk_queue_failed_record{ 87 | queue_id = QueueID, 88 | qsk_queue_record = Task, 89 | time = now(), 90 | reason = Reason}). 91 | 92 | %%-------------------------------------------------------------------- 93 | %% run_task 94 | %%-------------------------------------------------------------------- 95 | run_task(Task, State) -> 96 | 97 | ?DEBUG("queuesk_pool_scheduler:run_task:begin", []), 98 | 99 | QueueRec = State#state.queue_rec, 100 | QueueID = QueueRec#qsk_queue_registery.queue_id, 101 | TaskTimeout = Task#qsk_queue_record.timeout, 102 | TaskRetry = Task#qsk_queue_record.retry, 103 | 104 | {WorkerPID, WorkerRef} = spawn_monitor(fun() -> do_run_task(Task, State) end), 105 | 106 | receive 107 | {'DOWN', WorkerRef, process, WorkerPID, done} -> 108 | ?DEBUG("queuesk_pool_scheduler:run_task:receive:done", []), 109 | done_task(QueueID, Task), 110 | ok; 111 | {'DOWN', WorkerRef, process, WorkerPID, Reason} -> 112 | ?DEBUG("queuesk_pool_scheduler:run_task:receive:{~p}", [Reason]), 113 | NewTaskRetry = TaskRetry - 1, 114 | case NewTaskRetry > 0 of 115 | true -> 116 | NewTask = Task#qsk_queue_record{retry = NewTaskRetry}, 117 | ?DEBUG("queuesk_pool_scheduler:run_task:receive:{~p}:retry", 118 | [Reason]), 119 | submit_task(QueueID, NewTask), 120 | ok; 121 | false -> 122 | failed_task(QueueID, Task, {error, Reason}), 123 | ?DEBUG("queuesk_pool_scheduler:run_task:receive:{~p}:dont_retry", 124 | [Reason]), 125 | ok 126 | end 127 | after TaskTimeout -> 128 | ?DEBUG("queuesk_pool_scheduler:run_task:timeout", []), 129 | erlang:demonitor(WorkerRef, [flush]), 130 | exit(WorkerPID, kill), 131 | NewTaskRetry = TaskRetry - 1, 132 | case NewTaskRetry > 0 of 133 | true -> 134 | NewTask = Task#qsk_queue_record{retry = NewTaskRetry}, 135 | ?DEBUG("queuesk_pool_scheduler:run_task:timeout:retry", []), 136 | submit_task(QueueID, NewTask), 137 | ok; 138 | false -> 139 | failed_task(QueueID, Task, timeout), 140 | ?DEBUG("queuesk_pool_scheduler:run_task:timeout:dont_retry", []), 141 | ok 142 | end 143 | 144 | end. 145 | 146 | %%-------------------------------------------------------------------- 147 | %% do_run_task (quarantine) 148 | %%-------------------------------------------------------------------- 149 | do_run_task(Task, _State) -> 150 | ?DEBUG("queuesk_pool_scheduler:do_run_task:begin", []), 151 | TaskFunc = Task#qsk_queue_record.task, 152 | case apply(TaskFunc, []) of 153 | ok -> 154 | ?DEBUG("queuesk_pool_scheduler:do_run_task:exit:ok", []), 155 | exit(done); 156 | Reason -> 157 | ?DEBUG("queuesk_pool_scheduler:do_run_task:exit:{~p}", [Reason]), 158 | exit(Reason) 159 | end. 160 | -------------------------------------------------------------------------------- /test/queuesk_SUITE.erl: -------------------------------------------------------------------------------- 1 | -module(queuesk_SUITE). 2 | 3 | -include_lib("common_test/include/ct.hrl"). 4 | -include_lib("eunit/include/eunit.hrl"). 5 | 6 | -include_lib("Queuesk/include/queuesk.hrl"). 7 | 8 | -compile(export_all). 9 | 10 | -define(SELF, ct_running_process). 11 | 12 | %%==================================================================== 13 | %% CT Callbacks 14 | %%==================================================================== 15 | 16 | %%-------------------------------------------------------------------- 17 | %% suite | groups | all 18 | %%-------------------------------------------------------------------- 19 | suite() -> [{timetrap, {seconds, 60*60}}]. 20 | 21 | groups() -> []. 22 | 23 | all() -> 24 | [test_queue_api, 25 | test_task_api]. 26 | 27 | %%-------------------------------------------------------------------- 28 | %% init_per_suite | end_per_suite 29 | %%-------------------------------------------------------------------- 30 | init_per_suite(Config) -> 31 | application:start(queuesk), 32 | Config. 33 | 34 | end_per_suite(_Config) -> 35 | application:stop(queuesk), 36 | ok. 37 | 38 | %%-------------------------------------------------------------------- 39 | %% init_per_group | end_per_group 40 | %%-------------------------------------------------------------------- 41 | init_per_group(_group, Config) -> 42 | Config. 43 | 44 | end_per_group(_group, Config) -> 45 | Config. 46 | 47 | %%-------------------------------------------------------------------- 48 | %% init_per_testcase | end_per_testcase 49 | %%-------------------------------------------------------------------- 50 | init_per_testcase(_TestCase, Config) -> 51 | register(?SELF, self()), 52 | Config. 53 | 54 | end_per_testcase(_TestCase, Config) -> 55 | unregister(?SELF), 56 | Config. 57 | 58 | %%==================================================================== 59 | %% Test Cases 60 | %%==================================================================== 61 | 62 | %%-------------------------------------------------------------------- 63 | %% test_queue_api 64 | %%-------------------------------------------------------------------- 65 | test_queue_api(_Config) -> 66 | 67 | %%---------------------------------------------------------------- 68 | %% add and get queue 69 | %%---------------------------------------------------------------- 70 | 71 | ?assertEqual({ok, qsk_queue_test1}, 72 | queuesk:queue_add(test1, [])), 73 | 74 | ?assertEqual({ok, #qsk_queue_registery{queue_name = test1, 75 | queue_id = qsk_queue_test1, 76 | type = priority, 77 | persist = false, 78 | schedulers = 1}}, 79 | queuesk:queue_get(test1)), 80 | 81 | ?assertEqual({ok, qsk_queue_test2}, 82 | queuesk:queue_add(test2, [{persist, false}])), 83 | 84 | ?assertEqual({ok, #qsk_queue_registery{queue_name = test2, 85 | queue_id = qsk_queue_test2, 86 | type = priority, 87 | persist = false, 88 | schedulers = 1}}, 89 | queuesk:queue_get(test2)), 90 | 91 | ?assertEqual({ok, qsk_queue_test3}, 92 | queuesk:queue_add(test3, [{persist, true}])), 93 | 94 | ?assertEqual({ok, #qsk_queue_registery{queue_name = test3, 95 | queue_id = qsk_queue_test3, 96 | type = priority, 97 | persist = true, 98 | schedulers = 1}}, 99 | queuesk:queue_get(test3)), 100 | 101 | %%---------------------------------------------------------------- 102 | %% get queue id 103 | %%---------------------------------------------------------------- 104 | 105 | ?assertEqual({ok, qsk_queue_test1}, queuesk:queue_get_id(test1)), 106 | 107 | %%---------------------------------------------------------------- 108 | %% list queues 109 | %%---------------------------------------------------------------- 110 | 111 | ?assertEqual([#qsk_queue_registery{queue_name = test1, 112 | queue_id = qsk_queue_test1, 113 | type = priority, 114 | persist = false, 115 | schedulers = 1}, 116 | #qsk_queue_registery{queue_name = test2, 117 | queue_id = qsk_queue_test2, 118 | type = priority, 119 | persist = false, 120 | schedulers = 1}, 121 | #qsk_queue_registery{queue_name = test3, 122 | queue_id = qsk_queue_test3, 123 | type = priority, 124 | persist = true, 125 | schedulers = 1}], 126 | queuesk:queue_list()), 127 | 128 | %%---------------------------------------------------------------- 129 | %% remove queue 130 | %%---------------------------------------------------------------- 131 | 132 | ?assertEqual(ok, queuesk:queue_remove(test1)), 133 | 134 | ?assertEqual(not_exist, queuesk:queue_get(test1)), 135 | 136 | ?assertEqual([#qsk_queue_registery{queue_name = test2, 137 | queue_id = qsk_queue_test2, 138 | type = priority, 139 | persist = false, 140 | schedulers = 1}, 141 | #qsk_queue_registery{queue_name = test3, 142 | queue_id = qsk_queue_test3, 143 | type = priority, 144 | persist = true, 145 | schedulers = 1}], 146 | queuesk:queue_list()), 147 | 148 | ok. 149 | 150 | %%-------------------------------------------------------------------- 151 | %% test_task_api 152 | %%-------------------------------------------------------------------- 153 | test_task_api(_Config) -> 154 | 155 | %%---------------------------------------------------------------- 156 | %% push task 157 | %%---------------------------------------------------------------- 158 | 159 | ?assertEqual({ok, qsk_queue_test4}, 160 | queuesk:queue_add(test4, [])), 161 | 162 | Self = self(), 163 | TasksNumber = 2, 164 | 165 | lists:foreach(fun(_) -> 166 | ?assertEqual(ok, 167 | queuesk:task_push(qsk_queue_test4, 168 | fun() -> Self ! task, ok end, 169 | [{priority, 1}, {retry, 3}, 170 | {timeout, 3000}])) 171 | end, 172 | lists:seq(1, TasksNumber)), 173 | 174 | ?assertEqual(ok, count_done_tasks(TasksNumber)), 175 | 176 | ok. 177 | 178 | %%==================================================================== 179 | %% Controlling functions 180 | %%==================================================================== 181 | 182 | %%-------------------------------------------------------------------- 183 | %% count done tasks 184 | %%-------------------------------------------------------------------- 185 | count_done_tasks(Number) 186 | when Number > 0 -> 187 | receive 188 | task -> 189 | count_done_tasks(Number - 1) 190 | end; 191 | count_done_tasks(_) -> 192 | ok. 193 | 194 | %%-------------------------------------------------------------------- 195 | %% pause 196 | %%-------------------------------------------------------------------- 197 | pause() -> 198 | receive 199 | stop -> 200 | ok 201 | end. 202 | 203 | %%-------------------------------------------------------------------- 204 | %% next 205 | %%-------------------------------------------------------------------- 206 | next() -> 207 | catch ?SELF ! stop, 208 | ok. 209 | -------------------------------------------------------------------------------- /src/queuesk.erl: -------------------------------------------------------------------------------- 1 | -module(queuesk). 2 | 3 | -export([start/0, 4 | stop/0]). 5 | 6 | -export([queue_add/2, 7 | queue_remove/1, 8 | queue_list/0, 9 | queue_get/1, 10 | queue_make_id/1, 11 | queue_get_id/1]). 12 | 13 | -export([task_push/3, 14 | task_pop/1, 15 | task_peek/1, 16 | task_remove/2]). 17 | 18 | -include("queuesk.hrl"). 19 | 20 | -type queue_opts() :: {persist, true | false}. 21 | -type task_func() :: fun(() -> nok | ok). 22 | -type task_id() :: {integer(), erlang:timestamp()}. 23 | -type task_opts() :: {priority, integer()} 24 | | {retry, integer()} 25 | | {timeout, integer()}. 26 | 27 | %%=================================================================== 28 | %% Queue API 29 | %%=================================================================== 30 | 31 | %%-------------------------------------------------------------------- 32 | %% start 33 | %%-------------------------------------------------------------------- 34 | start() -> 35 | application:start(?MODULE). 36 | 37 | %%-------------------------------------------------------------------- 38 | %% stop 39 | %%-------------------------------------------------------------------- 40 | stop() -> 41 | application:stop(?MODULE). 42 | 43 | %%-------------------------------------------------------------------- 44 | %% queue_add 45 | %%-------------------------------------------------------------------- 46 | -spec queue_add(QueueName :: atom(), Opts :: [queue_opts()]) -> ok. 47 | queue_add(QueueName, Opts) 48 | when is_atom(QueueName), 49 | is_list(Opts) -> 50 | 51 | %% @TODO: don't let use reserved names like 'registery', 'record' and 'failed_record' 52 | 53 | {ok, DefaultType} = queuesk_utils:get_config(default_queue_type), 54 | {ok, DefaultPersist} = queuesk_utils:get_config(default_queue_persist), 55 | {ok, DefaultSchedulers} = queuesk_utils:get_config(default_queue_schedulers), 56 | 57 | QueueID = queue_make_id(QueueName), 58 | 59 | NewOpts = [{queue_id, QueueID}, 60 | {type, proplists:get_value(type, Opts, DefaultType)}, 61 | {persist, proplists:get_value(persist, Opts, DefaultPersist)}, 62 | {schedulers, proplists:get_value(schedulers, Opts, DefaultSchedulers)}], 63 | 64 | do_queue_add(QueueName, NewOpts). 65 | 66 | do_queue_add(QueueName, Opts) -> 67 | 68 | %% @TODO: make following action transactional and idempotent 69 | 70 | QueueID = proplists:get_value(queue_id, Opts), 71 | Type = proplists:get_value(type, Opts), 72 | Persist = proplists:get_value(persist, Opts), 73 | Schedulers = proplists:get_value(schedulers, Opts), 74 | Storage = case proplists:get_value(persist, Opts) of 75 | true -> 76 | disc_copies; 77 | false -> 78 | ram_copies 79 | end, 80 | 81 | Result = mnesia:create_table(QueueID, 82 | [{type, ordered_set}, 83 | {Storage, [node()]}, 84 | {attributes, 85 | record_info(fields, 86 | qsk_queue_schema)}]), 87 | 88 | case Result of 89 | {atomic, ok} -> 90 | Queue = #qsk_queue_registery{ 91 | queue_id = QueueID, 92 | queue_name = QueueName, 93 | type = Type, 94 | persist = Persist, 95 | schedulers = Schedulers}, 96 | 97 | ok = mnesia:dirty_write(Queue), 98 | {ok, _QueueSchedulerPID} = queuesk_pool_sup:add_scheduler(Queue), 99 | 100 | {ok, QueueID}; 101 | Else -> 102 | Else 103 | end. 104 | 105 | %%-------------------------------------------------------------------- 106 | %% queue_remove 107 | %%-------------------------------------------------------------------- 108 | -spec queue_remove(QueueName :: atom()) -> ok. 109 | queue_remove(QueueName) -> 110 | {ok, QueueID} = queue_get_id(QueueName), 111 | {atomic, ok} = mnesia:delete_table(QueueID), 112 | ok = mnesia:dirty_delete({qsk_queue_registery, QueueName}), 113 | ok. 114 | 115 | %%-------------------------------------------------------------------- 116 | %% queue_get 117 | %%-------------------------------------------------------------------- 118 | -spec queue_get(QueueName :: atom()) -> {ok, QueueRecord :: #qsk_queue_registery{}} | not_exist. 119 | queue_get(QueueName) -> 120 | case mnesia:dirty_read({qsk_queue_registery, QueueName}) of 121 | [QueueRec] -> 122 | {ok, QueueRec}; 123 | [] -> 124 | not_exist 125 | end. 126 | 127 | %%-------------------------------------------------------------------- 128 | %% queue_get_id 129 | %%-------------------------------------------------------------------- 130 | -spec queue_get_id(QueueName :: atom()) -> {ok, QueueID :: atom()} | not_exit. 131 | queue_get_id(QueueName) -> 132 | case queue_get(QueueName) of 133 | {ok, #qsk_queue_registery{queue_id = QueueID}} -> 134 | {ok, QueueID}; 135 | _ -> 136 | not_exist 137 | end. 138 | 139 | %%-------------------------------------------------------------------- 140 | %% queue_make_id 141 | %%------------------------------------------------------------------- 142 | queue_make_id(QueueName) -> 143 | {ok, QueueIDPrefix} = queuesk_utils:get_config(queue_id_prefix), 144 | list_to_atom( 145 | atom_to_list(QueueIDPrefix) 146 | ++ "_" 147 | ++ atom_to_list(QueueName)). 148 | 149 | %%-------------------------------------------------------------------- 150 | %% queue_list 151 | %%-------------------------------------------------------------------- 152 | -spec queue_list() -> [#qsk_queue_registery{}]. 153 | queue_list() -> 154 | mnesia:dirty_select(qsk_queue_registery, [{'_',[],['$_']}]). 155 | 156 | %%=================================================================== 157 | %% Task API 158 | %%=================================================================== 159 | 160 | %%-------------------------------------------------------------------- 161 | %% task_push 162 | %%-------------------------------------------------------------------- 163 | -spec task_push(QueueID :: atom(), Func :: task_func(), Opts :: [task_opts()]) -> ok. 164 | task_push(QueueID, TaskFunc, Opts) -> 165 | 166 | %% @NOTE: 167 | %% TaskFunc must return ok if it's task was done 168 | %% correctly. Otherwise the scheduler will retry it. 169 | 170 | %% @TODO: add api for pushing MFAs (local or external), 171 | %% in addition to FUN terms for efficiency 172 | 173 | {ok, DefaultTaskPriority} = queuesk_utils:get_config(default_task_priority), 174 | {ok, DefaultTaskRetry} = queuesk_utils:get_config(default_task_retry), 175 | {ok, DefaultTaskTimeout} = queuesk_utils:get_config(default_task_timeout), 176 | 177 | TaskPriority = proplists:get_value(priority, Opts, DefaultTaskPriority), 178 | TaskRetry = proplists:get_value(retry, Opts, DefaultTaskRetry), 179 | TaskTimeout = proplists:get_value(timeout, Opts, DefaultTaskTimeout), 180 | 181 | TaskRec = #qsk_queue_record{priority = {TaskPriority, ?NOW_TIMESTAMP}, 182 | retry = TaskRetry, 183 | timeout = TaskTimeout, 184 | task = TaskFunc, 185 | queue_id = QueueID}, 186 | 187 | ok = mnesia:dirty_write(?SPECIFIC_QUEUE_REC(TaskRec)), 188 | ok = queuesk_pool_scheduler:submit_task(QueueID, TaskRec), 189 | ok. 190 | 191 | %%-------------------------------------------------------------------- 192 | %% task_pop 193 | %%-------------------------------------------------------------------- 194 | -spec task_pop(QueueID :: atom()) -> #qsk_queue_record{} | empty. 195 | task_pop(QueueID) -> 196 | Key = mnesia:dirty_first(QueueID), 197 | mnesia:activity( 198 | transaction, 199 | fun() -> 200 | case mnesia:read(QueueID, Key) of 201 | [Rec] -> 202 | ok = mnesia:delete({QueueID, Key}), 203 | Rec; 204 | [] -> 205 | empty 206 | end 207 | end). 208 | 209 | %%-------------------------------------------------------------------- 210 | %% task_peek 211 | %%-------------------------------------------------------------------- 212 | -spec task_peek(QueueID :: atom()) -> #qsk_queue_record{} | empty. 213 | task_peek(QueueID) -> 214 | Key = mnesia:dirty_first(QueueID), 215 | case mnesia:dirty_read(QueueID, Key) of 216 | [Rec] -> 217 | Rec; 218 | [] -> 219 | empy 220 | end. 221 | 222 | %%-------------------------------------------------------------------- 223 | %% task_remove 224 | %%-------------------------------------------------------------------- 225 | -spec task_remove(QueueID :: atom(), Key :: task_id()) -> ok. 226 | task_remove(QueueID, Key) -> 227 | mnesia:dirty_delete({QueueID, Key}). 228 | --------------------------------------------------------------------------------