├── .github └── workflows │ └── erlang.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── bin └── rebar3 ├── include └── swirl.hrl ├── rebar.config ├── rebar.lock ├── src ├── swirl.app.src ├── swirl.erl ├── swirl_code_server.erl ├── swirl_config.erl ├── swirl_ets_manager.erl ├── swirl_flow.erl ├── swirl_mapper.erl ├── swirl_ql.erl ├── swirl_ql_lexer.xrl ├── swirl_ql_parser.yrl ├── swirl_reducer.erl ├── swirl_stream.erl ├── swirl_sup.erl ├── swirl_tracker.erl └── swirl_utils.erl └── test ├── swirl_flow_example.erl ├── swirl_flow_tests.erl ├── swirl_ql_tests.erl └── test.hrl /.github/workflows/erlang.yml: -------------------------------------------------------------------------------- 1 | name: Erlang CI 2 | on: [push] 3 | jobs: 4 | build: 5 | runs-on: ubuntu-latest 6 | strategy: 7 | matrix: 8 | erlang: [24, 25, 26] 9 | 10 | container: 11 | image: erlang:${{ matrix.erlang }} 12 | 13 | steps: 14 | - uses: actions/checkout@v1 15 | - name: Compile 16 | run: make compile 17 | - name: Run xref 18 | run: make xref 19 | - name: Run eunit 20 | run: make eunit 21 | - name: Run dialyzer 22 | run: make dialyzer 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | doc/README.md 2 | erl_crash.dump 3 | fprofx.* 4 | src/swirl_ql_lexer.erl 5 | src/swirl_ql_parser.erl 6 | test/*.beam 7 | _build 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013-2024 Louis-Philippe Gauthier 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | CACHEGRIND=qcachegrind 2 | REBAR3=$(shell which rebar3) 3 | ifeq ($(REBAR3),) 4 | REBAR3=./bin/rebar3 5 | endif 6 | 7 | all: compile 8 | 9 | clean: 10 | @echo "Running rebar3 clean..." 11 | @$(REBAR3) clean -a 12 | 13 | compile: 14 | @echo "Running rebar3 compile..." 15 | @$(REBAR3) as compile compile 16 | 17 | dialyzer: 18 | @echo "Running rebar3 dialyze..." 19 | @$(REBAR3) dialyzer 20 | 21 | edoc: 22 | @echo "Running rebar3 edoc..." 23 | @$(REBAR3) as edoc edoc 24 | 25 | eunit: 26 | @echo "Running rebar3 eunit..." 27 | @$(REBAR3) do eunit -cv, cover -v 28 | 29 | profile: 30 | @echo "Profiling..." 31 | @$(REBAR3) as test compile 32 | @erl -noshell \ 33 | -pa _build/test/lib/*/ebin \ 34 | -eval 'swirl_profile:fprofx()' \ 35 | -eval 'init:stop()' 36 | @_build/test/lib/fprofx/erlgrindx -p fprofx.analysis 37 | @$(CACHEGRIND) fprofx.cgrind 38 | 39 | test: xref eunit dialyzer 40 | 41 | xref: 42 | @echo "Running rebar3 xref..." 43 | @$(REBAR3) xref 44 | 45 | .PHONY: clean compile dialyzer edoc eunit profile xref 46 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # swirl 2 | 3 | High Performance Erlang Stream Processor 4 | 5 | ### Requirements 6 | 7 | * Erlang 22.0 + 8 | 9 | ### Environment variables 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
NameTypeDefaultDescription
mappers_maxpos_integer()100maximum number of mappers
reducers_maxpos_integer()100maximum number of reducers
31 | 32 | ## Examples 33 | 34 | #### Starting a flow 35 | 36 | ```erlang 37 | ok = application:start(swirl), 38 | 39 | FlowMod = swirl_flow_example, 40 | FlowOpts = [ 41 | {stream_names, [delivery]}, 42 | {stream_filter, "exchange_id = 3 AND bidder_id IS NOT NULL"} 43 | ], 44 | MapperNodes = [node()], 45 | ReducerNode = node(), 46 | {ok, Flow} = swirl_flow:start(FlowMod, FlowOpts, MapperNodes, ReducerNode), 47 | 48 | StreamName = delivery, 49 | Event = #{exchange_id => 1, bidder_id => 10}, 50 | 51 | swirl_stream:emit(StreamName, Event), 52 | 53 | ok = swirl_flow:stop(Flow) 54 | ``` 55 | 56 | #### Implementing a flow 57 | 58 | ```erlang 59 | -module(swirl_flow_example). 60 | -include_lib("swirl/include/swirl.hrl"). 61 | 62 | -behavior(swirl_flow). 63 | -export([ 64 | map/3, 65 | reduce/3, 66 | output/4 67 | ]). 68 | 69 | %% swirl_flow callbacks 70 | map(StreamName, Event, _MapperOpts) -> 71 | Type = ?L(type, Event), 72 | ExchangeId = ?L(exchange_id, Event), 73 | BidderId = ?L(bidder_id, Event), 74 | 75 | Key = {Type, StreamName, ExchangeId, BidderId}, 76 | CounterIncrements = {1, 10}, 77 | 78 | {Key, CounterIncrements}. 79 | 80 | reduce(_Flow, Row, _ReducerOpts) -> 81 | Row. 82 | 83 | output(_Flow, _Period, Rows, OutputOpts) -> 84 | %% do something with the output 85 | io:format("rows: ~p~n", [Rows]), 86 | ``` 87 | 88 | #### Stream Filter 89 | 90 | ```erlang 91 | exchange_id = 3 AND bidder_id IS NOT NULL 92 | flight_id in (10, 12, 23) OR tag_id = 20 93 | buyer_id notnull AND seller_id > 103 94 | ``` 95 | 96 | #### Swirl QL 97 | 98 | variables: 99 | 100 | ``` 101 | atom() 102 | ``` 103 | values: 104 | 105 | ``` 106 | integer() | float() | binary() 107 | ``` 108 | boolean operators: 109 | 110 | ``` 111 | 'and' | 'or' 112 | ``` 113 | comparison operators: 114 | 115 | ``` 116 | '<' | '<=' | '=' | '>=' | '>' | '<>' 117 | ``` 118 | inclusion operators: 119 | 120 | ``` 121 | in | notin 122 | ``` 123 | null operators: 124 | 125 | ``` 126 | null | notnull 127 | ``` 128 | 129 | ## TODO 130 | * node discovery 131 | * boolean expression indexing 132 | 133 | ## Tests 134 | 135 | ```makefile 136 | make dialyzer 137 | make elvis 138 | make eunit 139 | make xref 140 | ``` 141 | 142 | ## License 143 | 144 | ```license 145 | The MIT License (MIT) 146 | 147 | Copyright (c) 2013-2024 Louis-Philippe Gauthier 148 | 149 | Permission is hereby granted, free of charge, to any person obtaining a copy of 150 | this software and associated documentation files (the "Software"), to deal in 151 | the Software without restriction, including without limitation the rights to 152 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 153 | the Software, and to permit persons to whom the Software is furnished to do so, 154 | subject to the following conditions: 155 | 156 | The above copyright notice and this permission notice shall be included in all 157 | copies or substantial portions of the Software. 158 | 159 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 160 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 161 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 162 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 163 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 164 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 165 | ``` 166 | -------------------------------------------------------------------------------- /bin/rebar3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lpgauth/swirl/9eeed82163482d923077ebfe1a2f028575c19bd5/bin/rebar3 -------------------------------------------------------------------------------- /include/swirl.hrl: -------------------------------------------------------------------------------- 1 | %% macros 2 | -define(DEFAULT_HEARTBEAT, timer:seconds(5)). 3 | -define(DEFAULT_MAPPERS_MAX, 100). 4 | -define(DEFAULT_MAPPER_WINDOW, timer:seconds(1)). 5 | -define(DEFAULT_REDUCERS_MAX, 100). 6 | -define(DEFAULT_REDUCER_SKIP, false). 7 | -define(DEFAULT_REDUCER_WINDOW, timer:seconds(1)). 8 | -define(DEFAULT_WINDOW_SYNC, false). 9 | 10 | -define(TABLE_NAME_CODE_SERVER, swirl_code_server). 11 | -define(TABLE_NAME_FLOWS, swirl_flows). 12 | -define(TABLE_NAME_MAPPERS, swirl_mappers). 13 | -define(TABLE_NAME_REDUCERS, swirl_reducers). 14 | -define(TABLE_NAME_STREAMS, swirl_streams). 15 | 16 | -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). 17 | -define(L(Key, List), swirl_utils:lookup(Key, List)). 18 | -define(L(Key, List, Default), swirl_utils:lookup(Key, List, Default)). 19 | -define(NULL, undefined). 20 | 21 | %% records 22 | -record(flow, { 23 | id :: binary(), 24 | module :: module(), 25 | module_vsn :: undefined | module_vsn(), 26 | stream_filter :: undefined | string(), 27 | stream_names :: undefined | stream_names(), 28 | mapper_window :: undefined | pos_integer(), 29 | mapper_nodes :: undefined | [node()], 30 | mapper_opts :: mapper_opts(), 31 | reducer_window :: undefined | pos_integer(), 32 | reducer_node :: node(), 33 | reducer_opts :: reducer_opts(), 34 | reducer_skip :: undefined | boolean(), 35 | output_opts :: output_opts(), 36 | heartbeat :: undefined | pos_integer(), 37 | window_sync :: undefined | boolean(), 38 | started_at :: undefined | erlang:timestamp(), 39 | start_node :: node() 40 | }). 41 | 42 | -record(stream, { 43 | flow_id :: binary(), 44 | flow_mod :: module(), 45 | flow_mod_vsn :: module_vsn(), 46 | start_node :: node(), 47 | exp_tree :: undefined | exp_tree(), 48 | mapper_opts :: mapper_opts(), 49 | table_id :: ets:tab() 50 | }). 51 | 52 | -record(period, { 53 | start_at :: pos_integer(), 54 | end_at :: pos_integer() 55 | }). 56 | 57 | %% types 58 | -type flow() :: #flow {}. 59 | -type flow_opts() :: {heartbeat, pos_integer()} | 60 | {mapper_opts, mapper_opts()} | 61 | {mapper_window, pos_integer()} | 62 | {output_opts, output_opts()} | 63 | {reducer_opts, reducer_opts()} | 64 | {reducer_skip, boolean()} | 65 | {reducer_window, pos_integer()} | 66 | {stream_filter, string()} | 67 | {stream_names, stream_names()} | 68 | {window_sync, boolean()}. 69 | -type mapper_opts() :: term(). 70 | -type module_vsn() :: pos_integer(). 71 | -type output_opts() :: term(). 72 | -type period() :: #period {}. 73 | -type reducer_opts() :: term(). 74 | -type row() :: tuple(). 75 | -type stream() :: #stream {}. 76 | -type stream_name() :: atom(). 77 | -type stream_names() :: [stream_name()]. 78 | -type update() :: {tuple(), tuple()}. 79 | 80 | -type event() :: [{atom(), value()}]. 81 | 82 | -type boolean_op() :: 'and' | 'or'. 83 | -type comparison_op() :: '<' | '<=' | '=' | '>=' | '>' | '<>'. 84 | -type inclusion_op() :: in | notin. 85 | -type null_op() :: null | notnull. 86 | -type variable() :: atom(). 87 | -type value() :: integer() | float() | binary(). 88 | 89 | -type exp_tree() :: {boolean_op(), exp_tree(), exp_tree()} | 90 | {comparison_op(), variable(), value()} | 91 | {inclusion_op(), variable(), [value(), ...]} | 92 | {null_op(), variable()}. 93 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | {cover_export_enabled, true}. 2 | {cover_excl_mods, []}. 3 | 4 | {deps, [ 5 | {uuid, ".*", 6 | {git, "https://github.com/okeuday/uuid.git", {branch, "master"}}} 7 | ]}. 8 | 9 | {edoc_opts, [ 10 | {app_default, "http://www.erlang.org/doc/man"}, 11 | {doclet, edown_doclet}, 12 | {image, ""}, 13 | {includes, ["include"]}, 14 | {preprocess, true}, 15 | {stylesheet, ""}, 16 | {title, "swirl"} 17 | ]}. 18 | 19 | {erl_opts, [ 20 | debug_info 21 | ]}. 22 | 23 | {profiles, [ 24 | {compile, [ 25 | {erl_opts, [ 26 | warnings_as_errors, 27 | warn_export_all, 28 | warn_export_vars, 29 | % warn_missing_spec, 30 | warn_obsolete_guard, 31 | warn_shadow_vars, 32 | % warn_untyped_record, 33 | warn_unused_import, 34 | warn_unused_vars 35 | ]} 36 | ]}, 37 | {edoc, [ 38 | {deps, [ 39 | {edown, 40 | {git, "https://github.com/uwiger/edown.git", {tag, "0.8.4"}}} 41 | ]} 42 | ]}, 43 | {test, [ 44 | {deps, [ 45 | {fprofx, 46 | {git, "https://github.com/ransomr/fprofx.git", {branch, "master"}}} 47 | ]}, 48 | {src_dirs, ["src", "test"]} 49 | ]} 50 | ]}. 51 | 52 | {xref_checks, [ 53 | deprecated_functions, 54 | deprecated_function_calls, 55 | locals_not_used, 56 | undefined_functions, 57 | undefined_function_calls 58 | ]}. 59 | 60 | {xref_ignores, [{swirl_ql_parser, return_error, 2}]}. 61 | -------------------------------------------------------------------------------- /rebar.lock: -------------------------------------------------------------------------------- 1 | [{<<"quickrand">>, 2 | {git,"https://github.com/okeuday/quickrand.git", 3 | {ref,"65332de501998764f437c3ffe05d744f582d7622"}}, 4 | 1}, 5 | {<<"uuid">>, 6 | {git,"https://github.com/okeuday/uuid.git", 7 | {ref,"63e32cdad70693495163ab131456905e827a5e36"}}, 8 | 0}]. 9 | -------------------------------------------------------------------------------- /src/swirl.app.src: -------------------------------------------------------------------------------- 1 | {application, swirl, [ 2 | {description, "stream processor"}, 3 | {vsn, "0.2.5"}, 4 | {registered, [ 5 | swirl_ets_manager, 6 | swirl_sup, 7 | swirl_tracker 8 | ]}, 9 | {applications, [ 10 | kernel, 11 | stdlib, 12 | uuid 13 | ]}, 14 | {mod, {swirl, []}}, 15 | {env, []} 16 | ]}. 17 | -------------------------------------------------------------------------------- /src/swirl.erl: -------------------------------------------------------------------------------- 1 | -module(swirl). 2 | 3 | %% public 4 | -export([ 5 | start/0 6 | ]). 7 | 8 | -behaviour(application). 9 | -export([ 10 | start/2, 11 | stop/1 12 | ]). 13 | 14 | %% public 15 | -spec start() -> ok. 16 | start() -> 17 | {ok, _Apps} = application:ensure_all_started(?MODULE), 18 | ok. 19 | 20 | %% application callbacks 21 | start(_StartType, _StartArgs) -> 22 | swirl_sup:start_link(). 23 | 24 | stop(_State) -> 25 | ok. 26 | -------------------------------------------------------------------------------- /src/swirl_code_server.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_code_server). 2 | -include("swirl.hrl"). 3 | 4 | %% public 5 | -export([ 6 | get_module/3, 7 | start_link/0, 8 | version/1 9 | ]). 10 | 11 | -behaviour(gen_server). 12 | -export([ 13 | init/1, 14 | handle_call/3, 15 | handle_cast/2, 16 | handle_info/2, 17 | terminate/2, 18 | code_change/3 19 | ]). 20 | 21 | -define(TABLE_OPTS, [public, named_table, {write_concurrency, true}]). 22 | -define(SERVER, ?MODULE). 23 | 24 | -record(state, {}). 25 | 26 | %% public 27 | -spec get_module(node(), module(), module_vsn()) -> ok. 28 | get_module(Node, Module, ModuleVsn) -> 29 | KeyVal = {{Module, ModuleVsn}, true}, 30 | case ets:insert_new(?TABLE_NAME_CODE_SERVER, KeyVal) of 31 | true -> 32 | message(Node, {get_module, Module, ModuleVsn, node()}); 33 | false -> ok 34 | end. 35 | 36 | -spec start_link() -> {ok, pid()}. 37 | start_link() -> 38 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). 39 | 40 | -spec version(module()) -> {ok, module_vsn()} | {error, term()}. 41 | version(Module) -> 42 | try Module:module_info(attributes) of 43 | Attributes -> 44 | [ModuleVsn] = ?L(vsn, Attributes), 45 | {ok, ModuleVsn} 46 | catch 47 | error:undef -> 48 | {error, undef} 49 | end. 50 | 51 | %% gen_server callbacks 52 | init([]) -> 53 | process_flag(trap_exit, true), 54 | swirl_ets_manager:table(?TABLE_NAME_CODE_SERVER, ?TABLE_OPTS, ?SERVER), 55 | {ok, #state {}}. 56 | 57 | handle_call(Request, _From, State) -> 58 | io:format("unexpected message: ~p~n", [Request]), 59 | {reply, ok, State}. 60 | 61 | handle_cast(Msg, State) -> 62 | io:format("unexpected message: ~p~n", [Msg]), 63 | {noreply, State}. 64 | 65 | handle_info({'ETS-TRANSFER', _TableId, _Pid, _Data}, State) -> 66 | {noreply, State}; 67 | handle_info({get_module, Module, ModuleVsn, Node}, State) -> 68 | {ok, ModuleVsn} = version(Module), 69 | {Module, Binary, Filename} = code:get_object_code(Module), 70 | message(Node, {load_module, Module, Binary, Filename}), 71 | {noreply, State}; 72 | handle_info({load_module, Module, Binary, Filename}, State) -> 73 | {module, Module} = code:load_binary(Module, Filename, Binary), 74 | {noreply, State}; 75 | handle_info(Info, State) -> 76 | io:format("unexpected message: ~p~n", [Info]), 77 | {noreply, State}. 78 | 79 | terminate(_Reason, _State) -> 80 | ok. 81 | 82 | code_change(_OldVsn, State, _Extra) -> 83 | {ok, State}. 84 | 85 | %% private 86 | message(Node, Msg) -> 87 | {?SERVER, Node} ! Msg, 88 | ok. 89 | -------------------------------------------------------------------------------- /src/swirl_config.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_config). 2 | -include("swirl.hrl"). 3 | 4 | %% public 5 | -export([ 6 | flows/0, 7 | flows_count/0, 8 | mappers/0, 9 | mappers_count/0, 10 | mappers_max/0, 11 | reducers/0, 12 | reducers_count/0, 13 | reducers_max/0 14 | ]). 15 | 16 | %% public 17 | -spec flows() -> list(tuple()). 18 | flows() -> 19 | swirl_utils:tab2list(?TABLE_NAME_FLOWS). 20 | 21 | -spec flows_count() -> non_neg_integer(). 22 | flows_count() -> 23 | select_count_all(?TABLE_NAME_FLOWS). 24 | 25 | -spec mappers() -> list(tuple()). 26 | mappers() -> 27 | swirl_utils:tab2list(?TABLE_NAME_MAPPERS). 28 | 29 | -spec mappers_count() -> non_neg_integer(). 30 | mappers_count() -> 31 | select_count_all(?TABLE_NAME_MAPPERS). 32 | 33 | -spec mappers_max() -> non_neg_integer(). 34 | mappers_max() -> 35 | application:get_env(swirl, mappers_max, ?DEFAULT_MAPPERS_MAX). 36 | 37 | -spec reducers() -> list(tuple()). 38 | reducers() -> 39 | swirl_utils:tab2list(?TABLE_NAME_REDUCERS). 40 | 41 | -spec reducers_count() -> non_neg_integer(). 42 | reducers_count() -> 43 | select_count_all(?TABLE_NAME_REDUCERS). 44 | 45 | -spec reducers_max() -> non_neg_integer(). 46 | reducers_max() -> 47 | application:get_env(swirl, reducers_max, ?DEFAULT_REDUCERS_MAX). 48 | 49 | %% private 50 | select_count_all(TableId) -> 51 | ets:select_count(TableId, [{'_', [], [true]}]). 52 | -------------------------------------------------------------------------------- /src/swirl_ets_manager.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_ets_manager). 2 | -include("swirl.hrl"). 3 | 4 | %% internal 5 | -export([ 6 | new_table/3, 7 | start_link/0, 8 | table/3 9 | ]). 10 | 11 | -behaviour(gen_server). 12 | -export([ 13 | init/1, 14 | handle_call/3, 15 | handle_cast/2, 16 | handle_info/2, 17 | terminate/2, 18 | code_change/3 19 | ]). 20 | 21 | -define(SERVER, ?MODULE). 22 | 23 | -record(state, { 24 | tables 25 | }). 26 | 27 | %% internal 28 | -spec new_table(atom(), list(atom() | tuple()), atom() | pid()) -> ok. 29 | new_table(Name, Options, Server) -> 30 | gen_server:cast(?SERVER, {new_table, {Name, Options, Server}}). 31 | 32 | -spec start_link() -> {ok, pid()}. 33 | start_link() -> 34 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). 35 | 36 | -spec table(atom(), list(atom() | tuple()), atom() | pid()) -> ok. 37 | table(Name, Options, Server) -> 38 | gen_server:cast(?SERVER, {table, {Name, Options, Server}}). 39 | 40 | %% gen_server callbacks 41 | init([]) -> 42 | {ok, #state { 43 | tables = dict:new() 44 | }}. 45 | 46 | handle_call(Request, _From, State) -> 47 | io:format("unexpected message: ~p~n", [Request]), 48 | {reply, ok, State}. 49 | 50 | handle_cast({new_table, {Name, Options, Server} = Data}, #state { 51 | tables = Tables 52 | } = State) -> 53 | 54 | TableId = ets_new(Name, Options, Data), 55 | Tables2 = dict:erase({Server, Name}, Tables), 56 | Tables3 = dict:append({Server, Name}, TableId, Tables2), 57 | ets_give_away(Server, TableId, Data), 58 | 59 | {noreply, State#state { 60 | tables = Tables3 61 | }}; 62 | handle_cast({table, {Name, Options, Server} = Data}, #state { 63 | tables = Tables 64 | } = State) -> 65 | 66 | TableId = 67 | case swirl_utils:safe_dict_fetch({Server, Name}, Tables) of 68 | undefined -> ets_new(Name, Options, Data); 69 | [TableId2] -> TableId2 70 | end, 71 | Tables2 = dict:erase({Server, Name}, Tables), 72 | Tables3 = dict:append({Server, Name}, TableId, Tables2), 73 | ets_give_away(Server, TableId, Data), 74 | 75 | {noreply, State#state { 76 | tables = Tables3 77 | }}; 78 | handle_cast(Msg, State) -> 79 | io:format("unexpected message: ~p~n", [Msg]), 80 | {noreply, State}. 81 | 82 | handle_info({'ETS-TRANSFER', _TableId, _Pid, _Data}, State) -> 83 | {noreply, State}; 84 | handle_info(Msg, State) -> 85 | io:format("unexpected message: ~p~n", [Msg]), 86 | {noreply, State}. 87 | 88 | terminate(_Reason, _State) -> 89 | ok. 90 | 91 | code_change(_OldVsn, State, _Extra) -> 92 | {ok, State}. 93 | 94 | %% private 95 | ets_new(Name, Options, Data) -> 96 | TableId = ets:new(Name, Options), 97 | true = ets:setopts(TableId, {heir, self(), Data}), 98 | TableId. 99 | 100 | ets_give_away(Server, TableId, Data) -> 101 | ServerPid = server_pid(Server), 102 | ets:give_away(TableId, ServerPid, Data). 103 | 104 | server_pid(Server) when is_pid(Server) -> 105 | Server; 106 | server_pid(Server) when is_atom(Server) -> 107 | whereis(Server). 108 | -------------------------------------------------------------------------------- /src/swirl_flow.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_flow). 2 | -include("swirl.hrl"). 3 | 4 | %% public 5 | -export([ 6 | start/4, 7 | stop/1 8 | ]). 9 | 10 | %% inernal 11 | -export([ 12 | lookup/1, 13 | register/1, 14 | unregister/1 15 | ]). 16 | 17 | %% callback 18 | -callback map(stream_name(), event(), mapper_opts()) -> 19 | list(update()) | update() | ignore. 20 | 21 | -callback reduce(flow(), row(), reducer_opts()) -> 22 | update() | ignore. 23 | 24 | -callback output(flow(), period(), list(row()), output_opts()) -> 25 | ok. 26 | 27 | %% public 28 | -spec start(atom(), [flow_opts()], [node()], node()) -> 29 | {ok, flow()} | {error, flow_mod_undef | {bad_flow_opts, list()}}. 30 | 31 | start(FlowMod, FlowOpts, MapperNodes, ReducerNode) -> 32 | case flow(FlowMod, FlowOpts, MapperNodes, ReducerNode) of 33 | {ok, Flow} -> 34 | ok = swirl_tracker:start_reducer(Flow), 35 | ok = swirl_tracker:start_mappers(Flow), 36 | {ok, Flow}; 37 | {error, Reason} -> 38 | {error, Reason} 39 | end. 40 | 41 | -spec stop(flow()) -> 42 | ok. 43 | 44 | stop(#flow {} = Flow) -> 45 | ok = swirl_tracker:stop_mappers(Flow), 46 | ok = swirl_tracker:stop_reducer(Flow), 47 | ok. 48 | 49 | %% internal 50 | -spec lookup(binary() | flow()) -> 51 | undefined | flow(). 52 | 53 | lookup(FlowId) when is_binary(FlowId) -> 54 | lookup(#flow {id = FlowId}); 55 | lookup(#flow {} = Flow) -> 56 | swirl_tracker:lookup(?TABLE_NAME_FLOWS, key(Flow)). 57 | 58 | -spec register(flow()) -> 59 | true. 60 | 61 | register(#flow {} = Flow) -> 62 | swirl_tracker:register(?TABLE_NAME_FLOWS, key(Flow), Flow). 63 | 64 | -spec unregister(flow()) -> 65 | true. 66 | 67 | unregister(#flow {} = Flow) -> 68 | swirl_tracker:unregister(?TABLE_NAME_FLOWS, key(Flow)). 69 | 70 | %% private 71 | flow(Module, Options, MapperNodes, ReducerNode) -> 72 | case swirl_code_server:version(Module) of 73 | {ok, ModuleVsn} -> 74 | case verify_options(Options) of 75 | ok -> 76 | {ok, new_flow_rec(Module, ModuleVsn, Options, 77 | MapperNodes, ReducerNode)}; 78 | {error, Reason} -> 79 | {error, Reason} 80 | end; 81 | {error, undef} -> 82 | {error, flow_mod_undef} 83 | end. 84 | 85 | key(#flow {id = Id}) -> Id. 86 | 87 | new_flow_rec(Module, ModuleVsn, Options, MapperNodes, 88 | ReducerNode) -> 89 | 90 | #flow { 91 | id = swirl_utils:uuid(), 92 | module = Module, 93 | module_vsn = ModuleVsn, 94 | start_node = node(), 95 | heartbeat = ?L(heartbeat, Options, ?DEFAULT_HEARTBEAT), 96 | window_sync = ?L(window_sync, Options, ?DEFAULT_WINDOW_SYNC), 97 | mapper_window = ?L(mapper_window, Options, ?DEFAULT_MAPPER_WINDOW), 98 | mapper_nodes = MapperNodes, 99 | mapper_opts = ?L(mapper_opts, Options, []), 100 | reducer_window = ?L(reducer_window, Options, ?DEFAULT_REDUCER_WINDOW), 101 | reducer_node = ReducerNode, 102 | reducer_opts = ?L(reducer_opts, Options, []), 103 | reducer_skip = ?L(reducer_skip, Options, ?DEFAULT_REDUCER_SKIP), 104 | output_opts = ?L(output_opts, Options, []), 105 | stream_filter = ?L(stream_filter, Options), 106 | stream_names = ?L(stream_names, Options, []), 107 | started_at = os:timestamp() 108 | }. 109 | 110 | verify_options(FlowOpts) -> 111 | verify_options(FlowOpts, []). 112 | 113 | verify_options([{heartbeat, Heartbeat} | Options], Errors) 114 | when is_integer(Heartbeat) -> 115 | verify_options(Options, Errors); 116 | verify_options([{mapper_opts, _} | Options], Errors) -> 117 | verify_options(Options, Errors); 118 | verify_options([{mapper_window, MapperWindow} | Options], Errors) 119 | when is_integer(MapperWindow) -> 120 | verify_options(Options, Errors); 121 | verify_options([{output_opts, _} | Options], Errors) -> 122 | verify_options(Options, Errors); 123 | verify_options([{reducer_opts, _} | Options], Errors) -> 124 | verify_options(Options, Errors); 125 | verify_options([{reducer_skip, ReducerSkip} | Options], Errors) 126 | when is_boolean(ReducerSkip) -> 127 | verify_options(Options, Errors); 128 | verify_options([{reducer_window, ReducerWindow} | Options], Errors) 129 | when is_integer(ReducerWindow) -> 130 | verify_options(Options, Errors); 131 | verify_options([{stream_filter, undefined} | Options], Errors) -> 132 | verify_options(Options, Errors); 133 | verify_options([{stream_filter, StreamFilter} = Option | Options], Errors) -> 134 | case swirl_ql:parse(StreamFilter) of 135 | {ok, _ExpTree} -> 136 | verify_options(Options, Errors); 137 | {error, _Reason} -> 138 | verify_options(Options, [Option | Errors]) 139 | end; 140 | verify_options([{stream_names, StreamNames} | Options], Errors) 141 | when is_list(StreamNames)-> 142 | verify_options(Options, Errors); 143 | verify_options([{window_sync, WindowSync} | Options], Errors) 144 | when is_boolean(WindowSync) -> 145 | verify_options(Options, Errors); 146 | verify_options([Option | Options], Errors) -> 147 | verify_options(Options, [Option | Errors]); 148 | verify_options([], []) -> 149 | ok; 150 | verify_options([], Errors) -> 151 | {error, {bad_flow_opts, Errors}}. 152 | -------------------------------------------------------------------------------- /src/swirl_mapper.erl: -------------------------------------------------------------------------------- 1 | %% TODO: add module vsn check to auto-refresh old modules from the code server 2 | 3 | -module(swirl_mapper). 4 | -include("swirl.hrl"). 5 | 6 | -compile({no_auto_import, [ 7 | unregister/1 8 | ]}). 9 | 10 | %% internal 11 | -export([ 12 | lookup/1, 13 | map/3, 14 | register/1, 15 | start/1, 16 | unregister/1 17 | ]). 18 | 19 | -behaviour(gen_server). 20 | -export([ 21 | init/1, 22 | handle_call/3, 23 | handle_cast/2, 24 | handle_info/2, 25 | terminate/2, 26 | code_change/3 27 | ]). 28 | 29 | -define(TABLE_NAME, mapper_rows). 30 | -define(TABLE_OPTS, [public, {write_concurrency, true}]). 31 | -define(SERVER, ?MODULE). 32 | 33 | -record(state, { 34 | flow, 35 | table_id, 36 | window_timer, 37 | window_tstamp, 38 | hbeat_timer, 39 | hbeat_tstamp 40 | }). 41 | 42 | %% internal 43 | -spec lookup(binary() | flow()) -> undefined | pid. 44 | lookup(FlowId) when is_binary(FlowId) -> 45 | lookup(#flow {id = FlowId}); 46 | lookup(#flow {} = Flow) -> 47 | swirl_tracker:lookup(?TABLE_NAME_MAPPERS, key(Flow)). 48 | 49 | -spec map(atom(), event(), stream()) -> ok. 50 | map(StreamName, Event, #stream { 51 | flow_mod = FlowMod, 52 | flow_mod_vsn = FlowModVsn, 53 | start_node = StartNode, 54 | mapper_opts = MapperOpts, 55 | table_id = TableId 56 | }) -> 57 | 58 | try FlowMod:map(StreamName, Event, MapperOpts) of 59 | Updates when is_list(Updates) -> 60 | [update(TableId, Key, Counters) || {Key, Counters} <- Updates]; 61 | {Key, Counters} -> 62 | update(TableId, Key, Counters); 63 | ignore -> 64 | ok 65 | catch 66 | error:undef -> 67 | swirl_code_server:get_module(StartNode, FlowMod, FlowModVsn) 68 | end. 69 | 70 | -spec register(flow()) -> true. 71 | register(#flow {} = Flow) -> 72 | swirl_tracker:register(?TABLE_NAME_MAPPERS, key(Flow), self()). 73 | 74 | -spec start(flow()) -> {ok, pid()} | {error, mappers_max}. 75 | start(#flow {} = Flow) -> 76 | MappersCount = swirl_config:mappers_count(), 77 | MappersMax = swirl_config:mappers_max(), 78 | case lookup(Flow) of 79 | undefined when MappersCount < MappersMax -> 80 | start_link(Flow); 81 | _Else -> ok 82 | end. 83 | 84 | -spec unregister(flow()) -> true. 85 | unregister(#flow {} = Flow) -> 86 | swirl_tracker:unregister(?TABLE_NAME_MAPPERS, key(Flow)). 87 | 88 | %% gen_server callbacks 89 | init(#flow {} = Flow) -> 90 | process_flag(trap_exit, true), 91 | register(Flow), 92 | swirl_flow:register(Flow), 93 | 94 | self() ! flush, 95 | self() ! heartbeat, 96 | 97 | {ok, #state { 98 | flow = Flow, 99 | hbeat_tstamp = swirl_utils:unix_tstamp_ms() 100 | }}. 101 | 102 | handle_call(Request, _From, State) -> 103 | io:format("unexpected message: ~p~n", [Request]), 104 | {reply, ok, State}. 105 | 106 | handle_cast(Msg, State) -> 107 | io:format("unexpected message: ~p~n", [Msg]), 108 | {noreply, State}. 109 | 110 | handle_info(flush, #state { 111 | flow = #flow { 112 | mapper_window = Window, 113 | window_sync = Sync 114 | } = Flow, 115 | table_id = TableId, 116 | window_tstamp = Tstamp 117 | } = State) -> 118 | 119 | Tstamp2 = swirl_utils:unix_tstamp_ms(), 120 | WindowTimer = swirl_utils:new_timer(Window, flush, Sync), 121 | NewTableId = ets:new(?TABLE_NAME, ?TABLE_OPTS), 122 | swirl_stream:register(Flow, NewTableId), 123 | 124 | Period = #period {start_at = Tstamp, end_at = Tstamp2}, 125 | spawn(fun() -> flush_window(Flow, Period, TableId) end), 126 | 127 | {noreply, State#state { 128 | table_id = NewTableId, 129 | window_tstamp = Tstamp2, 130 | window_timer = WindowTimer 131 | }}; 132 | handle_info(heartbeat, #state { 133 | flow = #flow { 134 | heartbeat = Hbeat 135 | }, 136 | hbeat_tstamp = Tstamp 137 | } = State) -> 138 | 139 | Tstamp2 = swirl_utils:unix_tstamp_ms(), 140 | HbeatTimer = swirl_utils:new_timer(Hbeat, heartbeat, true), 141 | Delta = Tstamp2 - Tstamp, 142 | 143 | case Delta > 2 * Hbeat of 144 | true -> 145 | {stop, normal, State}; 146 | false -> 147 | {noreply, State#state { 148 | hbeat_timer = HbeatTimer 149 | }} 150 | end; 151 | handle_info({ping, ReducerNode}, #state { 152 | flow = #flow { 153 | id = FlowId, 154 | reducer_node = ReducerNode 155 | } 156 | } = State) -> 157 | 158 | Msg = {pong, node()}, 159 | swirl_tracker:message(ReducerNode, FlowId, Msg), 160 | 161 | {noreply, State#state { 162 | hbeat_tstamp = swirl_utils:unix_tstamp_ms() 163 | }}; 164 | handle_info(stop, State) -> 165 | {stop, normal, State}; 166 | handle_info(Msg, State) -> 167 | io:format("unexpected message: ~p~n", [Msg]), 168 | {noreply, State}. 169 | 170 | terminate(_Reason, #state { 171 | flow = Flow, 172 | window_timer = WindowTimer, 173 | hbeat_timer = HbeatTimer 174 | }) -> 175 | 176 | swirl_stream:unregister(Flow), 177 | swirl_flow:unregister(Flow), 178 | unregister(Flow), 179 | timer:cancel(WindowTimer), 180 | timer:cancel(HbeatTimer), 181 | ok. 182 | 183 | code_change(_OldVsn, State, _Extra) -> 184 | {ok, State}. 185 | 186 | %% private 187 | flush_window(_Flow, _Period, undefined) -> 188 | ok; 189 | flush_window(#flow { 190 | id = FlowId, 191 | reducer_node = ReducerNode 192 | }, Period, TableId) -> 193 | 194 | Rows = swirl_utils:tab2list(TableId), 195 | Msg = {mapper_window, Period, Rows}, 196 | swirl_tracker:message(ReducerNode, FlowId, Msg), 197 | swirl_utils:safe_ets_delete(TableId). 198 | 199 | key(#flow {id = Id}) -> Id. 200 | 201 | start_link(Flow) -> 202 | gen_server:start_link(?MODULE, Flow, []). 203 | 204 | -spec update(ets:tab(), tuple(), tuple()) -> ok. 205 | update(TableId, Key, Counters) -> 206 | swirl_utils:safe_ets_increment(TableId, Key, Counters). 207 | -------------------------------------------------------------------------------- /src/swirl_ql.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_ql). 2 | -include("swirl.hrl"). 3 | -compile([native]). 4 | 5 | -export([ 6 | evaluate/2, 7 | parse/1 8 | ]). 9 | 10 | %% public 11 | -spec evaluate(exp_tree(), event()) -> boolean(). 12 | evaluate({'and', A, B}, Vars) -> 13 | evaluate(A, Vars) andalso evaluate(B, Vars); 14 | evaluate({'or', A, B}, Vars) -> 15 | evaluate(A, Vars) orelse evaluate(B, Vars); 16 | evaluate({comp, Comparator, Var, Value}, Vars) -> 17 | compare(Comparator, ?L(Var, Vars), Value); 18 | evaluate({in, Var, List}, Vars) -> 19 | lists:member(?L(Var, Vars), List); 20 | evaluate({notin, Var, List}, Vars) -> 21 | not lists:member(?L(Var, Vars), List); 22 | evaluate({in_var, Item, Var}, Vars) -> 23 | lists:member(Item, ?L(Var, Vars)); 24 | evaluate({notin_var, Item, Var}, Vars) -> 25 | not lists:member(Item, ?L(Var, Vars)); 26 | evaluate({null, Var}, Vars) -> 27 | ?L(Var, Vars) =:= ?NULL; 28 | evaluate({notnull, Var}, Vars) -> 29 | ?L(Var, Vars) =/= ?NULL. 30 | 31 | -spec parse(string() | binary()) -> {ok, exp_tree()} | {error, term()}. 32 | parse(String) when is_binary(String) -> 33 | parse(binary_to_list(String)); 34 | parse(String) when is_list(String) -> 35 | case swirl_ql_lexer:string(String) of 36 | {ok, Tokens, _} -> 37 | case swirl_ql_parser:parse(Tokens) of 38 | {ok, ExpTree} -> {ok, ExpTree}; 39 | {error, Reason} -> {error, Reason} 40 | end; 41 | {error, Reason, _} -> {error, Reason} 42 | end. 43 | 44 | %% private 45 | compare(_Comp, undefined, _B) -> 46 | false; 47 | compare('<', A, B) when is_number(A) and 48 | is_number(B) -> 49 | A < B; 50 | compare('<=', A, B) when is_number(A) and 51 | is_number(B) -> 52 | A =< B; 53 | compare('=', A, B) when is_number(A) or 54 | is_binary(A) and 55 | is_number(B) or 56 | is_binary(B) -> 57 | A == B; 58 | compare('>=', A, B) when is_number(A) and 59 | is_number(B) -> 60 | A >= B; 61 | compare('>', A, B) when is_number(A) and 62 | is_number(B) -> 63 | A > B; 64 | compare('<>', A, B) when is_number(A) or 65 | is_binary(A) and 66 | is_number(B) or 67 | is_binary(B) -> 68 | A /= B. 69 | -------------------------------------------------------------------------------- /src/swirl_ql_lexer.xrl: -------------------------------------------------------------------------------- 1 | Definitions. 2 | 3 | D = [0-9] 4 | L = [A-Za-z_][A-Za-z0-9_.]* 5 | WS = ([\000-\s]|%.*) 6 | C = (<|<=|=|>=|>|<>) 7 | P = [(),] 8 | 9 | Rules. 10 | 11 | {C} : {token, {comp, TokenLine, atom(TokenChars)}}. 12 | {D}+\.{D}+? : {token, {float, TokenLine, list_to_float(TokenChars)}}. 13 | {D}+ : {token, {int, TokenLine, list_to_integer(TokenChars)}}. 14 | {P} : {token, {atom(TokenChars), TokenLine}}. 15 | '[^']*' : {token, {string, TokenLine, quoted_bitstring(TokenChars, TokenLen)}}. 16 | "[^"]*" : {token, {string, TokenLine, quoted_bitstring(TokenChars, TokenLen)}}. 17 | {L}+ : word(TokenLine, TokenChars). 18 | {WS}+ : skip_token. 19 | 20 | Erlang code. 21 | 22 | -dialyzer({nowarn_function, yyrev/2}). 23 | 24 | atom(TokenChars) -> 25 | list_to_atom(string:to_lower(TokenChars)). 26 | 27 | quoted_bitstring(TokenChars, TokenLen) -> 28 | list_to_bitstring(lists:sublist(TokenChars, 2, TokenLen - 2)). 29 | 30 | reserved_word('and') -> true; 31 | reserved_word('in') -> true; 32 | reserved_word('is') -> true; 33 | reserved_word('not') -> true; 34 | reserved_word('null') -> true; 35 | reserved_word('or') -> true; 36 | reserved_word(_) -> false. 37 | 38 | word(TokenLine, TokenChars) -> 39 | Word = atom(TokenChars), 40 | case reserved_word(Word) of 41 | true -> {token, {Word, TokenLine}}; 42 | false -> {token, {var, TokenLine, Word}} 43 | end. 44 | -------------------------------------------------------------------------------- /src/swirl_ql_parser.yrl: -------------------------------------------------------------------------------- 1 | Nonterminals where_clause search_cond search_cond2 search_cond3 search_cond4 predicate comparsion_pred in_pred 2 | test_for_null_pred atom_commalist scalar_exp column_ref atom literal. 3 | 4 | Terminals comp int float '(' ')' ',' string 'and' in is 'not' null 'or' var. 5 | 6 | Rootsymbol where_clause. 7 | 8 | where_clause -> search_cond : '$1'. 9 | 10 | search_cond -> search_cond2 'or' search_cond : {'or', '$1', '$3'}. 11 | search_cond -> search_cond2 : '$1'. 12 | 13 | search_cond2 -> search_cond3 'and' search_cond2 : {'and', '$1', '$3'}. 14 | search_cond2 -> search_cond3 : '$1'. 15 | 16 | search_cond3 -> '(' search_cond ')' : '$2'. 17 | search_cond3 -> search_cond4 : '$1'. 18 | 19 | search_cond4 -> predicate : '$1'. 20 | 21 | predicate -> comparsion_pred : '$1'. 22 | predicate -> in_pred : '$1'. 23 | predicate -> test_for_null_pred : '$1'. 24 | 25 | comparsion_pred -> scalar_exp comp scalar_exp : {comp, value('$2'), '$1', '$3'}. 26 | 27 | in_pred -> scalar_exp 'not' in '(' atom_commalist ')' : {notin, '$1', '$5'}. 28 | in_pred -> scalar_exp in '(' atom_commalist ')' : {in, '$1', '$4'}. 29 | in_pred -> scalar_exp 'not' in column_ref : {notin_var, '$1', '$4'}. 30 | in_pred -> scalar_exp in column_ref : {in_var, '$1', '$3'}. 31 | 32 | test_for_null_pred -> column_ref is not null : {notnull, '$1'}. 33 | test_for_null_pred -> column_ref is null : {null, '$1'}. 34 | 35 | scalar_exp -> atom : '$1'. 36 | scalar_exp -> column_ref : '$1'. 37 | scalar_exp -> '(' scalar_exp ')' : '$2'. 38 | 39 | atom -> literal : '$1'. 40 | 41 | column_ref -> var : value('$1'). 42 | 43 | literal -> int : value('$1'). 44 | literal -> float : value('$1'). 45 | literal -> string : value('$1'). 46 | 47 | atom_commalist -> atom : ['$1']. 48 | atom_commalist -> atom_commalist ',' atom : flatten(['$1', '$3']). 49 | 50 | Erlang code. 51 | 52 | flatten(List) -> lists:flatten(List). 53 | value({_, _, Value}) -> Value. 54 | -------------------------------------------------------------------------------- /src/swirl_reducer.erl: -------------------------------------------------------------------------------- 1 | %% TODO: add module vsn check to auto-refresh old modules from the code server 2 | 3 | -module(swirl_reducer). 4 | -include("swirl.hrl"). 5 | 6 | -compile({no_auto_import, [ 7 | unregister/1 8 | ]}). 9 | 10 | %% internal 11 | -export([ 12 | lookup/1, 13 | register/1, 14 | start/1, 15 | unregister/1 16 | ]). 17 | 18 | -behaviour(gen_server). 19 | -export([ 20 | init/1, 21 | handle_call/3, 22 | handle_cast/2, 23 | handle_info/2, 24 | terminate/2, 25 | code_change/3 26 | ]). 27 | 28 | -define(TABLE_NAME, reducer_rows). 29 | -define(TABLE_OPTS, [public, {write_concurrency, true}]). 30 | -define(SERVER, ?MODULE). 31 | 32 | -record(state, { 33 | flow, 34 | table_id, 35 | window_timer, 36 | window_tstamp, 37 | hbeat_timer, 38 | hbeat_nodes 39 | }). 40 | 41 | %% internal 42 | -spec lookup(binary() | flow()) -> undefined | pid(). 43 | lookup(FlowId) when is_binary(FlowId) -> 44 | lookup(#flow {id = FlowId}); 45 | lookup(#flow {} = Flow) -> 46 | swirl_tracker:lookup(?TABLE_NAME_REDUCERS, key(Flow)). 47 | 48 | -spec register(flow()) -> true. 49 | register(#flow {} = Flow) -> 50 | swirl_tracker:register(?TABLE_NAME_REDUCERS, key(Flow), self()). 51 | 52 | -spec start(flow()) -> {ok, pid()} | {error, reducers_max}. 53 | start(#flow {} = Flow) -> 54 | ReducersCount = swirl_config:reducers_count(), 55 | ReducersMax = swirl_config:reducers_max(), 56 | case lookup(Flow) of 57 | undefined when ReducersCount < ReducersMax -> 58 | start_link(Flow); 59 | _Else -> 60 | {error, reducers_max} 61 | end. 62 | 63 | -spec unregister(flow()) -> true. 64 | unregister(#flow {} = Flow) -> 65 | swirl_tracker:unregister(?TABLE_NAME_REDUCERS, key(Flow)). 66 | 67 | %% gen_server callbacks 68 | init(#flow {mapper_nodes = MapperNodes} = Flow) -> 69 | process_flag(trap_exit, true), 70 | register(Flow), 71 | swirl_flow:register(Flow), 72 | 73 | self() ! flush, 74 | self() ! heartbeat, 75 | 76 | {ok, #state { 77 | flow = Flow, 78 | hbeat_nodes = MapperNodes 79 | }}. 80 | handle_call(Request, _From, State) -> 81 | io:format("unexpected message: ~p~n", [Request]), 82 | {reply, ok, State}. 83 | 84 | handle_cast(Msg, State) -> 85 | io:format("unexpected message: ~p~n", [Msg]), 86 | {noreply, State}. 87 | 88 | handle_info(flush, #state { 89 | flow = #flow { 90 | reducer_window = Window, 91 | window_sync = Sync 92 | } = Flow, 93 | table_id = TableId, 94 | window_tstamp = Tstamp 95 | } = State) -> 96 | 97 | Tstamp2 = swirl_utils:unix_tstamp_ms(), 98 | WindowTimer = swirl_utils:new_timer(Window, flush, Sync), 99 | NewTableId = ets:new(?TABLE_NAME, ?TABLE_OPTS), 100 | Period = #period {start_at = Tstamp, end_at = Tstamp2}, 101 | spawn(fun() -> flush_window(Flow, Period, TableId) end), 102 | 103 | {noreply, State#state { 104 | table_id = NewTableId, 105 | window_tstamp = Tstamp2, 106 | window_timer = WindowTimer 107 | }}; 108 | handle_info(heartbeat, #state { 109 | flow = #flow { 110 | id = FlowId, 111 | heartbeat = Hbeat, 112 | mapper_nodes = MapperNodes 113 | } = Flow, 114 | hbeat_nodes = HbeatNodes 115 | } = State) -> 116 | 117 | HbeatTimer = swirl_utils:new_timer(Hbeat, heartbeat, true), 118 | 119 | DeadNodes = lists:filter(fun(Node) -> 120 | not lists:member(Node, HbeatNodes) 121 | end, MapperNodes), 122 | 123 | FlowProp = swirl_utils:record_to_proplist(Flow), 124 | Msg = {start_mapper, FlowProp}, 125 | [swirl_tracker:message(Node, FlowId, Msg) || Node <- DeadNodes], 126 | 127 | Msg2 = {ping, node()}, 128 | [swirl_tracker:message(Node, FlowId, Msg2) || Node <- MapperNodes], 129 | 130 | {noreply, State#state { 131 | hbeat_timer = HbeatTimer, 132 | hbeat_nodes = [] 133 | }}; 134 | handle_info({pong, MapperNode}, #state { 135 | hbeat_nodes = HbeatNodes 136 | } = State) -> 137 | 138 | {noreply, State#state { 139 | hbeat_nodes = [MapperNode | HbeatNodes] 140 | }}; 141 | handle_info(stop, State) -> 142 | {stop, normal, State}; 143 | handle_info({mapper_window, Period, Rows}, #state { 144 | table_id = TableId 145 | } = State) -> 146 | 147 | spawn(fun() -> map_rows(Period, Rows, TableId) end), 148 | {noreply, State}; 149 | handle_info({ping, Node}, #state {flow = #flow {id = FlowId}} = State) -> 150 | swirl_tracker:message(Node, FlowId, pong), 151 | {noreply, State}; 152 | handle_info(Msg, State) -> 153 | io:format("unexpected message: ~p~n", [Msg]), 154 | {noreply, State}. 155 | 156 | terminate(_Reason, #state { 157 | flow = Flow, 158 | window_timer = WindowTimer 159 | }) -> 160 | 161 | swirl_flow:unregister(Flow), 162 | unregister(Flow), 163 | timer:cancel(WindowTimer), 164 | ok. 165 | 166 | code_change(_OldVsn, State, _Extra) -> 167 | {ok, State}. 168 | 169 | %% private 170 | flush_window(_Flow, _Period, undefined) -> 171 | ok; 172 | flush_window(#flow { 173 | reducer_skip = ReducerSkip 174 | } = Flow, Period, TableId) -> 175 | 176 | Rows = swirl_utils:tab2list(TableId), 177 | true = ets:delete(TableId), 178 | ReducedRows = reduce_rows(Flow, Rows, ReducerSkip), 179 | output(Flow, Period, ReducedRows). 180 | 181 | key(#flow {id = FlowId}) -> FlowId. 182 | 183 | map_rows(_Period, [], _TableId) -> 184 | ok; 185 | map_rows(Period, [H | T], TableId) -> 186 | [Key | Counters] = tuple_to_list(H), 187 | swirl_utils:safe_ets_increment(TableId, Key, Counters), 188 | map_rows(Period, T, TableId). 189 | 190 | -spec output(flow(), period(), list(row())) -> ok. 191 | output(#flow { 192 | module = Module, 193 | module_vsn = ModuleVsn, 194 | start_node = StartNode, 195 | output_opts = OutputOpts 196 | } = Flow, Period, Rows) -> 197 | 198 | try Module:output(Flow, Period, Rows, OutputOpts) 199 | catch 200 | error:undef -> 201 | swirl_code_server:get_module(StartNode, Module, ModuleVsn) 202 | end. 203 | 204 | -spec reduce(flow(), row()) -> ignore | update(). 205 | reduce(#flow { 206 | module = Module, 207 | module_vsn = ModuleVsn, 208 | start_node = StartNode, 209 | reducer_opts = ReducerOpts 210 | } = Flow, Row) -> 211 | 212 | try Module:reduce(Flow, Row, ReducerOpts) of 213 | ignore -> ignore; 214 | {_Key, _Counters} = Update -> Update 215 | catch 216 | error:undef -> 217 | swirl_code_server:get_module(StartNode, Module, ModuleVsn), 218 | ignore 219 | end. 220 | 221 | reduce_rows(_Flow, [], _ReducerSkip) -> 222 | []; 223 | reduce_rows(Flow, [Row | T], ReducerSkip) -> 224 | [Key | Counters] = tuple_to_list(Row), 225 | Row2 = {Key, list_to_tuple(Counters)}, 226 | case ReducerSkip of 227 | true -> 228 | [Row2 | reduce_rows(Flow, T, ReducerSkip)]; 229 | false -> 230 | case reduce(Flow, Row2) of 231 | ignore -> 232 | reduce_rows(Flow, T, ReducerSkip); 233 | Row3 -> 234 | [Row3 | reduce_rows(Flow, T, ReducerSkip)] 235 | end 236 | end. 237 | 238 | start_link(Flow) -> 239 | gen_server:start_link(?MODULE, Flow, []). 240 | -------------------------------------------------------------------------------- /src/swirl_stream.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_stream). 2 | -include("swirl.hrl"). 3 | -compile([native]). 4 | 5 | %% public 6 | -export([ 7 | emit/2 8 | ]). 9 | 10 | %% internal 11 | -export([ 12 | lookup/1, 13 | register/2, 14 | unregister/1 15 | ]). 16 | 17 | %% public 18 | -spec emit(stream_name(), event()) -> ok. 19 | emit(StreamName, Event) -> 20 | evaluate(StreamName, Event, lookup(StreamName)). 21 | 22 | %% internal 23 | -spec lookup(stream_name()) -> [tuple()]. 24 | lookup(StreamName) -> 25 | LookupSpec = match_lookup_spec(StreamName), 26 | ets:select(?TABLE_NAME_STREAMS, LookupSpec). 27 | 28 | -spec register(flow(), ets:tab()) -> true. 29 | register(#flow { 30 | id = FlowId, 31 | module = FlowMod, 32 | module_vsn = FlowModVsn, 33 | start_node = StartNode, 34 | stream_filter = StreamFilter, 35 | stream_names = StreamNames, 36 | mapper_opts = MapperOpts 37 | } = Flow, TableId) -> 38 | 39 | ok =:= lists:foreach(fun (StreamName) -> 40 | Key = key(Flow, StreamName), 41 | Stream = #stream { 42 | flow_id = FlowId, 43 | flow_mod = FlowMod, 44 | flow_mod_vsn = FlowModVsn, 45 | start_node = StartNode, 46 | exp_tree = expession_tree(StreamFilter), 47 | mapper_opts = MapperOpts, 48 | table_id = TableId 49 | }, 50 | KeyValue = {Key, Stream}, 51 | ets:insert(?TABLE_NAME_STREAMS, KeyValue) 52 | end, StreamNames). 53 | 54 | -spec unregister(flow()) -> true. 55 | unregister(#flow {stream_names = StreamNames} = Flow) -> 56 | ok =:= lists:foreach(fun (StreamName) -> 57 | DeleteSpec = match_delete_spec(Flow, StreamName), 58 | ets:match_delete(?TABLE_NAME_STREAMS, DeleteSpec) 59 | end, StreamNames). 60 | 61 | %% private 62 | evaluate(_StreamName, _Event, []) -> 63 | ok; 64 | evaluate(StreamName, Event, [#stream { 65 | exp_tree = undefined 66 | } = Stream | T]) -> 67 | 68 | swirl_mapper:map(StreamName, Event, Stream), 69 | evaluate(StreamName, Event, T); 70 | evaluate(StreamName, Event, [#stream { 71 | exp_tree = ExpTree 72 | } = Stream | T]) -> 73 | 74 | case swirl_ql:evaluate(ExpTree, Event) of 75 | true -> 76 | swirl_mapper:map(StreamName, Event, Stream); 77 | false -> ok 78 | end, 79 | evaluate(StreamName, Event, T). 80 | 81 | expession_tree(undefined) -> 82 | undefined; 83 | expession_tree(StreamFilter) -> 84 | {ok, ExpTree} = swirl_ql:parse(StreamFilter), 85 | ExpTree. 86 | 87 | key(#flow {id = FlowId}, StreamName) -> 88 | {FlowId, StreamName}. 89 | 90 | match_lookup_spec(StreamName) -> 91 | [{{{'$1', '$2'}, '$3'}, [{'orelse', {'=:=', '$2', StreamName}, 92 | {'=:=', '$2', undefined}}], ['$3']}]. 93 | 94 | match_delete_spec(#flow {id = FlowId}, StreamName) -> 95 | {{FlowId, StreamName}, '_'}. 96 | -------------------------------------------------------------------------------- /src/swirl_sup.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_sup). 2 | -include("swirl.hrl"). 3 | 4 | %% internal 5 | -export([ 6 | start_link/0 7 | ]). 8 | 9 | -behaviour(supervisor). 10 | -export([ 11 | init/1 12 | ]). 13 | 14 | %% internal 15 | -spec start_link() -> {ok, pid()}. 16 | start_link() -> 17 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 18 | 19 | %% supervisor callbacks 20 | init([]) -> 21 | Workers = [ 22 | ?CHILD(swirl_ets_manager, worker), 23 | ?CHILD(swirl_code_server, worker), 24 | ?CHILD(swirl_tracker, worker) 25 | ], 26 | {ok, {{one_for_one, 5, 10}, Workers}}. 27 | -------------------------------------------------------------------------------- /src/swirl_tracker.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_tracker). 2 | -include("swirl.hrl"). 3 | 4 | -compile({no_auto_import, [ 5 | register/2, 6 | unregister/1 7 | ]}). 8 | 9 | %% internal 10 | -export([ 11 | lookup/2, 12 | message/3, 13 | register/3, 14 | start_link/0, 15 | start_mappers/1, 16 | start_reducer/1, 17 | stop_mappers/1, 18 | stop_reducer/1, 19 | unregister/2 20 | ]). 21 | 22 | -behaviour(gen_server). 23 | -export([ 24 | init/1, 25 | handle_call/3, 26 | handle_cast/2, 27 | handle_info/2, 28 | terminate/2, 29 | code_change/3 30 | ]). 31 | 32 | -define(TABLE_OPTS, [public, named_table, {read_concurrency, true}]). 33 | -define(SERVER, ?MODULE). 34 | 35 | -record(state, {}). 36 | 37 | %% internal 38 | -spec lookup(ets:tab(), term()) -> term(). 39 | lookup(TableId, Key) -> 40 | swirl_utils:safe_ets_lookup_element(TableId, Key). 41 | 42 | -spec message(node(), binary(), term()) -> ok. 43 | message(Node, FlowId, Msg) -> 44 | {?SERVER, Node} ! {flow, FlowId, Msg}, 45 | ok. 46 | 47 | -spec register(ets:tab(), term(), term()) -> true. 48 | register(TableId, Key, Value) -> 49 | ets:insert(TableId, {Key, Value}). 50 | 51 | -spec start_link() -> {'ok', pid()}. 52 | start_link() -> 53 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). 54 | 55 | -spec start_mappers(flow()) -> ok. 56 | start_mappers(#flow {id = FlowId, mapper_nodes = MapperNodes} = Flow) -> 57 | FlowProp = swirl_utils:record_to_proplist(Flow), 58 | [message(Node, FlowId, {start_mapper, FlowProp}) || Node <- MapperNodes], 59 | ok. 60 | 61 | -spec start_reducer(flow()) -> ok. 62 | start_reducer(#flow {id = FlowId, reducer_node = ReducerNode} = Flow) -> 63 | FlowProp = swirl_utils:record_to_proplist(Flow), 64 | message(ReducerNode, FlowId, {start_reducer, FlowProp}). 65 | 66 | -spec stop_mappers(flow()) -> ok. 67 | stop_mappers(#flow {id = FlowId, mapper_nodes = MapperNodes}) -> 68 | [message(Node, FlowId, stop_mapper) || Node <- MapperNodes], 69 | ok. 70 | 71 | -spec stop_reducer(flow()) -> ok. 72 | stop_reducer(#flow {id = FlowId, reducer_node = ReducerNode}) -> 73 | message(ReducerNode, FlowId, stop_reducer), 74 | ok. 75 | 76 | -spec unregister(ets:tab(), term()) -> true. 77 | unregister(TableId, Key) -> 78 | ets:delete(TableId, Key). 79 | 80 | %% gen_server callbacks 81 | init([]) -> 82 | process_flag(trap_exit, true), 83 | [swirl_ets_manager:table(Name, ?TABLE_OPTS, ?SERVER) || Name <- [ 84 | ?TABLE_NAME_FLOWS, 85 | ?TABLE_NAME_MAPPERS, 86 | ?TABLE_NAME_REDUCERS, 87 | ?TABLE_NAME_STREAMS 88 | ]], 89 | {ok, #state {}}. 90 | 91 | handle_call(Request, _From, State) -> 92 | io:format("unexpected message: ~p~n", [Request]), 93 | {reply, ok, State}. 94 | 95 | handle_cast(Msg, State) -> 96 | io:format("unexpected message: ~p~n", [Msg]), 97 | {noreply, State}. 98 | 99 | handle_info({'ETS-TRANSFER', _TableId, _Pid, {_TableName, _Options, ?SERVER}}, 100 | State) -> 101 | 102 | {noreply, State}; 103 | handle_info({'EXIT', _Pid, normal}, State) -> 104 | {noreply, State}; 105 | handle_info({flow, FlowId, Msg}, State) -> 106 | handler_flow_msg(FlowId, Msg, State); 107 | handle_info(Info, State) -> 108 | io:format("unexpected message: ~p~n", [Info]), 109 | {noreply, State}. 110 | 111 | terminate(_Reason, _State) -> 112 | ok. 113 | 114 | code_change(_OldVsn, State, _Extra) -> 115 | {ok, State}. 116 | 117 | %% private 118 | handler_flow_msg(FlowId, {mapper_window, _Period, _Rows} = Msg, State) -> 119 | message(swirl_reducer:lookup(FlowId), Msg), 120 | {noreply, State}; 121 | handler_flow_msg(FlowId, {ping, _Node} = Msg, State) -> 122 | message(swirl_mapper:lookup(FlowId), Msg), 123 | {noreply, State}; 124 | handler_flow_msg(FlowId, {pong, _Node} = Msg, State) -> 125 | message(swirl_reducer:lookup(FlowId), Msg), 126 | {noreply, State}; 127 | handler_flow_msg(_FlowId, {start_mapper, FlowProp}, State) -> 128 | Flow = swirl_utils:proplist_to_record(FlowProp, flow), 129 | swirl_mapper:start(Flow), 130 | {noreply, State}; 131 | handler_flow_msg(_FlowId, {start_reducer, FlowProp}, State) -> 132 | Flow = swirl_utils:proplist_to_record(FlowProp, flow), 133 | swirl_reducer:start(Flow), 134 | {noreply, State}; 135 | handler_flow_msg(FlowId, stop_mapper, State) -> 136 | message(swirl_mapper:lookup(FlowId), stop), 137 | {noreply, State}; 138 | handler_flow_msg(FlowId, stop_reducer, State) -> 139 | message(swirl_reducer:lookup(FlowId), stop), 140 | {noreply, State}. 141 | 142 | message(undefined, _Msg) -> 143 | ok; 144 | message(Pid, Msg) when is_pid(Pid) -> 145 | Pid ! Msg. 146 | -------------------------------------------------------------------------------- /src/swirl_utils.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_utils). 2 | -include("swirl.hrl"). 3 | 4 | %% public 5 | -export([ 6 | lookup/2, 7 | lookup/3, 8 | new_timer/2, 9 | new_timer/3, 10 | proplist_to_record/2, 11 | record_to_proplist/1, 12 | safe_dict_fetch/2, 13 | safe_ets_delete/1, 14 | safe_ets_increment/3, 15 | safe_ets_lookup_element/2, 16 | tab2list/1, 17 | unix_tstamp_ms/0, 18 | update_op/1, 19 | uuid/0 20 | ]). 21 | 22 | %% public 23 | -spec lookup(term(), [{term(), term()}]) -> 24 | term(). 25 | 26 | lookup(Key, List) -> 27 | lookup(Key, List, undefined). 28 | 29 | -spec lookup(term(), [{term(), term()}], term()) -> 30 | term(). 31 | 32 | lookup(Key, List, Default) -> 33 | case lists:keyfind(Key, 1, List) of 34 | false -> Default; 35 | {_, Value} -> Value 36 | end. 37 | 38 | -spec maybe_tuple_to_list(tuple() | list()) -> 39 | list(). 40 | 41 | maybe_tuple_to_list(Tuple) when is_tuple(Tuple) -> 42 | tuple_to_list(Tuple); 43 | maybe_tuple_to_list(List) when is_list(List) -> 44 | List. 45 | 46 | -spec new_timer(pos_integer(), term()) -> 47 | erlang:reference(). 48 | 49 | new_timer(Time, Msg) -> 50 | new_timer(Time, Msg, false). 51 | 52 | -spec new_timer(pos_integer(), term(), boolean()) -> 53 | erlang:reference(). 54 | 55 | new_timer(Time, Msg, true) -> 56 | Delta = unix_tstamp_ms() rem Time, 57 | erlang:send_after(Time - Delta, self(), Msg); 58 | new_timer(Time, Msg, false) -> 59 | erlang:send_after(Time, self(), Msg). 60 | 61 | -spec proplist_to_record([{atom(), term()}], atom()) -> 62 | tuple(). 63 | 64 | proplist_to_record(Proplist, Record) -> 65 | Fields = [lookup(Field, Proplist) || Field <- record_info(Record)], 66 | list_to_tuple([Record | Fields]). 67 | 68 | record_to_proplist(#flow {} = Flow) -> 69 | lists:zip(record_info(flow), tl(tuple_to_list(Flow))). 70 | 71 | safe_dict_fetch(Key, Dict) -> 72 | try dict:fetch(Key, Dict) 73 | catch 74 | error:badarg -> 75 | undefined 76 | end. 77 | 78 | safe_ets_delete(TableId) -> 79 | try ets:delete(TableId) 80 | catch 81 | error:badarg -> 82 | ok 83 | end. 84 | 85 | safe_ets_increment(TableId, Key, Counters) -> 86 | UpdateOp = update_op(Counters), 87 | try ets:update_counter(TableId, Key, UpdateOp) 88 | catch 89 | error:badarg -> 90 | safe_ets_insert(TableId, Key, Counters) 91 | end. 92 | 93 | safe_ets_insert(TableId, Key, Counters) -> 94 | try 95 | New = list_to_tuple([Key] ++ maybe_tuple_to_list(Counters)), 96 | ets:insert(TableId, New) 97 | catch 98 | error:badarg -> 99 | ok 100 | end. 101 | 102 | safe_ets_lookup_element(TableId, Key) -> 103 | try ets:lookup_element(TableId, Key, 2) 104 | catch 105 | error:badarg -> 106 | undefined 107 | end. 108 | 109 | tab2list(Tid) -> 110 | lists:append(match_all(ets:match_object(Tid, '_', 500))). 111 | 112 | unix_tstamp_ms() -> 113 | {Mega, Sec, Micro} = os:timestamp(), 114 | (Mega * 1000000000 + Sec * 1000) + trunc(Micro / 1000). 115 | 116 | update_op(Counters) when is_tuple(Counters) -> 117 | update_op(tuple_to_list(Counters), 2); 118 | update_op(Counters) when is_list(Counters) -> 119 | update_op(Counters, 2). 120 | 121 | uuid() -> 122 | {Uuid, _UuidState} = uuid:get_v1(uuid:new(self(), os)), 123 | Uuid. 124 | 125 | %% private 126 | match_all('$end_of_table') -> 127 | []; 128 | match_all({Match, Continuation}) -> 129 | [Match | match_all(ets:match_object(Continuation))]. 130 | 131 | record_info(flow) -> 132 | record_info(fields, flow). 133 | 134 | update_op([], _Pos) -> 135 | []; 136 | update_op([Counter | T], Pos) -> 137 | [{Pos, Counter} | update_op(T, Pos + 1)]. 138 | -------------------------------------------------------------------------------- /test/swirl_flow_example.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_flow_example). 2 | -include("../include/swirl.hrl"). 3 | 4 | -behavior(swirl_flow). 5 | -export([ 6 | map/3, 7 | reduce/3, 8 | output/4 9 | ]). 10 | 11 | %% swirl_flow callbacks 12 | map(StreamName, Event, _MapperOpts) -> 13 | {{?L(type, Event), StreamName, ?L(exchange_id, Event), 14 | ?L(bidder_id, Event)}, {1, 10}}. 15 | 16 | reduce(_Flow, Row, _ReducerOpts) -> 17 | Row. 18 | 19 | output(_Flow, _Period, Rows, OutputOpts) -> 20 | case ?L(send_to , OutputOpts) of 21 | undefined -> ok; 22 | Pid -> Pid ! Rows 23 | end. 24 | -------------------------------------------------------------------------------- /test/swirl_flow_tests.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_flow_tests). 2 | -include("test.hrl"). 3 | 4 | -compile(export_all). 5 | 6 | %% runners 7 | swirl_test_() -> 8 | {setup, 9 | fun () -> setup() end, 10 | fun (_) -> cleanup() end, 11 | [ 12 | ?T(test_flow) 13 | ]}. 14 | 15 | %% tests 16 | test_flow() -> 17 | {ok, Flow} = swirl_flow:start(swirl_flow_example, [ 18 | {heartbeat, timer:seconds(10)}, 19 | {mapper_opts, []}, 20 | {mapper_window, timer:seconds(1)}, 21 | {output_opts, [{send_to, self()}]}, 22 | {reducer_opts, []}, 23 | {reducer_skip, true}, 24 | {reducer_window, timer:seconds(1)}, 25 | {stream_filter, "exchange_id = 3"}, 26 | {stream_names, [delivery, requests]}, 27 | {window_sync, true} 28 | ], [node()], node()), 29 | 30 | timer:sleep(timer:seconds(1)), 31 | 32 | swirl_stream:emit(delivery, [{type, start}, {exchange_id, 1}, 33 | {bidder_id, 10}]), 34 | swirl_stream:emit(delivery, [{type, start}, {exchange_id, 3}, 35 | {bidder_id, 1}]), 36 | swirl_stream:emit(delivery, [{type, start}, {exchange_id, 3}, 37 | {bidder_id, 10}]), 38 | swirl_stream:emit(requests, [{type, start}, {exchange_id, 3}, 39 | {bidder_id, 50}]), 40 | 41 | Rows = receive_loop(), 42 | Expected = [ 43 | {{start, delivery, 3, 1}, {1, 10}}, 44 | {{start, delivery, 3, 10}, {1, 10}}, 45 | {{start, requests, 3, 50}, {1, 10}} 46 | ], 47 | 48 | ?assertEqual(Expected, lists:usort(Rows)), 49 | swirl_flow:stop(Flow). 50 | 51 | %% utils 52 | cleanup() -> 53 | error_logger:tty(false), 54 | application:stop(swirl), 55 | error_logger:tty(true). 56 | 57 | receive_loop() -> 58 | receive 59 | [] -> receive_loop(); 60 | Aggregates -> Aggregates 61 | end. 62 | 63 | setup() -> 64 | error_logger:tty(false), 65 | application:stop(swirl), 66 | swirl:start(), 67 | error_logger:tty(true). 68 | 69 | test(Test) -> 70 | {atom_to_list(Test), ?MODULE, Test}. 71 | -------------------------------------------------------------------------------- /test/swirl_ql_tests.erl: -------------------------------------------------------------------------------- 1 | -module(swirl_ql_tests). 2 | -include("test.hrl"). 3 | 4 | -compile(export_all). 5 | 6 | % runners 7 | swirl_test_() -> 8 | {inparallel, [ 9 | ?T(test_evaluate), 10 | ?T(test_parse) 11 | ]}. 12 | 13 | % tests 14 | test_evaluate() -> 15 | % comp predictate 16 | assert_eval({comp, '=', bidder_id, 1}, [{bidder_id, 1}]), 17 | assert_not_eval({comp, '=', bidder_id, 1}, [{bidder_id, 2}]), 18 | assert_eval({comp, '<', price, 100}, [{price, 60}]), 19 | assert_not_eval({comp, '<', price, 100}, [{price, 160}]), 20 | assert_eval({comp, '<=', price, 100}, [{price, 100}]), 21 | assert_not_eval({comp, '<=', price, 100}, [{price, 160}]), 22 | assert_eval({comp, '>=', price, 100}, [{price, 100}]), 23 | assert_not_eval({comp, '>=', price, 160}, [{price, 100}]), 24 | assert_eval({comp, '>', price, 100}, [{price, 160}]), 25 | assert_not_eval({comp, '>', price, 100}, [{price, 60}]), 26 | assert_eval({comp, '<>', price, 100}, [{price, 160}]), 27 | assert_not_eval({comp, '<>', price, 100}, [{price, 100}]), 28 | 29 | % in predictate 30 | assert_eval({in, exchange_id, [1 , 2]}, [{exchange_id, 2}]), 31 | assert_not_eval({in, exchange_id, [1 , 2]}, [{exchange_id, 3}]), 32 | assert_eval({notin, exchange_id, [1 , 2]}, [{exchange_id, 3}]), 33 | assert_not_eval({notin, exchange_id, [1 , 2]}, [{exchange_id, 2}]), 34 | assert_eval({in_var, 54, segment_ids}, [{segment_ids, [12, 54]}]), 35 | assert_not_eval({in_var, 54, segment_ids}, [{segment_ids, [12]}]), 36 | assert_eval({notin_var, 54, segment_ids}, [{segment_ids, [12]}]), 37 | assert_not_eval({notin_var, 54, segment_ids}, [{segment_ids, [12, 54]}]), 38 | 39 | % null predictate 40 | assert_eval({null, exchange_id}, []), 41 | assert_not_eval({null, exchange_id}, [{exchange_id, 3}]), 42 | assert_eval({notnull, exchange_id}, [{exchange_id, 3}]), 43 | assert_not_eval({notnull, exchange_id}, [{exchange_id, ?NULL}]), 44 | 45 | % and 46 | assert_eval({'and', {comp, '=', bidder_id, 1}, {comp, '=', bidder_id, 1}}, 47 | [{bidder_id, 1}]), 48 | assert_not_eval({'and', {comp, '=', bidder_id, 1}, 49 | {comp, '=', exchange_id, 1}}, [{bidder_id, 1}, {exchange_id, 2}]), 50 | 51 | % or 52 | assert_eval({'or', {comp, '=', bidder_id, 2}, {comp, '=', bidder_id, 1}}, 53 | [{bidder_id, 1}]), 54 | assert_not_eval({'or', {comp, '=', bidder_id, 2}, 55 | {comp, '=', bidder_id, 3}}, [{bidder_id, 1}]). 56 | 57 | test_parse() -> 58 | assert_parse({comp, '=', bidder_id, 1}, "bidder_id = 1"), 59 | assert_parse({comp, '=', domain, <<"ebay.ca">>}, "domain = 'ebay.ca'"), 60 | assert_parse({comp, '=', domain, <<"ebay.ca">>}, "domain = \"ebay.ca\""), 61 | assert_parse({in, exchange_id, [1, 2, 3]}, "exchange_id IN (1, 2, 3)"), 62 | assert_parse({in_var, 4, segment_ids}, "4 IN segment_ids"), 63 | assert_parse({notin_var, 8, segment_ids}, "8 NOT IN segment_ids"), 64 | assert_parse({'and', {comp, '=', bidder_id, 1}, 65 | {'or', {notin, exchange_id, [1 , 2]}, 66 | {comp, '=', domain, <<"ebay.ca">>}}}, 67 | "bidder_id = 1 AND (exchange_id NOT IN (1, 2) OR domain = 'ebay.ca')"). 68 | 69 | %% test_utils 70 | assert_eval(ExpTree, Vars) -> 71 | ?assert(swirl_ql:evaluate(ExpTree, Vars)). 72 | 73 | assert_not_eval(ExpTree, Vars) -> 74 | ?assertNot(swirl_ql:evaluate(ExpTree, Vars)). 75 | 76 | assert_parse(Expected, Expression) -> 77 | {ok, ExpTree} = swirl_ql:parse(Expression), 78 | ?assertEqual(Expected, ExpTree). 79 | 80 | test(Test) -> 81 | {atom_to_list(Test), ?MODULE, Test}. 82 | -------------------------------------------------------------------------------- /test/test.hrl: -------------------------------------------------------------------------------- 1 | -include_lib("eunit/include/eunit.hrl"). 2 | -include_lib("swirl/include/swirl.hrl"). 3 | 4 | -define(T, fun (Test) -> test(Test) end). 5 | --------------------------------------------------------------------------------