├── rebar3 ├── .gitignore ├── include ├── riak_pipe_log.hrl ├── riak_pipe_debug.hrl └── riak_pipe.hrl ├── Makefile ├── src ├── riak_pipe.app.src ├── riak_pipe_qcover_sup.erl ├── riak_pipe_fun.erl ├── riak_pipe_w_fwd.erl ├── riak_pipe_app.erl ├── riak_pipe_w_pass.erl ├── riak_pipe_qcover_fsm.erl ├── riak_pipe_w_tee.erl ├── riak_pipe_sup.erl ├── riak_pipe_vnode_worker_sup.erl ├── riak_pipe_w_xform.erl ├── riak_pipe_stat.erl ├── riak_pipe_fitting_sup.erl ├── riak_pipe_cinfo.erl ├── riak_pipe_log.erl ├── riak_pipe_builder_sup.erl ├── riak_pipe_w_rec_countdown.erl ├── riak_pipe_w_crash.erl ├── riak_pipe_sink.erl ├── riak_pipe_v.erl ├── riak_pipe_w_reduce.erl ├── riak_pipe_builder.erl ├── riak_pipe_vnode_worker.erl └── riak_pipe_fitting.erl ├── rebar.config ├── .github └── workflows │ └── erlang.yml ├── tools.mk ├── eqc ├── reduce_fitting_pulse_sink_sup.erl ├── reduce_fitting_pulse_sink.erl ├── riak_pipe_fitting_eqc.erl └── reduce_fitting_pulse.erl ├── riak_pipe_monitors.dot ├── priv └── app.slave0.config └── LICENSE /rebar3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/basho/riak_pipe/HEAD/rebar3 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | ebin/* 3 | .eunit/* 4 | doc/* 5 | deps/* 6 | erl_crash.dump 7 | EUnit-SASL.log 8 | .local_dialyzer_plt 9 | .rebar/ 10 | .rebar3 11 | rebar.lock 12 | _build 13 | .eqc-info 14 | -------------------------------------------------------------------------------- /include/riak_pipe_log.hrl: -------------------------------------------------------------------------------- 1 | -define(T(Details, Extra, Msg), 2 | riak_pipe_log:trace(Details, [?MODULE|Extra], Msg)). 3 | -define(T_ERR(Details, Props), 4 | riak_pipe_log:trace(Details, [?MODULE, error], {error, Props})). 5 | -define(L(Details, Msg), 6 | riak_pipe_log:log(Details, Msg)). 7 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: compile rel cover test dialyzer 2 | REBAR=./rebar3 3 | 4 | compile: 5 | $(REBAR) compile 6 | 7 | clean: 8 | $(REBAR) clean 9 | 10 | cover: test 11 | $(REBAR) cover 12 | 13 | test: compile 14 | $(REBAR) as test do eunit 15 | 16 | dialyzer: 17 | $(REBAR) dialyzer 18 | 19 | xref: 20 | $(REBAR) xref 21 | 22 | check: test dialyzer xref 23 | -------------------------------------------------------------------------------- /src/riak_pipe.app.src: -------------------------------------------------------------------------------- 1 | {application, riak_pipe, 2 | [ 3 | {description, "Riak Pipeline"}, 4 | {vsn, git}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | sasl, 10 | riak_core, 11 | cluster_info, 12 | exometer_core 13 | ]}, 14 | {mod, { riak_pipe_app, []}}, 15 | {env, []} 16 | ]}. 17 | -------------------------------------------------------------------------------- /include/riak_pipe_debug.hrl: -------------------------------------------------------------------------------- 1 | -ifdef(DEBUG). 2 | -define(DP(M), error_logger:info_msg("~p:~p "++M, [?MODULE, ?LINE])). 3 | -define(DPV(X), error_logger:info_msg("~p:~p -- ~p =~n ~p", 4 | [?MODULE, ?LINE, ??X, X])). 5 | 6 | -define(DPF(M,X), error_logger:info_msg("~p:~p "++M, 7 | [?MODULE, ?LINE]++X)). 8 | -else. 9 | -define(DP(M), noop). 10 | -define(DPV(X), noop). 11 | -define(DPF(M,X), noop). 12 | -endif. 13 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang -*- 2 | {minimum_otp_vsn, "22.0"}. 3 | 4 | {erl_opts, [warnings_as_errors, 5 | debug_info, 6 | {platform_define, "^[0-9]+", namespaced_types}]}. 7 | 8 | {edoc_opts, [{preprocess, true}]}. 9 | {cover_enabled, true}. 10 | 11 | {xref_checks,[undefined_function_calls,undefined_functions,locals_not_used]}. 12 | 13 | {profiles, [ 14 | {gha, [{erl_opts, [{d, 'GITHUBEXCLUDE'}]}]} 15 | ]}. 16 | 17 | {deps, [ 18 | {riak_core, ".*", {git, "https://github.com/basho/riak_core.git", {branch, "develop"}}} 19 | ]}. 20 | 21 | {plugins, [{eqc_rebar, {git, "https://github.com/Quviq/eqc-rebar", {branch, "master"}}}]}. 22 | -------------------------------------------------------------------------------- /include/riak_pipe.hrl: -------------------------------------------------------------------------------- 1 | -record(fitting, 2 | { 3 | pid :: pid(), 4 | ref :: reference() | undefined, 5 | chashfun :: riak_pipe_vnode:chashfun() |undefined, 6 | nval :: riak_pipe_vnode:nval() | undefined 7 | }). 8 | 9 | -record(fitting_details, 10 | { 11 | fitting :: #fitting{}, 12 | name :: term(), 13 | module :: atom(), 14 | arg :: term() | undefined, 15 | output :: #fitting{}, 16 | options :: riak_pipe:exec_opts(), 17 | q_limit :: pos_integer() 18 | }). 19 | 20 | -record(fitting_spec, 21 | { 22 | name :: term(), 23 | module :: atom(), 24 | arg :: term() |undefined, 25 | chashfun = {chash, key_of} :: riak_pipe_vnode:chashfun(), 26 | nval = 1 :: riak_pipe_vnode:nval(), 27 | q_limit = 64 :: pos_integer() 28 | }). 29 | 30 | -record(pipe, 31 | { 32 | builder :: pid(), 33 | fittings :: [{Name::term(), #fitting{}}], 34 | sink :: #fitting{} 35 | }). 36 | 37 | -record(pipe_result, 38 | { 39 | ref, 40 | from, 41 | result 42 | }). 43 | 44 | -record(pipe_eoi, 45 | { 46 | ref 47 | }). 48 | 49 | -record(pipe_log, 50 | { 51 | ref, 52 | from, 53 | msg 54 | }). 55 | -------------------------------------------------------------------------------- /.github/workflows/erlang.yml: -------------------------------------------------------------------------------- 1 | name: Erlang CI 2 | 3 | on: 4 | push: 5 | branches: [ develop ] 6 | pull_request: 7 | branches: [ develop ] 8 | 9 | 10 | jobs: 11 | 12 | build: 13 | 14 | name: Test on ${{ matrix.os }} with OTP ${{ matrix.otp }} 15 | runs-on: ${{ matrix.os }} 16 | 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | otp: [22, 24, 25] 21 | os: [ubuntu-latest] 22 | # OTP lower than 23 does not run on ubuntu-latest (22.04), see 23 | # https://github.com/erlef/setup-beam#compatibility-between-operating-system-and-erlangotp 24 | exclude: 25 | - otp: 22 26 | os: ubuntu-latest 27 | include: 28 | - otp: 22 29 | os: ubuntu-20.04 30 | 31 | steps: 32 | - uses: lukka/get-cmake@latest 33 | - uses: actions/checkout@v2 34 | - name: Install dependencies (Ubuntu) 35 | if: ${{ startsWith(matrix.os, 'ubuntu') }} 36 | run: | 37 | sudo apt-get -qq update 38 | sudo apt-get -qq install libsnappy-dev libc6-dev 39 | - name: Install Erlang/OTP 40 | uses: erlef/setup-beam@v1 41 | with: 42 | otp-version: ${{ matrix.otp }} 43 | - name: Compile 44 | run: ./rebar3 compile 45 | - name: Run xref and dialyzer 46 | run: ./rebar3 do xref, dialyzer 47 | - name: Run eunit 48 | run: ./rebar3 as gha do eunit 49 | -------------------------------------------------------------------------------- /src/riak_pipe_qcover_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_pipe_qcover_sup: supervise the riak_pipe qcover state machines. 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc supervise the riak_pipe qcover state machines 24 | 25 | -module(riak_pipe_qcover_sup). 26 | 27 | -behaviour(supervisor). 28 | 29 | -export([start_qcover_fsm/1]). 30 | -export([start_link/0]). 31 | -export([init/1]). 32 | 33 | start_qcover_fsm(Args) -> 34 | supervisor:start_child(?MODULE, Args). 35 | 36 | %% @spec start_link() -> ServerRet 37 | %% @doc API for starting the supervisor. 38 | start_link() -> 39 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 40 | 41 | %% @spec init([]) -> SupervisorTree 42 | %% @doc supervisor callback. 43 | init([]) -> 44 | FsmSpec = {undefined, 45 | {riak_core_coverage_fsm, start_link, [riak_pipe_qcover_fsm]}, 46 | temporary, 5000, worker, [riak_pipe_qcover_fsm]}, 47 | 48 | {ok, {{simple_one_for_one, 10, 10}, [FsmSpec]}}. 49 | -------------------------------------------------------------------------------- /src/riak_pipe_fun.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Utilities for dealing with anonymous funs. This is most 22 | %% useful for upgrading from older pipes using funs to newer pipes 23 | %% using modfun tuples. 24 | 25 | -module(riak_pipe_fun). 26 | 27 | -export([compat_apply/2]). 28 | 29 | %% @doc Attempt to evaluate Fun(Inputs...). If the evaluation throws a 30 | %% badfun error, ask the module that defines the function for an 31 | %% alternate implementation, by calling `Module:compat_fun/1', and 32 | %% then evaluates that instead. 33 | compat_apply(Fun, Inputs) -> 34 | try 35 | apply(Fun, Inputs) 36 | catch error:{badfun,Fun} -> 37 | {module, Module} = erlang:fun_info(Fun, module), 38 | try Module:compat_fun(Fun) of 39 | {ok, CompatFun} -> 40 | apply(CompatFun, Inputs); 41 | error -> 42 | %% just to re-raise the original badfun 43 | apply(Fun, Inputs) 44 | catch error:undef -> 45 | %% just to re-raise the original badfun 46 | apply(Fun, Inputs) 47 | end 48 | end. 49 | -------------------------------------------------------------------------------- /tools.mk: -------------------------------------------------------------------------------- 1 | REBAR ?= ./rebar 2 | 3 | test: compile 4 | ${REBAR} eunit skip_deps=true 5 | 6 | docs: 7 | ${REBAR} doc skip_deps=true 8 | 9 | xref: compile 10 | ${REBAR} xref skip_deps=true 11 | 12 | PLT ?= $(HOME)/.combo_dialyzer_plt 13 | LOCAL_PLT = .local_dialyzer_plt 14 | DIALYZER_FLAGS ?= -Wunmatched_returns 15 | 16 | ${PLT}: compile 17 | @if [ -f $(PLT) ]; then \ 18 | dialyzer --check_plt --plt $(PLT) --apps $(DIALYZER_APPS) && \ 19 | dialyzer --add_to_plt --plt $(PLT) --output_plt $(PLT) --apps $(DIALYZER_APPS) ; test $$? -ne 1; \ 20 | else \ 21 | dialyzer --build_plt --output_plt $(PLT) --apps $(DIALYZER_APPS); test $$? -ne 1; \ 22 | fi 23 | 24 | ${LOCAL_PLT}: compile 25 | @if [ -d deps ]; then \ 26 | if [ -f $(LOCAL_PLT) ]; then \ 27 | dialyzer --check_plt --plt $(LOCAL_PLT) deps/*/ebin && \ 28 | dialyzer --add_to_plt --plt $(LOCAL_PLT) --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ 29 | else \ 30 | dialyzer --build_plt --output_plt $(LOCAL_PLT) deps/*/ebin ; test $$? -ne 1; \ 31 | fi \ 32 | fi 33 | 34 | dialyzer: ${PLT} ${LOCAL_PLT} 35 | @echo "==> $(shell basename $(shell pwd)) (dialyzer)" 36 | @if [ -f $(LOCAL_PLT) ]; then \ 37 | PLTS="$(PLT) $(LOCAL_PLT)"; \ 38 | else \ 39 | PLTS=$(PLT); \ 40 | fi; \ 41 | if [ -f dialyzer.ignore-warnings ]; then \ 42 | if [ $$(grep -cvE '[^[:space:]]' dialyzer.ignore-warnings) -ne 0 ]; then \ 43 | echo "ERROR: dialyzer.ignore-warnings contains a blank/empty line, this will match all messages!"; \ 44 | exit 1; \ 45 | fi; \ 46 | dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin > dialyzer_warnings ; \ 47 | egrep -v "^[[:space:]]*(done|Checking|Proceeding|Compiling)" dialyzer_warnings | grep -F -f dialyzer.ignore-warnings -v > dialyzer_unhandled_warnings ; \ 48 | cat dialyzer_unhandled_warnings ; \ 49 | [ $$(cat dialyzer_unhandled_warnings | wc -l) -eq 0 ] ; \ 50 | else \ 51 | dialyzer $(DIALYZER_FLAGS) --plts $${PLTS} -c ebin; \ 52 | fi 53 | 54 | cleanplt: 55 | @echo 56 | @echo "Are you sure? It takes several minutes to re-build." 57 | @echo Deleting $(PLT) and $(LOCAL_PLT) in 5 seconds. 58 | @echo 59 | sleep 5 60 | rm $(PLT) 61 | rm $(LOCAL_PLT) 62 | 63 | -------------------------------------------------------------------------------- /src/riak_pipe_w_fwd.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc This worker always returns `forward_preflist' for each input, 22 | %% in order to get the worker to send the input on to the next 23 | %% vnode in the preflist. It is used by the vnode to disperse 24 | %% enqueued inputs for a worker, if the worker fails, and then 25 | %% also fails to restart. 26 | -module(riak_pipe_w_fwd). 27 | -behaviour(riak_pipe_vnode_worker). 28 | 29 | -export([init/2, 30 | process/3, 31 | done/1]). 32 | 33 | -include("riak_pipe.hrl"). 34 | -include("riak_pipe_log.hrl"). 35 | 36 | -record(state, {fd :: riak_pipe_fitting:details()}). 37 | -opaque state() :: #state{}. 38 | -export_type([state/0]). 39 | 40 | %% @doc Initialization just stows the fitting details in the module's 41 | %% state, for sending traces in {@link process/3}. 42 | -spec init(riak_pipe_vnode:partition(), 43 | riak_pipe_fitting:details()) -> 44 | {ok, state()}. 45 | init(_Partition, FittingDetails) -> 46 | {ok, #state{fd=FittingDetails}}. 47 | 48 | %% @doc Process just requests that `Input' be forwarded to the next 49 | %% vnode in its preflist. 50 | -spec process(term(), boolean(), state()) -> {forward_preflist, state()}. 51 | process(_Input, _Last, #state{fd=FittingDetails}=State) -> 52 | ?T(FittingDetails, [], {forwarding, _Input}), 53 | {forward_preflist, State}. 54 | 55 | %% @doc Unused. 56 | -spec done(state()) -> ok. 57 | done(_State) -> 58 | ok. 59 | -------------------------------------------------------------------------------- /src/riak_pipe_app.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Start and stop handling of the `riak_pipe' application. 22 | 23 | -module(riak_pipe_app). 24 | 25 | -behaviour(application). 26 | 27 | %% Application callbacks 28 | -export([start/2, stop/1]). 29 | 30 | %% =================================================================== 31 | %% Application callbacks 32 | %% =================================================================== 33 | 34 | %% @doc Start the `riak_pipe' application. 35 | %% 36 | %% The `riak_core' application should already be started. This 37 | %% function will register the `riak_pipe_vnode' module to setup 38 | %% the riak_pipe vnode master, and will also announce the 39 | %% riak_pipe service to the node watcher. 40 | %% 41 | %% If cluster_info has also been started, this function will 42 | %% register the `riak_pipe_cinfo' module with it. 43 | -spec start(term(), term()) -> {ok, pid()} | {error, term()}. 44 | start(_StartType, _StartArgs) -> 45 | %% startup mostly copied from riak_kv 46 | catch cluster_info:register_app(riak_pipe_cinfo), 47 | 48 | case riak_pipe_sup:start_link() of 49 | {ok, Pid} -> 50 | riak_core:register(riak_pipe, [ 51 | {vnode_module, riak_pipe_vnode}, 52 | {stat_mod, riak_pipe_stat} 53 | ]), 54 | {ok, Pid}; 55 | {error, Reason} -> 56 | {error, Reason} 57 | end. 58 | 59 | %% @doc Unused. 60 | -spec stop(term()) -> ok. 61 | stop(_State) -> 62 | ok. 63 | -------------------------------------------------------------------------------- /src/riak_pipe_w_pass.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Simple pass-thru fitting. Just passes its input directly to 22 | %% its output. This fitting should work with any consistent-hash 23 | %% function. It ignores its argument, and requires no archiving 24 | %% for handoff. 25 | -module(riak_pipe_w_pass). 26 | -behaviour(riak_pipe_vnode_worker). 27 | 28 | -export([init/2, 29 | process/3, 30 | done/1]). 31 | 32 | -include("riak_pipe.hrl"). 33 | -include("riak_pipe_log.hrl"). 34 | 35 | -record(state, {p :: riak_pipe_vnode:partition(), 36 | fd :: riak_pipe_fitting:details()}). 37 | -opaque state() :: #state{}. 38 | -export_type([state/0]). 39 | 40 | %% @doc Initialization just stows the partition and fitting details in 41 | %% the module's state, for sending outputs in {@link process/3}. 42 | -spec init(riak_pipe_vnode:partition(), 43 | riak_pipe_fitting:details()) -> 44 | {ok, state()}. 45 | init(Partition, FittingDetails) -> 46 | {ok, #state{p=Partition, fd=FittingDetails}}. 47 | 48 | %% @doc Process just sends `Input' directly to the next fitting. This 49 | %% function also generates two trace messages: `{processing, 50 | %% Input}' before sending the output, and `{processed, Input}' after 51 | %% the blocking output send has returned. This can be useful for 52 | %% dropping in another pipeline to watching data move through it. 53 | -spec process(term(), boolean(), state()) -> {ok, state()}. 54 | process(Input, _Last, #state{p=Partition, fd=FD}=State) -> 55 | if FD#fitting_details.arg == black_hole -> 56 | {ok, State}; 57 | true -> 58 | ?T(FD, [], {processing, Input}), 59 | ok = riak_pipe_vnode_worker:send_output(Input, Partition, FD), 60 | ?T(FD, [], {processed, Input}), 61 | {ok, State} 62 | end. 63 | 64 | %% @doc Unused. 65 | -spec done(state()) -> ok. 66 | done(_State) -> 67 | ok. 68 | -------------------------------------------------------------------------------- /src/riak_pipe_qcover_fsm.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% riak_pipe_qcover_fsm: enqueueing on a covering set of vnodes 4 | %% 5 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 6 | %% 7 | %% This file is provided to you under the Apache License, 8 | %% Version 2.0 (the "License"); you may not use this file 9 | %% except in compliance with the License. You may obtain 10 | %% a copy of the License at 11 | %% 12 | %% http://www.apache.org/licenses/LICENSE-2.0 13 | %% 14 | %% Unless required by applicable law or agreed to in writing, 15 | %% software distributed under the License is distributed on an 16 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | %% KIND, either express or implied. See the License for the 18 | %% specific language governing permissions and limitations 19 | %% under the License. 20 | %% 21 | %% ------------------------------------------------------------------- 22 | 23 | %% @doc The qcover fsm manages enqueueing the same work on a set of 24 | %% vnodes that covers the space of data with a replication factor 25 | %% of N. This can be used, for example, to ask 1/Xth of the 26 | %% cluster to list keys for data stored in Riak KV with N=X. 27 | %% 28 | %% When using qcover to send `Input', workers for the fitting 29 | %% will receive input of the form `{cover, FilterVNodes, Input}'. 30 | %% 31 | %% IMPORTANT: Fittings that will receive their inputs via this 32 | %% qcover fsm should be started with their nval=1. Pipe nval 33 | %% failover should not be used with qcover inputs, because the 34 | %% vnode filter is meant only for the vnode that initially 35 | %% receives the request. 36 | %% 37 | %% Note also that the chashfun fitting parameter has no effect 38 | %% when using qcover. Coverage is computed purely via the 39 | %% qcover's NVal. 40 | 41 | -module(riak_pipe_qcover_fsm). 42 | 43 | -behaviour(riak_core_coverage_fsm). 44 | 45 | -export([init/2, 46 | process_results/2, 47 | finish/2]). 48 | 49 | -include("riak_pipe.hrl"). 50 | 51 | -record(state, {from}). 52 | 53 | init(From, [#pipe{fittings=[{_Name, Fitting}|_]}, Input, NVal]) -> 54 | Req = {Fitting, Input}, 55 | {Req, %% Request 56 | all, %% VNodeSelector 57 | NVal, %% NVal 58 | 1, %% PrimaryVNodeCoverage 59 | riak_pipe, %% NodeCheckService 60 | riak_pipe_vnode_master, %% VNodeMaster 61 | infinity, %% Timeout 62 | #state{from=From}}. %% State 63 | 64 | process_results(ok, State) -> 65 | %% this 'done' means that the vnode that sent this reply is done, 66 | %% not that the entire request is done 67 | {done, State}; 68 | process_results({error, Reason}, _State) -> 69 | {error, Reason}. 70 | 71 | finish({error, Reason}, #state{from={raw, Ref, Client}}=State) -> 72 | Client ! {Ref, {error, Reason}}, 73 | {stop, normal, State}; 74 | finish(clean, #state{from={raw, Ref, Client}}=State) -> 75 | Client ! {Ref, done}, 76 | {stop, normal, State}. 77 | -------------------------------------------------------------------------------- /src/riak_pipe_w_tee.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Send inputs to another fitting (often the sink) in addition to 22 | %% the output. 23 | %% 24 | %% If the argument to this fitting is the atom `sink', every 25 | %% input will be sent to the sink. If the argument is a 26 | %% `#fitting{}' record, every input will be sent to that fitting. 27 | -module(riak_pipe_w_tee). 28 | -behaviour(riak_pipe_vnode_worker). 29 | 30 | -export([init/2, 31 | process/3, 32 | done/1, 33 | validate_arg/1]). 34 | 35 | -include("riak_pipe.hrl"). 36 | -include("riak_pipe_log.hrl"). 37 | 38 | -record(state, {p :: riak_pipe_vnode:partition(), 39 | fd :: riak_pipe_fitting:details()}). 40 | -opaque state() :: #state{}. 41 | -export_type([state/0]). 42 | 43 | %% @doc Init just stashes the `Partition' and `FittingDetails' for later. 44 | -spec init(riak_pipe_vnode:partition(), riak_pipe_fitting:details()) -> 45 | {ok, state()}. 46 | init(Partition, FittingDetails) -> 47 | {ok, #state{p=Partition, fd=FittingDetails}}. 48 | 49 | %% @doc Processing an input involves sending it to both the fitting 50 | %% specified by the argument (possibly the sink), and to the 51 | %% output. 52 | -spec process(term(), boolean(), state()) -> {ok, state()}. 53 | process(Input, _Last, #state{p=Partition, fd=FittingDetails}=State) -> 54 | Tee = case FittingDetails#fitting_details.arg of 55 | sink -> 56 | proplists:get_value( 57 | sink, FittingDetails#fitting_details.options); 58 | #fitting{}=Fitting -> 59 | Fitting 60 | end, 61 | ok = riak_pipe_vnode_worker:send_output( 62 | Input, Partition, FittingDetails, Tee, infinity), 63 | ok = riak_pipe_vnode_worker:send_output(Input, Partition, FittingDetails), 64 | {ok, State}. 65 | 66 | %% @doc Unused. 67 | -spec done(state()) -> ok. 68 | done(_State) -> 69 | ok. 70 | 71 | %% @doc Check that the fitting's argument is either the atom `sink' or 72 | %% a `#fitting{}' record. 73 | -spec validate_arg(term()) -> ok | {error, iolist()}. 74 | validate_arg(sink) -> ok; 75 | validate_arg(#fitting{}) -> ok; 76 | validate_arg(Other) -> 77 | {error, io_lib:format("~p requires a fitting record," 78 | " or the atom 'sink'" 79 | " as its argument, not a ~p", 80 | [?MODULE, riak_pipe_v:type_of(Other)])}. 81 | -------------------------------------------------------------------------------- /src/riak_pipe_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Supervisor for the `riak_pipe' application. 22 | 23 | -module(riak_pipe_sup). 24 | 25 | -behaviour(supervisor). 26 | 27 | %% API 28 | -export([start_link/0]). 29 | 30 | %% Supervisor callbacks 31 | -export([init/1]). 32 | 33 | %% =================================================================== 34 | %% API functions 35 | %% =================================================================== 36 | 37 | %% @doc Start the supervisor. It will be registered under the atom 38 | %% `riak_pipe_sup'. 39 | -spec start_link() -> {ok, pid()} | ignore | {error, term()}. 40 | start_link() -> 41 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 42 | 43 | %% =================================================================== 44 | %% Supervisor callbacks 45 | %% =================================================================== 46 | 47 | %% @doc Initialize the supervisor, and start children. 48 | %% 49 | %% Three children are started immediately: 50 | %%
  1. 51 | %% The vnode master for riak_pipe vnodes (registered under 52 | %% `riak_pipe_vnode_master'). 53 | %%
  2. 54 | %% The pipe builder supervisor (registered under 55 | %% `riak_pipe_builder_sup'). 56 | %%
  3. 57 | %% The pipe fitting supervisor (registred under 58 | %% `riak_pipe_fitting_sup'). 59 | %%
. 60 | -spec init([]) -> {ok, {{supervisor:strategy(), 61 | pos_integer(), 62 | pos_integer()}, 63 | [ supervisor:child_spec() ]}}. 64 | init([]) -> 65 | %% ordsets = enabled traces are represented as ordsets in fitting_details 66 | %% sets = '' sets '' 67 | riak_core_capability:register( 68 | {riak_pipe, trace_format}, [ordsets, sets], sets), 69 | 70 | VMaster = {riak_pipe_vnode_master, 71 | {riak_core_vnode_master, start_link, [riak_pipe_vnode]}, 72 | permanent, 5000, worker, [riak_core_vnode_master]}, 73 | BSup = {riak_pipe_builder_sup, 74 | {riak_pipe_builder_sup, start_link, []}, 75 | permanent, 5000, supervisor, [riak_pipe_builder_sup]}, 76 | FSup = {riak_pipe_fitting_sup, 77 | {riak_pipe_fitting_sup, start_link, []}, 78 | permanent, 5000, supervisor, [riak_pipe_fitting_sup]}, 79 | CSup = {riak_pipe_qcover_sup, 80 | {riak_pipe_qcover_sup, start_link, []}, 81 | permanent, 5000, supervisor, [riak_pipe_qcover_sup]}, 82 | {ok, { {one_for_one, 5, 10}, [VMaster, BSup, FSup, CSup]} }. 83 | -------------------------------------------------------------------------------- /src/riak_pipe_vnode_worker_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Supervisor for a vnode's worker processes. One of these is 22 | %% started per vnode. 23 | -module(riak_pipe_vnode_worker_sup). 24 | 25 | -behaviour(supervisor). 26 | 27 | %% API 28 | -export([start_link/2]). 29 | -export([start_worker/2, 30 | terminate_worker/2]). 31 | 32 | %% Supervisor callbacks 33 | -export([init/1]). 34 | 35 | -include("riak_pipe.hrl"). 36 | 37 | %%%=================================================================== 38 | %%% API functions 39 | %%%=================================================================== 40 | 41 | %% @doc Start the supervisor. 42 | -spec start_link(riak_pipe_vnode:partition(), pid()) -> 43 | {ok, pid()} | ignore | {error, term()}. 44 | start_link(Partition, VnodePid) -> 45 | supervisor:start_link(?MODULE, [Partition, VnodePid]). 46 | 47 | %% @doc Start a new worker under the supervisor. 48 | -spec start_worker(pid(), riak_pipe_fitting:details()) -> {ok, pid()}. 49 | start_worker(Supervisor, Details) -> 50 | supervisor:start_child(Supervisor, [Details]). 51 | 52 | %% @doc Stop a worker immediately 53 | -spec terminate_worker(pid(), pid()) -> ok | {error, term()}. 54 | terminate_worker(Supervisor, WorkerPid) -> 55 | supervisor:terminate_child(Supervisor, WorkerPid). 56 | 57 | %%%=================================================================== 58 | %%% Supervisor callbacks 59 | %%%=================================================================== 60 | 61 | %% @doc Initialize the supervisor. This is a `simple_one_for_one', 62 | %% whose child spec is for starting `riak_pipe_vnode_worker' 63 | %% FSMs. 64 | -spec init([riak_pipe_vnode:partition() | pid()]) -> 65 | {ok, {{supervisor:strategy(), 66 | pos_integer(), 67 | pos_integer()}, 68 | [ supervisor:child_spec() ]}}. 69 | init([Partition, VnodePid]) -> 70 | RestartStrategy = simple_one_for_one, 71 | MaxRestarts = 1000, 72 | MaxSecondsBetweenRestarts = 3600, 73 | 74 | SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, 75 | 76 | Restart = temporary, 77 | Shutdown = 2000, 78 | Type = worker, 79 | 80 | AChild = {undefined, % no registered name 81 | {riak_pipe_vnode_worker, start_link, [Partition, VnodePid]}, 82 | Restart, Shutdown, Type, [riak_pipe_vnode_worker]}, 83 | 84 | {ok, {SupFlags, [AChild]}}. 85 | 86 | %%%=================================================================== 87 | %%% Internal functions 88 | %%%=================================================================== 89 | -------------------------------------------------------------------------------- /src/riak_pipe_w_xform.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Alter an input, and send the alteration as output. For each 22 | %% input, the fitting evaluates a function (passed as the fitting 23 | %% argument). That function should accept three parameters: 24 | %%
25 | %% `Input' :: term() 26 | %%
27 | %% Whatever the upstream fitting has sent. 28 | %%
29 | %% `Partition' :: riak_pipe_vnode:partition() 30 | %%
31 | %% The partition of the vnode on which this worker is running. 32 | %% Necessary for logging and sending output. 33 | %%
34 | %% `FittingDetails' :: #fitting_details{} 35 | %%
36 | %% The details for the fitting. Also necessary for logging and 37 | %% sending output. 38 | %%
39 | %% 40 | %% The function should use `Partition' and `FittingDetails' along 41 | %% with {@link riak_pipe_vnode_worker:send_output/3} to send 42 | %% whatever outputs it likes (this allows the fitting to be used 43 | %% as "filter" as well as "map"). 44 | -module(riak_pipe_w_xform). 45 | -behaviour(riak_pipe_vnode_worker). 46 | 47 | -export([init/2, 48 | process/3, 49 | done/1, 50 | validate_arg/1]). 51 | 52 | -include("riak_pipe.hrl"). 53 | -include("riak_pipe_log.hrl"). 54 | 55 | -record(state, {p :: riak_pipe_vnode:partition(), 56 | fd :: riak_pipe_fitting:details()}). 57 | -opaque state() :: #state{}. 58 | -export_type([state/0]). 59 | 60 | %% @doc Init just stashes the `Partition' and `FittingDetails' for later. 61 | -spec init(riak_pipe_vnode:partition(), riak_pipe_fitting:details()) -> 62 | {ok, state()}. 63 | init(Partition, FittingDetails) -> 64 | {ok, #state{p=Partition, fd=FittingDetails}}. 65 | 66 | %% @doc Process evaluates the fitting's argument function. 67 | -spec process(term(), boolean(), state()) -> {ok, state()}. 68 | process(Input, _Last, #state{p=Partition, fd=FittingDetails}=State) -> 69 | Fun = FittingDetails#fitting_details.arg, 70 | ok = riak_pipe_fun:compat_apply( 71 | Fun, [Input, Partition, FittingDetails]), 72 | {ok, State}. 73 | 74 | %% @doc Unused. 75 | -spec done(state()) -> ok. 76 | done(_State) -> 77 | ok. 78 | 79 | %% @doc Check that the argument is a valid function of arity 3. See 80 | %% {@link riak_pipe_v_validate_function/3}. 81 | -spec validate_arg(term()) -> ok | {error, iolist()}. 82 | validate_arg(Fun) when is_function(Fun) -> 83 | riak_pipe_v:validate_function("arg", 3, Fun); 84 | validate_arg(Fun) -> 85 | {error, io_lib:format("~p requires a function as argument, not a ~p", 86 | [?MODULE, riak_pipe_v:type_of(Fun)])}. 87 | -------------------------------------------------------------------------------- /eqc/reduce_fitting_pulse_sink_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Support module for reduce_fitting_pulse tests - supervisor for 22 | %% fsm-style sink, like Riak KV uses. 23 | -module(reduce_fitting_pulse_sink_sup). 24 | 25 | -behaviour(supervisor). 26 | 27 | %% API 28 | -export([start_link/0]). 29 | -export([start_sink/2, 30 | terminate_sink/1]). 31 | 32 | %% Supervisor callbacks 33 | -export([init/1]). 34 | 35 | -ifdef(PULSE). 36 | -include_lib("pulse/include/pulse.hrl"). 37 | %% have to transform the 'receive' of the work results 38 | -compile({parse_transform, pulse_instrument}). 39 | %% don't trasnform toplevel test functions 40 | -compile({pulse_replace_module,[{supervisor,pulse_supervisor}]}). 41 | -endif. 42 | 43 | %%%=================================================================== 44 | %%% API functions 45 | %%%=================================================================== 46 | 47 | %% @doc Start the supervisor. 48 | -spec start_link() -> {ok, pid()} | ignore | {error, term()}. 49 | start_link() -> 50 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 51 | 52 | %% @doc Start a new worker under the supervisor. 53 | -spec start_sink(pid(), reference()) -> {ok, pid()}. 54 | start_sink(Owner, Ref) -> 55 | supervisor:start_child(?MODULE, [Owner, Ref]). 56 | 57 | %% @doc Stop a worker immediately 58 | -spec terminate_sink(pid()) -> ok | {error, term()}. 59 | terminate_sink(Sink) -> 60 | supervisor:terminate_child(?MODULE, Sink). 61 | 62 | %%%=================================================================== 63 | %%% Supervisor callbacks 64 | %%%=================================================================== 65 | 66 | %% @doc Initialize the supervisor. This is a `simple_one_for_one', 67 | %% whose child spec is for starting `riak_kv_mrc_sink' FSMs. 68 | -spec init([]) -> {ok, {{supervisor:strategy(), 69 | pos_integer(), 70 | pos_integer()}, 71 | [ supervisor:child_spec() ]}}. 72 | init([]) -> 73 | RestartStrategy = simple_one_for_one, 74 | MaxRestarts = 1000, 75 | MaxSecondsBetweenRestarts = 3600, 76 | 77 | SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, 78 | 79 | Restart = temporary, 80 | Shutdown = 2000, 81 | Type = worker, 82 | 83 | AChild = {undefined, % no registered name 84 | {reduce_fitting_pulse_sink, start_link, []}, 85 | Restart, Shutdown, Type, [reduce_fitting_pulse_sink]}, 86 | 87 | {ok, {SupFlags, [AChild]}}. 88 | 89 | %%%=================================================================== 90 | %%% Internal functions 91 | %%%=================================================================== 92 | -------------------------------------------------------------------------------- /riak_pipe_monitors.dot: -------------------------------------------------------------------------------- 1 | // A map of the links and monitors among Riak Pipe processes 2 | 3 | // This map shows the basic web of monitors and links present in the 4 | // system when: 5 | // - a pipeline "A" is setup with two fittings, "1" and "2" 6 | // - a vnode is handling inputs for both of those fittings 7 | // (only one vnode is shown, for simplicity in the diagram) 8 | 9 | // Edges are colored according to the type of link/monitor (or the 10 | // special-case link of supervision): 11 | // supervises = [color="#009900"]; 12 | // links = [color="#990000"]; 13 | // monitors = [color="#000099"]; 14 | 15 | // Render using Graphviz: 16 | // dot -Tpng -oriak_pipe_monitors.png riak_pipe_monitors.dot 17 | 18 | // Notes: 19 | // - the links from riak_pipe_vnode to riak_pipe_vnode_worker_A* were 20 | // put in place before riak_core_vnode supported handle_info; they 21 | // should be changed to monitors now 22 | digraph { 23 | // application-immortal processes 24 | subgraph apps { 25 | rank=min; 26 | riak_pipe_sup 27 | riak_core_sup 28 | } 29 | 30 | subgraph immortal { 31 | {rank=same; 32 | riak_pipe_vnode_master 33 | riak_pipe_builder_sup 34 | riak_pipe_fitting_sup 35 | riak_core_vnode_sup} 36 | 37 | // started at application start time 38 | riak_pipe_sup -> riak_pipe_vnode_master [color="#009900"]; 39 | riak_pipe_sup -> riak_pipe_builder_sup [color="#009900"]; 40 | riak_pipe_sup -> riak_pipe_fitting_sup [color="#009900"]; 41 | 42 | // external to pipe 43 | riak_core_sup -> riak_core_vnode_sup [color="#009900"]; 44 | } 45 | 46 | // application-mortal processes 47 | // started as vnode requests arrive 48 | subgraph mortal { 49 | riak_pipe_vnode 50 | riak_pipe_vnode_worker_sup 51 | 52 | riak_core_vnode_sup -> riak_pipe_vnode [color="#009900"]; 53 | riak_pipe_vnode_master -> riak_pipe_vnode [color="#000099"]; 54 | 55 | riak_pipe_vnode -> riak_pipe_vnode_worker_sup [color="#990000"]; 56 | } 57 | 58 | // the builder/fitting "meta" processes 59 | subgraph pipeline_meta_A { 60 | riak_pipe_builder_A 61 | 62 | riak_pipe_fitting_A1 63 | riak_pipe_fitting_A2 64 | 65 | // started as pipelines are set up 66 | riak_pipe_builder_sup -> riak_pipe_builder_A [color="#009900"]; 67 | 68 | riak_pipe_fitting_sup -> riak_pipe_fitting_A1 [color="#009900"]; 69 | riak_pipe_fitting_sup -> riak_pipe_fitting_A2 [color="#009900"]; 70 | 71 | riak_pipe_builder_A -> riak_pipe_fitting_A1 [color="#000099"]; 72 | riak_pipe_builder_A -> riak_pipe_fitting_A2 [color="#000099"]; 73 | 74 | riak_pipe_fitting_A1 -> riak_pipe_builder_A [color="#000099"]; 75 | riak_pipe_fitting_A2 -> riak_pipe_builder_A [color="#000099"]; 76 | 77 | // builder also monitors the process receiving outputs 78 | riak_pipe_builder_A -> sink_process_A [color="#000099"]; 79 | } 80 | 81 | // the processes actually processing inputs 82 | subgraph pipeline_A { 83 | riak_pipe_vnode_worker_A1 84 | riak_pipe_vnode_worker_A2 85 | 86 | riak_pipe_vnode_worker_sup -> riak_pipe_vnode_worker_A1 [color="#009900"]; 87 | riak_pipe_vnode_worker_sup -> riak_pipe_vnode_worker_A2 [color="#009900"]; 88 | 89 | riak_pipe_vnode -> riak_pipe_vnode_worker_A1 [color="#990000"]; 90 | riak_pipe_vnode -> riak_pipe_vnode_worker_A2 [color="#990000"]; 91 | 92 | riak_pipe_vnode -> riak_pipe_fitting_A1 [color="#000099"]; 93 | riak_pipe_vnode -> riak_pipe_fitting_A2 [color="#000099"]; 94 | 95 | riak_pipe_fitting_A1 -> riak_pipe_vnode [color="#000099"]; 96 | riak_pipe_fitting_A2 -> riak_pipe_vnode [color="#000099"]; 97 | } 98 | } -------------------------------------------------------------------------------- /src/riak_pipe_stat.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% Copyright (c) 2007-2011 Basho Technologies, Inc. All Rights Reserved. 3 | %% 4 | %% This file is provided to you under the Apache License, 5 | %% Version 2.0 (the "License"); you may not use this file 6 | %% except in compliance with the License. You may obtain 7 | %% a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, 12 | %% software distributed under the License is distributed on an 13 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | %% KIND, either express or implied. See the License for the 15 | %% specific language governing permissions and limitations 16 | %% under the License. 17 | %% 18 | %% ------------------------------------------------------------------- 19 | %% 20 | %% @doc Collector for various pipe stats. 21 | -module(riak_pipe_stat). 22 | 23 | -behaviour(gen_server). 24 | 25 | %% API 26 | -export([start_link /0, register_stats/0, 27 | get_stats/0, 28 | update/1, 29 | stats/0]). 30 | 31 | %% gen_server callbacks 32 | -export([init/1, handle_call/3, handle_cast/2, handle_info/2, 33 | terminate/2, code_change/3]). 34 | 35 | -define(SERVER, ?MODULE). 36 | -define(APP, riak_pipe). 37 | -define(PFX, riak_core_stat:prefix()). 38 | 39 | -type stat_type() :: counter | spiral. 40 | -type stat_options() :: [tuple()]. 41 | -type stat_aliases() :: [{exometer:datapoint(), atom()}]. 42 | 43 | %% ------------------------------------------------------------------- 44 | %% API 45 | %% ------------------------------------------------------------------- 46 | 47 | start_link() -> 48 | gen_server:start_link({local, ?SERVER}, ?MODULE, [], []). 49 | 50 | register_stats() -> 51 | riak_core_stat:register_stats(?APP, stats()). 52 | 53 | %% @doc Return current aggregation of all stats. 54 | -spec get_stats() -> proplists:proplist(). 55 | get_stats() -> 56 | riak_core_stat:get_stats(?APP). 57 | 58 | update(Arg) -> 59 | gen_server:cast(?SERVER, {update, Arg}). 60 | 61 | %% gen_server 62 | 63 | init([]) -> 64 | register_stats(), 65 | {ok, ok}. 66 | 67 | handle_call(_Req, _From, State) -> 68 | {reply, ok, State}. 69 | 70 | handle_cast({update, {create, Pid}}, State) -> 71 | erlang:monitor(process, Pid), 72 | do_update(create), 73 | {noreply, State}; 74 | handle_cast({update, Arg}, State) -> 75 | do_update(Arg), 76 | {noreply, State}; 77 | handle_cast(_Req, State) -> 78 | {noreply, State}. 79 | 80 | handle_info({'DOWN', _Ref, process, _Pid, _Reason}, State) -> 81 | do_update(destroy), 82 | {noreply, State}; 83 | handle_info(_Info, State) -> 84 | {noreply, State}. 85 | 86 | terminate(_Reason, _State) -> 87 | ok. 88 | 89 | code_change(_OldVsn, State, _Extra) -> 90 | {ok, State}. 91 | 92 | %% @doc Update the given `Stat'. 93 | -spec do_update(term()) -> ok. 94 | do_update(create) -> 95 | ok = exometer:update([?PFX, ?APP, pipeline, create], 1), 96 | exometer:update([?PFX, ?APP, pipeline, active], 1); 97 | do_update(create_error) -> 98 | exometer:update([?PFX, ?APP, pipeline, create, error], 1); 99 | do_update(destroy) -> 100 | exometer:update([?PFX, ?APP, pipeline, active], -1). 101 | 102 | %% ------------------------------------------------------------------- 103 | %% Private 104 | %% ------------------------------------------------------------------- 105 | -spec stats() -> [{riak_core_stat_q:path(), stat_type(), stat_options(), 106 | stat_aliases()}]. 107 | stats() -> 108 | [ 109 | {[pipeline, create], spiral, [], [{count, pipeline_create_count}, 110 | {one, pipeline_create_one}]}, 111 | {[pipeline, create, error], spiral, [], [{count, pipeline_create_error_count}, 112 | {one, pipeline_create_error_one}]}, 113 | {[pipeline, active], counter, [], [{value, pipeline_active}]} 114 | ]. 115 | -------------------------------------------------------------------------------- /src/riak_pipe_fitting_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Supervisor of fitting processes. 22 | -module(riak_pipe_fitting_sup). 23 | 24 | -behaviour(supervisor). 25 | 26 | %% API 27 | -export([start_link/0]). 28 | -export([add_fitting/4, 29 | terminate_fitting/1]). 30 | 31 | %% Supervisor callbacks 32 | -export([init/1]). 33 | 34 | -include("riak_pipe.hrl"). 35 | -include("riak_pipe_debug.hrl"). 36 | 37 | -ifdef(PULSE). 38 | -include_lib("pulse/include/pulse.hrl"). 39 | %% have to transform the 'receive' of the work results 40 | -compile({parse_transform, pulse_instrument}). 41 | %% don't trasnform toplevel test functions 42 | -compile({pulse_replace_module,[{supervisor,pulse_supervisor}]}). 43 | -endif. 44 | 45 | -define(SERVER, ?MODULE). 46 | 47 | %%%=================================================================== 48 | %%% API functions 49 | %%%=================================================================== 50 | 51 | %% @doc Start the supervisor. It will be registered under the atom 52 | %% `riak_pipe_fitting_sup'. 53 | -spec start_link() -> {ok, pid()} | ignore | {error, term()}. 54 | start_link() -> 55 | supervisor:start_link({local, ?SERVER}, ?MODULE, []). 56 | 57 | %% @doc Start a new fitting coordinator under this supervisor. 58 | -spec add_fitting(pid(), 59 | riak_pipe:fitting_spec(), 60 | riak_pipe:fitting(), 61 | riak_pipe:exec_opts()) -> 62 | {ok, pid(), riak_pipe:fitting()}. 63 | add_fitting(Builder, Spec, Output, Options) -> 64 | ?DPF("Adding fitting for ~p", [Spec]), 65 | supervisor:start_child(?SERVER, [Builder, Spec, Output, Options]). 66 | 67 | %% @doc Terminate a coordinator immediately. Useful for tearing down 68 | %% pipelines that may be otherwise swamped with messages from 69 | %% restarting workers. 70 | -spec terminate_fitting(riak_pipe:fitting()) -> ok | {error, term()}. 71 | terminate_fitting(#fitting{pid=Pid}) -> 72 | supervisor:terminate_child(?SERVER, Pid). 73 | 74 | %%%=================================================================== 75 | %%% Supervisor callbacks 76 | %%%=================================================================== 77 | 78 | %% @doc Initialize this supervisor. This is a `simple_one_for_one', 79 | %% whose child spec is for starting `riak_pipe_fitting' FSMs. 80 | -spec init([]) -> {ok, {{supervisor:strategy(), 81 | pos_integer(), 82 | pos_integer()}, 83 | [ supervisor:child_spec() ]}}. 84 | init([]) -> 85 | RestartStrategy = simple_one_for_one, 86 | MaxRestarts = 1000, 87 | MaxSecondsBetweenRestarts = 3600, 88 | 89 | SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, 90 | 91 | Restart = temporary, 92 | Shutdown = 2000, 93 | Type = worker, 94 | 95 | Child = {undefined, 96 | {riak_pipe_fitting, start_link, []}, 97 | Restart, Shutdown, Type, [riak_pipe_fitting]}, 98 | 99 | {ok, {SupFlags, [Child]}}. 100 | 101 | %%%=================================================================== 102 | %%% Internal functions 103 | %%%=================================================================== 104 | -------------------------------------------------------------------------------- /src/riak_pipe_cinfo.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc `cluster_info' interrogation module. 22 | -module(riak_pipe_cinfo). 23 | 24 | -export([cluster_info_init/0, 25 | cluster_info_generator_funs/0]). 26 | 27 | -include("riak_pipe.hrl"). 28 | -include("riak_pipe_debug.hrl"). 29 | 30 | %% @doc Unused. 31 | -spec cluster_info_init() -> ok. 32 | cluster_info_init() -> 33 | ok. 34 | 35 | %% @doc Get the list of cluster_info-generating functions. Current 36 | %% functions print information about pipelines, and vnode queues. 37 | -spec cluster_info_generator_funs() -> 38 | [ {Name::string(), Interrogator::function()} ]. 39 | cluster_info_generator_funs() -> 40 | [ 41 | {"Riak Pipelines", fun pipelines/1}, 42 | {"Riak Pipeline Vnode Queues", fun queues/1} 43 | ]. 44 | 45 | %% @doc Print information about the active pipelines. 46 | -spec pipelines(ClusterInfoHandle::term()) -> ok. 47 | pipelines(C) -> 48 | Pipes = riak_pipe_builder_sup:builder_pids(), 49 | cluster_info:format(C, "Pipelines active: ~b~n", [length(Pipes)]), 50 | _ = [ pipeline(C, Pipe) || Pipe <- Pipes], 51 | ok. 52 | 53 | %% @doc Print information about the given pipeline. 54 | %% The pid given should be that of the builder of the pipeline. 55 | -spec pipeline(ClusterInfoHandle::term(), Builder::pid()) -> ok. 56 | pipeline(C, Pipe) -> 57 | case riak_pipe_builder:fitting_pids(Pipe) of 58 | {ok, Fits} -> 59 | cluster_info:format(C, " - ~p fittings: ~b alive~n", 60 | [Pipe, length(Fits)]), 61 | [ fitting(C, Fit) || Fit <- Fits ]; 62 | gone -> 63 | cluster_info:format(C, " - ~p *gone*~n", [Pipe]) 64 | end, 65 | ok. 66 | 67 | %% @doc Print information about the given fitting. 68 | %% The pid given should be that of the fitting. 69 | -spec fitting(ClusterInfoHandle::term(), Fitting::pid()) -> ok. 70 | fitting(C, Fit) -> 71 | case riak_pipe_fitting:workers(Fit) of 72 | {ok, Workers} -> 73 | %% TODO: add 'name' from details? maybe module/etc. too? 74 | cluster_info:format(C, " + ~p worker partitions: ~b~n", 75 | [Fit, length(Workers)]), 76 | [ fitting_worker(C, W) || W <- Workers ]; 77 | gone -> 78 | cluster_info:format(C, " + ~p *gone*~n", [Fit]) 79 | end, 80 | ok. 81 | 82 | %% @doc Print the ring partition index of a vnode doing work for some 83 | %% fitting. The `Worker' should be the index to print. 84 | -spec fitting_worker(ClusterInfoHandle::term(), 85 | Worker::riak_pipe_vnode:partition()) -> ok. 86 | fitting_worker(C, W) -> 87 | cluster_info:format(C, " * ~p~n", [W]), 88 | ok. 89 | 90 | %% @doc Print information about each active `riak_pipe' vnode. 91 | -spec queues(ClusterInfoHandle::term()) -> ok. 92 | queues(C) -> 93 | VnodePids = riak_core_vnode_master:all_nodes(riak_pipe_vnode), 94 | cluster_info:format(C, "Vnodes active: ~b~n", [length(VnodePids)]), 95 | _ = [ queues(C, V) || V <- VnodePids], 96 | ok. 97 | 98 | %% @doc Print information about the workers on the given vnode. 99 | -spec queues(ClusterInfoHandle::term(), Vnode::pid()) -> ok. 100 | queues(C, V) -> 101 | {Partition, Workers} = riak_pipe_vnode:status(V), 102 | cluster_info:format(C, " - ~p workers: ~b~n", 103 | [Partition, length(Workers)]), 104 | _ = [ queue(C, W) || W <- Workers ], 105 | ok. 106 | 107 | %% @doc Print information about the given worker. 108 | -spec queue(ClusterInfoHandle::term(), [{atom(), term()}]) -> ok. 109 | queue(C, WorkerProps) -> 110 | cluster_info:format(C, " + ~p~n", [WorkerProps]), 111 | ok. 112 | -------------------------------------------------------------------------------- /src/riak_pipe_log.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Logging support for pipes. 22 | 23 | -module(riak_pipe_log). 24 | 25 | -export([log/2, 26 | trace/3]). 27 | 28 | -include("riak_pipe.hrl"). 29 | -include_lib("kernel/include/logger.hrl"). 30 | 31 | -export_type([trace_filter/0]). 32 | 33 | -ifdef(namespaced_types). 34 | -type riak_pipe_log_set() :: sets:set(). 35 | -else. 36 | -type riak_pipe_log_set() :: set(). 37 | -endif. 38 | 39 | -type trace_filter() :: all | riak_pipe_log_set() | trace_compiled(). 40 | -type trace_compiled() :: ordsets:ordset(term()). 41 | 42 | %% @doc Log the given message, if logging is enabled, to the specified 43 | %% log target. Logging is enabled and directed via the `log' 44 | %% option passed to {@link riak_pipe:exec/2}. If the option was 45 | %% set to `sink', log messages are sent to the sink. If the 46 | %% option was set to `sasl', log messages are printed via 47 | %% `error_logger' to the SASL log. If the option was set to 48 | %% `logger', log messages are printed via `logger' to the Riak 49 | %% node's log. If no option was given, log messages are 50 | %% discarded. 51 | -spec log(riak_pipe_fitting:details(), term()) -> ok. 52 | log(#fitting_details{options=O, name=N}, Msg) -> 53 | case proplists:get_value(log, O) of 54 | undefined -> 55 | ok; %% no logging 56 | sink -> 57 | Sink = proplists:get_value(sink, O), 58 | riak_pipe_sink:log(N, Sink, Msg, O); 59 | {sink, Sink} -> 60 | riak_pipe_sink:log(N, Sink, Msg, O); 61 | logger -> 62 | ?LOG_INFO( 63 | "~s: ~P", 64 | [riak_pipe_fitting:format_name(N), Msg, 9]); 65 | sasl -> 66 | error_logger:info_msg( 67 | "Pipe log -- ~s:~n~P", 68 | [riak_pipe_fitting:format_name(N), Msg, 9]) 69 | end. 70 | 71 | %% @doc Log a trace message. If any of the `Types' given matches any 72 | %% of the types in the `trace' option that was passed to {@link 73 | %% riak_pipe:exec/2} (or if `trace' was set to `all'), the trace 74 | %% message will be sent to the log target (if logging is enabled; 75 | %% see {@link log/2}). If no `trace' option was given, or no 76 | %% type matches, the message is discarded. 77 | %% 78 | %% The `node()' and the name of the fitting will be added to the 79 | %% `Types' list - the calling function does not need to specify 80 | %% them. 81 | -spec trace(riak_pipe_fitting:details(), [term()], term()) -> ok. 82 | trace(#fitting_details{options=O, name=N}=FD, Types, Msg) -> 83 | TraceOn = case proplists:get_value(trace, O) of 84 | all -> 85 | {true, all}; 86 | undefined -> 87 | false; 88 | EnabledSet when is_list(EnabledSet) -> 89 | %% ordsets (post 1.2) 90 | find_enabled([N|Types], EnabledSet); 91 | EnabledSet -> 92 | %% sets (1.2 and earlier) 93 | OS = ordsets:from_list(sets:to_list(EnabledSet)), 94 | find_enabled([N|Types], OS) 95 | end, 96 | case TraceOn of 97 | {true, Traces} -> log(FD, {trace, Traces, Msg}); 98 | false -> ok 99 | end. 100 | 101 | -spec find_enabled(list(), trace_compiled()) -> 102 | {true, list()} | false. 103 | find_enabled(Types, Enabled) -> 104 | MatchSet = ordsets:from_list([node()|Types]), 105 | Intersection = ordsets:intersection(Enabled, MatchSet), 106 | case Intersection of 107 | [] -> false; 108 | _ -> {true, Intersection} 109 | end. 110 | -------------------------------------------------------------------------------- /src/riak_pipe_builder_sup.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Supervisor for the pipe builder processes. 22 | %% 23 | %% This supervisor is mostly convenience for later investigation, 24 | %% to learn how many pipelines are active. No restart strategy 25 | %% is used. 26 | -module(riak_pipe_builder_sup). 27 | 28 | -behaviour(supervisor). 29 | 30 | %% API 31 | -export([start_link/0]). 32 | -export([new_pipeline/2, 33 | pipelines/0, 34 | builder_pids/0]). 35 | 36 | %% Supervisor callbacks 37 | -export([init/1]). 38 | 39 | -include("riak_pipe.hrl"). 40 | 41 | -ifdef(PULSE). 42 | -include_lib("pulse/include/pulse.hrl"). 43 | %% have to transform the 'receive' of the work results 44 | -compile({parse_transform, pulse_instrument}). 45 | %% don't trasnform toplevel test functions 46 | -compile({pulse_replace_module,[{supervisor,pulse_supervisor}]}). 47 | -endif. 48 | 49 | -define(SERVER, ?MODULE). 50 | 51 | %%%=================================================================== 52 | %%% API functions 53 | %%%=================================================================== 54 | 55 | %% @doc Start the supervisor. It will be registered under the atom 56 | %% `riak_pipe_builder_sup'. 57 | -spec start_link() -> {ok, pid()} | ignore | {error, term()}. 58 | start_link() -> 59 | supervisor:start_link({local, ?SERVER}, ?MODULE, []). 60 | 61 | %% @doc Start a new pipeline builder. Starts the builder process 62 | %% under this supervisor. 63 | -spec new_pipeline([#fitting_spec{}], riak_pipe:exec_opts()) -> 64 | {ok, Pipe::#pipe{}} | {error, Reason::term()}. 65 | new_pipeline(Spec, Options) -> 66 | case supervisor:start_child(?MODULE, [Spec, Options]) of 67 | {ok, Pid, Ref} -> 68 | case riak_pipe_builder:pipeline(Pid) of 69 | {ok, #pipe{sink=#fitting{ref=Ref}}=Pipe} -> 70 | riak_pipe_stat:update({create, Pid}), 71 | {ok, Pipe}; 72 | _ -> 73 | riak_pipe_stat:update(create_error), 74 | {error, startup_failure} 75 | end; 76 | Error -> 77 | riak_pipe_stat:update(create_error), 78 | Error 79 | end. 80 | 81 | %% @doc Get the list of pipelines hosted on this node. 82 | -spec pipelines() -> [#pipe{}]. 83 | pipelines() -> 84 | Children = builder_pids(), 85 | Responses = [ riak_pipe_builder:pipeline(BuilderPid) 86 | || BuilderPid <- Children ], 87 | %% filter out gone responses 88 | [ P || {ok, #pipe{}=P} <- Responses ]. 89 | 90 | %% @doc Get information about the builders supervised here. 91 | -spec builder_pids() -> [term()]. 92 | builder_pids() -> 93 | [ Pid || {_, Pid, _, _} <- supervisor:which_children(?SERVER) ]. 94 | 95 | %%%=================================================================== 96 | %%% Supervisor callbacks 97 | %%%=================================================================== 98 | 99 | %% @doc Initialize this supervisor. This is a `simple_one_for_one', 100 | %% whose child spec is for starting `riak_pipe_builder' FSMs. 101 | -spec init([]) -> {ok, {{supervisor:strategy(), 102 | pos_integer(), 103 | pos_integer()}, 104 | [ supervisor:child_spec() ]}}. 105 | init([]) -> 106 | RestartStrategy = simple_one_for_one, 107 | MaxRestarts = 1000, 108 | MaxSecondsBetweenRestarts = 3600, 109 | 110 | SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, 111 | 112 | Restart = temporary, 113 | Shutdown = brutal_kill, 114 | Type = worker, 115 | 116 | Child = {undefined, {riak_pipe_builder, start_link, []}, 117 | Restart, Shutdown, Type, [riak_pipe_builder]}, 118 | 119 | {ok, {SupFlags, [Child]}}. 120 | 121 | %%%=================================================================== 122 | %%% Internal functions 123 | %%%=================================================================== 124 | -------------------------------------------------------------------------------- /eqc/reduce_fitting_pulse_sink.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Support module for reduce_fitting_pulse tests - implements a 22 | %% pipe fsm-style sink. 23 | -module(reduce_fitting_pulse_sink). 24 | 25 | -compile([export_all, nowarn_export_all]). 26 | 27 | -compile([{nowarn_deprecated_function, 28 | [{gen_fsm, start_link, 3}, 29 | {gen_fsm, sync_send_event, 3}, 30 | {gen_fsm, reply, 2}]}]). 31 | 32 | %% compiler gets angry if behavior functions are not exported explicitly 33 | -export([init/1,handle_event/3,handle_sync_event/4,terminate/3, 34 | handle_info/3,code_change/4]). 35 | 36 | -ifdef(PULSE). 37 | -include_lib("pulse/include/pulse.hrl"). 38 | %% have to transform the 'receive' of the work results 39 | -compile({parse_transform, pulse_instrument}). 40 | %% don't trasnform toplevel test functions 41 | -compile({pulse_replace_module,[{gen_fsm,pulse_gen_fsm}]}). 42 | -endif. 43 | 44 | -include("include/riak_pipe.hrl"). 45 | 46 | -behaviour(gen_fsm). 47 | 48 | -record(state, {monitor, owner, ref, builder, results=[], logs=[], waiter}). 49 | 50 | start_link(Owner, Ref) -> 51 | gen_fsm:start_link(?MODULE, [Owner, Ref], []). 52 | 53 | use_pipe(Sink, Ref, Pipe) -> 54 | gen_fsm:sync_send_event(Sink, {use_pipe, Ref, Pipe}, infinity). 55 | 56 | all_results(Sink, Ref) -> 57 | gen_fsm:sync_send_event(Sink, {all_results, Ref}, infinity). 58 | 59 | init([Owner, Ref]) -> 60 | Monitor = erlang:monitor(process, Owner), 61 | {ok, 62 | wait_pipe, 63 | #state{monitor=Monitor, owner=Owner, ref=Ref}}. 64 | 65 | wait_pipe({use_pipe, Ref, #pipe{builder=Builder}}, 66 | _From, 67 | #state{ref=Ref}=State) -> 68 | Monitor = erlang:monitor(process, Builder), 69 | {reply, 70 | ok, 71 | accumulating, 72 | State#state{builder={Monitor, Builder}}}. 73 | 74 | accumulating(#pipe_result{ref=Ref}=R, 75 | #state{ref=Ref, results=Results}=State) -> 76 | {next_state, accumulating, State#state{results=[R|Results]}}; 77 | accumulating(#pipe_log{ref=Ref}=L, 78 | #state{ref=Ref, logs=Logs}=State) -> 79 | {next_state, accumulating, State#state{logs=[L|Logs]}}; 80 | accumulating(#pipe_eoi{ref=Ref}, 81 | #state{ref=Ref, waiter=Waiter}=State) -> 82 | case Waiter of 83 | undefined -> 84 | {next_state, waiting, State}; 85 | _ -> 86 | gen_fsm:reply(Waiter, State#state.results), 87 | {stop, normal, State} 88 | end. 89 | 90 | accumulating({all_results, Ref}, From, #state{ref=Ref}=State) -> 91 | {next_state, accumulating, State#state{waiter=From}}; 92 | accumulating(Other, _From, State) -> 93 | case accumulating(Other, State) of 94 | {next_state, NextStateName, NextState} -> 95 | {reply, ok, NextStateName, NextState}; 96 | {stop, Reason, State} -> 97 | {stop, Reason, ok, State}; 98 | Response -> 99 | Response 100 | end. 101 | 102 | waiting({all_results, Ref}, 103 | _From, 104 | #state{ref=Ref, results=Results}=State) -> 105 | {stop, normal, Results, State}. 106 | 107 | handle_info({'DOWN', Monitor, process, Owner, _Reason}, 108 | _StateName, 109 | #state{monitor=Monitor, owner=Owner}=State) -> 110 | {stop, normal, State}; 111 | handle_info({'DOWN', Monitor, process, Builder, Reason}, 112 | StateName, 113 | #state{builder={Monitor, Builder}}=State) -> 114 | case Reason of 115 | normal -> 116 | %% mimicking riak_kv_mrc_sink for the moment 117 | {next_state, StateName, State}; 118 | _ -> 119 | %% also mimicking riak_kv_mrc_sink 120 | {stop, normal, State} 121 | end. 122 | 123 | handle_event(_, _, State) -> 124 | {stop, unknown_message, State}. 125 | 126 | handle_sync_event(_, _, _, State) -> 127 | {stop, unknown_message, State}. 128 | 129 | code_change(_Old, StateName, State, _Extra) -> 130 | {ok, StateName, State}. 131 | 132 | terminate(_Reason, _StateName, _State) -> 133 | ok. 134 | -------------------------------------------------------------------------------- /src/riak_pipe_w_rec_countdown.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Proof of concept for recursive input (fitting sending output 22 | %% to itself). When this fitting receives an input, it passes 23 | %% that input to its output, and also passes `Input-1' to itself 24 | %% as input until the input is `0'. Thus, sending `3' as the 25 | %% input to this fitting, would result in the outputs `3', `2', 26 | %% `1', and `0'. That is: 27 | %% 28 | %%``` 29 | %% Spec = [#fitting_spec{name=counter, 30 | %% module=riak_pipe_w_rec_countdown}], 31 | %% {ok, Pipe} = riak_pipe:exec(Spec, []), 32 | %% riak_pipe:queue_work(Pipe, 3), 33 | %% riak_pipe:eoi(Pipe), 34 | %% {eoi, Results, []} = riak_pipe:collect_results(Pipe). 35 | %% [{counter,0},{counter,1},{counter,2},{counter,3}] = Results. 36 | %%''' 37 | %% 38 | %% This fitting should work with any consistent-hash function. 39 | %% It requires no archiving for handoff. 40 | %% 41 | %% If the argument is the atom `testeoi', then the final 42 | %% recursive input (`0') will be sent three times, with no delay 43 | %% before the second case and a 1-second delay before the third. 44 | %% These two sends should test the behavior of vnode enqueueing 45 | %% while attempting to force a worker to `done'. If all `eoi' 46 | %% handling is done properly, then `0' should appear three times 47 | %% in the result list. The `testeoi' case should go like this: 48 | %% 49 | %%``` 50 | %% Spec = [#fitting_spec{name=counter, 51 | %% module=riak_pipe_w_rec_countdown, 52 | %% arg=testeoi}], 53 | %% Options = [{trace,[restart]},{log,sink}], 54 | %% {ok, Pipe} = riak_pipe:exec(Spec, Options), 55 | %% riak_pipe:queue_work(Pipe, 3), 56 | %% riak_pipe:eoi(Pipe), 57 | %% {eoi, Results, Trace} = riak_pipe:collect_results(Pipe). 58 | %% [{counter,0},{counter,0},{counter,0}, 59 | %% {counter,1},{counter,2},{counter,3}] = Results. 60 | %% [{counter,{trace,[restart],{vnode,{restart,_}}}}] = Trace. 61 | %%''' 62 | %% 63 | %% If `Results' contains less than three instances of 64 | %% `{counter,0}', then the test failed. If `Trace' is empty, the 65 | %% done/eoi race was not triggered, and the test should be 66 | %% re-run. 67 | %% 68 | %% NOTE: This test code has been copied to the EUnit tests in riak_pipe.erl, 69 | %% into the basic_test_() collection. 70 | 71 | -module(riak_pipe_w_rec_countdown). 72 | -behaviour(riak_pipe_vnode_worker). 73 | 74 | -export([init/2, 75 | process/3, 76 | done/1]). 77 | 78 | -include("riak_pipe.hrl"). 79 | -include("riak_pipe_log.hrl"). 80 | 81 | -record(state, {p :: riak_pipe_vnode:partition(), 82 | fd :: riak_pipe_fitting:details()}). 83 | -opaque state() :: #state{}. 84 | -export_type([state/0]). 85 | 86 | %% @doc Initialization just stows the partition and fitting details in 87 | %% the module's state, for sending outputs in {@link process/3}. 88 | -spec init(riak_pipe_vnode:partition(), 89 | riak_pipe_fitting:details()) -> 90 | {ok, state()}. 91 | init(Partition, FittingDetails) -> 92 | {ok, #state{p=Partition, fd=FittingDetails}}. 93 | 94 | %% @doc Process sends `Input' directly to the next fitting, and also 95 | %% `Input-1' back to this fitting as new input. 96 | -spec process(term(), boolean(), state()) -> {ok, state()}. 97 | process(Input, _Last, #state{p=Partition, fd=FittingDetails}=State) -> 98 | ?T(FittingDetails, [], {input, Input, Partition}), 99 | ok = riak_pipe_vnode_worker:send_output(Input, Partition, FittingDetails), 100 | if Input =< 0 -> 101 | ok; 102 | Input == 1, FittingDetails#fitting_details.arg == testeoi -> 103 | ?T(FittingDetails, [], {zero1, Partition}), 104 | ok = riak_pipe_vnode_worker:recurse_input( 105 | 0, Partition, FittingDetails), 106 | ?T(FittingDetails, [], {zero2, Partition}), 107 | ok = riak_pipe_vnode_worker:recurse_input( 108 | 0, Partition, FittingDetails), 109 | timer:sleep(1000), 110 | ?T(FittingDetails, [], {zero3, Partition}), 111 | ok = riak_pipe_vnode_worker:recurse_input( 112 | 0, Partition, FittingDetails); 113 | true -> 114 | ?T(FittingDetails, [], {recinput, Input-1, Partition}), 115 | ok = riak_pipe_vnode_worker:recurse_input( 116 | Input-1, Partition, FittingDetails) 117 | end, 118 | {ok, State}. 119 | 120 | %% @doc Unused. 121 | -spec done(state()) -> ok. 122 | done(_State) -> 123 | ?T(_State#state.fd, [], {done, _State#state.p}), 124 | ok. 125 | -------------------------------------------------------------------------------- /src/riak_pipe_w_crash.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc It's what we do: crash. 22 | 23 | -module(riak_pipe_w_crash). 24 | -behaviour(riak_pipe_vnode_worker). 25 | 26 | -export([init/2, 27 | process/3, 28 | done/1]). 29 | 30 | -include("riak_pipe.hrl"). 31 | -include("riak_pipe_log.hrl"). 32 | 33 | -record(state, {p :: riak_pipe_vnode:partition(), 34 | fd :: riak_pipe_fitting:details()}). 35 | -opaque state() :: #state{}. 36 | -export_type([state/0]). 37 | 38 | %% name of the table reMEMbering restarts 39 | -define(MEM, ?MODULE). 40 | 41 | %% @doc Initialization just stows the partition and fitting details in 42 | %% the module's state, for sending outputs in {@link process/3}. 43 | -spec init(riak_pipe_vnode:partition(), 44 | riak_pipe_fitting:details()) -> 45 | {ok, state()}. 46 | init(Partition, FittingDetails) -> 47 | case FittingDetails#fitting_details.arg of 48 | init_exit -> 49 | exit(crash); 50 | init_badreturn -> 51 | crash; 52 | init_restartfail -> 53 | case is_restart(Partition, FittingDetails) of 54 | true -> 55 | restart_crash; 56 | false -> 57 | {ok, #state{p=Partition, fd=FittingDetails}} 58 | end; 59 | _ -> 60 | {ok, #state{p=Partition, fd=FittingDetails}} 61 | end. 62 | 63 | is_restart(Partition, FittingDetails) -> 64 | Fitting = FittingDetails#fitting_details.fitting, 65 | %% set the fitting coordinator as the heir, such that the ets 66 | %% table survives when this worker exits, but gets cleaned 67 | %% up when the pipeline shuts down 68 | case (catch ets:new(?MEM, [set, {keypos, 1}, 69 | named_table, public, 70 | {heir, Fitting#fitting.pid, ok}])) of 71 | {'EXIT',{badarg,_}} -> 72 | %% table was already created 73 | ok; 74 | ?MEM -> 75 | %% table is now created 76 | ok 77 | end, 78 | case ets:lookup(?MEM, Partition) of 79 | [] -> 80 | %% no record - not restart; 81 | %% make a record for the next start to check 82 | ets:insert(?MEM, {Partition, true}), 83 | false; 84 | [{Partition, true}] -> 85 | true 86 | end. 87 | 88 | %% @doc Process just sends `Input' directly to the next fitting. This 89 | %% function also generates two trace messages: `{processing, 90 | %% Input}' before sending the output, and `{processed, Input}' after 91 | %% the blocking output send has returned. This can be useful for 92 | %% dropping in another pipeline to watching data move through it. 93 | -spec process(term(), boolean(), state()) -> {ok, state()}. 94 | process(Input, _Last, #state{p=Partition, fd=FittingDetails}=State) -> 95 | ?T(FittingDetails, [], {processing, Input}), 96 | case FittingDetails#fitting_details.arg of 97 | Input -> 98 | if Input == init_restartfail -> 99 | %% "worker restart failure, input forwarding" test in 100 | %% riak_pipe exploits this timer:sleep to test both 101 | %% moments of forwarding: those items left in a 102 | %% failed-restart worker's queue, as well as those items 103 | %% sent to a worker that is already forwarding 104 | timer:sleep(1000); 105 | true -> ok 106 | end, 107 | ?T(FittingDetails, [], {crashing, Input}), 108 | exit(process_input_crash); 109 | {recurse_done_pause, _} -> 110 | %% "restart after eoi" test in riak_pipe uses this 111 | %% behavior see done/1 for more details 112 | ok = case Input of 113 | [_] -> ok; 114 | [_|More] -> 115 | timer:sleep(100), 116 | riak_pipe_vnode_worker:recurse_input( 117 | More, Partition, FittingDetails) 118 | end, 119 | {ok, State}; 120 | _Other -> 121 | ok = riak_pipe_vnode_worker:send_output( 122 | Input, Partition, FittingDetails), 123 | ?T(FittingDetails, [], {processed, Input}), 124 | {ok, State} 125 | end. 126 | 127 | %% @doc Unused. 128 | -spec done(state()) -> ok. 129 | done(#state{fd=FittingDetails}) -> 130 | case FittingDetails#fitting_details.arg of 131 | {recurse_done_pause, Time} -> 132 | %% "restart after eoi" test in riak_pipe exploits this 133 | %% sleep to get more input queued while the worker is 134 | %% shutting down 135 | timer:sleep(Time); 136 | _ -> 137 | ok 138 | end. 139 | -------------------------------------------------------------------------------- /src/riak_pipe_sink.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Methods for sending messages to the sink. 22 | %% 23 | %% Sink messages are delivered as three record types: 24 | %% `#pipe_result{}', `#pipe_log{}', and `#pipe_eoi{}'. 25 | -module(riak_pipe_sink). 26 | 27 | -compile({nowarn_deprecated_function, 28 | [{gen_fsm, send_event, 2}, 29 | {gen_fsm, sync_send_event, 3}]}). 30 | 31 | -export([ 32 | result/4, 33 | log/4, 34 | eoi/2, 35 | valid_sink_type/1 36 | ]). 37 | 38 | -include("riak_pipe.hrl"). 39 | 40 | -ifdef(PULSE). 41 | -include_lib("pulse/include/pulse.hrl"). 42 | %% have to transform the 'receive' of the work results 43 | -compile({parse_transform, pulse_instrument}). 44 | %% don't trasnform toplevel test functions 45 | -compile({pulse_replace_module,[{gen_fsm,pulse_gen_fsm}]}). 46 | -endif. 47 | 48 | -export_type([sink_type/0]). 49 | -type sink_type() :: raw 50 | | {fsm, Period::integer(), Timeout::timeout()}. 51 | 52 | %% @doc Send a result to the sink (used by worker processes). The 53 | %% result is delivered as a `#pipe_result{}' record in the sink 54 | %% process's mailbox. 55 | -spec result(term(), Sink::riak_pipe:fitting(), term(), 56 | riak_pipe:exec_opts()) -> 57 | ok | {error, term()}. 58 | result(From, #fitting{pid=Pid, ref=Ref, chashfun=sink}, Output, Opts) -> 59 | send_to_sink(Pid, 60 | #pipe_result{ref=Ref, from=From, result=Output}, 61 | sink_type(Opts)). 62 | 63 | %% @doc Send a log message to the sink (used by worker processes and 64 | %% fittings). The message is delivered as a `#pipe_log{}' record 65 | %% in the sink process's mailbox. 66 | -spec log(term(), Sink::riak_pipe:fitting(), term(), list()) -> 67 | ok | {error, term()}. 68 | log(From, #fitting{pid=Pid, ref=Ref, chashfun=sink}, Msg, Opts) -> 69 | send_to_sink(Pid, #pipe_log{ref=Ref, from=From, msg=Msg}, 70 | sink_type(Opts)). 71 | 72 | %% @doc Send an end-of-inputs message to the sink (used by fittings). 73 | %% The message is delivered as a `#pipe_eoi{}' record in the sink 74 | %% process's mailbox. 75 | -spec eoi(Sink::riak_pipe:fitting(), list()) -> 76 | ok | {error, term()}. 77 | eoi(#fitting{pid=Pid, ref=Ref, chashfun=sink}, Opts) -> 78 | send_to_sink(Pid, #pipe_eoi{ref=Ref}, 79 | sink_type(Opts)). 80 | 81 | %% @doc Learn the type of sink we're dealing with from the execution 82 | %% options. 83 | -spec sink_type(riak_pipe:exec_opts()) -> sink_type(). 84 | sink_type(Opts) -> 85 | case lists:keyfind(sink_type, 1, Opts) of 86 | {_, Type} -> 87 | Type; 88 | false -> 89 | raw 90 | end. 91 | 92 | %% @doc Validate the type of sink given in the execution 93 | %% options. Returns `true' if the type is valid, or `{false, Type}' if 94 | %% invalid, where `Type' is what was found. 95 | -spec valid_sink_type(riak_pipe:exec_opts()) -> true | {false, term()}. 96 | valid_sink_type(Opts) -> 97 | case lists:keyfind(sink_type, 1, Opts) of 98 | {_, {fsm, Period, Timeout}} 99 | when (is_integer(Period) orelse Period == infinity), 100 | (is_integer(Timeout) orelse Timeout == infinity) -> 101 | true; 102 | %% other types as needed (fsm_async, for example) can go here 103 | {_, raw} -> 104 | true; 105 | false -> 106 | true; 107 | Other -> 108 | {false, Other} 109 | end. 110 | 111 | %% @doc Do the right kind of communication, given the sink type. 112 | -spec send_to_sink(pid(), 113 | #pipe_result{} | #pipe_log{} | #pipe_eoi{}, 114 | sink_type()) -> 115 | ok | {error, term()}. 116 | send_to_sink(Pid, Msg, raw) -> 117 | Pid ! Msg, 118 | ok; 119 | send_to_sink(Pid, Msg, {fsm, Period, Timeout}) -> 120 | case get(sink_sync) of 121 | undefined -> 122 | %% never sync for an 'infinity' Period, but always sync 123 | %% first send for any other Period, to prevent worker 124 | %% restart from overwhelming the sink 125 | send_to_sink_fsm(Pid, Msg, Timeout, Period /= infinity, 0); 126 | Count -> 127 | %% integer is never > than atom, so X is not > 'infinity' 128 | send_to_sink_fsm(Pid, Msg, Timeout, Count >= Period, Count) 129 | end. 130 | 131 | send_to_sink_fsm(Pid, Msg, _Timeout, false, Count) -> 132 | gen_fsm:send_event(Pid, Msg), 133 | put(sink_sync, Count+1), 134 | ok; 135 | send_to_sink_fsm(Pid, Msg, Timeout, true, _Count) -> 136 | try 137 | gen_fsm:sync_send_event(Pid, Msg, Timeout), 138 | put(sink_sync, 0), 139 | ok 140 | catch 141 | exit:{timeout,_} -> 142 | {error, timeout}; 143 | exit:{_Reason,{gen_fsm,sync_send_event,_}} -> 144 | %% we don't care why it died, just that it did ('noproc' 145 | %% and 'normal' have been seen; others could be possible) 146 | {error, sink_died}; 147 | exit:{_Reason,{pulse_gen_fsm,sync_send_event,_}} -> 148 | %% the pulse parse transform won't catch just the atom 'gen_fsm' 149 | {error, sink_died} 150 | end. 151 | 152 | -------------------------------------------------------------------------------- /src/riak_pipe_v.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Helpers for validating inputs. 22 | 23 | -module(riak_pipe_v). 24 | 25 | -export([validate_module/2, 26 | validate_function/3, 27 | type_of/1]). 28 | 29 | %% @doc Validate that `Module' is an atom that names a loaded or 30 | %% loadable module. If a module is already loaded under that 31 | %% name, or {@link code:load_file/1} is able to load one, the 32 | %% atom `ok' is returned. If no module is found, and `{error, 33 | %% Reason}' tuple is returned. (`Label' is used in the error 34 | %% message). 35 | -spec validate_module(string(), term()) -> ok | {error, iolist()}. 36 | validate_module(Label, Module) when is_atom(Module) -> 37 | case code:ensure_loaded(Module) of 38 | {module, Module} -> ok; 39 | {error, Error} -> 40 | {error, io_lib:format( 41 | "~s must be a valid module name" 42 | " (failed to load ~p: ~p)", 43 | [Label, Module, Error])} 44 | end; 45 | validate_module(Label, Module) -> 46 | {error, io_lib:format("~s must be an atom, not a ~p", 47 | [Label, type_of(Module)])}. 48 | 49 | %% @doc Validate that `Fun' is a function of arity `Arity'. 50 | %% 51 | %% If the function is of type `local' (anonymous functions, and 52 | %% functions named via `fun Name/Arity'), validation completes 53 | %% onces the arity is checked. 54 | %% 55 | %% If the function is of type `external' (functions named via 56 | %% `fun Module:Function/Arity'), then it is also verified that 57 | %% the module is loaded or loadable (see {@link 58 | %% validate_module/2}) and that it exports the named function. 59 | %% 60 | %% If validation completes successfully, the atom `ok' is 61 | %% returned. If validation failes, an `{error, Reason}' tuple is 62 | %% returned. (`Label' is used in the error message). 63 | -spec validate_function(string(), integer(), any() | {atom(), atom()}) -> 64 | ok | {error, iolist()}. 65 | validate_function(Label, Arity, {Module, Function}) 66 | when is_atom(Module), is_atom(Function) -> 67 | validate_exported_function(Label, Arity, Module, Function); 68 | validate_function(Label, Arity, Fun) when is_function(Fun) -> 69 | Info = erlang:fun_info(Fun), 70 | case proplists:get_value(arity, Info) of 71 | Arity -> 72 | case proplists:get_value(type, Info) of 73 | local -> 74 | %% reference was validated by compiler 75 | ok; 76 | external -> 77 | Module = proplists:get_value(module, Info), 78 | Function = proplists:get_value(name, Info), 79 | validate_exported_function( 80 | Label, Arity, Module, Function) 81 | end; 82 | N -> 83 | {error, io_lib:format("~s must be of arity ~b, not ~b", 84 | [Label, Arity, N])} 85 | end; 86 | validate_function(Label, Arity, Fun) -> 87 | {error, io_lib:format( 88 | "~s must be a function or {Mod, Fun} (arity ~b), not a ~p", 89 | [Label, Arity, type_of(Fun)])}. 90 | 91 | %% @doc Validate an exported function. See {@link validate_function/3}. 92 | -spec validate_exported_function(string(), integer(), atom(), atom()) -> 93 | ok | {error, iolist()}. 94 | validate_exported_function(Label, Arity, Module, Function) -> 95 | case validate_module("", Module) of 96 | ok -> 97 | Exports = Module:module_info(exports), 98 | case lists:member({Function,Arity}, Exports) of 99 | true -> 100 | ok; 101 | false -> 102 | {error, io_lib:format( 103 | "~s specifies ~p:~p/~b, which is not exported", 104 | [Label, Module, Function, Arity])} 105 | end; 106 | {error,Error} -> 107 | {error, io_lib:format("invalid module named in ~s function:~n~s", 108 | [Label, Error])} 109 | end. 110 | 111 | %% @doc Determine the type of a term. For example: 112 | %% ``` 113 | %% number = riak_pipe_v:type_of(1). 114 | %% atom = riak_pipe_v:type_of(a). 115 | %% pid = riak_pipe_v:type_of(self()). 116 | %% function = riak_pipe_v:type_of(fun() -> ok end). 117 | %% ''' 118 | -type checkable_types() :: 119 | pid | reference | list | tuple | atom | number | binary | function. 120 | 121 | -spec type_of(term()) -> checkable_types() | unidentified. 122 | type_of(Pid) when is_pid(Pid) -> 123 | pid; 124 | type_of(Ref) when is_reference(Ref) -> 125 | reference; 126 | type_of(List) when is_list(List) -> 127 | list; 128 | type_of(Tuple) when is_tuple(Tuple) -> 129 | tuple; 130 | type_of(Atom) when is_atom(Atom) -> 131 | atom; 132 | type_of(Number) when is_number(Number) -> 133 | number; 134 | type_of(Binary) when is_binary(Binary) -> 135 | binary; 136 | type_of(Fun) when is_function(Fun) -> 137 | function; 138 | type_of(_Other) -> 139 | unidentified. 140 | 141 | -ifdef(TEST). 142 | -include_lib("eunit/include/eunit.hrl"). 143 | 144 | check_types_test() -> 145 | P = self(), 146 | ?assert(pid == type_of(P)), 147 | L = [], 148 | ?assert(list == type_of(L)), 149 | T = {}, 150 | ?assert(tuple == type_of(T)), 151 | M = #{}, 152 | ?assert(unidentified == type_of(M)). 153 | 154 | 155 | -endif. 156 | -------------------------------------------------------------------------------- /priv/app.slave0.config: -------------------------------------------------------------------------------- 1 | %% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% ex: ts=4 sw=4 et 3 | [ 4 | %% Riak Core config 5 | {riak_core, [ 6 | %% Default location of ringstate 7 | {ring_state_dir, "data/ring"}, 8 | 9 | %% http is a list of IP addresses and TCP ports that the Riak 10 | %% HTTP interface will bind. 11 | {http, [ {"127.0.0.1", 38391 } ]}, 12 | 13 | %% https is a list of IP addresses and TCP ports that the Riak 14 | %% HTTPS interface will bind. 15 | %{https, [{ "127.0.0.1", 38091 }]}, 16 | 17 | %% default cert and key locations for https can be overridden 18 | %% with the ssl config variable 19 | %{ssl, [ 20 | % {certfile, "etc/cert.pem"}, 21 | % {keyfile, "etc/key.pem"} 22 | % ]}, 23 | 24 | %% riak_handoff_port is the TCP port that Riak uses for 25 | %% intra-cluster data handoff. 26 | {handoff_port, 38131 }, 27 | 28 | %% To encrypt riak_core intra-cluster data handoff traffic, 29 | %% uncomment the following line and edit its path to an 30 | %% appropriate certfile and keyfile. (This example uses a 31 | %% single file with both items concatenated together.) 32 | %{handoff_ssl_options, [{certfile, "/tmp/erlserver.pem"}]}, 33 | 34 | %% Platform-specific installation paths (substituted by rebar) 35 | {platform_bin_dir, "./bin"}, 36 | {platform_data_dir, "./data"}, 37 | {platform_etc_dir, "./etc"}, 38 | {platform_lib_dir, "./lib"}, 39 | {platform_log_dir, "./log"} 40 | ]}, 41 | 42 | %% Riak KV config 43 | {riak_kv, [ 44 | %% Storage_backend specifies the Erlang module defining the storage 45 | %% mechanism that will be used on this node. 46 | {storage_backend, riak_kv_bitcask_backend}, 47 | 48 | %% pb_ip is the IP address that the Riak Protocol Buffers interface 49 | %% will bind to. If this is undefined, the interface will not run. 50 | {pb_ip, "127.0.0.1" }, 51 | 52 | %% pb_port is the TCP port that the Riak Protocol Buffers interface 53 | %% will bind to 54 | {pb_port, 8081 }, 55 | 56 | %% pb_backlog is the maximum length to which the queue of pending 57 | %% connections may grow. If set, it must be an integer >= 0. 58 | %% By default the value is 5. If you anticipate a huge number of 59 | %% connections being initialised *simultaneously*, set this number 60 | %% higher. 61 | %% {pb_backlog, 64}, 62 | 63 | %% raw_name is the first part of all URLS used by the Riak raw HTTP 64 | %% interface. See riak_web.erl and raw_http_resource.erl for 65 | %% details. 66 | %{raw_name, "riak"}, 67 | 68 | %% mapred_name is URL used to submit map/reduce requests to Riak. 69 | {mapred_name, "mapred"}, 70 | 71 | %% directory used to store a transient queue for pending 72 | %% map tasks 73 | {mapred_queue_dir, "data/mr_queue" }, 74 | 75 | %% Each of the following entries control how many Javascript 76 | %% virtual machines are available for executing map, reduce, 77 | %% pre- and post-commit hook functions. 78 | {map_js_vm_count, 8 }, 79 | {reduce_js_vm_count, 6 }, 80 | {hook_js_vm_count, 2 }, 81 | 82 | %% Number of items the mapper will fetch in one request. 83 | %% Larger values can impact read/write performance for 84 | %% non-MapReduce requests. 85 | {mapper_batch_size, 5}, 86 | 87 | %% js_max_vm_mem is the maximum amount of memory, in megabytes, 88 | %% allocated to the Javascript VMs. If unset, the default is 89 | %% 8MB. 90 | {js_max_vm_mem, 8}, 91 | 92 | %% js_thread_stack is the maximum amount of thread stack, in megabyes, 93 | %% allocate to the Javascript VMs. If unset, the default is 16MB. 94 | %% NOTE: This is not the same as the C thread stack. 95 | {js_thread_stack, 16}, 96 | 97 | %% Number of objects held in the MapReduce cache. These will be 98 | %% ejected when the cache runs out of room or the bucket/key 99 | %% pair for that entry changes 100 | {map_cache_size, 10000}, 101 | 102 | %% js_source_dir should point to a directory containing Javascript 103 | %% source files which will be loaded by Riak when it initializes 104 | %% Javascript VMs. 105 | %{js_source_dir, "/tmp/js_source"}, 106 | 107 | %% riak_stat enables the use of the "riak-admin status" command to 108 | %% retrieve information the Riak node for performance and debugging needs 109 | {riak_kv_stat, true} 110 | ]}, 111 | 112 | %% Bitcask Config 113 | {bitcask, [ 114 | {data_root, "data/bitcask"} 115 | ]}, 116 | 117 | %% Luwak Config 118 | {luwak, [ 119 | {enabled, false} 120 | ]}, 121 | 122 | %% Riak_err Config 123 | {riak_err, [ 124 | %% Info/error/warning reports larger than this will be considered 125 | %% too big to be formatted safely with the user-supplied format 126 | %% string. 127 | {term_max_size, 65536}, 128 | 129 | %% Limit the total size of formatted info/error/warning reports. 130 | {fmt_max_bytes, 65536} 131 | ]}, 132 | 133 | %% riak_sysmon config 134 | {riak_sysmon, [ 135 | %% To disable forwarding events of a particular type, use a 136 | %% limit of 0. 137 | {process_limit, 30}, 138 | {port_limit, 30}, 139 | 140 | %% Finding reasonable limits for a given workload is a matter 141 | %% of experimentation. 142 | {gc_ms_limit, 50}, 143 | {heap_word_limit, 10485760} 144 | ]}, 145 | 146 | %% SASL config 147 | {sasl, [ 148 | {sasl_error_logger, {file, "log/sasl-error.log"}}, 149 | {errlog_type, error}, 150 | {error_logger_mf_dir, "log/sasl"}, % Log directory 151 | {error_logger_mf_maxbytes, 10485760}, % 10 MB max file size 152 | {error_logger_mf_maxfiles, 5} % 5 files max 153 | ]} 154 | ]. 155 | -------------------------------------------------------------------------------- /src/riak_pipe_w_reduce.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc A "reduce"-like fitting (in the MapReduce sense). Really more 22 | %% like a keyed list-fold. This fitting expects inputs of the 23 | %% form `{Key, Value}'. For each input, the fitting evaluates a 24 | %% function (its argument) on the `Value' and any previous result 25 | %% for that `Key', or `[]' (the empty list) if that `Key' has 26 | %% never been seen by this worker. When `done' is finally 27 | %% received, the fitting sends each key-value pair it has 28 | %% evaluated as an input to the next fittin. 29 | %% 30 | %% The intent is that a fitting might receive a stream of inputs 31 | %% like `{a, 1}, {b, 2}, {a 3}, {b, 4}' and send on results like 32 | %% `{a, 4}, {b, 6}' by using a simple "sum" function. 33 | %% 34 | %% This function expects a function as its argument. The function 35 | %% should be arity-4 and expect the arguments: 36 | %%
37 | %% `Key' :: term() 38 | %%
39 | %% Whatever aggregation key is necessary for the algorithm. 40 | %%
41 | %% `InAccumulator' :: [term()] 42 | %%
43 | %% A list composed of the new input, cons'd on the front of the 44 | %% result of the last evaluation for this `Key' (or the empty list 45 | %% if this is the first evaluation for the key). 46 | %%
47 | %% `Partition' :: riak_pipe_vnode:partition() 48 | %%
49 | %% The partition of the vnode on which this worker is running. 50 | %% (Useful for logging, or sending other output.) 51 | %%
52 | %% `FittingDetails' :: #fitting_details{} 53 | %%
54 | %% The details of this fitting. 55 | %% (Useful for logging, or sending other output.) 56 | %%
57 | %% 58 | %% The function should return a tuple of the form `{ok, 59 | %% NewAccumulator}', where `NewAccumulator' is a list, onto which 60 | %% the next input will be cons'd. For example, the function to 61 | %% sum values for a key, as described above might look like: 62 | %% ``` 63 | %% fun(_Key, Inputs, _Partition, _FittingDetails) -> 64 | %% {ok, [lists:sum(Inputs)]} 65 | %% end 66 | %% ''' 67 | %% 68 | %% The preferred consistent-hash function for this fitting is 69 | %% {@link chashfun/1}. It hashes the input `Key'. Any other 70 | %% partition function should work, but beware that a function 71 | %% that sends values for the same `Key' to different partitions 72 | %% will result in fittings down the pipe receiving multiple 73 | %% results for the `Key'. 74 | %% 75 | %% This fitting produces as its archive, the store of evaluation 76 | %% results for the keys it has seen. To merge handoff values, 77 | %% the lists stored with each key are concatenated, and the 78 | %% reduce function is re-evaluated. 79 | -module(riak_pipe_w_reduce). 80 | -behaviour(riak_pipe_vnode_worker). 81 | 82 | -export([init/2, 83 | process/3, 84 | done/1, 85 | archive/1, 86 | handoff/2, 87 | validate_arg/1]). 88 | -export([chashfun/1]). 89 | 90 | -include_lib("kernel/include/logger.hrl"). 91 | 92 | -include("riak_pipe.hrl"). 93 | 94 | -ifdef(namespaced_types). 95 | -type riak_pipe_w_reduce_dict() :: dict:dict(). 96 | -else. 97 | -type riak_pipe_w_reduce_dict() :: dict(). 98 | -endif. 99 | 100 | -record(state, {accs :: riak_pipe_w_reduce_dict(), 101 | p :: riak_pipe_vnode:partition(), 102 | fd :: riak_pipe_fitting:details()}). 103 | -opaque state() :: #state{}. 104 | -export_type([state/0]). 105 | 106 | %% @doc Setup creates the store for evaluation results (a dict()) and 107 | %% stashes away the `Partition' and `FittingDetails' for later. 108 | -spec init(riak_pipe_vnode:partition(), 109 | riak_pipe_fitting:details()) -> 110 | {ok, state()}. 111 | init(Partition, FittingDetails) -> 112 | {ok, #state{accs=dict:new(), p=Partition, fd=FittingDetails}}. 113 | 114 | %% @doc Process looks up the previous result for the `Key', and then 115 | %% evaluates the funtion on that with the new `Input'. 116 | -spec process({term(), term()}, boolean(), state()) -> {ok, state()}. 117 | process({Key, Input}, _Last, #state{accs=Accs}=State) -> 118 | case dict:find(Key, Accs) of 119 | {ok, OldAcc} -> ok; 120 | error -> OldAcc=[] 121 | end, 122 | InAcc = [Input|OldAcc], 123 | case reduce(Key, InAcc, State) of 124 | {ok, OutAcc} -> 125 | {ok, State#state{accs=dict:store(Key, OutAcc, Accs)}}; 126 | {error, {Type, Error, Trace}} -> 127 | %%TODO: forward 128 | ?LOG_ERROR( 129 | "~p:~p reducing:~n ~P~n ~P", 130 | [Type, Error, InAcc, 2, Trace, 5]), 131 | {ok, State} 132 | end. 133 | 134 | %% @doc Unless the aggregation function sends its own outputs, done/1 135 | %% is where all outputs are sent. 136 | -spec done(state()) -> ok. 137 | done(#state{accs=Accs, p=Partition, fd=FittingDetails}) -> 138 | _ = [ ok = riak_pipe_vnode_worker:send_output(A, Partition, FittingDetails) 139 | || A <- dict:to_list(Accs)], 140 | ok. 141 | 142 | %% @doc The archive is just the store (dict()) of evaluation results. 143 | -spec archive(state()) -> {ok, riak_pipe_w_reduce_dict()}. 144 | archive(#state{accs=Accs}) -> 145 | %% just send state of reduce so far 146 | {ok, Accs}. 147 | 148 | %% @doc The handoff merge is simple a dict:merge, where entries for 149 | %% the same key are concatenated. The reduce function is also 150 | %% re-evaluated for the key, such that {@link done/1} still has 151 | %% the correct value to send, even if no more inputs arrive. 152 | -spec handoff(riak_pipe_w_reduce_dict(), state()) -> {ok, state()}. 153 | handoff(HandoffAccs, #state{accs=Accs}=State) -> 154 | %% for each Acc, add to local accs; 155 | NewAccs = dict:merge(fun(K, HA, A) -> 156 | handoff_acc(K, HA, A, State) 157 | end, 158 | HandoffAccs, Accs), 159 | {ok, State#state{accs=NewAccs}}. 160 | 161 | %% @doc The dict:merge function for handoff. Handles the reducing. 162 | -spec handoff_acc(term(), [term()], [term()], state()) -> [term()]. 163 | handoff_acc(Key, HandoffAccs, LocalAccs, State) -> 164 | InAcc = HandoffAccs++LocalAccs, 165 | case reduce(Key, InAcc, State) of 166 | {ok, OutAcc} -> 167 | OutAcc; 168 | {error, {Type, Error, Trace}} -> 169 | ?LOG_ERROR( 170 | "~p:~p reducing handoff:~n ~P~n ~P", 171 | [Type, Error, InAcc, 2, Trace, 5]), 172 | LocalAccs %% don't completely barf 173 | end. 174 | 175 | %% @doc Actually evaluate the aggregation function. 176 | -spec reduce(term(), [term()], state()) -> 177 | {ok, [term()]} | {error, {term(), term(), term()}}. 178 | reduce(Key, InAcc, #state{p=Partition, fd=FittingDetails}) -> 179 | Fun = FittingDetails#fitting_details.arg, 180 | try 181 | {ok, OutAcc} = Fun(Key, InAcc, Partition, FittingDetails), 182 | true = is_list(OutAcc), %%TODO: nicer error 183 | {ok, OutAcc} 184 | catch Class:Reason:Stacktrace -> 185 | {error, {Class, Reason, Stacktrace}} 186 | end. 187 | 188 | %% @doc Check that the arg is a valid arity-4 function. See {@link 189 | %% riak_pipe_v:validate_function/3}. 190 | -spec validate_arg(term()) -> ok | {error, iolist()}. 191 | validate_arg(Fun) when is_function(Fun) -> 192 | riak_pipe_v:validate_function("arg", 4, Fun); 193 | validate_arg(Fun) -> 194 | {error, io_lib:format("~p requires a function as argument, not a ~p", 195 | [?MODULE, riak_pipe_v:type_of(Fun)])}. 196 | 197 | %% @doc The preferred hashing function. Chooses a partition based 198 | %% on the hash of the `Key'. 199 | -spec chashfun({term(), term()}) -> riak_pipe_vnode:chash(). 200 | chashfun({Key,_}) -> 201 | chash:key_of(Key). 202 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | -------------------------------------------------------------------------------- /eqc/riak_pipe_fitting_eqc.erl: -------------------------------------------------------------------------------- 1 | %%-------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %%-------------------------------------------------------------------- 20 | 21 | %% @doc Exercise riak_pipe_fitting. 22 | -module(riak_pipe_fitting_eqc). 23 | 24 | -compile([export_all, nowarn_export_all]). 25 | 26 | -ifdef(EQC). 27 | 28 | -include("include/riak_pipe.hrl"). 29 | 30 | -include_lib("eqc/include/eqc.hrl"). 31 | -include_lib("eqc/include/eqc_fsm.hrl"). 32 | -include_lib("eunit/include/eunit.hrl"). 33 | 34 | -define(QC_OUT(P), 35 | eqc:on_output(fun(Str, Args) -> io:format(user, Str, Args) end, P)). 36 | 37 | -record(state, { 38 | vnodes = [] :: [pid()] %% the "vnode" processes 39 | }). 40 | 41 | -record(setup, { 42 | builder :: pid(), 43 | sink :: #fitting{}, 44 | fitting :: #fitting{}, 45 | fitting_mon :: pid() 46 | }). 47 | 48 | -type sink_result() :: {ok, Messages::list()} 49 | | {error, Reason::term()}. 50 | -type vnode_result() :: {ok, GotDetails::boolean, Messages::list()} 51 | | {error, Reason::term()}. 52 | 53 | -record(output, { 54 | sink :: sink_result(), %% messages the sink received 55 | vnodes :: [vnode_result()] %% messages each vnode received 56 | }). 57 | 58 | 59 | %% @doc Make sure that all vnodes that obtain details from a fitting 60 | %% get an eoi message from the fitting. 61 | prop_eoi() -> 62 | ?FORALL(Cmds, commands(?MODULE), 63 | begin 64 | Setup = setup_fitting(), 65 | try 66 | Result = run_commands(?MODULE, Cmds, 67 | [{fitting, Setup#setup.fitting}]), 68 | 69 | Output = gather_output(Result, Setup), 70 | aggregate(zip(state_names(element(1, Result)), 71 | command_names(Cmds)), 72 | ?WHENFAIL( 73 | print_helpful(Result, Output, Setup), 74 | check_result(Result, Output, Setup))) 75 | after 76 | cleanup_fitting(Setup) 77 | end 78 | end). 79 | 80 | print_helpful({_,{Last,_},R}, Output, Setup) -> 81 | FittingStatus = fitting_is_alive(Setup), 82 | io:format("fitting ~p~n", [FittingStatus]), 83 | io:format("Reason: ~p~nLast: ~p~nSink: ~p~nVnodes: ~p~n", 84 | [R, Last, Output#output.sink, Output#output.vnodes]). 85 | 86 | %% @doc start a builder and sink for support, then start the fitting 87 | %% we will test 88 | setup_fitting() -> 89 | Builder = spawn_link(?MODULE, fake_builder, []), 90 | Sink = spawn_link(?MODULE, fake_sink, []), 91 | SinkFitting = #fitting{pid=Sink, ref=make_ref(), chashfun=sink}, 92 | Spec = #fitting_spec{name=eqc}, 93 | Options = [], 94 | {ok, _Pid, Fitting} = riak_pipe_fitting:start_link( 95 | Builder, Spec, SinkFitting, Options), 96 | 97 | Pid = monitor_fitting_pid(Fitting), 98 | 99 | #setup{builder=Builder, 100 | sink=SinkFitting, 101 | fitting=Fitting, 102 | fitting_mon=Pid}. 103 | 104 | %% @doc ask the sink and the vnodes for all messages they have received 105 | gather_output({_, {Last, S}, _}, Setup) -> 106 | case Last of 107 | stopping -> 108 | %% make sure the fitting finishes sending 109 | %% messages before we go digging 110 | ok = wait_for_fitting_exit(Setup#setup.fitting); 111 | _ -> 112 | ok 113 | end, 114 | {ok, SMsgs} = get_sink_msgs(Setup#setup.sink), 115 | VMsgs = [get_vnode_msgs(V) || V <- S#state.vnodes], 116 | [begin unlink(V), exit(V, kill) end || V <- S#state.vnodes], 117 | #output{sink=SMsgs, 118 | vnodes=VMsgs}. 119 | 120 | check_result({_,{stopping,_},R}, Output, Setup) -> 121 | %% if the test decided to transition from running to stopping... 122 | Eoi = #pipe_eoi{ref=(Setup#setup.sink)#fitting.ref}, 123 | IfDetailsAlsoEoi = 124 | fun({ok, true, M}) -> 125 | %% all vnodes with details must receive eoi 126 | lists:member({cmd_eoi, Setup#setup.fitting}, M); 127 | ({ok, false, M}) -> 128 | %% all vnodes without details must receive nothing 129 | [] == M; 130 | (_) -> 131 | %% unrecognized message 132 | false 133 | end, 134 | %% test must have finished ok 135 | ok == R andalso 136 | %% sink must have recevied an eoi 137 | lists:member(Eoi, Output#output.sink) andalso 138 | lists:all(IfDetailsAlsoEoi, Output#output.vnodes); 139 | check_result({_,{running,_},R}, Output, Setup) -> 140 | %% if the test did not decide to transition from running to 141 | %% stopping (it never sent eoi to the fitting)... 142 | %% (is this part of the test useful?) 143 | 144 | {FittingAlive, FittingStatus} = fitting_is_alive(Setup), 145 | NoVnodeMessages = fun({ok, _, M}) -> [] == M; 146 | (_) -> false 147 | end, 148 | %% test must have finished ok 149 | case (ok == R andalso 150 | %% the fitting process must still be running 151 | FittingAlive andalso 152 | %% the sink must have received nothing 153 | [] == Output#output.sink andalso 154 | %% the vnodes must have received nothing 155 | lists:all(NoVnodeMessages, Output#output.vnodes)) of 156 | true -> true; 157 | false -> 158 | {{result_ok, R==ok}, 159 | {fitting_alive, FittingAlive, FittingStatus}, 160 | {sink_no_recv, Output#output.sink==[]}, 161 | {vnode_no_recv, lists:all(NoVnodeMessages, Output#output.vnodes)} 162 | } 163 | end. 164 | 165 | %% tear down all of our processes 166 | cleanup_fitting(Setup) -> 167 | stop_sink(Setup#setup.sink), 168 | stop_builder(Setup#setup.builder), 169 | erlang:unlink((Setup#setup.fitting)#fitting.pid), 170 | exit((Setup#setup.fitting)#fitting.pid, kill). 171 | 172 | initial_state() -> 173 | running. 174 | 175 | initial_state_data() -> 176 | #state{}. 177 | 178 | %%% STATES 179 | 180 | %% make the test stay in running a bit more often: 181 | %% with equal weights, aggregate shows 85% stopping state; 182 | %% with 10x running->running, aggregate shows 60% stopping, 40% running 183 | weight(running, running, _Call) -> 10; 184 | weight(_, _, _Call) -> 1. 185 | 186 | %% these are available in all states 187 | nontransitional(_s) -> 188 | %% always okay to start a vnode 189 | [{history, {call, ?MODULE, start_vnode, [{var, fitting}]}}]. 190 | 191 | running(S) -> 192 | [{stopping, {call, riak_pipe_fitting, eoi, [{var, fitting}]}}] 193 | ++nontransitional(S). 194 | 195 | stopping(S) -> 196 | nontransitional(S). 197 | 198 | precondition(_From,_To,_S,_Call) -> 199 | true. 200 | 201 | next_state_data(_F,_T,S,R,{call,?MODULE,start_vnode,_}) -> 202 | S#state{vnodes=[R|S#state.vnodes]}; 203 | next_state_data(_,_,S,_,_) -> 204 | S. 205 | 206 | postcondition(_, _, _, {call,?MODULE,start_vnode,_}, R) -> 207 | case is_pid(R) of 208 | true -> 209 | true; 210 | false -> 211 | {?MODULE, start_vnode, result, not_pid, R} 212 | end; 213 | postcondition(_, _, _, {call,riak_pipe_fitting,eoi,_}, R) -> 214 | case(R) of 215 | ok -> 216 | true; 217 | Other -> 218 | {riak_pipe_fitting, eoi, result, not_ok, Other} 219 | end; 220 | postcondition(_, _, _, _, _) -> 221 | true. 222 | 223 | %% MOCKS 224 | 225 | fake_builder() -> 226 | receive 227 | stop -> ok 228 | end. 229 | 230 | stop_builder(B) -> 231 | B ! stop. 232 | 233 | %% just accumulates messages 234 | fake_sink() -> 235 | fake_sink([]). 236 | fake_sink(Msgs) -> 237 | receive 238 | stop -> 239 | ok; 240 | {get, Ref, Pid} -> 241 | Pid ! {msgs, Ref, Msgs}, 242 | fake_sink([]); 243 | Any -> 244 | fake_sink([Any|Msgs]) 245 | end. 246 | 247 | stop_sink(#fitting{pid=S}) -> 248 | S ! stop. 249 | 250 | get_sink_msgs(#fitting{pid=S}) -> 251 | R = make_ref(), 252 | M = erlang:monitor(process, S), 253 | S ! {get, R, self()}, 254 | receive 255 | {msgs, R, Msgs} -> 256 | erlang:demonitor(M, [flush]), 257 | {ok, Msgs}; 258 | {'DOWN', M, process, S, _} -> 259 | {error, down} 260 | after 5000 -> 261 | {error, timeout} 262 | end. 263 | 264 | wait_for_fitting_exit(#fitting{pid=Pid}) -> 265 | M = erlang:monitor(process, Pid), 266 | receive 267 | {'DOWN', M, process, Pid, _} -> 268 | ok 269 | end. 270 | 271 | monitor_fitting_pid(#fitting{pid=Pid}) -> 272 | case is_process_alive(Pid) of 273 | true -> 274 | MonRef = erlang:monitor(process, Pid), 275 | spawn_link(?MODULE, fitting_monitor, [MonRef, Pid, true]); 276 | false -> 277 | exit(no_fitting_pid) 278 | end. 279 | 280 | fitting_monitor(Ref, Pid, Down) -> 281 | receive 282 | {'DOWN', Ref, process, Pid, Reason} -> 283 | fitting_monitor(Ref, Pid, Reason); 284 | {From, status} -> 285 | From ! {self(), Down}, 286 | fitting_monitor(Ref, Pid, Down) 287 | end. 288 | 289 | fitting_is_alive(#setup{fitting_mon=Pid}) -> 290 | Pid ! {self(), status}, 291 | FittingAlive = receive 292 | {Pid, Down} -> 293 | Down 294 | end, 295 | case FittingAlive of 296 | true -> 297 | {true, alive}; 298 | Other -> 299 | {false, Other} 300 | end. 301 | 302 | start_vnode(#fitting{}=F) -> 303 | P = spawn_link(?MODULE, fake_vnode, [F, self()]), 304 | receive 305 | {up, P} -> P 306 | end. 307 | 308 | %% ask for details, then enter message receive loop 309 | fake_vnode(#fitting{}=F, Test) -> 310 | %% TODO: define Partition, instead of make_ref() 311 | D = case riak_pipe_fitting:get_details(F, make_ref()) of 312 | {ok, #fitting_details{}} -> true; 313 | gone -> false 314 | end, 315 | Test ! {up, self()}, 316 | fake_vnode_loop(F, D, []). 317 | 318 | fake_vnode_loop(F, D, Msgs) -> 319 | receive 320 | {'$gen_event', 321 | {riak_vnode_req_v1,_, _,Msg}} -> 322 | riak_pipe_fitting:worker_done(F), 323 | fake_vnode_loop(F, D, [Msg|Msgs]); 324 | {msgs, Ref, Pid} -> 325 | Pid ! {msgs, Ref, D, Msgs}, 326 | fake_vnode_loop(F, D, []) 327 | end. 328 | 329 | get_vnode_msgs(V) -> 330 | R = make_ref(), 331 | M = erlang:monitor(process, V), 332 | V ! {msgs, R, self()}, 333 | receive 334 | {msgs, R, D, Msgs} -> 335 | erlang:demonitor(M, [flush]), 336 | {ok, D, Msgs}; 337 | {'DOWN', M, process, V, _} -> 338 | {error, down} 339 | end. 340 | 341 | -endif. 342 | -------------------------------------------------------------------------------- /eqc/reduce_fitting_pulse.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2013 Basho Technologies, Inc. All Rights Reserved. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Test riak_pipe_fitting for no-input, reduce-once-anyway 22 | %% behavior, possibly related to github issues 48 and 49. 23 | -module(reduce_fitting_pulse). 24 | 25 | -include("include/riak_pipe.hrl"). 26 | -include("include/riak_pipe_log.hrl"). 27 | 28 | -behaviour(riak_pipe_vnode_worker). 29 | -ifdef(EQC). 30 | -include_lib("eqc/include/eqc.hrl"). 31 | -endif. 32 | -include_lib("eunit/include/eunit.hrl"). 33 | 34 | %% riak_pipe_worker behavior 35 | -export([no_input_run_reduce_once/0, 36 | init/2, 37 | process/3, 38 | done/1]). 39 | 40 | %% console debugging convenience 41 | -compile([export_all, nowarn_export_all]). 42 | 43 | -ifdef(PULSE). 44 | -include_lib("pulse/include/pulse.hrl"). 45 | %% have to transform the 'receive' of the work results 46 | -compile({parse_transform, pulse_instrument}). 47 | %% don't trasnform toplevel test functions 48 | -compile({pulse_skip,[{death_test_,0}]}). 49 | -endif. 50 | 51 | %%% Worker Definition This is a rough copy of important parts of 52 | %% riak_kv_w_reduce. The important bit is that it defines 53 | %% no_input_run_reduce_once/0 as true, so that the fitting process 54 | %% will run these functions itself, instead of evaulating them in a 55 | %% vnode worker. 56 | 57 | no_input_run_reduce_once() -> 58 | true. 59 | 60 | init(Partition, #fitting_details{}=Details) -> 61 | maybe_send_trace(Details), 62 | {ok, {Partition, Details}}. 63 | 64 | process(_Input, _Last, State) -> 65 | %% we're only concerned with the zero-worker case, which is 66 | %% easiest to replicate with zero inputs 67 | throw(this_should_not_be_part_of_the_test), 68 | {ok, State}. 69 | 70 | done(State) -> 71 | maybe_send_output(State), 72 | ok. 73 | 74 | maybe_send_trace({_, #fitting_details{arg=Arg}=Details}) -> 75 | case lists:member(send_trace, Arg) of 76 | true -> 77 | ?T(Details, [maybe_send_trace], sending_trace); 78 | false -> 79 | ok 80 | end. 81 | 82 | maybe_send_output({Partition, #fitting_details{arg=Arg}=Details}) -> 83 | case lists:member(send_output, Arg) of 84 | true -> 85 | riak_pipe_vnode_worker:send_output( 86 | an_output, Partition, Details); 87 | false -> 88 | ok 89 | end. 90 | 91 | %% none of these tests make sense if PULSE is not used 92 | -ifdef(PULSE). 93 | 94 | %% @doc Nothing should ever cause the fitting to exit abnormally. 95 | %% This is a bit of a wide net to cast, with more options than are 96 | %% likely necessary to find edge cases. It was constructed while 97 | %% searching for the unknown source of github issues 48 and 49. 98 | %% Although only a small corner provoked failures, such coverage may 99 | %% be useful when making changes in the future, so it's staying in. 100 | prop_fitting_dies_normal() -> 101 | ?SETUP(fun() -> 102 | pulse:start(), 103 | fun() -> pulse:stop() end 104 | end, 105 | ?FORALL({Seed, Trace, Output, SinkType, 106 | {Eoi, Destroy, AlsoDestroySink, ExitPoint}}, 107 | {pulse:seed(), bool(), bool(), oneof([fsm, raw]), 108 | ?LET({Eoi, Destroy}, 109 | {bool(), bool()}, 110 | {Eoi, Destroy, 111 | %% to mimick riak_kv HTTP&PB endpoints, we only 112 | %% destroy the sink if we also destroy the pipe 113 | oneof([Destroy, false]), 114 | oneof([before_eoi] 115 | %% can't exit after eoi if there's no eoi 116 | ++ [ after_eoi || Eoi] 117 | %% similarly for destroy 118 | ++ [ after_destroy || Destroy] 119 | %% and "never" exiting requires at least one of 120 | %% eoi or destroy, or the test will hang 121 | ++ [ never || Eoi or Destroy])})}, 122 | 123 | %% this is a gigantic table to collect, but useful to see anyway 124 | collect({Trace, Output, SinkType, 125 | Eoi, Destroy, AlsoDestroySink, ExitPoint}, 126 | 127 | begin 128 | ExitReasons = 129 | fitting_exit_reason( 130 | Seed, Trace, Output, SinkType, 131 | Eoi, Destroy, AlsoDestroySink, ExitPoint), 132 | ?WHENFAIL( 133 | io:format(user, "Exit Reasons: ~p~n", [ExitReasons]), 134 | exit_reason_is_normalish( 135 | proplists:get_value(fitting, ExitReasons)) andalso 136 | proplists:get_value(client, ExitReasons) == ExitPoint) 137 | end))). 138 | 139 | exit_reason_is_normalish(normal) -> 140 | %% the fitting chose to exit on its own 141 | true; 142 | exit_reason_is_normalish(shutdown) -> 143 | %% the fitting's supervisor shut it down 144 | true. 145 | 146 | %% @doc This is a nice convenience to have when re-running failures 147 | %% manually. 148 | fitting_exit_reason(Seed, Trace, Output, SinkType, 149 | Eoi, Destroy, AlsoDestroySink, ExitPoint) -> 150 | pulse:run_with_seed( 151 | fun() -> 152 | fitting_exit_reason(Trace, Output, SinkType, 153 | Eoi, Destroy, AlsoDestroySink, ExitPoint) 154 | end, 155 | Seed). 156 | 157 | -endif. %% PULSE 158 | 159 | %% PULSE is not *required* to run this code. The code is much more 160 | %% interesting with PULSE enabled, but it may be useful to run without 161 | %% PULSE to sanity check. 162 | 163 | %% @doc The actual run. 164 | fitting_exit_reason(Trace, Output, SinkType, 165 | Eoi, Destroy, AlsoDestroySink, ExitPoint) -> 166 | Supervisors = [riak_pipe_builder_sup, 167 | riak_pipe_fitting_sup, 168 | reduce_fitting_pulse_sink_sup], 169 | [ {ok,_} = Sup:start_link() || Sup <- Supervisors ], 170 | 171 | %% we want the client linked to this process so that it dies if 172 | %% the test dies, but we don't want death to spread the other way 173 | erlang:process_flag(trap_exit, true), 174 | ClientRef = make_ref(), 175 | Self = self(), 176 | Client = spawn_link( 177 | fun() -> 178 | pipe_client({ClientRef, Self}, Trace, Output, SinkType, 179 | Eoi, Destroy, AlsoDestroySink, ExitPoint) 180 | end), 181 | 182 | Pipe = receive 183 | {ClientRef, PipeDef} -> 184 | PipeDef 185 | end, 186 | 187 | %% monitor before the client proceeds with the test, to 188 | %% ensure we get the actual exit reason, instead of a 189 | %% bogus noproc 190 | FittingMonitor = monitor(process, fitting_process(Pipe)), 191 | 192 | %% we're not checking the builder at the moment, but watching for 193 | %% this 'DOWN' prevents us from killing the builder supervisor 194 | %% before the builder exits 195 | BuilderMonitor = monitor(process, builder_process(Pipe)), 196 | 197 | %% for 'raw' sink type, Sink and Client are the same, but our 198 | %% checks should be fine with that 199 | SinkMonitor = monitor(process, sink_process(Pipe)), 200 | Client ! {ClientRef, ok}, 201 | 202 | Reasons = receive_exits([{fitting, FittingMonitor}, 203 | {builder, BuilderMonitor}, 204 | {sink, SinkMonitor}, 205 | {client, Client}]), 206 | 207 | [ begin 208 | unlink(whereis(Sup)), 209 | exit(whereis(Sup), kill) 210 | end 211 | || Sup <- Supervisors ], 212 | Reasons. 213 | 214 | fitting_process(#pipe{fittings=[{fake_reduce, #fitting{pid=Pid}}]}) -> 215 | Pid. 216 | 217 | builder_process(#pipe{builder=Builder}) -> 218 | Builder. 219 | 220 | sink_process(#pipe{sink=#fitting{pid=Pid}}) -> 221 | Pid. 222 | 223 | receive_exits([]) -> 224 | []; 225 | receive_exits(Waiting) -> 226 | {Tag, Reason} = receive 227 | {'DOWN', Monitor, process, _Pid, R} -> 228 | {Monitor, R}; 229 | {'EXIT', Pid, R} -> 230 | {Pid, R} 231 | end, 232 | case lists:keytake(Tag, 2, Waiting) of 233 | {value, {Name, Tag}, Rest} -> 234 | [{Name, Reason}|receive_exits(Rest)]; 235 | false -> 236 | receive_exits(Waiting) 237 | end. 238 | 239 | pipe_client({Ref, Test}, Trace, Output, SinkType, 240 | Eoi, Destroy, AlsoDestroySink, ExitPoint) -> 241 | Options = case SinkType of 242 | fsm -> 243 | %% mimicking riak_kv here, using a supervisor 244 | %% for the sink instead of bare linking, in case 245 | %% that makes a difference (though it doesn't 246 | %% seem to) 247 | {ok, S} = reduce_fitting_pulse_sink_sup:start_sink( 248 | self(), Ref), 249 | [{sink, #fitting{pid=S, ref=Ref, chashfun=sink}}, 250 | {sink_type, {fsm, 1, infinity}}]; 251 | raw -> 252 | [{sink, #fitting{pid=self(), ref=Ref, chashfun=sink}}, 253 | {sink_type, raw}] 254 | end, 255 | 256 | {ok, Pipe} = riak_pipe:exec( 257 | [#fitting_spec{name=fake_reduce, 258 | module=?MODULE, 259 | arg=[send_trace || Trace]++ 260 | [send_output || Output]}], 261 | Options), 262 | 263 | case SinkType of 264 | fsm -> 265 | reduce_fitting_pulse_sink:use_pipe( 266 | sink_process(Pipe), Ref, Pipe); 267 | raw -> 268 | ok 269 | end, 270 | 271 | Test ! {Ref, Pipe}, 272 | receive {Ref, ok} -> ok end, 273 | 274 | maybe_exit(before_eoi, ExitPoint), 275 | 276 | case Eoi of 277 | true -> 278 | riak_pipe:eoi(Pipe), 279 | maybe_exit(after_eoi, ExitPoint); 280 | false -> ok 281 | end, 282 | 283 | case Destroy of 284 | true -> 285 | riak_pipe:destroy(Pipe), 286 | case {AlsoDestroySink, SinkType} of 287 | {true, fsm} -> 288 | reduce_fitting_pulse_sink_sup:terminate_sink( 289 | sink_process(Pipe)); 290 | _ -> 291 | %% *this* process is the sink if we're not using fsm 292 | %% (to mimick earlier Riak KV versions) 293 | ok 294 | end, 295 | maybe_exit(after_destroy, ExitPoint); 296 | false -> 297 | case SinkType of 298 | fsm -> 299 | %% we don't care if this call fails, just that it 300 | %% doesn't return until the sink has finished its 301 | %% work; mimicking riak_kv endpoints that don't 302 | %% wait for results if they destroy 303 | catch reduce_fitting_pulse_sink:all_results( 304 | sink_process(Pipe), Ref); 305 | raw -> 306 | %% *this* process is the sink if we're not using fsm 307 | %% (to mimick earlier Riak KV versions) 308 | receive #pipe_eoi{ref=Ref} -> ok end 309 | end 310 | end, 311 | 312 | %% we can't let the exit be 'normal', because the test process 313 | %% will never see it (it's linking not monitoring) 314 | exit(ExitPoint). 315 | 316 | maybe_exit(Now, Now) -> 317 | exit(Now); 318 | maybe_exit(_Now, _NotNow) -> 319 | ok. 320 | -------------------------------------------------------------------------------- /src/riak_pipe_builder.erl: -------------------------------------------------------------------------------- 1 | %%-------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %%-------------------------------------------------------------------- 20 | 21 | %% @doc The builder starts and monitors the fitting processes. 22 | %% 23 | %% This startup process is how each fitting learns about the 24 | %% fitting that follows it. The builder is also the process that 25 | %% the client asks to find the head fitting. 26 | -module(riak_pipe_builder). 27 | 28 | -compile({nowarn_deprecated_function, 29 | [{gen_fsm, start_link, 3}, 30 | {gen_fsm, sync_send_event, 2}, 31 | {gen_fsm, sync_send_event, 3}, 32 | {gen_fsm, sync_send_all_state_event, 2}]}). 33 | 34 | -behaviour(gen_fsm). 35 | 36 | %% API 37 | -export([start_link/2]). 38 | -export([fitting_pids/1, 39 | pipeline/1, 40 | destroy/1]). 41 | 42 | %% gen_fsm callbacks 43 | -export([init/1, 44 | wait_pipeline_shutdown/2, 45 | wait_pipeline_shutdown/3, 46 | handle_event/3, 47 | handle_sync_event/4, 48 | handle_info/3, 49 | terminate/3, 50 | code_change/4]). 51 | 52 | -include("riak_pipe.hrl"). 53 | -include("riak_pipe_debug.hrl"). 54 | 55 | -include_lib("kernel/include/logger.hrl"). 56 | 57 | -ifdef(PULSE). 58 | -include_lib("pulse/include/pulse.hrl"). 59 | %% have to transform the 'receive' of the work results 60 | -compile({parse_transform, pulse_instrument}). 61 | %% don't trasnform toplevel test functions 62 | -compile({pulse_replace_module,[{gen_fsm,pulse_gen_fsm}]}). 63 | -endif. 64 | 65 | -record(state, {options :: riak_pipe:exec_opts(), 66 | pipe :: #pipe{}, 67 | alive :: [{#fitting{}, reference()}], % monitor ref 68 | sinkmon :: reference()}). % monitor ref 69 | 70 | -opaque state() :: #state{}. 71 | -export_type([state/0]). 72 | 73 | %%%=================================================================== 74 | %%% API 75 | %%%=================================================================== 76 | 77 | %% @doc Start a builder to setup the pipeline described by `Spec'. 78 | -spec start_link([riak_pipe:fitting_spec()], riak_pipe:exec_opts()) -> 79 | {ok, pid(), reference()} | ignore | {error, term()}. 80 | start_link(Spec, Options) -> 81 | case gen_fsm:start_link(?MODULE, [Spec, Options], []) of 82 | {ok, Pid} -> 83 | {sink, #fitting{ref=Ref}} = lists:keyfind(sink, 1, Options), 84 | {ok, Pid, Ref}; 85 | Error -> 86 | Error 87 | end. 88 | 89 | %% @doc Get the list of pids for fittings that this builder started. 90 | %% If the builder terminated before this call was made, the 91 | %% function returns the atom `gone'. 92 | -spec fitting_pids(pid()) -> {ok, FittingPids::[pid()]} | gone. 93 | fitting_pids(Builder) -> 94 | try 95 | {ok, gen_fsm:sync_send_all_state_event(Builder, fittings)} 96 | catch exit:{noproc, _} -> 97 | gone 98 | end. 99 | 100 | %% @doc Get the `#pipe{}' record describing the pipeline created by 101 | %% this builder. This function will block until the builder has 102 | %% finished building the pipeline. 103 | -spec pipeline(pid()) -> {ok, #pipe{}} | gone. 104 | pipeline(BuilderPid) -> 105 | gen_fsm:sync_send_event(BuilderPid, pipeline). 106 | 107 | %% @doc Shutdown the pipeline built by this builder. 108 | -spec destroy(pid()) -> ok. 109 | destroy(BuilderPid) -> 110 | try 111 | gen_fsm:sync_send_event(BuilderPid, destroy, infinity) 112 | catch exit:_Reason -> 113 | %% the builder exited before the call completed, 114 | %% since we were shutting it down anyway, this is ok 115 | ok 116 | end. 117 | 118 | %%%=================================================================== 119 | %%% gen_fsm callbacks 120 | %%%=================================================================== 121 | 122 | %% @doc Initialize the builder fsm (gen_fsm callback). 123 | -spec init([ [riak_pipe:fitting_spec()] | riak_pipe:exec_opts() ]) -> 124 | {ok, wait_pipeline_shutdown, state()}. 125 | init([Spec, Options]) -> 126 | {sink, #fitting{ref=Ref}=Sink} = lists:keyfind(sink, 1, Options), 127 | SinkMon = erlang:monitor(process, Sink#fitting.pid), 128 | Fittings = start_fittings(Spec, Options), 129 | NamedFittings = lists:zip( 130 | [ N || #fitting_spec{name=N} <- Spec ], 131 | [ F || {F, _R} <- Fittings ]), 132 | Pipe = #pipe{builder=self(), 133 | fittings=NamedFittings, 134 | sink=Sink}, 135 | put(eunit, [{module, ?MODULE}, 136 | {ref, Ref}, 137 | {spec, Spec}, 138 | {options, Options}, 139 | {fittings, Fittings}]), 140 | {ok, wait_pipeline_shutdown, 141 | #state{options=Options, 142 | pipe=Pipe, 143 | alive=Fittings, 144 | sinkmon=SinkMon}}. 145 | 146 | %% @doc All fittings have been started, and the builder is just 147 | %% monitoring the pipeline (and replying to clients looking 148 | %% for the head fitting). 149 | -spec wait_pipeline_shutdown(term(), state()) -> 150 | {next_state, wait_pipeline_shutdown, state()}. 151 | wait_pipeline_shutdown(_Event, State) -> 152 | {next_state, wait_pipeline_shutdown, State}. 153 | 154 | %% @doc A client is asking for the fittings. Respond. 155 | -spec wait_pipeline_shutdown(pipeline | destroy, term(), state()) -> 156 | {reply, 157 | {ok, #pipe{}}, 158 | wait_pipeline_shutdown, 159 | state()} 160 | |{stop, normal, ok, state()}. 161 | wait_pipeline_shutdown(pipeline, _From, #state{pipe=Pipe}=State) -> 162 | %% everything is started - reply now 163 | {reply, {ok, Pipe}, wait_pipeline_shutdown, State}; 164 | wait_pipeline_shutdown(destroy, _From, State) -> 165 | %% client asked to shutdown this pipe immediately 166 | {stop, normal, ok, State}; 167 | wait_pipeline_shutdown(_, _, State) -> 168 | %% unknown message - reply {error, unknown} to get rid of it 169 | {reply, {error, unknown}, wait_pipeline_shutdown, State}. 170 | 171 | %% @doc Unused. 172 | -spec handle_event(term(), atom(), state()) -> 173 | {next_state, atom(), state()}. 174 | handle_event(_Event, StateName, State) -> 175 | {next_state, StateName, State}. 176 | 177 | %% @doc The only sync event recognized in all states is `fittings', 178 | %% which retrieves a count of fittings waiting to be started, 179 | %% and pids for fittings already started. 180 | -spec handle_sync_event(fittings, term(), atom(), state()) -> 181 | {reply, 182 | FittingPids::[pid()], 183 | StateName::atom(), 184 | state()}. 185 | handle_sync_event(fittings, _From, StateName, 186 | #state{alive=Alive}=State) -> 187 | Reply = [ Pid || {#fitting{pid=Pid},_Ref} <- Alive ], 188 | {reply, Reply, StateName, State}; 189 | handle_sync_event(_Event, _From, StateName, State) -> 190 | Reply = ok, 191 | {reply, Reply, StateName, State}. 192 | 193 | %% @doc The only non-gen_fsm message this process expects are `'DOWN'' 194 | %% messages from monitoring the fittings it has started. When 195 | %% normal `'DOWN'' messages have been received from all monitored 196 | %% fittings, this gen_fsm stops with reason `normal'. If an 197 | %% error `'DOWN'' message is received for any fitting, this 198 | %% process exits immediately, with an error reason. 199 | -spec handle_info({'DOWN', reference(), process, pid(), term()}, 200 | atom(), state()) -> 201 | {next_state, atom(), state()} 202 | | {stop, term(), state()}. 203 | handle_info({'DOWN', Ref, process, Pid, Reason}, StateName, 204 | #state{alive=Alive}=State) -> 205 | %% stages should exit normally in order, 206 | %% but messages may be delivered out-of-order 207 | case lists:keytake(Ref, 2, Alive) of 208 | {value, {#fitting{pid=Pid}, Ref}, Rest} -> 209 | %% one of our fittings died 210 | case Reason of 211 | normal -> ok; 212 | _ -> 213 | ?LOG_WARNING("~p: Fitting worker ~p died. Reason: ~p", 214 | [StateName, Pid, Reason]) 215 | end, 216 | maybe_shutdown(Reason, 217 | StateName, 218 | State#state{alive=Rest}); 219 | false -> 220 | case (State#state.sinkmon == Ref) andalso 221 | (((State#state.pipe)#pipe.sink)#fitting.pid == Pid) of 222 | true -> 223 | %% the sink died - kill the pipe, since it has 224 | %% nowhere to send its output 225 | 226 | %% Exit normal here because an abnormal sink exit 227 | %% should have generated its own error log, and a 228 | %% normal sink exit should not generate spam. 229 | {stop, normal, State}; 230 | false -> 231 | %% this wasn't meant for us - ignore 232 | {next_state, StateName, State} 233 | end 234 | end; 235 | handle_info(_Info, StateName, State) -> 236 | {next_state, StateName, State}. 237 | 238 | %% @doc Decide whether to shutdown, or continue waiting for `'DOWN'' 239 | %% messages from other fittings. 240 | -spec maybe_shutdown(term(), atom(), state()) -> 241 | {stop, normal, state()} 242 | | {stop, {fitting_exited_abnormally, term()}, state()} 243 | | {next_state, wait_pipeline_shutdown, state()}. 244 | maybe_shutdown(normal, wait_pipeline_shutdown, #state{alive=[]}=S) -> 245 | %% all fittings stopped normally, and we were waiting for them 246 | {stop, normal, S}; 247 | maybe_shutdown(normal, wait_pipeline_shutdown, State) -> 248 | %% fittings are beginning to stop, but we're still waiting on some 249 | {next_state, wait_pipeline_shutdown, State}; 250 | maybe_shutdown(Reason, _StateName, State) -> 251 | %% some fitting exited abnormally 252 | %% (either non-normal status, or before we were ready) 253 | %% explode! 254 | {stop, {fitting_exited_abnormally, Reason}, State}. 255 | 256 | %% @doc Terminate any fittings that are still alive. 257 | -spec terminate(term(), atom(), state()) -> ok. 258 | terminate(_Reason, _StateName, #state{alive=Alive}) -> 259 | %% this is a brutal kill of each fitting, just in case that fitting 260 | %% is otherwise swamped with stop/restart messages from its workers 261 | _ = [ _ = riak_pipe_fitting_sup:terminate_fitting(F) || {F,_R} <- Alive ], 262 | ok. 263 | 264 | %% @doc Unused. 265 | -spec code_change(term(), atom(), state(), term()) -> 266 | {ok, atom(), state()}. 267 | code_change(_OldVsn, StateName, State, _Extra) -> 268 | {ok, StateName, State}. 269 | 270 | %%%=================================================================== 271 | %%% Internal functions 272 | %%%=================================================================== 273 | 274 | 275 | %% @doc Start and monitor all of the fittings for this builder's 276 | %% pipeline. 277 | -spec start_fittings([riak_pipe:fitting_spec()], 278 | riak_pipe:exec_opts()) -> 279 | [{riak_pipe:fitting(), reference()}]. 280 | start_fittings(Spec, Options) -> 281 | [Tail|Rest] = lists:reverse(Spec), 282 | ClientOutput = client_output(Options), 283 | lists:foldl(fun(FitSpec, [{Output,_}|_]=Acc) -> 284 | [start_fitting(FitSpec, Output, Options)|Acc] 285 | end, 286 | [start_fitting(Tail, ClientOutput, Options)], 287 | Rest). 288 | 289 | %% @doc Start a new fitting, as specified by `Spec', sending its 290 | %% output to `Output'. 291 | -spec start_fitting(riak_pipe:fitting_spec(), 292 | riak_pipe:fitting(), 293 | riak_pipe:exec_opts()) -> 294 | {riak_pipe:fitting(), reference()}. 295 | start_fitting(Spec, Output, Options) -> 296 | ?DPF("Starting fitting for ~p", [Spec]), 297 | {ok, Pid, Fitting} = riak_pipe_fitting_sup:add_fitting( 298 | self(), Spec, Output, Options), 299 | Ref = erlang:monitor(process, Pid), 300 | {Fitting, Ref}. 301 | 302 | %% @doc Find the sink in the options passed. 303 | -spec client_output(riak_pipe:exec_opts()) -> riak_pipe:fitting(). 304 | client_output(Options) -> 305 | proplists:get_value(sink, Options). 306 | -------------------------------------------------------------------------------- /src/riak_pipe_vnode_worker.erl: -------------------------------------------------------------------------------- 1 | %% ------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %% ------------------------------------------------------------------- 20 | 21 | %% @doc Basic worker process implementation, to be parameterized with 22 | %% fitting implementation module. 23 | %% 24 | %% Modules that implement this behavior need to export at least 25 | %% three functions: 26 | %% 27 | %% ``` 28 | %% init(Partition :: riak_pipe_vnode:partition(), 29 | %% FittingDetails :: riak_pipe_fitting:details()) 30 | %% -> {ok, ModuleState :: term()} 31 | %% ''' 32 | %% 33 | %% The `init/2' function is called when the worker starts. The 34 | %% module should do whatever it needs to get ready before 35 | %% processing inputs. The module will probably also want to 36 | %% store the `Parition' and `FittingDetails' arguments in its 37 | %% state, as it will need them to send outputs later. The 38 | %% `ModuleState' returned from this function will be passed to 39 | %% the `process/3' function later. 40 | %% 41 | %% ``` 42 | %% process(Input :: term(), 43 | %% LastInPreflist :: boolean(), 44 | %% ModuleState :: term()) 45 | %% -> {ok, NewModuleState :: term()} 46 | %% |forward_preflist. 47 | %% ''' 48 | %% 49 | %% The `process/3' function is called once for each input 50 | %% delivered to the worker. The module should do whatever 51 | %% processing is needed, sending outputs if appropriate. The 52 | %% `NewModuleState' returned from `process/3' will be passed back 53 | %% in on the next input. 54 | %% 55 | %% `LastInPreflist' is an indicator as to whether this worker is 56 | %% the last in the partition preflist for this input. When this 57 | %% parameter is `false', the function may return the atom 58 | %% `forward_preflist' to have the input sent to the next vnode in 59 | %% the prefence list. When is parameter is `true', returning 60 | %% `forward_preflist' will cause an error trace message to be 61 | %% generated, with the reason `preflist_exhausted'. 62 | %% 63 | %% ``` 64 | %% done(ModuleState :: term()) -> ok. 65 | %% ''' 66 | %% 67 | %% The `done/1' function is called when all inputs have been 68 | %% processed, and the end-of-inputs flag has been received from 69 | %% the fitting. The module should clean up any resources it 70 | %% needs to clean up, and send any outputs it still needs to 71 | %% send. When `done/1' returns, the worker will terminate. 72 | %% 73 | %% There are also four optional functions that a worker behavior 74 | %% module can export: 75 | %% 76 | %% ``` 77 | %% validate_arg(Arg :: term()) -> ok | {error, Reason :: iolist()}. 78 | %% ''' 79 | %% 80 | %% The `validate_arg/1' function is called before a pipeline is 81 | %% constructed. If the behavior module exports this function, 82 | %% then it will be evaluated on the value of the `arg' field of a 83 | %% `#fitting_spec{}' record that points to this module. If the 84 | %% argument is valid, this function should return the atom `ok'. 85 | %% If the argument is invalid, the function should return an 86 | %% error tuple, with the Reason being a printable iolist. 87 | %% 88 | %% ``` 89 | %% archive(ModuleState :: term()) -> {ok, Archive :: term()}. 90 | %% ''' 91 | %% 92 | %% The `archive/1' function is called when the vnode that owns 93 | %% this worker is being handed off to another node. The worker 94 | %% should produce some erlang term that represents its state. 95 | %% This `Archive' term will be passed to the `handoff/2' function 96 | %% of the module, by the worker running on the handoff target. 97 | %% 98 | %% ``` 99 | %% handoff(Archive :: term(), 100 | %% ModuleState :: term()) -> 101 | %% {ok, NewModuleState :: term()}. 102 | %% ''' 103 | %% 104 | %% The `handoff/2' function is called when a vnode receives a 105 | %% handoff archive from another vnode. The module should "merge" 106 | %% the `Archive' with its `ModuleState' (in whatever sense 107 | %% "merge" may mean for this fitting), and return the resulting 108 | %% `NewModuleState'. 109 | %% 110 | %% ``` 111 | %% no_input_run_reduce_once() -> boolean(). 112 | %% ''' 113 | %% 114 | %% If present and returns `true', then in the case that a fitting has 115 | %% no input (as measured by having zero workers), then a "fake" worker 116 | %% will be started for the express purpose of running its computation 117 | %% once and sending some output downstream. Right now, the only 118 | %% fitting that needs this feature is riak_kv_w_reduce.erl, which 119 | %% needs the capability to run its reduce function once (with input of 120 | %% an empty list) in order to maintain full compatibility with Riak 121 | %% KV's Map/Reduce. 122 | %% 123 | %% For riak_kv_w_reduce.erl and any other pipe behavior callback 124 | %% module where this function returns `true', the 125 | %% `#fitting_details.options' property list will contain the property 126 | %% `pipe_fitting_no_input' to indicate that the fitting has no input. 127 | %% 128 | -module(riak_pipe_vnode_worker). 129 | 130 | -compile({nowarn_deprecated_function, 131 | [{gen_fsm, start_link, 3}, 132 | {gen_fsm, send_event, 2}]}). 133 | 134 | -behaviour(gen_fsm). 135 | 136 | %% API 137 | -export([start_link/3]). 138 | -export([send_input/2, 139 | recurse_input/3, 140 | recurse_input/4, 141 | send_handoff/2, 142 | send_archive/1, 143 | send_output/3, 144 | send_output/4, 145 | send_output/5]). 146 | 147 | %% gen_fsm callbacks 148 | -export([ 149 | init/1, 150 | initial_input_request/2, 151 | wait_for_input/2, 152 | handle_event/3, 153 | handle_sync_event/4, 154 | handle_info/3, 155 | terminate/3, 156 | code_change/4 157 | ]). 158 | 159 | -include("riak_pipe.hrl"). 160 | -include("riak_pipe_log.hrl"). 161 | 162 | -record(state, {partition :: riak_pipe_vnode:partition(), 163 | details :: riak_pipe_fitting:details(), 164 | vnode :: pid(), 165 | modstate :: term()}). 166 | -opaque state() :: #state{}. 167 | -export_type([state/0]). 168 | 169 | -type callback_state() :: term(). 170 | 171 | -callback init(riak_pipe_vnode:partition(), 172 | riak_pipe_fitting:details()) -> 173 | {ok, callback_state()}. 174 | -callback process(term(), boolean(), callback_state()) -> 175 | {ok, callback_state()} | 176 | {forward_preflist, callback_state()} | 177 | {{error, term()}, callback_state()}. 178 | -callback done(callback_state()) -> ok. 179 | 180 | %%%=================================================================== 181 | %%% API 182 | %%%=================================================================== 183 | 184 | %% @doc Start a worker for the specified fitting+vnode. 185 | -spec start_link(riak_pipe_vnode:partition(), 186 | pid(), 187 | riak_pipe_fitting:details()) -> 188 | {ok, pid()} | ignore | {error, term()}. 189 | start_link(Partition, VnodePid, FittingDetails) -> 190 | gen_fsm:start_link(?MODULE, [Partition, VnodePid, FittingDetails], []). 191 | 192 | %% @doc Send input to the worker. Note: this should only be called 193 | %% by the vnode that owns the worker, as the result of the worker 194 | %% asking for its next input. 195 | -spec send_input(pid(), done | {term(), riak_core_apl:preflist()}) -> ok. 196 | send_input(WorkerPid, Input) -> 197 | gen_fsm:send_event(WorkerPid, {input, Input}). 198 | 199 | %% @doc Ask the worker to merge handoff data from an archived worker. 200 | %% Note: this should only be called by the vnode that owns the 201 | %% worker, as the result of the worker asking for its next input 202 | %% when the vnode has received handoff data for the worker's 203 | %% fitting. 204 | -spec send_handoff(pid(), Archive::term()) -> ok. 205 | send_handoff(WorkerPid, Handoff) -> 206 | gen_fsm:send_event(WorkerPid, {handoff, Handoff}). 207 | 208 | %% @doc Ask the worker to archive itself. The worker will send the 209 | %% archive data to the owning vnode when it has done so. Once 210 | %% it has sent the archive, the worker shuts down normally. 211 | -spec send_archive(pid()) -> ok. 212 | send_archive(WorkerPid) -> 213 | gen_fsm:send_event(WorkerPid, archive). 214 | 215 | %% @equiv send_output(Output, FromPartition, Details, infinity) 216 | send_output(Output, FromPartition, Details) -> 217 | send_output(Output, FromPartition, Details, infinity). 218 | 219 | %% @doc Send output from the given fitting to the next output down the 220 | %% line. `FromPartition' is used in the case that the next 221 | %% fitting's partition function is `follow'. 222 | -spec send_output(term(), 223 | riak_pipe_vnode:partition(), 224 | riak_pipe_fitting:details(), 225 | riak_pipe_vnode:qtimeout()) -> 226 | ok | {error, term()}. 227 | send_output(Output, FromPartition, 228 | #fitting_details{output=Fitting}=Details, 229 | Timeout) -> 230 | send_output(Output, FromPartition, Details, Fitting, Timeout). 231 | 232 | %% @equiv recurse_input(Input, FromPartition, Details, noblock) 233 | recurse_input(Input, FromPartition, Details) -> 234 | recurse_input(Input, FromPartition, Details, noblock). 235 | 236 | %% @doc Send a new input from this fitting, to itself. This can be 237 | %% used to write fittings that perform recursive calculation, 238 | %% where steps in the recursion might be done in parallel. 239 | %% 240 | %% For example, when walking intermediate tree nodes, using 241 | %% `recurse_input/3' to send children to other vnodes, instead of 242 | %% processing them in the same worker, may be a useful strategy. 243 | %% 244 | %% WARNING: Using recurse_input with a `Timeout' of `infinity' is 245 | %% discouraged, unless you can guarantee that the queues for a 246 | %% fitting will never be full. Otherwise, it's possible to 247 | %% deadlock a fitting by blocking on enqueueing an input for a 248 | %% worker that is blocking on enqueueing an input for the sender 249 | %% (circular blocking). Use `noblock' and handle timeout 250 | %% failures to prevent deadlock. 251 | %% 252 | %% Internal details: This works because of the nature of the 253 | %% blocking enqueue operation. It is guaranteed that as long as 254 | %% this worker is alive, the fitting for which it works will not 255 | %% receive all of its `done' messages. So, the vnode that 256 | %% enqueues this input will still be able to ask the fitting for 257 | %% details, and the fitting will know that it has to wait on that 258 | %% vnode. 259 | -spec recurse_input(term(), 260 | riak_pipe_vnode:partition(), 261 | riak_pipe_fitting:details(), 262 | riak_pipe_vnode:qtimeout()) -> 263 | ok | {error, term()}. 264 | recurse_input(Input, FromPartition, Details, Timeout) -> 265 | recurse_input(Input, FromPartition, Details, Timeout, []). 266 | 267 | -spec recurse_input(term(), 268 | riak_pipe_vnode:partition(), 269 | riak_pipe_fitting:details(), 270 | riak_pipe_vnode:qtimeout(), 271 | riak_core_apl:preflist()) -> 272 | ok | {error, term()}. 273 | recurse_input(Input, FromPartition, 274 | #fitting_details{fitting=Fitting}=Details, 275 | Timeout, UsedPreflist) -> 276 | send_output(Input, FromPartition, Details, 277 | Fitting, Timeout, UsedPreflist). 278 | 279 | %% @doc Send output from the given fitting to a specific fitting. 280 | %% This is most often used to send output to the sink, but also 281 | %% happens to be the internal implementation of {@link 282 | %% send_output/3}. 283 | -spec send_output(term(), riak_pipe_vnode:partition(), 284 | riak_pipe_fitting:details(), riak_pipe:fitting(), 285 | riak_pipe_vnode:qtimeout()) -> 286 | ok | {error, term()}. 287 | send_output(Output, FromPartition, Details, FittingOverride, Timeout) -> 288 | send_output(Output, FromPartition, Details, FittingOverride, Timeout, []). 289 | 290 | -spec send_output(term(), riak_pipe_vnode:partition(), 291 | riak_pipe_fitting:details(), riak_pipe:fitting(), 292 | riak_pipe_vnode:qtimeout(), 293 | riak_core_apl:preflist()) -> 294 | ok | {error, term()}. 295 | send_output(Output, FromPartition, 296 | #fitting_details{name=Name, options=Opts}=_Details, 297 | FittingOverride, 298 | Timeout, UsedPreflist) -> 299 | case FittingOverride#fitting.chashfun of 300 | sink -> 301 | riak_pipe_sink:result(Name, FittingOverride, Output, Opts); 302 | follow -> 303 | %% TODO: should 'follow' use the original preflist (in 304 | %% case of failover)? 305 | riak_pipe_vnode:queue_work( 306 | FittingOverride, Output, Timeout, UsedPreflist, 307 | riak_pipe_vnode:hash_for_partition(FromPartition)); 308 | _ -> 309 | riak_pipe_vnode:queue_work( 310 | FittingOverride, Output, Timeout, UsedPreflist) 311 | end. 312 | 313 | %%%=================================================================== 314 | %%% gen_fsm callbacks 315 | %%%=================================================================== 316 | 317 | %% @doc Initialize the worker. This function calls the implementing 318 | %% module's init function. If that init function fails, the 319 | %% worker stops with an `{init_failed, Type, Error}' reason. 320 | -spec init([riak_pipe_vnode:partition() | pid() 321 | | riak_pipe_fitting:details()]) -> 322 | {ok, initial_input_request, state(), 0} 323 | | {stop, {init_failed, term(), term()}}. 324 | init([Partition, VnodePid, #fitting_details{module=Module}=FittingDetails]) -> 325 | try 326 | put(eunit, [{module, ?MODULE}, 327 | {partition, Partition}, 328 | {VnodePid, VnodePid}, 329 | {details, FittingDetails}]), 330 | {ok, ModState} = Module:init(Partition, FittingDetails), 331 | {ok, initial_input_request, 332 | #state{partition=Partition, 333 | details=FittingDetails, 334 | vnode=VnodePid, 335 | modstate=ModState}, 336 | 0} 337 | catch Type:Error -> 338 | {stop, {init_failed, Type, Error}} 339 | end. 340 | 341 | %% @doc The worker has just started, and should request its first 342 | %% input from its owning vnode. This is done after a zero 343 | %% timeout instead of in the init function to get around the 344 | %% deadlock that would result from having the worker wait for a 345 | %% message from the vnode, which is waiting for a response from 346 | %% this process. 347 | -spec initial_input_request(timeout, state()) -> 348 | {next_state, wait_for_input, state()}. 349 | initial_input_request(timeout, State) -> 350 | request_input(State), 351 | {next_state, wait_for_input, State}. 352 | 353 | %% @doc The worker has requested its next input item, and is waiting 354 | %% for it. 355 | %% 356 | %% If the input is `done', due to end-of-inputs from the fitting, 357 | %% then the implementing module's `done' function is evaluated, 358 | %% the the worker terminates normally. 359 | %% 360 | %% If the input is any regular input, then the implementing 361 | %% module's `process' function is evaluated. When it finishes, 362 | %% the next input is requested from the vnode. 363 | %% 364 | %% If the input is a handoff from another vnode, the worker asks 365 | %% the implementing module to merge the archive, if the worker 366 | %% exports that functionality. 367 | %% 368 | %% If the input is a request to archive, the worker asks the 369 | %% implementing module to archive itself, if the worker exports 370 | %% that functionality. When the archiving process has finished, 371 | %% the worker terminates normally. 372 | -spec wait_for_input({input, done | {term(), riak_core_apl:preflist()}} 373 | |{handoff, term()} 374 | |archive, 375 | state()) -> 376 | {next_state, wait_for_input, state()} 377 | | {stop, normal, state()}. 378 | wait_for_input({input, done}, State) -> 379 | ok = process_done(State), 380 | {stop, normal, State}; %%TODO: monitor 381 | wait_for_input({input, {Input, UsedPreflist}}, State) -> 382 | NewState = process_input(Input, UsedPreflist, State), 383 | request_input(NewState), 384 | {next_state, wait_for_input, NewState}; 385 | wait_for_input({handoff, HandoffState}, State) -> 386 | %% receiving handoff from another vnode 387 | NewState = handoff(HandoffState, State), 388 | request_input(NewState), 389 | {next_state, wait_for_input, NewState}; 390 | wait_for_input(archive, State) -> 391 | %% sending handoff to another vnode 392 | Archive = archive(State), 393 | reply_archive(Archive, State), 394 | {stop, normal, State}. 395 | 396 | %% @doc Unused. 397 | -spec handle_event(term(), atom(), state()) -> 398 | {next_state, atom(), state()}. 399 | handle_event(_Event, StateName, State) -> 400 | {next_state, StateName, State}. 401 | 402 | %% @doc Unused. 403 | -spec handle_sync_event(term(), term(), atom(), state()) -> 404 | {reply, ok, atom(), state()}. 405 | handle_sync_event(_Event, _From, StateName, State) -> 406 | Reply = ok, 407 | {reply, Reply, StateName, State}. 408 | 409 | %% @doc Unused. 410 | -spec handle_info(term(), atom(), state()) -> 411 | {next_state, atom(), state()}. 412 | handle_info(_Info, StateName, State) -> 413 | {next_state, StateName, State}. 414 | 415 | %% @doc Unused. 416 | -spec terminate(term(), atom(), state()) -> ok. 417 | terminate(_Reason, _StateName, _State) -> 418 | ok. 419 | 420 | %% @doc Unused. 421 | -spec code_change(term(), atom(), state(), term()) -> 422 | {ok, atom(), state()}. 423 | code_change(_OldVsn, StateName, State, _Extra) -> 424 | {ok, StateName, State}. 425 | 426 | %%%=================================================================== 427 | %%% Internal functions 428 | %%%=================================================================== 429 | 430 | %% @doc Ask the vnode for this worker's next input. The input will be 431 | %% sent as an event later. 432 | -spec request_input(state()) -> ok. 433 | request_input(#state{vnode=Vnode, details=Details}) -> 434 | riak_pipe_vnode:next_input(Vnode, Details#fitting_details.fitting). 435 | 436 | %% @doc Process an input - call the implementing module's `process/3' 437 | %% function. 438 | -spec process_input(term(), riak_core_apl:preflist(), state()) -> state(). 439 | process_input(Input, UsedPreflist, 440 | #state{details=FD, modstate=ModState}=State) -> 441 | Module = FD#fitting_details.module, 442 | NVal = case (FD#fitting_details.fitting)#fitting.nval of 443 | NValInt when is_integer(NValInt) -> NValInt; 444 | {NValMod, NValFun} -> NValMod:NValFun(Input); 445 | %% 1.0.x compatibility 446 | NValFun -> 447 | riak_pipe_fun:compat_apply(NValFun, [Input]) 448 | end, 449 | try 450 | {Result, NewModState} = Module:process(Input, 451 | length(UsedPreflist) == NVal, 452 | ModState), 453 | case Result of 454 | ok -> 455 | ok; 456 | forward_preflist -> 457 | forward_preflist(Input, UsedPreflist, State); 458 | {error, RError} -> 459 | processing_error( 460 | result, RError, FD, ModState, Module, State, Input) 461 | end, 462 | State#state{modstate=NewModState} 463 | catch Type:Error -> 464 | processing_error(Type, Error, FD, ModState, Module, State, Input), 465 | exit(processing_error) 466 | end. 467 | 468 | %% @private 469 | processing_error(Type, Error, FD, ModState, Module, State, Input) -> 470 | Fields = record_info(fields, fitting_details), 471 | FieldPos = lists:zip(Fields, lists:seq(2, length(Fields)+1)), 472 | DsList = [{Field, element(Pos, FD)} || {Field, Pos} <- FieldPos], 473 | ?T_ERR(FD, [{module, Module}, 474 | {partition, State#state.partition}, 475 | {details, DsList}, 476 | {type, Type}, 477 | {error, Error}, 478 | {input, Input}, 479 | {modstate, ModState}, 480 | {stack, erlang:process_info(self(), [current_stacktrace])}]). 481 | 482 | %% @doc Process a done (end-of-inputs) message - call the implementing 483 | %% module's `done/1' function. 484 | -spec process_done(state()) -> ok. 485 | process_done(#state{details=FD, modstate=ModState}) -> 486 | Module = FD#fitting_details.module, 487 | Module:done(ModState). 488 | 489 | %% @doc Process a handoff message - call the implementing module's 490 | %% `handoff/2' function, if exported. 491 | -spec handoff(term(), state()) -> state(). 492 | handoff(HandoffArchive, #state{details=FD, modstate=ModState}=State) -> 493 | Module = FD#fitting_details.module, 494 | case lists:member({handoff, 2}, Module:module_info(exports)) of 495 | true -> 496 | {ok, NewModState} = Module:handoff(HandoffArchive, ModState), 497 | State#state{modstate=NewModState}; 498 | false -> 499 | %% module doesn't bother handing off state 500 | State 501 | end. 502 | 503 | %% @doc Process an archive request - call the implementing module's 504 | %% `archive/1' fucntion, if exported. The atom `undefined' is if 505 | %% archive/1 is not exported. 506 | -spec archive(state()) -> term(). 507 | archive(#state{details=FD, modstate=ModState}) -> 508 | Module = FD#fitting_details.module, 509 | case lists:member({archive, 1}, Module:module_info(exports)) of 510 | true -> 511 | {ok, Archive} = Module:archive(ModState), 512 | Archive; 513 | false -> 514 | %% module doesn't bother handif off state 515 | undefined 516 | end. 517 | 518 | %% @doc Send the archive to the vnode after it has been generated. 519 | -spec reply_archive(term(), state()) -> ok. 520 | reply_archive(Archive, #state{vnode=Vnode, details=Details}) -> 521 | riak_pipe_vnode:reply_archive(Vnode, 522 | Details#fitting_details.fitting, 523 | Archive). 524 | 525 | %% @doc Instead of processing this input here, forward it to the next 526 | %% vnode in its preflist. 527 | %% 528 | %% If that forwarding fails, an error trace message will be sent 529 | %% under the filter `forward_preflist', listing the error and the 530 | %% input. 531 | -spec forward_preflist(term(), riak_core_apl:preflist(), state()) -> ok. 532 | forward_preflist(Input, UsedPreflist, 533 | #state{partition=Partition, 534 | details=FittingDetails}=State) -> 535 | case recurse_input(Input, Partition, FittingDetails, 536 | noblock, UsedPreflist) of 537 | ok -> ok; 538 | {error, Error} -> 539 | processing_error(forward_preflist, Error, FittingDetails, 540 | State#state.modstate, 541 | FittingDetails#fitting_details.module, 542 | State, Input) 543 | end. 544 | -------------------------------------------------------------------------------- /src/riak_pipe_fitting.erl: -------------------------------------------------------------------------------- 1 | %%-------------------------------------------------------------------- 2 | %% 3 | %% Copyright (c) 2011 Basho Technologies, Inc. 4 | %% 5 | %% This file is provided to you under the Apache License, 6 | %% Version 2.0 (the "License"); you may not use this file 7 | %% except in compliance with the License. You may obtain 8 | %% a copy of the License at 9 | %% 10 | %% http://www.apache.org/licenses/LICENSE-2.0 11 | %% 12 | %% Unless required by applicable law or agreed to in writing, 13 | %% software distributed under the License is distributed on an 14 | %% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | %% KIND, either express or implied. See the License for the 16 | %% specific language governing permissions and limitations 17 | %% under the License. 18 | %% 19 | %%-------------------------------------------------------------------- 20 | 21 | %% @doc The coordinator process that hold the details for the fitting. 22 | %% This process also manages the end-of-inputs synchronization 23 | %% for this stage of the pipeline. 24 | -module(riak_pipe_fitting). 25 | 26 | -compile({nowarn_deprecated_function, 27 | [{gen_fsm, start_link, 3}, 28 | {gen_fsm, send_event, 2}, 29 | {gen_fsm, sync_send_event, 2}, 30 | {gen_fsm, sync_send_all_state_event, 2}]}). 31 | 32 | -behaviour(gen_fsm). 33 | 34 | %% API 35 | -export([start_link/4]). 36 | -export([eoi/1, 37 | get_details/2, 38 | worker_done/1, 39 | workers/1]). 40 | -export([validate_fitting/1, 41 | format_name/1]). 42 | 43 | %% gen_fsm callbacks 44 | -export([init/1, 45 | wait_upstream_eoi/2, wait_upstream_eoi/3, 46 | wait_workers_done/3, 47 | handle_event/3, 48 | handle_sync_event/4, 49 | handle_info/3, 50 | terminate/3, 51 | code_change/4]). 52 | 53 | -include_lib("kernel/include/logger.hrl"). 54 | 55 | -include("riak_pipe.hrl"). 56 | -include("riak_pipe_log.hrl"). 57 | -include("riak_pipe_debug.hrl"). 58 | 59 | -ifdef(TEST). 60 | -include_lib("eunit/include/eunit.hrl"). 61 | -endif. 62 | 63 | -ifdef(PULSE). 64 | -include_lib("pulse/include/pulse.hrl"). 65 | %% have to transform the 'receive' of the work results 66 | -compile({parse_transform, pulse_instrument}). 67 | %% don't trasnform toplevel test functions 68 | -compile({pulse_replace_module,[{gen_fsm,pulse_gen_fsm}]}). 69 | -endif. 70 | 71 | -record(worker, {partition :: riak_pipe_vnode:partition(), 72 | pid :: pid(), 73 | monitor :: reference()}). 74 | -record(state, {builder :: pid(), 75 | details :: #fitting_details{}, 76 | workers :: [#worker{}], 77 | ref :: reference()}). %% to avoid digging two levels 78 | 79 | -opaque state() :: #state{}. 80 | 81 | -export_type([state/0, details/0]). 82 | -type details() :: #fitting_details{}. 83 | 84 | %%%=================================================================== 85 | %%% API 86 | %%%=================================================================== 87 | 88 | %% @doc Start the coordinator, according to the `Spec' given. The 89 | %% coordinator will register with `Builder' and will request its 90 | %% outputs to be processed under the `Output' fitting. 91 | -spec start_link(pid(), 92 | riak_pipe:fitting_spec(), 93 | riak_pipe:fitting(), 94 | riak_pipe:exec_opts()) -> 95 | {ok, pid(), riak_pipe:fitting()} | ignore | {error, term()}. 96 | start_link(Builder, Spec, Output, Options) -> 97 | case gen_fsm:start_link(?MODULE, [Builder, Spec, Output, Options], []) of 98 | {ok, Pid} -> 99 | {ok, Pid, fitting_record(Pid, Spec, Output)}; 100 | Error -> 101 | Error 102 | end. 103 | 104 | %% @doc Send an end-of-inputs message to the specified coordinator. 105 | -spec eoi(riak_pipe:fitting()) -> ok. 106 | eoi(#fitting{pid=Pid, ref=Ref, chashfun=C}) when C =/= sink -> 107 | gen_fsm:send_event(Pid, {eoi, Ref}). 108 | 109 | %% @doc Request the details about this fitting. The ring partition 110 | %% index of the vnode requesting the details is included such 111 | %% that the coordinator can inform the vnode of end-of-inputs later. 112 | %% This function assumes that it is being called from the vnode 113 | %% process, so the `self()' can be used to give the coordinator 114 | %% a pid to monitor. 115 | -spec get_details(riak_pipe:fitting(), riak_pipe_vnode:partition()) -> 116 | {ok, details()} | gone. 117 | get_details(#fitting{pid=Pid, ref=Ref}, Partition) -> 118 | try 119 | gen_fsm:sync_send_event(Pid, {get_details, Ref, Partition, self()}) 120 | catch exit:_ -> 121 | %% catching all exit types here , since we don't care 122 | %% whether the coordinator was gone before we asked ('noproc') 123 | %% or if it went away before responding ('normal' or other 124 | %% exit reason) 125 | gone 126 | end. 127 | 128 | %% @doc Tell the coordinator that this worker is done. This function 129 | %% assumes that it is being called from the vnode process, so 130 | %% that `self()' can be used to inform the coordinator of which 131 | %% worker is done. 132 | -spec worker_done(riak_pipe:fitting()) -> ok | gone. 133 | worker_done(#fitting{pid=Pid, ref=Ref}) -> 134 | try 135 | gen_fsm:sync_send_event(Pid, {done, Ref, self()}) 136 | catch exit:_ -> 137 | %% catching all exit types here , since we don't care 138 | %% whether the coordinator was gone before we asked ('noproc') 139 | %% or if it went away before responding ('normal' or other 140 | %% exit reason) 141 | gone 142 | end. 143 | 144 | %% @doc Get the list of ring partition indexes (vnodes) that are doing 145 | %% work for this coordinator. 146 | -spec workers(pid()) -> {ok, [riak_pipe_vnode:partition()]} | gone. 147 | workers(Fitting) -> 148 | try 149 | {ok, gen_fsm:sync_send_all_state_event(Fitting, workers)} 150 | catch exit:_ -> 151 | %% catching all exit types here , since we don't care 152 | %% whether the coordinator was gone before we asked ('noproc') 153 | %% or if it went away before responding ('normal' or other 154 | %% exit reason) 155 | gone 156 | end. 157 | 158 | %%%=================================================================== 159 | %%% gen_fsm callbacks 160 | %%%=================================================================== 161 | 162 | %% @doc Initialize the coordinator. This function monitors the 163 | %% builder process, so it will tear down if the builder exits. 164 | -spec init([pid() | riak_pipe:fitting_spec() | riak_pipe:fitting() 165 | | riak_pipe:exec_opts()]) -> 166 | {ok, wait_upstream_eoi, state()}. 167 | init([Builder, 168 | #fitting_spec{name=Name, module=Module, arg=Arg, q_limit=QLimit}=Spec, 169 | Output, 170 | Options]) -> 171 | Fitting = fitting_record(self(), Spec, Output), 172 | Details = #fitting_details{fitting=Fitting, 173 | name=Name, 174 | module=Module, 175 | arg=Arg, 176 | output=Output, 177 | options=Options, 178 | q_limit=QLimit}, 179 | 180 | ?T(Details, [], {fitting, init_started}), 181 | 182 | erlang:monitor(process, Builder), 183 | 184 | ?T(Details, [], {fitting, init_finished}), 185 | 186 | put(eunit, [{module, ?MODULE}, 187 | {fitting, Fitting}, 188 | {details, Details}, 189 | {builder, Builder}]), 190 | {ok, wait_upstream_eoi, 191 | #state{builder=Builder, details=Details, workers=[], 192 | ref=Output#fitting.ref}}. 193 | 194 | %% @doc The coordinator is just hanging out, serving details and waiting 195 | %% for end-of-inputs. 196 | %% 197 | %% When it gets eoi, it forwards the signal to its workers, and 198 | %% then begins waiting for them to respond done. If it has no 199 | %% workers when it receives end-of-inputs, the coordinator stops 200 | %% immediately. 201 | -spec wait_upstream_eoi(eoi, state()) -> 202 | {stop, normal, state()} 203 | | {next_state, wait_workers_done, state()}. 204 | wait_upstream_eoi({eoi, Ref}, 205 | #state{ref=Ref, workers=[], details=Details}=State) -> 206 | ?T(Details, [eoi], {fitting, receive_eoi}), 207 | %% No workers to stop 208 | try 209 | %% To assist some fittings, such as riak_kv_w_reduce, we need 210 | %% to fake spinning up a single worker and have it send its 211 | %% result downstream (which is done as a side-effect of 212 | %% calling wait_for_input()). 213 | #fitting_details{module=Module, options=Os0} = Details, 214 | true = Module:no_input_run_reduce_once(), 215 | Os = [pipe_fitting_no_input|Os0], 216 | {ok, WState1} = Module:init(0, Details#fitting_details{options=Os}), 217 | _ = Module:done(WState1) 218 | catch 219 | error:_ -> % undef or badmatch 220 | ok 221 | end, 222 | forward_eoi(State), 223 | {stop, normal, State}; 224 | wait_upstream_eoi({eoi, Ref}, 225 | #state{ref=Ref, workers=Workers, details=Details}=State) -> 226 | ?T(Details, [eoi], {fitting, receive_eoi}), 227 | _ = [ riak_pipe_vnode:eoi(Pid, Details#fitting_details.fitting) 228 | || #worker{pid=Pid} <- Workers ], 229 | {next_state, wait_workers_done, State}; 230 | wait_upstream_eoi(_, State) -> 231 | %% unknown message - ignore 232 | {next_state, wait_upstream_eoi, State}. 233 | 234 | 235 | %% @doc The coordinator is just hanging out, serving details and waiting 236 | %% for end-of-inputs. 237 | %% 238 | %% When it gets a request for the fitting details, it sets up 239 | %% a monitor for the working vnode, and responds with details. 240 | %% 241 | %% The coordinator may receive a `done' message from a vnode before 242 | %% eoi has been sent, if handoff causes the worker to relocate. 243 | %% In this case, the coordinator simply demonitors the vnode, and 244 | %% removes it from its worker list. 245 | -spec wait_upstream_eoi({get_details, riak_pipe_vnode:partition(), pid()}, 246 | term(), state()) -> 247 | {reply, {ok, details()}, wait_upstream_eoi, state()}; 248 | ({done, pid()}, term(), state()) -> 249 | {reply, ok, wait_upstream_eoi, state()}. 250 | wait_upstream_eoi({get_details, Ref, Partition, Pid}=M, _From, 251 | #state{ref=Ref}=State) -> 252 | ?T(State#state.details, [get_details], {fitting, M}), 253 | NewState = add_worker(Partition, Pid, State), 254 | {reply, 255 | {ok, State#state.details}, 256 | wait_upstream_eoi, 257 | NewState}; 258 | wait_upstream_eoi({done, Ref, Pid}=M, _From, #state{ref=Ref}=State) -> 259 | %% handoff caused early done 260 | ?T(State#state.details, [done], {early_fitting, M}), 261 | case lists:keytake(Pid, #worker.pid, State#state.workers) of 262 | {value, Worker, Rest} -> 263 | erlang:demonitor(Worker#worker.monitor); 264 | false -> 265 | Rest = State#state.workers 266 | end, 267 | %% don't check for empty Rest like in wait_workers_done, though 268 | %% because we haven't seen eoi yet 269 | {reply, ok, wait_upstream_eoi, State#state{workers=Rest}}; 270 | wait_upstream_eoi(_, _, State) -> 271 | %% unknown message - reply {error, unknown} to get rid of it 272 | {reply, {error, unknown}, wait_upstream_eoi, State}. 273 | 274 | %% @doc The coordinator has forwarded the end-of-inputs signal to all of 275 | %% the vnodes working for it, and is waiting for done responses. 276 | %% 277 | %% When the coordinator receives a done response, it demonitors 278 | %% the vnode that sent it, and removes it from its worker list. 279 | %% If there are no more responses to wait for, the coordinator 280 | %% forwards the end-of-inputs signal to the coordinator for the 281 | %% next fitting in the pipe, and then shuts down normally. 282 | %% 283 | %% If the coordinator receives a request for details from a vnode 284 | %% while in this state, it responds with the detail as usual, 285 | %% but also immediately sends end-of-inputs to that vnode. 286 | -spec wait_workers_done({get_details, riak_pipe_vnode:partition(), pid()}, 287 | term(), state()) -> 288 | {reply, {ok, details()}, wait_workers_done, state()}; 289 | ({done, pid()}, term(), state()) -> 290 | {reply, ok, wait_workers_done, state()} 291 | | {stop, normal, ok, state()}. 292 | wait_workers_done({get_details, Ref, Partition, Pid}=M, _From, 293 | #state{ref=Ref}=State) -> 294 | %% handoff caused a late get_details 295 | ?T(State#state.details, [get_details], {late_fitting, M}), 296 | %% send details, and monitor as usual 297 | NewState = add_worker(Partition, Pid, State), 298 | %% also send eoi, to have worker immediately finish up 299 | Details = NewState#state.details, 300 | riak_pipe_vnode:eoi(Pid, Details#fitting_details.fitting), 301 | {reply, 302 | {ok, NewState#state.details}, 303 | wait_workers_done, 304 | NewState}; 305 | wait_workers_done({done, Ref, Pid}=M, _From, #state{ref=Ref}=State) -> 306 | ?T(State#state.details, [done], {fitting, M}), 307 | case lists:keytake(Pid, #worker.pid, State#state.workers) of 308 | {value, Worker, Rest} -> 309 | erlang:demonitor(Worker#worker.monitor); 310 | false -> 311 | Rest = State#state.workers 312 | end, 313 | case Rest of 314 | [] -> 315 | forward_eoi(State), 316 | {stop, normal, ok, State#state{workers=[]}}; 317 | _ -> 318 | {reply, ok, wait_workers_done, State#state{workers=Rest}} 319 | end; 320 | wait_workers_done(_, _, State) -> 321 | %% unknown message - reply {error, unknown} to get rid of it 322 | {reply, {error, unknown}, wait_workers_done, State}. 323 | 324 | %% @doc Unused. 325 | -spec handle_event(term(), atom(), state()) -> 326 | {next_state, atom(), state()}. 327 | handle_event(_Event, StateName, State) -> 328 | {next_state, StateName, State}. 329 | 330 | %% @doc The only sync event handled in all states is `workers', which 331 | %% retrieves a list of ring partition indexes that have requested 332 | %% the fitting details (i.e. that are doing work for this 333 | %% coordinator). 334 | -spec handle_sync_event(workers, term(), atom(), state()) -> 335 | {reply, [riak_pipe_vnode:partition()], atom(), state()}. 336 | handle_sync_event(workers, _From, StateName, #state{workers=Workers}=State) -> 337 | Partitions = [ P || #worker{partition=P} <- Workers ], 338 | {reply, Partitions, StateName, State}; 339 | handle_sync_event({test_crash, Fun},_,_,_) -> 340 | %% Only test-enabled client sends this. 341 | %% See riak_test's rt_pipe:crash_fitting/2 and pipe_verify_* tests 342 | Fun(); 343 | handle_sync_event(_Event, _From, StateName, State) -> 344 | Reply = ok, 345 | {reply, Reply, StateName, State}. 346 | 347 | %% @doc The non-gen_fsm message that this process expects is 'DOWN'. 348 | %% 349 | %% 'DOWN' messages are received when monitored vnodes exit. In 350 | %% that case, the vnode is removed from the worker list. If that 351 | %% was also the last vnode we were waiting on a `done' message 352 | %% from, also forward `eoi' and shut down the coordinator. 353 | -spec handle_info({'DOWN', reference(), term(), term(), term()}, 354 | atom(), state()) -> 355 | {next_state, atom(), state()} 356 | |{stop, normal, state()}. 357 | handle_info({'DOWN', _Ref, process, Builder, _Reason}, 358 | _StateName, 359 | #state{builder=Builder}=State) -> 360 | %% if the builder exits, stop immediately 361 | {stop, normal, State}; 362 | handle_info({'DOWN', Ref, _, _, _}, StateName, State) -> 363 | case lists:keytake(Ref, #worker.monitor, State#state.workers) of 364 | {value, Worker, Rest} -> 365 | ?T(State#state.details, [done, 'DOWN'], 366 | {vnode_failure, Worker#worker.partition}), 367 | %% check whether this coordinator was just waiting on a final 368 | %% 'done' and stop if so (because anything left in that 369 | %% vnode's worker queue is lost) 370 | case {StateName, Rest} of 371 | {wait_workers_done, []} -> 372 | forward_eoi(State), 373 | {stop, normal, State#state{workers=[]}}; 374 | _ -> 375 | {next_state, StateName, State#state{workers=Rest}} 376 | end; 377 | false -> 378 | %% looks like a misdirected down notification - ignore 379 | {next_state, StateName, State} 380 | end; 381 | handle_info(_Info, StateName, State) -> 382 | {next_state, StateName, State}. 383 | 384 | %% @doc Unused. 385 | -spec terminate(term(), atom(), state()) -> ok. 386 | terminate(_Reason, _StateName, _State) -> 387 | ok. 388 | 389 | %% @doc Unused. 390 | -spec code_change(term(), atom(), state(), term()) -> 391 | {ok, atom(), state()}. 392 | code_change(_OldVsn, StateName, State, _Extra) -> 393 | {ok, StateName, State}. 394 | 395 | %%%=================================================================== 396 | %%% Internal functions 397 | %%%=================================================================== 398 | 399 | %% @doc Construct a #fitting{} record, given this coordinator's pid, 400 | %% its fitting spec, and output destination. 401 | -spec fitting_record(pid(), 402 | Spec::riak_pipe:fitting_spec(), 403 | Output::riak_pipe:fitting()) -> 404 | riak_pipe:fitting(). 405 | fitting_record(Pid, 406 | #fitting_spec{chashfun=HashFun, nval=NVal}, 407 | #fitting{ref=Ref}) -> 408 | #fitting{pid=Pid, ref=Ref, chashfun=HashFun, nval=NVal}. 409 | 410 | %% @doc Send the end-of-inputs signal to the next coordinator. 411 | -spec forward_eoi(state()) -> ok. 412 | forward_eoi(#state{details=Details}) -> 413 | ?T(Details, [eoi], {fitting, send_eoi}), 414 | case Details#fitting_details.output of 415 | #fitting{chashfun=sink}=Sink -> 416 | riak_pipe_sink:eoi(Sink, Details#fitting_details.options); 417 | #fitting{}=Fitting -> 418 | riak_pipe_fitting:eoi(Fitting) 419 | end. 420 | 421 | %% @doc Monitor the given vnode, and add it to our list of workers. 422 | -spec add_worker(riak_pipe_vnode:partition(), pid(), state()) -> state(). 423 | add_worker(Partition, Pid, State) -> 424 | %% check if we're already monitoring this pid before setting up a 425 | %% new monitor (in case pid re-requests details) 426 | case worker_by_partpid(Partition, Pid, State) of 427 | {ok, _Worker} -> 428 | %% already monitoring 429 | State; 430 | none -> 431 | Ref = erlang:monitor(process, Pid), 432 | State#state{workers=[#worker{partition=Partition, 433 | pid=Pid, 434 | monitor=Ref} 435 | |State#state.workers]} 436 | end. 437 | 438 | %% @doc Find a worker's entry in the worker list by its ring 439 | %% partition index and pid. 440 | -spec worker_by_partpid(riak_pipe_vnode:partition(), pid(), state()) -> 441 | {ok, #worker{}} | none. 442 | worker_by_partpid(Partition, Pid, #state{workers=Workers}) -> 443 | case [ W || #worker{partition=A, pid=I}=W <- Workers, 444 | A == Partition, I == Pid] of 445 | [#worker{}=Worker] -> {ok, Worker}; 446 | [] -> none 447 | end. 448 | 449 | %% @doc Ensure that a fitting specification is valid. This function 450 | %% will check that the module is an atom that names a valid 451 | %% module (see {@link riak_pipe_v:validate_module/2}), that the 452 | %% arg is valid for the module (see {@link validate_argument/2}), 453 | %% and that the partition function is of the proper form (see 454 | %% {@link validate_chashfun/1}). It also checks that nval is 455 | %% undefined or a postive integer. 456 | %% 457 | %% If all components are valid, the atom `ok' is returned. If 458 | %% any piece is invalid, `{badarg, #fitting_spec.name, ErrorMsg}' 459 | %% is thrown. 460 | -spec validate_fitting(riak_pipe:fitting_spec()) -> ok. 461 | validate_fitting(#fitting_spec{name=Name, 462 | module=Module, 463 | arg=Arg, 464 | chashfun=HashFun, 465 | nval=NVal}) -> 466 | case riak_pipe_v:validate_module("module", Module) of 467 | ok -> ok; 468 | {error, ModError} -> 469 | ?LOG_ERROR( 470 | "Invalid module in fitting spec \"~s\": ~s", 471 | [format_name(Name), ModError]), 472 | throw({badarg, Name, ModError}) 473 | end, 474 | case validate_argument(Module, Arg) of 475 | ok -> ok; 476 | {error, ArgError} -> 477 | ?LOG_ERROR( 478 | "Invalid module argument in fitting spec \"~s\": ~s", 479 | [format_name(Name), ArgError]), 480 | throw({badarg, Name, ArgError}) 481 | end, 482 | case validate_chashfun(HashFun) of 483 | ok -> ok; 484 | {error, PFError} -> 485 | ?LOG_ERROR( 486 | "Invalid chashfun in fitting spec \"~s\": ~s", 487 | [format_name(Name), PFError]), 488 | throw({badarg, Name, PFError}) 489 | end, 490 | case validate_nval(NVal) of 491 | ok -> ok; 492 | {error, NVError} -> 493 | ?LOG_ERROR( 494 | "Invalid nval in fitting spec \"~s\": ~s", 495 | [format_name(Name), NVError]), 496 | throw({badarg, Name, NVError}) 497 | end; 498 | validate_fitting(Other) -> 499 | ?LOG_ERROR( 500 | "Invalid fitting_spec given (expected fitting_spec record):~n~P", 501 | [Other, 3]), 502 | throw({badarg, undefined, "not a fitting_spec record"}). 503 | 504 | %% @doc Validate initialization `Arg' for the given `Module' by calling 505 | %% `Module:validate_arg(Arg)', if it exists. This function assumes 506 | %% that `Module' has already been validate. 507 | -spec validate_argument(module(), term()) -> ok | {error, string()}. 508 | validate_argument(Module, Arg) -> 509 | case lists:member({validate_arg, 1}, Module:module_info(exports)) of 510 | true -> 511 | try 512 | Module:validate_arg(Arg) 513 | catch Type:Error -> 514 | {error, io_lib:format( 515 | "failed to validate module argument: ~p:~p", 516 | [Type, Error])} 517 | end; 518 | false -> 519 | ok %% don't force modules to validate their args 520 | end. 521 | 522 | %% @doc Validate the consistent hashing function. This must be the 523 | %% atom `follow', a static hash as a 160-bit binary, or a valid 524 | %% funtion of arity 1 (see {@link riak_pipe_v:validate_function/3}). 525 | -spec validate_chashfun(follow | riak_pipe_vnode:chashfun()) -> 526 | ok | {error, string()}. 527 | validate_chashfun(follow) -> 528 | ok; 529 | validate_chashfun(Hash) when is_binary(Hash) -> 530 | case byte_size(Hash) of 531 | 20 -> 532 | % consistent hashes are 160 bits 533 | ok; 534 | Other -> 535 | {error, io_lib:format( 536 | "expected a 160-bit binary, found ~p bits", [Other])} 537 | end; 538 | validate_chashfun(HashFun) -> 539 | riak_pipe_v:validate_function("chashfun", 1, HashFun). 540 | 541 | %% @doc Validate the nval parameter. This must either be a positive 542 | %% integer, or a function of arity 1 (that produces a positive 543 | %% integer). The function may be specified anonymously or as a 544 | %% {Mod, Fun} tuple. 545 | -spec validate_nval(term()) -> ok | {error, string()}. 546 | validate_nval(NVal) when is_integer(NVal) -> 547 | if NVal > 0 -> ok; 548 | true -> 549 | {error, io_lib:format( 550 | "expected a positive integer, found ~p", [NVal])} 551 | end; 552 | validate_nval(NVal) when is_function(NVal) -> 553 | riak_pipe_v:validate_function("nval", 1, NVal); 554 | validate_nval({Mod, Fun}) when is_atom(Mod), is_atom(Fun) -> 555 | riak_pipe_v:validate_function("nval", 1, {Mod, Fun}); 556 | validate_nval(NVal) -> 557 | {error, io_lib:format( 558 | "expected a positive integer," 559 | " or a function or {Mod, Fun} of arity 1; not a ~p", 560 | [riak_pipe_v:type_of(NVal)])}. 561 | 562 | %% @doc Coerce a fitting name into a printable string. 563 | -spec format_name(term()) -> iolist(). 564 | format_name(Name) when is_binary(Name) -> 565 | Name; 566 | format_name(Name) when is_list(Name) -> 567 | case is_iolist(Name) of 568 | true -> Name; 569 | false -> io_lib:format("~p", [Name]) 570 | end; 571 | format_name(Name) -> 572 | io_lib:format("~p", [Name]). 573 | 574 | %% @doc Determine if a term is an iolist. 575 | -spec is_iolist(term()) -> boolean(). 576 | is_iolist(Name) when is_list(Name) -> 577 | lists:all(fun is_iolist/1, Name); 578 | is_iolist(Name) when is_binary(Name) -> 579 | true; 580 | is_iolist(Name) when is_integer(Name), Name >= 0, Name =< 255 -> 581 | true; 582 | is_iolist(_) -> 583 | false. 584 | 585 | -ifdef(TEST). 586 | validate_test_() -> 587 | [{"very bad fitting", 588 | %% undefined name because it's not a fitting_spec 589 | ?_assertMatch({badarg, undefined, _Msg}, 590 | (catch riak_pipe_fitting:validate_fitting(x)))}, 591 | {"bad fitting module", 592 | ?_assertMatch({badarg, empty_pass, _Msg}, 593 | (catch riak_pipe_fitting:validate_fitting( 594 | #fitting_spec{name=empty_pass, 595 | module=does_not_exist})))}, 596 | {"bad fitting argument", 597 | ?_assertMatch({badarg, empty_pass, _Msg}, 598 | (catch riak_pipe_fitting:validate_fitting( 599 | #fitting_spec{name=empty_pass, 600 | module=riak_pipe_w_reduce, 601 | arg=bogus_arg})))}, 602 | {"good partfun", 603 | ?_assertEqual(ok, 604 | (catch 605 | riak_pipe_fitting:validate_fitting( 606 | #fitting_spec{name=empty_pass, 607 | module=riak_pipe_w_pass, 608 | chashfun=follow})))}, 609 | {"bad partfun", 610 | ?_assertMatch({badarg, empty_pass, _Msg}, 611 | (catch riak_pipe_fitting:validate_fitting( 612 | #fitting_spec{name=empty_pass, 613 | module=riak_pipe_w_pass, 614 | chashfun=fun(_,_) -> 0 end})))}, 615 | {"format_name binary", 616 | ?_assertEqual(<<"foo">>, 617 | riak_pipe_fitting:format_name(<<"foo">>))}, 618 | {"format_name string", 619 | ?_assertEqual("foo", 620 | riak_pipe_fitting:format_name("foo"))}, 621 | {"format_name atom", 622 | ?_assertEqual("[foo]", 623 | lists:flatten(riak_pipe_fitting:format_name([foo])))} 624 | ]. 625 | 626 | -endif. 627 | --------------------------------------------------------------------------------