├── 2011 ├── riak-core-conflict-resolution │ ├── README.md │ ├── meta │ └── rts │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── progski.access.log.gz │ │ ├── rebar │ │ ├── rebar.config │ │ ├── rel │ │ ├── files │ │ │ ├── app.config │ │ │ ├── erl │ │ │ ├── nodetool │ │ │ ├── rts │ │ │ ├── rts-admin │ │ │ └── vm.args │ │ ├── reltool.config │ │ ├── vars.config │ │ └── vars │ │ │ ├── dev1.config │ │ │ ├── dev2.config │ │ │ └── dev3.config │ │ ├── replay │ │ └── src │ │ ├── rts.app.src │ │ ├── rts.erl │ │ ├── rts.hrl │ │ ├── rts_app.erl │ │ ├── rts_console.erl │ │ ├── rts_entry_vnode.erl │ │ ├── rts_get_fsm.erl │ │ ├── rts_get_fsm_sup.erl │ │ ├── rts_obj.erl │ │ ├── rts_stat_vnode.erl │ │ ├── rts_sup.erl │ │ ├── rts_vnode.erl │ │ ├── rts_wm_entry.erl │ │ ├── rts_write_fsm.erl │ │ └── rts_write_fsm_sup.erl ├── riak-core-first-multinode │ ├── README.md │ ├── meta │ └── mfmn │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── rebar │ │ ├── rebar.config │ │ ├── rel │ │ ├── files │ │ │ ├── app.config │ │ │ ├── erl │ │ │ ├── mfmn │ │ │ ├── mfmn-admin │ │ │ ├── nodetool │ │ │ └── vm.args │ │ ├── reltool.config │ │ ├── vars.config │ │ └── vars │ │ │ ├── dev1.config │ │ │ ├── dev2.config │ │ │ └── dev3.config │ │ └── src │ │ ├── mfmn.app.src │ │ ├── mfmn.erl │ │ ├── mfmn.hrl │ │ ├── mfmn_app.erl │ │ ├── mfmn_console.erl │ │ ├── mfmn_sup.erl │ │ └── mfmn_vnode.erl ├── riak-core-the-coordinator │ ├── README.md │ ├── meta │ └── rts │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── progski.access.log.gz │ │ ├── rebar │ │ ├── rebar.config │ │ ├── rel │ │ ├── files │ │ │ ├── app.config │ │ │ ├── erl │ │ │ ├── nodetool │ │ │ ├── rts │ │ │ ├── rts-admin │ │ │ └── vm.args │ │ ├── reltool.config │ │ ├── vars.config │ │ └── vars │ │ │ ├── dev1.config │ │ │ ├── dev2.config │ │ │ └── dev3.config │ │ ├── replay │ │ └── src │ │ ├── rts.app.src │ │ ├── rts.erl │ │ ├── rts.hrl │ │ ├── rts_app.erl │ │ ├── rts_console.erl │ │ ├── rts_entry_vnode.erl │ │ ├── rts_get_fsm.erl │ │ ├── rts_get_fsm_sup.erl │ │ ├── rts_stat_vnode.erl │ │ ├── rts_sup.erl │ │ ├── rts_vnode.erl │ │ ├── rts_wm_entry.erl │ │ ├── rts_write_fsm.erl │ │ └── rts_write_fsm_sup.erl ├── riak-core-the-vnode │ ├── README.md │ ├── meta │ └── rts │ │ ├── .gitignore │ │ ├── Makefile │ │ ├── progski.access.log.gz │ │ ├── rebar │ │ ├── rebar.config │ │ ├── rel │ │ ├── files │ │ │ ├── app.config │ │ │ ├── erl │ │ │ ├── nodetool │ │ │ ├── rts │ │ │ ├── rts-admin │ │ │ └── vm.args │ │ ├── reltool.config │ │ ├── vars.config │ │ └── vars │ │ │ ├── dev1.config │ │ │ ├── dev2.config │ │ │ └── dev3.config │ │ ├── replay │ │ └── src │ │ ├── rts.app.src │ │ ├── rts.erl │ │ ├── rts.hrl │ │ ├── rts_app.erl │ │ ├── rts_console.erl │ │ ├── rts_entry_vnode.erl │ │ ├── rts_stat_vnode.erl │ │ ├── rts_sup.erl │ │ ├── rts_vnode.erl │ │ └── rts_wm_entry.erl └── riak-search-inline-fields │ ├── .gitignore │ ├── Makefile │ ├── README.md │ ├── RUN_BENCHMARKS.md │ ├── init.sh │ ├── meta │ ├── naive.config │ ├── results │ ├── naive-non-inline.jpg │ ├── naive.png │ ├── scoped-filter.png │ ├── scoped-non-inline.jpg │ └── scoped.png │ ├── schemas │ ├── tweets-schema-inline.txt │ └── tweets-schema.txt │ ├── scoped-filter.config │ ├── scoped.config │ └── upload.py └── README.md /2011/riak-core-conflict-resolution/meta: -------------------------------------------------------------------------------- 1 | published 2011-06-20 -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | .eunit 3 | deps/* 4 | ebin 5 | rel/rts 6 | dev 7 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: deps rel 2 | 3 | all: deps compile 4 | 5 | compile: 6 | ./rebar compile 7 | 8 | deps: 9 | ./rebar get-deps 10 | 11 | clean: 12 | ./rebar clean 13 | 14 | distclean: clean devclean relclean 15 | ./rebar delete-deps 16 | 17 | rel: all 18 | ./rebar generate 19 | 20 | relclean: 21 | rm -rf rel/rts 22 | 23 | devrel: dev1 dev2 dev3 24 | 25 | devclean: 26 | rm -rf dev 27 | 28 | dev1 dev2 dev3: 29 | mkdir -p dev 30 | (cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@.config) -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/progski.access.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-core-conflict-resolution/rts/progski.access.log.gz -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-core-conflict-resolution/rts/rebar -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [debug_info, fail_on_warning]}. 2 | 3 | {sub_dirs, ["rel"]}. 4 | 5 | {deps, [{riak_core, "1.0.*", 6 | {git, "git://github.com/basho/riak_core", {tag,"1.0.0"}}}, 7 | {statebox, "0.2.1", 8 | {git, "git://github.com/mochi/statebox.git", {tag, "v0.2.1"}}} 9 | ]}. 10 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/files/app.config: -------------------------------------------------------------------------------- 1 | [ 2 | %% Riak Core config 3 | {riak_core, [ 4 | %% Default location of ringstate 5 | {ring_state_dir, "{{ring_state_dir}}"}, 6 | 7 | %% http is a list of IP addresses and TCP ports that the Riak 8 | %% HTTP interface will bind. 9 | {http, [ {"{{web_ip}}", {{web_port}} } ]}, 10 | 11 | %% riak_handoff_port is the TCP port that Riak uses for 12 | %% intra-cluster data handoff. 13 | {handoff_port, {{handoff_port}} } 14 | ]}, 15 | 16 | %% SASL config 17 | {sasl, [ 18 | {sasl_error_logger, {file, "log/sasl-error.log"}}, 19 | {errlog_type, error}, 20 | {error_logger_mf_dir, "log/sasl"}, % Log directory 21 | {error_logger_mf_maxbytes, 10485760}, % 10 MB max file size 22 | {error_logger_mf_maxfiles, 5} % 5 files max 23 | ]} 24 | ]. 25 | 26 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/files/erl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## This script replaces the default "erl" in erts-VSN/bin. This is necessary 4 | ## as escript depends on erl and in turn, erl depends on having access to a 5 | ## bootscript (start.boot). Note that this script is ONLY invoked as a side-effect 6 | ## of running escript -- the embedded node bypasses erl and uses erlexec directly 7 | ## (as it should). 8 | ## 9 | ## Note that this script makes the assumption that there is a start_clean.boot 10 | ## file available in $ROOTDIR/release/VSN. 11 | 12 | # Determine the abspath of where this script is executing from. 13 | ERTS_BIN_DIR=$(cd ${0%/*} && pwd) 14 | 15 | # Now determine the root directory -- this script runs from erts-VSN/bin, 16 | # so we simply need to strip off two dirs from the end of the ERTS_BIN_DIR 17 | # path. 18 | ROOTDIR=${ERTS_BIN_DIR%/*/*} 19 | 20 | # Parse out release and erts info 21 | START_ERL=`cat $ROOTDIR/releases/start_erl.data` 22 | ERTS_VSN=${START_ERL% *} 23 | APP_VSN=${START_ERL#* } 24 | 25 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 26 | EMU=beam 27 | PROGNAME=`echo $0 | sed 's/.*\\///'` 28 | CMD="$BINDIR/erlexec" 29 | export EMU 30 | export ROOTDIR 31 | export BINDIR 32 | export PROGNAME 33 | 34 | exec $CMD -boot $ROOTDIR/releases/$APP_VSN/start_clean ${1+"$@"} -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/files/nodetool: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% ex: ft=erlang ts=4 sw=4 et 3 | %% ------------------------------------------------------------------- 4 | %% 5 | %% nodetool: Helper Script for interacting with live nodes 6 | %% 7 | %% ------------------------------------------------------------------- 8 | 9 | main(Args) -> 10 | ok = start_epmd(), 11 | %% Extract the args 12 | {RestArgs, TargetNode} = process_args(Args, [], undefined), 13 | 14 | %% See if the node is currently running -- if it's not, we'll bail 15 | case {net_kernel:hidden_connect_node(TargetNode), net_adm:ping(TargetNode)} of 16 | {true, pong} -> 17 | ok; 18 | {_, pang} -> 19 | io:format("Node ~p not responding to pings.\n", [TargetNode]), 20 | halt(1) 21 | end, 22 | 23 | case RestArgs of 24 | ["ping"] -> 25 | %% If we got this far, the node already responsed to a ping, so just dump 26 | %% a "pong" 27 | io:format("pong\n"); 28 | ["stop"] -> 29 | io:format("~p\n", [rpc:call(TargetNode, init, stop, [], 60000)]); 30 | ["restart"] -> 31 | io:format("~p\n", [rpc:call(TargetNode, init, restart, [], 60000)]); 32 | ["reboot"] -> 33 | io:format("~p\n", [rpc:call(TargetNode, init, reboot, [], 60000)]); 34 | ["rpc", Module, Function | RpcArgs] -> 35 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), 36 | [RpcArgs], 60000) of 37 | ok -> 38 | ok; 39 | {badrpc, Reason} -> 40 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), 41 | halt(1); 42 | _ -> 43 | halt(1) 44 | end; 45 | ["rpcterms", Module, Function, ArgsAsString] -> 46 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), 47 | consult(ArgsAsString), 60000) of 48 | {badrpc, Reason} -> 49 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), 50 | halt(1); 51 | Other -> 52 | io:format("~p\n", [Other]) 53 | end; 54 | Other -> 55 | io:format("Other: ~p\n", [Other]), 56 | io:format("Usage: nodetool {ping|stop|restart|reboot}\n") 57 | end, 58 | net_kernel:stop(). 59 | 60 | process_args([], Acc, TargetNode) -> 61 | {lists:reverse(Acc), TargetNode}; 62 | process_args(["-setcookie", Cookie | Rest], Acc, TargetNode) -> 63 | erlang:set_cookie(node(), list_to_atom(Cookie)), 64 | process_args(Rest, Acc, TargetNode); 65 | process_args(["-name", TargetName | Rest], Acc, _) -> 66 | ThisNode = append_node_suffix(TargetName, "_maint_"), 67 | {ok, _} = net_kernel:start([ThisNode, longnames]), 68 | process_args(Rest, Acc, nodename(TargetName)); 69 | process_args(["-sname", TargetName | Rest], Acc, _) -> 70 | ThisNode = append_node_suffix(TargetName, "_maint_"), 71 | {ok, _} = net_kernel:start([ThisNode, shortnames]), 72 | process_args(Rest, Acc, nodename(TargetName)); 73 | process_args([Arg | Rest], Acc, Opts) -> 74 | process_args(Rest, [Arg | Acc], Opts). 75 | 76 | 77 | start_epmd() -> 78 | [] = os:cmd(epmd_path() ++ " -daemon"), 79 | ok. 80 | 81 | epmd_path() -> 82 | ErtsBinDir = filename:dirname(escript:script_name()), 83 | Name = "epmd", 84 | case os:find_executable(Name, ErtsBinDir) of 85 | false -> 86 | case os:find_executable(Name) of 87 | false -> 88 | io:format("Could not find epmd.~n"), 89 | halt(1); 90 | GlobalEpmd -> 91 | GlobalEpmd 92 | end; 93 | Epmd -> 94 | Epmd 95 | end. 96 | 97 | 98 | nodename(Name) -> 99 | case string:tokens(Name, "@") of 100 | [_Node, _Host] -> 101 | list_to_atom(Name); 102 | [Node] -> 103 | [_, Host] = string:tokens(atom_to_list(node()), "@"), 104 | list_to_atom(lists:concat([Node, "@", Host])) 105 | end. 106 | 107 | append_node_suffix(Name, Suffix) -> 108 | case string:tokens(Name, "@") of 109 | [Node, Host] -> 110 | list_to_atom(lists:concat([Node, Suffix, os:getpid(), "@", Host])); 111 | [Node] -> 112 | list_to_atom(lists:concat([Node, Suffix, os:getpid()])) 113 | end. 114 | 115 | 116 | %% 117 | %% Given a string or binary, parse it into a list of terms, ala file:consult/0 118 | %% 119 | consult(Str) when is_list(Str) -> 120 | consult([], Str, []); 121 | consult(Bin) when is_binary(Bin)-> 122 | consult([], binary_to_list(Bin), []). 123 | 124 | consult(Cont, Str, Acc) -> 125 | case erl_scan:tokens(Cont, Str, 0) of 126 | {done, Result, Remaining} -> 127 | case Result of 128 | {ok, Tokens, _} -> 129 | {ok, Term} = erl_parse:parse_term(Tokens), 130 | consult([], Remaining, [Term | Acc]); 131 | {eof, _Other} -> 132 | lists:reverse(Acc); 133 | {error, Info, _} -> 134 | {error, Info} 135 | end; 136 | {more, Cont1} -> 137 | consult(Cont1, eof, Acc) 138 | end. 139 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/files/rts: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # -*- tab-width:4;indent-tabs-mode:nil -*- 3 | # ex: ts=4 sw=4 et 4 | 5 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd) 6 | 7 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*} 8 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc 9 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log 10 | # Note the trailing slash on $PIPE_DIR/ 11 | PIPE_DIR=/tmp/$RUNNER_BASE_DIR/ 12 | RUNNER_USER= 13 | 14 | # Make sure this script is running as the appropriate user 15 | if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then 16 | exec sudo -u $RUNNER_USER -i $0 $@ 17 | fi 18 | 19 | # Make sure CWD is set to runner base dir 20 | cd $RUNNER_BASE_DIR 21 | 22 | # Make sure log directory exists 23 | mkdir -p $RUNNER_LOG_DIR 24 | 25 | # Extract the target node name from node.args 26 | NAME_ARG=`grep -e '-[s]*name' $RUNNER_ETC_DIR/vm.args` 27 | if [ -z "$NAME_ARG" ]; then 28 | echo "vm.args needs to have either -name or -sname parameter." 29 | exit 1 30 | fi 31 | 32 | # Extract the target cookie 33 | COOKIE_ARG=`grep -e '-setcookie' $RUNNER_ETC_DIR/vm.args` 34 | if [ -z "$COOKIE_ARG" ]; then 35 | echo "vm.args needs to have a -setcookie parameter." 36 | exit 1 37 | fi 38 | 39 | # Identify the script name 40 | SCRIPT=`basename $0` 41 | 42 | # Parse out release and erts info 43 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` 44 | ERTS_VSN=${START_ERL% *} 45 | APP_VSN=${START_ERL#* } 46 | 47 | # Add ERTS bin dir to our path 48 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin 49 | 50 | # Setup command to control the node 51 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" 52 | 53 | # Check the first argument for instructions 54 | case "$1" in 55 | start) 56 | # Make sure there is not already a node running 57 | RES=`$NODETOOL ping` 58 | if [ "$RES" = "pong" ]; then 59 | echo "Node is already running!" 60 | exit 1 61 | fi 62 | HEART_COMMAND="$RUNNER_BASE_DIR/bin/$SCRIPT start" 63 | export HEART_COMMAND 64 | mkdir -p $PIPE_DIR 65 | shift # remove $1 66 | $ERTS_PATH/run_erl -daemon $PIPE_DIR $RUNNER_LOG_DIR "exec $RUNNER_BASE_DIR/bin/$SCRIPT console $@" 2>&1 67 | ;; 68 | 69 | stop) 70 | # Wait for the node to completely stop... 71 | case `uname -s` in 72 | Linux|Darwin|FreeBSD|DragonFly|NetBSD|OpenBSD) 73 | # PID COMMAND 74 | PID=`ps ax -o pid= -o command=|\ 75 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'` 76 | ;; 77 | SunOS) 78 | # PID COMMAND 79 | PID=`ps -ef -o pid= -o args=|\ 80 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'` 81 | ;; 82 | CYGWIN*) 83 | # UID PID PPID TTY STIME COMMAND 84 | PID=`ps -efW|grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $2}'` 85 | ;; 86 | esac 87 | $NODETOOL stop 88 | while `kill -0 $PID 2>/dev/null`; 89 | do 90 | sleep 1 91 | done 92 | ;; 93 | 94 | restart) 95 | ## Restart the VM without exiting the process 96 | $NODETOOL restart 97 | ;; 98 | 99 | reboot) 100 | ## Restart the VM completely (uses heart to restart it) 101 | $NODETOOL reboot 102 | ;; 103 | 104 | ping) 105 | ## See if the VM is alive 106 | $NODETOOL ping 107 | ;; 108 | 109 | attach) 110 | # Make sure a node IS running 111 | RES=`$NODETOOL ping` 112 | if [ "$RES" != "pong" ]; then 113 | echo "Node is not running!" 114 | exit 1 115 | fi 116 | 117 | shift 118 | $ERTS_PATH/to_erl $PIPE_DIR 119 | ;; 120 | 121 | console|console_clean) 122 | # .boot file typically just $SCRIPT (ie, the app name) 123 | # however, for debugging, sometimes start_clean.boot is useful: 124 | case "$1" in 125 | console) BOOTFILE=$SCRIPT ;; 126 | console_clean) BOOTFILE=start_clean ;; 127 | esac 128 | # Setup beam-required vars 129 | ROOTDIR=$RUNNER_BASE_DIR 130 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 131 | EMU=beam 132 | PROGNAME=`echo $0 | sed 's/.*\\///'` 133 | CMD="$BINDIR/erlexec -boot $RUNNER_BASE_DIR/releases/$APP_VSN/$BOOTFILE -embedded -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -- ${1+"$@"}" 134 | export EMU 135 | export ROOTDIR 136 | export BINDIR 137 | export PROGNAME 138 | 139 | # Dump environment info for logging purposes 140 | echo "Exec: $CMD" 141 | echo "Root: $ROOTDIR" 142 | 143 | # Log the startup 144 | logger -t "$SCRIPT[$$]" "Starting up" 145 | 146 | # Start the VM 147 | exec $CMD 148 | ;; 149 | 150 | *) 151 | echo "Usage: $SCRIPT {start|stop|restart|reboot|ping|console|console_clean|attach}" 152 | exit 1 153 | ;; 154 | esac 155 | 156 | exit 0 157 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/files/rts-admin: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd) 4 | RUNNER_SCRIPT=${0##*/} 5 | 6 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*} 7 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc 8 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log 9 | RUNNER_USER= 10 | 11 | # Make sure this script is running as the appropriate user 12 | if [ "$RUNNER_USER" -a "x$LOGNAME" != "x$RUNNER_USER" ]; then 13 | type -p sudo > /dev/null 2>&1 14 | if [ $? -ne 0 ]; then 15 | echo "sudo doesn't appear to be installed and your EUID isn't $RUNNER_USER" 1>&2 16 | exit 1 17 | fi 18 | echo "Attempting to restart script through sudo -u $RUNNER_USER" 19 | exec sudo -u $RUNNER_USER -i $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ 20 | fi 21 | 22 | # Make sure CWD is set to runner base dir 23 | cd $RUNNER_BASE_DIR 24 | 25 | # Extract the target node name from node.args 26 | NAME_ARG=`grep -e '-[s]*name' $RUNNER_ETC_DIR/vm.args` 27 | if [ -z "$NAME_ARG" ]; then 28 | echo "vm.args needs to have either -name or -sname parameter." 29 | exit 1 30 | fi 31 | 32 | # Learn how to specify node name for connection from remote nodes 33 | echo "$NAME_ARG" | grep '^-sname' > /dev/null 2>&1 34 | if [ "X$?" = "X0" ]; then 35 | NAME_PARAM="-sname" 36 | NAME_HOST="" 37 | else 38 | NAME_PARAM="-name" 39 | echo "$NAME_ARG" | grep '@.*' > /dev/null 2>&1 40 | if [ "X$?" = "X0" ]; then 41 | NAME_HOST=`echo "${NAME_ARG}" | sed -e 's/.*(@.*)$//'` 42 | else 43 | NAME_HOST="" 44 | fi 45 | fi 46 | 47 | # Extract the target cookie 48 | COOKIE_ARG=`grep '-setcookie' $RUNNER_ETC_DIR/vm.args` 49 | if [ -z "$COOKIE_ARG" ]; then 50 | echo "vm.args needs to have a -setcookie parameter." 51 | exit 1 52 | fi 53 | 54 | # Identify the script name 55 | SCRIPT=`basename $0` 56 | 57 | # Parse out release and erts info 58 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` 59 | ERTS_VSN=${START_ERL% *} 60 | APP_VSN=${START_ERL#* } 61 | 62 | # Add ERTS bin dir to our path 63 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin 64 | 65 | # Setup command to control the node 66 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" 67 | 68 | # Check the first argument for instructions 69 | case "$1" in 70 | join) 71 | # Make sure the local node IS running 72 | RES=`$NODETOOL ping` 73 | if [ "$RES" != "pong" ]; then 74 | echo "Node is not running!" 75 | exit 1 76 | fi 77 | 78 | shift 79 | 80 | $NODETOOL rpc rts_console join $@ 81 | ;; 82 | 83 | leave) 84 | # Make sure the local node is running 85 | RES=`$NODETOOL ping` 86 | if [ "$RES" != "pong" ]; then 87 | echo "Node is not running!" 88 | exit 1 89 | fi 90 | 91 | shift 92 | $NODETOOL rpc rts__console leave $@ 93 | ;; 94 | 95 | remove) 96 | if [ $# -ne 2 ]; then 97 | echo "Usage: $SCRIPT remove " 98 | exit 1 99 | fi 100 | 101 | RES=`$NODETOOL ping` 102 | if [ "$RES" != "pong" ]; then 103 | echo "Node is not running!" 104 | exit 1 105 | fi 106 | 107 | shift 108 | $NODETOOL rpc rts_console remove $@ 109 | ;; 110 | 111 | ringready) 112 | # Make sure the local node IS running 113 | RES=`$NODETOOL ping` 114 | if [ "$RES" != "pong" ]; then 115 | echo "Node is not running!" 116 | exit 1 117 | fi 118 | shift 119 | 120 | $NODETOOL rpc rts_console ringready $@ 121 | ;; 122 | 123 | *) 124 | echo "Usage: $SCRIPT { join | leave | reip | ringready | remove }" 125 | exit 1 126 | ;; 127 | esac 128 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/files/vm.args: -------------------------------------------------------------------------------- 1 | 2 | ## Name of the node 3 | -name {{node}} 4 | 5 | ## Cookie for distributed erlang 6 | -setcookie rts 7 | 8 | ## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive 9 | ## (Disabled by default..use with caution!) 10 | ##-heart 11 | 12 | ## Enable kernel poll and a few async threads 13 | +K true 14 | +A 5 15 | 16 | ## Increase number of concurrent ports/sockets 17 | -env ERL_MAX_PORTS 4096 18 | 19 | ## Tweak GC to run more often 20 | -env ERL_FULLSWEEP_AFTER 10 21 | 22 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/reltool.config: -------------------------------------------------------------------------------- 1 | {sys, [ 2 | {lib_dirs, ["../../", "../deps/"]}, 3 | {rel, "rts", "1", 4 | [ 5 | kernel, 6 | stdlib, 7 | sasl, 8 | rts 9 | ]}, 10 | {rel, "start_clean", "", 11 | [ 12 | kernel, 13 | stdlib 14 | ]}, 15 | {boot_rel, "rts"}, 16 | {profile, embedded}, 17 | {excl_sys_filters, ["^bin/.*", 18 | "^erts.*/bin/(dialyzer|typer)"]}, 19 | {app, sasl, [{incl_cond, include}]}, 20 | {app, rts, [{incl_cond, include}]} 21 | ]}. 22 | 23 | {target_dir, "rts"}. 24 | 25 | {overlay_vars, "vars.config"}. 26 | 27 | {overlay, [ 28 | {mkdir, "data/ring"}, 29 | {mkdir, "log/sasl"}, 30 | {copy, "files/erl", "{{erts_vsn}}/bin/erl"}, 31 | {copy, "files/nodetool", "{{erts_vsn}}/bin/nodetool"}, 32 | {template, "files/app.config", "etc/app.config"}, 33 | {template, "files/vm.args", "etc/vm.args"}, 34 | {template, "files/rts", "bin/rts"}, 35 | {template, "files/rts-admin", "bin/rts-admin"} 36 | ]}. 37 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/vars.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8888"}. 7 | {handoff_port, "8099"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/vars/dev1.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8881"}. 7 | {handoff_port, "8101"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts1@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/vars/dev2.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8882"}. 7 | {handoff_port, "8102"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts2@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/rel/vars/dev3.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8883"}. 7 | {handoff_port, "8103"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts3@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/replay: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # Usage: 4 | # gunzip access.log.gz | ./replay [--devrel] client 5 | # ./replay [--devrel] client < access.log 6 | 7 | PORTS[0]=8888 8 | MAX=0 9 | 10 | while [ $# -gt 0 ] 11 | do 12 | case $1 in 13 | --devrel | -d ) 14 | PORTS[0]=8881 15 | PORTS[1]=8882 16 | PORTS[2]=8883 17 | MAX=2 18 | ;; 19 | *) 20 | break 21 | ;; 22 | esac 23 | shift 24 | done 25 | 26 | CLIENT=$1 27 | shift 28 | 29 | node=0 30 | while read line 31 | do 32 | curl -X POST \ 33 | -H 'content-type: text/plain' \ 34 | "http://localhost:${PORTS[$node]}/rts/entry/$CLIENT" \ 35 | -d "$line" 36 | 37 | if [ $node -eq $MAX ] 38 | then 39 | node=0 40 | else 41 | ((node++)) 42 | fi 43 | done -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts.app.src: -------------------------------------------------------------------------------- 1 | {application, rts, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | riak_core 10 | ]}, 11 | {mod, { rts_app, []}}, 12 | {env, []} 13 | ]}. -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts.erl: -------------------------------------------------------------------------------- 1 | %% @doc Interface into the Real Time Statistics application. 2 | -module(rts). 3 | -include("rts.hrl"). 4 | -include_lib("riak_core/include/riak_core_vnode.hrl"). 5 | 6 | -export([ 7 | ping/0, 8 | entry/2, 9 | get/2, 10 | get/3, 11 | set/3, 12 | append/3, 13 | incr/2, 14 | incrby/3, 15 | sadd/3, 16 | srem/3 17 | ]). 18 | 19 | -export([get_dbg_preflist/2, 20 | get_dbg_preflist/3, 21 | dbg_op/5, dbg_op/6]). 22 | -define(TIMEOUT, 5000). 23 | 24 | %%%=================================================================== 25 | %%% API 26 | %%%=================================================================== 27 | 28 | % @doc Pings a random vnode to make sure communication is functional 29 | ping() -> 30 | DocIdx = riak_core_util:chash_key({<<"ping">>, term_to_binary(now())}), 31 | PrefList = riak_core_apl:get_primary_apl(DocIdx, 1, rts), 32 | [{IndexNode, _Type}] = PrefList, 33 | riak_core_vnode_master:sync_spawn_command(IndexNode, ping, rts_vnode_master). 34 | 35 | %% @doc Process an entry. 36 | entry(Client, Entry) -> 37 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 38 | term_to_binary(now())}), 39 | PrefList = riak_core_apl:get_apl(DocIdx, 1, rts_entry), 40 | [IdxNode] = PrefList, 41 | rts_entry_vnode:entry(IdxNode, Client, Entry). 42 | 43 | %% @doc Get a stat's value. 44 | get(Client, StatName) -> 45 | get(Client, StatName, []). 46 | 47 | get(Client, StatName, Opts) -> 48 | {ok, ReqID} = rts_get_fsm:get(Client, StatName, Opts), 49 | {ok, Val} = wait_for_reqid(ReqID, ?TIMEOUT), 50 | pretty_print(Val). 51 | 52 | get_dbg_preflist(Client, StatName) -> 53 | [get_dbg_preflist(Client, StatName, N) || N <- lists:seq(1,3)]. 54 | 55 | get_dbg_preflist(Client, StatName, N) -> 56 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 57 | list_to_binary(StatName)}), 58 | Preflist = riak_core_apl:get_apl(DocIdx, ?N, rts_stat), 59 | IdxNode = lists:nth(N, Preflist), 60 | {ok, req_id, _, Val} = 61 | riak_core_vnode_master:sync_command(IdxNode, 62 | {get, req_id, StatName}, 63 | rts_stat_vnode_master), 64 | {IdxNode, Val}. 65 | 66 | %% @doc Set a stat's value, replacing the current value. 67 | set(Client, StatName, Val) -> 68 | do_write(Client, StatName, set, Val). 69 | 70 | %% @doc Append to a stat's value. 71 | append(Client, StatName, Val) -> 72 | do_write(Client, StatName, append, Val). 73 | 74 | %% @doc Increment the stat's value by 1. 75 | incr(Client, StatName) -> 76 | do_write(Client, StatName, incr). 77 | 78 | %% @doc Increment the stat's value by Val. 79 | incrby(Client, StatName, Val) -> 80 | do_write(Client, StatName, incrby, Val). 81 | 82 | %% @doc Add a member to the stat's set. 83 | sadd(Client, StatName, Val) -> 84 | do_write(Client, StatName, sadd, Val). 85 | 86 | %% @doc Remove a member from the stat's set. 87 | srem(Client, StatName, Val) -> 88 | do_write(Client, StatName, srem, Val). 89 | 90 | %% @doc Fake a partitioned `Op' to the given `Nodes' from the given 91 | %% `Coordinator'. That is, this op will act as if the given nodes are 92 | %% partitioned from the rest of the cluster. Let the replies fall on 93 | %% the caller's mailbox. 94 | -spec dbg_op(atom(), node(), [node()], string(), string()) -> ok. 95 | dbg_op(Op, Coordinator, Nodes, Client, StatName) -> 96 | dbg_op(Op, Coordinator, Nodes, Client, StatName, undefined). 97 | 98 | -spec dbg_op(atom(), node(), [node()], string(), string(), term()) -> ok. 99 | dbg_op(Op, Coordinator, Nodes, Client, StatName, Val) -> 100 | ReqID = mk_reqid(), 101 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 102 | list_to_binary(StatName)}), 103 | Preflist = riak_core_apl:get_apl(DocIdx, ?N, rts_stat), 104 | P = fun({_Idx,Node}) -> 105 | lists:member(Node, [Coordinator|Nodes]) 106 | end, 107 | Targets = lists:filter(P, Preflist), 108 | case Val of 109 | undefined -> 110 | rts_stat_vnode:Op(Targets, {ReqID, Coordinator}, StatName); 111 | _ -> 112 | rts_stat_vnode:Op(Targets, {ReqID, Coordinator}, StatName, Val) 113 | end. 114 | 115 | %%%=================================================================== 116 | %%% Internal Functions 117 | %%%=================================================================== 118 | do_write(Client, StatName, Op) -> 119 | {ok, ReqID} = rts_write_fsm:write(Client, StatName, Op), 120 | wait_for_reqid(ReqID, ?TIMEOUT). 121 | 122 | do_write(Client, StatName, Op, Val) -> 123 | {ok, ReqID} = rts_write_fsm:write(Client, StatName, Op, Val), 124 | wait_for_reqid(ReqID, ?TIMEOUT). 125 | 126 | wait_for_reqid(ReqID, Timeout) -> 127 | receive 128 | {ReqID, ok} -> ok; 129 | {ReqID, ok, Val} -> {ok, Val} 130 | after Timeout -> 131 | {error, timeout} 132 | end. 133 | 134 | mk_reqid() -> erlang:phash2(erlang:now()). 135 | 136 | pretty_print(#incr{total=Total}) -> Total; 137 | pretty_print(Val) when element(1, Val) == statebox -> 138 | pretty_print(statebox:value(Val)); 139 | pretty_print(Val) when element(1, Val) == set -> sets:to_list(Val); 140 | pretty_print(Val) -> Val. 141 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts.hrl: -------------------------------------------------------------------------------- 1 | -define(PRINT(Var), io:format("DEBUG: ~p:~p - ~p~n~n ~p~n~n", [?MODULE, ?LINE, ??Var, Var])). 2 | 3 | -define(N, 3). 4 | -define(R, 2). 5 | -define(W, 2). 6 | 7 | -define(DEFAULT_TIMEOUT, 10000). 8 | -define(STATEBOX_EXPIRE, 60000). 9 | 10 | -record(incr, {total :: pos_integer(), 11 | counts :: dict()}). 12 | -type incr() :: #incr{}. 13 | 14 | -type val() :: incr() | statebox:statebox(). 15 | 16 | -record(rts_obj, {val :: val(), 17 | vclock :: vclock:vclock()}). 18 | 19 | -type rts_obj() :: #rts_obj{} | not_found. 20 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_app.erl: -------------------------------------------------------------------------------- 1 | -module(rts_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | case rts_sup:start_link() of 14 | {ok, Pid} -> 15 | ok = riak_core:register_vnode_module(rts_vnode), 16 | ok = riak_core_node_watcher:service_up(rts, self()), 17 | 18 | ok = riak_core:register_vnode_module(rts_entry_vnode), 19 | ok = riak_core_node_watcher:service_up(rts_entry, self()), 20 | 21 | ok = riak_core:register_vnode_module(rts_stat_vnode), 22 | ok = riak_core_node_watcher:service_up(rts_stat, self()), 23 | 24 | EntryRoute = {["rts", "entry", client], rts_wm_entry, []}, 25 | webmachine_router:add_route(EntryRoute), 26 | 27 | {ok, Pid}; 28 | {error, Reason} -> 29 | {error, Reason} 30 | end. 31 | 32 | stop(_State) -> 33 | ok. 34 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_console.erl: -------------------------------------------------------------------------------- 1 | %% @doc Interface for rts-admin commands. 2 | -module(rts_console). 3 | -export([join/1, 4 | leave/1, 5 | remove/1, 6 | ringready/1]). 7 | 8 | join([NodeStr]) -> 9 | try 10 | case riak_core:join(NodeStr) of 11 | ok -> 12 | io:format("Sent join request to ~s\n", [NodeStr]), 13 | ok; 14 | {error, not_reachable} -> 15 | io:format("Node ~s is not reachable!\n", [NodeStr]), 16 | error; 17 | {error, different_ring_sizes} -> 18 | io:format("Failed: ~s has a different ring_creation_size~n", 19 | [NodeStr]), 20 | error 21 | end 22 | catch 23 | Exception:Reason -> 24 | lager:error("Join failed ~p:~p", [Exception, 25 | Reason]), 26 | io:format("Join failed, see log for details~n"), 27 | error 28 | end. 29 | 30 | 31 | leave([]) -> 32 | remove_node(node()). 33 | 34 | 35 | remove([Node]) -> 36 | remove_node(list_to_atom(Node)). 37 | 38 | remove_node(Node) when is_atom(Node) -> 39 | try 40 | case catch(riak_core:remove_from_cluster(Node)) of 41 | {'EXIT', {badarg, [{erlang, hd, [[]]}|_]}} -> 42 | %% This is a workaround because 43 | %% riak_core_gossip:remove_from_cluster doesn't check if 44 | %% the result of subtracting the current node from the 45 | %% cluster member list results in the empty list. When 46 | %% that code gets refactored this can probably go away. 47 | io:format("Leave failed, this node is the only member.~n"), 48 | error; 49 | Res -> 50 | io:format(" ~p\n", [Res]) 51 | end 52 | catch 53 | Exception:Reason -> 54 | lager:error("Leave failed ~p:~p", [Exception, 55 | Reason]), 56 | io:format("Leave failed, see log for details~n"), 57 | error 58 | end. 59 | 60 | 61 | -spec(ringready([]) -> ok | error). 62 | ringready([]) -> 63 | try 64 | case riak_core_status:ringready() of 65 | {ok, Nodes} -> 66 | io:format("TRUE All nodes agree on the ring ~p\n", [Nodes]); 67 | {error, {different_owners, N1, N2}} -> 68 | io:format("FALSE Node ~p and ~p list different partition owners\n", [N1, N2]), 69 | error; 70 | {error, {nodes_down, Down}} -> 71 | io:format("FALSE ~p down. All nodes need to be up to check.\n", [Down]), 72 | error 73 | end 74 | catch 75 | Exception:Reason -> 76 | lager:error("Ringready failed ~p:~p", [Exception, 77 | Reason]), 78 | io:format("Ringready failed, see log for details~n"), 79 | error 80 | end. 81 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_entry_vnode.erl: -------------------------------------------------------------------------------- 1 | %% @doc A vnode to crunch incoming log entries. Attempt to match each 2 | %% log entry against a registry of regexps. If a regexp matches then 3 | %% execute its corresponding trigger function passing it the {Client, 4 | %% Entry, Regexp} as well as the resulting match. The trigger 5 | %% function can then choose to take an action such as update a 6 | %% statistic via the `rts_stat_vnode'. 7 | %% 8 | %% Since this vnode is purely for computation there is no need to 9 | %% worry about handoff. 10 | -module(rts_entry_vnode). 11 | -behaviour(riak_core_vnode). 12 | -include("rts.hrl"). 13 | 14 | -export([start_vnode/1, 15 | init/1, 16 | terminate/2, 17 | handle_command/3, 18 | is_empty/1, 19 | delete/1, 20 | handle_handoff_command/3, 21 | handoff_starting/2, 22 | handoff_cancelled/1, 23 | handoff_finished/2, 24 | handle_handoff_data/2, 25 | encode_handoff_item/2, 26 | handle_coverage/4, 27 | handle_exit/3]). 28 | 29 | %% match handlers 30 | -export([ 31 | combined_lf/2 32 | ]). 33 | 34 | %% API 35 | -export([entry/3]). 36 | 37 | -record(state, { 38 | partition, 39 | reg %% registry [regexp => fun] 40 | }). 41 | 42 | -define(MASTER, rts_entry_vnode_master). 43 | -define(COMBINED_LF, "(\\d+\\.\\d+\\.\\d+\\.\\d+) (.*) (.*) (\\[.*\\]) \"(.*)\" (\\d+) (.*) \"(.*)\" \"(.*)\""). 44 | 45 | %%%=================================================================== 46 | %%% API 47 | %%%=================================================================== 48 | 49 | start_vnode(I) -> 50 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 51 | 52 | entry(IdxNode, Client, Entry) -> 53 | riak_core_vnode_master:command(IdxNode, 54 | {entry, Client, Entry}, 55 | ?MASTER). 56 | 57 | %%%=================================================================== 58 | %%% Callbacks 59 | %%%=================================================================== 60 | 61 | init([Partition]) -> 62 | Reg = [ 63 | {?COMBINED_LF, fun ?MODULE:combined_lf/2} 64 | ], 65 | {ok, #state { partition=Partition, reg=Reg }}. 66 | 67 | handle_command({entry, Client, Entry}, _Sender, #state{reg=Reg}=State) -> 68 | io:format("~p~n", [{entry, State#state.partition}]), 69 | lists:foreach(match(Client, Entry), Reg), 70 | {noreply, State}. 71 | 72 | handle_handoff_command(_Message, _Sender, State) -> 73 | {noreply, State}. 74 | 75 | handoff_starting(_TargetNode, _State) -> 76 | {true, _State}. 77 | 78 | handoff_cancelled(State) -> 79 | {ok, State}. 80 | 81 | handoff_finished(_TargetNode, State) -> 82 | {ok, State}. 83 | 84 | handle_handoff_data(_Data, State) -> 85 | {reply, ok, State}. 86 | 87 | encode_handoff_item(_ObjectName, _ObjectValue) -> 88 | <<>>. 89 | 90 | is_empty(State) -> 91 | {true, State}. 92 | 93 | delete(State) -> 94 | {ok, State}. 95 | 96 | handle_coverage(_Req, _KeySpaces, _Sender, State) -> 97 | {stop, not_implemented, State}. 98 | 99 | handle_exit(_Pid, _Reason, _State) -> 100 | {noreply, _State}. 101 | 102 | terminate(_Reason, _State) -> 103 | ok. 104 | 105 | %%%=================================================================== 106 | %%% Internal Functions 107 | %%%=================================================================== 108 | 109 | match(Client, Entry) -> 110 | fun({Regexp, Fun}) -> 111 | case re:run(Entry, Regexp, [{capture, all, list}]) of 112 | nomatch -> ignore; 113 | {match, Match} -> Fun({Client, Entry, Regexp}, Match) 114 | end 115 | end. 116 | 117 | %%%=================================================================== 118 | %%% Match Handlers 119 | %%%=================================================================== 120 | 121 | combined_lf({Client, _Entry, _Regexp}, [_Entry, _Host, _, _User, _Time, Req, Code, BodySize, _Referer, Agent]) -> 122 | rts:sadd(Client, "agents", Agent), 123 | rts:incrby(Client, "total_sent", list_to_integer(BodySize)), 124 | [Method, _Resource, _Protocol] = string:tokens(Req, " "), 125 | rts:incr(Client, Method), 126 | case Code of 127 | [$2, _, _] -> 128 | rts:incr(Client, "200"); 129 | [$3, _, _] -> 130 | rts:incr(Client, "300"); 131 | [$4, _, _] -> 132 | rts:incr(Client, "400"); 133 | [$5, _, _] -> 134 | rts:incr(Client, "500") 135 | end, 136 | rts:incr(Client, "total_reqs"). 137 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_get_fsm.erl: -------------------------------------------------------------------------------- 1 | %% @doc The coordinator for stat get operations. The key here is to 2 | %% generate the preflist just like in wrtie_fsm and then query each 3 | %% replica and wait until a quorum is met. 4 | -module(rts_get_fsm). 5 | -behavior(gen_fsm). 6 | -include("rts.hrl"). 7 | 8 | %% API 9 | -export([start_link/5, get/3]). 10 | 11 | %% Callbacks 12 | -export([init/1, code_change/4, handle_event/3, handle_info/3, 13 | handle_sync_event/4, terminate/3]). 14 | 15 | %% States 16 | -export([prepare/2, execute/2, waiting/2, wait_for_n/2, finalize/2]). 17 | 18 | -export([reconcile/1]). 19 | 20 | -record(state, {req_id, 21 | from, 22 | client, 23 | stat_name, 24 | preflist, 25 | num_r=0, 26 | replies=[], 27 | r=?R, 28 | timeout=?DEFAULT_TIMEOUT}). 29 | 30 | -type idx_node() :: {integer(), node()}. 31 | -type vnode_reply() :: {idx_node(), rts_obj() | not_found}. 32 | 33 | %%%=================================================================== 34 | %%% API 35 | %%%=================================================================== 36 | 37 | start_link(ReqID, From, Client, StatName, Opts) -> 38 | gen_fsm:start_link(?MODULE, [ReqID, From, Client, StatName, Opts], []). 39 | 40 | get(Client, StatName, Opts) -> 41 | ReqID = mk_reqid(), 42 | rts_get_fsm_sup:start_get_fsm([ReqID, self(), Client, StatName, Opts]), 43 | {ok, ReqID}. 44 | 45 | %%%=================================================================== 46 | %%% States 47 | %%%=================================================================== 48 | 49 | %% Intiailize state data. 50 | init([ReqId, From, Client, StatName, Opts]) -> 51 | R = proplists:get_value(r, Opts, ?R), 52 | SD = #state{req_id=ReqId, 53 | from=From, 54 | client=Client, 55 | stat_name=StatName, 56 | r=R}, 57 | {ok, prepare, SD, 0}. 58 | 59 | %% @doc Calculate the Preflist. 60 | prepare(timeout, SD0=#state{client=Client, 61 | stat_name=StatName}) -> 62 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 63 | list_to_binary(StatName)}), 64 | Prelist = riak_core_apl:get_apl(DocIdx, ?N, rts_stat), 65 | SD = SD0#state{preflist=Prelist}, 66 | {next_state, execute, SD, 0}. 67 | 68 | %% @doc Execute the get reqs. 69 | execute(timeout, SD0=#state{req_id=ReqId, 70 | stat_name=StatName, 71 | preflist=Prelist}) -> 72 | rts_stat_vnode:get(Prelist, ReqId, StatName), 73 | {next_state, waiting, SD0}. 74 | 75 | %% @doc Wait for R replies and then respond to From (original client 76 | %% that called `rts:get/2'). 77 | waiting({ok, ReqID, IdxNode, Obj}, 78 | SD0=#state{from=From, num_r=NumR0, replies=Replies0, 79 | r=R, timeout=Timeout}) -> 80 | NumR = NumR0 + 1, 81 | Replies = [{IdxNode, Obj}|Replies0], 82 | SD = SD0#state{num_r=NumR,replies=Replies}, 83 | 84 | if 85 | NumR =:= R -> 86 | Reply = rts_obj:val(merge(Replies)), 87 | From ! {ReqID, ok, Reply}, 88 | 89 | if NumR =:= ?N -> {next_state, finalize, SD, 0}; 90 | true -> {next_state, wait_for_n, SD, Timeout} 91 | end; 92 | true -> {next_state, waiting, SD} 93 | end. 94 | 95 | wait_for_n({ok, _ReqID, IdxNode, Obj}, 96 | SD0=#state{num_r=?N - 1, replies=Replies0, stat_name=_StatName}) -> 97 | Replies = [{IdxNode, Obj}|Replies0], 98 | {next_state, finalize, SD0#state{num_r=?N, replies=Replies}, 0}; 99 | 100 | wait_for_n({ok, _ReqID, IdxNode, Obj}, 101 | SD0=#state{num_r=NumR0, replies=Replies0, 102 | stat_name=_StatName, timeout=Timeout}) -> 103 | NumR = NumR0 + 1, 104 | Replies = [{IdxNode, Obj}|Replies0], 105 | {next_state, wait_for_n, SD0#state{num_r=NumR, replies=Replies}, Timeout}; 106 | 107 | %% TODO partial repair? 108 | wait_for_n(timeout, SD) -> 109 | {stop, timeout, SD}. 110 | 111 | finalize(timeout, SD=#state{replies=Replies, stat_name=StatName}) -> 112 | MObj = merge(Replies), 113 | case needs_repair(MObj, Replies) of 114 | true -> 115 | repair(StatName, MObj, Replies), 116 | {stop, normal, SD}; 117 | false -> 118 | {stop, normal, SD} 119 | end. 120 | 121 | handle_info(_Info, _StateName, StateData) -> 122 | {stop,badmsg,StateData}. 123 | 124 | handle_event(_Event, _StateName, StateData) -> 125 | {stop,badmsg,StateData}. 126 | 127 | handle_sync_event(_Event, _From, _StateName, StateData) -> 128 | {stop,badmsg,StateData}. 129 | 130 | code_change(_OldVsn, StateName, State, _Extra) -> {ok, StateName, State}. 131 | 132 | terminate(_Reason, _SN, _SD) -> 133 | ok. 134 | 135 | %%%=================================================================== 136 | %%% Internal Functions 137 | %%%=================================================================== 138 | 139 | %% @impure 140 | mk_reqid() -> erlang:phash2(erlang:now()). 141 | 142 | %% @pure 143 | %% 144 | %% @doc Given a list of `Replies' return the merged value. 145 | -spec merge([vnode_reply()]) -> rts_obj() | not_found. 146 | merge(Replies) -> 147 | Objs = [Obj || {_,Obj} <- Replies], 148 | rts_obj:merge(Objs). 149 | 150 | %% @pure 151 | %% 152 | %% @doc Reconcile conflicts among conflicting values. 153 | -spec reconcile([A::any()]) -> A::any(). 154 | reconcile([#incr{}|_]=Vals) -> 155 | Get = fun(K, L) -> proplists:get_value(K, L, 0) end, 156 | Counts = [dict:to_list(V#incr.counts) || V <- Vals], 157 | Nodes = unique(lists:flatten([[Node || {Node,_} <- C] || C <- Counts])), 158 | MaxCounts = [{Node, lists:max([Get(Node, C) || C <- Counts])} 159 | || Node <- Nodes], 160 | Total = lists:sum([lists:max([Get(Node, C) || C <- Counts]) 161 | || Node <- Nodes]), 162 | #incr{total=Total, counts=dict:from_list(MaxCounts)}; 163 | 164 | reconcile([V|_]=Vals) when element(1, V) == statebox -> statebox:merge(Vals). 165 | 166 | 167 | %% @pure 168 | %% 169 | %% @doc Given the merged object `MObj' and a list of `Replies' 170 | %% determine if repair is needed. 171 | -spec needs_repair(any(), [vnode_reply()]) -> boolean(). 172 | needs_repair(MObj, Replies) -> 173 | Objs = [Obj || {_,Obj} <- Replies], 174 | lists:any(different(MObj), Objs). 175 | 176 | %% @pure 177 | different(A) -> fun(B) -> not rts_obj:equal(A,B) end. 178 | 179 | %% @impure 180 | %% 181 | %% @doc Repair any vnodes that do not have the correct object. 182 | -spec repair(string(), rts_obj(), [vnode_reply()]) -> io. 183 | repair(_, _, []) -> io; 184 | 185 | repair(StatName, MObj, [{IdxNode,Obj}|T]) -> 186 | case rts_obj:equal(MObj, Obj) of 187 | true -> repair(StatName, MObj, T); 188 | false -> 189 | rts_stat_vnode:repair(IdxNode, StatName, MObj), 190 | repair(StatName, MObj, T) 191 | end. 192 | 193 | %% pure 194 | %% 195 | %% @doc Given a list return the set of unique values. 196 | -spec unique([A::any()]) -> [A::any()]. 197 | unique(L) -> 198 | sets:to_list(sets:from_list(L)). 199 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_get_fsm_sup.erl: -------------------------------------------------------------------------------- 1 | %% @doc Supervise the rts_get FSM. 2 | -module(rts_get_fsm_sup). 3 | -behavior(supervisor). 4 | 5 | -export([start_get_fsm/1, 6 | start_link/0]). 7 | -export([init/1]). 8 | 9 | start_get_fsm(Args) -> 10 | supervisor:start_child(?MODULE, Args). 11 | 12 | start_link() -> 13 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 14 | 15 | init([]) -> 16 | GetFsm = {undefined, 17 | {rts_get_fsm, start_link, []}, 18 | temporary, 5000, worker, [rts_get_fsm]}, 19 | {ok, {{simple_one_for_one, 10, 10}, [GetFsm]}}. 20 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_obj.erl: -------------------------------------------------------------------------------- 1 | %% @doc A suite of functions that operate on the algebraic data type 2 | %% `rts_obj'. 3 | %% 4 | %% TODO Possibly move type/record defs in there and use accessor funs 5 | %% and opaque types. 6 | -module(rts_obj). 7 | -export([ancestors/1, children/1, equal/1, equal/2, merge/1, unique/1, 8 | update/3]). 9 | -export([val/1, vclock/1]). 10 | 11 | -include("rts.hrl"). 12 | 13 | %% @pure 14 | %% 15 | %% @doc Given a list of `rts_obj()' return a list of all the 16 | %% ancestors. Ancestors are objects that all the other objects in the 17 | %% list have descent from. 18 | -spec ancestors([rts_obj()]) -> [rts_obj()]. 19 | ancestors(Objs0) -> 20 | Objs = [O || O <- Objs0, O /= not_found], 21 | As = [[O2 || O2 <- Objs, 22 | ancestor(O2#rts_obj.vclock, 23 | O1#rts_obj.vclock)] || O1 <- Objs], 24 | unique(lists:flatten(As)). 25 | 26 | %% @pure 27 | %% 28 | %% @doc Predicate to determine if `Va' is ancestor of `Vb'. 29 | -spec ancestor(vclock:vclock(), vclock:vclock()) -> boolean(). 30 | ancestor(Va, Vb) -> 31 | vclock:descends(Vb, Va) andalso (vclock:descends(Va, Vb) == false). 32 | 33 | %% @pure 34 | %% 35 | %% @doc Given a list of `rts_obj()' return a list of the children 36 | %% objects. Children are the descendants of all others objects. 37 | children(Objs) -> 38 | unique(Objs) -- ancestors(Objs). 39 | 40 | %% @pure 41 | %% 42 | %% @doc Predeicate to determine if `ObjA' and `ObjB' are equal. 43 | -spec equal(ObjA::rts_obj(), ObjB::rts_obj()) -> boolean(). 44 | equal(#rts_obj{vclock=A}, #rts_obj{vclock=B}) -> vclock:equal(A,B); 45 | equal(not_found, not_found) -> true; 46 | equal(_, _) -> false. 47 | 48 | %% @pure 49 | %% 50 | %% @doc Closure around `equal/2' for use with HOFs (damn verbose 51 | %% Erlang). 52 | -spec equal(ObjA::rts_obj()) -> fun((ObjB::rts_obj()) -> boolean()). 53 | equal(ObjA) -> 54 | fun(ObjB) -> equal(ObjA, ObjB) end. 55 | 56 | %% @pure 57 | %% 58 | %% @doc Merge the list of `Objs', calling the appropriate reconcile 59 | %% fun if there are siblings. 60 | -spec merge([rts_obj()]) -> rts_obj(). 61 | merge([not_found|_]=Objs) -> 62 | P = fun(X) -> X == not_found end, 63 | case lists:all(P, Objs) of 64 | true -> not_found; 65 | false -> merge(lists:dropwhile(P, Objs)) 66 | end; 67 | 68 | merge([#rts_obj{}|_]=Objs) -> 69 | case rts_obj:children(Objs) of 70 | [] -> not_found; 71 | [Child] -> Child; 72 | Chldrn -> 73 | Val = rts_get_fsm:reconcile(lists:map(fun val/1, Chldrn)), 74 | MergedVC = vclock:merge(lists:map(fun vclock/1, Chldrn)), 75 | #rts_obj{val=Val, vclock=MergedVC} 76 | end. 77 | 78 | %% @pure 79 | %% 80 | %% @doc Given a list of `Objs' return the list of uniques. 81 | -spec unique([rts_obj()]) -> [rts_obj()]. 82 | unique(Objs) -> 83 | F = fun(not_found, Acc) -> 84 | Acc; 85 | (Obj, Acc) -> 86 | case lists:any(equal(Obj), Acc) of 87 | true -> Acc; 88 | false -> [Obj|Acc] 89 | end 90 | end, 91 | lists:foldl(F, [], Objs). 92 | 93 | %% @pure 94 | %% 95 | %% @doc Given a `Val' update the `Obj'. The `Updater' is the name of 96 | %% the entity performing the update. 97 | -spec update(val(), node(), rts_obj()) -> rts_obj(). 98 | update(Val, Updater, #rts_obj{vclock=VClock0}=Obj0) -> 99 | VClock = vclock:increment(Updater, VClock0), 100 | Obj0#rts_obj{val=Val, vclock=VClock}. 101 | 102 | -spec val(rts_obj()) -> any(). 103 | val(#rts_obj{val=Val}) -> Val; 104 | val(not_found) -> not_found. 105 | 106 | %% @pure 107 | %% 108 | %% @doc Given a vclock type `Obj' retrieve the vclock. 109 | -spec vclock(rts_obj()) -> vclock:vclock(). 110 | vclock(#rts_obj{vclock=VC}) -> VC. 111 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_stat_vnode.erl: -------------------------------------------------------------------------------- 1 | %% @doc A vnode to handle get & put commands for stat data. The vnode 2 | %% requests will be hashed on Client and StatName and will use a 3 | %% coordinator to enforce N/R/W values. 4 | -module(rts_stat_vnode). 5 | -behaviour(riak_core_vnode). 6 | -include("rts.hrl"). 7 | -include_lib("riak_core/include/riak_core_vnode.hrl"). 8 | -export([start_vnode/1, 9 | init/1, 10 | terminate/2, 11 | handle_command/3, 12 | is_empty/1, 13 | delete/1, 14 | handle_handoff_command/3, 15 | handoff_starting/2, 16 | handoff_cancelled/1, 17 | handoff_finished/2, 18 | handle_handoff_data/2, 19 | encode_handoff_item/2, 20 | handle_coverage/4, 21 | handle_exit/3]). 22 | 23 | -export([ 24 | get/3, 25 | set/4, 26 | repair/3, 27 | incr/3, 28 | incrby/4, 29 | append/4, 30 | sadd/4, 31 | srem/4 32 | ]). 33 | 34 | -record(state, {partition, stats, node}). 35 | 36 | -define(MASTER, rts_stat_vnode_master). 37 | -define(sync(PrefList, Command, Master), 38 | riak_core_vnode_master:sync_command(PrefList, Command, Master)). 39 | 40 | %%%=================================================================== 41 | %%% API 42 | %%%=================================================================== 43 | 44 | start_vnode(I) -> 45 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 46 | 47 | get(Preflist, ReqID, StatName) -> 48 | riak_core_vnode_master:command(Preflist, 49 | {get, ReqID, StatName}, 50 | {fsm, undefined, self()}, 51 | ?MASTER). 52 | 53 | set(Preflist, Identity, StatName, Val) -> 54 | riak_core_vnode_master:command(Preflist, 55 | {set, Identity, StatName, Val}, 56 | ?MASTER). 57 | 58 | %% @doc Attempt to repair -- fire and forget. 59 | repair(IdxNode, StatName, Obj) -> 60 | riak_core_vnode_master:command(IdxNode, 61 | {repair, undefined, StatName, Obj}, 62 | ignore, 63 | ?MASTER). 64 | 65 | %% TODO: I have to look at the Sender stuff more closely again 66 | incr(Preflist, Identity, StatName) -> 67 | riak_core_vnode_master:command(Preflist, 68 | {incrby, Identity, StatName, 1}, 69 | {fsm, undefined, self()}, 70 | ?MASTER). 71 | 72 | incrby(Preflist, Identity, StatName, Val) -> 73 | riak_core_vnode_master:command(Preflist, 74 | {incrby, Identity, StatName, Val}, 75 | {fsm, undefined, self()}, 76 | ?MASTER). 77 | 78 | append(Preflist, Identity, StatName, Val) -> 79 | riak_core_vnode_master:command(Preflist, 80 | {append, Identity, StatName, Val}, 81 | {fsm, undefined, self()}, 82 | ?MASTER). 83 | 84 | sadd(Preflist, Identity, StatName, Val) -> 85 | riak_core_vnode_master:command(Preflist, 86 | {sadd, Identity, StatName, Val}, 87 | {fsm, undefined, self()}, 88 | ?MASTER). 89 | 90 | srem(Preflist, Identity, StatName, Val) -> 91 | riak_core_vnode_master:command(Preflist, 92 | {srem, Identity, StatName, Val}, 93 | {fsm, undefined, self()}, 94 | ?MASTER). 95 | 96 | %%%=================================================================== 97 | %%% Callbacks 98 | %%%=================================================================== 99 | 100 | init([Partition]) -> 101 | {ok, #state { partition=Partition, stats=dict:new(), node=node() }}. 102 | 103 | handle_command({get, ReqID, StatName}, _Sender, 104 | #state{stats=Stats, partition=Partition, node=Node}=State) -> 105 | Reply = 106 | case dict:find(StatName, Stats) of 107 | error -> 108 | not_found; 109 | {ok, Found} -> 110 | Found 111 | end, 112 | {reply, {ok, ReqID, {Partition,Node}, Reply}, State}; 113 | 114 | handle_command({set, {ReqID, _}, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 115 | Stats = dict:store(StatName, Val, Stats0), 116 | {reply, {ok, ReqID}, State#state{stats=Stats}}; 117 | 118 | handle_command({repair, undefined, StatName, Obj}, _Sender, #state{stats=Stats0}=State) -> 119 | error_logger:error_msg("repair performed ~p~n", [Obj]), 120 | Stats = dict:store(StatName, Obj, Stats0), 121 | {noreply, State#state{stats=Stats}}; 122 | 123 | handle_command({incrby, {ReqID, Coordinator}, StatName, IncrBy}, _Sender, #state{stats=Stats0}=State) -> 124 | Obj = 125 | case dict:find(StatName, Stats0) of 126 | {ok, #rts_obj{val=#incr{total=T0, counts=C0}}=O} -> 127 | T = T0 + IncrBy, 128 | C = dict:update_counter(Coordinator, IncrBy, C0), 129 | Val = #incr{total=T, counts=C}, 130 | rts_obj:update(Val, Coordinator, O); 131 | error -> 132 | Val = #incr{total=IncrBy, 133 | counts=dict:from_list([{Coordinator, IncrBy}])}, 134 | VC0 = vclock:fresh(), 135 | VC = vclock:increment(Coordinator, VC0), 136 | #rts_obj{val=Val, vclock=VC} 137 | end, 138 | Stats = dict:store(StatName, Obj, Stats0), 139 | {reply, {ok, ReqID}, State#state{stats=Stats}}; 140 | 141 | handle_command({append, {ReqID, _}, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 142 | Stats = try dict:append(StatName, Val, Stats0) 143 | catch _:_ -> dict:store(StatName, [Val], Stats0) 144 | end, 145 | {reply, {ok, ReqID}, State#state{stats=Stats}}; 146 | 147 | handle_command({sadd, {ReqID, Coordinator}, StatName, Val}, 148 | _Sender, #state{stats=Stats0}=State) -> 149 | SB = 150 | case dict:find(StatName, Stats0) of 151 | {ok, #rts_obj{val=SB0}=O} -> 152 | SB1 = statebox:modify({sets, add_element, [Val]}, SB0), 153 | SB2 = statebox:expire(?STATEBOX_EXPIRE, SB1), 154 | rts_obj:update(SB2, Coordinator, O); 155 | error -> 156 | SB0 = statebox:new(fun sets:new/0), 157 | SB1 = statebox:modify({sets, add_element, [Val]}, SB0), 158 | VC0 = vclock:fresh(), 159 | VC = vclock:increment(Coordinator, VC0), 160 | #rts_obj{val=SB1, vclock=VC} 161 | end, 162 | Stats = dict:store(StatName, SB, Stats0), 163 | {reply, {ok, ReqID}, State#state{stats=Stats}}; 164 | 165 | handle_command({srem, {ReqID, Coordinator}, StatName, Val}, 166 | _Sender, #state{stats=Stats0}=State) -> 167 | SB = 168 | case dict:find(StatName, Stats0) of 169 | {ok, #rts_obj{val=SB0}=O} -> 170 | SB1 = statebox:modify({sets, del_element, [Val]}, SB0), 171 | SB2 = statebox:expire(?STATEBOX_EXPIRE, SB1), 172 | rts_obj:update(SB2, Coordinator, O); 173 | error -> 174 | SB0 = statebox:new(fun sets:new/0), 175 | SB1 = statebox:modify({sets, del_element, [Val]}, SB0), 176 | VC0 = vclock:fresh(), 177 | VC = vclock:increment(Coordinator, VC0), 178 | #rts_obj{val=SB1, vclock=VC} 179 | end, 180 | Stats = dict:store(StatName, SB, Stats0), 181 | {reply, {ok, ReqID}, State#state{stats=Stats}}. 182 | 183 | handle_handoff_command(?FOLD_REQ{foldfun=Fun, acc0=Acc0}, _Sender, State) -> 184 | Acc = dict:fold(Fun, Acc0, State#state.stats), 185 | {reply, Acc, State}. 186 | 187 | handoff_starting(_TargetNode, _State) -> 188 | {true, _State}. 189 | 190 | handoff_cancelled(State) -> 191 | {ok, State}. 192 | 193 | handoff_finished(_TargetNode, State) -> 194 | {ok, State}. 195 | 196 | handle_handoff_data(Data, #state{stats=Stats0}=State) -> 197 | {StatName, HObj} = binary_to_term(Data), 198 | MObj = 199 | case dict:find(StatName, Stats0) of 200 | {ok, Obj} -> rts_obj:merge([Obj,HObj]); 201 | error -> HObj 202 | end, 203 | Stats = dict:store(StatName, MObj, Stats0), 204 | {reply, ok, State#state{stats=Stats}}. 205 | 206 | encode_handoff_item(StatName, Val) -> 207 | term_to_binary({StatName,Val}). 208 | 209 | is_empty(State) -> 210 | case dict:size(State#state.stats) of 211 | 0 -> {true, State}; 212 | _ -> {false, State} 213 | end. 214 | 215 | delete(State) -> 216 | {ok, State}. 217 | 218 | handle_coverage(_Req, _KeySpaces, _Sender, State) -> 219 | {stop, not_implemented, State}. 220 | 221 | handle_exit(_Pid, _Reason, _State) -> 222 | {noreply, _State}. 223 | 224 | terminate(_Reason, _State) -> 225 | ok. 226 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_sup.erl: -------------------------------------------------------------------------------- 1 | -module(rts_sup). 2 | 3 | -behaviour(supervisor). 4 | 5 | %% API 6 | -export([start_link/0]). 7 | 8 | %% Supervisor callbacks 9 | -export([init/1]). 10 | 11 | %% =================================================================== 12 | %% API functions 13 | %% =================================================================== 14 | 15 | start_link() -> 16 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 17 | 18 | %% =================================================================== 19 | %% Supervisor callbacks 20 | %% =================================================================== 21 | 22 | init(_Args) -> 23 | VMaster = { rts_vnode_master, 24 | {riak_core_vnode_master, start_link, [rts_vnode]}, 25 | permanent, 5000, worker, [riak_core_vnode_master]}, 26 | 27 | Entry = {rts_entry_vnode_master, 28 | {riak_core_vnode_master, start_link, [rts_entry_vnode]}, 29 | permanent, 5000, worker, [riak_core_vnode_master]}, 30 | 31 | Stat = {rts_stat_vnode_master, 32 | {riak_core_vnode_master, start_link, [rts_stat_vnode]}, 33 | permanent, 5000, worker, [riak_core_vnode_master]}, 34 | 35 | WriteFSMs = {rts_write_fsm_sup, 36 | {rts_write_fsm_sup, start_link, []}, 37 | permanent, infinity, supervisor, [rts_write_fsm_sup]}, 38 | 39 | GetFSMs = {rts_get_fsm_sup, 40 | {rts_get_fsm_sup, start_link, []}, 41 | permanent, infinity, supervisor, [rts_get_fsm_sup]}, 42 | 43 | {ok, 44 | {{one_for_one, 5, 10}, 45 | [VMaster, Entry, Stat, WriteFSMs, GetFSMs]}}. 46 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_vnode.erl: -------------------------------------------------------------------------------- 1 | -module(rts_vnode). 2 | -behaviour(riak_core_vnode). 3 | -include("rts.hrl"). 4 | 5 | -export([start_vnode/1, 6 | init/1, 7 | terminate/2, 8 | handle_command/3, 9 | is_empty/1, 10 | delete/1, 11 | handle_handoff_command/3, 12 | handoff_starting/2, 13 | handoff_cancelled/1, 14 | handoff_finished/2, 15 | handle_handoff_data/2, 16 | encode_handoff_item/2, 17 | handle_coverage/4, 18 | handle_exit/3]). 19 | 20 | -record(state, {partition}). 21 | 22 | %% API 23 | start_vnode(I) -> 24 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 25 | 26 | init([Partition]) -> 27 | {ok, #state { partition=Partition }}. 28 | 29 | % Sample command: respond to a ping 30 | handle_command(ping, _Sender, State) -> 31 | {reply, {pong, State#state.partition}, State}; 32 | handle_command(Message, _Sender, State) -> 33 | ?PRINT({unhandled_command, Message}), 34 | {noreply, State}. 35 | 36 | handle_handoff_command(_Message, _Sender, State) -> 37 | {noreply, State}. 38 | 39 | handoff_starting(_TargetNode, State) -> 40 | {true, State}. 41 | 42 | handoff_cancelled(State) -> 43 | {ok, State}. 44 | 45 | handoff_finished(_TargetNode, State) -> 46 | {ok, State}. 47 | 48 | handle_handoff_data(_Data, State) -> 49 | {reply, ok, State}. 50 | 51 | encode_handoff_item(_ObjectName, _ObjectValue) -> 52 | <<>>. 53 | 54 | is_empty(State) -> 55 | {true, State}. 56 | 57 | delete(State) -> 58 | {ok, State}. 59 | 60 | handle_coverage(_Req, _KeySpaces, _Sender, State) -> 61 | {stop, not_implemented, State}. 62 | 63 | handle_exit(_Pid, _Reason, _State) -> 64 | {noreply, _State}. 65 | 66 | terminate(_Reason, _State) -> 67 | ok. 68 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_wm_entry.erl: -------------------------------------------------------------------------------- 1 | %% @doc Webmachine resource to handle incoming log entries. 2 | %% 3 | %% == PUT /rts/entry/Client == 4 | %% 5 | %% Write an entry for the given Client. 6 | %% 7 | %% Content-type must be 'text/plain' and the body should be _one_ 8 | %% line. 9 | -module(rts_wm_entry). 10 | -export([ 11 | init/1, 12 | allowed_methods/2, 13 | process_post/2 14 | ]). 15 | 16 | -include_lib("webmachine/include/webmachine.hrl"). 17 | 18 | init(_Props) -> 19 | {ok, none}. 20 | 21 | allowed_methods(_RD, _Ctx) -> 22 | {['POST'], _RD, _Ctx}. 23 | 24 | process_post(RD, _Ctx) -> 25 | case wrq:get_req_header("content-type", RD) of 26 | "text/plain" -> 27 | Client = wrq:path_info(client, RD), 28 | Entry = binary_to_list(wrq:req_body(RD)), 29 | ok = rts:entry(Client, Entry), 30 | {true, RD, _Ctx}; 31 | _ -> 32 | {{halt, 415}, RD, _Ctx} 33 | end. 34 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_write_fsm.erl: -------------------------------------------------------------------------------- 1 | %% @doc The coordinator for stat write opeartions. This example will 2 | %% show how to properly replicate your data in Riak Core by making use 3 | %% of the _preflist_. 4 | -module(rts_write_fsm). 5 | -behavior(gen_fsm). 6 | -include("rts.hrl"). 7 | 8 | %% API 9 | -export([start_link/5, start_link/6, write/3, write/4]). 10 | 11 | %% Callbacks 12 | -export([init/1, code_change/4, handle_event/3, handle_info/3, 13 | handle_sync_event/4, terminate/3]). 14 | 15 | %% States 16 | -export([prepare/2, execute/2, waiting/2]). 17 | 18 | %% req_id: The request id so the caller can verify the response. 19 | %% 20 | %% sender: The pid of the sender so a reply can be made. 21 | %% 22 | %% client: The external entity that wrote the log entry that produced 23 | %% this stat. 24 | %% 25 | %% state_name: The name of the statistic. 26 | %% 27 | %% op: The stat op, one of [set, incr, incr_by, append, sadd] 28 | %% 29 | %% prelist: The preflist for the given {Client, StatName} pair. 30 | %% 31 | %% num_w: The number of successful write replies. 32 | -record(state, {req_id :: pos_integer(), 33 | cordinator :: node(), 34 | from :: pid(), 35 | client :: string(), 36 | stat_name :: string(), 37 | op :: atom(), 38 | val = undefined :: term() | undefined, 39 | preflist :: riak_core_apl:preflist2(), 40 | num_w = 0 :: non_neg_integer()}). 41 | 42 | %%%=================================================================== 43 | %%% API 44 | %%%=================================================================== 45 | 46 | start_link(ReqID, From, Client, StatName, Op) -> 47 | start_link(ReqID, From, Client, StatName, Op, undefined). 48 | 49 | start_link(ReqID, From, Client, StatName, Op, Val) -> 50 | gen_fsm:start_link(?MODULE, [ReqID, From, Client, StatName, Op, Val], []). 51 | 52 | write(Client, StatName, Op) -> 53 | write(Client, StatName, Op, undefined). 54 | 55 | write(Client, StatName, Op, Val) -> 56 | ReqID = mk_reqid(), 57 | rts_write_fsm_sup:start_write_fsm([ReqID, self(), Client, StatName, Op, Val]), 58 | {ok, ReqID}. 59 | 60 | %%%=================================================================== 61 | %%% States 62 | %%%=================================================================== 63 | 64 | %% @doc Initialize the state data. 65 | init([ReqID, From, Client, StatName, Op, Val]) -> 66 | SD = #state{req_id=ReqID, 67 | cordinator=node(), 68 | from=From, 69 | client=Client, 70 | stat_name=StatName, 71 | op=Op, 72 | val=Val}, 73 | {ok, prepare, SD, 0}. 74 | 75 | %% @doc Prepare the write by calculating the _preference list_. 76 | prepare(timeout, SD0=#state{client=Client, 77 | stat_name=StatName}) -> 78 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 79 | list_to_binary(StatName)}), 80 | Preflist = riak_core_apl:get_apl(DocIdx, ?N, rts_stat), 81 | SD = SD0#state{preflist=Preflist}, 82 | {next_state, execute, SD, 0}. 83 | 84 | %% @doc Execute the write request and then go into waiting state to 85 | %% verify it has meets consistency requirements. 86 | execute(timeout, SD0=#state{req_id=ReqID, 87 | cordinator=Cordinator, 88 | stat_name=StatName, 89 | op=Op, 90 | val=Val, 91 | preflist=Preflist}) -> 92 | case Val of 93 | undefined -> 94 | rts_stat_vnode:Op(Preflist, {ReqID, Cordinator}, StatName); 95 | _ -> 96 | rts_stat_vnode:Op(Preflist, {ReqID, Cordinator}, StatName, Val) 97 | end, 98 | {next_state, waiting, SD0}. 99 | 100 | %% @doc Wait for W write reqs to respond. 101 | waiting({ok, ReqID}, SD0=#state{from=From, num_w=NumW0}) -> 102 | NumW = NumW0 + 1, 103 | SD = SD0#state{num_w=NumW}, 104 | if 105 | NumW =:= ?W -> 106 | From ! {ReqID, ok}, 107 | {stop, normal, SD}; 108 | true -> {next_state, waiting, SD} 109 | end. 110 | 111 | handle_info(_Info, _StateName, StateData) -> 112 | {stop,badmsg,StateData}. 113 | 114 | handle_event(_Event, _StateName, StateData) -> 115 | {stop,badmsg,StateData}. 116 | 117 | handle_sync_event(_Event, _From, _StateName, StateData) -> 118 | {stop,badmsg,StateData}. 119 | 120 | code_change(_OldVsn, StateName, State, _Extra) -> {ok, StateName, State}. 121 | 122 | terminate(_Reason, _SN, _SD) -> 123 | ok. 124 | 125 | %%%=================================================================== 126 | %%% Internal Functions 127 | %%%=================================================================== 128 | 129 | mk_reqid() -> erlang:phash2(erlang:now()). 130 | -------------------------------------------------------------------------------- /2011/riak-core-conflict-resolution/rts/src/rts_write_fsm_sup.erl: -------------------------------------------------------------------------------- 1 | %% @doc Supervise the rts_write FSM. 2 | -module(rts_write_fsm_sup). 3 | -behavior(supervisor). 4 | 5 | -export([start_write_fsm/1, 6 | start_link/0]). 7 | -export([init/1]). 8 | 9 | start_write_fsm(Args) -> 10 | supervisor:start_child(?MODULE, Args). 11 | 12 | start_link() -> 13 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 14 | 15 | init([]) -> 16 | WriteFsm = {undefined, 17 | {rts_write_fsm, start_link, []}, 18 | temporary, 5000, worker, [rts_write_fsm]}, 19 | {ok, {{simple_one_for_one, 10, 10}, [WriteFsm]}}. 20 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/README.md: -------------------------------------------------------------------------------- 1 | Riak Core, First Multinode 2 | ========== 3 | 4 | Questions about Riak Core seem to be occurring with more frequency on the riak-users mailing list. Recently [jnewlend](https://github.com/jnewland) threw up a [Riak Core template](https://github.com/websterclay/rebar_riak_core) to be used with [Rebar](https://github.com/basho/rebar). I thought this was excellent work and ran with it. 5 | 6 | While I liked what Mr. Newland had started I felt it could use a little more meat on its bones. He created a template to create a Riak Core application but left it as an exercise to the user to do the rest such as build a "multinode" capable release. With knowledge of Erlang, Rebar, and peeking at the Riak source code this is certainly doable but it's a real PITA when you just want to get something up and running. I've started [my own fork](https://github.com/rzezeski/rebar_riak_core/tree/multinode) that allows one to standup a _multinode_ release fairly easily. 7 | 8 | Um, Ryan...What The Hell is Multinode? 9 | ---------- 10 | 11 | So I kind of just threw the term _multinode_ at you like it's colloquial language or something; let me try to explain. In general, when someone says _node_ I generally think of a physical machine...you know a big black box with sharp corners, loud fans, and maybe a blue neon light if you built it yourself, or maybe it's a sleek, quiet, beautiful piece of solid aluminum if you bought it from a company named after a piece of fruit (no laughing, I own both). I digress. Back to multinode. 12 | 13 | In Erlang, a node refers to an instance of an Erlang Virtual Machine. You can have multiple Virtual Machines running at once on the same machine just like you can run more than one instance of a JVM on your local machine. Now, if you've read anything about Erlang you've probably heard it's really great at this whole "distributed" thing. No? Well go RTFM and come back, please. In fact, Erlang has a _distributed_ mode, which is to say that it is started in such a way that it can be joined with other nodes on the same physical machine or on that abused mac mini you have sitting in the corner (if you're counting that's three computers I own...not enough). 14 | 15 | Like some famous person said "one is the loneliest number" so Erlang nodes tend to form gangs. Not the gangs like in West Side Story where they walk in-step snapping their fingers, but still cool nonetheless. When more than one node makes up a cluster I call it a _multinode_. That's all I mean. I guess it might have been easier to just call it a "cluster," but multinode sounds way cooler; and besides I'm too lazy to change it now. 16 | 17 | TLDR, Start Here 18 | ---------- 19 | 20 | Enough of me attempting to be witty (is it possible to be witty in written form?). The first thing you need to do is grab the templates and drop them in `~/.rebar/templates`. 21 | 22 | git clone git://github.com/rzezeski/rebar_riak_core.git 23 | mkdir -p ~/.rebar/templates 24 | cp rebar_riak_core/* ~/.rebar/templates 25 | ls ~/.rebar/templates 26 | 27 | Without executing the above steps rebar won't be able to find the _riak\_core\_multinode_ template (can I configure rebar to look in other dirs for templates?). If you don't have rebar don't worry, we'll get to that. Next, make a directory to house your new multinode capable Riak Core application. 28 | 29 | mkdir mfmn 30 | cd mfmn 31 | 32 | The mfmn stands for "My First MultiNode". Original, I know. Next, you need rebar. 33 | 34 | wget http://cloud.github.com/downloads/basho/rebar/rebar && chmod u+x rebar 35 | 36 | Great, now create your new multinode app. 37 | 38 | ./rebar create template=riak_core_multinode appid=mfmn nodeid=mfmn 39 | 40 | Here is an excerpt of what the output should look like: 41 | 42 | ==> mfmn (create) 43 | Writing rel/reltool.config 44 | Writing rel/files/erl 45 | Writing rel/files/nodetool 46 | Writing rel/files/mfmn 47 | ... 48 | 49 | Let me break that last command down a bit: 50 | 51 | * `create`: This is a rebar _command_, and it tells rebar that you want to create a skeleton from the template and vars which you pass as argument. 52 | 53 | * `template`: The name of the template used to build the skeleton. 54 | 55 | * `appid` & `nodeid`: These are a little harder to explain without going into Erlang OTP, applications and releases. The multinode template not only assumes you want to build a local application (and by local I mean in the same repo) but also that you want to build a release around this application. If you have no clue what that means then just ignore this for now and specify the same value for both. 56 | 57 | Congrats, you have the start of a Riak Core application that can be deployed to multiple nodes and joined together to form a _multinode_ cluster. Lets start 'er up. 58 | 59 | make rel 60 | ./rel/mfmn/bin/mfmn console 61 | 62 | At this point you have a single node of _mfmn_ running, now for the moment you've been waiting for! 63 | 64 | mfmn:ping(). 65 | 66 | You should see something like `{pong,365375409332725729550921208179070754913983135744}` returned but the really large number probably won't be the same. Try running that command a few times back to back. Notice how the number changes each time? That's a _partition_ in Riak Core parlance, and the fact that it's changing means your ping request is getting distributed across the various _vnodes_ in the one node cluster. "Wait..wha? What the hell is a vnode Ryan?" Glad you asked. 67 | 68 | This example has pretty much sidestepped explaining anything about Riak Core. Yes, when you call `mfmn:ping()` you are using a Riak Core based application but that doesn't tell you much. While trying to write a post that explains Riak Core in more depth I realized that it might be easier if I could establish some sort of common ground. I created this Rebar template for that reason. In my next post I plan to discuss the idea of a _vnode_ and how to implement one. In the meantime, I'll discuss some more things you can do with the toy project you just created. 69 | 70 | devrel 71 | ---------- 72 | 73 | Above I showed you how to start a single node with the console at the foreground, but this isn't typically how other Riak Core based applications like Riak are tested. Instead, there is something called a _devrel_ that allows one to easily stand up a local 3-node cluster. Lucky for you I included this in the multinode template. 74 | 75 | make devrel 76 | 77 | This command is very similar to `rel` but instead creates 3 separate instances under the `dev/` dir; check it out. 78 | 79 | ls dev 80 | 81 | Now, lets start all the nodes. 82 | 83 | for d in dev/dev*; do $d/bin/mfmn start; done 84 | 85 | There's no output so let's make sure they are indeed up. 86 | 87 | for d in dev/dev*; do $d/bin/mfmn ping; done 88 | 89 | You should see three `pong` replies. Now, at this point, it is worth saying that you have three **INDIVIDUAL** mfmn nodes running. They are **NOT** aware of each other yet and if this were a Riak KV cluster you could store data in one node and the other node will have no idea it's there. In order to form the cluster you have to _join_ the nodes. Don't worry, you only have to join them once. If a node, or the entire cluster, goes down it will remember the other nodes it's joined to. 90 | 91 | for d in dev/dev{2,3}; do $d/bin/mfmn-admin join mfmn1@127.0.0.1; done 92 | 93 | Finally, to make sure they really all agree on the shape of the cluster you can ask if the _ring_ is "ready." 94 | 95 | To verify you have a 3 node cluster you can run the `member_status` command. 96 | 97 | ./dev/dev1/bin/mfmn-admin member_status 98 | 99 | Now you can attach to the shell of one of the nodes and run the `ping` command. 100 | 101 | ./dev/dev2/bin/mfmn attach 102 | mfmn:ping(). 103 | 104 | To stop all the nodes just transpose `start` for `stop`. 105 | 106 | for d in dev/dev*; do $d/bin/mfmn stop; done 107 | 108 | This is a "Working Blog" 109 | ---------- 110 | 111 | It's my intention that this will be a _working_ blog. That means that I will strive to place the emphasis on working code over prose. If you look you'll see a `mfmn` dir that contains everything needed for turnkey operation. That is, you should be able to go in there and just type `make` to get things going. This means you can skip the rigmarole of downloading the templates, copying them, downloading rebar, etc. It also means you can eff up your local copy without worry cause there will always be a pristine one here; as long as I never delete my repo :) 112 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/meta: -------------------------------------------------------------------------------- 1 | published 2011-04-04 -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | .eunit 3 | deps/* 4 | ebin 5 | rel/mfmn 6 | dev 7 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/Makefile: -------------------------------------------------------------------------------- 1 | all: deps compile 2 | 3 | compile: 4 | ./rebar compile 5 | 6 | deps: 7 | ./rebar get-deps 8 | 9 | clean: 10 | ./rebar clean 11 | 12 | distclean: clean devclean relclean 13 | ./rebar delete-deps 14 | 15 | rel: all 16 | ./rebar generate 17 | 18 | devrel: dev1 dev2 dev3 19 | 20 | dev1 dev2 dev3: 21 | mkdir -p dev 22 | (cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@.config) -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-core-first-multinode/mfmn/rebar -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [debug_info, fail_on_warning]}. 2 | 3 | {sub_dirs, ["rel"]}. 4 | 5 | {deps, [{riak_core, "1.0.*", 6 | {git, "git://github.com/basho/riak_core", {tag,"1.0.0"}}} 7 | ]}. 8 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/files/app.config: -------------------------------------------------------------------------------- 1 | [ 2 | %% Riak Core config 3 | {riak_core, [ 4 | %% Default location of ringstate 5 | {ring_state_dir, "{{ring_state_dir}}"}, 6 | 7 | %% http is a list of IP addresses and TCP ports that the Riak 8 | %% HTTP interface will bind. 9 | {http, [ {"{{web_ip}}", {{web_port}} } ]}, 10 | 11 | %% https is a list of IP addresses and TCP ports that the Riak 12 | %% HTTPS interface will bind. 13 | %{https, [{ "{{web_ip}}", {{web_port}} }]}, 14 | 15 | %% default cert and key locations for https can be overridden 16 | %% with the ssl config variable 17 | %{ssl, [ 18 | % {certfile, "etc/cert.pem"}, 19 | % {keyfile, "etc/key.pem"} 20 | % ]}, 21 | 22 | %% riak_handoff_port is the TCP port that Riak uses for 23 | %% intra-cluster data handoff. 24 | {handoff_port, {{handoff_port}} } 25 | ]}, 26 | 27 | %% SASL config 28 | {sasl, [ 29 | {sasl_error_logger, {file, "log/sasl-error.log"}}, 30 | {errlog_type, error}, 31 | {error_logger_mf_dir, "log/sasl"}, % Log directory 32 | {error_logger_mf_maxbytes, 10485760}, % 10 MB max file size 33 | {error_logger_mf_maxfiles, 5} % 5 files max 34 | ]} 35 | ]. 36 | 37 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/files/erl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## This script replaces the default "erl" in erts-VSN/bin. This is necessary 4 | ## as escript depends on erl and in turn, erl depends on having access to a 5 | ## bootscript (start.boot). Note that this script is ONLY invoked as a side-effect 6 | ## of running escript -- the embedded node bypasses erl and uses erlexec directly 7 | ## (as it should). 8 | ## 9 | ## Note that this script makes the assumption that there is a start_clean.boot 10 | ## file available in $ROOTDIR/release/VSN. 11 | 12 | # Determine the abspath of where this script is executing from. 13 | ERTS_BIN_DIR=$(cd ${0%/*} && pwd) 14 | 15 | # Now determine the root directory -- this script runs from erts-VSN/bin, 16 | # so we simply need to strip off two dirs from the end of the ERTS_BIN_DIR 17 | # path. 18 | ROOTDIR=${ERTS_BIN_DIR%/*/*} 19 | 20 | # Parse out release and erts info 21 | START_ERL=`cat $ROOTDIR/releases/start_erl.data` 22 | ERTS_VSN=${START_ERL% *} 23 | APP_VSN=${START_ERL#* } 24 | 25 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 26 | EMU=beam 27 | PROGNAME=`echo $0 | sed 's/.*\\///'` 28 | CMD="$BINDIR/erlexec" 29 | export EMU 30 | export ROOTDIR 31 | export BINDIR 32 | export PROGNAME 33 | 34 | exec $CMD -boot $ROOTDIR/releases/$APP_VSN/start_clean ${1+"$@"} -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/files/mfmn: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # -*- tab-width:4;indent-tabs-mode:nil -*- 3 | # ex: ts=4 sw=4 et 4 | 5 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd) 6 | 7 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*} 8 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc 9 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log 10 | # Note the trailing slash on $PIPE_DIR/ 11 | PIPE_DIR=/tmp/$RUNNER_BASE_DIR/ 12 | RUNNER_USER= 13 | 14 | # Make sure this script is running as the appropriate user 15 | if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then 16 | exec sudo -u $RUNNER_USER -i $0 $@ 17 | fi 18 | 19 | # Make sure CWD is set to runner base dir 20 | cd $RUNNER_BASE_DIR 21 | 22 | # Make sure log directory exists 23 | mkdir -p $RUNNER_LOG_DIR 24 | 25 | # Extract the target node name from node.args 26 | NAME_ARG=`grep -e '-[s]*name' $RUNNER_ETC_DIR/vm.args` 27 | if [ -z "$NAME_ARG" ]; then 28 | echo "vm.args needs to have either -name or -sname parameter." 29 | exit 1 30 | fi 31 | 32 | # Extract the target cookie 33 | COOKIE_ARG=`grep -e '-setcookie' $RUNNER_ETC_DIR/vm.args` 34 | if [ -z "$COOKIE_ARG" ]; then 35 | echo "vm.args needs to have a -setcookie parameter." 36 | exit 1 37 | fi 38 | 39 | # Identify the script name 40 | SCRIPT=`basename $0` 41 | 42 | # Parse out release and erts info 43 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` 44 | ERTS_VSN=${START_ERL% *} 45 | APP_VSN=${START_ERL#* } 46 | 47 | # Add ERTS bin dir to our path 48 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin 49 | 50 | # Setup command to control the node 51 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" 52 | 53 | # Check the first argument for instructions 54 | case "$1" in 55 | start) 56 | # Make sure there is not already a node running 57 | RES=`$NODETOOL ping` 58 | if [ "$RES" = "pong" ]; then 59 | echo "Node is already running!" 60 | exit 1 61 | fi 62 | HEART_COMMAND="$RUNNER_BASE_DIR/bin/$SCRIPT start" 63 | export HEART_COMMAND 64 | mkdir -p $PIPE_DIR 65 | shift # remove $1 66 | $ERTS_PATH/run_erl -daemon $PIPE_DIR $RUNNER_LOG_DIR "exec $RUNNER_BASE_DIR/bin/$SCRIPT console $@" 2>&1 67 | ;; 68 | 69 | stop) 70 | # Wait for the node to completely stop... 71 | case `uname -s` in 72 | Linux|Darwin|FreeBSD|DragonFly|NetBSD|OpenBSD) 73 | # PID COMMAND 74 | PID=`ps ax -o pid= -o command=|\ 75 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'` 76 | ;; 77 | SunOS) 78 | # PID COMMAND 79 | PID=`ps -ef -o pid= -o args=|\ 80 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'` 81 | ;; 82 | CYGWIN*) 83 | # UID PID PPID TTY STIME COMMAND 84 | PID=`ps -efW|grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $2}'` 85 | ;; 86 | esac 87 | $NODETOOL stop 88 | while `kill -0 $PID 2>/dev/null`; 89 | do 90 | sleep 1 91 | done 92 | ;; 93 | 94 | restart) 95 | ## Restart the VM without exiting the process 96 | $NODETOOL restart 97 | ;; 98 | 99 | reboot) 100 | ## Restart the VM completely (uses heart to restart it) 101 | $NODETOOL reboot 102 | ;; 103 | 104 | ping) 105 | ## See if the VM is alive 106 | $NODETOOL ping 107 | ;; 108 | 109 | attach) 110 | # Make sure a node IS running 111 | RES=`$NODETOOL ping` 112 | if [ "$RES" != "pong" ]; then 113 | echo "Node is not running!" 114 | exit 1 115 | fi 116 | 117 | shift 118 | $ERTS_PATH/to_erl $PIPE_DIR 119 | ;; 120 | 121 | console|console_clean) 122 | # .boot file typically just $SCRIPT (ie, the app name) 123 | # however, for debugging, sometimes start_clean.boot is useful: 124 | case "$1" in 125 | console) BOOTFILE=$SCRIPT ;; 126 | console_clean) BOOTFILE=start_clean ;; 127 | esac 128 | # Setup beam-required vars 129 | ROOTDIR=$RUNNER_BASE_DIR 130 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 131 | EMU=beam 132 | PROGNAME=`echo $0 | sed 's/.*\\///'` 133 | CMD="$BINDIR/erlexec -boot $RUNNER_BASE_DIR/releases/$APP_VSN/$BOOTFILE -embedded -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -- ${1+"$@"}" 134 | export EMU 135 | export ROOTDIR 136 | export BINDIR 137 | export PROGNAME 138 | 139 | # Dump environment info for logging purposes 140 | echo "Exec: $CMD" 141 | echo "Root: $ROOTDIR" 142 | 143 | # Log the startup 144 | logger -t "$SCRIPT[$$]" "Starting up" 145 | 146 | # Start the VM 147 | exec $CMD 148 | ;; 149 | 150 | *) 151 | echo "Usage: $SCRIPT {start|stop|restart|reboot|ping|console|console_clean|attach}" 152 | exit 1 153 | ;; 154 | esac 155 | 156 | exit 0 157 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/files/mfmn-admin: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd) 4 | RUNNER_SCRIPT=${0##*/} 5 | 6 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*} 7 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc 8 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log 9 | RUNNER_USER= 10 | 11 | # Make sure this script is running as the appropriate user 12 | if [ "$RUNNER_USER" -a "x$LOGNAME" != "x$RUNNER_USER" ]; then 13 | type -p sudo > /dev/null 2>&1 14 | if [ $? -ne 0 ]; then 15 | echo "sudo doesn't appear to be installed and your EUID isn't $RUNNER_USER" 1>&2 16 | exit 1 17 | fi 18 | echo "Attempting to restart script through sudo -u $RUNNER_USER" 19 | exec sudo -u $RUNNER_USER -i $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ 20 | fi 21 | 22 | # Make sure CWD is set to runner base dir 23 | cd $RUNNER_BASE_DIR 24 | 25 | # Extract the target node name from node.args 26 | NAME_ARG=`grep -e '-[s]*name' $RUNNER_ETC_DIR/vm.args` 27 | if [ -z "$NAME_ARG" ]; then 28 | echo "vm.args needs to have either -name or -sname parameter." 29 | exit 1 30 | fi 31 | 32 | # Learn how to specify node name for connection from remote nodes 33 | echo "$NAME_ARG" | grep '^-sname' > /dev/null 2>&1 34 | if [ "X$?" = "X0" ]; then 35 | NAME_PARAM="-sname" 36 | NAME_HOST="" 37 | else 38 | NAME_PARAM="-name" 39 | echo "$NAME_ARG" | grep '@.*' > /dev/null 2>&1 40 | if [ "X$?" = "X0" ]; then 41 | NAME_HOST=`echo "${NAME_ARG}" | sed -e 's/.*(@.*)$//'` 42 | else 43 | NAME_HOST="" 44 | fi 45 | fi 46 | 47 | # Extract the target cookie 48 | COOKIE_ARG=`grep '-setcookie' $RUNNER_ETC_DIR/vm.args` 49 | if [ -z "$COOKIE_ARG" ]; then 50 | echo "vm.args needs to have a -setcookie parameter." 51 | exit 1 52 | fi 53 | 54 | # Identify the script name 55 | SCRIPT=`basename $0` 56 | 57 | # Parse out release and erts info 58 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` 59 | ERTS_VSN=${START_ERL% *} 60 | APP_VSN=${START_ERL#* } 61 | 62 | # Add ERTS bin dir to our path 63 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin 64 | 65 | # Setup command to control the node 66 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" 67 | 68 | # Check the first argument for instructions 69 | case "$1" in 70 | join) 71 | # Make sure the local node IS running 72 | RES=`$NODETOOL ping` 73 | if [ "$RES" != "pong" ]; then 74 | echo "Node is not running!" 75 | exit 1 76 | fi 77 | 78 | shift 79 | 80 | $NODETOOL rpc mfmn_console join $@ 81 | ;; 82 | 83 | leave) 84 | # Make sure the local node is running 85 | RES=`$NODETOOL ping` 86 | if [ "$RES" != "pong" ]; then 87 | echo "Node is not running!" 88 | exit 1 89 | fi 90 | 91 | shift 92 | $NODETOOL rpc mfmn__console leave $@ 93 | ;; 94 | 95 | remove) 96 | if [ $# -ne 2 ]; then 97 | echo "Usage: $SCRIPT remove " 98 | exit 1 99 | fi 100 | 101 | RES=`$NODETOOL ping` 102 | if [ "$RES" != "pong" ]; then 103 | echo "Node is not running!" 104 | exit 1 105 | fi 106 | 107 | shift 108 | $NODETOOL rpc mfmn_console remove $@ 109 | ;; 110 | 111 | ringready) 112 | # Make sure the local node IS running 113 | RES=`$NODETOOL ping` 114 | if [ "$RES" != "pong" ]; then 115 | echo "Node is not running!" 116 | exit 1 117 | fi 118 | shift 119 | 120 | $NODETOOL rpc mfmn_console ringready $@ 121 | ;; 122 | 123 | *) 124 | echo "Usage: $SCRIPT { join | leave | reip | ringready | remove }" 125 | exit 1 126 | ;; 127 | esac 128 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/files/nodetool: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% ex: ft=erlang ts=4 sw=4 et 3 | %% ------------------------------------------------------------------- 4 | %% 5 | %% nodetool: Helper Script for interacting with live nodes 6 | %% 7 | %% ------------------------------------------------------------------- 8 | 9 | main(Args) -> 10 | ok = start_epmd(), 11 | %% Extract the args 12 | {RestArgs, TargetNode} = process_args(Args, [], undefined), 13 | 14 | %% See if the node is currently running -- if it's not, we'll bail 15 | case {net_kernel:hidden_connect_node(TargetNode), net_adm:ping(TargetNode)} of 16 | {true, pong} -> 17 | ok; 18 | {_, pang} -> 19 | io:format("Node ~p not responding to pings.\n", [TargetNode]), 20 | halt(1) 21 | end, 22 | 23 | case RestArgs of 24 | ["ping"] -> 25 | %% If we got this far, the node already responsed to a ping, so just dump 26 | %% a "pong" 27 | io:format("pong\n"); 28 | ["stop"] -> 29 | io:format("~p\n", [rpc:call(TargetNode, init, stop, [], 60000)]); 30 | ["restart"] -> 31 | io:format("~p\n", [rpc:call(TargetNode, init, restart, [], 60000)]); 32 | ["reboot"] -> 33 | io:format("~p\n", [rpc:call(TargetNode, init, reboot, [], 60000)]); 34 | ["rpc", Module, Function | RpcArgs] -> 35 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), 36 | [RpcArgs], 60000) of 37 | ok -> 38 | ok; 39 | {badrpc, Reason} -> 40 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), 41 | halt(1); 42 | _ -> 43 | halt(1) 44 | end; 45 | ["rpcterms", Module, Function, ArgsAsString] -> 46 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), 47 | consult(ArgsAsString), 60000) of 48 | {badrpc, Reason} -> 49 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), 50 | halt(1); 51 | Other -> 52 | io:format("~p\n", [Other]) 53 | end; 54 | Other -> 55 | io:format("Other: ~p\n", [Other]), 56 | io:format("Usage: nodetool {ping|stop|restart|reboot}\n") 57 | end, 58 | net_kernel:stop(). 59 | 60 | process_args([], Acc, TargetNode) -> 61 | {lists:reverse(Acc), TargetNode}; 62 | process_args(["-setcookie", Cookie | Rest], Acc, TargetNode) -> 63 | erlang:set_cookie(node(), list_to_atom(Cookie)), 64 | process_args(Rest, Acc, TargetNode); 65 | process_args(["-name", TargetName | Rest], Acc, _) -> 66 | ThisNode = append_node_suffix(TargetName, "_maint_"), 67 | {ok, _} = net_kernel:start([ThisNode, longnames]), 68 | process_args(Rest, Acc, nodename(TargetName)); 69 | process_args(["-sname", TargetName | Rest], Acc, _) -> 70 | ThisNode = append_node_suffix(TargetName, "_maint_"), 71 | {ok, _} = net_kernel:start([ThisNode, shortnames]), 72 | process_args(Rest, Acc, nodename(TargetName)); 73 | process_args([Arg | Rest], Acc, Opts) -> 74 | process_args(Rest, [Arg | Acc], Opts). 75 | 76 | 77 | start_epmd() -> 78 | [] = os:cmd(epmd_path() ++ " -daemon"), 79 | ok. 80 | 81 | epmd_path() -> 82 | ErtsBinDir = filename:dirname(escript:script_name()), 83 | Name = "epmd", 84 | case os:find_executable(Name, ErtsBinDir) of 85 | false -> 86 | case os:find_executable(Name) of 87 | false -> 88 | io:format("Could not find epmd.~n"), 89 | halt(1); 90 | GlobalEpmd -> 91 | GlobalEpmd 92 | end; 93 | Epmd -> 94 | Epmd 95 | end. 96 | 97 | 98 | nodename(Name) -> 99 | case string:tokens(Name, "@") of 100 | [_Node, _Host] -> 101 | list_to_atom(Name); 102 | [Node] -> 103 | [_, Host] = string:tokens(atom_to_list(node()), "@"), 104 | list_to_atom(lists:concat([Node, "@", Host])) 105 | end. 106 | 107 | append_node_suffix(Name, Suffix) -> 108 | case string:tokens(Name, "@") of 109 | [Node, Host] -> 110 | list_to_atom(lists:concat([Node, Suffix, os:getpid(), "@", Host])); 111 | [Node] -> 112 | list_to_atom(lists:concat([Node, Suffix, os:getpid()])) 113 | end. 114 | 115 | 116 | %% 117 | %% Given a string or binary, parse it into a list of terms, ala file:consult/0 118 | %% 119 | consult(Str) when is_list(Str) -> 120 | consult([], Str, []); 121 | consult(Bin) when is_binary(Bin)-> 122 | consult([], binary_to_list(Bin), []). 123 | 124 | consult(Cont, Str, Acc) -> 125 | case erl_scan:tokens(Cont, Str, 0) of 126 | {done, Result, Remaining} -> 127 | case Result of 128 | {ok, Tokens, _} -> 129 | {ok, Term} = erl_parse:parse_term(Tokens), 130 | consult([], Remaining, [Term | Acc]); 131 | {eof, _Other} -> 132 | lists:reverse(Acc); 133 | {error, Info, _} -> 134 | {error, Info} 135 | end; 136 | {more, Cont1} -> 137 | consult(Cont1, eof, Acc) 138 | end. 139 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/files/vm.args: -------------------------------------------------------------------------------- 1 | 2 | ## Name of the node 3 | -name {{node}} 4 | 5 | ## Cookie for distributed erlang 6 | -setcookie mfmn 7 | 8 | ## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive 9 | ## (Disabled by default..use with caution!) 10 | ##-heart 11 | 12 | ## Enable kernel poll and a few async threads 13 | +K true 14 | +A 5 15 | 16 | ## Increase number of concurrent ports/sockets 17 | -env ERL_MAX_PORTS 4096 18 | 19 | ## Tweak GC to run more often 20 | -env ERL_FULLSWEEP_AFTER 10 21 | 22 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/reltool.config: -------------------------------------------------------------------------------- 1 | {sys, [ 2 | {lib_dirs, ["../../", "../deps/"]}, 3 | {rel, "mfmn", "1", 4 | [ 5 | kernel, 6 | stdlib, 7 | sasl, 8 | mfmn 9 | ]}, 10 | {rel, "start_clean", "", 11 | [ 12 | kernel, 13 | stdlib 14 | ]}, 15 | {boot_rel, "mfmn"}, 16 | {profile, embedded}, 17 | {excl_sys_filters, ["^bin/.*", 18 | "^erts.*/bin/(dialyzer|typer)"]}, 19 | {app, sasl, [{incl_cond, include}]}, 20 | {app, mfmn, [{incl_cond, include}]} 21 | ]}. 22 | 23 | {target_dir, "mfmn"}. 24 | 25 | {overlay_vars, "vars.config"}. 26 | 27 | {overlay, [ 28 | {mkdir, "data/ring"}, 29 | {mkdir, "log/sasl"}, 30 | {copy, "files/erl", "{{erts_vsn}}/bin/erl"}, 31 | {copy, "files/nodetool", "{{erts_vsn}}/bin/nodetool"}, 32 | {template, "files/app.config", "etc/app.config"}, 33 | {template, "files/vm.args", "etc/vm.args"}, 34 | {template, "files/mfmn", "bin/mfmn"}, 35 | {template, "files/mfmn-admin", "bin/mfmn-admin"} 36 | ]}. 37 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/vars.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8098"}. 7 | {handoff_port, "8099"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "mfmn@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/vars/dev1.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8091"}. 7 | {handoff_port, "8101"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "mfmn1@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/vars/dev2.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8092"}. 7 | {handoff_port, "8102"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "mfmn2@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/rel/vars/dev3.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8093"}. 7 | {handoff_port, "8103"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "mfmn3@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/src/mfmn.app.src: -------------------------------------------------------------------------------- 1 | {application, mfmn, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | riak_core 10 | ]}, 11 | {mod, { mfmn_app, []}}, 12 | {env, []} 13 | ]}. -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/src/mfmn.erl: -------------------------------------------------------------------------------- 1 | -module(mfmn). 2 | -include("mfmn.hrl"). 3 | -include_lib("riak_core/include/riak_core_vnode.hrl"). 4 | 5 | -export([ 6 | ping/0 7 | ]). 8 | 9 | %% Public API 10 | 11 | % @doc Pings a random vnode to make sure communication is functional 12 | ping() -> 13 | DocIdx = riak_core_util:chash_key({<<"ping">>, term_to_binary(now())}), 14 | PrefList = riak_core_apl:get_primary_apl(DocIdx, 1, mfmn), 15 | [{IndexNode, _Type}] = PrefList, 16 | riak_core_vnode_master:sync_spawn_command(IndexNode, ping, mfmn_vnode_master). 17 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/src/mfmn.hrl: -------------------------------------------------------------------------------- 1 | -define(PRINT(Var), io:format("DEBUG: ~p:~p - ~p~n~n ~p~n~n", [?MODULE, ?LINE, ??Var, Var])). -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/src/mfmn_app.erl: -------------------------------------------------------------------------------- 1 | -module(mfmn_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | case mfmn_sup:start_link() of 14 | {ok, Pid} -> 15 | ok = riak_core:register_vnode_module(mfmn_vnode), 16 | ok = riak_core_node_watcher:service_up(mfmn, self()), 17 | {ok, Pid}; 18 | {error, Reason} -> 19 | {error, Reason} 20 | end. 21 | 22 | stop(_State) -> 23 | ok. 24 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/src/mfmn_console.erl: -------------------------------------------------------------------------------- 1 | %% @doc Interface for mfmn-admin commands. 2 | -module(mfmn_console). 3 | -export([join/1, 4 | leave/1, 5 | remove/1, 6 | ringready/1]). 7 | 8 | join([NodeStr]) -> 9 | case do_join(NodeStr) of 10 | ok -> 11 | io:format("Sent join request to ~s 12 | ", [NodeStr]), 13 | ok; 14 | {error, not_reachable} -> 15 | io:format("Node ~s is not reachable! 16 | ", [NodeStr]), 17 | error; 18 | {error, different_ring_sizes} -> 19 | io:format("Failed: ~s has a different ring_creation_size~n", 20 | [NodeStr]), 21 | error 22 | end; 23 | join(_) -> 24 | io:format("Join requires a node to join with. 25 | "), 26 | error. 27 | 28 | do_join(NodeStr) when is_list(NodeStr) -> 29 | do_join(riak_core_util:str_to_node(NodeStr)); 30 | do_join(Node) when is_atom(Node) -> 31 | {ok, OurRingSize} = application:get_env(riak_core, ring_creation_size), 32 | case net_adm:ping(Node) of 33 | pong -> 34 | case rpc:call(Node, 35 | application, 36 | get_env, 37 | [riak_core, ring_creation_size]) of 38 | {ok, OurRingSize} -> 39 | riak_core_gossip:send_ring(Node, node()); 40 | _ -> 41 | {error, different_ring_sizes} 42 | end; 43 | pang -> 44 | {error, not_reachable} 45 | end. 46 | 47 | leave([]) -> 48 | remove_node(node()). 49 | 50 | remove([Node]) -> 51 | remove_node(list_to_atom(Node)). 52 | 53 | remove_node(Node) when is_atom(Node) -> 54 | Res = riak_core_gossip:remove_from_cluster(Node), 55 | io:format("~p 56 | ", [Res]). 57 | 58 | -spec(ringready([]) -> ok | error). 59 | ringready([]) -> 60 | case ringready() of 61 | {ok, Nodes} -> 62 | io:format("TRUE All nodes agree on the ring ~p 63 | ", [Nodes]); 64 | {error, {different_owners, N1, N2}} -> 65 | io:format("FALSE Node ~p and ~p list different partition owners 66 | ", [N1, N2]), 67 | error; 68 | {error, {nodes_down, Down}} -> 69 | io:format("FALSE ~p down. All nodes need to be up to check. 70 | ", [Down]), 71 | error 72 | end. 73 | 74 | -spec(ringready() -> {ok, [atom()]} | {error, any()}). 75 | ringready() -> 76 | case get_rings() of 77 | {[], Rings} -> 78 | {N1,R1}=hd(Rings), 79 | case rings_match(hash_ring(R1), tl(Rings)) of 80 | true -> 81 | Nodes = [N || {N,_} <- Rings], 82 | {ok, Nodes}; 83 | 84 | {false, N2} -> 85 | {error, {different_owners, N1, N2}} 86 | end; 87 | 88 | {Down, _Rings} -> 89 | {error, {nodes_down, Down}} 90 | end. 91 | 92 | %% Retrieve the rings for all other nodes by RPC 93 | get_rings() -> 94 | {RawRings, Down} = riak_core_util:rpc_every_member( 95 | riak_core_ring_manager, get_my_ring, [], 30000), 96 | Rings = orddict:from_list([{riak_core_ring:owner_node(R), R} || {ok, R} <- RawRings]), 97 | {lists:sort(Down), Rings}. 98 | 99 | %% Produce a hash of the 'chash' portion of the ring 100 | hash_ring(R) -> 101 | erlang:phash2(riak_core_ring:all_owners(R)). 102 | 103 | %% Check if all rings match given a hash and a list of [{N,P}] to check 104 | rings_match(_, []) -> 105 | true; 106 | rings_match(R1hash, [{N2, R2} | Rest]) -> 107 | case hash_ring(R2) of 108 | R1hash -> 109 | rings_match(R1hash, Rest); 110 | _ -> 111 | {false, N2} 112 | end. 113 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/src/mfmn_sup.erl: -------------------------------------------------------------------------------- 1 | -module(mfmn_sup). 2 | 3 | -behaviour(supervisor). 4 | 5 | %% API 6 | -export([start_link/0]). 7 | 8 | %% Supervisor callbacks 9 | -export([init/1]). 10 | 11 | %% =================================================================== 12 | %% API functions 13 | %% =================================================================== 14 | 15 | start_link() -> 16 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 17 | 18 | %% =================================================================== 19 | %% Supervisor callbacks 20 | %% =================================================================== 21 | 22 | init(_Args) -> 23 | VMaster = { mfmn_vnode_master, 24 | {riak_core_vnode_master, start_link, [mfmn_vnode]}, 25 | permanent, 5000, worker, [riak_core_vnode_master]}, 26 | 27 | { ok, 28 | { {one_for_one, 5, 10}, 29 | [VMaster]}}. 30 | -------------------------------------------------------------------------------- /2011/riak-core-first-multinode/mfmn/src/mfmn_vnode.erl: -------------------------------------------------------------------------------- 1 | -module(mfmn_vnode). 2 | -behaviour(riak_core_vnode). 3 | -include("mfmn.hrl"). 4 | 5 | -export([start_vnode/1, 6 | init/1, 7 | terminate/2, 8 | handle_command/3, 9 | is_empty/1, 10 | delete/1, 11 | handle_handoff_command/3, 12 | handoff_starting/2, 13 | handoff_cancelled/1, 14 | handoff_finished/2, 15 | handle_handoff_data/2, 16 | encode_handoff_item/2]). 17 | 18 | -record(state, {partition}). 19 | 20 | %% API 21 | start_vnode(I) -> 22 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 23 | 24 | init([Partition]) -> 25 | {ok, #state { partition=Partition }}. 26 | 27 | % Sample command: respond to a ping 28 | handle_command(ping, _Sender, State) -> 29 | {reply, {pong, State#state.partition}, State}; 30 | handle_command(Message, _Sender, State) -> 31 | ?PRINT({unhandled_command, Message}), 32 | {noreply, State}. 33 | 34 | handle_handoff_command(_Message, _Sender, State) -> 35 | {noreply, State}. 36 | 37 | handoff_starting(_TargetNode, State) -> 38 | {true, State}. 39 | 40 | handoff_cancelled(State) -> 41 | {ok, State}. 42 | 43 | handoff_finished(_TargetNode, State) -> 44 | {ok, State}. 45 | 46 | handle_handoff_data(_Data, State) -> 47 | {reply, ok, State}. 48 | 49 | encode_handoff_item(_ObjectName, _ObjectValue) -> 50 | <<>>. 51 | 52 | is_empty(State) -> 53 | {true, State}. 54 | 55 | delete(State) -> 56 | {ok, State}. 57 | 58 | terminate(_Reason, _State) -> 59 | ok. 60 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/meta: -------------------------------------------------------------------------------- 1 | published 2011-04-19 -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | .eunit 3 | deps/* 4 | ebin 5 | rel/rts 6 | dev 7 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: deps rel 2 | 3 | all: deps compile 4 | 5 | compile: 6 | ./rebar compile 7 | 8 | deps: 9 | ./rebar get-deps 10 | 11 | clean: 12 | ./rebar clean 13 | 14 | distclean: clean devclean relclean 15 | ./rebar delete-deps 16 | 17 | rel: all 18 | ./rebar generate 19 | 20 | relclean: 21 | rm -rf rel/rts 22 | 23 | devrel: dev1 dev2 dev3 24 | 25 | devclean: 26 | rm -rf dev 27 | 28 | dev1 dev2 dev3: 29 | mkdir -p dev 30 | (cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@.config) -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/progski.access.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-core-the-coordinator/rts/progski.access.log.gz -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-core-the-coordinator/rts/rebar -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [debug_info, fail_on_warning]}. 2 | 3 | {sub_dirs, ["rel"]}. 4 | 5 | {deps, [{riak_core, "1.0.*", 6 | {git, "git://github.com/basho/riak_core", {tag,"1.0.0"}}} 7 | ]}. 8 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/files/app.config: -------------------------------------------------------------------------------- 1 | [ 2 | %% Riak Core config 3 | {riak_core, [ 4 | %% Default location of ringstate 5 | {ring_state_dir, "{{ring_state_dir}}"}, 6 | 7 | %% http is a list of IP addresses and TCP ports that the Riak 8 | %% HTTP interface will bind. 9 | {http, [ {"{{web_ip}}", {{web_port}} } ]}, 10 | 11 | %% riak_handoff_port is the TCP port that Riak uses for 12 | %% intra-cluster data handoff. 13 | {handoff_port, {{handoff_port}} } 14 | ]}, 15 | 16 | %% SASL config 17 | {sasl, [ 18 | {sasl_error_logger, {file, "log/sasl-error.log"}}, 19 | {errlog_type, error}, 20 | {error_logger_mf_dir, "log/sasl"}, % Log directory 21 | {error_logger_mf_maxbytes, 10485760}, % 10 MB max file size 22 | {error_logger_mf_maxfiles, 5} % 5 files max 23 | ]} 24 | ]. 25 | 26 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/files/erl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## This script replaces the default "erl" in erts-VSN/bin. This is necessary 4 | ## as escript depends on erl and in turn, erl depends on having access to a 5 | ## bootscript (start.boot). Note that this script is ONLY invoked as a side-effect 6 | ## of running escript -- the embedded node bypasses erl and uses erlexec directly 7 | ## (as it should). 8 | ## 9 | ## Note that this script makes the assumption that there is a start_clean.boot 10 | ## file available in $ROOTDIR/release/VSN. 11 | 12 | # Determine the abspath of where this script is executing from. 13 | ERTS_BIN_DIR=$(cd ${0%/*} && pwd) 14 | 15 | # Now determine the root directory -- this script runs from erts-VSN/bin, 16 | # so we simply need to strip off two dirs from the end of the ERTS_BIN_DIR 17 | # path. 18 | ROOTDIR=${ERTS_BIN_DIR%/*/*} 19 | 20 | # Parse out release and erts info 21 | START_ERL=`cat $ROOTDIR/releases/start_erl.data` 22 | ERTS_VSN=${START_ERL% *} 23 | APP_VSN=${START_ERL#* } 24 | 25 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 26 | EMU=beam 27 | PROGNAME=`echo $0 | sed 's/.*\\///'` 28 | CMD="$BINDIR/erlexec" 29 | export EMU 30 | export ROOTDIR 31 | export BINDIR 32 | export PROGNAME 33 | 34 | exec $CMD -boot $ROOTDIR/releases/$APP_VSN/start_clean ${1+"$@"} -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/files/nodetool: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% ex: ft=erlang ts=4 sw=4 et 3 | %% ------------------------------------------------------------------- 4 | %% 5 | %% nodetool: Helper Script for interacting with live nodes 6 | %% 7 | %% ------------------------------------------------------------------- 8 | 9 | main(Args) -> 10 | ok = start_epmd(), 11 | %% Extract the args 12 | {RestArgs, TargetNode} = process_args(Args, [], undefined), 13 | 14 | %% See if the node is currently running -- if it's not, we'll bail 15 | case {net_kernel:hidden_connect_node(TargetNode), net_adm:ping(TargetNode)} of 16 | {true, pong} -> 17 | ok; 18 | {_, pang} -> 19 | io:format("Node ~p not responding to pings.\n", [TargetNode]), 20 | halt(1) 21 | end, 22 | 23 | case RestArgs of 24 | ["ping"] -> 25 | %% If we got this far, the node already responsed to a ping, so just dump 26 | %% a "pong" 27 | io:format("pong\n"); 28 | ["stop"] -> 29 | io:format("~p\n", [rpc:call(TargetNode, init, stop, [], 60000)]); 30 | ["restart"] -> 31 | io:format("~p\n", [rpc:call(TargetNode, init, restart, [], 60000)]); 32 | ["reboot"] -> 33 | io:format("~p\n", [rpc:call(TargetNode, init, reboot, [], 60000)]); 34 | ["rpc", Module, Function | RpcArgs] -> 35 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), 36 | [RpcArgs], 60000) of 37 | ok -> 38 | ok; 39 | {badrpc, Reason} -> 40 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), 41 | halt(1); 42 | _ -> 43 | halt(1) 44 | end; 45 | ["rpcterms", Module, Function, ArgsAsString] -> 46 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), 47 | consult(ArgsAsString), 60000) of 48 | {badrpc, Reason} -> 49 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), 50 | halt(1); 51 | Other -> 52 | io:format("~p\n", [Other]) 53 | end; 54 | Other -> 55 | io:format("Other: ~p\n", [Other]), 56 | io:format("Usage: nodetool {ping|stop|restart|reboot}\n") 57 | end, 58 | net_kernel:stop(). 59 | 60 | process_args([], Acc, TargetNode) -> 61 | {lists:reverse(Acc), TargetNode}; 62 | process_args(["-setcookie", Cookie | Rest], Acc, TargetNode) -> 63 | erlang:set_cookie(node(), list_to_atom(Cookie)), 64 | process_args(Rest, Acc, TargetNode); 65 | process_args(["-name", TargetName | Rest], Acc, _) -> 66 | ThisNode = append_node_suffix(TargetName, "_maint_"), 67 | {ok, _} = net_kernel:start([ThisNode, longnames]), 68 | process_args(Rest, Acc, nodename(TargetName)); 69 | process_args(["-sname", TargetName | Rest], Acc, _) -> 70 | ThisNode = append_node_suffix(TargetName, "_maint_"), 71 | {ok, _} = net_kernel:start([ThisNode, shortnames]), 72 | process_args(Rest, Acc, nodename(TargetName)); 73 | process_args([Arg | Rest], Acc, Opts) -> 74 | process_args(Rest, [Arg | Acc], Opts). 75 | 76 | 77 | start_epmd() -> 78 | [] = os:cmd(epmd_path() ++ " -daemon"), 79 | ok. 80 | 81 | epmd_path() -> 82 | ErtsBinDir = filename:dirname(escript:script_name()), 83 | Name = "epmd", 84 | case os:find_executable(Name, ErtsBinDir) of 85 | false -> 86 | case os:find_executable(Name) of 87 | false -> 88 | io:format("Could not find epmd.~n"), 89 | halt(1); 90 | GlobalEpmd -> 91 | GlobalEpmd 92 | end; 93 | Epmd -> 94 | Epmd 95 | end. 96 | 97 | 98 | nodename(Name) -> 99 | case string:tokens(Name, "@") of 100 | [_Node, _Host] -> 101 | list_to_atom(Name); 102 | [Node] -> 103 | [_, Host] = string:tokens(atom_to_list(node()), "@"), 104 | list_to_atom(lists:concat([Node, "@", Host])) 105 | end. 106 | 107 | append_node_suffix(Name, Suffix) -> 108 | case string:tokens(Name, "@") of 109 | [Node, Host] -> 110 | list_to_atom(lists:concat([Node, Suffix, os:getpid(), "@", Host])); 111 | [Node] -> 112 | list_to_atom(lists:concat([Node, Suffix, os:getpid()])) 113 | end. 114 | 115 | 116 | %% 117 | %% Given a string or binary, parse it into a list of terms, ala file:consult/0 118 | %% 119 | consult(Str) when is_list(Str) -> 120 | consult([], Str, []); 121 | consult(Bin) when is_binary(Bin)-> 122 | consult([], binary_to_list(Bin), []). 123 | 124 | consult(Cont, Str, Acc) -> 125 | case erl_scan:tokens(Cont, Str, 0) of 126 | {done, Result, Remaining} -> 127 | case Result of 128 | {ok, Tokens, _} -> 129 | {ok, Term} = erl_parse:parse_term(Tokens), 130 | consult([], Remaining, [Term | Acc]); 131 | {eof, _Other} -> 132 | lists:reverse(Acc); 133 | {error, Info, _} -> 134 | {error, Info} 135 | end; 136 | {more, Cont1} -> 137 | consult(Cont1, eof, Acc) 138 | end. 139 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/files/rts: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # -*- tab-width:4;indent-tabs-mode:nil -*- 3 | # ex: ts=4 sw=4 et 4 | 5 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd) 6 | 7 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*} 8 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc 9 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log 10 | # Note the trailing slash on $PIPE_DIR/ 11 | PIPE_DIR=/tmp/$RUNNER_BASE_DIR/ 12 | RUNNER_USER= 13 | 14 | # Make sure this script is running as the appropriate user 15 | if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then 16 | exec sudo -u $RUNNER_USER -i $0 $@ 17 | fi 18 | 19 | # Make sure CWD is set to runner base dir 20 | cd $RUNNER_BASE_DIR 21 | 22 | # Make sure log directory exists 23 | mkdir -p $RUNNER_LOG_DIR 24 | 25 | # Extract the target node name from node.args 26 | NAME_ARG=`grep -e '-[s]*name' $RUNNER_ETC_DIR/vm.args` 27 | if [ -z "$NAME_ARG" ]; then 28 | echo "vm.args needs to have either -name or -sname parameter." 29 | exit 1 30 | fi 31 | 32 | # Extract the target cookie 33 | COOKIE_ARG=`grep -e '-setcookie' $RUNNER_ETC_DIR/vm.args` 34 | if [ -z "$COOKIE_ARG" ]; then 35 | echo "vm.args needs to have a -setcookie parameter." 36 | exit 1 37 | fi 38 | 39 | # Identify the script name 40 | SCRIPT=`basename $0` 41 | 42 | # Parse out release and erts info 43 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` 44 | ERTS_VSN=${START_ERL% *} 45 | APP_VSN=${START_ERL#* } 46 | 47 | # Add ERTS bin dir to our path 48 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin 49 | 50 | # Setup command to control the node 51 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" 52 | 53 | # Check the first argument for instructions 54 | case "$1" in 55 | start) 56 | # Make sure there is not already a node running 57 | RES=`$NODETOOL ping` 58 | if [ "$RES" = "pong" ]; then 59 | echo "Node is already running!" 60 | exit 1 61 | fi 62 | HEART_COMMAND="$RUNNER_BASE_DIR/bin/$SCRIPT start" 63 | export HEART_COMMAND 64 | mkdir -p $PIPE_DIR 65 | shift # remove $1 66 | $ERTS_PATH/run_erl -daemon $PIPE_DIR $RUNNER_LOG_DIR "exec $RUNNER_BASE_DIR/bin/$SCRIPT console $@" 2>&1 67 | ;; 68 | 69 | stop) 70 | # Wait for the node to completely stop... 71 | case `uname -s` in 72 | Linux|Darwin|FreeBSD|DragonFly|NetBSD|OpenBSD) 73 | # PID COMMAND 74 | PID=`ps ax -o pid= -o command=|\ 75 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'` 76 | ;; 77 | SunOS) 78 | # PID COMMAND 79 | PID=`ps -ef -o pid= -o args=|\ 80 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'` 81 | ;; 82 | CYGWIN*) 83 | # UID PID PPID TTY STIME COMMAND 84 | PID=`ps -efW|grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $2}'` 85 | ;; 86 | esac 87 | $NODETOOL stop 88 | while `kill -0 $PID 2>/dev/null`; 89 | do 90 | sleep 1 91 | done 92 | ;; 93 | 94 | restart) 95 | ## Restart the VM without exiting the process 96 | $NODETOOL restart 97 | ;; 98 | 99 | reboot) 100 | ## Restart the VM completely (uses heart to restart it) 101 | $NODETOOL reboot 102 | ;; 103 | 104 | ping) 105 | ## See if the VM is alive 106 | $NODETOOL ping 107 | ;; 108 | 109 | attach) 110 | # Make sure a node IS running 111 | RES=`$NODETOOL ping` 112 | if [ "$RES" != "pong" ]; then 113 | echo "Node is not running!" 114 | exit 1 115 | fi 116 | 117 | shift 118 | $ERTS_PATH/to_erl $PIPE_DIR 119 | ;; 120 | 121 | console|console_clean) 122 | # .boot file typically just $SCRIPT (ie, the app name) 123 | # however, for debugging, sometimes start_clean.boot is useful: 124 | case "$1" in 125 | console) BOOTFILE=$SCRIPT ;; 126 | console_clean) BOOTFILE=start_clean ;; 127 | esac 128 | # Setup beam-required vars 129 | ROOTDIR=$RUNNER_BASE_DIR 130 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 131 | EMU=beam 132 | PROGNAME=`echo $0 | sed 's/.*\\///'` 133 | CMD="$BINDIR/erlexec -boot $RUNNER_BASE_DIR/releases/$APP_VSN/$BOOTFILE -embedded -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -- ${1+"$@"}" 134 | export EMU 135 | export ROOTDIR 136 | export BINDIR 137 | export PROGNAME 138 | 139 | # Dump environment info for logging purposes 140 | echo "Exec: $CMD" 141 | echo "Root: $ROOTDIR" 142 | 143 | # Log the startup 144 | logger -t "$SCRIPT[$$]" "Starting up" 145 | 146 | # Start the VM 147 | exec $CMD 148 | ;; 149 | 150 | *) 151 | echo "Usage: $SCRIPT {start|stop|restart|reboot|ping|console|console_clean|attach}" 152 | exit 1 153 | ;; 154 | esac 155 | 156 | exit 0 157 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/files/rts-admin: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd) 4 | RUNNER_SCRIPT=${0##*/} 5 | 6 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*} 7 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc 8 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log 9 | RUNNER_USER= 10 | 11 | # Make sure this script is running as the appropriate user 12 | if [ "$RUNNER_USER" -a "x$LOGNAME" != "x$RUNNER_USER" ]; then 13 | type -p sudo > /dev/null 2>&1 14 | if [ $? -ne 0 ]; then 15 | echo "sudo doesn't appear to be installed and your EUID isn't $RUNNER_USER" 1>&2 16 | exit 1 17 | fi 18 | echo "Attempting to restart script through sudo -u $RUNNER_USER" 19 | exec sudo -u $RUNNER_USER -i $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ 20 | fi 21 | 22 | # Make sure CWD is set to runner base dir 23 | cd $RUNNER_BASE_DIR 24 | 25 | # Extract the target node name from node.args 26 | NAME_ARG=`grep -e '-[s]*name' $RUNNER_ETC_DIR/vm.args` 27 | if [ -z "$NAME_ARG" ]; then 28 | echo "vm.args needs to have either -name or -sname parameter." 29 | exit 1 30 | fi 31 | 32 | # Learn how to specify node name for connection from remote nodes 33 | echo "$NAME_ARG" | grep '^-sname' > /dev/null 2>&1 34 | if [ "X$?" = "X0" ]; then 35 | NAME_PARAM="-sname" 36 | NAME_HOST="" 37 | else 38 | NAME_PARAM="-name" 39 | echo "$NAME_ARG" | grep '@.*' > /dev/null 2>&1 40 | if [ "X$?" = "X0" ]; then 41 | NAME_HOST=`echo "${NAME_ARG}" | sed -e 's/.*(@.*)$//'` 42 | else 43 | NAME_HOST="" 44 | fi 45 | fi 46 | 47 | # Extract the target cookie 48 | COOKIE_ARG=`grep '-setcookie' $RUNNER_ETC_DIR/vm.args` 49 | if [ -z "$COOKIE_ARG" ]; then 50 | echo "vm.args needs to have a -setcookie parameter." 51 | exit 1 52 | fi 53 | 54 | # Identify the script name 55 | SCRIPT=`basename $0` 56 | 57 | # Parse out release and erts info 58 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` 59 | ERTS_VSN=${START_ERL% *} 60 | APP_VSN=${START_ERL#* } 61 | 62 | # Add ERTS bin dir to our path 63 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin 64 | 65 | # Setup command to control the node 66 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" 67 | 68 | # Check the first argument for instructions 69 | case "$1" in 70 | join) 71 | # Make sure the local node IS running 72 | RES=`$NODETOOL ping` 73 | if [ "$RES" != "pong" ]; then 74 | echo "Node is not running!" 75 | exit 1 76 | fi 77 | 78 | shift 79 | 80 | $NODETOOL rpc rts_console join $@ 81 | ;; 82 | 83 | leave) 84 | # Make sure the local node is running 85 | RES=`$NODETOOL ping` 86 | if [ "$RES" != "pong" ]; then 87 | echo "Node is not running!" 88 | exit 1 89 | fi 90 | 91 | shift 92 | $NODETOOL rpc rts__console leave $@ 93 | ;; 94 | 95 | remove) 96 | if [ $# -ne 2 ]; then 97 | echo "Usage: $SCRIPT remove " 98 | exit 1 99 | fi 100 | 101 | RES=`$NODETOOL ping` 102 | if [ "$RES" != "pong" ]; then 103 | echo "Node is not running!" 104 | exit 1 105 | fi 106 | 107 | shift 108 | $NODETOOL rpc rts_console remove $@ 109 | ;; 110 | 111 | ringready) 112 | # Make sure the local node IS running 113 | RES=`$NODETOOL ping` 114 | if [ "$RES" != "pong" ]; then 115 | echo "Node is not running!" 116 | exit 1 117 | fi 118 | shift 119 | 120 | $NODETOOL rpc rts_console ringready $@ 121 | ;; 122 | 123 | *) 124 | echo "Usage: $SCRIPT { join | leave | reip | ringready | remove }" 125 | exit 1 126 | ;; 127 | esac 128 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/files/vm.args: -------------------------------------------------------------------------------- 1 | 2 | ## Name of the node 3 | -name {{node}} 4 | 5 | ## Cookie for distributed erlang 6 | -setcookie rts 7 | 8 | ## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive 9 | ## (Disabled by default..use with caution!) 10 | ##-heart 11 | 12 | ## Enable kernel poll and a few async threads 13 | +K true 14 | +A 5 15 | 16 | ## Increase number of concurrent ports/sockets 17 | -env ERL_MAX_PORTS 4096 18 | 19 | ## Tweak GC to run more often 20 | -env ERL_FULLSWEEP_AFTER 10 21 | 22 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/reltool.config: -------------------------------------------------------------------------------- 1 | {sys, [ 2 | {lib_dirs, ["../../", "../deps/"]}, 3 | {rel, "rts", "1", 4 | [ 5 | kernel, 6 | stdlib, 7 | sasl, 8 | rts 9 | ]}, 10 | {rel, "start_clean", "", 11 | [ 12 | kernel, 13 | stdlib 14 | ]}, 15 | {boot_rel, "rts"}, 16 | {profile, embedded}, 17 | {excl_sys_filters, ["^bin/.*", 18 | "^erts.*/bin/(dialyzer|typer)"]}, 19 | {app, sasl, [{incl_cond, include}]}, 20 | {app, rts, [{incl_cond, include}]} 21 | ]}. 22 | 23 | {target_dir, "rts"}. 24 | 25 | {overlay_vars, "vars.config"}. 26 | 27 | {overlay, [ 28 | {mkdir, "data/ring"}, 29 | {mkdir, "log/sasl"}, 30 | {copy, "files/erl", "{{erts_vsn}}/bin/erl"}, 31 | {copy, "files/nodetool", "{{erts_vsn}}/bin/nodetool"}, 32 | {template, "files/app.config", "etc/app.config"}, 33 | {template, "files/vm.args", "etc/vm.args"}, 34 | {template, "files/rts", "bin/rts"}, 35 | {template, "files/rts-admin", "bin/rts-admin"} 36 | ]}. 37 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/vars.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8888"}. 7 | {handoff_port, "8099"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/vars/dev1.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8881"}. 7 | {handoff_port, "8101"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts1@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/vars/dev2.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8882"}. 7 | {handoff_port, "8102"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts2@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/rel/vars/dev3.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8883"}. 7 | {handoff_port, "8103"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts3@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/replay: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # Usage: 4 | # gunzip access.log.gz | ./replay [--devrel] client 5 | # ./replay [--devrel] client < access.log 6 | 7 | PORTS[0]=8888 8 | MAX=0 9 | 10 | while [ $# -gt 0 ] 11 | do 12 | case $1 in 13 | --devrel | -d ) 14 | PORTS[0]=8881 15 | PORTS[1]=8882 16 | PORTS[2]=8883 17 | MAX=2 18 | ;; 19 | *) 20 | break 21 | ;; 22 | esac 23 | shift 24 | done 25 | 26 | CLIENT=$1 27 | shift 28 | 29 | node=0 30 | while read line 31 | do 32 | curl -X POST \ 33 | -H 'content-type: text/plain' \ 34 | "http://localhost:${PORTS[$node]}/rts/entry/$CLIENT" \ 35 | -d "$line" 36 | 37 | if [ $node -eq $MAX ] 38 | then 39 | node=0 40 | else 41 | ((node++)) 42 | fi 43 | done -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts.app.src: -------------------------------------------------------------------------------- 1 | {application, rts, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | riak_core 10 | ]}, 11 | {mod, { rts_app, []}}, 12 | {env, []} 13 | ]}. -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts.erl: -------------------------------------------------------------------------------- 1 | %% @doc Interface into the Real Time Statistics application. 2 | -module(rts). 3 | -include("rts.hrl"). 4 | -include_lib("riak_core/include/riak_core_vnode.hrl"). 5 | 6 | -export([ 7 | ping/0, 8 | entry/2, 9 | get/2, 10 | set/3, 11 | append/3, 12 | incr/2, 13 | incrby/3, 14 | sadd/3 15 | ]). 16 | -export([get_dbg_preflist/2, 17 | get_dbg_preflist/3]). 18 | -define(TIMEOUT, 5000). 19 | 20 | %%%=================================================================== 21 | %%% API 22 | %%%=================================================================== 23 | 24 | % @doc Pings a random vnode to make sure communication is functional 25 | ping() -> 26 | DocIdx = riak_core_util:chash_key({<<"ping">>, term_to_binary(now())}), 27 | PrefList = riak_core_apl:get_primary_apl(DocIdx, 1, rts), 28 | [{IndexNode, _Type}] = PrefList, 29 | riak_core_vnode_master:sync_spawn_command(IndexNode, ping, rts_vnode_master). 30 | 31 | %% @doc Process an entry. 32 | entry(Client, Entry) -> 33 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 34 | term_to_binary(now())}), 35 | PrefList = riak_core_apl:get_apl(DocIdx, 1, rts_entry), 36 | [IdxNode] = PrefList, 37 | rts_entry_vnode:entry(IdxNode, Client, Entry). 38 | 39 | %% @doc Get a stat's value. 40 | get(Client, StatName) -> 41 | {ok, ReqID} = rts_get_fsm:get(Client, StatName), 42 | wait_for_reqid(ReqID, ?TIMEOUT). 43 | 44 | get_dbg_preflist(Client, StatName) -> 45 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 46 | list_to_binary(StatName)}), 47 | riak_core_apl:get_apl(DocIdx, ?N, rts_stat). 48 | 49 | get_dbg_preflist(Client, StatName, N) -> 50 | IdxNode = lists:nth(N, get_dbg_preflist(Client, StatName)), 51 | {ok, req_id, Val} = 52 | riak_core_vnode_master:sync_command(IdxNode, 53 | {get, req_id, StatName}, 54 | rts_stat_vnode_master), 55 | [IdxNode, Val]. 56 | 57 | %% @doc Set a stat's value, replacing the current value. 58 | set(Client, StatName, Val) -> 59 | do_write(Client, StatName, set, Val). 60 | 61 | %% @doc Append to a stat's value. 62 | append(Client, StatName, Val) -> 63 | do_write(Client, StatName, append, Val). 64 | 65 | %% @doc Increment the stat's value by 1. 66 | incr(Client, StatName) -> 67 | do_write(Client, StatName, incr). 68 | 69 | %% @doc Increment the stat's value by Val. 70 | incrby(Client, StatName, Val) -> 71 | do_write(Client, StatName, incrby, Val). 72 | 73 | %% @doc Add a memeber to the stat's set. 74 | sadd(Client, StatName, Val) -> 75 | do_write(Client, StatName, sadd, Val). 76 | 77 | %%%=================================================================== 78 | %%% Internal Functions 79 | %%%=================================================================== 80 | do_write(Client, StatName, Op) -> 81 | {ok, ReqID} = rts_write_fsm:write(Client, StatName, Op), 82 | wait_for_reqid(ReqID, ?TIMEOUT). 83 | 84 | do_write(Client, StatName, Op, Val) -> 85 | {ok, ReqID} = rts_write_fsm:write(Client, StatName, Op, Val), 86 | wait_for_reqid(ReqID, ?TIMEOUT). 87 | 88 | wait_for_reqid(ReqID, Timeout) -> 89 | receive 90 | {ReqID, ok} -> ok; 91 | {ReqID, ok, Val} -> {ok, Val} 92 | after Timeout -> 93 | {error, timeout} 94 | end. 95 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts.hrl: -------------------------------------------------------------------------------- 1 | -define(PRINT(Var), io:format("DEBUG: ~p:~p - ~p~n~n ~p~n~n", [?MODULE, ?LINE, ??Var, Var])). 2 | 3 | -define(N, 3). 4 | -define(R, 2). 5 | -define(W, 2). 6 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_app.erl: -------------------------------------------------------------------------------- 1 | -module(rts_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | case rts_sup:start_link() of 14 | {ok, Pid} -> 15 | ok = riak_core:register_vnode_module(rts_vnode), 16 | ok = riak_core_node_watcher:service_up(rts, self()), 17 | 18 | ok = riak_core:register_vnode_module(rts_entry_vnode), 19 | ok = riak_core_node_watcher:service_up(rts_entry, self()), 20 | 21 | ok = riak_core:register_vnode_module(rts_stat_vnode), 22 | ok = riak_core_node_watcher:service_up(rts_stat, self()), 23 | 24 | EntryRoute = {["rts", "entry", client], rts_wm_entry, []}, 25 | webmachine_router:add_route(EntryRoute), 26 | 27 | {ok, Pid}; 28 | {error, Reason} -> 29 | {error, Reason} 30 | end. 31 | 32 | stop(_State) -> 33 | ok. 34 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_console.erl: -------------------------------------------------------------------------------- 1 | %% @doc Interface for rts-admin commands. 2 | -module(rts_console). 3 | -export([join/1, 4 | leave/1, 5 | remove/1, 6 | ringready/1]). 7 | 8 | join([NodeStr]) -> 9 | try 10 | case riak_core:join(NodeStr) of 11 | ok -> 12 | io:format("Sent join request to ~s\n", [NodeStr]), 13 | ok; 14 | {error, not_reachable} -> 15 | io:format("Node ~s is not reachable!\n", [NodeStr]), 16 | error; 17 | {error, different_ring_sizes} -> 18 | io:format("Failed: ~s has a different ring_creation_size~n", 19 | [NodeStr]), 20 | error 21 | end 22 | catch 23 | Exception:Reason -> 24 | lager:error("Join failed ~p:~p", [Exception, 25 | Reason]), 26 | io:format("Join failed, see log for details~n"), 27 | error 28 | end. 29 | 30 | 31 | leave([]) -> 32 | remove_node(node()). 33 | 34 | 35 | remove([Node]) -> 36 | remove_node(list_to_atom(Node)). 37 | 38 | remove_node(Node) when is_atom(Node) -> 39 | try 40 | case catch(riak_core:remove_from_cluster(Node)) of 41 | {'EXIT', {badarg, [{erlang, hd, [[]]}|_]}} -> 42 | %% This is a workaround because 43 | %% riak_core_gossip:remove_from_cluster doesn't check if 44 | %% the result of subtracting the current node from the 45 | %% cluster member list results in the empty list. When 46 | %% that code gets refactored this can probably go away. 47 | io:format("Leave failed, this node is the only member.~n"), 48 | error; 49 | Res -> 50 | io:format(" ~p\n", [Res]) 51 | end 52 | catch 53 | Exception:Reason -> 54 | lager:error("Leave failed ~p:~p", [Exception, 55 | Reason]), 56 | io:format("Leave failed, see log for details~n"), 57 | error 58 | end. 59 | 60 | 61 | -spec(ringready([]) -> ok | error). 62 | ringready([]) -> 63 | try 64 | case riak_core_status:ringready() of 65 | {ok, Nodes} -> 66 | io:format("TRUE All nodes agree on the ring ~p\n", [Nodes]); 67 | {error, {different_owners, N1, N2}} -> 68 | io:format("FALSE Node ~p and ~p list different partition owners\n", [N1, N2]), 69 | error; 70 | {error, {nodes_down, Down}} -> 71 | io:format("FALSE ~p down. All nodes need to be up to check.\n", [Down]), 72 | error 73 | end 74 | catch 75 | Exception:Reason -> 76 | lager:error("Ringready failed ~p:~p", [Exception, 77 | Reason]), 78 | io:format("Ringready failed, see log for details~n"), 79 | error 80 | end. 81 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_entry_vnode.erl: -------------------------------------------------------------------------------- 1 | %% @doc A vnode to crunch incoming log entries. Attempt to match each 2 | %% log entry against a registry of regexps. If a regexp matches then 3 | %% execute its corresponding trigger function passing it the {Client, 4 | %% Entry, Regexp} as well as the resulting match. The trigger 5 | %% function can then choose to take an action such as update a 6 | %% statistic via the `rts_stat_vnode'. 7 | %% 8 | %% Since this vnode is purely for computation there is no need to 9 | %% worry about handoff. 10 | -module(rts_entry_vnode). 11 | -behaviour(riak_core_vnode). 12 | -include("rts.hrl"). 13 | 14 | -export([start_vnode/1, 15 | init/1, 16 | terminate/2, 17 | handle_command/3, 18 | is_empty/1, 19 | delete/1, 20 | handle_handoff_command/3, 21 | handoff_starting/2, 22 | handoff_cancelled/1, 23 | handoff_finished/2, 24 | handle_handoff_data/2, 25 | encode_handoff_item/2, 26 | handle_coverage/4, 27 | handle_exit/3]). 28 | 29 | %% match handlers 30 | -export([ 31 | combined_lf/2 32 | ]). 33 | 34 | %% API 35 | -export([entry/3]). 36 | 37 | -record(state, { 38 | partition, 39 | reg %% registry [regexp => fun] 40 | }). 41 | 42 | -define(MASTER, rts_entry_vnode_master). 43 | -define(COMBINED_LF, "(\\d+\\.\\d+\\.\\d+\\.\\d+) (.*) (.*) (\\[.*\\]) \"(.*)\" (\\d+) (.*) \"(.*)\" \"(.*)\""). 44 | 45 | %%%=================================================================== 46 | %%% API 47 | %%%=================================================================== 48 | 49 | start_vnode(I) -> 50 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 51 | 52 | entry(IdxNode, Client, Entry) -> 53 | riak_core_vnode_master:command(IdxNode, 54 | {entry, Client, Entry}, 55 | ?MASTER). 56 | 57 | %%%=================================================================== 58 | %%% Callbacks 59 | %%%=================================================================== 60 | 61 | init([Partition]) -> 62 | Reg = [ 63 | {?COMBINED_LF, fun ?MODULE:combined_lf/2} 64 | ], 65 | {ok, #state { partition=Partition, reg=Reg }}. 66 | 67 | handle_command({entry, Client, Entry}, _Sender, #state{reg=Reg}=State) -> 68 | io:format("~p~n", [{entry, State#state.partition}]), 69 | lists:foreach(match(Client, Entry), Reg), 70 | {noreply, State}. 71 | 72 | handle_handoff_command(_Message, _Sender, State) -> 73 | {noreply, State}. 74 | 75 | handoff_starting(_TargetNode, _State) -> 76 | {true, _State}. 77 | 78 | handoff_cancelled(State) -> 79 | {ok, State}. 80 | 81 | handoff_finished(_TargetNode, State) -> 82 | {ok, State}. 83 | 84 | handle_handoff_data(_Data, State) -> 85 | {reply, ok, State}. 86 | 87 | encode_handoff_item(_ObjectName, _ObjectValue) -> 88 | <<>>. 89 | 90 | is_empty(State) -> 91 | {true, State}. 92 | 93 | delete(State) -> 94 | {ok, State}. 95 | 96 | handle_coverage(_Req, _KeySpaces, _Sender, State) -> 97 | {stop, not_implemented, State}. 98 | 99 | handle_exit(_Pid, _Reason, State) -> 100 | {noreply, State}. 101 | 102 | terminate(_Reason, _State) -> 103 | ok. 104 | 105 | %%%=================================================================== 106 | %%% Internal Functions 107 | %%%=================================================================== 108 | 109 | match(Client, Entry) -> 110 | fun({Regexp, Fun}) -> 111 | case re:run(Entry, Regexp, [{capture, all, list}]) of 112 | nomatch -> ignore; 113 | {match, Match} -> Fun({Client, Entry, Regexp}, Match) 114 | end 115 | end. 116 | 117 | %%%=================================================================== 118 | %%% Match Handlers 119 | %%%=================================================================== 120 | 121 | combined_lf({Client, _Entry, _Regexp}, [_Entry, _Host, _, _User, _Time, Req, Code, BodySize, _Referer, Agent]) -> 122 | rts:sadd(Client, "agents", Agent), 123 | rts:incrby(Client, "total_sent", list_to_integer(BodySize)), 124 | [Method, _Resource, _Protocol] = string:tokens(Req, " "), 125 | rts:incr(Client, Method), 126 | case Code of 127 | [$2, _, _] -> 128 | rts:incr(Client, "200"); 129 | [$3, _, _] -> 130 | rts:incr(Client, "300"); 131 | [$4, _, _] -> 132 | rts:incr(Client, "400"); 133 | [$5, _, _] -> 134 | rts:incr(Client, "500") 135 | end, 136 | rts:incr(Client, "total_reqs"). 137 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_get_fsm.erl: -------------------------------------------------------------------------------- 1 | %% @doc The coordinator for stat get operations. The key here is to 2 | %% generate the preflist just like in wrtie_fsm and then query each 3 | %% replica and wait until a quorum is met. 4 | -module(rts_get_fsm). 5 | -behavior(gen_fsm). 6 | -include("rts.hrl"). 7 | 8 | %% API 9 | -export([start_link/4, get/2]). 10 | 11 | %% Callbacks 12 | -export([init/1, code_change/4, handle_event/3, handle_info/3, 13 | handle_sync_event/4, terminate/3]). 14 | 15 | %% States 16 | -export([prepare/2, execute/2, waiting/2]). 17 | 18 | -record(state, {req_id, 19 | from, 20 | client, 21 | stat_name, 22 | preflist, 23 | num_r=0, 24 | replies=[]}). 25 | 26 | %%%=================================================================== 27 | %%% API 28 | %%%=================================================================== 29 | 30 | start_link(ReqID, From, Client, StatName) -> 31 | gen_fsm:start_link(?MODULE, [ReqID, From, Client, StatName], []). 32 | 33 | get(Client, StatName) -> 34 | ReqID = mk_reqid(), 35 | rts_get_fsm_sup:start_get_fsm([ReqID, self(), Client, StatName]), 36 | {ok, ReqID}. 37 | 38 | %%%=================================================================== 39 | %%% States 40 | %%%=================================================================== 41 | 42 | %% Intiailize state data. 43 | init([ReqId, From, Client, StatName]) -> 44 | SD = #state{req_id=ReqId, 45 | from=From, 46 | client=Client, 47 | stat_name=StatName}, 48 | {ok, prepare, SD, 0}. 49 | 50 | %% @doc Calculate the Preflist. 51 | prepare(timeout, SD0=#state{client=Client, 52 | stat_name=StatName}) -> 53 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 54 | list_to_binary(StatName)}), 55 | Prelist = riak_core_apl:get_apl(DocIdx, ?N, rts_stat), 56 | SD = SD0#state{preflist=Prelist}, 57 | {next_state, execute, SD, 0}. 58 | 59 | %% @doc Execute the get reqs. 60 | execute(timeout, SD0=#state{req_id=ReqId, 61 | stat_name=StatName, 62 | preflist=Prelist}) -> 63 | rts_stat_vnode:get(Prelist, ReqId, StatName), 64 | {next_state, waiting, SD0}. 65 | 66 | %% @doc Wait for R replies and then respond to From (original client 67 | %% that called `rts:get/2'). 68 | %% TODO: read repair...or another blog post? 69 | waiting({ok, ReqID, Val}, SD0=#state{from=From, num_r=NumR0, replies=Replies0}) -> 70 | NumR = NumR0 + 1, 71 | Replies = [Val|Replies0], 72 | SD = SD0#state{num_r=NumR,replies=Replies}, 73 | if 74 | NumR =:= ?R -> 75 | Reply = 76 | case lists:any(different(Val), Replies) of 77 | true -> 78 | Replies; 79 | false -> 80 | Val 81 | end, 82 | From ! {ReqID, ok, Reply}, 83 | {stop, normal, SD}; 84 | true -> {next_state, waiting, SD} 85 | end. 86 | 87 | handle_info(_Info, _StateName, StateData) -> 88 | {stop,badmsg,StateData}. 89 | 90 | handle_event(_Event, _StateName, StateData) -> 91 | {stop,badmsg,StateData}. 92 | 93 | handle_sync_event(_Event, _From, _StateName, StateData) -> 94 | {stop,badmsg,StateData}. 95 | 96 | code_change(_OldVsn, StateName, State, _Extra) -> {ok, StateName, State}. 97 | 98 | terminate(_Reason, _SN, _SD) -> 99 | ok. 100 | 101 | %%%=================================================================== 102 | %%% Internal Functions 103 | %%%=================================================================== 104 | 105 | different(A) -> fun(B) -> A =/= B end. 106 | 107 | mk_reqid() -> erlang:phash2(erlang:now()). 108 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_get_fsm_sup.erl: -------------------------------------------------------------------------------- 1 | %% @doc Supervise the rts_get FSM. 2 | -module(rts_get_fsm_sup). 3 | -behavior(supervisor). 4 | 5 | -export([start_get_fsm/1, 6 | start_link/0]). 7 | -export([init/1]). 8 | 9 | start_get_fsm(Args) -> 10 | supervisor:start_child(?MODULE, Args). 11 | 12 | start_link() -> 13 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 14 | 15 | init([]) -> 16 | GetFsm = {undefined, 17 | {rts_get_fsm, start_link, []}, 18 | temporary, 5000, worker, [rts_get_fsm]}, 19 | {ok, {{simple_one_for_one, 10, 10}, [GetFsm]}}. 20 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_stat_vnode.erl: -------------------------------------------------------------------------------- 1 | %% @doc A vnode to handle get & put commands for stat data. The vnode 2 | %% requests will be hashed on Client and StatName and will use a 3 | %% coordinator to enforce N/R/W values. 4 | -module(rts_stat_vnode). 5 | -behaviour(riak_core_vnode). 6 | -include("rts.hrl"). 7 | -include_lib("riak_core/include/riak_core_vnode.hrl"). 8 | -export([start_vnode/1, 9 | init/1, 10 | terminate/2, 11 | handle_command/3, 12 | is_empty/1, 13 | delete/1, 14 | handle_handoff_command/3, 15 | handoff_starting/2, 16 | handoff_cancelled/1, 17 | handoff_finished/2, 18 | handle_handoff_data/2, 19 | encode_handoff_item/2, 20 | handle_coverage/4, 21 | handle_exit/3]). 22 | 23 | -export([ 24 | get/3, 25 | set/4, 26 | incr/3, 27 | incrby/4, 28 | append/4, 29 | sadd/4 30 | ]). 31 | 32 | -record(state, {partition, stats}). 33 | 34 | -define(MASTER, rts_stat_vnode_master). 35 | -define(sync(PrefList, Command, Master), 36 | riak_core_vnode_master:sync_command(PrefList, Command, Master)). 37 | 38 | %%%=================================================================== 39 | %%% API 40 | %%%=================================================================== 41 | 42 | start_vnode(I) -> 43 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 44 | 45 | get(Preflist, ReqID, StatName) -> 46 | riak_core_vnode_master:command(Preflist, 47 | {get, ReqID, StatName}, 48 | {fsm, undefined, self()}, 49 | ?MASTER). 50 | 51 | set(Preflist, ReqID, StatName, Val) -> 52 | riak_core_vnode_master:command(Preflist, 53 | {set, ReqID, StatName, Val}, 54 | ?MASTER). 55 | 56 | %% TODO: I have to look at the Sender stuff more closely again 57 | incr(Preflist, ReqID, StatName) -> 58 | riak_core_vnode_master:command(Preflist, 59 | {incr, ReqID, StatName}, 60 | {fsm, undefined, self()}, 61 | ?MASTER). 62 | 63 | incrby(Preflist, ReqID, StatName, Val) -> 64 | riak_core_vnode_master:command(Preflist, 65 | {incrby, ReqID, StatName, Val}, 66 | {fsm, undefined, self()}, 67 | ?MASTER). 68 | 69 | append(Preflist, ReqID, StatName, Val) -> 70 | riak_core_vnode_master:command(Preflist, 71 | {append, ReqID, StatName, Val}, 72 | ?MASTER). 73 | 74 | sadd(Preflist, ReqID, StatName, Val) -> 75 | riak_core_vnode_master:command(Preflist, 76 | {sadd, ReqID, StatName, Val}, 77 | ?MASTER). 78 | 79 | %%%=================================================================== 80 | %%% Callbacks 81 | %%%=================================================================== 82 | 83 | init([Partition]) -> 84 | {ok, #state { partition=Partition, stats=dict:new() }}. 85 | 86 | handle_command({get, ReqID, StatName}, _Sender, #state{stats=Stats}=State) -> 87 | Reply = 88 | case dict:find(StatName, Stats) of 89 | error -> 90 | not_found; 91 | {ok, Found} -> 92 | Found 93 | end, 94 | {reply, {ok, ReqID, Reply}, State}; 95 | 96 | handle_command({set, ReqID, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 97 | Stats = dict:store(StatName, Val, Stats0), 98 | {reply, {ok, ReqID}, State#state{stats=Stats}}; 99 | 100 | handle_command({incr, ReqID, StatName}, _Sender, #state{stats=Stats0}=State) -> 101 | Stats = dict:update_counter(StatName, 1, Stats0), 102 | {reply, {ok, ReqID}, State#state{stats=Stats}}; 103 | 104 | handle_command({incrby, ReqID, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 105 | Stats = dict:update_counter(StatName, Val, Stats0), 106 | {reply, {ok, ReqID}, State#state{stats=Stats}}; 107 | 108 | handle_command({append, ReqID, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 109 | Stats = try dict:append(StatName, Val, Stats0) 110 | catch _:_ -> dict:store(StatName, [Val], Stats0) 111 | end, 112 | {reply, {ok, ReqID}, State#state{stats=Stats}}; 113 | 114 | handle_command({sadd, ReqID, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 115 | F = fun(S) -> 116 | sets:add_element(Val, S) 117 | end, 118 | Stats = dict:update(StatName, F, sets:from_list([Val]), Stats0), 119 | {reply, {ok, ReqID}, State#state{stats=Stats}}. 120 | 121 | handle_handoff_command(?FOLD_REQ{foldfun=Fun, acc0=Acc0}, _Sender, State) -> 122 | Acc = dict:fold(Fun, Acc0, State#state.stats), 123 | {reply, Acc, State}. 124 | 125 | handoff_starting(_TargetNode, _State) -> 126 | {true, _State}. 127 | 128 | handoff_cancelled(State) -> 129 | {ok, State}. 130 | 131 | handoff_finished(_TargetNode, State) -> 132 | {ok, State}. 133 | 134 | handle_handoff_data(Data, #state{stats=Stats0}=State) -> 135 | {StatName, Val} = binary_to_term(Data), 136 | Stats = dict:store(StatName, Val, Stats0), 137 | {reply, ok, State#state{stats=Stats}}. 138 | 139 | encode_handoff_item(StatName, Val) -> 140 | term_to_binary({StatName,Val}). 141 | 142 | is_empty(State) -> 143 | case dict:size(State#state.stats) of 144 | 0 -> {true, State}; 145 | _ -> {false, State} 146 | end. 147 | 148 | delete(State) -> 149 | {ok, State}. 150 | 151 | handle_coverage(_Req, _KeySpaces, _Sender, State) -> 152 | {stop, not_implemented, State}. 153 | 154 | handle_exit(_Pid, _Reason, State) -> 155 | {noreply, State}. 156 | 157 | terminate(_Reason, _State) -> 158 | ok. 159 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_sup.erl: -------------------------------------------------------------------------------- 1 | -module(rts_sup). 2 | 3 | -behaviour(supervisor). 4 | 5 | %% API 6 | -export([start_link/0]). 7 | 8 | %% Supervisor callbacks 9 | -export([init/1]). 10 | 11 | %% =================================================================== 12 | %% API functions 13 | %% =================================================================== 14 | 15 | start_link() -> 16 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 17 | 18 | %% =================================================================== 19 | %% Supervisor callbacks 20 | %% =================================================================== 21 | 22 | init(_Args) -> 23 | VMaster = { rts_vnode_master, 24 | {riak_core_vnode_master, start_link, [rts_vnode]}, 25 | permanent, 5000, worker, [riak_core_vnode_master]}, 26 | 27 | Entry = {rts_entry_vnode_master, 28 | {riak_core_vnode_master, start_link, [rts_entry_vnode]}, 29 | permanent, 5000, worker, [riak_core_vnode_master]}, 30 | 31 | Stat = {rts_stat_vnode_master, 32 | {riak_core_vnode_master, start_link, [rts_stat_vnode]}, 33 | permanent, 5000, worker, [riak_core_vnode_master]}, 34 | 35 | WriteFSMs = {rts_write_fsm_sup, 36 | {rts_write_fsm_sup, start_link, []}, 37 | permanent, infinity, supervisor, [rts_write_fsm_sup]}, 38 | 39 | GetFSMs = {rts_get_fsm_sup, 40 | {rts_get_fsm_sup, start_link, []}, 41 | permanent, infinity, supervisor, [rts_get_fsm_sup]}, 42 | 43 | {ok, 44 | {{one_for_one, 5, 10}, 45 | [VMaster, Entry, Stat, WriteFSMs, GetFSMs]}}. 46 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_vnode.erl: -------------------------------------------------------------------------------- 1 | -module(rts_vnode). 2 | -behaviour(riak_core_vnode). 3 | -include("rts.hrl"). 4 | 5 | -export([start_vnode/1, 6 | init/1, 7 | terminate/2, 8 | handle_command/3, 9 | is_empty/1, 10 | delete/1, 11 | handle_handoff_command/3, 12 | handoff_starting/2, 13 | handoff_cancelled/1, 14 | handoff_finished/2, 15 | handle_handoff_data/2, 16 | encode_handoff_item/2, 17 | handle_coverage/4, 18 | handle_exit/3]). 19 | 20 | -record(state, {partition}). 21 | 22 | %% API 23 | start_vnode(I) -> 24 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 25 | 26 | init([Partition]) -> 27 | {ok, #state { partition=Partition }}. 28 | 29 | % Sample command: respond to a ping 30 | handle_command(ping, _Sender, State) -> 31 | {reply, {pong, State#state.partition}, State}; 32 | handle_command(Message, _Sender, State) -> 33 | ?PRINT({unhandled_command, Message}), 34 | {noreply, State}. 35 | 36 | handle_handoff_command(_Message, _Sender, State) -> 37 | {noreply, State}. 38 | 39 | handoff_starting(_TargetNode, State) -> 40 | {true, State}. 41 | 42 | handoff_cancelled(State) -> 43 | {ok, State}. 44 | 45 | handoff_finished(_TargetNode, State) -> 46 | {ok, State}. 47 | 48 | handle_handoff_data(_Data, State) -> 49 | {reply, ok, State}. 50 | 51 | encode_handoff_item(_ObjectName, _ObjectValue) -> 52 | <<>>. 53 | 54 | is_empty(State) -> 55 | {true, State}. 56 | 57 | delete(State) -> 58 | {ok, State}. 59 | 60 | handle_exit(_Pid, _Reason, _State) -> 61 | {noreply, _State}. 62 | 63 | handle_coverage(_Req, _KeySpaces, _Sender, State) -> 64 | {stop, not_implemented, State}. 65 | 66 | terminate(_Reason, _State) -> 67 | ok. 68 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_wm_entry.erl: -------------------------------------------------------------------------------- 1 | %% @doc Webmachine resource to handle incoming log entries. 2 | %% 3 | %% == PUT /rts/entry/Client == 4 | %% 5 | %% Write an entry for the given Client. 6 | %% 7 | %% Content-type must be 'text/plain' and the body should be _one_ 8 | %% line. 9 | -module(rts_wm_entry). 10 | -export([ 11 | init/1, 12 | allowed_methods/2, 13 | process_post/2 14 | ]). 15 | 16 | -include_lib("webmachine/include/webmachine.hrl"). 17 | 18 | init(_Props) -> 19 | {ok, none}. 20 | 21 | allowed_methods(_RD, _Ctx) -> 22 | {['POST'], _RD, _Ctx}. 23 | 24 | process_post(RD, _Ctx) -> 25 | case wrq:get_req_header("content-type", RD) of 26 | "text/plain" -> 27 | Client = wrq:path_info(client, RD), 28 | Entry = binary_to_list(wrq:req_body(RD)), 29 | ok = rts:entry(Client, Entry), 30 | {true, RD, _Ctx}; 31 | _ -> 32 | {{halt, 415}, RD, _Ctx} 33 | end. 34 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_write_fsm.erl: -------------------------------------------------------------------------------- 1 | %% @doc The coordinator for stat write opeartions. This example will 2 | %% show how to properly replicate your data in Riak Core by making use 3 | %% of the _preflist_. 4 | -module(rts_write_fsm). 5 | -behavior(gen_fsm). 6 | -include("rts.hrl"). 7 | 8 | %% API 9 | -export([start_link/5, start_link/6, write/3, write/4]). 10 | 11 | %% Callbacks 12 | -export([init/1, code_change/4, handle_event/3, handle_info/3, 13 | handle_sync_event/4, terminate/3]). 14 | 15 | %% States 16 | -export([prepare/2, execute/2, waiting/2]). 17 | 18 | %% req_id: The request id so the caller can verify the response. 19 | %% 20 | %% sender: The pid of the sender so a reply can be made. 21 | %% 22 | %% client: The external entity that wrote the log entry that produced 23 | %% this stat. 24 | %% 25 | %% state_name: The name of the statistic. 26 | %% 27 | %% op: The stat op, one of [set, incr, incr_by, append, sadd] 28 | %% 29 | %% prelist: The preflist for the given {Client, StatName} pair. 30 | %% 31 | %% num_w: The number of successful write replies. 32 | -record(state, {req_id :: pos_integer(), 33 | from :: pid(), 34 | client :: string(), 35 | stat_name :: string(), 36 | op :: atom(), 37 | val = undefined :: term() | undefined, 38 | preflist :: riak_core_apl:preflist2(), 39 | num_w = 0 :: non_neg_integer()}). 40 | 41 | %%%=================================================================== 42 | %%% API 43 | %%%=================================================================== 44 | 45 | start_link(ReqID, From, Client, StatName, Op) -> 46 | start_link(ReqID, From, Client, StatName, Op, undefined). 47 | 48 | start_link(ReqID, From, Client, StatName, Op, Val) -> 49 | gen_fsm:start_link(?MODULE, [ReqID, From, Client, StatName, Op, Val], []). 50 | 51 | write(Client, StatName, Op) -> 52 | write(Client, StatName, Op, undefined). 53 | 54 | write(Client, StatName, Op, Val) -> 55 | ReqID = mk_reqid(), 56 | rts_write_fsm_sup:start_write_fsm([ReqID, self(), Client, StatName, Op, Val]), 57 | {ok, ReqID}. 58 | 59 | %%%=================================================================== 60 | %%% States 61 | %%%=================================================================== 62 | 63 | %% @doc Initialize the state data. 64 | init([ReqID, From, Client, StatName, Op, Val]) -> 65 | SD = #state{req_id=ReqID, 66 | from=From, 67 | client=Client, 68 | stat_name=StatName, 69 | op=Op, 70 | val=Val}, 71 | {ok, prepare, SD, 0}. 72 | 73 | %% @doc Prepare the write by calculating the _preference list_. 74 | prepare(timeout, SD0=#state{client=Client, 75 | stat_name=StatName}) -> 76 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 77 | list_to_binary(StatName)}), 78 | Preflist = riak_core_apl:get_apl(DocIdx, ?N, rts_stat), 79 | SD = SD0#state{preflist=Preflist}, 80 | {next_state, execute, SD, 0}. 81 | 82 | %% @doc Execute the write request and then go into waiting state to 83 | %% verify it has meets consistency requirements. 84 | execute(timeout, SD0=#state{req_id=ReqID, 85 | stat_name=StatName, 86 | op=Op, 87 | val=Val, 88 | preflist=Preflist}) -> 89 | case Val of 90 | undefined -> 91 | rts_stat_vnode:Op(Preflist, ReqID, StatName); 92 | _ -> 93 | rts_stat_vnode:Op(Preflist, ReqID, StatName, Val) 94 | end, 95 | {next_state, waiting, SD0}. 96 | 97 | %% @doc Wait for W write reqs to respond. 98 | waiting({ok, ReqID}, SD0=#state{from=From, num_w=NumW0}) -> 99 | NumW = NumW0 + 1, 100 | SD = SD0#state{num_w=NumW}, 101 | if 102 | NumW =:= ?W -> 103 | From ! {ReqID, ok}, 104 | {stop, normal, SD}; 105 | true -> {next_state, waiting, SD} 106 | end. 107 | 108 | handle_info(_Info, _StateName, StateData) -> 109 | {stop,badmsg,StateData}. 110 | 111 | handle_event(_Event, _StateName, StateData) -> 112 | {stop,badmsg,StateData}. 113 | 114 | handle_sync_event(_Event, _From, _StateName, StateData) -> 115 | {stop,badmsg,StateData}. 116 | 117 | code_change(_OldVsn, StateName, State, _Extra) -> {ok, StateName, State}. 118 | 119 | terminate(_Reason, _SN, _SD) -> 120 | ok. 121 | 122 | %%%=================================================================== 123 | %%% Internal Functions 124 | %%%=================================================================== 125 | 126 | mk_reqid() -> erlang:phash2(erlang:now()). 127 | -------------------------------------------------------------------------------- /2011/riak-core-the-coordinator/rts/src/rts_write_fsm_sup.erl: -------------------------------------------------------------------------------- 1 | %% @doc Supervise the rts_write FSM. 2 | -module(rts_write_fsm_sup). 3 | -behavior(supervisor). 4 | 5 | -export([start_write_fsm/1, 6 | start_link/0]). 7 | -export([init/1]). 8 | 9 | start_write_fsm(Args) -> 10 | supervisor:start_child(?MODULE, Args). 11 | 12 | start_link() -> 13 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 14 | 15 | init([]) -> 16 | WriteFsm = {undefined, 17 | {rts_write_fsm, start_link, []}, 18 | temporary, 5000, worker, [rts_write_fsm]}, 19 | {ok, {{simple_one_for_one, 10, 10}, [WriteFsm]}}. 20 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/meta: -------------------------------------------------------------------------------- 1 | published 2011-04-07 -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/.gitignore: -------------------------------------------------------------------------------- 1 | *.beam 2 | .eunit 3 | deps/* 4 | ebin 5 | rel/rts 6 | dev 7 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: deps 2 | 3 | all: deps compile 4 | 5 | compile: 6 | ./rebar compile 7 | 8 | deps: 9 | ./rebar get-deps 10 | 11 | clean: 12 | ./rebar clean 13 | 14 | distclean: clean devclean relclean 15 | ./rebar delete-deps 16 | 17 | rel: all 18 | ./rebar generate 19 | 20 | relclean: 21 | rm -rf rel/rts 22 | 23 | devrel: dev1 dev2 dev3 24 | 25 | dev1 dev2 dev3: 26 | mkdir -p dev 27 | (cd rel && ../rebar generate target_dir=../dev/$@ overlay_vars=vars/$@.config) 28 | 29 | devclean: 30 | rm -rf dev -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/progski.access.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-core-the-vnode/rts/progski.access.log.gz -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-core-the-vnode/rts/rebar -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rebar.config: -------------------------------------------------------------------------------- 1 | {erl_opts, [debug_info, fail_on_warning]}. 2 | 3 | {sub_dirs, ["rel"]}. 4 | 5 | {deps, [{riak_core, "1.0.*", 6 | {git, "git://github.com/basho/riak_core", {tag,"1.0.0"}}} 7 | ]}. 8 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/files/app.config: -------------------------------------------------------------------------------- 1 | [ 2 | %% Riak Core config 3 | {riak_core, [ 4 | %% Default location of ringstate 5 | {ring_state_dir, "{{ring_state_dir}}"}, 6 | 7 | %% http is a list of IP addresses and TCP ports that the Riak 8 | %% HTTP interface will bind. 9 | {http, [ {"{{web_ip}}", {{web_port}} } ]}, 10 | 11 | %% riak_handoff_port is the TCP port that Riak uses for 12 | %% intra-cluster data handoff. 13 | {handoff_port, {{handoff_port}} } 14 | ]}, 15 | 16 | %% SASL config 17 | {sasl, [ 18 | {sasl_error_logger, {file, "log/sasl-error.log"}}, 19 | {errlog_type, error}, 20 | {error_logger_mf_dir, "log/sasl"}, % Log directory 21 | {error_logger_mf_maxbytes, 10485760}, % 10 MB max file size 22 | {error_logger_mf_maxfiles, 5} % 5 files max 23 | ]} 24 | ]. 25 | 26 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/files/erl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## This script replaces the default "erl" in erts-VSN/bin. This is necessary 4 | ## as escript depends on erl and in turn, erl depends on having access to a 5 | ## bootscript (start.boot). Note that this script is ONLY invoked as a side-effect 6 | ## of running escript -- the embedded node bypasses erl and uses erlexec directly 7 | ## (as it should). 8 | ## 9 | ## Note that this script makes the assumption that there is a start_clean.boot 10 | ## file available in $ROOTDIR/release/VSN. 11 | 12 | # Determine the abspath of where this script is executing from. 13 | ERTS_BIN_DIR=$(cd ${0%/*} && pwd) 14 | 15 | # Now determine the root directory -- this script runs from erts-VSN/bin, 16 | # so we simply need to strip off two dirs from the end of the ERTS_BIN_DIR 17 | # path. 18 | ROOTDIR=${ERTS_BIN_DIR%/*/*} 19 | 20 | # Parse out release and erts info 21 | START_ERL=`cat $ROOTDIR/releases/start_erl.data` 22 | ERTS_VSN=${START_ERL% *} 23 | APP_VSN=${START_ERL#* } 24 | 25 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 26 | EMU=beam 27 | PROGNAME=`echo $0 | sed 's/.*\\///'` 28 | CMD="$BINDIR/erlexec" 29 | export EMU 30 | export ROOTDIR 31 | export BINDIR 32 | export PROGNAME 33 | 34 | exec $CMD -boot $ROOTDIR/releases/$APP_VSN/start_clean ${1+"$@"} -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/files/nodetool: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- 2 | %% ex: ft=erlang ts=4 sw=4 et 3 | %% ------------------------------------------------------------------- 4 | %% 5 | %% nodetool: Helper Script for interacting with live nodes 6 | %% 7 | %% ------------------------------------------------------------------- 8 | 9 | main(Args) -> 10 | ok = start_epmd(), 11 | %% Extract the args 12 | {RestArgs, TargetNode} = process_args(Args, [], undefined), 13 | 14 | %% See if the node is currently running -- if it's not, we'll bail 15 | case {net_kernel:hidden_connect_node(TargetNode), net_adm:ping(TargetNode)} of 16 | {true, pong} -> 17 | ok; 18 | {_, pang} -> 19 | io:format("Node ~p not responding to pings.\n", [TargetNode]), 20 | halt(1) 21 | end, 22 | 23 | case RestArgs of 24 | ["ping"] -> 25 | %% If we got this far, the node already responsed to a ping, so just dump 26 | %% a "pong" 27 | io:format("pong\n"); 28 | ["stop"] -> 29 | io:format("~p\n", [rpc:call(TargetNode, init, stop, [], 60000)]); 30 | ["restart"] -> 31 | io:format("~p\n", [rpc:call(TargetNode, init, restart, [], 60000)]); 32 | ["reboot"] -> 33 | io:format("~p\n", [rpc:call(TargetNode, init, reboot, [], 60000)]); 34 | ["rpc", Module, Function | RpcArgs] -> 35 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), 36 | [RpcArgs], 60000) of 37 | ok -> 38 | ok; 39 | {badrpc, Reason} -> 40 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), 41 | halt(1); 42 | _ -> 43 | halt(1) 44 | end; 45 | ["rpcterms", Module, Function, ArgsAsString] -> 46 | case rpc:call(TargetNode, list_to_atom(Module), list_to_atom(Function), 47 | consult(ArgsAsString), 60000) of 48 | {badrpc, Reason} -> 49 | io:format("RPC to ~p failed: ~p\n", [TargetNode, Reason]), 50 | halt(1); 51 | Other -> 52 | io:format("~p\n", [Other]) 53 | end; 54 | Other -> 55 | io:format("Other: ~p\n", [Other]), 56 | io:format("Usage: nodetool {ping|stop|restart|reboot}\n") 57 | end, 58 | net_kernel:stop(). 59 | 60 | process_args([], Acc, TargetNode) -> 61 | {lists:reverse(Acc), TargetNode}; 62 | process_args(["-setcookie", Cookie | Rest], Acc, TargetNode) -> 63 | erlang:set_cookie(node(), list_to_atom(Cookie)), 64 | process_args(Rest, Acc, TargetNode); 65 | process_args(["-name", TargetName | Rest], Acc, _) -> 66 | ThisNode = append_node_suffix(TargetName, "_maint_"), 67 | {ok, _} = net_kernel:start([ThisNode, longnames]), 68 | process_args(Rest, Acc, nodename(TargetName)); 69 | process_args(["-sname", TargetName | Rest], Acc, _) -> 70 | ThisNode = append_node_suffix(TargetName, "_maint_"), 71 | {ok, _} = net_kernel:start([ThisNode, shortnames]), 72 | process_args(Rest, Acc, nodename(TargetName)); 73 | process_args([Arg | Rest], Acc, Opts) -> 74 | process_args(Rest, [Arg | Acc], Opts). 75 | 76 | 77 | start_epmd() -> 78 | [] = os:cmd(epmd_path() ++ " -daemon"), 79 | ok. 80 | 81 | epmd_path() -> 82 | ErtsBinDir = filename:dirname(escript:script_name()), 83 | Name = "epmd", 84 | case os:find_executable(Name, ErtsBinDir) of 85 | false -> 86 | case os:find_executable(Name) of 87 | false -> 88 | io:format("Could not find epmd.~n"), 89 | halt(1); 90 | GlobalEpmd -> 91 | GlobalEpmd 92 | end; 93 | Epmd -> 94 | Epmd 95 | end. 96 | 97 | 98 | nodename(Name) -> 99 | case string:tokens(Name, "@") of 100 | [_Node, _Host] -> 101 | list_to_atom(Name); 102 | [Node] -> 103 | [_, Host] = string:tokens(atom_to_list(node()), "@"), 104 | list_to_atom(lists:concat([Node, "@", Host])) 105 | end. 106 | 107 | append_node_suffix(Name, Suffix) -> 108 | case string:tokens(Name, "@") of 109 | [Node, Host] -> 110 | list_to_atom(lists:concat([Node, Suffix, os:getpid(), "@", Host])); 111 | [Node] -> 112 | list_to_atom(lists:concat([Node, Suffix, os:getpid()])) 113 | end. 114 | 115 | 116 | %% 117 | %% Given a string or binary, parse it into a list of terms, ala file:consult/0 118 | %% 119 | consult(Str) when is_list(Str) -> 120 | consult([], Str, []); 121 | consult(Bin) when is_binary(Bin)-> 122 | consult([], binary_to_list(Bin), []). 123 | 124 | consult(Cont, Str, Acc) -> 125 | case erl_scan:tokens(Cont, Str, 0) of 126 | {done, Result, Remaining} -> 127 | case Result of 128 | {ok, Tokens, _} -> 129 | {ok, Term} = erl_parse:parse_term(Tokens), 130 | consult([], Remaining, [Term | Acc]); 131 | {eof, _Other} -> 132 | lists:reverse(Acc); 133 | {error, Info, _} -> 134 | {error, Info} 135 | end; 136 | {more, Cont1} -> 137 | consult(Cont1, eof, Acc) 138 | end. 139 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/files/rts: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # -*- tab-width:4;indent-tabs-mode:nil -*- 3 | # ex: ts=4 sw=4 et 4 | 5 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd) 6 | 7 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*} 8 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc 9 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log 10 | # Note the trailing slash on $PIPE_DIR/ 11 | PIPE_DIR=/tmp/$RUNNER_BASE_DIR/ 12 | RUNNER_USER= 13 | 14 | # Make sure this script is running as the appropriate user 15 | if [ ! -z "$RUNNER_USER" ] && [ `whoami` != "$RUNNER_USER" ]; then 16 | exec sudo -u $RUNNER_USER -i $0 $@ 17 | fi 18 | 19 | # Make sure CWD is set to runner base dir 20 | cd $RUNNER_BASE_DIR 21 | 22 | # Make sure log directory exists 23 | mkdir -p $RUNNER_LOG_DIR 24 | 25 | # Extract the target node name from node.args 26 | NAME_ARG=`grep -e '-[s]*name' $RUNNER_ETC_DIR/vm.args` 27 | if [ -z "$NAME_ARG" ]; then 28 | echo "vm.args needs to have either -name or -sname parameter." 29 | exit 1 30 | fi 31 | 32 | # Extract the target cookie 33 | COOKIE_ARG=`grep -e '-setcookie' $RUNNER_ETC_DIR/vm.args` 34 | if [ -z "$COOKIE_ARG" ]; then 35 | echo "vm.args needs to have a -setcookie parameter." 36 | exit 1 37 | fi 38 | 39 | # Identify the script name 40 | SCRIPT=`basename $0` 41 | 42 | # Parse out release and erts info 43 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` 44 | ERTS_VSN=${START_ERL% *} 45 | APP_VSN=${START_ERL#* } 46 | 47 | # Add ERTS bin dir to our path 48 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin 49 | 50 | # Setup command to control the node 51 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" 52 | 53 | # Check the first argument for instructions 54 | case "$1" in 55 | start) 56 | # Make sure there is not already a node running 57 | RES=`$NODETOOL ping` 58 | if [ "$RES" = "pong" ]; then 59 | echo "Node is already running!" 60 | exit 1 61 | fi 62 | HEART_COMMAND="$RUNNER_BASE_DIR/bin/$SCRIPT start" 63 | export HEART_COMMAND 64 | mkdir -p $PIPE_DIR 65 | shift # remove $1 66 | $ERTS_PATH/run_erl -daemon $PIPE_DIR $RUNNER_LOG_DIR "exec $RUNNER_BASE_DIR/bin/$SCRIPT console $@" 2>&1 67 | ;; 68 | 69 | stop) 70 | # Wait for the node to completely stop... 71 | case `uname -s` in 72 | Linux|Darwin|FreeBSD|DragonFly|NetBSD|OpenBSD) 73 | # PID COMMAND 74 | PID=`ps ax -o pid= -o command=|\ 75 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'` 76 | ;; 77 | SunOS) 78 | # PID COMMAND 79 | PID=`ps -ef -o pid= -o args=|\ 80 | grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $1}'` 81 | ;; 82 | CYGWIN*) 83 | # UID PID PPID TTY STIME COMMAND 84 | PID=`ps -efW|grep "$RUNNER_BASE_DIR/.*/[b]eam"|awk '{print $2}'` 85 | ;; 86 | esac 87 | $NODETOOL stop 88 | while `kill -0 $PID 2>/dev/null`; 89 | do 90 | sleep 1 91 | done 92 | ;; 93 | 94 | restart) 95 | ## Restart the VM without exiting the process 96 | $NODETOOL restart 97 | ;; 98 | 99 | reboot) 100 | ## Restart the VM completely (uses heart to restart it) 101 | $NODETOOL reboot 102 | ;; 103 | 104 | ping) 105 | ## See if the VM is alive 106 | $NODETOOL ping 107 | ;; 108 | 109 | attach) 110 | # Make sure a node IS running 111 | RES=`$NODETOOL ping` 112 | if [ "$RES" != "pong" ]; then 113 | echo "Node is not running!" 114 | exit 1 115 | fi 116 | 117 | shift 118 | $ERTS_PATH/to_erl $PIPE_DIR 119 | ;; 120 | 121 | console|console_clean) 122 | # .boot file typically just $SCRIPT (ie, the app name) 123 | # however, for debugging, sometimes start_clean.boot is useful: 124 | case "$1" in 125 | console) BOOTFILE=$SCRIPT ;; 126 | console_clean) BOOTFILE=start_clean ;; 127 | esac 128 | # Setup beam-required vars 129 | ROOTDIR=$RUNNER_BASE_DIR 130 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 131 | EMU=beam 132 | PROGNAME=`echo $0 | sed 's/.*\\///'` 133 | CMD="$BINDIR/erlexec -boot $RUNNER_BASE_DIR/releases/$APP_VSN/$BOOTFILE -embedded -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -- ${1+"$@"}" 134 | export EMU 135 | export ROOTDIR 136 | export BINDIR 137 | export PROGNAME 138 | 139 | # Dump environment info for logging purposes 140 | echo "Exec: $CMD" 141 | echo "Root: $ROOTDIR" 142 | 143 | # Log the startup 144 | logger -t "$SCRIPT[$$]" "Starting up" 145 | 146 | # Start the VM 147 | exec $CMD 148 | ;; 149 | 150 | *) 151 | echo "Usage: $SCRIPT {start|stop|restart|reboot|ping|console|console_clean|attach}" 152 | exit 1 153 | ;; 154 | esac 155 | 156 | exit 0 157 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/files/rts-admin: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | RUNNER_SCRIPT_DIR=$(cd ${0%/*} && pwd) 4 | RUNNER_SCRIPT=${0##*/} 5 | 6 | RUNNER_BASE_DIR=${RUNNER_SCRIPT_DIR%/*} 7 | RUNNER_ETC_DIR=$RUNNER_BASE_DIR/etc 8 | RUNNER_LOG_DIR=$RUNNER_BASE_DIR/log 9 | RUNNER_USER= 10 | 11 | # Make sure this script is running as the appropriate user 12 | if [ "$RUNNER_USER" -a "x$LOGNAME" != "x$RUNNER_USER" ]; then 13 | type -p sudo > /dev/null 2>&1 14 | if [ $? -ne 0 ]; then 15 | echo "sudo doesn't appear to be installed and your EUID isn't $RUNNER_USER" 1>&2 16 | exit 1 17 | fi 18 | echo "Attempting to restart script through sudo -u $RUNNER_USER" 19 | exec sudo -u $RUNNER_USER -i $RUNNER_SCRIPT_DIR/$RUNNER_SCRIPT $@ 20 | fi 21 | 22 | # Make sure CWD is set to runner base dir 23 | cd $RUNNER_BASE_DIR 24 | 25 | # Extract the target node name from node.args 26 | NAME_ARG=`grep -e '-[s]*name' $RUNNER_ETC_DIR/vm.args` 27 | if [ -z "$NAME_ARG" ]; then 28 | echo "vm.args needs to have either -name or -sname parameter." 29 | exit 1 30 | fi 31 | 32 | # Learn how to specify node name for connection from remote nodes 33 | echo "$NAME_ARG" | grep '^-sname' > /dev/null 2>&1 34 | if [ "X$?" = "X0" ]; then 35 | NAME_PARAM="-sname" 36 | NAME_HOST="" 37 | else 38 | NAME_PARAM="-name" 39 | echo "$NAME_ARG" | grep '@.*' > /dev/null 2>&1 40 | if [ "X$?" = "X0" ]; then 41 | NAME_HOST=`echo "${NAME_ARG}" | sed -e 's/.*(@.*)$//'` 42 | else 43 | NAME_HOST="" 44 | fi 45 | fi 46 | 47 | # Extract the target cookie 48 | COOKIE_ARG=`grep '-setcookie' $RUNNER_ETC_DIR/vm.args` 49 | if [ -z "$COOKIE_ARG" ]; then 50 | echo "vm.args needs to have a -setcookie parameter." 51 | exit 1 52 | fi 53 | 54 | # Identify the script name 55 | SCRIPT=`basename $0` 56 | 57 | # Parse out release and erts info 58 | START_ERL=`cat $RUNNER_BASE_DIR/releases/start_erl.data` 59 | ERTS_VSN=${START_ERL% *} 60 | APP_VSN=${START_ERL#* } 61 | 62 | # Add ERTS bin dir to our path 63 | ERTS_PATH=$RUNNER_BASE_DIR/erts-$ERTS_VSN/bin 64 | 65 | # Setup command to control the node 66 | NODETOOL="$ERTS_PATH/escript $ERTS_PATH/nodetool $NAME_ARG $COOKIE_ARG" 67 | 68 | # Check the first argument for instructions 69 | case "$1" in 70 | join) 71 | # Make sure the local node IS running 72 | RES=`$NODETOOL ping` 73 | if [ "$RES" != "pong" ]; then 74 | echo "Node is not running!" 75 | exit 1 76 | fi 77 | 78 | shift 79 | 80 | $NODETOOL rpc rts_console join $@ 81 | ;; 82 | 83 | leave) 84 | # Make sure the local node is running 85 | RES=`$NODETOOL ping` 86 | if [ "$RES" != "pong" ]; then 87 | echo "Node is not running!" 88 | exit 1 89 | fi 90 | 91 | shift 92 | $NODETOOL rpc rts__console leave $@ 93 | ;; 94 | 95 | remove) 96 | if [ $# -ne 2 ]; then 97 | echo "Usage: $SCRIPT remove " 98 | exit 1 99 | fi 100 | 101 | RES=`$NODETOOL ping` 102 | if [ "$RES" != "pong" ]; then 103 | echo "Node is not running!" 104 | exit 1 105 | fi 106 | 107 | shift 108 | $NODETOOL rpc rts_console remove $@ 109 | ;; 110 | 111 | ringready) 112 | # Make sure the local node IS running 113 | RES=`$NODETOOL ping` 114 | if [ "$RES" != "pong" ]; then 115 | echo "Node is not running!" 116 | exit 1 117 | fi 118 | shift 119 | 120 | $NODETOOL rpc rts_console ringready $@ 121 | ;; 122 | 123 | *) 124 | echo "Usage: $SCRIPT { join | leave | reip | ringready | remove }" 125 | exit 1 126 | ;; 127 | esac 128 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/files/vm.args: -------------------------------------------------------------------------------- 1 | 2 | ## Name of the node 3 | -name {{node}} 4 | 5 | ## Cookie for distributed erlang 6 | -setcookie rts 7 | 8 | ## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive 9 | ## (Disabled by default..use with caution!) 10 | ##-heart 11 | 12 | ## Enable kernel poll and a few async threads 13 | +K true 14 | +A 5 15 | 16 | ## Increase number of concurrent ports/sockets 17 | -env ERL_MAX_PORTS 4096 18 | 19 | ## Tweak GC to run more often 20 | -env ERL_FULLSWEEP_AFTER 10 21 | 22 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/reltool.config: -------------------------------------------------------------------------------- 1 | {sys, [ 2 | {lib_dirs, ["../../", "../deps/"]}, 3 | {rel, "rts", "1", 4 | [ 5 | kernel, 6 | stdlib, 7 | sasl, 8 | rts 9 | ]}, 10 | {rel, "start_clean", "", 11 | [ 12 | kernel, 13 | stdlib 14 | ]}, 15 | {boot_rel, "rts"}, 16 | {profile, embedded}, 17 | {excl_sys_filters, ["^bin/.*", 18 | "^erts.*/bin/(dialyzer|typer)"]}, 19 | {app, sasl, [{incl_cond, include}]}, 20 | {app, rts, [{incl_cond, include}]} 21 | ]}. 22 | 23 | {target_dir, "rts"}. 24 | 25 | {overlay_vars, "vars.config"}. 26 | 27 | {overlay, [ 28 | {mkdir, "data/ring"}, 29 | {mkdir, "log/sasl"}, 30 | {copy, "files/erl", "{{erts_vsn}}/bin/erl"}, 31 | {copy, "files/nodetool", "{{erts_vsn}}/bin/nodetool"}, 32 | {template, "files/app.config", "etc/app.config"}, 33 | {template, "files/vm.args", "etc/vm.args"}, 34 | {template, "files/rts", "bin/rts"}, 35 | {template, "files/rts-admin", "bin/rts-admin"} 36 | ]}. 37 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/vars.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8888"}. 7 | {handoff_port, "8099"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/vars/dev1.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8881"}. 7 | {handoff_port, "8101"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts1@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/vars/dev2.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8882"}. 7 | {handoff_port, "8102"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts2@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/rel/vars/dev3.config: -------------------------------------------------------------------------------- 1 | %% 2 | %% etc/app.config 3 | %% 4 | {ring_state_dir, "data/ring"}. 5 | {web_ip, "127.0.0.1"}. 6 | {web_port, "8883"}. 7 | {handoff_port, "8103"}. 8 | 9 | %% 10 | %% etc/vm.args 11 | %% 12 | {node, "rts3@127.0.0.1"}. 13 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/replay: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # Usage: 4 | # gunzip -c access.log.gz | ./replay [--devrel] client 5 | # ./replay [--devrel] client < access.log 6 | 7 | PORTS[0]=8888 8 | MAX=0 9 | 10 | while [ $# -gt 0 ] 11 | do 12 | case $1 in 13 | --devrel | -d ) 14 | PORTS[0]=8881 15 | PORTS[1]=8882 16 | PORTS[2]=8883 17 | MAX=2 18 | ;; 19 | *) 20 | break 21 | ;; 22 | esac 23 | shift 24 | done 25 | 26 | CLIENT=$1 27 | shift 28 | 29 | node=0 30 | while read line 31 | do 32 | curl -X POST \ 33 | -H 'content-type: text/plain' \ 34 | "http://localhost:${PORTS[$node]}/rts/entry/$CLIENT" \ 35 | -d "$line" 36 | 37 | if [ $node -eq $MAX ] 38 | then 39 | node=0 40 | else 41 | ((node++)) 42 | fi 43 | done -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts.app.src: -------------------------------------------------------------------------------- 1 | {application, rts, 2 | [ 3 | {description, ""}, 4 | {vsn, "1"}, 5 | {registered, []}, 6 | {applications, [ 7 | kernel, 8 | stdlib, 9 | riak_core 10 | ]}, 11 | {mod, { rts_app, []}}, 12 | {env, []} 13 | ]}. -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts.erl: -------------------------------------------------------------------------------- 1 | %% @doc Interface into the Real Time Statistics application. 2 | -module(rts). 3 | -include("rts.hrl"). 4 | -include_lib("riak_core/include/riak_core_vnode.hrl"). 5 | 6 | -export([ 7 | ping/0, 8 | entry/2, 9 | get/2, 10 | set/3, 11 | append/3, 12 | incr/2, 13 | incrby/3, 14 | sadd/3 15 | ]). 16 | 17 | %%%=================================================================== 18 | %%% API 19 | %%%=================================================================== 20 | 21 | % @doc Pings a random vnode to make sure communication is functional 22 | ping() -> 23 | DocIdx = riak_core_util:chash_key({<<"ping">>, term_to_binary(now())}), 24 | PrefList = riak_core_apl:get_primary_apl(DocIdx, 1, rts), 25 | [{IndexNode, _Type}] = PrefList, 26 | riak_core_vnode_master:sync_spawn_command(IndexNode, ping, rts_vnode_master). 27 | 28 | %% @doc Process an entry. 29 | %% 30 | %% TODO: Coordinator to provide N/R/W 31 | entry(Client, Entry) -> 32 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), 33 | term_to_binary(now())}), 34 | PrefList = riak_core_apl:get_apl(DocIdx, 1, rts_entry), 35 | [IdxNode] = PrefList, 36 | rts_entry_vnode:entry(IdxNode, Client, Entry). 37 | 38 | %% @doc Get a stat's value. 39 | get(Client, StatName) -> 40 | rts_stat_vnode:get(get_idxnode(Client, StatName), StatName). 41 | 42 | %% @doc Set a stat's value, replacing the current value. 43 | set(Client, StatName, Val) -> 44 | rts_stat_vnode:set(get_idxnode(Client, StatName), StatName, Val). 45 | 46 | %% @doc Append to a stat's value. 47 | append(Client, StatName, Val) -> 48 | rts_state_vnode:append(get_idxnode(Client, StatName), StatName, Val). 49 | 50 | %% @doc Increment the stat's value by 1. 51 | incr(Client, StatName) -> 52 | rts_stat_vnode:incr(get_idxnode(Client, StatName), StatName). 53 | 54 | %% @doc Increment the stat's value by Val. 55 | incrby(Client, StatName, Val) -> 56 | rts_stat_vnode:incrby(get_idxnode(Client, StatName), StatName, Val). 57 | 58 | %% @doc Add a memeber to the stat's set. 59 | sadd(Client, StatName, Val) -> 60 | rts_stat_vnode:sadd(get_idxnode(Client, StatName), StatName, Val). 61 | 62 | %%%=================================================================== 63 | %%% Internal Functions 64 | %%%=================================================================== 65 | 66 | get_idxnode(Client, StatName) -> 67 | DocIdx = riak_core_util:chash_key({list_to_binary(Client), list_to_binary(StatName)}), 68 | hd(riak_core_apl:get_apl(DocIdx, 1, rts_stat)). 69 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts.hrl: -------------------------------------------------------------------------------- 1 | -define(PRINT(Var), io:format("DEBUG: ~p:~p - ~p~n~n ~p~n~n", [?MODULE, ?LINE, ??Var, Var])). -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts_app.erl: -------------------------------------------------------------------------------- 1 | -module(rts_app). 2 | 3 | -behaviour(application). 4 | 5 | %% Application callbacks 6 | -export([start/2, stop/1]). 7 | 8 | %% =================================================================== 9 | %% Application callbacks 10 | %% =================================================================== 11 | 12 | start(_StartType, _StartArgs) -> 13 | case rts_sup:start_link() of 14 | {ok, Pid} -> 15 | ok = riak_core:register_vnode_module(rts_vnode), 16 | ok = riak_core_node_watcher:service_up(rts, self()), 17 | 18 | ok = riak_core:register_vnode_module(rts_entry_vnode), 19 | ok = riak_core_node_watcher:service_up(rts_entry, self()), 20 | 21 | ok = riak_core:register_vnode_module(rts_stat_vnode), 22 | ok = riak_core_node_watcher:service_up(rts_stat, self()), 23 | 24 | EntryRoute = {["rts", "entry", client], rts_wm_entry, []}, 25 | webmachine_router:add_route(EntryRoute), 26 | 27 | {ok, Pid}; 28 | {error, Reason} -> 29 | {error, Reason} 30 | end. 31 | 32 | stop(_State) -> 33 | ok. 34 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts_console.erl: -------------------------------------------------------------------------------- 1 | %% @doc Interface for rts-admin commands. 2 | -module(rts_console). 3 | -export([join/1, 4 | leave/1, 5 | remove/1, 6 | ringready/1]). 7 | 8 | join([NodeStr]) -> 9 | try 10 | case riak_core:join(NodeStr) of 11 | ok -> 12 | io:format("Sent join request to ~s\n", [NodeStr]), 13 | ok; 14 | {error, not_reachable} -> 15 | io:format("Node ~s is not reachable!\n", [NodeStr]), 16 | error; 17 | {error, different_ring_sizes} -> 18 | io:format("Failed: ~s has a different ring_creation_size~n", 19 | [NodeStr]), 20 | error 21 | end 22 | catch 23 | Exception:Reason -> 24 | lager:error("Join failed ~p:~p", [Exception, 25 | Reason]), 26 | io:format("Join failed, see log for details~n"), 27 | error 28 | end. 29 | 30 | leave([]) -> 31 | try 32 | case riak_core:leave() of 33 | ok -> 34 | io:format("Success: ~p will shutdown after handing off " 35 | "its data~n", [node()]), 36 | ok; 37 | {error, already_leaving} -> 38 | io:format("~p is already in the process of leaving the " 39 | "cluster.~n", [node()]), 40 | ok; 41 | {error, not_member} -> 42 | io:format("Failed: ~p is not a member of the cluster.~n", 43 | [node()]), 44 | error; 45 | {error, only_member} -> 46 | io:format("Failed: ~p is the only member.~n", [node()]), 47 | error 48 | end 49 | catch 50 | Exception:Reason -> 51 | lager:error("Leave failed ~p:~p", [Exception, Reason]), 52 | io:format("Leave failed, see log for details~n"), 53 | error 54 | end. 55 | 56 | remove([Node]) -> 57 | remove_node(list_to_atom(Node)). 58 | 59 | remove_node(Node) when is_atom(Node) -> 60 | try 61 | case catch(riak_core:remove_from_cluster(Node)) of 62 | {'EXIT', {badarg, [{erlang, hd, [[]]}|_]}} -> 63 | %% This is a workaround because 64 | %% riak_core_gossip:remove_from_cluster doesn't check if 65 | %% the result of subtracting the current node from the 66 | %% cluster member list results in the empty list. When 67 | %% that code gets refactored this can probably go away. 68 | io:format("Leave failed, this node is the only member.~n"), 69 | error; 70 | Res -> 71 | io:format(" ~p\n", [Res]) 72 | end 73 | catch 74 | Exception:Reason -> 75 | lager:error("Leave failed ~p:~p", [Exception, 76 | Reason]), 77 | io:format("Leave failed, see log for details~n"), 78 | error 79 | end. 80 | 81 | 82 | -spec(ringready([]) -> ok | error). 83 | ringready([]) -> 84 | try 85 | case riak_core_status:ringready() of 86 | {ok, Nodes} -> 87 | io:format("TRUE All nodes agree on the ring ~p\n", [Nodes]); 88 | {error, {different_owners, N1, N2}} -> 89 | io:format("FALSE Node ~p and ~p list different partition owners\n", [N1, N2]), 90 | error; 91 | {error, {nodes_down, Down}} -> 92 | io:format("FALSE ~p down. All nodes need to be up to check.\n", [Down]), 93 | error 94 | end 95 | catch 96 | Exception:Reason -> 97 | lager:error("Ringready failed ~p:~p", [Exception, 98 | Reason]), 99 | io:format("Ringready failed, see log for details~n"), 100 | error 101 | end. 102 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts_entry_vnode.erl: -------------------------------------------------------------------------------- 1 | %% @doc A vnode to crunch incoming log entries. Attempt to match each 2 | %% log entry against a registry of regexps. If a regexp matches then 3 | %% execute its corresponding trigger function passing it the {Client, 4 | %% Entry, Regexp} as well as the resulting match. The trigger 5 | %% function can then choose to take an action such as update a 6 | %% statistic via the `rts_stat_vnode'. 7 | %% 8 | %% Since this vnode is purely for computation there is no need to 9 | %% worry about handoff. 10 | -module(rts_entry_vnode). 11 | -behaviour(riak_core_vnode). 12 | -include("rts.hrl"). 13 | 14 | -export([start_vnode/1, 15 | init/1, 16 | terminate/2, 17 | handle_command/3, 18 | is_empty/1, 19 | delete/1, 20 | handle_handoff_command/3, 21 | handoff_starting/2, 22 | handoff_cancelled/1, 23 | handoff_finished/2, 24 | handle_handoff_data/2, 25 | encode_handoff_item/2, 26 | handle_coverage/4, 27 | handle_exit/3]). 28 | 29 | %% match handlers 30 | -export([ 31 | combined_lf/2 32 | ]). 33 | 34 | %% API 35 | -export([entry/3]). 36 | 37 | -record(state, { 38 | partition, 39 | reg %% registry [regexp => fun] 40 | }). 41 | 42 | -define(MASTER, rts_entry_vnode_master). 43 | -define(COMBINED_LF, "(\\d+\\.\\d+\\.\\d+\\.\\d+) (.*) (.*) (\\[.*\\]) \"(.*)\" (\\d+) (.*) \"(.*)\" \"(.*)\""). 44 | 45 | %%%=================================================================== 46 | %%% API 47 | %%%=================================================================== 48 | 49 | start_vnode(I) -> 50 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 51 | 52 | entry(IdxNode, Client, Entry) -> 53 | riak_core_vnode_master:command(IdxNode, 54 | {entry, Client, Entry}, 55 | ?MASTER). 56 | 57 | %%%=================================================================== 58 | %%% Callbacks 59 | %%%=================================================================== 60 | 61 | init([Partition]) -> 62 | Reg = [ 63 | {?COMBINED_LF, fun ?MODULE:combined_lf/2} 64 | ], 65 | {ok, #state { partition=Partition, reg=Reg }}. 66 | 67 | handle_command({entry, Client, Entry}, _Sender, #state{reg=Reg}=State) -> 68 | io:format("~p~n", [{entry, State#state.partition}]), 69 | lists:foreach(match(Client, Entry), Reg), 70 | {noreply, State}. 71 | 72 | handle_handoff_command(_Message, _Sender, State) -> 73 | {noreply, State}. 74 | 75 | handoff_starting(_TargetNode, _State) -> 76 | {true, _State}. 77 | 78 | handoff_cancelled(State) -> 79 | {ok, State}. 80 | 81 | handoff_finished(_TargetNode, State) -> 82 | {ok, State}. 83 | 84 | handle_handoff_data(_Data, State) -> 85 | {reply, ok, State}. 86 | 87 | encode_handoff_item(_ObjectName, _ObjectValue) -> 88 | <<>>. 89 | 90 | is_empty(State) -> 91 | {true, State}. 92 | 93 | delete(State) -> 94 | {ok, State}. 95 | 96 | handle_coverage(_Req, _KeySpaces, _Sender, State) -> 97 | {stop, not_implemented, State}. 98 | 99 | handle_exit(_Pid, _Reason, State) -> 100 | {noreply, State}. 101 | 102 | terminate(_Reason, _State) -> 103 | ok. 104 | 105 | %%%=================================================================== 106 | %%% Internal Functions 107 | %%%=================================================================== 108 | 109 | match(Client, Entry) -> 110 | fun({Regexp, Fun}) -> 111 | case re:run(Entry, Regexp, [{capture, all, list}]) of 112 | nomatch -> ignore; 113 | {match, Match} -> Fun({Client, Entry, Regexp}, Match) 114 | end 115 | end. 116 | 117 | %%%=================================================================== 118 | %%% Match Handlers 119 | %%%=================================================================== 120 | 121 | combined_lf({Client, _Entry, _Regexp}, [_Entry, _Host, _, _User, _Time, Req, Code, BodySize, _Referer, Agent]) -> 122 | rts:sadd(Client, "agents", Agent), 123 | rts:incrby(Client, "total_sent", list_to_integer(BodySize)), 124 | [Method, _Resource, _Protocol] = string:tokens(Req, " "), 125 | rts:incr(Client, Method), 126 | case Code of 127 | [$2, _, _] -> 128 | rts:incr(Client, "200"); 129 | [$3, _, _] -> 130 | rts:incr(Client, "300"); 131 | [$4, _, _] -> 132 | rts:incr(Client, "400"); 133 | [$5, _, _] -> 134 | rts:incr(Client, "500") 135 | end, 136 | rts:incr(Client, "total_reqs"). 137 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts_stat_vnode.erl: -------------------------------------------------------------------------------- 1 | %% @doc A vnode to handle get & put commands for stat data. The vnode 2 | %% requests will be hashed on Client and StatName and will use a 3 | %% coordinator to enforce N/R/W values. 4 | -module(rts_stat_vnode). 5 | -behaviour(riak_core_vnode). 6 | -include("rts.hrl"). 7 | -include_lib("riak_core/include/riak_core_vnode.hrl"). 8 | -export([start_vnode/1, 9 | init/1, 10 | terminate/2, 11 | handle_command/3, 12 | is_empty/1, 13 | delete/1, 14 | handle_handoff_command/3, 15 | handoff_starting/2, 16 | handoff_cancelled/1, 17 | handoff_finished/2, 18 | handle_handoff_data/2, 19 | encode_handoff_item/2, 20 | handle_coverage/4, 21 | handle_exit/3]). 22 | 23 | -export([ 24 | get/2, 25 | set/3, 26 | incr/2, 27 | incrby/3, 28 | append/3, 29 | sadd/3 30 | ]). 31 | 32 | -record(state, {partition, stats}). 33 | 34 | -define(MASTER, rts_stat_vnode_master). 35 | -define(sync(PrefList, Command, Master), 36 | riak_core_vnode_master:sync_command(PrefList, Command, Master)). 37 | 38 | %%%=================================================================== 39 | %%% API 40 | %%%=================================================================== 41 | 42 | start_vnode(I) -> 43 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 44 | 45 | get(IdxNode, StatName) -> 46 | ?sync(IdxNode, {get, StatName}, ?MASTER). 47 | 48 | set(IdxNode, StatName, Val) -> 49 | ?sync(IdxNode, {set, StatName, Val}, ?MASTER). 50 | 51 | incr(IdxNode, StatName) -> 52 | ?sync(IdxNode, {incr, StatName}, ?MASTER). 53 | 54 | incrby(IdxNode, StatName, Val) -> 55 | ?sync(IdxNode, {incrby, StatName, Val}, ?MASTER). 56 | 57 | append(IdxNode, StatName, Val) -> 58 | ?sync(IdxNode, {append, StatName, Val}, ?MASTER). 59 | 60 | sadd(IdxNode, StatName, Val) -> 61 | ?sync(IdxNode, {sadd, StatName, Val}, ?MASTER). 62 | 63 | %%%=================================================================== 64 | %%% Callbacks 65 | %%%=================================================================== 66 | 67 | init([Partition]) -> 68 | {ok, #state { partition=Partition, stats=dict:new() }}. 69 | 70 | handle_command({get, StatName}, _Sender, #state{stats=Stats}=State) -> 71 | Reply = 72 | case dict:find(StatName, Stats) of 73 | error -> 74 | not_found; 75 | Found -> 76 | Found 77 | end, 78 | {reply, Reply, State}; 79 | 80 | handle_command({set, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 81 | Stats = dict:store(StatName, Val, Stats0), 82 | {reply, ok, State#state{stats=Stats}}; 83 | 84 | handle_command({incr, StatName}, _Sender, #state{stats=Stats0}=State) -> 85 | Stats = dict:update_counter(StatName, 1, Stats0), 86 | {reply, ok, State#state{stats=Stats}}; 87 | 88 | handle_command({incrby, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 89 | Stats = dict:update_counter(StatName, Val, Stats0), 90 | {reply, ok, State#state{stats=Stats}}; 91 | 92 | handle_command({append, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 93 | Stats = try dict:append(StatName, Val, Stats0) 94 | catch _:_ -> dict:store(StatName, [Val], Stats0) 95 | end, 96 | {reply, ok, State#state{stats=Stats}}; 97 | 98 | handle_command({sadd, StatName, Val}, _Sender, #state{stats=Stats0}=State) -> 99 | F = fun(S) -> 100 | sets:add_element(Val, S) 101 | end, 102 | Stats = dict:update(StatName, F, sets:from_list([Val]), Stats0), 103 | {reply, ok, State#state{stats=Stats}}. 104 | 105 | handle_handoff_command(?FOLD_REQ{foldfun=Fun, acc0=Acc0}, _Sender, State) -> 106 | Acc = dict:fold(Fun, Acc0, State#state.stats), 107 | {reply, Acc, State}. 108 | 109 | handoff_starting(_TargetNode, _State) -> 110 | {true, _State}. 111 | 112 | handoff_cancelled(State) -> 113 | {ok, State}. 114 | 115 | handoff_finished(_TargetNode, State) -> 116 | {ok, State}. 117 | 118 | handle_handoff_data(Data, #state{stats=Stats0}=State) -> 119 | {StatName, Val} = binary_to_term(Data), 120 | Stats = dict:store(StatName, Val, Stats0), 121 | {reply, ok, State#state{stats=Stats}}. 122 | 123 | encode_handoff_item(StatName, Val) -> 124 | term_to_binary({StatName,Val}). 125 | 126 | is_empty(State) -> 127 | case dict:size(State#state.stats) of 128 | 0 -> {true, State}; 129 | _ -> {false, State} 130 | end. 131 | 132 | delete(State) -> 133 | {ok, State}. 134 | 135 | handle_coverage(_Req, _KeySpaces, _Sender, State) -> 136 | {stop, not_implemented, State}. 137 | 138 | handle_exit(_Pid, _Reason, State) -> 139 | {noreply, State}. 140 | 141 | terminate(_Reason, _State) -> 142 | ok. 143 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts_sup.erl: -------------------------------------------------------------------------------- 1 | -module(rts_sup). 2 | 3 | -behaviour(supervisor). 4 | 5 | %% API 6 | -export([start_link/0]). 7 | 8 | %% Supervisor callbacks 9 | -export([init/1]). 10 | 11 | %% =================================================================== 12 | %% API functions 13 | %% =================================================================== 14 | 15 | start_link() -> 16 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 17 | 18 | %% =================================================================== 19 | %% Supervisor callbacks 20 | %% =================================================================== 21 | 22 | init(_Args) -> 23 | VMaster = { rts_vnode_master, 24 | {riak_core_vnode_master, start_link, [rts_vnode]}, 25 | permanent, 5000, worker, [riak_core_vnode_master]}, 26 | 27 | Entry = {rts_entry_vnode_master, 28 | {riak_core_vnode_master, start_link, [rts_entry_vnode]}, 29 | permanent, 5000, worker, [riak_core_vnode_master]}, 30 | 31 | Stat = {rts_stat_vnode_master, 32 | {riak_core_vnode_master, start_link, [rts_stat_vnode]}, 33 | permanent, 5000, worker, [riak_core_vnode_master]}, 34 | 35 | {ok, 36 | {{one_for_one, 5, 10}, 37 | [VMaster, Entry, Stat]}}. 38 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts_vnode.erl: -------------------------------------------------------------------------------- 1 | -module(rts_vnode). 2 | -behaviour(riak_core_vnode). 3 | -include("rts.hrl"). 4 | 5 | -export([start_vnode/1, 6 | init/1, 7 | terminate/2, 8 | handle_command/3, 9 | is_empty/1, 10 | delete/1, 11 | handle_handoff_command/3, 12 | handoff_starting/2, 13 | handoff_cancelled/1, 14 | handoff_finished/2, 15 | handle_handoff_data/2, 16 | encode_handoff_item/2, 17 | handle_coverage/4, 18 | handle_exit/3]). 19 | 20 | -record(state, {partition}). 21 | 22 | %% API 23 | start_vnode(I) -> 24 | riak_core_vnode_master:get_vnode_pid(I, ?MODULE). 25 | 26 | init([Partition]) -> 27 | {ok, #state { partition=Partition }}. 28 | 29 | % Sample command: respond to a ping 30 | handle_command(ping, _Sender, State) -> 31 | {reply, {pong, State#state.partition}, State}; 32 | handle_command(Message, _Sender, State) -> 33 | ?PRINT({unhandled_command, Message}), 34 | {noreply, State}. 35 | 36 | handle_handoff_command(_Message, _Sender, State) -> 37 | {noreply, State}. 38 | 39 | handoff_starting(_TargetNode, State) -> 40 | {true, State}. 41 | 42 | handoff_cancelled(State) -> 43 | {ok, State}. 44 | 45 | handoff_finished(_TargetNode, State) -> 46 | {ok, State}. 47 | 48 | handle_handoff_data(_Data, State) -> 49 | {reply, ok, State}. 50 | 51 | encode_handoff_item(_ObjectName, _ObjectValue) -> 52 | <<>>. 53 | 54 | is_empty(State) -> 55 | {true, State}. 56 | 57 | delete(State) -> 58 | {ok, State}. 59 | 60 | handle_coverage(_Req, _KeySpaces, _Sender, State) -> 61 | {stop, not_implemented, State}. 62 | 63 | handle_exit(_Pid, _Reason, State) -> 64 | {noreply, State}. 65 | 66 | terminate(_Reason, _State) -> 67 | ok. 68 | -------------------------------------------------------------------------------- /2011/riak-core-the-vnode/rts/src/rts_wm_entry.erl: -------------------------------------------------------------------------------- 1 | %% @doc Webmachine resource to handle incoming log entries. 2 | %% 3 | %% == PUT /rts/entry/Client == 4 | %% 5 | %% Write an entry for the given Client. 6 | %% 7 | %% Content-type must be 'text/plain' and the body should be _one_ 8 | %% line. 9 | -module(rts_wm_entry). 10 | -export([ 11 | init/1, 12 | allowed_methods/2, 13 | process_post/2 14 | ]). 15 | 16 | -include_lib("webmachine/include/webmachine.hrl"). 17 | 18 | init(_Props) -> 19 | {ok, none}. 20 | 21 | allowed_methods(_RD, _Ctx) -> 22 | {['POST'], _RD, _Ctx}. 23 | 24 | process_post(RD, _Ctx) -> 25 | case wrq:get_req_header("content-type", RD) of 26 | "text/plain" -> 27 | Client = wrq:path_info(client, RD), 28 | Entry = binary_to_list(wrq:req_body(RD)), 29 | ok = rts:entry(Client, Entry), 30 | {true, RD, _Ctx}; 31 | _ -> 32 | {{halt, 415}, RD, _Ctx} 33 | end. 34 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/.gitignore: -------------------------------------------------------------------------------- 1 | earthquake 2 | tests 3 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/Makefile: -------------------------------------------------------------------------------- 1 | BB ?= /Users/rzezeski/work/basho_bench 2 | RIAK_PATH ?= /Users/rzezeski/work/riak/dev/dev1 3 | PBC_PORT ?= 8081 4 | SCHEMA ?= schemas/tweets-schema.txt 5 | 6 | .PHONY: init results run run-non-inline run_bench 7 | 8 | init: 9 | ./init.sh $(RIAK_PATH) $(SCHEMA) $(PBC_PORT) 10 | 11 | # First prime system by running naive, then measure. 12 | run: 13 | $(BB)/basho_bench naive.config 14 | 15 | $(BB)/basho_bench naive.config 16 | cp -r tests/current tests/naive 17 | 18 | $(BB)/basho_bench scoped.config 19 | cp -r tests/current tests/scoped 20 | 21 | $(BB)/basho_bench scoped-filter.config 22 | cp -r tests/current tests/scoped-filter 23 | 24 | run-non-inline: 25 | $(BB)/basho_bench naive.config 26 | 27 | $(BB)/basho_bench naive.config 28 | cp -r tests/current tests/naive 29 | 30 | $(BB)/basho_bench scoped.config 31 | cp -r tests/current tests/scoped 32 | 33 | run_bench: 34 | $(BB)/basho_bench $(CONFIG) 35 | 36 | # Build the pretty graphs, you need R installed 37 | results: 38 | $(BB)/priv/summary.r -i tests/naive 39 | cp tests/naive/summary.png tests/naive/naive.png 40 | 41 | $(BB)/priv/summary.r -i tests/scoped 42 | cp tests/scoped/summary.png tests/scoped/scoped.png 43 | 44 | $(BB)/priv/summary.r -i tests/scoped-filter 45 | cp tests/scoped-filter/summary.png tests/scoped-filter/scoped-filter.png -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/RUN_BENCHMARKS.md: -------------------------------------------------------------------------------- 1 | Run Benchmarks 2 | ========== 3 | 4 | In keeping tradition with my "working blog" title I've included these 5 | instructions to enable you to run the benchmarks on your own hardware. 6 | 7 | ### Prereqs ### 8 | 9 | Since I already assumed familiarity with Search I'm going to assume 10 | you already have it installed or you know how to build a devrel from 11 | scratch. Remember, Search is integrated with Riak now. 12 | 13 | You'll need the Python [client] [pc] to load the data. 14 | 15 | You'll need [basho bench] [bb] to run the benchmarks. 16 | 17 | Finally, you'll need a local copy of the Riak Search [code] [rsc] 18 | because it contains the bench driver needed to run the benchmarks. 19 | 20 | Then you should edit the Makefile and .config files to correspond to 21 | your environment. 22 | 23 | * `BB` - Path to Basho Bench 24 | 25 | * `RIAK_PATH` - That path to your Riak "install." If you used devrel 26 | then point it to `/dev/dev1`. The key is that the `bin` 27 | dir exist in this directory. 28 | 29 | * `PBC_PORT` - The Protocol Buffers port of any of your nodes. If 30 | using devrel then set it to `8081`. 31 | 32 | For example... 33 | 34 | BB ?= /Users/rzezeski/work/basho_bench 35 | RIAK_PATH ?= /Users/rzezeski/work/riak/dev/dev1 36 | PBC_PORT ?= 8081 37 | 38 | And in the .config files... 39 | 40 | {source_dir, "/Users/rzezeski/work/riak_search/src"}. 41 | 42 | ### Load Corpus ### 43 | 44 | To load the data run the following. Keep in mind this will take 45 | several minutes to load and you will see a few errors that you can 46 | safely ignore. It would be a good time to step away for a beer, a 47 | smoke, or an apple if you listened to your mother as a child. 48 | 49 | make init 50 | 51 | 52 | ### Run the Benchmarks ### 53 | 54 | A total of 4 runs will occur. The first to prime the system and then 55 | the three queries. Each run will take 5 minutes to finish. 56 | 57 | make run 58 | 59 | 60 | ### Generate the Pretty Graphs ### 61 | 62 | If you want to see pretty graphs run the following command. It should 63 | create PNG images for each run. 64 | 65 | make results 66 | 67 | 68 | [pc]: https://github.com/basho/riak-python-client 69 | 70 | [bb]: http://wiki.basho.com/Benchmarking-with-Basho-Bench.html 71 | 72 | [rsc]: https://github.com/basho/riak_search 73 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/init.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | #> Usage: 4 | #> init RIAK_PATH SCHEMA [PBC_PORT] 5 | 6 | set -e 7 | 8 | if [ ! -e earthquake ] 9 | then 10 | echo "The tweet corpus must first be downloaded and extracted." 11 | echo "See http://www.infochimps.com/datasets/twitter-haiti-earthquake-data" 12 | exit 1 13 | fi 14 | 15 | if [ $# -lt 2 ] 16 | then 17 | grep '#>' $0 | tr -d '#>' | sed '$d' 18 | exit 1 19 | fi 20 | 21 | RIAK=$1 22 | SCHEMA=$2 23 | PORT=${3:-8087} 24 | i=0 25 | 26 | echo "Install schema $SCHEMA..." 27 | $RIAK/bin/search-cmd set-schema tweets $SCHEMA 28 | 29 | echo "Install Search precommit hook..." 30 | $RIAK/bin/search-cmd install tweets 31 | 32 | printf "Upload data" 33 | python upload.py $PORT < earthquake 34 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/meta: -------------------------------------------------------------------------------- 1 | published 2011-07-16 2 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/naive.config: -------------------------------------------------------------------------------- 1 | {mode, {rate, 10}}. 2 | {concurrent, 2}. 3 | {duration, 5}. 4 | {operations, [{search,1}]}. 5 | {driver, basho_bench_driver_riaksearch}. 6 | {source_dir, "/Users/rzezeski/work/riak_search/src"}. 7 | 8 | {riaksearch_node, 'bench@127.0.0.1'}. 9 | {riaksearch_remotenodes, 10 | ['dev1@127.0.0.1', 'dev2@127.0.0.1', 'dev3@127.0.0.1', 'dev2@127.0.0.1']}. 11 | {riaksearch_query, ["tweets", "text:earthquake"]}. 12 | {riaksearch_expected, 62805}. 13 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/results/naive-non-inline.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-search-inline-fields/results/naive-non-inline.jpg -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/results/naive.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-search-inline-fields/results/naive.png -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/results/scoped-filter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-search-inline-fields/results/scoped-filter.png -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/results/scoped-non-inline.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-search-inline-fields/results/scoped-non-inline.jpg -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/results/scoped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rzezeski/try-try-try/c5d99f29fb3380f8653efdd1aa6a8f52143a9717/2011/riak-search-inline-fields/results/scoped.png -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/schemas/tweets-schema-inline.txt: -------------------------------------------------------------------------------- 1 | %% Schema for tweets 2 | { 3 | schema, 4 | [ 5 | {version, "1.1"}, 6 | {n_val, 2}, 7 | {default_field, "value"}, 8 | {analyzer_factory, {erlang, text_analyzers, standard_analyzer_factory}} 9 | ], 10 | [ 11 | %% Person who tweeted 12 | {field, [ 13 | {aliases, ["who"]}, 14 | {name, "user_screen_name"} 15 | ]}, 16 | 17 | %% What they tweeted 18 | {field, [ 19 | {aliases, ["what"]}, 20 | {inline, true}, 21 | {name, "text"} 22 | ]}, 23 | 24 | %% When they tweeted 25 | {field, [ 26 | {aliases, ["when"]}, 27 | {name, "created_at"} 28 | ]}, 29 | 30 | {dynamic_field, [ 31 | {name, "*"}, 32 | {skip, true} 33 | ]} 34 | ] 35 | }. 36 | 37 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/schemas/tweets-schema.txt: -------------------------------------------------------------------------------- 1 | %% Schema for tweets 2 | { 3 | schema, 4 | [ 5 | {version, "1.1"}, 6 | {n_val, 2}, 7 | {default_field, "value"}, 8 | {analyzer_factory, {erlang, text_analyzers, standard_analyzer_factory}} 9 | ], 10 | [ 11 | %% Person who tweeted 12 | {field, [ 13 | {aliases, ["who"]}, 14 | {name, "user_screen_name"} 15 | ]}, 16 | 17 | %% What they tweeted 18 | {field, [ 19 | {aliases, ["what"]}, 20 | {name, "text"} 21 | ]}, 22 | 23 | %% When they tweeted 24 | {field, [ 25 | {aliases, ["when"]}, 26 | {name, "created_at"} 27 | ]}, 28 | 29 | {dynamic_field, [ 30 | {name, "*"}, 31 | {skip, true} 32 | ]} 33 | ] 34 | }. 35 | 36 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/scoped-filter.config: -------------------------------------------------------------------------------- 1 | {mode, {rate, 10}}. 2 | {concurrent, 2}. 3 | {duration, 5}. 4 | {operations, [{search,1}]}. 5 | {driver, basho_bench_driver_riaksearch}. 6 | {source_dir, "/Users/rzezeski/work/riak_search/src"}. 7 | 8 | {riaksearch_node, 'bench@127.0.0.1'}. 9 | {riaksearch_remotenodes, 10 | ['dev1@127.0.0.1', 'dev2@127.0.0.1', 'dev3@127.0.0.1', 'dev2@127.0.0.1']}. 11 | {riaksearch_query, 12 | ["tweets", "created_at:[20100113T032200 TO 20100113T032500]", 13 | "text:earthquake"]}. 14 | {riaksearch_expected, 318}. 15 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/scoped.config: -------------------------------------------------------------------------------- 1 | {mode, {rate, 10}}. 2 | {concurrent, 2}. 3 | {duration, 5}. 4 | {operations, [{search,1}]}. 5 | {driver, basho_bench_driver_riaksearch}. 6 | {source_dir, "/Users/rzezeski/work/riak_search/src"}. 7 | 8 | {riaksearch_node, 'bench@127.0.0.1'}. 9 | {riaksearch_remotenodes, 10 | ['dev1@127.0.0.1', 'dev2@127.0.0.1', 'dev3@127.0.0.1', 'dev2@127.0.0.1']}. 11 | {riaksearch_query, 12 | ["tweets", 13 | "text:earthquake AND created_at:[20100113T032200 TO 20100113T032500]"]}. 14 | {riaksearch_expected, 318}. 15 | -------------------------------------------------------------------------------- /2011/riak-search-inline-fields/upload.py: -------------------------------------------------------------------------------- 1 | # Python isn't my 1st, 2nd, 3rd, or even 10th language, so please 2 | # forgive me if it's not pretty/idiomatic. 3 | 4 | # Usage: head -10 earthquake | python upload.py [PORT] 5 | # python upload.py [PORT] < earthquake 6 | import sys 7 | import json 8 | import riak 9 | import datetime 10 | 11 | def main(): 12 | if len(sys.argv) == 1: 13 | inf = sys.stdin 14 | port = 8087 15 | else: 16 | inf = sys.stdin 17 | port = int(sys.argv[1]) 18 | 19 | i = 0 20 | c = riak.RiakClient(port=port, transport_class=riak.RiakPbcTransport) 21 | b = c.bucket('tweets') 22 | 23 | for line in inf: 24 | 25 | try: 26 | j = json.loads(line) 27 | ts = datetime.datetime.strptime(j['created_at'], '%a %b %d %H:%M:%S +0000 %Y') 28 | j['created_at'] = ts.strftime('%Y%m%dT%H%M%S') 29 | (b.new(str(i), data=j)).store() 30 | except ValueError as (strerror): 31 | print "Couldn't decode: %s" % line 32 | print "Error: ", strerror 33 | 34 | i += 1 35 | if i % 10 == 0: 36 | sys.stdout.write('.') 37 | sys.stdout.flush() 38 | 39 | print 'OK' 40 | 41 | if __name__ == '__main__': 42 | main() 43 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | This is my new "working blog" inspired by [homoiconic](https://github.com/raganwald/homoiconic). 2 | 3 | The focus here is on providing working code that can be downloaded, compiled and run by the reader while also providing supporting text, in the form of a blog post, that helps to illustrate how the code works. That is, it's pretty much the same as a traditional blog but with an emphasis on complete, working code examples rather than simply just code excerpts. 4 | 5 | I titled the blog _try try try_ after a verse in the song _12341234_ from the Album [Keasbey Nights](http://www.amazon.com/Keasbey-Nights-Catch-22/dp/B000005ZFF/ref=sr_1_1?s=music&ie=UTF8&qid=1301883715&sr=1-1) by New Jersey Ska band Catch 22. It reminds me of a great quote from \_Why that has resonated with me at such a deep level and has reminded me that the most important thing you can do in anything you want to be good at is to try. 6 | 7 | > When you don’t create things, you become defined by your tastes rather than ability. Your tastes only narrow and exclude people. so create. 8 | --------------------------------------------------------------------------------