├── .gitignore ├── Makefile ├── README.md ├── rebar ├── src ├── bench.erl ├── redo.app.src ├── redo.erl ├── redo_block.erl ├── redo_concurrency_test.erl ├── redo_logging.hrl ├── redo_redis_proto.erl └── redo_uri.erl └── test ├── redo_block_tests.erl └── redo_tests.erl /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | src/*.swp 3 | ebin 4 | .eunit 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | ./rebar compile 3 | 4 | clean: 5 | ./rebar clean 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Philosophy 2 | 3 | > If you wish to write a redis client from scratch, you must first invent the universe. 4 | 5 | ### About 6 | 7 | Redo is a pipelined redis client written in Erlang. It lacks any sort of syntactic sugar. The only API function is redo:cmd, which takes a raw redis command. 8 | 9 | ### Build 10 | 11 | $ make 12 | 13 | ### Test 14 | 15 | #### Unit tests 16 | 17 | $ ./rebar eunit suite=redo 18 | 19 | #### Local read benchmark 20 | 21 | $ erl -pa ebin 22 | 1> bench:sync(1000). 23 | 91ms 24 | 10989 req/sec 25 | 26 | 2> bench:async(1000, 100). 27 | 38ms 28 | 26315 req/sec 29 | 30 | #### Concurrency test 31 | 32 | $ erl -pa ebin 33 | 1> redo_concurrency_test:run(20, 100). %% 20 pids, 100 random operations performed per pid 34 | 35 | ### Start 36 | 37 | No arguments: register process as "redo" 38 | 39 | $ erl -pa ebin 40 | 1> redo:start_link(). 41 | {ok, <0.33.0> 42 | 2> whereis(redo). 43 | <0.33.0> 44 | 3> redo:cmd(["PING"]). 45 | <<"PONG">> 46 | 47 | Register with custom name 48 | 49 | erl -pa ebin 50 | 1> redo:start_link(myclient). 51 | {ok,<0.33.0>} 52 | 2> whereis(myclient). 53 | <0.33.0> 54 | 8> redo:cmd(myclient, ["PING"]). 55 | <<"PONG">> 56 | 57 | Start anonymous Redo process 58 | 59 | erl -pa ebin 60 | 1> {ok, Pid} = redo:start_link(undefined). 61 | {ok,<0.33.0>} 62 | 2> redo:cmd(Pid, ["PING"]). 63 | <<"PONG">> 64 | 65 | Specifying connection options 66 | 67 | erl -pa ebin 68 | 1> redo:start_link([{host, "localhost"}, {port, 6379}]). 69 | {ok,<0.33.0>} 70 | 2> redo:cmd(["PING"]). 71 | <<"PONG">> 72 | 73 | 3> redo:start_link(myclient, [{host, "localhost"}, {port, 6379}]). 74 | {ok,<0.37.0>} 75 | 4> redo:cmd(myclient, ["PING"]). 76 | <<"PONG">> 77 | 78 | ### Commands 79 | 80 | erl -pa ebin 81 | 1> redo:start_link(). 82 | <0.33.0> 83 | 2> redo:cmd(["SET", "foo"]). 84 | {error,<<"ERR wrong number of arguments for 'set' command">>} 85 | 3> redo:cmd(["SET", "foo", "bar"]). 86 | <<"OK">> 87 | 4> redo:cmd(["GET", "foo"]). 88 | <<"bar">> 89 | 5> redo:cmd(["HMSET", "hfoo", "ONE", "ABC", "TWO", "DEF"]). 90 | <<"OK">> 91 | 6> redo:cmd(["HGETALL", "hfoo"]). 92 | [<<"ONE">>,<<"ABC">>,<<"TWO">>,<<"DEF">>] 93 | 94 | ### Pipelined commands 95 | 96 | 1> redo:start_link(). 97 | <0.33.> 98 | 2> redo:cmd([["GET", "foo"], ["HGETALL", "hfoo"]]). 99 | [<<"bar">>, [<<"ONE">>,<<"ABC">>,<<"TWO">>,<<"DEF">>]] 100 | 101 | ### Pub/Sub 102 | 103 | $ erl -pa ebin 104 | 1> redo:start_link(). 105 | {ok,<0.33.0>} 106 | 2> Ref = redo:subscribe("chfoo"). 107 | #Ref<0.0.0.42> 108 | 3> (fun() -> receive {Ref, Res} -> Res after 10000 -> timeout end end)(). 109 | [<<"subscribe">>,<<"chfoo">>,1] 110 | 4> redo:start_link(client). 111 | {ok,<0.39.0>} 112 | 5> redo:cmd(client, ["PUBLISH", "chfoo", "hello"]). 113 | 1 114 | 6> (fun() -> receive {Ref, Res} -> Res after 10000 -> timeout end end)(). 115 | [<<"message">>,<<"chfoo">>,<<"hello">>] 116 | 117 | %% restart redis server... 118 | 119 | 7> (fun() -> receive {Ref, Res} -> Res after 10000 -> timeout end end)(). 120 | closed 121 | 8> f(Ref). 122 | ok 123 | 9> Ref = redo:subscribe("chfoo"). 124 | #Ref<0.0.0.68> 125 | 10> (fun() -> receive {Ref, Res} -> Res after 10000 -> timeout end end)(). 126 | [<<"subscribe">>,<<"chfoo">>,1] 127 | 11> redo:cmd(client, ["PUBLISH", "chfoo", "hello again"]). 128 | 1 129 | 12> (fun() -> receive {Ref, Res} -> Res after 10000 -> timeout end end)(). 130 | [<<"message">>,<<"chfoo">>,<<"hello again">>] 131 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jkvor/redo/7c7eaef4cd65271e2fc4ea88587e848407cf0762/rebar -------------------------------------------------------------------------------- /src/bench.erl: -------------------------------------------------------------------------------- 1 | -module(bench). 2 | -compile(export_all). 3 | 4 | sync(Num) -> 5 | Redo = setup(), 6 | A = now(), 7 | ok = loop(Redo, Num), 8 | B = now(), 9 | print(Num,A,B), 10 | ok. 11 | 12 | async(Num, Concurrency) -> 13 | Redo = setup(), 14 | Self = self(), 15 | A = now(), 16 | Pids = [spawn_link(fun() -> loop(Redo, Num div Concurrency), Self ! {self(), done} end) || _ <- lists:seq(1, Concurrency)], 17 | [receive {Pid, done} -> ok end || Pid <- Pids], 18 | B = now(), 19 | print(Num,A,B), 20 | ok. 21 | 22 | setup() -> 23 | {ok, Redo} = redo:start_link(undefined, []), 24 | redo:cmd(Redo, ["SET", "foo", "bar"]), 25 | Redo. 26 | 27 | print(Num,A,B) -> 28 | Microsecs = timer:now_diff(B,A), 29 | Time = Microsecs div Num, 30 | PerSec = 1000000 div Time, 31 | io:format("~wms~n~w req/sec~n", [Microsecs div 1000, PerSec]). 32 | 33 | loop(_Pid, 0) -> ok; 34 | 35 | loop(Pid, Count) -> 36 | <<"bar">> = redo:cmd(Pid, ["GET", "foo"]), 37 | loop(Pid, Count-1). 38 | -------------------------------------------------------------------------------- /src/redo.app.src: -------------------------------------------------------------------------------- 1 | {application, redo, 2 | [ 3 | {description, "Pipelined Redis Erlang Driver"}, 4 | {vsn, "1.1.0"}, 5 | {registered, []}, 6 | {applications, [kernel,stdlib]}, 7 | {env, []} 8 | ]}. 9 | -------------------------------------------------------------------------------- /src/redo.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(redo). 24 | -behaviour(gen_server). 25 | 26 | -include("redo_logging.hrl"). 27 | 28 | %% gen_server callbacks 29 | -export([init/1, handle_call/3, handle_cast/2, 30 | handle_info/2, terminate/2, code_change/3]). 31 | 32 | -export([start_link/0, start_link/1, start_link/2, 33 | cmd/1, cmd/2, cmd/3, subscribe/1, subscribe/2, 34 | shutdown/1, async_cmd/1, async_cmd/2]). 35 | 36 | -record(state, {host, port, pass, db, sock, queue, subscriber, cancelled, acc, buffer, reconnect}). 37 | 38 | -define(TIMEOUT, 30000). 39 | 40 | -spec start_link() -> {ok, pid()} | {error, term()}. 41 | start_link() -> 42 | start_link([]). 43 | 44 | -spec start_link(atom() | list()) -> 45 | {ok, pid()} | 46 | {error, term()}. 47 | start_link(Name) when is_atom(Name) -> 48 | start_link(Name, []); 49 | 50 | start_link(Opts) when is_list(Opts) -> 51 | start_link(?MODULE, Opts). 52 | 53 | -spec start_link(atom(), list()) -> {ok, pid()} | {error, term()}. 54 | start_link(undefined, Opts) when is_list(Opts) -> 55 | gen_server:start_link(?MODULE, [Opts], []); 56 | 57 | start_link(Name, Opts) when is_atom(Name), is_list(Opts) -> 58 | gen_server:start_link({local, Name}, ?MODULE, [Opts], []). 59 | 60 | cmd(Cmd) -> 61 | cmd(?MODULE, Cmd, ?TIMEOUT). 62 | 63 | cmd(NameOrPid, Cmd) -> 64 | cmd(NameOrPid, Cmd, ?TIMEOUT). 65 | 66 | -type redis_value() :: integer() | binary(). 67 | -type response() :: redis_value() | [redis_value()] | {'error', Reason::term()}. 68 | 69 | -spec cmd(atom() | pid(), list() | binary(), integer()) -> 70 | response() | [response()]. 71 | 72 | cmd(NameOrPid, Cmd, Timeout) when is_integer(Timeout); Timeout =:= infinity -> 73 | %% format commands to be sent to redis 74 | Packets = redo_redis_proto:package(Cmd), 75 | 76 | %% send the commands and receive back 77 | %% unique refs for each packet sent 78 | Refs = gen_server:call(NameOrPid, {cmd, Packets}, 2000), 79 | receive_resps(NameOrPid, Refs, Timeout). 80 | 81 | async_cmd(Cmd) -> 82 | async_cmd(?MODULE, Cmd). 83 | 84 | async_cmd(NameOrPid, Cmd) -> 85 | Packets = redo_redis_proto:package(Cmd), 86 | gen_server:call(NameOrPid, {cmd, Packets}). 87 | 88 | receive_resps(_NameOrPid, {error, Err}, _Timeout) -> 89 | {error, Err}; 90 | 91 | receive_resps(NameOrPid, [Ref], Timeout) -> 92 | %% for a single packet, receive a single reply 93 | receive_resp(NameOrPid, Ref, Timeout); 94 | 95 | receive_resps(NameOrPid, Refs, Timeout) -> 96 | %% for multiple packets, build a list of replies 97 | [receive_resp(NameOrPid, Ref, Timeout) || Ref <- Refs]. 98 | 99 | receive_resp(NameOrPid, Ref, Timeout) -> 100 | receive 101 | %% the connection to the redis server was closed 102 | {Ref, closed} -> 103 | {error, closed}; 104 | {Ref, Val} -> 105 | Val 106 | %% after the timeout expires, cancel the command and return 107 | after Timeout -> 108 | gen_server:cast(NameOrPid, {cancel, Ref}), 109 | {error, timeout} 110 | end. 111 | 112 | -spec subscribe(list() | binary()) -> reference() | {error, term()}. 113 | subscribe(Channel) -> 114 | subscribe(?MODULE, Channel). 115 | 116 | -spec subscribe(atom() | pid(), list() | binary()) -> reference() | {error, term()}. 117 | subscribe(NameOrPid, Channel) -> 118 | Packet = redo_redis_proto:package(["SUBSCRIBE", Channel]), 119 | gen_server:call(NameOrPid, {subscribe, Packet}, 2000). 120 | 121 | shutdown(NameOrPid) -> 122 | gen_server:cast(NameOrPid, shutdown). 123 | 124 | %%==================================================================== 125 | %% gen_server callbacks 126 | %%==================================================================== 127 | 128 | %%-------------------------------------------------------------------- 129 | %% Function: init(Args) -> {ok, State} | 130 | %% {ok, State, Timeout} | 131 | %% ignore | 132 | %% {stop, Reason} 133 | %% Description: Initiates the server 134 | %%-------------------------------------------------------------------- 135 | init([Opts]) -> 136 | State = init_state(Opts), 137 | case connect(State) of 138 | State1 when is_record(State1, state) -> 139 | {ok, State1}; 140 | Err -> 141 | {stop, Err} 142 | end. 143 | 144 | %%-------------------------------------------------------------------- 145 | %% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} | 146 | %% {reply, Reply, State, Timeout} | 147 | %% {noreply, State} | 148 | %% {noreply, State, Timeout} | 149 | %% {stop, Reason, Reply, State} | 150 | %% {stop, Reason, State} 151 | %% Description: Handling call messages 152 | %%-------------------------------------------------------------------- 153 | handle_call({cmd, Packets}, {From, _Ref}, #state{subscriber=undefined, queue=Queue}=State) -> 154 | case test_connection(State) of 155 | State1 when is_record(State1, state) -> 156 | %% send each packet to redis 157 | %% and generate a unique ref per packet 158 | Refs = lists:foldl( 159 | fun(Packet, Refs) when is_list(Refs) -> 160 | case gen_tcp:send(State1#state.sock, Packet) of 161 | ok -> [erlang:make_ref()|Refs]; 162 | Err -> Err 163 | end; 164 | (_Packet, Err) -> 165 | Err 166 | end, [], Packets), 167 | 168 | %% enqueue the client pid/refs 169 | case Refs of 170 | List when is_list(List) -> 171 | Refs1 = lists:reverse(Refs), 172 | Queue1 = lists:foldl( 173 | fun(Ref, Acc) -> 174 | queue:in({From, Ref}, Acc) 175 | end, Queue, Refs1), 176 | {reply, Refs1, State1#state{queue=Queue1}}; 177 | Err -> 178 | {stop, Err, State1} 179 | end; 180 | _Err -> 181 | %% failed to connect, retry 182 | {reply, {error, closed}, State#state{sock=undefined}, 1000} 183 | end; 184 | 185 | handle_call({cmd, _Packets}, _From, State = #state{}) -> 186 | {reply, {error, subscriber_mode}, State}; 187 | 188 | handle_call({subscribe, Packet}, {From, _Ref}, State = #state{}) -> 189 | case test_connection(State) of 190 | State1 when is_record(State1, state) -> 191 | case gen_tcp:send(State1#state.sock, Packet) of 192 | ok -> 193 | Ref = erlang:make_ref(), 194 | {reply, Ref, State1#state{subscriber={From, Ref}}}; 195 | Err -> 196 | {stop, Err, State1} 197 | end; 198 | _Err -> 199 | %% failed to connect, retry 200 | {reply, {error, closed}, State#state{sock=undefined}, 1000} 201 | end; 202 | 203 | %% state doesn't match. Likely outdated or an error. 204 | %% Moving from 1.1.0 to here sees the addition of one field 205 | handle_call(Msg, From, OldState) when 1+tuple_size(OldState) =:= tuple_size(#state{}), 206 | state =:= element(1,OldState) -> 207 | {ok, State} = code_change("v1.1.0", OldState, internal_detection), 208 | handle_call(Msg, From, State); 209 | 210 | handle_call(_Msg, _From, State = #state{}) -> 211 | {reply, unknown_message, State}. 212 | 213 | 214 | %%-------------------------------------------------------------------- 215 | %% Function: handle_cast(Msg, State) -> {noreply, State} | 216 | %% {noreply, State, Timeout} | 217 | %% {stop, Reason, State} 218 | %% Description: Handling cast messages 219 | %%-------------------------------------------------------------------- 220 | handle_cast(shutdown, State = #state{sock=Sock}) -> 221 | case is_port(Sock) of 222 | true -> 223 | catch gen_tcp:close(Sock); 224 | false -> ok 225 | end, 226 | State1 = close_connection(State#state{sock=undefined}), 227 | {stop, shutdown, State1}; 228 | 229 | handle_cast({cancel, Ref}, #state{cancelled=Cancelled}=State) -> 230 | {noreply, State#state{cancelled=[Ref|Cancelled]}}; 231 | 232 | %% state doesn't match. Likely outdated or an error. 233 | %% Moving from 1.1.0 to here sees the addition of one field 234 | handle_cast(Msg, OldState) when 1+tuple_size(OldState) =:= tuple_size(#state{}), 235 | state =:= element(1,OldState) -> 236 | {ok, State} = code_change("v1.1.0", OldState, internal_detection), 237 | handle_cast(Msg, State); 238 | 239 | handle_cast(_Msg, State) -> 240 | {noreply, State}. 241 | 242 | %%-------------------------------------------------------------------- 243 | %% Function: handle_info(Info, State) -> {noreply, State} | 244 | %% {noreply, State, Timeout} | 245 | %% {stop, Reason, State} 246 | %% Description: Handling all non call/cast messages 247 | %%-------------------------------------------------------------------- 248 | handle_info({tcp, Sock, Data}, #state{sock=Sock, buffer=Buffer}=State) -> 249 | %% compose the packet to be processed by combining 250 | %% the leftover buffer with the new data packet 251 | Packet = packet(Buffer, Data), 252 | case process_packet(State, Packet) of 253 | {ok, State1} -> 254 | %% accept next incoming packet 255 | inet:setopts(Sock, [{active, once}]), 256 | {noreply, State1}; 257 | Err -> 258 | {stop, Err, State} 259 | end; 260 | 261 | handle_info({tcp_closed, Sock}, #state{sock=Sock, reconnect=false}=State) -> 262 | State1 = close_connection(State), 263 | {stop, tcp_closed, State1}; 264 | 265 | handle_info({tcp_closed, Sock}, #state{sock=Sock}=State) -> 266 | State1 = close_connection(State), 267 | 268 | %% reconnect to redis 269 | case connect(State1) of 270 | State2 when is_record(State2, state) -> 271 | {noreply, State2}; 272 | _Err -> 273 | {noreply, State1#state{sock=undefined}, 1000} 274 | end; 275 | 276 | handle_info({tcp_error, Sock, Reason}, #state{sock=Sock}=State) -> 277 | ?WARN("at=tcp_error sock=~p reason=~p", 278 | [Sock, Reason]), 279 | CState = close_connection(State#state{sock=undefined}), 280 | {stop, normal, CState}; 281 | 282 | %% attempt to reconnect to redis 283 | handle_info(timeout, State = #state{}) -> 284 | case connect(State) of 285 | State1 when is_record(State1, state) -> 286 | {noreply, State1}; 287 | _Err -> 288 | {noreply, State#state{sock=undefined}, 1000} 289 | end; 290 | 291 | %% state doesn't match. Likely outdated or an error. 292 | %% Moving from 1.1.0 to here sees the addition of one field 293 | handle_info(Msg, OldState) when 1+tuple_size(OldState) =:= tuple_size(#state{}), 294 | state =:= element(1,OldState) -> 295 | {ok, State} = code_change("v1.1.0", OldState, internal_detection), 296 | handle_info(Msg, State); 297 | 298 | handle_info(_Info, State) -> 299 | {noreply, State}. 300 | 301 | %%-------------------------------------------------------------------- 302 | %% Function: terminate(Reason, State) -> void() 303 | %% Description: This function is called by a gen_server when it is about to 304 | %% terminate. It should be the opposite of Module:init/1 and do any necessary 305 | %% cleaning up. When it returns, the gen_server terminates with Reason. 306 | %% The return value is ignored. 307 | %%-------------------------------------------------------------------- 308 | terminate(_Reason, _State) -> 309 | ok. 310 | 311 | %%-------------------------------------------------------------------- 312 | %% Func: code_change(OldVsn, State, Extra) -> {ok, NewState} 313 | %% Description: Convert process state when code is changed 314 | %%-------------------------------------------------------------------- 315 | code_change("v1.1.0", OldState, internal_detection) -> 316 | {state, Host, Port, Pass, Db, Sock, Queue, Subscriber, Cancelled, 317 | Acc, Buffer} = OldState, 318 | %% we make the reconnection set to 'true' given that's the default 319 | %% value set in init_state/1 320 | {ok, #state{host=Host, port=Port, pass=Pass, db=Db, sock=Sock, 321 | queue=Queue, subscriber=Subscriber, cancelled=Cancelled, 322 | acc=Acc, buffer=Buffer, reconnect=true}}; 323 | code_change(_OldVsn, State, _Extra) -> 324 | {ok, State}. 325 | 326 | %%-------------------------------------------------------------------- 327 | %% Internal functions 328 | %%-------------------------------------------------------------------- 329 | init_state(Opts) -> 330 | Host = proplists:get_value(host, Opts, "localhost"), 331 | Port = proplists:get_value(port, Opts, 6379), 332 | Pass = proplists:get_value(pass, Opts), 333 | Db = proplists:get_value(db, Opts, 0), 334 | Recn = proplists:get_value(reconnect, Opts, true), 335 | #state{ 336 | host = Host, 337 | port = Port, 338 | pass = Pass, 339 | db = Db, 340 | queue = queue:new(), 341 | cancelled = [], 342 | acc = [], 343 | buffer = {raw, <<>>}, 344 | reconnect = Recn 345 | }. 346 | 347 | connect(#state{host=Host, port=Port, pass=Pass, db=Db}=State) -> 348 | case connect_socket(Host, Port) of 349 | {ok, Sock} -> 350 | case auth(Sock, Pass) of 351 | ok -> 352 | case select_db(Sock, Db) of 353 | ok -> 354 | inet:setopts(Sock, [{active, once}]), 355 | State#state{sock=Sock}; 356 | Err -> 357 | Err 358 | end; 359 | Err -> 360 | Err 361 | end; 362 | Err -> 363 | Err 364 | end. 365 | 366 | connect_socket(Host, Port) -> 367 | SockOpts = [binary, {active, false}, {keepalive, true}, {nodelay, true}], 368 | gen_tcp:connect(Host, Port, SockOpts). 369 | 370 | auth(_Sock, Pass) when Pass == <<>>; Pass == undefined -> 371 | ok; 372 | 373 | auth(Sock, Pass) -> 374 | case gen_tcp:send(Sock, [<<"AUTH ">>, Pass, <<"\r\n">>]) of 375 | ok -> 376 | case gen_tcp:recv(Sock, 0) of 377 | {ok, <<"+OK\r\n">>} -> ok; 378 | {ok, Err} -> {error, Err}; 379 | Err -> Err 380 | end; 381 | Err -> 382 | Err 383 | end. 384 | 385 | select_db(_Sock, 0) -> 386 | ok; 387 | 388 | select_db(Sock, Db) -> 389 | case gen_tcp:send(Sock, hd(redo_redis_proto:package(["SELECT", Db]))) of 390 | ok -> 391 | case gen_tcp:recv(Sock, 0) of 392 | {ok, <<"+OK\r\n">>} -> ok; 393 | {ok, Err} -> {error, Err}; 394 | Err -> Err 395 | end; 396 | Err -> 397 | Err 398 | end. 399 | 400 | test_connection(#state{sock=undefined}=State) -> 401 | connect(State); 402 | 403 | test_connection(State) -> 404 | State. 405 | 406 | process_packet(#state{acc=Acc, queue=Queue, subscriber=Subscriber}=State, Packet) -> 407 | case redo_redis_proto:parse(Acc, Packet) of 408 | %% the reply has been received in its entirety 409 | {ok, Result, Rest} -> 410 | case queue:out(Queue) of 411 | {{value, {Pid, Ref}}, Queue1} -> 412 | send_response(Pid, Ref, Result, Rest, State, Queue1); 413 | {empty, Queue1} -> 414 | case Subscriber of 415 | {Pid, Ref} -> 416 | send_response(Pid, Ref, Result, Rest, State, Queue1); 417 | undefined -> 418 | {error, queue_empty} 419 | end 420 | end; 421 | %% the current reply packet ended abruptly 422 | %% we must wait for the next data packet 423 | {eof, Acc1, Rest} -> 424 | {ok, State#state{acc=Acc1, buffer=Rest}} 425 | end. 426 | 427 | send_response(Pid, Ref, Result, Rest, State, Queue) -> 428 | Pid ! {Ref, Result}, 429 | case Rest of 430 | {raw, <<>>} -> 431 | %% we have reached the end of this tcp packet 432 | %% wait for the next incoming packet 433 | {ok, State#state{acc=[], queue=Queue, buffer = {raw, <<>>}}}; 434 | _ -> 435 | %% there is still data left in this packet 436 | %% we may begin processing the next reply 437 | process_packet(State#state{acc=[], queue=Queue}, packet(Rest, <<>>)) 438 | end. 439 | 440 | packet({raw, Buffer}, Data) -> 441 | {raw, <>}; 442 | 443 | packet({multi_bulk, N, Buffer}, Data) -> 444 | {multi_bulk, N, <>}. 445 | 446 | close_connection(State = #state{queue=Queue}) -> 447 | %% notify all waiting pids that the connection is closed 448 | %% so that they may try resending their requests 449 | [Pid ! {Ref, closed} || {Pid, Ref} <- queue:to_list(Queue)], 450 | 451 | %% notify subscriber pid of disconnect 452 | case State#state.subscriber of 453 | {Pid, Ref} -> Pid ! {Ref, closed}; 454 | _ -> ok 455 | end, 456 | 457 | %% reset the state 458 | State#state{ 459 | queue = queue:new(), 460 | cancelled = [], 461 | buffer = {raw, <<>>} 462 | }. 463 | -------------------------------------------------------------------------------- /src/redo_block.erl: -------------------------------------------------------------------------------- 1 | %% Manages multiple requests and timers. May or may not kill it if the task 2 | %% takes too long, and then just start a new worker instead. 3 | %% 4 | %% Subscription mode isn't supported yet. 5 | -module(redo_block). 6 | -behaviour(gen_server). 7 | 8 | %% gen_server callbacks 9 | -export([init/1, handle_call/3, handle_cast/2, 10 | handle_info/2, terminate/2, code_change/3]). 11 | 12 | -export([start_link/0, start_link/1, start_link/2, 13 | cmd/1, cmd/2, cmd/3, shutdown/1]). 14 | -export([reply/2]). 15 | 16 | -record(state, {opts=[], refs, acc, 17 | worker, worker_status=free, 18 | timers, queue}). 19 | 20 | -define(TIMEOUT, 5000). 21 | 22 | %%%%%%%%%%%%%%%%% 23 | %%% INTERFACE %%% 24 | %%%%%%%%%%%%%%%%% 25 | start_link() -> 26 | start_link([]). 27 | 28 | start_link(Name) when is_atom(Name) -> 29 | start_link(Name, []); 30 | start_link(Opts) when is_list(Opts) -> 31 | start_link(?MODULE, Opts). 32 | 33 | start_link(undefined, Opts) when is_list(Opts) -> 34 | gen_server:start_link(?MODULE, Opts, []); 35 | start_link(Name, Opts) when is_atom(Name), is_list(Opts) -> 36 | gen_server:start_link({local, Name}, ?MODULE, Opts, []). 37 | 38 | 39 | cmd(Cmd) -> 40 | cmd(?MODULE, Cmd, ?TIMEOUT). 41 | 42 | cmd(NameOrPid, Cmd) -> 43 | cmd(NameOrPid, Cmd, ?TIMEOUT). 44 | 45 | cmd(NameOrPid, Cmd, Timeout) -> 46 | %% format commands to be sent to redis | now done in the worker 47 | %% Packets = redo_redis_proto:package(Cmd), 48 | gen_server:call(NameOrPid, {cmd, Cmd, Timeout}, infinity). 49 | 50 | shutdown(NameOrPid) -> 51 | gen_server:cast(NameOrPid, shutdown). 52 | 53 | reply({Pid, TimerRef}, Response) -> 54 | gen_server:cast(Pid, {done, self(), TimerRef, Response}). 55 | 56 | %%%%%%%%%%%%%%%%% 57 | %%% CALLBACKS %%% 58 | %%%%%%%%%%%%%%%%% 59 | init(Opts) -> 60 | process_flag(trap_exit, true), % deal with worker failures 61 | {ok, Worker} = redo:start_link(undefined, Opts), 62 | {ok, #state{opts=Opts, 63 | worker=Worker, worker_status=free, 64 | timers=gb_trees:empty(), queue=queue:new()}}. 65 | 66 | handle_call({cmd, Cmd, TimeOut}, From, S=#state{worker_status=free, timers=T, queue=Q}) -> 67 | TimerRef = erlang:start_timer(TimeOut, self(), From), 68 | handle_info(timeout, S#state{timers=gb_trees:insert(TimerRef, {From,Cmd}, T), 69 | queue=queue:in(TimerRef,Q)}); 70 | handle_call({cmd, Cmd, TimeOut}, From, S=#state{worker_status=busy, timers=T, queue=Q}) -> 71 | TimerRef = erlang:start_timer(TimeOut, self(), From), 72 | {noreply, S#state{timers=gb_trees:insert(TimerRef, {From,Cmd}, T), 73 | queue=queue:in(TimerRef,Q)}, 0}. 74 | 75 | handle_cast({done, TimerRef, Result}, S=#state{timers=T, queue=Q}) -> 76 | erlang:cancel_timer(TimerRef), 77 | case queue:out(Q) of 78 | {empty, _} -> % response likely came too late 79 | {noreply, S#state{worker_status=free, refs=undefined, acc=undefined}}; 80 | {{value,TimerRef}, Queue} -> % done, drop top element 81 | {From, _} = gb_trees:get(TimerRef, T), 82 | gen_server:reply(From, Result), 83 | {noreply, S#state{worker_status = free, 84 | refs = undefined, 85 | acc = undefined, 86 | timers = gb_trees:delete(TimerRef, T), 87 | queue = Queue}, 0}; 88 | {{value,_Ref}, _} -> % race condition, timed out answer? 89 | %% Let's clean up just in case 90 | Queue = queue:filter(fun(Ref) -> Ref =/= TimerRef end, Q), 91 | {noreply, S#state{worker_status = free, 92 | timers = gb_trees:delete_any(TimerRef, T), 93 | queue = Queue}, 0} 94 | end; 95 | handle_cast(shutdown, State) -> 96 | {stop, {shutdown,cast}, State}. 97 | 98 | handle_info(timeout, S=#state{worker_status=busy}) -> 99 | %% Task scheduled. We'll get a message when it's done 100 | {noreply, S}; 101 | handle_info(timeout, S=#state{worker_status=free}) -> 102 | case schedule(S) of 103 | {ok, State} -> {noreply, State}; 104 | {error, Reason} -> {stop, {worker,Reason}, S} 105 | end; 106 | handle_info({timeout, TimerRef, From}, S=#state{worker=Pid, timers=T, queue=Q}) -> 107 | case queue:out(Q) of 108 | {empty, _} -> % potential bad timer management? 109 | {noreply, S}; 110 | {{value,TimerRef}, Queue} -> % currently being handled 111 | unlink(Pid), 112 | exit(Pid, shutdown), 113 | {From, _} = gb_trees:get(TimerRef, T), 114 | gen_server:reply(From, {error, timeout}), 115 | case schedule(S#state{worker = undefined, 116 | worker_status = free, 117 | refs = undefined, 118 | acc = undefined, 119 | timers = gb_trees:delete(TimerRef, T), 120 | queue = Queue}) of 121 | {ok, State} -> {noreply, State}; 122 | {error, Reason} -> {stop, {worker,Reason}, S} 123 | end; 124 | {_, _} -> % somewhere in the queue, or maybe not if raced before cancel 125 | case gb_trees:lookup(TimerRef, T) of 126 | none -> 127 | {noreply, S}; 128 | {value, {From, _}} -> % same as the one we got 129 | gen_server:reply(From, {error, timeout}), 130 | Queue = queue:filter(fun(Ref) -> Ref =/= TimerRef end, Q), 131 | {noreply, S#state{timers = gb_trees:delete(TimerRef, T), 132 | queue = Queue}} 133 | end 134 | end; 135 | handle_info({'EXIT', Pid, _Reason}, S=#state{worker=Pid}) -> 136 | %% The worker died. We retry until the timeout fails and the task is pulled 137 | %% from the queue. 138 | case schedule(S#state{worker=undefined, worker_status=free, 139 | refs=undefined, acc=undefined}) of 140 | {ok, State} -> {noreply, State}; 141 | {error, Reason} -> {stop, {worker, Reason}, S} 142 | end; 143 | 144 | %% Handling responses from redo 145 | handle_info({Ref, Val}, S=#state{refs=[Ref], acc=Acc, queue=Q}) -> 146 | {value, TimerRef} = queue:peek(Q), 147 | Result = case {Val,Acc} of % if Acc [], one single req, drop list. 148 | {closed, []} -> {error,closed}; 149 | {closed, _} -> lists:reverse([{error,closed} | Acc]); 150 | {_, []} -> Val; 151 | _ -> lists:reverse([Val|Acc]) 152 | end, 153 | gen_server:cast(self(), {done, TimerRef, Result}), 154 | {noreply, S#state{refs=undefined, acc=undefined}}; 155 | handle_info({Ref, closed}, S=#state{refs=[Ref|Refs], acc=Acc}) -> 156 | {noreply, S#state{refs=Refs, acc=[{error, closed} | Acc]}}; 157 | handle_info({Ref, Val}, S=#state{refs=[Ref|Refs], acc=Acc}) -> 158 | {noreply, S#state{refs=Refs, acc=[Val | Acc]}}; 159 | handle_info({_BadRef,_}, State) -> 160 | {noreply, State}. 161 | 162 | terminate(_, #state{worker=undefined}) -> 163 | ok; 164 | terminate(_, #state{worker=Pid}) -> 165 | exit(Pid, shutdown), 166 | ok. 167 | 168 | code_change(_OldVsn, State, _Extra) -> 169 | {ok, State}. 170 | 171 | %%%%%%%%%%%%%%% 172 | %%% PRIVATE %%% 173 | %%%%%%%%%%%%%%% 174 | schedule(S=#state{worker=undefined, opts=Opts}) -> 175 | case redo:start_link(undefined, Opts) of 176 | {ok, Worker} -> 177 | schedule(S#state{worker=Worker, worker_status=free}); 178 | {stop, Err} -> 179 | {error, Err} 180 | end; 181 | schedule(S=#state{worker=Pid, worker_status=free, timers=T, queue=Q}) -> 182 | case queue:peek(Q) of 183 | empty -> % nothing to schedule, keep idling. 184 | {ok, S}; 185 | {value,TimerRef} -> 186 | {_From, Cmd} = gb_trees:get(TimerRef, T), 187 | Refs = redo:async_cmd(Pid, Cmd), 188 | {ok, S#state{worker_status=busy, refs=Refs, acc=[]}} 189 | end. 190 | 191 | -------------------------------------------------------------------------------- /src/redo_concurrency_test.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(redo_concurrency_test). 24 | -export([run/2]). 25 | 26 | run(NumPids, NumOps) -> 27 | io:format("================================================================================~n"), 28 | io:format("== THIS TEST RELIES ON A LOCAL REDIS SERVER RUNNING ON localhost:6379 ==~n"), 29 | io:format("================================================================================~n"), 30 | 31 | random:seed(now()), 32 | 33 | case whereis(redo) of 34 | undefined -> 35 | {ok, _Pid} = redo:start_link(); 36 | _ -> 37 | ok 38 | end, 39 | 40 | <<"OK">> = redo:cmd(["SELECT", "6"]), 41 | <<"OK">> = redo:cmd(["FLUSHDB"]), 42 | 43 | Self = self(), 44 | Pids = [spawn_link(fun() -> io:format("spawned ~w~n", [N]), worker(Self, N, NumOps) end) || N <- lists:seq(1, NumPids)], 45 | 46 | [receive {Pid, N, done} -> io:format("finished ~w~n", [N]) end || Pid <- Pids], 47 | 48 | ok. 49 | 50 | worker(Parent, N, 0) -> 51 | Parent ! {self(), N, done}; 52 | 53 | worker(Parent, N, NumOps) -> 54 | StrN = integer_to_list(N), 55 | StrOp = integer_to_list(NumOps), 56 | case random:uniform(100) of 57 | R when R > 0, R =< 24 -> 58 | Key = iolist_to_binary([StrN, ":", StrOp, ":STRING"]), 59 | Val = iolist_to_binary(["STRING:", StrN, ":", StrOp]), 60 | <<"OK">> = redo:cmd(["SET", Key, Val]), 61 | Val = redo:cmd(["GET", Key]); 62 | R when R > 24, R =< 48 -> 63 | Key = iolist_to_binary([StrN, ":", StrOp, ":SET"]), 64 | Val1 = iolist_to_binary(["SET:1:", StrN, ":", StrOp]), 65 | Val2 = iolist_to_binary(["SET:2:", StrN, ":", StrOp]), 66 | 1 = redo:cmd(["SADD", Key, Val1]), 67 | 1 = redo:cmd(["SADD", Key, Val2]), 68 | Set = redo:cmd(["SMEMBERS", Key]), 69 | true = lists:member(Val1, Set), 70 | true = lists:member(Val2, Set); 71 | R when R > 48, R =< 72 -> 72 | Key = iolist_to_binary([StrN, ":", StrOp, ":HASH"]), 73 | Vals = [iolist_to_binary(["HASH:1:", StrN, ":Key:", StrOp]), 74 | iolist_to_binary(["HASH:1:", StrN, ":Val:", StrOp]), 75 | iolist_to_binary(["HASH:2:", StrN, ":Key", StrOp]), 76 | iolist_to_binary(["HASH:2:", StrN, ":Val", StrOp])], 77 | <<"OK">> = redo:cmd(["HMSET", Key | Vals]), 78 | Vals = redo:cmd(["HGETALL", Key]); 79 | R when R > 72, R =< 98 -> 80 | Key1 = iolist_to_binary([StrN, ":", StrOp, ":PIPESET"]), 81 | Val1 = iolist_to_binary(["PIPESET:1:", StrN, ":", StrOp]), 82 | Val2 = iolist_to_binary(["PIPESET:2:", StrN, ":", StrOp]), 83 | 1 = redo:cmd(["SADD", Key1, Val1]), 84 | 1 = redo:cmd(["SADD", Key1, Val2]), 85 | Key2 = iolist_to_binary([StrN, ":", StrOp, ":PIPEHASH"]), 86 | Vals = [iolist_to_binary(["PIPEHASH:1:", StrN, ":Key:", StrOp]), 87 | iolist_to_binary(["PIPEHASH:1:", StrN, ":Val:", StrOp]), 88 | iolist_to_binary(["PIPEHASH:2:", StrN, ":Key", StrOp]), 89 | iolist_to_binary(["PIPEHASH:2:", StrN, ":Val", StrOp])], 90 | <<"OK">> = redo:cmd(["HMSET", Key2 | Vals]), 91 | [Set, Hash] = redo:cmd([["SMEMBERS", Key1], ["HGETALL", Key2]]), 92 | true = lists:member(Val1, Set), 93 | true = lists:member(Val2, Set), 94 | Vals = Hash; 95 | R when R > 98 -> 96 | Keys = [begin 97 | Key = iolist_to_binary([StrN, ":", StrOp, ":", string:right(integer_to_list(ItemNum), 200, $0)]), 98 | <<"OK">> = redo:cmd(["SET", Key, "0"]), 99 | Key 100 | end || ItemNum <- lists:seq(1,1000)], 101 | Keys1 = redo:cmd(["KEYS", iolist_to_binary([StrN, ":", StrOp, ":*"])]), 102 | L = length(Keys), 103 | L1 = length(Keys1), 104 | L = L1 105 | end, 106 | worker(Parent, N, NumOps-1). 107 | 108 | -------------------------------------------------------------------------------- /src/redo_logging.hrl: -------------------------------------------------------------------------------- 1 | %%% Author : Geoff Cant 2 | %%% Description : Logging macros 3 | %%% Created : 13 Jan 2006 by Geoff Cant 4 | 5 | -ifndef(logging_macros). 6 | -define(logging_macros, true). 7 | 8 | -define(INFO(Format, Args), 9 | io:format("pid=~p m=~p ln=~p class=info " ++ Format ++ "~n", 10 | [self(), ?MODULE, ?LINE | Args])). 11 | -define(WARN(Format, Args), 12 | io:format("pid=~p m=~p ln=~p class=warn " ++ Format ++ "~n", 13 | [self(), ?MODULE, ?LINE | Args])). 14 | -define(ERR(Format, Args), 15 | io:format("pid=~p m=~p ln=~p class=err " ++ Format ++ "~n", 16 | [self(), ?MODULE, ?LINE | Args])). 17 | 18 | -endif. %logging 19 | -------------------------------------------------------------------------------- /src/redo_redis_proto.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(redo_redis_proto). 24 | -export([package/1, parse/2]). 25 | 26 | -define(CRLF, <<"\r\n">>). 27 | 28 | -spec package(binary() | list()) -> list(). 29 | %% packet is already a binary 30 | package(Packet) when is_binary(Packet) -> 31 | [Packet]; 32 | 33 | %% list of strings - single cmd 34 | package([[Char|_]|_]=Args) when is_integer(Char) -> 35 | [build_request(Args)]; 36 | 37 | %% list of binaries - single cmd 38 | package([Bin|_]=Args) when is_binary(Bin) -> 39 | [build_request(Args)]; 40 | 41 | %% list of multiple cmds 42 | package(Args) when is_list(Args) -> 43 | build_pipelined_request(Args, []). 44 | 45 | build_request(Args) -> 46 | Count = length(Args), 47 | Args1 = [begin 48 | Arg1 = to_arg(Arg), 49 | [<<"$">>, integer_to_list(iolist_size(Arg1)), ?CRLF, Arg1, ?CRLF] 50 | end || Arg <- Args], 51 | iolist_to_binary(["*", integer_to_list(Count), ?CRLF, Args1, ?CRLF]). 52 | 53 | build_pipelined_request([], Acc) -> 54 | lists:reverse(Acc); 55 | 56 | build_pipelined_request([Args|Rest], Acc) -> 57 | build_pipelined_request(Rest, [build_request(Args)|Acc]). 58 | 59 | to_arg(List) when is_list(List) -> 60 | List; 61 | 62 | to_arg(Bin) when is_binary(Bin) -> 63 | Bin; 64 | 65 | to_arg(Int) when is_integer(Int) -> 66 | integer_to_list(Int); 67 | 68 | to_arg(Atom) when is_atom(Atom) -> 69 | atom_to_list(Atom). 70 | 71 | -spec parse(list(), {raw, binary()} | {multi_bulk, integer(), binary()}) -> 72 | {ok, undefined, {raw, binary()}} | 73 | {ok, binary(), {raw, binary()}} | 74 | {eof, list(), {raw, binary()}} | 75 | {ok, {error, term()}, {raw, binary()}}. 76 | %% Single line reply 77 | parse(Acc, {raw, <<"+", Rest/binary>> = Data}) -> 78 | case read_line(Rest) of 79 | {ok, Str, Rest1} -> 80 | {ok, Str, {raw, Rest1}}; 81 | {error, eof} -> 82 | {eof, Acc, {raw, Data}} 83 | end; 84 | 85 | %% Error msg reply 86 | parse(Acc, {raw, <<"-", Rest/binary>> = Data}) -> 87 | case read_line(Rest) of 88 | {ok, Err, Rest1} -> 89 | {ok, {error, Err}, {raw, Rest1}}; 90 | {error, eof} -> 91 | {eof, Acc, {raw, Data}} 92 | end; 93 | 94 | %% Integer reply 95 | parse(Acc, {raw, <<":", Rest/binary>> = Data}) -> 96 | case read_line(Rest) of 97 | {ok, Int, Rest1} -> 98 | Val = list_to_integer(binary_to_list(Int)), 99 | {ok, Val, {raw, Rest1}}; 100 | {error, eof} -> 101 | {eof, Acc, {raw, Data}} 102 | end; 103 | 104 | %% Bulk reply 105 | parse(Acc, {raw, <<"$", Rest/binary>> = Data}) -> 106 | case read_line(Rest) of 107 | {ok, BinSize, Rest1} -> 108 | Size = list_to_integer(binary_to_list(BinSize)), 109 | case Size >= 0 of 110 | true -> 111 | case Rest1 of 112 | <> -> 113 | {ok, Str, {raw, Rest2}}; 114 | _ -> 115 | {eof, Acc, {raw, Data}} 116 | end; 117 | false -> 118 | {ok, undefined, {raw, Rest1}} 119 | end; 120 | {error, eof} -> 121 | {eof, Acc, {raw, Data}} 122 | end; 123 | 124 | %% Multi bulk reply 125 | parse(Acc, {raw, <<"*", Rest/binary>> = Data}) -> 126 | case read_line(Rest) of 127 | {ok, BinNum, Rest1} -> 128 | Num = list_to_integer(binary_to_list(BinNum)), 129 | parse(Acc, {multi_bulk, Num, Rest1}); 130 | {error, eof} -> 131 | {eof, Acc, {raw, Data}} 132 | end; 133 | 134 | parse(Acc, {multi_bulk, Num, Data}) -> 135 | multi_bulk(Acc, Num, Data). 136 | 137 | read_line(Data) -> 138 | read_line(Data, <<>>). 139 | 140 | read_line(<<"\r\n", Rest/binary>>, Acc) -> 141 | {ok, Acc, Rest}; 142 | 143 | read_line(<<>>, _Acc) -> 144 | {error, eof}; 145 | 146 | read_line(<>, Acc) -> 147 | read_line(Rest, <>). 148 | 149 | multi_bulk(Acc, 0, Rest) -> 150 | {ok, lists:reverse(Acc), {raw, Rest}}; 151 | 152 | multi_bulk(Acc, Num, <<>>) -> 153 | {eof, Acc, {multi_bulk, Num, <<>>}}; 154 | 155 | multi_bulk(Acc, Num, Rest) -> 156 | case parse(Acc, {raw, Rest}) of 157 | {ok, Result, {raw, Rest1}} -> 158 | multi_bulk([Result|Acc], Num-1, Rest1); 159 | {eof, Acc1, _} -> 160 | {eof, Acc1, {multi_bulk, Num, Rest}} 161 | end. 162 | 163 | -------------------------------------------------------------------------------- /src/redo_uri.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(redo_uri). 24 | -export([parse/1]). 25 | 26 | -spec parse(binary() | list()) -> list(). 27 | parse(Url) when is_binary(Url) -> 28 | parse(binary_to_list(Url)); 29 | 30 | parse("redis://" ++ Rest) -> 31 | parse(Rest); 32 | 33 | parse(":" ++ Rest) -> 34 | parse(Rest); 35 | 36 | % redis://:password@localhost:6379/1 37 | parse(Url) when is_list(Url) -> 38 | {Pass, Rest1} = pass(Url), 39 | {Host, Rest2} = host(Rest1), 40 | {Port, Db} = port(Rest2), 41 | [{host, Host} || Host =/= ""] ++ 42 | [{port, Port} || Port =/= ""] ++ 43 | [{pass, Pass} || Pass =/= ""] ++ 44 | [{db, Db} || Db =/= ""]. 45 | 46 | pass("") -> 47 | {"", ""}; 48 | 49 | pass(Url) -> 50 | case string:tokens(Url, "@") of 51 | [Rest] -> {"", Rest}; 52 | [Pass, Rest] -> {Pass, Rest}; 53 | [Pass | Rest] -> {Pass, string:join(Rest, "@")} 54 | end. 55 | 56 | host("") -> 57 | {"", ""}; 58 | 59 | host(Url) -> 60 | case string:tokens(Url, ":") of 61 | [Rest] -> 62 | case string:tokens(Rest, "/") of 63 | [Host] -> {Host, ""}; 64 | [Host, Rest1] -> {Host, "/" ++ Rest1}; 65 | [Host | Rest1] -> {Host, string:join(Rest1, "/")} 66 | end; 67 | [Host, Rest] -> {Host, Rest}; 68 | [Host | Rest] -> {Host, string:join(Rest, ":")} 69 | end. 70 | 71 | port("") -> 72 | {"", ""}; 73 | 74 | port("/" ++ Rest) -> 75 | {"", Rest}; 76 | 77 | port(Url) -> 78 | case string:tokens(Url, "/") of 79 | [Port] -> {list_to_integer(Port), ""}; 80 | [Port, Rest] -> {list_to_integer(Port), Rest}; 81 | [Port | Rest] -> {list_to_integer(Port), string:join(Rest, "/")} 82 | end. 83 | -------------------------------------------------------------------------------- /test/redo_block_tests.erl: -------------------------------------------------------------------------------- 1 | -module(redo_block_tests). 2 | -include_lib("eunit/include/eunit.hrl"). 3 | 4 | %% copy/pasted 5 | -record(state, {opts=[], refs, acc, 6 | worker, worker_status=free, 7 | timers, queue}). 8 | 9 | -ifndef(HOST). 10 | -define(HOST, "127.0.0.1"). 11 | -endif. 12 | 13 | -ifndef(PORT). 14 | -define(PORT, 6379). 15 | -endif. 16 | 17 | -ifndef(PASSWORD). 18 | -define(PASSWORD, undefined). 19 | -endif. 20 | 21 | -ifndef(DB). 22 | -define(DB, undefined). 23 | -endif. 24 | 25 | config() -> 26 | %% This suite assumes there is a local redis available. Can be defined 27 | %% through macros. 28 | [{host, ?HOST}, {port, ?PORT}] ++ 29 | case ?PASSWORD of 30 | undefined -> []; 31 | _ -> [{pass, ?PASSWORD}] 32 | end ++ 33 | case ?DB of 34 | undefined -> []; 35 | _ -> [{db, ?DB}] 36 | end. 37 | 38 | start() -> 39 | application:start(sasl), 40 | {ok, Pid} = redo_block:start_link(undefined, config()), 41 | unlink(Pid), 42 | Pid. 43 | 44 | stop(Pid) -> 45 | redo_block:shutdown(Pid). 46 | 47 | seq_test_() -> 48 | {"Try basic commands in sequence, try to get the right value", 49 | {setup, 50 | fun start/0, 51 | fun stop/1, 52 | fun(Pid) -> 53 | Key = lists:flatten(io_lib:format("~p-~p", [make_ref(),now()])), 54 | Cmds = [redo_block:cmd(Pid, ["EXISTS", Key]), 55 | redo_block:cmd(Pid, ["SET", Key, "1"]), 56 | redo_block:cmd(Pid, ["GET", Key]), 57 | redo_block:cmd(Pid, ["SET", Key, "2"]), 58 | redo_block:cmd(Pid, ["GET", Key]), 59 | redo_block:cmd(Pid, ["DEL", Key]), 60 | redo_block:cmd(Pid, ["EXISTS", Key])], 61 | ?_assertEqual([0, <<"OK">>, <<"1">>, <<"OK">>, <<"2">>, 1, 0], 62 | Cmds) 63 | end}}. 64 | 65 | parallel_test_() -> 66 | {"Running tests in parallel. They end up being sequential, but all the " 67 | "results should be in the right order.", 68 | {setup, 69 | fun start/0, 70 | fun stop/1, 71 | fun(Pid) -> 72 | Parent = self(), 73 | Keygen = fun() -> 74 | iolist_to_binary(io_lib:format("~p-~p", [make_ref(),now()])) 75 | end, 76 | %% Takes a unique key, inserts it, then reads it. With hundreds of concurrent 77 | %% processes, if redo_block doesn't behave well, the val read might be different 78 | %% from the val written. 79 | Proc = fun() -> 80 | Key = Keygen(), 81 | redo_block:cmd(Pid, ["SET", Key, Key]), 82 | R = redo_block:cmd(Pid, ["GET", Key]), 83 | redo_block:cmd(Pid, ["DEL", Key]), 84 | Parent ! {Key, R} 85 | end, 86 | N = 10000, 87 | _ = [spawn_link(Proc) || _ <- lists:seq(1,N)], 88 | Res = [receive Msg -> Msg end || _ <- lists:seq(1,N)], 89 | ?_assertEqual(N, length([ok || {Key,Key} <- Res])) 90 | end}}. 91 | 92 | timeout_test_() -> 93 | {"A request timeout should kill the connection and start a new one " 94 | "without necessarily killing follow-up requests or messing up " 95 | "with the final ordering of responses", 96 | {setup, 97 | fun start/0, 98 | fun stop/1, 99 | fun(Pid) -> 100 | Key = lists:flatten(io_lib:format("~p-~p", [make_ref(),now()])), 101 | Cmds = [redo_block:cmd(Pid, ["EXISTS", Key]), 102 | redo_block:cmd(Pid, ["SET", Key, "1"]), 103 | redo_block:cmd(Pid, ["GET", Key], 0), 104 | redo_block:cmd(Pid, ["SET", Key, "2"]), 105 | redo_block:cmd(Pid, ["GET", Key]), 106 | redo_block:cmd(Pid, ["DEL", Key]), 107 | redo_block:cmd(Pid, ["EXISTS", Key])], 108 | ?_assertMatch([0, <<"OK">>, _TimeoutOrVal, <<"OK">>, <<"2">>, 1, 0], 109 | Cmds) 110 | end}}. 111 | 112 | parallel_timeout_test_() -> 113 | {"Running tests in parallel. They end up being sequential, but all the " 114 | "results should be in the right order, even if requests time out all " 115 | "the time.", 116 | {setup, 117 | fun start/0, 118 | fun stop/1, 119 | fun(Pid) -> 120 | Parent = self(), 121 | Keygen = fun() -> 122 | iolist_to_binary(io_lib:format("~p-~p", [make_ref(),now()])) 123 | end, 124 | %% Takes a unique key, inserts it, then reads it. With hundreds of concurrent 125 | %% processes, if redo_block doesn't behave well, the val read might be different 126 | %% from the val written. 127 | Proc = fun() -> 128 | Key = Keygen(), 129 | redo_block:cmd(Pid, ["SET", Key, Key], 30000), 130 | R1 = redo_block:cmd(Pid, ["GET", Key], 30000), 131 | R2 = redo_block:cmd(Pid, ["GET", Key], 0), 132 | R3 = redo_block:cmd(Pid, ["GET", Key], 30000), 133 | redo_block:cmd(Pid, ["DEL", Key], 60000), 134 | Parent ! {Key,R1,R2,R3} 135 | end, 136 | N = 10000, 137 | _ = [spawn_link(Proc) || _ <- lists:seq(1,N)], 138 | Res = [receive Msg -> Msg end || _ <- lists:seq(1,N)], 139 | ?_assertEqual(N, length([ok || {Key,Key,TimeoutOrVal,Key} <- Res, 140 | TimeoutOrVal == {error,timeout} orelse 141 | TimeoutOrVal == Key])) 142 | end}}. 143 | 144 | timeout_new_conn_test() -> 145 | Pid = start(), 146 | {_,_,_,Props1} = sys:get_status(Pid), 147 | _ = [redo_block:cmd(Pid, ["EXISTS", "1"], 0) || _ <- lists:seq(1,1000)], 148 | {_,_,_,Props2} = sys:get_status(Pid), 149 | stop(Pid), 150 | [#state{worker=Pid1}] = [State || X = [_|_] <- Props1, 151 | {data,[{"State", State}]} <- X], 152 | [#state{worker=Pid2}] = [State || X = [_|_] <- Props2, 153 | {data,[{"State", State}]} <- X], 154 | ?assertNotEqual(Pid1, Pid2). 155 | -------------------------------------------------------------------------------- /test/redo_tests.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2011 Jacob Vorreuter 2 | %% 3 | %% Permission is hereby granted, free of charge, to any person 4 | %% obtaining a copy of this software and associated documentation 5 | %% files (the "Software"), to deal in the Software without 6 | %% restriction, including without limitation the rights to use, 7 | %% copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | %% copies of the Software, and to permit persons to whom the 9 | %% Software is furnished to do so, subject to the following 10 | %% conditions: 11 | %% 12 | %% The above copyright notice and this permission notice shall be 13 | %% included in all copies or substantial portions of the Software. 14 | %% 15 | %% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | %% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | %% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | %% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | %% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | %% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | %% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | %% OTHER DEALINGS IN THE SOFTWARE. 23 | -module(redo_tests). 24 | -include_lib("eunit/include/eunit.hrl"). 25 | 26 | redis_uri_test() -> 27 | ?assertEqual([{host, "127.0.0.1"}, 28 | {port, 666}, 29 | {pass, "password"}, 30 | {db, "1"}], 31 | redo_uri:parse("redis://:password@127.0.0.1:666/1")), 32 | 33 | ?assertEqual([{host, "127.0.0.1"}, 34 | {port, 666}, 35 | {pass, "password"}, 36 | {db, "1"}], 37 | redo_uri:parse("redis://password@127.0.0.1:666/1")), 38 | 39 | ?assertEqual([{host, "127.0.0.1"}, 40 | {pass, "password"}], 41 | redo_uri:parse("redis://password@127.0.0.1")), 42 | 43 | ?assertEqual([{host, "127.0.0.1"}, 44 | {port, 666}, 45 | {db, "1"}], 46 | redo_uri:parse("redis://127.0.0.1:666/1")), 47 | 48 | ?assertEqual([{host, "127.0.0.1"}, 49 | {db, "1"}], 50 | redo_uri:parse("redis://127.0.0.1/1")), 51 | 52 | ?assertEqual([{host, "127.0.0.1"}, 53 | {port, 666}], 54 | redo_uri:parse("redis://127.0.0.1:666")), 55 | 56 | ?assertEqual([{host, "127.0.0.1"}], redo_uri:parse("redis://127.0.0.1")), 57 | 58 | ?assertEqual([], redo_uri:parse("redis://")), 59 | 60 | ?assertEqual([], redo_uri:parse("")), 61 | 62 | ok. 63 | 64 | redis_proto_test() -> 65 | redo_redis_proto:parse( 66 | fun(Val) -> ?assertEqual(undefined, Val) end, 67 | {raw, <<"$-1\r\n">>}), 68 | 69 | redo_redis_proto:parse( 70 | fun(Val) -> ?assertEqual(<<>>, Val) end, 71 | {raw, <<"$0\r\n\r\n">>}), 72 | 73 | redo_redis_proto:parse( 74 | fun(Val) -> ?assertEqual(<<"OK">>, Val) end, 75 | {raw, <<"+OK\r\n">>}), 76 | 77 | redo_redis_proto:parse( 78 | fun(Val) -> ?assertEqual({error, <<"Error">>}, Val) end, 79 | {raw, <<"-Error\r\n">>}), 80 | 81 | redo_redis_proto:parse( 82 | fun(Val) -> ?assertEqual(<<"FOO">>, Val) end, 83 | {raw, <<"$3\r\nFOO\r\n">>}), 84 | 85 | redo_redis_proto:parse( 86 | fun(Val) -> ?assertEqual(1234, Val) end, 87 | {raw, <<":1234\r\n">>}), 88 | 89 | ok. 90 | --------------------------------------------------------------------------------