├── .upgrade_from_build ├── rebar ├── test.sh ├── .gitignore ├── start.sh ├── rebar.config ├── rel ├── files │ ├── sys.config │ ├── flake │ ├── vm.args │ └── erl └── reltool.config ├── apps └── flake │ └── src │ ├── flake.app.src │ ├── flake.hrl │ ├── flake_app.erl │ ├── flake_harness.erl │ ├── flake.erl │ ├── flake_server.erl │ ├── persistent_timer.erl │ ├── flake_sup.erl │ └── flake_util.erl └── readme.md /.upgrade_from_build: -------------------------------------------------------------------------------- 1 | 21 2 | 3 | -------------------------------------------------------------------------------- /rebar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/boundary/flake/HEAD/rebar -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # rebar clean compile generate 4 | rebar eunit app=flake -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ebin/ 2 | .eunit/ 3 | *.dump 4 | deps/ 5 | rel/flake/ 6 | .DS_Store 7 | 8 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | rebar get-deps clean compile generate && ./rel/flake/bin/flake 4 | -------------------------------------------------------------------------------- /rebar.config: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | {sub_dirs, ["apps/flake", "rel"]}. 3 | {erl_opts, [debug_info]}. 4 | {cover_enabled, false}. 5 | -------------------------------------------------------------------------------- /rel/files/sys.config: -------------------------------------------------------------------------------- 1 | [ 2 | {flake, [ 3 | {interface, "en0"}, 4 | {timestamp_path, "/tmp/flake-timestamp-dets"}, 5 | {allowable_downtime, 2592000000} 6 | ]} 7 | ]. 8 | -------------------------------------------------------------------------------- /apps/flake/src/flake.app.src: -------------------------------------------------------------------------------- 1 | {application, flake, 2 | [ 3 | {description, "flake"}, 4 | {vsn, "0.7"}, 5 | {modules, []}, 6 | {registered, []}, 7 | {applications, [ 8 | kernel, 9 | stdlib, 10 | crypto 11 | ]}, 12 | {mod, { flake_app, []}}, 13 | {env, []} 14 | ]}. 15 | -------------------------------------------------------------------------------- /apps/flake/src/flake.hrl: -------------------------------------------------------------------------------- 1 | -ifdef(TEST). 2 | -define(LOG_INFO(F), _ = [F]). 3 | -define(LOG_ERROR(F), _ = [F]). 4 | -define(LOG_INFO_FORMAT(F, A), _ = [F, A]). 5 | -define(LOG_ERROR_FORMAT(F, A), _ = [F, A]). 6 | -else. 7 | -define(LOG_INFO(F), 8 | error_logger:info_msg(F)). 9 | -define(LOG_ERROR(F), 10 | error_logger:error_msg(F)). 11 | -define(LOG_INFO_FORMAT(F, A), 12 | error_logger:info_msg(F, A)). 13 | -define(LOG_ERROR_FORMAT(F, A), 14 | error_logger:error_msg(F, A)). 15 | -endif. 16 | -------------------------------------------------------------------------------- /rel/files/flake: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ERTS_BIN_DIR=$(cd ${0%/*} && pwd) 4 | 5 | export ROOTDIR=${ERTS_BIN_DIR%/*} 6 | 7 | START_ERL=`cat $ROOTDIR/releases/start_erl.data` 8 | ERTS_VSN=${START_ERL% *} 9 | APP_VSN=${START_ERL#* } 10 | 11 | export BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 12 | export EMU=beam 13 | export PROGNAME=`echo $0 | sed 's/.*\///'` 14 | 15 | exec $BINDIR/erlexec -boot $ROOTDIR/releases/$APP_VSN/flake \ 16 | -args_file $ROOTDIR/releases/vm.args \ 17 | -config $ROOTDIR/releases/sys.config -------------------------------------------------------------------------------- /rel/files/vm.args: -------------------------------------------------------------------------------- 1 | ## Name of the node 2 | -name flake@127.0.0.1 3 | #-name tmp 4 | 5 | ## Cookie for distributed erlang 6 | -setcookie flake 7 | 8 | ## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive 9 | ## (Disabled by default..use with caution!) 10 | ##-heart 11 | 12 | ## Enable kernel poll and a few async threads 13 | +K true 14 | +A 5 15 | 16 | ## Increase number of concurrent ports/sockets 17 | -env ERL_MAX_PORTS 4096 18 | 19 | ## Tweak GC to run more often 20 | -env ERL_FULLSWEEP_AFTER 10 21 | 22 | #+Bd -noinput -------------------------------------------------------------------------------- /rel/reltool.config: -------------------------------------------------------------------------------- 1 | {sys, [ 2 | {lib_dirs, ["../apps"]}, 3 | {rel, "flake", "0.1", 4 | [ 5 | kernel, 6 | stdlib, 7 | sasl, 8 | inets, 9 | crypto, 10 | flake 11 | ]}, 12 | {rel, "start_clean", "", 13 | [ 14 | kernel, 15 | stdlib 16 | ]}, 17 | {boot_rel, "flake"}, 18 | {profile, embedded}, 19 | {excl_sys_filters, ["^bin/.*", 20 | "^erts.*/bin/(dialyzer|typer)"]}, 21 | {excl_archive_filters, [".*"]}, 22 | {app, flake, [{incl_cond, include}]}, 23 | {app, sasl, [{incl_cond, include}]} 24 | ]}. 25 | 26 | {target_dir, "flake"}. 27 | {clean_files, "flake"}. 28 | {overlay, [ 29 | {copy, "files/erl", "{{erts_vsn}}/bin/erl"}, 30 | {copy, "files/sys.config", "releases/sys.config"}, 31 | {copy, "files/vm.args", "releases/vm.args"}, 32 | {copy, "files/flake", "bin/flake"} 33 | ]}. 34 | -------------------------------------------------------------------------------- /rel/files/erl: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ## This script replaces the default "erl" in erts-VSN/bin. This is necessary 4 | ## as escript depends on erl and in turn, erl depends on having access to a 5 | ## bootscript (start.boot). Note that this script is ONLY invoked as a side-effect 6 | ## of running escript -- the embedded node bypasses erl and uses erlexec directly 7 | ## (as it should). 8 | ## 9 | ## Note that this script makes the assumption that there is a start_clean.boot 10 | ## file available in $ROOTDIR/release/VSN. 11 | 12 | # Determine the abspath of where this script is executing from. 13 | ERTS_BIN_DIR=$(cd ${0%/*} && pwd) 14 | 15 | # Now determine the root directory -- this script runs from erts-VSN/bin, 16 | # so we simply need to strip off two dirs from the end of the ERTS_BIN_DIR 17 | # path. 18 | ROOTDIR=${ERTS_BIN_DIR%/*/*} 19 | 20 | # Parse out release and erts info 21 | START_ERL=`cat $ROOTDIR/releases/start_erl.data` 22 | ERTS_VSN=${START_ERL% *} 23 | APP_VSN=${START_ERL#* } 24 | 25 | BINDIR=$ROOTDIR/erts-$ERTS_VSN/bin 26 | EMU=beam 27 | PROGNAME=`echo $0 | sed 's/.*\\///'` 28 | CMD="$BINDIR/erlexec" 29 | export EMU 30 | export ROOTDIR 31 | export BINDIR 32 | export PROGNAME 33 | 34 | exec $CMD -boot $ROOTDIR/releases/$APP_VSN/start_clean ${1+"$@"} -------------------------------------------------------------------------------- /apps/flake/src/flake_app.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright 2012, Boundary 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | %% @doc Callbacks for the snowflake application. 18 | 19 | -module (flake_app). 20 | -author ('Dietrich Featherston '). 21 | 22 | -behaviour (application). 23 | 24 | -export([ 25 | start/2, 26 | stop/1 27 | ]). 28 | 29 | -include_lib ("eunit/include/eunit.hrl"). 30 | 31 | %% @spec start(_Type, _StartArgs) -> ServerRet 32 | %% @doc application start callback for snowflake. 33 | start(_Type, _StartArgs) -> 34 | flake_sup:start_link(). 35 | 36 | %% @spec stop(_State) -> ServerRet 37 | %% @doc application stop callback for snowflake. 38 | stop(_State) -> 39 | ok. 40 | -------------------------------------------------------------------------------- /apps/flake/src/flake_harness.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright 2012, Boundary 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module (flake_harness). 18 | -author ('Dietrich Featherston '). 19 | 20 | -export ([ 21 | generate/1, 22 | generate/2, 23 | timed_generate/1, 24 | timed_generate/2 25 | ]). 26 | 27 | -include_lib("eunit/include/eunit.hrl"). 28 | 29 | generate(N) -> 30 | generate_ids(N, undefined, []). 31 | 32 | timed_generate(N) -> 33 | ?debugTime("generating ids", generate(N)). 34 | 35 | generate(N, Base) -> 36 | generate_ids(N, Base, []). 37 | 38 | timed_generate(N, Base) -> 39 | ?debugTime("generating ids", generate(N, Base)). 40 | 41 | generate_ids(0, _Base, Acc) -> 42 | Acc; 43 | generate_ids(N, Base, Acc) -> 44 | {ok, Flake} = case Base of 45 | undefined -> 46 | flake_server:id(); 47 | _ -> 48 | flake_server:id(Base) 49 | end, 50 | generate_ids(N-1, Base, [Flake|Acc]). 51 | -------------------------------------------------------------------------------- /apps/flake/src/flake.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright 2012, Boundary 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module (flake). 18 | -author ('Dietrich Featherston '). 19 | 20 | %%==================================================================== 21 | %% API 22 | %%==================================================================== 23 | -export([ 24 | start/0, 25 | start_link/0, 26 | stop/0, 27 | get_config_value/2 28 | ]). 29 | 30 | -include ("flake.hrl"). 31 | 32 | -include_lib ("eunit/include/eunit.hrl"). 33 | 34 | %% @spec start_link() -> {ok,Pid::pid()} 35 | %% @doc Starts the app for inclusion in a supervisor tree 36 | start_link() -> 37 | flake_sup:start_link(). 38 | 39 | %% @spec start() -> ok 40 | %% @doc Start the snowflake server. 41 | start() -> 42 | application:start(flake). 43 | 44 | %% @spec stop() -> ok 45 | %% @doc Stop the snowflake server. 46 | stop() -> 47 | Res = application:stop(flake), 48 | Res. 49 | 50 | get_config_value(Key, Default) -> 51 | case application:get_env(flake, Key) of 52 | {ok, Value} -> Value; 53 | _ -> Default 54 | end. 55 | -------------------------------------------------------------------------------- /apps/flake/src/flake_server.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright 2012, Boundary 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module (flake_server). 18 | -author ('Dietrich Featherston '). 19 | 20 | -behaviour (gen_server). 21 | 22 | %% API 23 | -export ([ 24 | start_link/1, 25 | id/0, 26 | id/1 27 | ]). 28 | 29 | %% gen_server callbacks 30 | -export ([ 31 | init/1, 32 | handle_call/3, 33 | handle_cast/2, 34 | handle_info/2, 35 | terminate/2, 36 | code_change/3 37 | ]). 38 | 39 | -include_lib ("eunit/include/eunit.hrl"). 40 | 41 | -record (state, {max_time, worker_id, sequence}). 42 | 43 | 44 | %% ---------------------------------------------------------- 45 | %% API 46 | %% ---------------------------------------------------------- 47 | 48 | % start and link to a new flake id generator 49 | start_link(Config) -> 50 | gen_server:start_link({local,flake}, ?MODULE, Config, []). 51 | 52 | % generate a new snowflake id 53 | id() -> 54 | respond(gen_server:call(flake, get)). 55 | 56 | id(Base) -> 57 | respond(gen_server:call(flake, {get, Base})). 58 | 59 | respond({ok,Flake}) -> 60 | {ok,Flake}; 61 | respond(X) -> 62 | X. 63 | 64 | 65 | %% ---------------------------------------------------------- 66 | %% gen_server callbacks 67 | %% ---------------------------------------------------------- 68 | 69 | init([{worker_id, WorkerId}]) -> 70 | {ok, #state{max_time=flake_util:curr_time_millis(), worker_id=WorkerId, sequence=0}}. 71 | 72 | handle_call(get, _From, State = #state{max_time=MaxTime, worker_id=WorkerId, sequence=Sequence}) -> 73 | {Resp, S0} = get(flake_util:curr_time_millis(), MaxTime, WorkerId, Sequence, State), 74 | {reply, Resp, S0}; 75 | 76 | handle_call({get,Base}, _From, State = #state{max_time=MaxTime,worker_id=WorkerId,sequence=Sequence}) -> 77 | {Resp, S0} = get(flake_util:curr_time_millis(), MaxTime, WorkerId, Sequence, State), 78 | case Resp of 79 | {ok, Id} -> 80 | <> = Id, 81 | {reply, {ok, flake_util:as_list(IntId, Base)}, S0}; 82 | E -> 83 | {reply, E, S0} 84 | end; 85 | 86 | handle_call(X, _From, State) -> 87 | error_logger:error_msg("unrecognized msg in ~p:handle_call -> ~p~n",[?MODULE, X]), 88 | {reply, ok, State}. 89 | 90 | handle_cast(_, State) -> {noreply, State}. 91 | 92 | handle_info(_, State) -> {noreply, State}. 93 | 94 | terminate(_Reason, _State) -> ok. 95 | 96 | code_change(_, State, _) -> {ok, State}. 97 | 98 | %% clock hasn't moved, increment sequence 99 | get(Time,Time,WorkerId,Seq0,State) -> 100 | Sequence = Seq0 + 1, 101 | {{ok,flake_util:gen_id(Time,WorkerId,Sequence)},State#state{sequence=Sequence}}; 102 | %% clock has progressed, reset sequence 103 | get(CurrTime,MaxTime,WorkerId,_,State) when CurrTime > MaxTime -> 104 | {{ok, flake_util:gen_id(CurrTime, WorkerId, 0)}, State#state{max_time=CurrTime, sequence=0}}; 105 | %% clock is running backwards 106 | get(CurrTime, MaxTime, _WorkerId, _Sequence, State) when MaxTime > CurrTime -> 107 | {{error, clock_running_backwards}, State}. 108 | -------------------------------------------------------------------------------- /apps/flake/src/persistent_timer.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright 2012, Boundary 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module (persistent_timer). 18 | -author ('Dietrich Featherston '). 19 | 20 | -behaviour (gen_server). 21 | 22 | -export ([ 23 | start_link/1, 24 | write_timestamp/1, 25 | read_timestamp/1, 26 | get_last_timestamp/0 27 | ]). 28 | 29 | %% gen_server callbacks 30 | -export ([ 31 | init/1, 32 | handle_call/3, 33 | handle_cast/2, 34 | handle_info/2, 35 | terminate/2, 36 | code_change/3 37 | ]). 38 | 39 | -include_lib("eunit/include/eunit.hrl"). 40 | 41 | -record (state, {table, timer}). 42 | 43 | %% ---------------------------------------------------------- 44 | %% API 45 | %% ---------------------------------------------------------- 46 | 47 | % start and link to a new flake id generator 48 | start_link(Config) -> 49 | gen_server:start_link({local, ?MODULE}, ?MODULE, Config, []). 50 | 51 | get_last_timestamp() -> 52 | gen_server:call(?MODULE, get_last_timestamp). 53 | 54 | 55 | %% ---------------------------------------------------------- 56 | %% gen_server callbacks 57 | %% ---------------------------------------------------------- 58 | 59 | init(Config) -> 60 | Table = proplists:get_value(table, Config), 61 | Interval = proplists:get_value(interval,Config, 1000), 62 | {ok, TimerRef} = timer:send_interval(Interval, save), 63 | {ok,#state{table=Table, timer=TimerRef}}. 64 | 65 | handle_call(get_last_timestamp, _From, State = #state{table=Table}) -> 66 | {reply, read_timestamp(Table), State}. 67 | 68 | handle_cast(_, State) -> {noreply, State}. 69 | 70 | handle_info(save, State = #state{table=Table}) -> 71 | {ok, _} = write_timestamp(Table), 72 | {noreply, State}. 73 | 74 | terminate(_Reason, _State) -> ok. 75 | 76 | code_change(_, State, _) -> {ok, State}. 77 | 78 | %% ---------------------------------------------------------- 79 | %% utils 80 | %% ---------------------------------------------------------- 81 | 82 | %% write the current time stamp to disk 83 | %% {ok,Timestamp=int()} | {error,Reason} 84 | write_timestamp(Table) -> 85 | TS = flake_util:curr_time_millis(), 86 | ok = dets:insert(Table,{last_timestamp,TS}), 87 | {ok,TS}. 88 | 89 | %% read the timestamp from the given file. will write the current timestamp to disk if the file does not exist 90 | %% {ok,Timestamp=int()} | {error,Reason} 91 | read_timestamp(Table) -> 92 | case dets:lookup(Table,last_timestamp) of 93 | [{last_timestamp,TS}] when is_integer(TS) -> 94 | {ok,TS}; 95 | _ -> 96 | write_timestamp(Table) 97 | end. 98 | 99 | 100 | %% ---------------------------------------------------------- 101 | %% tests 102 | %% ---------------------------------------------------------- 103 | 104 | persistent_clock_test() -> 105 | {ok,Table} = 106 | dets:open_file(timestamp_table,[ 107 | {estimated_no_objects,10}, 108 | {type,set}, 109 | {file,"/tmp/timestamp-dets"} 110 | ]), 111 | {ok,TS0} = write_timestamp(Table), 112 | {ok,TS1} = read_timestamp(Table), 113 | ?assert(?debugVal(TS0) =:= ?debugVal(TS1)). 114 | -------------------------------------------------------------------------------- /apps/flake/src/flake_sup.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright 2012, Boundary 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module (flake_sup). 18 | -author('Dietrich Featherston '). 19 | -include("flake.hrl"). 20 | 21 | -include_lib("eunit/include/eunit.hrl"). 22 | 23 | -define (DEBUG,debug). 24 | 25 | -behaviour(supervisor). 26 | 27 | %% External exports 28 | -export([start_link/0, upgrade/0]). 29 | 30 | %% supervisor callbacks 31 | -export([init/1]). 32 | 33 | %% @spec start_link() -> ServerRet 34 | %% @doc API for starting the supervisor. 35 | start_link() -> 36 | supervisor:start_link({local, ?MODULE}, ?MODULE, []). 37 | 38 | %% @spec upgrade() -> ok 39 | %% @doc Add processes if necessary. 40 | upgrade() -> 41 | {ok, {_, Specs}} = init([]), 42 | 43 | Old = sets:from_list([Name || {Name, _, _, _} <- supervisor:which_children(?MODULE)]), 44 | New = sets:from_list([Name || {Name, _, _, _, _, _} <- Specs]), 45 | Kill = sets:subtract(Old, New), 46 | 47 | sets:fold( 48 | fun(Id, ok) -> 49 | supervisor:terminate_child(?MODULE, Id), 50 | supervisor:delete_child(?MODULE, Id), 51 | ok 52 | end, ok, Kill), 53 | 54 | [supervisor:start_child(?MODULE, Spec) || Spec <- Specs], 55 | ok. 56 | 57 | %% @spec init([]) -> SupervisorTree 58 | %% @doc supervisor callback. 59 | init([]) -> 60 | DefaultIf = flake_util:get_default_if(), 61 | If = flake:get_config_value(interface, DefaultIf), 62 | error_logger:info_msg("starting flake with hardware address of ~p as worker id~n", [If]), 63 | {ok,WorkerId} = flake_util:get_if_hw_int(If), 64 | error_logger:info_msg("using worker id: ~p~n", [WorkerId]), 65 | 66 | FlakeConfig = [ 67 | {worker_id, WorkerId} 68 | ], 69 | Flake = {flake, 70 | {flake_server, start_link, [FlakeConfig]}, 71 | permanent, 5000, worker, [flake_server]}, 72 | 73 | TimestampPath = flake:get_config_value(timestamp_path, "/tmp/flake-timestamp-dets"), 74 | AllowableDowntime = flake:get_config_value(allowable_downtime, 0), 75 | 76 | {ok, TimestampTable} = 77 | dets:open_file(timestamp_table,[ 78 | {estimated_no_objects, 10}, 79 | {type, set}, 80 | {file, TimestampPath} 81 | ]), 82 | 83 | {ok,TS} = persistent_timer:read_timestamp(TimestampTable), 84 | ?debugVal(TS), 85 | Now = flake_util:curr_time_millis(), 86 | ?debugVal(Now), 87 | TimeSinceLastRun = Now - TS, 88 | 89 | %% fail startup if 90 | %% 1) the clock time last recorded is later than the current time 91 | %% 2) the last recorded time is more than N ms in the past to prevent 92 | %% generating future ids in the event that the system clock is set to some point far in the future 93 | check_for_clock_error(Now >= TS, TimeSinceLastRun < AllowableDowntime), 94 | 95 | error_logger:info_msg("saving timestamps to ~p every 1s~n", [TimestampPath]), 96 | TimerConfig = [ 97 | {table, TimestampTable}, 98 | {interval, 1000} 99 | ], 100 | PersistentTimer = {persistent_timer, 101 | {persistent_timer,start_link,[TimerConfig]}, 102 | permanent, 5000, worker, [persistent_timer]}, 103 | 104 | {ok, { {one_for_one, 10, 10}, [Flake, PersistentTimer]} }. 105 | 106 | check_for_clock_error(true,true) -> 107 | ok; 108 | check_for_clock_error(false,_) -> 109 | error_logger:error_msg("system running backwards, failing startup of snowflake service~n"), 110 | exit(clock_running_backwards); 111 | check_for_clock_error(_,false) -> 112 | error_logger:error_msg("system clock too far advanced, failing startup of snowflake service~n"), 113 | exit(clock_advanced). 114 | -------------------------------------------------------------------------------- /apps/flake/src/flake_util.erl: -------------------------------------------------------------------------------- 1 | %%% 2 | %%% Copyright 2012, Boundary 3 | %%% 4 | %%% Licensed under the Apache License, Version 2.0 (the "License"); 5 | %%% you may not use this file except in compliance with the License. 6 | %%% You may obtain a copy of the License at 7 | %%% 8 | %%% http://www.apache.org/licenses/LICENSE-2.0 9 | %%% 10 | %%% Unless required by applicable law or agreed to in writing, software 11 | %%% distributed under the License is distributed on an "AS IS" BASIS, 12 | %%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | %%% See the License for the specific language governing permissions and 14 | %%% limitations under the License. 15 | %%% 16 | 17 | -module (flake_util). 18 | -author ('Dietrich Featherston '). 19 | 20 | -export ([ 21 | as_list/2, 22 | get_default_if/0, 23 | get_if_hw_int/1, 24 | hw_addr_to_int/1, 25 | curr_time_millis/0, 26 | gen_id/3 27 | ]). 28 | 29 | -include_lib("eunit/include/eunit.hrl"). 30 | 31 | %% get a reasonable default interface that has a valid mac address 32 | get_default_if() -> 33 | {ok, SysIfs} = inet:getifaddrs(), 34 | Ifs = [I || {I, Props} <- SysIfs, filter_if(Props)], 35 | hd(Ifs). 36 | 37 | % filter network interfaces 38 | filter_if(Props) -> 39 | HwAddr = proplists:get_value(hwaddr, Props), 40 | filter_hwaddr(HwAddr). 41 | 42 | % we exclude interfaces without a MAC address 43 | filter_hwaddr(undefined) -> 44 | false; 45 | % we exclude interfaces with a null MAC address, ex: loopback devices 46 | filter_hwaddr([0,0,0,0,0,0]) -> 47 | false; 48 | % all others are valid interfaces to pick from 49 | filter_hwaddr(_) -> 50 | true. 51 | 52 | %% get the mac/hardware address of the given interface as a 48-bit integer 53 | get_if_hw_int(undefined) -> 54 | {error, if_not_found}; 55 | get_if_hw_int(IfName) -> 56 | {ok, IfAddrs} = inet:getifaddrs(), 57 | IfProps = proplists:get_value(IfName, IfAddrs), 58 | case IfProps of 59 | undefined -> 60 | {error, if_not_found}; 61 | _ -> 62 | HwAddr = proplists:get_value(hwaddr, IfProps), 63 | {ok, hw_addr_to_int(HwAddr)} 64 | end. 65 | 66 | %% convert an array of 6 bytes into a 48-bit integer 67 | hw_addr_to_int(HwAddr) -> 68 | <> = erlang:list_to_binary(HwAddr), 69 | WorkerId. 70 | 71 | curr_time_millis() -> 72 | {MegaSec,Sec, MicroSec} = os:timestamp(), 73 | 1000000000*MegaSec + Sec*1000 + erlang:trunc(MicroSec/1000). 74 | 75 | gen_id(Time,WorkerId,Sequence) -> 76 | <>. 77 | 78 | %% 79 | %% n.b. - unique_id_62/0 and friends pulled from riak 80 | %% 81 | 82 | %% @spec integer_to_list(Integer :: integer(), Base :: integer()) -> 83 | %% string() 84 | %% @doc Convert an integer to its string representation in the given 85 | %% base. Bases 2-62 are supported. 86 | as_list(I, 10) -> 87 | erlang:integer_to_list(I); 88 | as_list(I, Base) 89 | when is_integer(I), 90 | is_integer(Base), 91 | Base >= 2, 92 | Base =< 1+$Z-$A+10+1+$z-$a -> 93 | if 94 | I < 0 -> 95 | [$-|as_list(-I, Base, [])]; 96 | true -> 97 | as_list(I, Base, []) 98 | end; 99 | as_list(I, Base) -> 100 | erlang:error(badarg, [I, Base]). 101 | 102 | %% @spec integer_to_list(integer(), integer(), stringing()) -> string() 103 | as_list(I0, Base, R0) -> 104 | D = I0 rem Base, 105 | I1 = I0 div Base, 106 | R1 = 107 | if 108 | D >= 36 -> 109 | [D-36+$a|R0]; 110 | D >= 10 -> 111 | [D-10+$A|R0]; 112 | true -> 113 | [D+$0|R0] 114 | end, 115 | if 116 | I1 =:= 0 -> 117 | R1; 118 | true -> 119 | as_list(I1, Base, R1) 120 | end. 121 | 122 | 123 | %% ---------------------------------------------------------- 124 | %% tests 125 | %% ---------------------------------------------------------- 126 | 127 | flake_test() -> 128 | TS = flake_util:curr_time_millis(), 129 | Worker = flake_util:hw_addr_to_int(lists:seq(1, 6)), 130 | Flake = flake_util:gen_id(TS, Worker, 0), 131 | <> = Flake, 132 | ?assert(?debugVal(Time) =:= TS), 133 | ?assert(?debugVal(Worker) =:= WorkerId), 134 | ?assert(?debugVal(Sequence) =:= 0), 135 | <> = Flake, 136 | ?debugVal(flake_util:as_list(FlakeInt, 62)), 137 | ok. 138 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Flake: A decentralized, k-ordered id generation service in Erlang 2 | 3 | 4 | Flake produces 128-bit, k-ordered ids (read time-ordered lexically). Run one on each node in your infrastructure and they will generate conflict-free ids on-demand without coordination. 5 | 6 | Read the original [post](http://archive.is/2015.07.08-082503/http://www.boundary.com/blog/2012/01/flake-a-decentralized-k-ordered-unique-id-generator-in-erlang/) on the Boundary blog. 7 | 8 | To get started 9 | 10 | git clone git://github.com/boundary/flake.git 11 | 12 | Then edit rel/files/sys.config to fit your environment. 13 | 14 | * interface - set to an available network interface from which to pull a 48-bit mac address as the worker id. 15 | * timestamp_path - set to a location where flake can periodically save the current time. If flake detects on startup that this file contains timestamps in the future or the distant past, it will refuse to startup. This is to prevent problematic ids from being distributed. 16 | * allowable_downtime - an added safeguard to prevent flake from starting up if it sees it hasn't run in a long period of time according to the system clock since this might be an indication that the clock has been skewed far into the future. 17 | 18 | Example configuration: 19 | 20 | ```erlang 21 | [ 22 | {flake, [ 23 | {interface, "en0"}, 24 | {timestamp_path, "/srv/flake/timestamp-dets"}, 25 | {allowable_downtime, 2592000000} 26 | ]} 27 | ] 28 | ``` 29 | 30 | Then simply run the server inline 31 | 32 | ./start.sh 33 | 34 | And use the embedded test harness to ensure that you are able to generate ids. 35 | 36 | Generate 1 id and receive the erlang binary 37 | (flake@localhost)1> flake_harness:generate(1). 38 | 39 | [<<0,0,1,52,212,33,45,67,16,154,221,94,14,143,0,0>>] 40 | 41 | Generate 10 base-62 encoded ids 42 | 43 | (flake@localhost)2> flake_harness:generate(10,62). 44 | 45 | ["8HFaR8qWtRlGDHnO57","8HFaR8qWtRlGDHnO56", 46 | "8HFaR8qWtRlGDHnO55","8HFaR8qWtRlGDHnO54", 47 | "8HFaR8qWtRlGDHnO53","8HFaR8qWtRlGDHnO52", 48 | "8HFaR8qAulTgCBd6Wp","8HFaR8qAulTgCBd6Wo", 49 | "8HFaR8qAulTgCBd6Wn","8HFaR8qAulTgCBd6Wm"] 50 | 51 | Time how long it takes to generate 100,000 ids 52 | 53 | (flake@localhost)3> flake_harness:timed_generate(100000). 54 | 55 | src/flake_harness.erl:33: generating ids: 0.402 s 56 | 57 | These last steps simple ensure that a flake application is up and running. Next we'll talk more about operational use. 58 | 59 | 60 | # Deployment 61 | 62 | Flake is a standalone application. Request ids with a gen_server:call from another Erlang VM (or application that speaks Erlang distribution a la [Scalang](https://github.com/boundary/scalang)). 63 | 64 | Example usage from your application. 65 | 66 | ```erlang 67 | flake() -> 68 | Node = {flake, flake@localhost}, 69 | {ok, FlakeId} = gen_server:call(Node, get), 70 | {ok, FlakeIdBase62} = gen_server:call(Node, {get,62}), 71 | %% example id decomposition for demonstration only 72 | <<_Time:64/integer,_WorkerId:48/integer,_Sequence:16/integer>> = FlakeId, 73 | FlakeId. 74 | ``` 75 | 76 | # Anatomy 77 | 78 | Flake ids are 128-bits wide described here from most significant to least significant bits. 79 | 80 | * 64-bit timestamp - milliseconds since the epoch (Jan 1 1970) 81 | * 48-bit worker id - MAC address from a configurable device 82 | * 16-bit sequence # - usually 0, incremented when more than one id is requested in the same millisecond and reset to 0 when the clock ticks forward 83 | 84 | 85 | # Roadmap 86 | 87 | * Bulk id generation 88 | * HTTP interface 89 | * Client library (Erlang, possibly others) 90 | * Provide an abstraction for the messaging format to avoid using gen_server calls as the public API. 91 | * Will generate a notional flake id (with an empty worker id) given a timestamp. This is useful for generating a lexical range of values that could have been generated in a span of time. At Boundary we store data in Riak keyed by flake ids and use keyfilters to page out blocks of data using this technique. 92 | 93 | 94 | # Frequently Asked Questions 95 | 96 | **What if I'm not using Erlang?** 97 | 98 | An HTTP interface is on the roadmap. For applications written in Scala, [Scalang](https://github.com/boundary/scalang) is an option. 99 | 100 | **How does this differ from snowflake developed at Twitter?** 101 | 102 | The differences stem primarily from the fact that Twitter snowflake ids are 64-bits wide. This means that additional coordination is required to pick a worker id (twitter does this via a ZooKeeper ensemble). Their scheme works great when your ids must fit into 64-bits. However this comes at the cost of additional coordination among nodes and a system that is generally a little more difficult to reason about. It is a fine system though and we were able to learn from it in our efforts. 103 | 104 | **How is flake different from rearranging the bits of a UUID-1 id?** 105 | 106 | First, successive UUID versions aim to make ids increasingly _opaque_ in nature through various means. We have actually found a great deal of utility in structurally transparent unique ids and that has motivated much of this work. The most transparent variant is UUID-1 (for which it has received a fair bit of criticism) and thus the nearest relative to a flake id. There are some important differences though. 107 | 108 | UUID-1 is an odd beast. First, the timestamp is based on the number of 100 nanosecond intervals since October 15, 1582. This is not how most of us familiar with Unix timestamps reason about time. If that isn't bad enough, the timestamp is an odd 60-bits in length with the most significant bits shifted to the least significant bits of the UUID. This property makes lexical ordering essentially meaningless. The remaining bits contain a clock id (initially set to a random number) and a node id (usually the MAC address). 109 | 110 | The first problem is the timestamp. We could rearrange the bits to get some k-ordering love, but reasoning on timestamps of this nature makes reasoning about the resulting ids more complex than it needs to be. This is why flake uses a standard 64-bit Unix timestamp, unaltered, as the most significant bits. 111 | 112 | The next problem is clock skew and protection against replaying ids for a time in the past. The UUID-1 spec dictates that the clock id be incremented in such an event, but this behavior is implementation-specific and we aren't aware of any Erlang implementations that met our safety goals. Flake durably writes a timestamp to a dets table periodically while running. Following a restart, flake will refuse to startup if the timestamp written there is from the future. Furthermore, flake will refuse to generate ids if it detects that the system clock is running backwards. 113 | 114 | **When are flake ids _not_ appropriate?** 115 | 116 | Flake ids are predictable by design. Don't use use flake to generate ids that you'd rather be unpredictable. Don't use flake to generate passwords, security tokens, or anything else you wouldn't want someone to be able to guess. 117 | 118 | Flake ids expose the identity of the machine which generated the id (by way of its MAC address) and the time at which it did so. This could be a problem for some security-sensitive applications. 119 | 120 | Don't do modulo 2 arithmetic on flake ids with the expectation of random distribution. The least significant 16-bits are usually going to be 0 on a machine that is generating an average of one or fewer ids per millisecond. 121 | 122 | 123 | --------------------------------------------------------------------------------