├── .gitignore ├── config └── config.exs ├── bench └── snowflake.exs ├── changelog.md ├── mix.lock ├── LICENSE ├── lib ├── snowflake.ex └── snowflake │ ├── generator.ex │ ├── util.ex │ └── helper.ex ├── mix.exs └── readme.md /.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /deps 3 | /.dialyzer 4 | erl_crash.dump 5 | *.ez 6 | doc 7 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :snowflake, 4 | nodes: ["127.0.0.1"], 5 | epoch: 1473562509301 6 | -------------------------------------------------------------------------------- /bench/snowflake.exs: -------------------------------------------------------------------------------- 1 | Application.ensure_all_started(:snowflake) 2 | 3 | # To test against snowflakex, uncomment the second line in the benchmark 4 | # and also add snowflakex as a dependency to your mix.exs 5 | 6 | Benchee.run(%{ 7 | "snowflake" => fn -> Snowflake.next_id() end, 8 | # "snowflakex" => fn -> Snowflakex.new() end 9 | }) 10 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v 1.0.3 4 | 5 | - Fix warnings. 6 | 7 | ## v 1.0.2 8 | 9 | - Added ability to define a specific machine_id instead of using the node list index mechanism. 10 | 11 | ## v 1.0.1 12 | 13 | - Introduced a helpful util function to get a snowflake for a specific time. 14 | 15 | ## v 1.0.0 16 | 17 | - Updated API, swapped Util and Helper module names to be more standard 18 | 19 | ## v 0.0.2 20 | 21 | - added Helper.real_timestamp_of_id 22 | - added default configuration so app doesn’t crash if user didn’t set config 23 | - added Erlang Node name as an option for nodes 24 | - updated documentation 25 | 26 | ## v 0.0.1 27 | 28 | Initial Release 29 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{"benchee": {:hex, :benchee, "0.6.0", "c2565506c621ee010e71d05f555e39a1b937e00810e284bc85463a4d4efc4b00", [:mix], [{:deep_merge, "~> 0.1", [hex: :deep_merge, optional: false]}]}, 2 | "deep_merge": {:hex, :deep_merge, "0.1.1", "c27866a7524a337b6a039eeb8dd4f17d458fd40fbbcb8c54661b71a22fffe846", [:mix], []}, 3 | "dialyxir": {:hex, :dialyxir, "0.4.4", "e93ff4affc5f9e78b70dc7bec7b07da44ae1ed3fef38e7113568dd30ad7b01d3", [:mix], []}, 4 | "earmark": {:hex, :earmark, "1.1.1", "433136b7f2e99cde88b745b3a0cfc3fbc81fe58b918a09b40fce7f00db4d8187", [:mix], []}, 5 | "ex_doc": {:hex, :ex_doc, "0.14.5", "c0433c8117e948404d93ca69411dd575ec6be39b47802e81ca8d91017a0cf83c", [:mix], [{:earmark, "~> 1.0", [hex: :earmark, optional: false]}]}, 6 | "snowflakex": {:hex, :snowflakex, "1.0.0", "a621dc8004625b178f4248cf5ea412c1f83d84f89b7bdc2479f4019bd26c34d4", [:mix], []}} 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Blitz Studios, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /lib/snowflake.ex: -------------------------------------------------------------------------------- 1 | defmodule Snowflake do 2 | @moduledoc """ 3 | Generates Snowflake IDs 4 | """ 5 | use Application 6 | 7 | def start(_type, _args) do 8 | import Supervisor.Spec 9 | 10 | children = [ 11 | worker(Snowflake.Generator, [Snowflake.Helper.epoch(), Snowflake.Helper.machine_id()]) 12 | ] 13 | 14 | Supervisor.start_link(children, strategy: :one_for_one) 15 | end 16 | 17 | @doc """ 18 | Generates a snowflake ID, each call is guaranteed to return a different ID 19 | that is sequantially larger than the previous ID. 20 | """ 21 | @spec next_id() :: {:ok, integer} | 22 | {:error, :backwards_clock} 23 | def next_id() do 24 | GenServer.call(Snowflake.Generator, :next_id) 25 | end 26 | 27 | @doc """ 28 | Returns the machine id of the current node. 29 | """ 30 | @spec machine_id() :: {:ok, integer} 31 | def machine_id() do 32 | GenServer.call(Snowflake.Generator, :machine_id) 33 | end 34 | 35 | @doc """ 36 | Returns the machine id of the current node. 37 | """ 38 | @spec set_machine_id(integer) :: {:ok, integer} 39 | def set_machine_id(machine_id) do 40 | GenServer.call(Snowflake.Generator, {:set_machine_id, machine_id}) 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Snowflake.Mixfile do 2 | use Mix.Project 3 | 4 | @version "1.0.4" 5 | @url "https://github.com/blitzstudios/snowflake" 6 | @maintainers ["Weixi Yen"] 7 | 8 | def project do 9 | [ 10 | name: "Snowflake", 11 | app: :snowflake, 12 | version: @version, 13 | source_url: @url, 14 | build_embedded: Mix.env == :prod, 15 | start_permanent: Mix.env == :prod, 16 | maintainers: @maintainers, 17 | description: "Elixir Snowflake ID Generator", 18 | elixir: "~> 1.3", 19 | package: package(), 20 | homepage_url: @url, 21 | docs: docs(), 22 | deps: deps() 23 | ] 24 | end 25 | 26 | def application do 27 | [applications: [], 28 | mod: {Snowflake, []}] 29 | end 30 | 31 | defp deps do 32 | [ 33 | {:dialyxir, "~> 0.4", only: :dev, runtime: false}, 34 | {:benchee, "~> 0.6", only: :dev}, 35 | {:ex_doc, "~> 0.14", only: :dev} 36 | ] 37 | end 38 | 39 | def docs do 40 | [ 41 | extras: ["README.md", "CHANGELOG.md"], 42 | source_ref: "v#{@version}" 43 | ] 44 | end 45 | 46 | defp package do 47 | [ 48 | maintainers: @maintainers, 49 | licenses: ["MIT"], 50 | links: %{github: @url}, 51 | files: ~w(lib) ++ ~w(CHANGELOG.md LICENSE mix.exs README.md) 52 | ] 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/snowflake/generator.ex: -------------------------------------------------------------------------------- 1 | defmodule Snowflake.Generator do 2 | @moduledoc false 3 | use GenServer 4 | 5 | @machine_id_overflow 1024 6 | @seq_overflow 4096 7 | 8 | def start_link(epoch, machine_id) when machine_id < @machine_id_overflow do 9 | state = {epoch, ts(epoch), machine_id, 0} 10 | GenServer.start_link(__MODULE__, state, name: __MODULE__) 11 | end 12 | 13 | def init(args) do 14 | {:ok, args} 15 | end 16 | 17 | def handle_call(:next_id, from, {epoch, prev_ts, machine_id, seq} = state) do 18 | case next_ts_and_seq(epoch, prev_ts, seq) do 19 | {:error, :seq_overflow} -> 20 | :timer.sleep(1) 21 | handle_call(:next_id, from, state) 22 | {:error, :backwards_clock} -> 23 | {:reply, {:error, :backwards_clock}, state} 24 | {:ok, new_ts, new_seq} -> 25 | new_state = {epoch, new_ts, machine_id, new_seq} 26 | {:reply, {:ok, create_id(new_ts, machine_id, new_seq)}, new_state} 27 | end 28 | end 29 | 30 | def handle_call(:machine_id, _from, {_epoch, _prev_ts, machine_id, _seq} = state) do 31 | {:reply, {:ok, machine_id}, state} 32 | end 33 | 34 | def handle_call({:set_machine_id, machine_id}, _from, {epoch, prev_ts, _old_machine_id, seq}) do 35 | {:reply, {:ok, machine_id}, {epoch, prev_ts, machine_id, seq}} 36 | end 37 | 38 | defp next_ts_and_seq(epoch, prev_ts, seq) do 39 | case ts(epoch) do 40 | ^prev_ts -> 41 | case seq + 1 do 42 | @seq_overflow -> {:error, :seq_overflow} 43 | next_seq -> {:ok, prev_ts, next_seq} 44 | end 45 | new_ts -> 46 | cond do 47 | new_ts < prev_ts -> {:error, :backwards_clock} 48 | true -> {:ok, new_ts, 0} 49 | end 50 | end 51 | end 52 | 53 | defp create_id(ts, machine_id, seq) do 54 | << new_id :: unsigned-integer-size(64)>> = << 55 | ts :: unsigned-integer-size(42), 56 | machine_id :: unsigned-integer-size(10), 57 | seq :: unsigned-integer-size(12) >> 58 | 59 | new_id 60 | end 61 | 62 | defp ts(epoch) do 63 | System.os_time(:millisecond) - epoch 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /lib/snowflake/util.ex: -------------------------------------------------------------------------------- 1 | defmodule Snowflake.Util do 2 | @moduledoc """ 3 | The Util module helps users work with snowflake IDs. 4 | 5 | Util module can do the following: 6 | - Deriving timestamp based on ID 7 | - Creating buckets based on days since epoch... 8 | - get real timestamp of any ID 9 | """ 10 | use Bitwise 11 | 12 | @doc """ 13 | First Snowflake for timestamp, useful if you have a timestamp and want 14 | to find snowflakes before or after a certain millesecond 15 | """ 16 | @spec first_snowflake_for_timestamp(integer) :: integer 17 | def first_snowflake_for_timestamp(timestamp) do 18 | ts = timestamp - Snowflake.Helper.epoch() 19 | 20 | << new_id :: unsigned-integer-size(64)>> = << 21 | ts :: unsigned-integer-size(42), 22 | 0 :: unsigned-integer-size(10), 23 | 0 :: unsigned-integer-size(12) >> 24 | 25 | new_id 26 | end 27 | 28 | @doc """ 29 | Get timestamp in ms from your config epoch from any snowflake ID 30 | """ 31 | @spec timestamp_of_id(integer) :: integer 32 | def timestamp_of_id(id) do 33 | id >>> 22 34 | end 35 | 36 | @doc """ 37 | Get timestamp from computer epoch - January 1, 1970, Midnight 38 | """ 39 | @spec real_timestamp_of_id(integer) :: integer 40 | def real_timestamp_of_id(id) do 41 | timestamp_of_id(id) + Snowflake.Helper.epoch() 42 | end 43 | 44 | @doc """ 45 | Get bucket value based on segments of N days 46 | """ 47 | @spec bucket(integer, atom, integer) :: integer 48 | def bucket(units, unit_type, id) do 49 | round(timestamp_of_id(id) / bucket_size(unit_type, units)) 50 | end 51 | 52 | @doc """ 53 | When no id is provided, we generate a bucket for the current time 54 | """ 55 | @spec bucket(integer, atom) :: integer 56 | def bucket(units, unit_type) do 57 | timestamp = System.os_time(:millisecond) - Snowflake.Helper.epoch() 58 | round(timestamp / bucket_size(unit_type, units)) 59 | end 60 | 61 | defp bucket_size(unit_type, units) do 62 | case unit_type do 63 | :hours -> 1000 * 60 * 60 * units 64 | _ -> 1000 * 60 * 60 * 24 * units # days is default 65 | end 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /lib/snowflake/helper.ex: -------------------------------------------------------------------------------- 1 | defmodule Snowflake.Helper do 2 | @moduledoc """ 3 | Utility functions intended for Snowflake application. 4 | epoch() and machine_id() are useful for inspecting in production. 5 | """ 6 | @default_config [ 7 | nodes: [], 8 | epoch: 0 9 | ] 10 | 11 | @doc """ 12 | Grabs epoch from config value 13 | """ 14 | @spec epoch() :: integer 15 | def epoch() do 16 | Application.get_env(:snowflake, :epoch) || @default_config[:epoch] 17 | end 18 | 19 | @doc """ 20 | Grabs hostname, fqdn, and ip addresses, then compares that list to the nodes 21 | config to find the intersection. 22 | """ 23 | @spec machine_id() :: integer 24 | def machine_id() do 25 | id = Application.get_env(:snowflake, :machine_id) 26 | machine_id(id) 27 | end 28 | 29 | defp machine_id(nil) do 30 | nodes = Application.get_env(:snowflake, :nodes) || @default_config[:nodes] 31 | host_addrs = [hostname(), fqdn(), Node.self()] ++ ip_addrs() 32 | 33 | case MapSet.intersection(MapSet.new(host_addrs), MapSet.new(nodes)) |> Enum.take(1) do 34 | [matching_node] -> Enum.find_index(nodes, fn node -> node == matching_node end) 35 | _ -> 1023 36 | end 37 | end 38 | 39 | defp machine_id(id) when id >= 0 and id < 1024, do: id 40 | defp machine_id(_id), do: machine_id(nil) 41 | 42 | defp ip_addrs() do 43 | case :inet.getifaddrs do 44 | {:ok, ifaddrs} -> 45 | ifaddrs 46 | |> Enum.flat_map(fn {_, kwlist} -> 47 | kwlist |> Enum.filter(fn {type, _} -> type == :addr end) 48 | end) 49 | |> Enum.filter(&(tuple_size(elem(&1, 1)) in [4, 6])) 50 | |> Enum.map(fn {_, addr} -> 51 | case addr do 52 | {a, b, c, d} -> [a, b, c, d] |> Enum.join(".") # ipv4 53 | {a, b, c, d, e, f} -> [a, b, c, d, e, f] |> Enum.join(":") # ipv6 54 | end 55 | end) 56 | _ -> [] 57 | end 58 | end 59 | 60 | defp hostname() do 61 | {:ok, name} = :inet.gethostname() 62 | to_string(name) 63 | end 64 | 65 | defp fqdn() do 66 | case :inet.get_rc[:domain] do 67 | nil -> nil 68 | domain -> hostname() <> "." <> to_string(domain) 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Snowflake 2 | 3 | A scalable, decentralized Snowflake generator in Elixir. 4 | 5 | ## Usage 6 | 7 | In your mix.exs file: 8 | 9 | ```elixir 10 | def deps do 11 | [{:snowflake, "~> 1.0.0"}] 12 | end 13 | ``` 14 | 15 | ```elixir 16 | def application do 17 | [applications: [:snowflake]] 18 | end 19 | ``` 20 | 21 | Specify the nodes in your config. If you're running a cluster, specify all the nodes in the cluster that snowflake runs on. 22 | 23 | - **nodes** can be Erlang Node Names, Public IPs, Private IPs, Hostnames, or FQDNs 24 | - **epoch** should not be changed once you begin generating IDs and want to maintain sorting 25 | - There should be no more than 1 snowflake generator per node, or you risk potential duplicate snowflakes on the same node. 26 | 27 | ```elixir 28 | config :snowflake, 29 | nodes: ["127.0.0.1", :'nonode@nohost'], # up to 1023 nodes 30 | epoch: 1142974214000 # don't change after you decide what your epoch is 31 | ``` 32 | 33 | Alternatively, you can specify a specific machine_id 34 | 35 | ```elixir 36 | config :snowflake, 37 | machine_id: 23, # values are 0 thru 1023 nodes 38 | epoch: 1142974214000 # don't change after you decide what your epoch is 39 | ``` 40 | 41 | Generating an ID is simple. 42 | 43 | ```elixir 44 | Snowflake.next_id() 45 | # => {:ok, 54974240033603584} 46 | ``` 47 | 48 | ## Util functions 49 | 50 | After generating snowflake IDs, you may want to use them to do other things. 51 | For example, deriving a bucket number from a snowflake to use as part of a 52 | composite key in Cassandra in the attempt to limit partition size. 53 | 54 | Lets say we want to know the current bucket for an ID that would be generated right now: 55 | ```elixir 56 | Snowflake.Util.bucket(30, :days) 57 | # => 5 58 | ``` 59 | 60 | Or if we want to know which bucket a snowflake ID should belong to, given we are 61 | bucketing by every 30 days. 62 | ```elixir 63 | Snowflake.Util.bucket(30, :days, 54974240033603584) 64 | # => 5 65 | ``` 66 | 67 | Or if we want to know how many ms elapsed from epoch 68 | ```elixir 69 | Snowflake.Util.timestamp_of_id(54974240033603584) 70 | # => 197588482172 71 | ``` 72 | 73 | Or if we want to know how many ms elapsed from computer epoch (January 1, 1970 midnight). We can use this to derive an actual calendar date. 74 | ```elixir 75 | Snowflake.Util.real_timestamp_of_id(54974240033603584) 76 | # => 1486669389497 77 | ``` 78 | 79 | ## NTP 80 | 81 | Keep your nodes in sync with [ntpd](https://en.wikipedia.org/wiki/Ntpd) or use 82 | your VM equivalent as snowflake depends on OS time. ntpd's job is to slow down 83 | or speed up the clock so that it syncs os time with your network time. 84 | 85 | ## Architecture 86 | 87 | Snowflake allows the user to specify the nodes in the cluster, each representing a machine. Snowflake at startup inspects itself for Node, IP and Host information and derives its machine_id from the location of itself in the list of nodes defined in the config. 88 | 89 | Machine ID is defaulted to **1023** if snowflake is not able to find itself within the specified config. It is important to specify the correct IPs / Hostnames / FQDNs for the nodes in a production environment to avoid any chance of snowflake collision. 90 | 91 | ## Benchmarks 92 | 93 | Consistently generates over 60,000 snowflakes per second on Macbook Pro 2.5 GHz Intel Core i7 w/ 16 GB RAM. 94 | 95 | ``` 96 | Benchmarking snowflake... 97 | Benchmarking snowflakex... 98 | 99 | Name ips average deviation median 100 | snowflake 316.51 K 3.16 μs ±503.52% 3.00 μs 101 | snowflakex 296.26 K 3.38 μs ±514.60% 3.00 μs 102 | 103 | Comparison: 104 | snowflake 316.51 K 105 | snowflakex 296.26 K - 1.07x slower 106 | ``` 107 | --------------------------------------------------------------------------------