├── .formatter.exs ├── .github └── dependabot.yml ├── .gitignore ├── LICENSE.md ├── README.md ├── config └── config.exs ├── lib ├── hlcid.ex └── hlcid │ └── application.ex ├── mix.exs ├── mix.lock └── test ├── hlcid_test.exs └── test_helper.exs /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] 4 | ] 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: mix 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "10:00" 8 | open-pull-requests-limit: 10 9 | ignore: 10 | - dependency-name: ex_doc 11 | versions: 12 | - 0.24.0 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | hlcid-*.tar 24 | 25 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Christopher Jon Keathley 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HLCID 2 | 3 | HLCID provides an API for generating 128 bit, k-ordered, globally unique ids without coordination. 4 | 5 | ## Why do I need this? 6 | 7 | Distributed systems often need to generate unique ids. But standard UUIDv4 ids don't provide any sorting capabilities. HLCIDs are globally unique and roughly time sorted. Servers can generate these ids with no coordination. 8 | 9 | ## How does it work? 10 | 11 | HLCID is an OTP application which runs on each of your nodes. You can ask it for an ID like so: 12 | 13 | ```elixir 14 | id = HLCID.generate() 15 | url_safe_id = HLCID.to_base64(id) 16 | => "AWy00jCrAABKaDWL0Z6heg==" 17 | ``` 18 | 19 | Each ID is a slightly modified [hybrid logical clock](https://cse.buffalo.edu/tech-reports/2014-04.pdf). Each timestamp is composed of 48 bits of physical time, 16 bits of logical time, and 64 bits of random bytes. When encoded as binaries each id maintains lexical ordering. 20 | 21 | HLCs automatically account for clock drift by utilizing both physical and logical time. But if the physical clock drifts beyond a given range - 300 seconds by default - you won't be able to generate IDs. If you run HLCID in production, you'll want to monitor NTP for clock drift. Because HLCs use logical time to avoid conflicts and provide one-way causality tracking, there is a limited number of ids available in a millisecond window. This limitation isn't a problem for our use case, but if you need to generate more then 65000 ids per node per millisecond, then HLCID won't work for you. 22 | 23 | ## Additional projects 24 | 25 | * [HLClock](https://github.com/toniqsystems/hlclock) - The library which provides the hybrid logical clocks. 26 | * [ecto_hlclock](https://github.com/toniqsystems/ecto_hlclock) - Provides Ecto migrations and types for using HLCs as a database id. 27 | * [HLClocksynchronizer](https://github.com/SimpleBet/hlclock_synchronizer) - Provides a GenServer for syncing hybrid logical clock timestamps between nodes using Phoenix.PubSub 28 | 29 | ## Alternatives 30 | 31 | While HLCID works for our use case, you should be aware of the alternative solutions and chose one that matches your needs. 32 | 33 | * Flake - Flake IDs heavily inspired HLCID. The main differences are that HLCID uses a purely random set of bytes instead of the machine name. If HLCIDs are broadcast across your cluster, they can be also be used for snapshots. 34 | * [KSUID](https://github.com/segmentio/ksuid) - These ids provide the same benefits of flake and HLCID, but KSUID allows for far more events in a given time window due to its use of randomness. It also has no bounds on the physical clock when means it can't become unavailable. 35 | 36 | ## Installation 37 | 38 | ```elixir 39 | def deps do 40 | [ 41 | {:hlcid, "~> 0.1.0"} 42 | ] 43 | end 44 | ``` 45 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # This configuration is loaded before any dependency and is restricted 6 | # to this project. If another project depends on this project, this 7 | # file won't be loaded nor affect the parent project. For this reason, 8 | # if you want to provide default values for your application for 9 | # third-party users, it should be done in your "mix.exs" file. 10 | 11 | # You can configure your application as: 12 | # 13 | # config :hlcid, key: :value 14 | # 15 | # and access this configuration in your application as: 16 | # 17 | # Application.get_env(:hlcid, :key) 18 | # 19 | # You can also configure a third-party app: 20 | # 21 | # config :logger, level: :info 22 | # 23 | 24 | # It is also possible to import configuration files, relative to this 25 | # directory. For example, you can emulate configuration per environment 26 | # by uncommenting the line below and defining dev.exs, test.exs and such. 27 | # Configuration from the imported file will override the ones defined 28 | # here (which is why it is important to import them last). 29 | # 30 | # import_config "#{Mix.env()}.exs" 31 | -------------------------------------------------------------------------------- /lib/hlcid.ex: -------------------------------------------------------------------------------- 1 | defmodule HLCID do 2 | @moduledoc """ 3 | HLCID is an OTP application which runs on each of your nodes. You can ask it 4 | for an ID like so: 5 | 6 | ```elixir 7 | id = HLCID.generate() 8 | url_safe_id = HLCID.to_base64(id) 9 | => "AWy00jCrAABKaDWL0Z6heg==" 10 | ``` 11 | 12 | Each ID is a slightly modified [hybrid logical clock](https://cse.buffalo.edu/tech-reports/2014-04.pdf). 13 | Each timestamp is composed of 48 bits of physical time, 16 bits of logical 14 | time, and 64 bits of random bytes. When encoded as binaries each id maintains 15 | lexical ordering. 16 | 17 | HLCs automatically account for clock drift by utilizing both physical and 18 | logical time. But if the physical clock drifts beyond a given range 19 | - 300 seconds by default - you won't be able to generate IDs. If you run 20 | HLCID in production, you'll want to monitor NTP for clock drift. Because HLCs 21 | use logical time to avoid conflicts and provide one-way causality tracking, 22 | there is a limited number of ids available in a millisecond window. This 23 | limitation isn't a problem for our use case, but if you need to generate 24 | more then 65000 ids per node per millisecond, than HLCID won't work for you. 25 | """ 26 | 27 | alias HLCID.{Clock} 28 | 29 | @doc """ 30 | Generates a new id. 31 | """ 32 | def generate do 33 | {:ok, ts} = HLClock.send_timestamp(Clock) 34 | 35 | ts 36 | end 37 | 38 | @doc """ 39 | Converts an id to url safe base64. 40 | """ 41 | def to_base64(ts) do 42 | ts 43 | |> HLClock.Timestamp.encode() 44 | |> Base.url_encode64() 45 | end 46 | 47 | @doc """ 48 | Converts a base64 binary back to a id 49 | """ 50 | def from_base64(ts) when is_binary(ts) do 51 | ts 52 | |> Base.url_decode64!() 53 | |> HLClock.Timestamp.decode() 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/hlcid/application.ex: -------------------------------------------------------------------------------- 1 | defmodule HLCID.Application do 2 | @moduledoc false 3 | 4 | use Application 5 | 6 | def start(_type, _args) do 7 | node_id = gen_node_id() 8 | 9 | children = [ 10 | {HLClock, name: HLCID.Clock, node_id: node_id} 11 | ] 12 | 13 | opts = [strategy: :one_for_one, name: HLCID.Supervisor] 14 | Supervisor.start_link(children, opts) 15 | end 16 | 17 | defp gen_node_id do 18 | 8 19 | |> :crypto.strong_rand_bytes() 20 | |> :crypto.bytes_to_integer() 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Hlcid.MixProject do 2 | use Mix.Project 3 | 4 | @version "0.1.0" 5 | 6 | def project do 7 | [ 8 | app: :hlcid, 9 | version: @version, 10 | elixir: "~> 1.8", 11 | start_permanent: Mix.env() == :prod, 12 | deps: deps(), 13 | description: description(), 14 | package: package(), 15 | deps: deps(), 16 | name: "HLCID", 17 | source_url: "https://github.com/keathley/hlcid", 18 | docs: docs() 19 | ] 20 | end 21 | 22 | # Run "mix help compile.app" to learn about applications. 23 | def application do 24 | [ 25 | extra_applications: [:logger, :crypto], 26 | mod: {HLCID.Application, []} 27 | ] 28 | end 29 | 30 | # Run "mix help deps" to learn about dependencies. 31 | defp deps do 32 | [ 33 | {:hlclock, "~> 1.0"}, 34 | {:ex_doc, "~> 0.19", only: :dev} 35 | ] 36 | end 37 | 38 | defp description do 39 | """ 40 | Generates unique, k-ordered ids based on hybrid logical clocks. 41 | """ 42 | end 43 | 44 | defp package do 45 | [ 46 | name: "hlcid", 47 | maintainers: ["Chris Keathley"], 48 | licenses: ["MIT"], 49 | links: %{"GitHub" => "https://github.com/keathley/hlcid"} 50 | ] 51 | end 52 | 53 | def docs do 54 | [ 55 | source_ref: "v#{@version}", 56 | source_url: "https://github.com/keathley/hlcid", 57 | main: "HLCID" 58 | ] 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "earmark": {:hex, :earmark, "1.4.4", "4821b8d05cda507189d51f2caeef370cf1e18ca5d7dfb7d31e9cafe6688106a4", [:mix], [], "hexpm", "1f93aba7340574847c0f609da787f0d79efcab51b044bb6e242cae5aca9d264d"}, 3 | "earmark_parser": {:hex, :earmark_parser, "1.4.29", "149d50dcb3a93d9f3d6f3ecf18c918fb5a2d3c001b5d3305c926cddfbd33355b", [:mix], [], "hexpm", "4902af1b3eb139016aed210888748db8070b8125c2342ce3dcae4f38dcc63503"}, 4 | "ex_doc": {:hex, :ex_doc, "0.29.1", "b1c652fa5f92ee9cf15c75271168027f92039b3877094290a75abcaac82a9f77", [:mix], [{:earmark_parser, "~> 1.4.19", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "b7745fa6374a36daf484e2a2012274950e084815b936b1319aeebcf7809574f6"}, 5 | "hlclock": {:hex, :hlclock, "1.0.0", "7a72fc7a20a9382499216227edf97a8b118e21fc3fcad0e81b8d10c616ce1431", [:mix], [], "hexpm", "d3f994336a7fcbc68bf08b14b2101b61e57bef82c032a6e05c1cdc753612c941"}, 6 | "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, 7 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.0", "f8c570a0d33f8039513fbccaf7108c5d750f47d8defd44088371191b76492b0b", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "28b2cbdc13960a46ae9a8858c4bebdec3c9a6d7b4b9e7f4ed1502f8159f338e7"}, 8 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, 9 | "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, 10 | "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm"}, 11 | } 12 | -------------------------------------------------------------------------------- /test/hlcid_test.exs: -------------------------------------------------------------------------------- 1 | defmodule HLCIDTest do 2 | use ExUnit.Case 3 | doctest HLCID 4 | 5 | @valid_base64 ~r"^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?$" 6 | 7 | test "can generate ids" do 8 | assert %HLClock.Timestamp{} = HLCID.generate() 9 | end 10 | 11 | test "can encode and decode to base64" do 12 | id = HLCID.generate() 13 | 14 | assert HLCID.to_base64(id) =~ @valid_base64 15 | 16 | assert id 17 | |> HLCID.to_base64() 18 | |> HLCID.from_base64() == id 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | --------------------------------------------------------------------------------