├── config ├── prod.exs ├── config.exs ├── test.exs └── dev.exs ├── test ├── test_helper.exs ├── support │ ├── tracer.ex │ ├── datadog_test_api_server.ex │ ├── util.ex │ └── traced_module.ex ├── api_server_mox_test.exs ├── adapter_test.exs └── api_server_test.exs ├── .github └── FUNDING.yml ├── .formatter.exs ├── .circleci └── config.yml ├── .gitignore ├── LICENSE ├── mix.exs ├── lib └── spandex_datadog │ ├── adapter.ex │ └── api_server.ex ├── CHANGELOG.md ├── mix.lock └── README.md /config/prod.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | 3 | Mox.defmock(HTTPoisonMock, for: HTTPoison.Base) 4 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [gregmefford] 4 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | # Used by "mix format" 2 | [ 3 | inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"], 4 | line_length: 120 5 | ] 6 | -------------------------------------------------------------------------------- /test/support/tracer.ex: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.Test.Support.Tracer do 2 | @moduledoc false 3 | use Spandex.Tracer, otp_app: :spandex_datadog 4 | end 5 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | import_config "./#{Mix.env()}.exs" 6 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :logger, :console, 4 | level: :debug, 5 | colors: [enabled: false], 6 | format: "$time $metadata[$level] $message\n", 7 | metadata: [:trace_id, :span_id] 8 | 9 | config :spandex_datadog, SpandexDatadog.Test.Support.Tracer, 10 | service: :spandex_test, 11 | adapter: SpandexDatadog.Adapter, 12 | sender: SpandexDatadog.Test.Support.TestApiServer, 13 | env: "test", 14 | resource: "default", 15 | service_version: "v1", 16 | services: [ 17 | spandex_test: :db 18 | ] 19 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :git_ops, 4 | mix_project: SpandexDatadog.MixProject, 5 | changelog_file: "CHANGELOG.md", 6 | repository_url: "https://github.com/spandex-project/spandex_datadog", 7 | types: [], 8 | # Instructs the tool to manage your mix version in your `mix.exs` file 9 | # See below for more information 10 | manage_mix_version?: true, 11 | # Instructs the tool to manage the version in your README.md 12 | # Pass in `true` to use `"README.md"` or a string to customize 13 | manage_readme_version: "README.md" 14 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Elixir CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-elixir/ for more details 4 | version: 2 5 | jobs: 6 | build: 7 | docker: 8 | - image: cimg/elixir:1.11 9 | environment: 10 | MIX_ENV: test 11 | working_directory: ~/spandex_datadog 12 | steps: 13 | - checkout 14 | - run: mix local.hex --force 15 | - run: mix local.rebar --force 16 | - run: mix format --check-formatted 17 | - run: mix deps.get 18 | - run: mix compile --warnings-as-errors 19 | - run: mix test 20 | -------------------------------------------------------------------------------- /test/support/datadog_test_api_server.ex: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.Test.Support.TestApiServer do 2 | @moduledoc """ 3 | Simply sends the data that would have been sent to datadog to self() as a message 4 | so that the test can assert on payloads that would have been sent to datadog 5 | """ 6 | 7 | alias Spandex.Trace 8 | alias SpandexDatadog.ApiServer 9 | 10 | def send_trace(trace, _opts \\ []) do 11 | send(self(), {:sent_datadog_spans, format(trace)}) 12 | end 13 | 14 | defp format(%Trace{spans: spans, priority: priority, baggage: baggage}) do 15 | Enum.map(spans, fn span -> ApiServer.format(span, priority, baggage) end) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # The directory Mix will write compiled artifacts to. 2 | /_build/ 3 | 4 | # If you run "mix test --cover", coverage assets end up here. 5 | /cover/ 6 | 7 | # The directory Mix downloads your dependencies sources to. 8 | /deps/ 9 | 10 | # Where third-party dependencies like ExDoc output generated docs. 11 | /doc/ 12 | 13 | # Ignore .fetch files in case you like to edit your project deps locally. 14 | /.fetch 15 | 16 | # If the VM crashes, it generates a dump, let's ignore it too. 17 | erl_crash.dump 18 | 19 | # Also ignore archive artifacts (built via "mix archive.build"). 20 | *.ez 21 | 22 | # Ignore package tarball (built via "mix hex.build"). 23 | spandex_datadog-*.tar 24 | 25 | # Temporary files for e.g. tests. 26 | /tmp/ 27 | -------------------------------------------------------------------------------- /test/support/util.ex: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.Test.Util do 2 | @moduledoc false 3 | 4 | def can_fail(func) do 5 | func.() 6 | rescue 7 | _exception -> nil 8 | end 9 | 10 | def find_span(name) when is_bitstring(name) do 11 | find_span(fn span -> span.name == name end) 12 | end 13 | 14 | def find_span(fun) when is_function(fun) do 15 | Enum.find(sent_spans(), fun) 16 | end 17 | 18 | def find_span(name, index) when is_bitstring(name) do 19 | find_span(fn span -> span.name == name end, index) 20 | end 21 | 22 | def find_span(fun, index) when is_function(fun) do 23 | sent_spans() 24 | |> Enum.filter(fun) 25 | |> Enum.at(index) 26 | end 27 | 28 | def sent_spans(timeout \\ 500) do 29 | receive do 30 | {:sent_datadog_spans, spans} -> 31 | send(self(), {:sent_datadog_spans, spans}) 32 | spans 33 | after 34 | timeout -> 35 | raise "No spans sent" 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Zachary Daniel 4 | Copyright (c) 2018 Greg Mefford 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.MixProject do 2 | use Mix.Project 3 | 4 | @source_url "https://github.com/spandex-project/spandex_datadog" 5 | @version "1.4.0" 6 | 7 | def project do 8 | [ 9 | app: :spandex_datadog, 10 | deps: deps(), 11 | description: "A datadog API adapter for spandex.", 12 | docs: docs(), 13 | elixir: "~> 1.6", 14 | elixirc_paths: elixirc_paths(Mix.env()), 15 | package: package(), 16 | start_permanent: Mix.env() == :prod, 17 | version: @version 18 | ] 19 | end 20 | 21 | def application do 22 | [ 23 | extra_applications: [:logger] 24 | ] 25 | end 26 | 27 | defp package do 28 | [ 29 | name: :spandex_datadog, 30 | maintainers: ["Greg Mefford"], 31 | licenses: ["MIT"], 32 | links: %{ 33 | "Changelog" => "https://hexdocs.pm/spandex_datadog/changelog.html", 34 | "GitHub" => @source_url, 35 | "Sponsor" => "https://github.com/sponsors/GregMefford" 36 | } 37 | ] 38 | end 39 | 40 | defp elixirc_paths(:test), do: ["lib", "test/support"] 41 | defp elixirc_paths(_), do: ["lib"] 42 | 43 | defp docs do 44 | [ 45 | extras: ["CHANGELOG.md", "README.md"], 46 | main: "readme", 47 | formatters: ["html"], 48 | skip_undefined_reference_warnings_on: ["CHANGELOG.md"] 49 | ] 50 | end 51 | 52 | defp deps do 53 | [ 54 | {:msgpax, "~> 2.2.1 or ~> 2.3"}, 55 | {:spandex, "~> 3.2"}, 56 | {:telemetry, "~> 0.4.2 or ~> 1.0"}, 57 | # Dev- and test-only deps 58 | {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, 59 | {:httpoison, "~> 0.13 or ~> 1.0 or ~> 2.0", only: :test}, 60 | {:mox, "~> 1.0", only: :test} 61 | ] 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /test/support/traced_module.ex: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.Test.TracedModule do 2 | @moduledoc false 3 | require Spandex 4 | 5 | require SpandexDatadog.Test.Support.Tracer 6 | alias SpandexDatadog.Test.Support.Tracer 7 | 8 | defmodule TestError do 9 | defexception [:message] 10 | end 11 | 12 | # Traces 13 | 14 | def trace_one_thing() do 15 | Tracer.trace "trace_one_thing/0" do 16 | do_one_thing() 17 | end 18 | end 19 | 20 | def trace_with_special_name() do 21 | Tracer.trace "special_name", service: :special_service do 22 | do_one_special_name_thing() 23 | end 24 | end 25 | 26 | def trace_one_error() do 27 | Tracer.trace "trace_one_error/0" do 28 | raise TestError, message: "trace_one_error" 29 | end 30 | end 31 | 32 | def error_two_deep() do 33 | Tracer.trace "error_two_deep/0" do 34 | error_one_deep() 35 | end 36 | end 37 | 38 | def two_fail_one_succeeds() do 39 | Tracer.trace "two_fail_one_succeeds/0" do 40 | try do 41 | _ = error_one_deep() 42 | rescue 43 | _ -> nil 44 | end 45 | 46 | _ = do_one_thing() 47 | _ = error_one_deep() 48 | end 49 | end 50 | 51 | # Spans 52 | 53 | def error_one_deep() do 54 | Tracer.span "error_one_deep/0" do 55 | raise TestError, message: "error_one_deep" 56 | end 57 | end 58 | 59 | def manually_span_one_thing() do 60 | Tracer.span "manually_span_one_thing/0" do 61 | :timer.sleep(100) 62 | end 63 | end 64 | 65 | def do_one_thing() do 66 | Tracer.span "do_one_thing/0" do 67 | :timer.sleep(100) 68 | end 69 | end 70 | 71 | def do_one_special_name_thing() do 72 | Tracer.span "special_name_span", service: :special_span_service do 73 | :timer.sleep(100) 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/spandex_datadog/adapter.ex: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.Adapter do 2 | @moduledoc """ 3 | A Datadog APM implementation for Spandex. 4 | """ 5 | 6 | @behaviour Spandex.Adapter 7 | 8 | require Logger 9 | 10 | alias Spandex.{ 11 | SpanContext, 12 | Tracer 13 | } 14 | 15 | @max_id 9_223_372_036_854_775_807 16 | 17 | @impl Spandex.Adapter 18 | def trace_id(), do: :rand.uniform(@max_id) 19 | 20 | @impl Spandex.Adapter 21 | def span_id(), do: trace_id() 22 | 23 | @impl Spandex.Adapter 24 | def now(), do: :os.system_time(:nano_seconds) 25 | 26 | @impl Spandex.Adapter 27 | @spec default_sender() :: SpandexDatadog.ApiServer 28 | def default_sender() do 29 | SpandexDatadog.ApiServer 30 | end 31 | 32 | @doc """ 33 | Fetches the Datadog-specific conn request headers if they are present. 34 | """ 35 | @impl Spandex.Adapter 36 | @spec distributed_context(conn :: Plug.Conn.t(), Tracer.opts()) :: 37 | {:ok, SpanContext.t()} 38 | | {:error, :no_distributed_trace} 39 | def distributed_context(%Plug.Conn{} = conn, _opts) do 40 | trace_id = get_first_header(conn, "x-datadog-trace-id") 41 | parent_id = get_first_header(conn, "x-datadog-parent-id") 42 | priority = get_first_header(conn, "x-datadog-sampling-priority") || 1 43 | 44 | if is_nil(trace_id) || is_nil(parent_id) do 45 | {:error, :no_distributed_trace} 46 | else 47 | {:ok, %SpanContext{trace_id: trace_id, parent_id: parent_id, priority: priority}} 48 | end 49 | end 50 | 51 | @impl Spandex.Adapter 52 | @spec distributed_context(headers :: Spandex.headers(), Tracer.opts()) :: 53 | {:ok, SpanContext.t()} 54 | | {:error, :no_distributed_trace} 55 | def distributed_context(headers, _opts) do 56 | trace_id = get_header(headers, "x-datadog-trace-id") 57 | parent_id = get_header(headers, "x-datadog-parent-id") 58 | priority = get_header(headers, "x-datadog-sampling-priority") || 1 59 | 60 | if is_nil(trace_id) || is_nil(parent_id) do 61 | {:error, :no_distributed_trace} 62 | else 63 | {:ok, %SpanContext{trace_id: trace_id, parent_id: parent_id, priority: priority}} 64 | end 65 | end 66 | 67 | @doc """ 68 | Injects Datadog-specific HTTP headers to represent the specified SpanContext 69 | """ 70 | @impl Spandex.Adapter 71 | @spec inject_context([{term(), term()}], SpanContext.t(), Tracer.opts()) :: [{term(), term()}] 72 | def inject_context(headers, %SpanContext{} = span_context, _opts) when is_list(headers) do 73 | span_context 74 | |> tracing_headers() 75 | |> Kernel.++(headers) 76 | end 77 | 78 | def inject_context(headers, %SpanContext{} = span_context, _opts) when is_map(headers) do 79 | span_context 80 | |> tracing_headers() 81 | |> Enum.into(%{}) 82 | |> Map.merge(headers) 83 | end 84 | 85 | # Private Helpers 86 | 87 | @spec get_first_header(Plug.Conn.t(), String.t()) :: integer() | nil 88 | defp get_first_header(conn, header_name) do 89 | conn 90 | |> Plug.Conn.get_req_header(header_name) 91 | |> List.first() 92 | |> parse_header() 93 | end 94 | 95 | @spec get_header(%{}, String.t()) :: integer() | nil 96 | defp get_header(headers, key) when is_map(headers) do 97 | Map.get(headers, key, nil) 98 | |> parse_header() 99 | end 100 | 101 | @spec get_header([], String.t()) :: integer() | nil 102 | defp get_header(headers, key) when is_list(headers) do 103 | Enum.find_value(headers, fn {k, v} -> if k == key, do: v end) 104 | |> parse_header() 105 | end 106 | 107 | defp parse_header(header) when is_bitstring(header) do 108 | case Integer.parse(header) do 109 | {int, _} -> int 110 | _ -> nil 111 | end 112 | end 113 | 114 | defp parse_header(_header), do: nil 115 | 116 | defp tracing_headers(%SpanContext{trace_id: trace_id, parent_id: parent_id, priority: priority}) do 117 | [ 118 | {"x-datadog-trace-id", to_string(trace_id)}, 119 | {"x-datadog-parent-id", to_string(parent_id)}, 120 | {"x-datadog-sampling-priority", to_string(priority)} 121 | ] 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /test/api_server_mox_test.exs: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.ApiServerMoxTest do 2 | use ExUnit.Case 3 | 4 | import Mox 5 | 6 | alias Spandex.Trace 7 | alias SpandexDatadog.ApiServer 8 | 9 | # Make sure mocks are verified when the test exits 10 | setup :verify_on_exit! 11 | 12 | # use global mox for simplicity, otherwise we have to configure allowances 13 | setup :set_mox_from_context 14 | setup :set_mox_global 15 | 16 | test "configured batch_size works properly" do 17 | # start our ApiServer GenServer 18 | opts = [ 19 | name: __MODULE__, 20 | http: HTTPoisonMock, 21 | batch_size: 5 22 | ] 23 | 24 | server = ExUnit.Callbacks.start_supervised!({ApiServer, opts}, restart: :temporary) 25 | 26 | # use a send call to async wait for the genserver to send a trace 27 | test_pid = self() 28 | 29 | trace = %Trace{id: "123"} 30 | 31 | # put 4 traces into the batch 32 | Enum.each(1..4, fn _ -> 33 | assert :ok = GenServer.call(server, {:send_trace, trace}) 34 | end) 35 | 36 | # expect a put request to send the traces out 37 | HTTPoisonMock 38 | |> expect(:put, fn "localhost:8126/v0.3/traces", _body, _options -> 39 | send(test_pid, :http_put_finished) 40 | {:ok, %HTTPoison.Response{}} 41 | end) 42 | 43 | # put the final trace that should trigger us to send the traces out 44 | assert :ok = GenServer.call(server, {:send_trace, trace}) 45 | 46 | assert_receive :http_put_finished, 100, "Failed to receive confirmation that our traces were sent." 47 | end 48 | 49 | test "remaining batched traces are flushed on GenServer.stop/3" do 50 | # start our ApiServer GenServer 51 | opts = [ 52 | name: __MODULE__, 53 | http: HTTPoisonMock, 54 | batch_size: 10 55 | ] 56 | 57 | server = ExUnit.Callbacks.start_supervised!({ApiServer, opts}, restart: :temporary) 58 | 59 | # use a send call to async wait for the genserver to send a trace 60 | test_pid = self() 61 | 62 | # put 1 trace in the batch 63 | trace = %Trace{id: "123"} 64 | assert :ok = GenServer.call(server, {:send_trace, trace}) 65 | 66 | # shut our ApiServer down and expect a final http_put to flush any traces left in the batch 67 | HTTPoisonMock 68 | |> expect(:put, fn "localhost:8126/v0.3/traces", _body, _options -> 69 | send(test_pid, :http_put_finished) 70 | {:ok, %HTTPoison.Response{}} 71 | end) 72 | 73 | assert :ok = GenServer.stop(server) 74 | assert_receive :http_put_finished, 100, "Failed to receive confirmation that our traces were sent." 75 | end 76 | 77 | test "remaining batched traces are sent on Process.exit because we are trapping exits" do 78 | # start our ApiServer GenServer 79 | opts = [ 80 | name: __MODULE__, 81 | http: HTTPoisonMock, 82 | trap_exits?: true, 83 | batch_size: 10 84 | ] 85 | 86 | # need to start this genserver unlinked to the test process since we'll be killing it 87 | # otherwise this test process will crash 88 | # {:ok, server} = GenServer.start(ApiServer, opts, name: __MODULE__) 89 | server = ExUnit.Callbacks.start_supervised!({ApiServer, opts}, restart: :temporary) 90 | 91 | # use a send call to async wait for the genserver to send a trace 92 | test_pid = self() 93 | 94 | # put 1 trace in the batch 95 | trace = %Trace{id: "123"} 96 | assert :ok = GenServer.call(server, {:send_trace, trace}) 97 | 98 | # shut our ApiServer down and expect a final http_put to flush any traces left in the batch 99 | HTTPoisonMock 100 | |> expect(:put, fn "localhost:8126/v0.3/traces", _body, _options -> 101 | send(test_pid, :http_put_finished) 102 | {:ok, %HTTPoison.Response{}} 103 | end) 104 | 105 | assert Process.exit(server, :shutdown) 106 | assert_receive :http_put_finished, 100, "Failed to receive confirmation that our traces were sent." 107 | end 108 | 109 | test "we don't wait to flush traces if trap_exits? by default" do 110 | # start our ApiServer GenServer 111 | opts = [ 112 | name: __MODULE__, 113 | http: HTTPoisonMock, 114 | batch_size: 10 115 | ] 116 | 117 | # need to start this genserver unlinked to the test process since we'll be killing it 118 | # otherwise this test process will crash 119 | # {:ok, server} = GenServer.start(ApiServer, opts, name: __MODULE__) 120 | server = ExUnit.Callbacks.start_supervised!({ApiServer, opts}, restart: :temporary) 121 | 122 | # use a send call to async wait for the genserver to send a trace 123 | test_pid = self() 124 | 125 | # put 1 trace in the batch 126 | trace = %Trace{id: "123"} 127 | assert :ok = GenServer.call(server, {:send_trace, trace}) 128 | 129 | # shut our ApiServer down and stup http put for testing 130 | HTTPoisonMock 131 | |> stub(:put, fn "localhost:8126/v0.3/traces", _body, _options -> 132 | send(test_pid, :http_put_finished) 133 | {:ok, %HTTPoison.Response{}} 134 | end) 135 | 136 | assert Process.exit(server, :shutdown) 137 | refute_receive :http_put_finished, 100, "Unexpected confirmation that our traces were sent." 138 | end 139 | end 140 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [1.4.0](https://github.com/spandex-project/spandex_datadog/compare/1.3.0...1.4.0) (2023-08-26) 4 | 5 | ## Features 6 | * Add option to flush remaining traces on shutdown by @SophisticaSean in https://github.com/spandex-project/spandex_datadog/pull/34 7 | * Support service version tagging by @gorkunov in https://github.com/spandex-project/spandex_datadog/pull/57 8 | * Add datadog meta headers by @DReigada in https://github.com/spandex-project/spandex_datadog/pull/60 9 | 10 | ## Bug Fixes 11 | * Handle boolean tag values safely by @DReigada in https://github.com/spandex-project/spandex_datadog/pull/61 12 | 13 | ## New Contributors 14 | * @SophisticaSean made their first contribution in https://github.com/spandex-project/spandex_datadog/pull/34 15 | * @gorkunov made their first contribution in https://github.com/spandex-project/spandex_datadog/pull/57 16 | * @DReigada made their first contribution in https://github.com/spandex-project/spandex_datadog/pull/61 17 | 18 | 19 | ## [1.3.0](https://github.com/spandex-project/spandex_datadog/compare/1.2.0...1.3.0) (2022-10-16) 20 | 21 | ## Features 22 | 23 | * Update supervision tree docs in README by @GregMefford in https://github.com/spandex-project/spandex_datadog/pull/46 24 | * Add rule_psr and limit_psr metrics to improve trace ingestion rate by @mrz in https://github.com/spandex-project/spandex_datadog/pull/45 25 | 26 | ## Bug Fixes 27 | 28 | * Fix typos by @kianmeng in https://github.com/spandex-project/spandex_datadog/pull/48 29 | 30 | ## New Contributors 31 | * @mrz made their first contribution in https://github.com/spandex-project/spandex_datadog/pull/45 32 | 33 | 34 | ## [1.2.0](https://github.com/spandex-project/spandex_datadog/compare/1.1.0...1.2.0) (2021-10-23) 35 | 36 | ### Features: 37 | 38 | * Handle structs explicitly when adding error type [#37](https://github.com/spandex-project/spandex_datadog/pull/37) 39 | * Misc doc generation changes [#40](https://github.com/spandex-project/spandex_datadog/pull/40) 40 | * Remove usage of the transitive dependency Optimal [#33](https://github.com/spandex-project/spandex_datadog/pull/33) 41 | * Update min version of telemetry [#43](https://github.com/spandex-project/spandex_datadog/pull/43) 42 | * add container id to ApiServer.State and send in header [#38](https://github.com/spandex-project/spandex_datadog/pull/38) 43 | 44 | ## [1.1.0](https://github.com/spandex-project/spandex_datadog/compare/1.0.0...1.1.0) (2021-01-19) 45 | 46 | ### Features: 47 | 48 | * Add Telemetry for ApiServer [#28](https://github.com/spandex-project/spandex_datadog/pull/28) 49 | 50 | 51 | 52 | ## [1.0.0](https://github.com/spandex-project/spandex_datadog/compare/0.6.0...1.0.0) (2020-05-22) 53 | ### Breaking Changes: 54 | 55 | * support distributed_context/2 with headers 56 | 57 | 58 | 59 | ## [0.6.0](https://github.com/spandex-project/spandex_datadog/compare/0.5.0...0.6.0) (2020-04-23) 60 | 61 | 62 | 63 | 64 | ### Features: 65 | 66 | * add support for app analytics (#23) 67 | 68 | ## [0.5.0](https://github.com/spandex-project/spandex_datadog/compare/0.4.1...0.5.0) (2019-11-25) 69 | 70 | 71 | 72 | 73 | ### Features: 74 | 75 | * Add X-Datadog-Trace-Count header (#22) 76 | 77 | ## [0.4.1](https://github.com/spandex-project/spandex_datadog/compare/0.4.0...0.4.1) (2019-10-4) 78 | 79 | 80 | 81 | 82 | ### Bug Fixes: 83 | 84 | * Ensure tags are converted to strings (#16) 85 | 86 | ## [0.4.0](https://github.com/spandex-project/spandex_datadog/compare/0.3.1...0.4.0) (2019-02-01) 87 | 88 | 89 | 90 | 91 | ### Features: 92 | 93 | * support elixir 1.8 via msgpax bump 94 | 95 | ## [0.3.1](https://github.com/spandex-project/spandex_datadog/compare/0.3.0...0.3.1) (2018-10-19) 96 | 97 | Initial release using automated changelog management 98 | 99 | # Changelog prior to automated change log management 100 | 101 | ## [0.3.0] (2018-09-16) 102 | 103 | [0.3.0]: https://github.com/spandex-project/spandex_datadog/compare/v0.3.0...v0.2.0 104 | 105 | ### Added 106 | - `SpandexDatadog.Adapter.inject_context/3` added to support the new version of 107 | the `Spandex.Adapter` behaviour. 108 | 109 | ## [0.2.0] (2018-08-31) 110 | 111 | [0.2.0]: https://github.com/spandex-project/spandex_datadog/compare/v0.2.0...v0.1.0 112 | 113 | ### Added 114 | - Priority sampling of distributed traces is now supported by sending the 115 | `priorty` field from the `Trace` along with each `Span` sent to Datadog, 116 | using the appropriate `_sampling_priority_v1` field under the `metrics` 117 | field. 118 | 119 | ### Changed 120 | - If the `env` option is not specified for a trace, it will no longer be sent 121 | to Datadog, This allows the Datadog trace collector configured default to be 122 | used, if desired. 123 | - `SpandexDatadog.Adapter.distributed_context/2` now returns a `Spandex.Trace` 124 | struct, including a `priority` based on the `x-datadog-sampling-priority` 125 | HTTP header. 126 | - `SpandexDatadog.ApiServer` now supports the `send_trace` function, taking a 127 | `Spandex.Trace` struct. 128 | 129 | ### Deprecated 130 | - `SpandexDatadog.ApiServer.send_spans/2` is deprecated in favor of 131 | `SpandexDatadog.ApiServer.send_trace/2`. 132 | 133 | ## [0.1.0] (2018-08-23) 134 | 135 | ### Added 136 | - Initial release of the `spandex_datadog` library separately from the 137 | `spandex` library. 138 | 139 | [0.1.0]: https://github.com/spandex-project/spandex_datadog/commit/3c217429ec5e79e77e05729f2a83d355eeab4996 140 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, 3 | "earmark_parser": {:hex, :earmark_parser, "1.4.33", "3c3fd9673bb5dcc9edc28dd90f50c87ce506d1f71b70e3de69aa8154bc695d44", [:mix], [], "hexpm", "2d526833729b59b9fdb85785078697c72ac5e5066350663e5be6a1182da61b8f"}, 4 | "ex_doc": {:hex, :ex_doc, "0.30.6", "5f8b54854b240a2b55c9734c4b1d0dd7bdd41f71a095d42a70445c03cf05a281", [:mix], [{:earmark_parser, "~> 1.4.31", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "bd48f2ddacf4e482c727f9293d9498e0881597eae6ddc3d9562bd7923375109f"}, 5 | "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, 6 | "httpoison": {:hex, :httpoison, "2.1.0", "655fd9a7b0b95ee3e9a3b535cf7ac8e08ef5229bab187fa86ac4208b122d934b", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "fc455cb4306b43827def4f57299b2d5ac8ac331cb23f517e734a4b78210a160c"}, 7 | "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, 8 | "makeup": {:hex, :makeup, "1.1.0", "6b67c8bc2882a6b6a445859952a602afc1a41c2e08379ca057c0f525366fc3ca", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "0a45ed501f4a8897f580eabf99a2e5234ea3e75a4373c8a52824f6e873be57a6"}, 9 | "makeup_elixir": {:hex, :makeup_elixir, "0.16.1", "cc9e3ca312f1cfeccc572b37a09980287e243648108384b97ff2b76e505c3555", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "e127a341ad1b209bd80f7bd1620a15693a9908ed780c3b763bccf7d200c767c6"}, 10 | "makeup_erlang": {:hex, :makeup_erlang, "0.1.2", "ad87296a092a46e03b7e9b0be7631ddcf64c790fa68a9ef5323b6cbb36affc72", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "f3f5a1ca93ce6e092d92b6d9c049bcda58a3b617a8d888f8e7231c85630e8108"}, 11 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, 12 | "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, 13 | "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, 14 | "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"}, 15 | "msgpax": {:hex, :msgpax, "2.4.0", "4647575c87cb0c43b93266438242c21f71f196cafa268f45f91498541148c15d", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "ca933891b0e7075701a17507c61642bf6e0407bb244040d5d0a58597a06369d2"}, 16 | "nimble_parsec": {:hex, :nimble_parsec, "1.3.1", "2c54013ecf170e249e9291ed0a62e5832f70a476c61da16f6aac6dca0189f2af", [:mix], [], "hexpm", "2682e3c0b2eb58d90c6375fc0cc30bc7be06f365bf72608804fb9cffa5e1b167"}, 17 | "optimal": {:hex, :optimal, "0.3.6", "46bbf52fbbbd238cda81e02560caa84f93a53c75620f1fe19e81e4ae7b07d1dd", [:mix], [], "hexpm", "1a06ea6a653120226b35b283a1cd10039550f2c566edcdec22b29316d73640fd"}, 18 | "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, 19 | "plug": {:hex, :plug, "1.14.2", "cff7d4ec45b4ae176a227acd94a7ab536d9b37b942c8e8fa6dfc0fff98ff4d80", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "842fc50187e13cf4ac3b253d47d9474ed6c296a8732752835ce4a86acdf68d13"}, 20 | "plug_crypto": {:hex, :plug_crypto, "1.2.5", "918772575e48e81e455818229bf719d4ab4181fcbf7f85b68a35620f78d89ced", [:mix], [], "hexpm", "26549a1d6345e2172eb1c233866756ae44a9609bd33ee6f99147ab3fd87fd842"}, 21 | "spandex": {:hex, :spandex, "3.2.0", "f8cd40146ea988c87f3c14054150c9a47ba17e53cd4515c00e1f93c29c45404d", [:mix], [{:decorator, "~> 1.2", [hex: :decorator, repo: "hexpm", optional: true]}, {:optimal, "~> 0.3.3", [hex: :optimal, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d0a7d5aef4c5af9cf5467f2003e8a5d8d2bdae3823a6cc95d776b9a2251d4d03"}, 22 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, 23 | "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, 24 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, 25 | } 26 | -------------------------------------------------------------------------------- /test/adapter_test.exs: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.Test.AdapterTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Spandex.SpanContext 5 | 6 | alias SpandexDatadog.{ 7 | Adapter, 8 | Test.TracedModule, 9 | Test.Util 10 | } 11 | 12 | test "a complete trace sends spans" do 13 | TracedModule.trace_one_thing() 14 | 15 | spans = Util.sent_spans() 16 | 17 | Enum.each(spans, fn span -> 18 | assert span.service == :spandex_test 19 | assert span.meta.env == "test" 20 | assert span.meta.version == "v1" 21 | end) 22 | end 23 | 24 | test "a trace can specify additional attributes" do 25 | TracedModule.trace_with_special_name() 26 | 27 | assert(Util.find_span("special_name").service == :special_service) 28 | end 29 | 30 | test "a span can specify additional attributes" do 31 | TracedModule.trace_with_special_name() 32 | 33 | assert(Util.find_span("special_name_span").service == :special_span_service) 34 | end 35 | 36 | test "a complete trace sends a top level span" do 37 | TracedModule.trace_one_thing() 38 | span = Util.find_span("trace_one_thing/0") 39 | refute is_nil(span) 40 | assert span.service == :spandex_test 41 | assert span.meta.env == "test" 42 | end 43 | 44 | test "a complete trace sends the internal spans as well" do 45 | TracedModule.trace_one_thing() 46 | 47 | assert(Util.find_span("do_one_thing/0") != nil) 48 | end 49 | 50 | test "the parent_id for a child span is correct" do 51 | TracedModule.trace_one_thing() 52 | 53 | assert(Util.find_span("trace_one_thing/0").span_id == Util.find_span("do_one_thing/0").parent_id) 54 | end 55 | 56 | test "a span is correctly notated as an error if an excepton occurs" do 57 | Util.can_fail(fn -> TracedModule.trace_one_error() end) 58 | 59 | assert(Util.find_span("trace_one_error/0").error == 1) 60 | end 61 | 62 | test "spans all the way up are correctly notated as an error" do 63 | Util.can_fail(fn -> TracedModule.error_two_deep() end) 64 | 65 | assert(Util.find_span("error_two_deep/0").error == 1) 66 | assert(Util.find_span("error_one_deep/0").error == 1) 67 | end 68 | 69 | test "successful sibling spans are not marked as failures when sibling fails" do 70 | Util.can_fail(fn -> TracedModule.two_fail_one_succeeds() end) 71 | 72 | assert(Util.find_span("error_one_deep/0", 0).error == 1) 73 | assert(Util.find_span("do_one_thing/0").error == 0) 74 | assert(Util.find_span("error_one_deep/0", 1).error == 1) 75 | end 76 | 77 | describe "distributed_context/2 with Plug.Conn" do 78 | test "returns a SpanContext struct" do 79 | conn = 80 | :get 81 | |> Plug.Test.conn("/") 82 | |> Plug.Conn.put_req_header("x-datadog-trace-id", "123") 83 | |> Plug.Conn.put_req_header("x-datadog-parent-id", "456") 84 | |> Plug.Conn.put_req_header("x-datadog-sampling-priority", "2") 85 | 86 | assert {:ok, %SpanContext{} = span_context} = Adapter.distributed_context(conn, []) 87 | assert span_context.trace_id == 123 88 | assert span_context.parent_id == 456 89 | assert span_context.priority == 2 90 | end 91 | 92 | test "priority defaults to 1 (i.e. we currently assume all distributed traces should be kept)" do 93 | conn = 94 | :get 95 | |> Plug.Test.conn("/") 96 | |> Plug.Conn.put_req_header("x-datadog-trace-id", "123") 97 | |> Plug.Conn.put_req_header("x-datadog-parent-id", "456") 98 | 99 | assert {:ok, %SpanContext{priority: 1}} = Adapter.distributed_context(conn, []) 100 | end 101 | 102 | test "returns an error when it cannot detect both a Trace ID and a Span ID" do 103 | conn = Plug.Test.conn(:get, "/") 104 | assert {:error, :no_distributed_trace} = Adapter.distributed_context(conn, []) 105 | end 106 | end 107 | 108 | describe "distributed_context/2 with Spandex.headers()" do 109 | test "returns a SpanContext struct when headers is a list" do 110 | headers = [{"x-datadog-trace-id", "123"}, {"x-datadog-parent-id", "456"}, {"x-datadog-sampling-priority", "2"}] 111 | 112 | assert {:ok, %SpanContext{} = span_context} = Adapter.distributed_context(headers, []) 113 | assert span_context.trace_id == 123 114 | assert span_context.parent_id == 456 115 | assert span_context.priority == 2 116 | end 117 | 118 | test "returns a SpanContext struct when headers is a map" do 119 | headers = %{ 120 | "x-datadog-trace-id" => "123", 121 | "x-datadog-parent-id" => "456", 122 | "x-datadog-sampling-priority" => "2" 123 | } 124 | 125 | assert {:ok, %SpanContext{} = span_context} = Adapter.distributed_context(headers, []) 126 | assert span_context.trace_id == 123 127 | assert span_context.parent_id == 456 128 | assert span_context.priority == 2 129 | end 130 | 131 | test "priority defaults to 1 (i.e. we currently assume all distributed traces should be kept)" do 132 | headers = %{ 133 | "x-datadog-trace-id" => "123", 134 | "x-datadog-parent-id" => "456" 135 | } 136 | 137 | assert {:ok, %SpanContext{priority: 1}} = Adapter.distributed_context(headers, []) 138 | end 139 | 140 | test "returns an error when it cannot detect both a Trace ID and a Span ID" do 141 | headers = %{} 142 | assert {:error, :no_distributed_trace} = Adapter.distributed_context(headers, []) 143 | end 144 | end 145 | 146 | describe "inject_context/3" do 147 | test "Prepends distributed tracing headers to an existing list of headers" do 148 | span_context = %SpanContext{trace_id: 123, parent_id: 456, priority: 10} 149 | headers = [{"header1", "value1"}, {"header2", "value2"}] 150 | 151 | result = Adapter.inject_context(headers, span_context, []) 152 | 153 | assert result == [ 154 | {"x-datadog-trace-id", "123"}, 155 | {"x-datadog-parent-id", "456"}, 156 | {"x-datadog-sampling-priority", "10"}, 157 | {"header1", "value1"}, 158 | {"header2", "value2"} 159 | ] 160 | end 161 | 162 | test "Merges distributed tracing headers with an existing map of headers" do 163 | span_context = %SpanContext{trace_id: 123, parent_id: 456, priority: 10} 164 | headers = %{"header1" => "value1", "header2" => "value2"} 165 | 166 | result = Adapter.inject_context(headers, span_context, []) 167 | 168 | assert result == %{ 169 | "x-datadog-trace-id" => "123", 170 | "x-datadog-parent-id" => "456", 171 | "x-datadog-sampling-priority" => "10", 172 | "header1" => "value1", 173 | "header2" => "value2" 174 | } 175 | end 176 | end 177 | end 178 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SpandexDatadog 2 | 3 | [![Package Version](https://img.shields.io/hexpm/v/spandex_datadog.svg)](https://hex.pm/packages/spandex_datadog) 4 | [![Hex Docs](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/spandex_datadog/) 5 | [![Downloads](https://img.shields.io/hexpm/dt/spandex_datadog.svg)](https://hex.pm/packages/spandex_datadog) 6 | [![License](https://img.shields.io/hexpm/l/spandex_datadog.svg)](https://github.com/spandex-project/spandex_datadog/blob/master/LICENSE) 7 | [![Last Updated](https://img.shields.io/github/last-commit/spandex-project/spandex_datadog.svg)](https://github.com/spandex-project/spandex_datadog/commits/master) 8 | 9 | A datadog adapter for the `:spandex` library. 10 | 11 | ## Installation 12 | 13 | The package can be installed by adding `:spandex_datadog` to your list of 14 | dependencies in `mix.exs`: 15 | 16 | ```elixir 17 | def deps do 18 | [ 19 | {:spandex_datadog, "~> 1.4"} 20 | ] 21 | end 22 | ``` 23 | 24 | To start the datadog adapter, add a worker to your application's supervisor 25 | 26 | ```elixir 27 | # Example configuration 28 | 29 | # Note: You should put ApiServer before any other children in the list that 30 | # might try to send traces before the ApiServer has started up, for example 31 | # Ecto.Repo and Phoenix.Endpoint 32 | 33 | spandex_opts = 34 | [ 35 | host: System.get_env("DATADOG_HOST") || "localhost", 36 | port: System.get_env("DATADOG_PORT") || 8126, 37 | batch_size: System.get_env("SPANDEX_BATCH_SIZE") || 10, 38 | sync_threshold: System.get_env("SPANDEX_SYNC_THRESHOLD") || 100, 39 | http: HTTPoison 40 | ] 41 | 42 | children = [ 43 | # ... 44 | {SpandexDatadog.ApiServer, spandex_opts}, 45 | MyApp.Repo, 46 | MyAppWeb.Endpoint, 47 | # ... 48 | ] 49 | 50 | opts = [strategy: :one_for_one, name: MyApp.Supervisor] 51 | Supvervisor.start_link(children, opts) 52 | ``` 53 | 54 | ## Distributed Tracing 55 | 56 | Distributed tracing is supported via headers `x-datadog-trace-id`, 57 | `x-datadog-parent-id`, and `x-datadog-sampling-priority`. If they are set, the 58 | `Spandex.Plug.StartTrace` plug will act accordingly, continuing that trace and 59 | span instead of starting a new one. *Both* `x-datadog-trace-id` and 60 | `x-datadog-parent-id` must be set for distributed tracing to work. You can 61 | learn more about the behavior of `x-datadog-sampling-priority` in the [Datadog 62 | priority sampling documentation]. 63 | 64 | [Datadog priority sampling documentation]: https://docs.datadoghq.com/tracing/getting_further/trace_sampling_and_storage/#priority-sampling-for-distributed-tracing 65 | 66 | ## Telemetry 67 | 68 | This library includes [telemetry] events that can be used to inspect the 69 | performance and operations involved in sending trace data to Datadog. Please 70 | refer to the [telemetry] documentation to see how to attach to these events 71 | using the standard conventions. The following events are currently exposed: 72 | 73 | [telemetry]: https://github.com/beam-telemetry/telemetry 74 | 75 | ### `[:spandex_datadog, :send_trace, :start]` 76 | 77 | This event is executed when the `SpandexDatadog.ApiServer.send_trace/2` API 78 | function is called, typically as a result of the `finish_trace/1` function 79 | being called on your `Tracer` implementation module. 80 | 81 | **NOTE:** This event is executed in the same process that is calling this API 82 | function, but at this point, the `Spandex.Trace` data has already been passed 83 | into the `send_trace` function, and thus the active trace can no longer be 84 | modified (for example, it is not possible to use this event to add a span to 85 | represent this API itself having been called). 86 | 87 | #### Measurements 88 | 89 | * `:system_time`: The time (in native units) this event executed. 90 | 91 | #### Metadata 92 | 93 | * `trace`: The current `Spandex.Trace` that is being sent to the `ApiServer`. 94 | 95 | ### `[:spandex_datadog, :send_trace, :stop]` 96 | 97 | This event is executed when the `SpandexDatadog.ApiServer.send_trace/2` API 98 | function completes normally. 99 | 100 | **NOTE:** This event is executed in the same process that is calling this API 101 | function, but at this point, the `Spandex.Trace` data has already been passed 102 | into the `send_trace` function, and thus the active trace can no longer be 103 | modified (for example, it is not possible to use this event to add a span to 104 | represent this API itself having been called). 105 | 106 | #### Measurements 107 | 108 | * `:duration`: The time (in native units) spent servicing the API call. 109 | 110 | #### Metadata 111 | 112 | * `trace`: The `Spandex.Trace` that was sent to the `ApiServer`. 113 | 114 | ### `[:spandex_datadog, :send_trace, :exception]` 115 | 116 | This event is executed when the `SpandexDatadog.ApiServer.send_trace/2` API 117 | function ends prematurely due to an error or exit. 118 | 119 | **NOTE:** This event is executed in the same process that is calling this API 120 | function, but at this point, the `Spandex.Trace` data has already been passed 121 | into the `send_trace` function, and thus the active trace can no longer be 122 | modified (for example, it is not possible to use this event to add a span to 123 | represent this API itself having been called). 124 | 125 | #### Measurements 126 | 127 | * `:duration`: The time (in native units) spent servicing the API call. 128 | 129 | #### Metadata 130 | 131 | * `trace`: The current `Spandex.Trace` that is being sent to the `ApiServer`. 132 | * `:kind`: The kind of exception raised. 133 | * `:error`: Error data associated with the relevant kind of exception. 134 | * `:stacktrace`: The stacktrace associated with the exception. 135 | 136 | ## API Sender Performance 137 | 138 | Originally, the library had an API server and spans were sent via 139 | `GenServer.cast`, but we've seen the need to introduce backpressure, and limit 140 | the overall amount of requests made. As such, the Datadog API sender accepts 141 | `batch_size` and `sync_threshold` options. 142 | 143 | Batch size refers to *traces*, not spans, so if you send a large amount of spans 144 | per trace, then you probably want to keep that number low. If you send only a 145 | few spans, then you could set it significantly higher. 146 | 147 | Sync threshold is how many _simultaneous_ HTTP pushes will be going to Datadog 148 | before it blocks/throttles your application by making the tracing call synchronous instead of async. 149 | 150 | Ideally, the sync threshold would be set to a point that you wouldn't reasonably reach often, but 151 | that is low enough to not cause systemic performance issues if you don't apply 152 | backpressure. 153 | 154 | A simple way to think about it is that if you are seeing 1000 155 | request per second and `batch_size` is 10, then you'll be making 100 156 | requests per second to Datadog (probably a bad config). 157 | With `sync_threshold` set to 10, only 10 of those requests can be 158 | processed concurrently before trace calls become synchronous. 159 | 160 | This concept of backpressure is very important, and strategies 161 | for switching to synchronous operation are often surprisingly far more 162 | performant than purely asynchronous strategies (and much more predictable). 163 | 164 | 165 | ## Copyright and License 166 | 167 | Copyright (c) 2021 Zachary Daniel & Greg Mefford 168 | 169 | Released under the MIT License, which can be found in the repository in [`LICENSE`](https://github.com/spandex-project/spandex_datadog/blob/master/LICENSE). 170 | -------------------------------------------------------------------------------- /test/api_server_test.exs: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.ApiServerTest do 2 | use ExUnit.Case, async: false 3 | 4 | import ExUnit.CaptureLog 5 | 6 | alias Spandex.{ 7 | Span, 8 | Trace 9 | } 10 | 11 | alias SpandexDatadog.ApiServer 12 | 13 | defmodule TestOkApiServer do 14 | def put(url, body, headers) do 15 | send(self(), {:put_datadog_spans, body |> Msgpax.unpack!() |> hd(), url, headers}) 16 | {:ok, %HTTPoison.Response{status_code: 200}} 17 | end 18 | end 19 | 20 | defmodule TestErrorApiServer do 21 | def put(url, body, headers) do 22 | send(self(), {:put_datadog_spans, body |> Msgpax.unpack!() |> hd(), url, headers}) 23 | {:error, %HTTPoison.Error{id: :foo, reason: :bar}} 24 | end 25 | end 26 | 27 | defmodule TestSlowApiServer do 28 | def put(_url, _body, _headers) do 29 | Process.sleep(500) 30 | {:error, :timeout} 31 | end 32 | end 33 | 34 | defmodule TelemetryRecorderPDict do 35 | def handle_event(event, measurements, metadata, _cfg) do 36 | Process.put(event, {measurements, metadata}) 37 | end 38 | end 39 | 40 | setup_all do 41 | {:ok, agent_pid} = Agent.start_link(fn -> 0 end) 42 | trace_id = 4_743_028_846_331_200_905 43 | 44 | {:ok, span_1} = 45 | Span.new( 46 | id: 4_743_028_846_331_200_906, 47 | start: 1_527_752_052_216_478_000, 48 | service: :foo, 49 | service_version: "v1", 50 | env: "local", 51 | name: "foo", 52 | trace_id: trace_id, 53 | completion_time: 1_527_752_052_216_578_000, 54 | tags: [is_foo: true, foo: "123", bar: 321, buz: :blitz, baz: {1, 2}, zyx: [xyz: {1, 2}]] 55 | ) 56 | 57 | {:ok, span_2} = 58 | Span.new( 59 | id: 4_743_029_846_331_200_906, 60 | start: 1_527_752_052_216_578_001, 61 | completion_time: 1_527_752_052_316_578_001, 62 | service: :bar, 63 | env: "local", 64 | name: "bar", 65 | trace_id: trace_id 66 | ) 67 | 68 | {:ok, span_3} = 69 | Span.new( 70 | id: 4_743_029_846_331_200_906, 71 | start: 1_527_752_052_216_578_001, 72 | completion_time: 1_527_752_052_316_578_001, 73 | service: :bar, 74 | env: "local", 75 | name: "bar", 76 | trace_id: trace_id, 77 | tags: [analytics_event: true] 78 | ) 79 | 80 | trace = %Trace{spans: [span_1, span_2, span_3]} 81 | 82 | { 83 | :ok, 84 | [ 85 | trace: trace, 86 | url: "localhost:8126/v0.3/traces", 87 | state: %ApiServer.State{ 88 | asynchronous_send?: false, 89 | host: "localhost", 90 | port: "8126", 91 | http: TestOkApiServer, 92 | verbose?: false, 93 | waiting_traces: [], 94 | batch_size: 1, 95 | agent_pid: agent_pid 96 | } 97 | ] 98 | } 99 | end 100 | 101 | describe "ApiServer.send_trace/2" do 102 | test "executes telemetry on success", %{trace: trace} do 103 | :telemetry.attach_many( 104 | "log-response-handler", 105 | [ 106 | [:spandex_datadog, :send_trace, :start], 107 | [:spandex_datadog, :send_trace, :stop], 108 | [:spandex_datadog, :send_trace, :exception] 109 | ], 110 | &TelemetryRecorderPDict.handle_event/4, 111 | nil 112 | ) 113 | 114 | assert {:ok, _pid} = ApiServer.start_link(http: TestOkApiServer, name: __MODULE__) 115 | 116 | ApiServer.send_trace(trace, name: __MODULE__) 117 | 118 | {start_measurements, start_metadata} = Process.get([:spandex_datadog, :send_trace, :start]) 119 | assert start_measurements[:system_time] 120 | assert trace == start_metadata[:trace] 121 | 122 | {stop_measurements, stop_metadata} = Process.get([:spandex_datadog, :send_trace, :stop]) 123 | assert stop_measurements[:duration] 124 | assert trace == stop_metadata[:trace] 125 | 126 | refute Process.get([:spandex_datadog, :send_trace, :exception]) 127 | end 128 | 129 | test "executes telemetry on exception", %{trace: trace} do 130 | :telemetry.attach_many( 131 | "log-response-handler", 132 | [ 133 | [:spandex_datadog, :send_trace, :start], 134 | [:spandex_datadog, :send_trace, :stop], 135 | [:spandex_datadog, :send_trace, :exception] 136 | ], 137 | &TelemetryRecorderPDict.handle_event/4, 138 | nil 139 | ) 140 | 141 | ApiServer.start_link(http: TestSlowApiServer, batch_size: 0, sync_threshold: 0) 142 | 143 | catch_exit(ApiServer.send_trace(trace, timeout: 1)) 144 | 145 | {start_measurements, start_metadata} = Process.get([:spandex_datadog, :send_trace, :start]) 146 | assert start_measurements[:system_time] 147 | assert trace == start_metadata[:trace] 148 | 149 | refute Process.get([:spandex_datadog, :send_trace, :stop]) 150 | 151 | {exception_measurements, exception_metadata} = Process.get([:spandex_datadog, :send_trace, :exception]) 152 | assert exception_measurements[:duration] 153 | assert trace == start_metadata[:trace] 154 | assert :exit == exception_metadata[:kind] 155 | assert nil == exception_metadata[:error] 156 | assert is_list(exception_metadata[:stacktrace]) 157 | end 158 | end 159 | 160 | describe "ApiServer.handle_call/3 - :send_trace" do 161 | test "doesn't log anything when verbose?: false", %{trace: trace, state: state, url: url} do 162 | log = 163 | capture_log(fn -> 164 | ApiServer.handle_call({:send_trace, trace}, self(), state) 165 | end) 166 | 167 | assert log == "" 168 | 169 | formatted = [ 170 | %{ 171 | "duration" => 100_000, 172 | "error" => 0, 173 | "meta" => %{ 174 | "bar" => "321", 175 | "baz" => "{1, 2}", 176 | "buz" => "blitz", 177 | "env" => "local", 178 | "foo" => "123", 179 | "is_foo" => "true", 180 | "version" => "v1", 181 | "zyx" => "[xyz: {1, 2}]" 182 | }, 183 | "metrics" => %{ 184 | "_sampling_priority_v1" => 1, 185 | "_dd.rule_psr" => 1.0, 186 | "_dd.limit_psr" => 1.0 187 | }, 188 | "name" => "foo", 189 | "resource" => "foo", 190 | "service" => "foo", 191 | "span_id" => 4_743_028_846_331_200_906, 192 | "start" => 1_527_752_052_216_478_000, 193 | "trace_id" => 4_743_028_846_331_200_905 194 | }, 195 | %{ 196 | "duration" => 100_000_000, 197 | "error" => 0, 198 | "meta" => %{ 199 | "env" => "local" 200 | }, 201 | "metrics" => %{ 202 | "_sampling_priority_v1" => 1, 203 | "_dd.rule_psr" => 1.0, 204 | "_dd.limit_psr" => 1.0 205 | }, 206 | "name" => "bar", 207 | "resource" => "bar", 208 | "service" => "bar", 209 | "span_id" => 4_743_029_846_331_200_906, 210 | "start" => 1_527_752_052_216_578_001, 211 | "trace_id" => 4_743_028_846_331_200_905 212 | }, 213 | %{ 214 | "duration" => 100_000_000, 215 | "error" => 0, 216 | "meta" => %{ 217 | "env" => "local" 218 | }, 219 | "metrics" => %{ 220 | "_dd1.sr.eausr" => 1, 221 | "_sampling_priority_v1" => 1, 222 | "_dd.rule_psr" => 1.0, 223 | "_dd.limit_psr" => 1.0 224 | }, 225 | "name" => "bar", 226 | "resource" => "bar", 227 | "service" => "bar", 228 | "span_id" => 4_743_029_846_331_200_906, 229 | "start" => 1_527_752_052_216_578_001, 230 | "trace_id" => 4_743_028_846_331_200_905 231 | } 232 | ] 233 | 234 | headers = [ 235 | {"Content-Type", "application/msgpack"}, 236 | {"Datadog-Meta-Lang", "elixir"}, 237 | {"Datadog-Meta-Lang-Version", System.version()}, 238 | {"Datadog-Meta-Tracer-Version", nil}, 239 | {"X-Datadog-Trace-Count", 1} 240 | ] 241 | 242 | assert_received {:put_datadog_spans, ^formatted, ^url, ^headers} 243 | end 244 | 245 | test "doesn't care about the response result", %{trace: trace, state: state, url: url} do 246 | state = 247 | state 248 | |> Map.put(:verbose?, true) 249 | |> Map.put(:http, TestErrorApiServer) 250 | 251 | [enqueue, processing, received_spans, response] = 252 | capture_log(fn -> 253 | {:reply, :ok, _} = ApiServer.handle_call({:send_trace, trace}, self(), state) 254 | end) 255 | |> String.split("\n") 256 | |> Enum.reject(fn s -> s == "" end) 257 | 258 | assert enqueue =~ ~r/Adding trace to stack with 3 spans/ 259 | 260 | assert processing =~ ~r/Sending 1 traces, 3 spans/ 261 | 262 | assert received_spans =~ ~r/Trace: \[%Spandex.Trace{/ 263 | 264 | formatted = [ 265 | %{ 266 | "duration" => 100_000, 267 | "error" => 0, 268 | "meta" => %{ 269 | "bar" => "321", 270 | "baz" => "{1, 2}", 271 | "buz" => "blitz", 272 | "env" => "local", 273 | "foo" => "123", 274 | "is_foo" => "true", 275 | "version" => "v1", 276 | "zyx" => "[xyz: {1, 2}]" 277 | }, 278 | "metrics" => %{ 279 | "_sampling_priority_v1" => 1, 280 | "_dd.rule_psr" => 1.0, 281 | "_dd.limit_psr" => 1.0 282 | }, 283 | "name" => "foo", 284 | "resource" => "foo", 285 | "service" => "foo", 286 | "span_id" => 4_743_028_846_331_200_906, 287 | "start" => 1_527_752_052_216_478_000, 288 | "trace_id" => 4_743_028_846_331_200_905 289 | }, 290 | %{ 291 | "duration" => 100_000_000, 292 | "error" => 0, 293 | "meta" => %{ 294 | "env" => "local" 295 | }, 296 | "metrics" => %{ 297 | "_sampling_priority_v1" => 1, 298 | "_dd.rule_psr" => 1.0, 299 | "_dd.limit_psr" => 1.0 300 | }, 301 | "name" => "bar", 302 | "resource" => "bar", 303 | "service" => "bar", 304 | "span_id" => 4_743_029_846_331_200_906, 305 | "start" => 1_527_752_052_216_578_001, 306 | "trace_id" => 4_743_028_846_331_200_905 307 | }, 308 | %{ 309 | "duration" => 100_000_000, 310 | "error" => 0, 311 | "meta" => %{ 312 | "env" => "local" 313 | }, 314 | "metrics" => %{ 315 | "_dd.rule_psr" => 1.0, 316 | "_dd.limit_psr" => 1.0, 317 | "_dd1.sr.eausr" => 1, 318 | "_sampling_priority_v1" => 1 319 | }, 320 | "name" => "bar", 321 | "resource" => "bar", 322 | "service" => "bar", 323 | "span_id" => 4_743_029_846_331_200_906, 324 | "start" => 1_527_752_052_216_578_001, 325 | "trace_id" => 4_743_028_846_331_200_905 326 | } 327 | ] 328 | 329 | assert response =~ ~r/Trace response: {:error, %HTTPoison.Error{id: :foo, reason: :bar}}/ 330 | assert_received {:put_datadog_spans, ^formatted, ^url, _} 331 | end 332 | end 333 | end 334 | -------------------------------------------------------------------------------- /lib/spandex_datadog/api_server.ex: -------------------------------------------------------------------------------- 1 | defmodule SpandexDatadog.ApiServer do 2 | @moduledoc """ 3 | Implements worker for sending spans to datadog as GenServer in order to send traces async. 4 | """ 5 | 6 | use GenServer 7 | require Logger 8 | 9 | alias Spandex.{ 10 | Span, 11 | Trace 12 | } 13 | 14 | defmodule State do 15 | @moduledoc false 16 | 17 | @type t :: %State{} 18 | 19 | defstruct [ 20 | :asynchronous_send?, 21 | :http, 22 | :url, 23 | :host, 24 | :port, 25 | :verbose?, 26 | :waiting_traces, 27 | :batch_size, 28 | :sync_threshold, 29 | :agent_pid, 30 | :container_id, 31 | :trap_exits? 32 | ] 33 | end 34 | 35 | # Same as HTTPoison.headers 36 | @type headers :: [{atom, binary}] | [{binary, binary}] | %{binary => binary} | any 37 | 38 | @headers [ 39 | {"Content-Type", "application/msgpack"}, 40 | {"Datadog-Meta-Lang", "elixir"}, 41 | {"Datadog-Meta-Lang-Version", System.version()}, 42 | {"Datadog-Meta-Tracer-Version", Application.spec(:spandex_datadog)[:vsn]} 43 | ] 44 | 45 | @default_opts [ 46 | host: "localhost", 47 | http: HTTPoison, 48 | port: 8126, 49 | verbose?: false, 50 | batch_size: 10, 51 | sync_threshold: 20, 52 | trap_exits?: false, 53 | name: __MODULE__, 54 | api_adapter: SpandexDatadog.ApiServer 55 | ] 56 | 57 | @doc """ 58 | Starts the ApiServer with given options. 59 | 60 | ## Options 61 | 62 | * `:http` - The HTTP module to use for sending spans to the agent. Defaults to `HTTPoison`. 63 | * `:host` - The host the agent can be reached at. Defaults to `"localhost"`. 64 | * `:port` - The port to use when sending traces to the agent. Defaults to `8126`. 65 | * `:verbose?` - Only to be used for debugging: All finished traces will be logged. Defaults to `false` 66 | * `:batch_size` - The number of traces that should be sent in a single batch. Defaults to `10`. 67 | * `:sync_threshold` - The maximum number of processes that may be sending traces at any one time. This adds backpressure. Defaults to `20`. 68 | * `:name` - The name the GenServer should have. Currently only used for testing. Defaults to `SpandexDatadog.ApiServer` 69 | * `:trap_exits?` - Whether or not to trap exits and attempt to flush traces on shutdown, defaults to `false`. Useful if you do frequent deploys and you don't want to lose traces each deploy. 70 | """ 71 | @spec start_link(opts :: Keyword.t()) :: GenServer.on_start() 72 | def start_link(opts \\ []) do 73 | opts = Keyword.merge(@default_opts, opts) 74 | 75 | GenServer.start_link(__MODULE__, opts, name: Keyword.get(opts, :name, __MODULE__)) 76 | end 77 | 78 | @doc false 79 | @spec init(opts :: Keyword.t()) :: {:ok, State.t()} 80 | def init(opts) do 81 | if opts[:trap_exits?] do 82 | Process.flag(:trap_exit, true) 83 | end 84 | 85 | {:ok, agent_pid} = Agent.start_link(fn -> 0 end) 86 | 87 | state = %State{ 88 | asynchronous_send?: true, 89 | host: opts[:host], 90 | port: opts[:port], 91 | verbose?: opts[:verbose?], 92 | http: opts[:http], 93 | waiting_traces: [], 94 | batch_size: opts[:batch_size], 95 | sync_threshold: opts[:sync_threshold], 96 | agent_pid: agent_pid, 97 | container_id: get_container_id() 98 | } 99 | 100 | {:ok, state} 101 | end 102 | 103 | @cgroup_uuid "[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}" 104 | @cgroup_ctnr "[0-9a-f]{64}" 105 | @cgroup_task "[0-9a-f]{32}-\\d+" 106 | @cgroup_regex Regex.compile!(".*(#{@cgroup_uuid}|#{@cgroup_ctnr}|#{@cgroup_task})(?:\\.scope)?$", "m") 107 | 108 | defp get_container_id() do 109 | with {:ok, file_binary} <- File.read("/proc/self/cgroup"), 110 | [_, container_id] <- Regex.run(@cgroup_regex, file_binary) do 111 | container_id 112 | else 113 | _ -> nil 114 | end 115 | end 116 | 117 | @doc """ 118 | Send spans asynchronously to DataDog. 119 | """ 120 | @spec send_trace(Trace.t(), Keyword.t()) :: :ok 121 | def send_trace(%Trace{} = trace, opts \\ []) do 122 | :telemetry.span([:spandex_datadog, :send_trace], %{trace: trace}, fn -> 123 | timeout = Keyword.get(opts, :timeout, 30_000) 124 | genserver_name = Keyword.get(opts, :name, __MODULE__) 125 | result = GenServer.call(genserver_name, {:send_trace, trace}, timeout) 126 | {result, %{trace: trace}} 127 | end) 128 | end 129 | 130 | @deprecated "Please use send_trace/2 instead" 131 | @doc false 132 | @spec send_spans([Span.t()], Keyword.t()) :: :ok 133 | def send_spans(spans, opts \\ []) when is_list(spans) do 134 | timeout = Keyword.get(opts, :timeout, 30_000) 135 | trace = %Trace{spans: spans} 136 | genserver_name = Keyword.get(opts, :name, __MODULE__) 137 | GenServer.call(genserver_name, {:send_trace, trace}, timeout) 138 | end 139 | 140 | @doc false 141 | def handle_call({:send_trace, trace}, _from, state) do 142 | state = 143 | state 144 | |> enqueue_trace(trace) 145 | |> maybe_flush_traces() 146 | 147 | {:reply, :ok, state} 148 | end 149 | 150 | @doc false 151 | def handle_info({:EXIT, _pid, reason}, state) do 152 | terminate(reason, state) 153 | {:noreply, %State{state | waiting_traces: []}} 154 | end 155 | 156 | @doc false 157 | def terminate(_reason, state) do 158 | # set batch_size to 0 to force any remaining traces to be flushed 159 | # set asynchronous_send? to false to send the last traces synchronously 160 | state 161 | |> Map.put(:batch_size, 0) 162 | |> Map.put(:asynchronous_send?, false) 163 | |> maybe_flush_traces() 164 | 165 | :ok 166 | end 167 | 168 | @spec send_and_log([Trace.t()], State.t()) :: :ok 169 | def send_and_log(traces, _state) when length(traces) < 1 do 170 | # no-op if there's no traces to send 171 | :ok 172 | end 173 | 174 | def send_and_log(traces, %{container_id: container_id, verbose?: verbose?} = state) do 175 | headers = @headers ++ [{"X-Datadog-Trace-Count", length(traces)}] 176 | headers = headers ++ List.wrap(if container_id, do: {"Datadog-Container-ID", container_id}) 177 | 178 | response = 179 | traces 180 | |> Enum.map(&format/1) 181 | |> encode() 182 | |> push(headers, state) 183 | 184 | if verbose? do 185 | Logger.debug(fn -> "Trace response: #{inspect(response)}" end) 186 | end 187 | 188 | :ok 189 | end 190 | 191 | @deprecated "Please use format/3 instead" 192 | @doc false 193 | @spec format(Trace.t()) :: map() 194 | def format(%Trace{spans: spans, priority: priority, baggage: baggage}) do 195 | Enum.map(spans, fn span -> format(span, priority, baggage) end) 196 | end 197 | 198 | @deprecated "Please use format/3 instead" 199 | @doc false 200 | @spec format(Span.t()) :: map() 201 | def format(%Span{} = span), do: format(span, 1, []) 202 | 203 | @spec format(Span.t(), integer(), Keyword.t()) :: map() 204 | def format(%Span{} = span, priority, _baggage) do 205 | %{ 206 | trace_id: span.trace_id, 207 | span_id: span.id, 208 | name: span.name, 209 | start: span.start, 210 | duration: (span.completion_time || SpandexDatadog.Adapter.now()) - span.start, 211 | parent_id: span.parent_id, 212 | error: error(span.error), 213 | resource: span.resource || span.name, 214 | service: span.service, 215 | type: span.type, 216 | meta: meta(span), 217 | metrics: 218 | metrics(span, %{ 219 | _sampling_priority_v1: priority, 220 | "_dd.rule_psr": 1.0, 221 | "_dd.limit_psr": 1.0 222 | }) 223 | } 224 | end 225 | 226 | # Private Helpers 227 | 228 | defp enqueue_trace(state, trace) do 229 | if state.verbose? do 230 | Logger.info(fn -> "Adding trace to stack with #{Enum.count(trace.spans)} spans" end) 231 | end 232 | 233 | %State{state | waiting_traces: [trace | state.waiting_traces]} 234 | end 235 | 236 | defp maybe_flush_traces(%{waiting_traces: traces, batch_size: size} = state) when length(traces) < size do 237 | state 238 | end 239 | 240 | defp maybe_flush_traces(state) do 241 | %{ 242 | asynchronous_send?: async?, 243 | verbose?: verbose?, 244 | waiting_traces: traces 245 | } = state 246 | 247 | if verbose? do 248 | span_count = Enum.reduce(traces, 0, fn trace, acc -> acc + length(trace.spans) end) 249 | Logger.info(fn -> "Sending #{length(traces)} traces, #{span_count} spans." end) 250 | Logger.debug(fn -> "Trace: #{inspect(traces)}" end) 251 | end 252 | 253 | if async? do 254 | if below_sync_threshold?(state) do 255 | Task.start(fn -> 256 | try do 257 | send_and_log(traces, state) 258 | after 259 | Agent.update(state.agent_pid, fn count -> count - 1 end) 260 | end 261 | end) 262 | else 263 | # We get benefits from running in a separate process (like better GC) 264 | # So we async/await here to mimic the behaviour above but still apply backpressure 265 | task = Task.async(fn -> send_and_log(traces, state) end) 266 | Task.await(task) 267 | end 268 | else 269 | send_and_log(traces, state) 270 | end 271 | 272 | %State{state | waiting_traces: []} 273 | end 274 | 275 | defp below_sync_threshold?(state) do 276 | Agent.get_and_update(state.agent_pid, fn count -> 277 | if count < state.sync_threshold do 278 | {true, count + 1} 279 | else 280 | {false, count} 281 | end 282 | end) 283 | end 284 | 285 | @spec meta(Span.t()) :: map 286 | defp meta(span) do 287 | %{} 288 | |> add_env_data(span) 289 | |> add_version_data(span) 290 | |> add_error_data(span) 291 | |> add_http_data(span) 292 | |> add_sql_data(span) 293 | |> add_tags(span) 294 | |> Enum.reject(fn {_k, v} -> is_nil(v) end) 295 | |> Enum.into(%{}) 296 | end 297 | 298 | @spec add_env_data(map, Span.t()) :: map 299 | defp add_env_data(meta, %Span{env: nil}), do: meta 300 | 301 | defp add_env_data(meta, %Span{env: env}) do 302 | Map.put(meta, :env, env) 303 | end 304 | 305 | @spec add_version_data(map, Span.t()) :: map 306 | defp add_version_data(meta, %Span{service_version: nil}), do: meta 307 | 308 | defp add_version_data(meta, %Span{service_version: version}) do 309 | Map.put(meta, :version, version) 310 | end 311 | 312 | @spec add_error_data(map, Span.t()) :: map 313 | defp add_error_data(meta, %{error: nil}), do: meta 314 | 315 | defp add_error_data(meta, %{error: error}) do 316 | meta 317 | |> add_error_type(error[:exception]) 318 | |> add_error_message(error[:exception]) 319 | |> add_error_stacktrace(error[:stacktrace]) 320 | end 321 | 322 | @spec add_error_type(map, Exception.t() | nil) :: map 323 | defp add_error_type(meta, %struct{}), do: Map.put(meta, "error.type", struct) 324 | defp add_error_type(meta, _), do: meta 325 | 326 | @spec add_error_message(map, Exception.t() | nil) :: map 327 | defp add_error_message(meta, nil), do: meta 328 | 329 | defp add_error_message(meta, exception), 330 | do: Map.put(meta, "error.msg", Exception.message(exception)) 331 | 332 | @spec add_error_stacktrace(map, list | nil) :: map 333 | defp add_error_stacktrace(meta, nil), do: meta 334 | 335 | defp add_error_stacktrace(meta, stacktrace), 336 | do: Map.put(meta, "error.stack", Exception.format_stacktrace(stacktrace)) 337 | 338 | @spec add_http_data(map, Span.t()) :: map 339 | defp add_http_data(meta, %{http: nil}), do: meta 340 | 341 | defp add_http_data(meta, %{http: http}) do 342 | status_code = 343 | if http[:status_code] do 344 | to_string(http[:status_code]) 345 | end 346 | 347 | meta 348 | |> Map.put("http.url", http[:url]) 349 | |> Map.put("http.status_code", status_code) 350 | |> Map.put("http.method", http[:method]) 351 | end 352 | 353 | @spec add_sql_data(map, Span.t()) :: map 354 | defp add_sql_data(meta, %{sql_query: nil}), do: meta 355 | 356 | defp add_sql_data(meta, %{sql_query: sql}) do 357 | meta 358 | |> Map.put("sql.query", sql[:query]) 359 | |> Map.put("sql.rows", sql[:rows]) 360 | |> Map.put("sql.db", sql[:db]) 361 | end 362 | 363 | @spec add_tags(map, Span.t()) :: map 364 | defp add_tags(meta, %{tags: nil}), do: meta 365 | 366 | defp add_tags(meta, %{tags: tags}) do 367 | tags = tags |> Keyword.delete(:analytics_event) 368 | 369 | Map.merge( 370 | meta, 371 | tags 372 | |> Enum.map(fn {k, v} -> {k, term_to_string(v)} end) 373 | |> Enum.into(%{}) 374 | ) 375 | end 376 | 377 | @spec metrics(Span.t(), map) :: map 378 | defp metrics(span, initial_value = %{}) do 379 | initial_value 380 | |> add_metrics(span) 381 | |> Enum.reject(fn {_k, v} -> is_nil(v) end) 382 | |> Enum.into(%{}) 383 | end 384 | 385 | @spec add_metrics(map, Span.t()) :: map 386 | defp add_metrics(metrics, %{tags: nil}), do: metrics 387 | 388 | defp add_metrics(metrics, %{tags: tags}) do 389 | with analytics_event <- tags |> Keyword.get(:analytics_event), 390 | true <- analytics_event != nil do 391 | Map.merge( 392 | metrics, 393 | %{"_dd1.sr.eausr" => 1} 394 | ) 395 | else 396 | _ -> 397 | metrics 398 | end 399 | end 400 | 401 | @spec error(nil | Keyword.t()) :: integer 402 | defp error(nil), do: 0 403 | 404 | defp error(keyword) do 405 | if Enum.any?(keyword, fn {_, v} -> not is_nil(v) end) do 406 | 1 407 | else 408 | 0 409 | end 410 | end 411 | 412 | @spec encode(data :: term) :: iodata | no_return 413 | defp encode(data), 414 | do: data |> deep_remove_nils() |> Msgpax.pack!(data) 415 | 416 | @spec push(body :: iodata(), headers, State.t()) :: any() 417 | defp push(body, headers, %State{http: http, host: host, port: port}), 418 | do: http.put("#{host}:#{port}/v0.3/traces", body, headers) 419 | 420 | @spec deep_remove_nils(term) :: term 421 | defp deep_remove_nils(term) when is_map(term) do 422 | term 423 | |> Enum.reject(fn {_k, v} -> is_nil(v) end) 424 | |> Enum.map(fn {k, v} -> {k, deep_remove_nils(v)} end) 425 | |> Enum.into(%{}) 426 | end 427 | 428 | defp deep_remove_nils(term) when is_list(term) do 429 | if Keyword.keyword?(term) do 430 | term 431 | |> Enum.reject(fn {_k, v} -> is_nil(v) end) 432 | |> Enum.map(fn {k, v} -> {k, deep_remove_nils(v)} end) 433 | else 434 | Enum.map(term, &deep_remove_nils/1) 435 | end 436 | end 437 | 438 | defp deep_remove_nils(term), do: term 439 | 440 | defp term_to_string(term) when is_boolean(term), do: inspect(term) 441 | defp term_to_string(term) when is_binary(term), do: term 442 | defp term_to_string(term) when is_atom(term), do: term 443 | defp term_to_string(term), do: inspect(term) 444 | end 445 | --------------------------------------------------------------------------------