├── lib ├── fast_ts.ex ├── riemann_proto.ex ├── fast_ts │ ├── index │ │ └── server.ex │ ├── router │ │ └── modules.ex │ ├── common.ex │ ├── stream │ │ ├── contex.ex │ │ └── pipeline.ex │ ├── server.ex │ ├── router.ex │ ├── supervisor.ex │ └── stream.ex └── riemann.proto ├── test ├── fast_ts_test.exs ├── test_helper.exs └── fast_ts │ └── stream │ ├── context_test.exs │ ├── pipeline_test.exs │ ├── stateless_filters_test.exs │ └── stateful_filters_test.exs ├── .gitignore ├── config ├── config.exs └── routes │ └── route.exs ├── LICENSE ├── mix.lock ├── mix.exs ├── TODO.md ├── Dockerfile └── README.md /lib/fast_ts.ex: -------------------------------------------------------------------------------- 1 | defmodule FastTS do 2 | use Application 3 | 4 | def start(_type, _args) do 5 | FastTS.Supervisor.start_link 6 | end 7 | 8 | end 9 | -------------------------------------------------------------------------------- /test/fast_ts_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FastTSTest do 2 | use ExUnit.Case 3 | doctest FastTS 4 | 5 | test "the truth" do 6 | assert 1 + 1 == 2 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/riemann_proto.ex: -------------------------------------------------------------------------------- 1 | defmodule RiemannProto do 2 | use Protobuf, from: Path.expand("riemann.proto", __DIR__) 3 | end 4 | 5 | # TODO: can we define a shorter name ? 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | _build 2 | cover 3 | deps 4 | .rebar 5 | erl_crash.dump 6 | *.ez 7 | 8 | # This is where I put experiments I want do not want to delete now, but do not want to commit 9 | attic 10 | 11 | # OS X 12 | .DS_Store 13 | 14 | # Intellij 15 | .idea 16 | fast_ts.iml 17 | 18 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | 3 | 4 | 5 | defmodule FastTSTestHelper do 6 | 7 | # We might use a lib for this in case we need more complex data to be generated. 8 | # there is faker and ExMachina. 9 | def create(:event), do: %RiemannProto.Event{metric_f: :random.uniform() * 10, state: Enum.random(["ok", "down"]), host: "h" , service: "s"} 10 | 11 | end 12 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | # This file is responsible for configuring your application 2 | # and its dependencies with the aid of the Mix.Config module. 3 | use Mix.Config 4 | 5 | # route_dir can be overriden from environment variable: FTS_ROUTE_DIR 6 | config :fast_ts, route_dir: "config/routes" 7 | 8 | # Configure MailMan to be able to send email 9 | config :mailman, 10 | relay: "localhost", 11 | port: 1025, 12 | auth: :never 13 | -------------------------------------------------------------------------------- /lib/fast_ts/index/server.ex: -------------------------------------------------------------------------------- 1 | defmodule FastTSIndexServer do 2 | @doc """ 3 | Create a new index. 4 | """ 5 | def start_link do 6 | Agent.start_link(fn -> HashDict.new end) 7 | end 8 | 9 | @doc """ 10 | Gets a value from the `index` by `key`. 11 | """ 12 | def get(index, key) do 13 | Agent.get(index, &HashDict.get(&1, key)) 14 | end 15 | 16 | @doc """ 17 | Puts the `value` for the given `key` in the `index`. 18 | """ 19 | def put(index, key, value) do 20 | Agent.update(index, &HashDict.put(&1, key, value)) 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2016 ProcessOne SARL 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{"bbmustache": {:hex, :bbmustache, "1.0.4"}, 2 | "cf": {:hex, :cf, "0.2.1"}, 3 | "dialyze": {:hex, :dialyze, "0.2.1"}, 4 | "earmark": {:hex, :earmark, "0.2.1"}, 5 | "eqc_ex": {:hex, :eqc_ex, "1.2.4"}, 6 | "erlware_commons": {:hex, :erlware_commons, "0.19.0"}, 7 | "ex_doc": {:hex, :ex_doc, "0.11.4"}, 8 | "exprotobuf": {:hex, :exprotobuf, "0.11.0"}, 9 | "exrm": {:hex, :exrm, "1.0.2"}, 10 | "gen_smtp": {:hex, :gen_smtp, "0.9.0"}, 11 | "getopt": {:hex, :getopt, "0.8.2"}, 12 | "gpb": {:hex, :gpb, "3.18.10"}, 13 | "mailman": {:hex, :mailman, "0.2.2"}, 14 | "providers": {:hex, :providers, "1.6.0"}, 15 | "relx": {:hex, :relx, "3.18.0"}} 16 | -------------------------------------------------------------------------------- /lib/fast_ts/router/modules.ex: -------------------------------------------------------------------------------- 1 | # TODO Module should be renamed registry 2 | defmodule FastTS.Router.Modules do 3 | @doc """ 4 | Router agent: Keeps and serves the list of router modules. 5 | """ 6 | def start_link do 7 | {:ok, _} = Agent.start_link(fn -> [] end, name: __MODULE__) 8 | end 9 | 10 | @doc """ 11 | Retrieve the list of register router modules. 12 | """ 13 | def get do 14 | Agent.get(__MODULE__, fn modules -> modules end) 15 | end 16 | 17 | @doc """ 18 | Register a module as FastTS router. 19 | """ 20 | def register(module) do 21 | Agent.update(__MODULE__, fn modules -> [module|modules] end) 22 | end 23 | 24 | def stop do 25 | Agent.stop(__MODULE__) 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /test/fast_ts/stream/context_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FastTS.Stream.ContextTest do 2 | use ExUnit.Case, async: true 3 | alias FastTS.Stream.Context 4 | 5 | test "stores values by key in context" do 6 | {:ok, context} = Context.start_link 7 | assert Context.get(context, "key1") == nil 8 | 9 | Context.put(context, "key1", 3) 10 | assert Context.get(context, "key1") == 3 11 | end 12 | 13 | test "can clear context" do 14 | {:ok, context} = Context.start_link 15 | assert Context.get(context, "key1") == nil 16 | 17 | Context.put(context, "key1", 3) 18 | assert Context.get(context, "key1") == 3 19 | 20 | Context.clear(context) 21 | assert Context.get(context, "key1") == nil 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /test/fast_ts/stream/pipeline_test.exs: -------------------------------------------------------------------------------- 1 | defmodule FastTS.Stream.PipelineTest do 2 | use ExUnit.Case 3 | 4 | test "Ignore empty pipeline" do 5 | assert FastTS.Stream.Pipeline.start_link(:empty, []) == :ignore 6 | end 7 | 8 | test "Create a one step pipeline" do 9 | test_pid = self 10 | start_fun = fn _ets_table, next_pid -> 11 | send test_pid, {:from, :started, self} 12 | fn(event) -> event end 13 | end 14 | 15 | {:ok, _pid} = FastTS.Stream.Pipeline.start_link(:pipe1, [{:stateful, start_fun}]) 16 | 17 | receive do 18 | {:from, :started, step1_pid} -> 19 | assert Process.alive?(step1_pid) 20 | after 5000 -> 21 | assert false, "pipeline process not properly started" 22 | end 23 | end 24 | 25 | end 26 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule FastTS.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [app: :fast_ts, 6 | version: "0.0.1", 7 | elixir: "~> 1.2", 8 | build_embedded: Mix.env == :prod, 9 | start_permanent: Mix.env == :prod, 10 | deps: deps(Mix.env)] 11 | end 12 | 13 | def application do 14 | [applications: [:logger, :exprotobuf, :gpb], 15 | mod: {FastTS, []} 16 | ] 17 | end 18 | 19 | defp deps(:prod) do 20 | [{:exprotobuf, "~> 0.11.0"}, 21 | {:mailman, "~> 0.2.2"}, 22 | # {:mailman, "~> 0.2.1"}, 23 | # {:mailman, git: "/Users/mremond/tmp/mailman", branch: "mailman_mix_config"}, 24 | {:exrm, "~> 1.0.0-rc7"}] 25 | end 26 | 27 | defp deps(_) do 28 | deps(:prod) ++ 29 | [{:dialyze, "~> 0.2.0"}, 30 | {:eqc_ex, "~> 1.2.3"} 31 | ] 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/fast_ts/common.ex: -------------------------------------------------------------------------------- 1 | defmodule Common do 2 | 3 | # :calendar.datetime_to_gregorian_seconds({{1970,1,1},{0,0,0}}) 4 | @epoch_ref_gregorian 62167219200 5 | 6 | def epoch_to_datetime(nil), do: :error 7 | def epoch_to_datetime(epoch) when is_integer(epoch), do: :calendar.gregorian_seconds_to_datetime(@epoch_ref_gregorian + epoch) 8 | def epoch_to_datetime(epoch) when is_list(epoch) do 9 | case Integer.parse(epoch) do 10 | :error -> :error 11 | int -> epoch_to_datetime(int) 12 | end 13 | end 14 | 15 | def epoch_to_string(epoch) do 16 | case epoch_to_datetime(epoch) do 17 | :error -> "" 18 | {{y,m,d},{h,mi,s}} -> "#{y}-#{rjust(m)}-#{rjust(d)} #{rjust(h)}:#{rjust(mi)}:#{rjust(s)}" 19 | end 20 | end 21 | 22 | defp rjust(i) do 23 | i 24 | |> Integer.to_string 25 | |> String.rjust(2, ?0) 26 | end 27 | 28 | end 29 | -------------------------------------------------------------------------------- /lib/fast_ts/stream/contex.ex: -------------------------------------------------------------------------------- 1 | defmodule FastTS.Stream.Context do 2 | 3 | @doc """ 4 | Starts a new context agent that will be associated to pipeline block. 5 | """ 6 | def start_link do 7 | Agent.start_link(fn -> %{} end) 8 | end 9 | 10 | @doc """ 11 | Gets a pipeline block value from context for a given key. 12 | """ 13 | def get(context, key) do 14 | Agent.get(context, &Map.get(&1, key)) 15 | end 16 | 17 | @doc """ 18 | Puts the `value` for the given `key` in the block context. 19 | """ 20 | def put(context, key, value) do 21 | Agent.update(context, &Map.put(&1, key, value)) 22 | end 23 | 24 | @doc """ 25 | Clear the block context. 26 | """ 27 | def clear(context) do 28 | Agent.update(context, fn _ -> %{} end) 29 | end 30 | 31 | @doc """ 32 | Get full context as map 33 | """ 34 | def get_all(context) do 35 | Agent.get(context, fn state -> state end) 36 | end 37 | 38 | end 39 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | # Multiple format 2 | 3 | To have a generic streaming tool, we need to support multiple format: 4 | 5 | * Metrics: We support the Riemann format for metrics. 6 | * Message: We support the Node Red format for messages. 7 | 8 | Some blocks can handle both, other type of blocks will just drop the 9 | content of unknown "packet" types. 10 | 11 | # Improvement for Elixir protobuf support 12 | 13 | ``` 14 | iex(6)> new = RiemannProto.Event.new 15 | %RiemannProto.Event{attributes: [], description: nil, host: nil, metric_d: nil, 16 | metric_f: nil, metric_sint64: nil, service: nil, state: nil, tags: [], 17 | time: nil, ttl: nil} 18 | iex(7)> new[:attributes] 19 | ** (UndefinedFunctionError) undefined function: RiemannProto.Event.fetch/2 20 | (fast_core) RiemannProto.Event.fetch(%RiemannProto.Event{attributes: [], description: nil, host: nil, metric_d: nil, metric_f: nil, metric_sint64: nil, service: nil, state: nil, tags: [], time: nil, ttl: nil}, :attributes) 21 | (elixir) lib/access.ex:72: Access.get/3 22 | ``` 23 | 24 | We need the generated proto beam to support fetch to have a nice getter syntax. 25 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM msaraiva/erlang:18.1 2 | 3 | RUN apk --update add erlang-crypto erlang-sasl && rm -rf /var/cache/apk/* 4 | 5 | ENV APP_NAME fast_ts 6 | ENV APP_VERSION "0.0.1" 7 | ENV PORT 5555 8 | ENV FTS_ROUTE_DIR /$APP_NAME/routes 9 | 10 | RUN mkdir -p /$APP_NAME 11 | ADD rel/$APP_NAME/bin /$APP_NAME/bin 12 | ADD rel/$APP_NAME/lib /$APP_NAME/lib 13 | ADD rel/$APP_NAME/releases/start_erl.data /$APP_NAME/releases/start_erl.data 14 | ADD rel/$APP_NAME/releases/$APP_VERSION/$APP_NAME.boot /$APP_NAME/releases/$APP_VERSION/$APP_NAME.boot 15 | ADD rel/$APP_NAME/releases/$APP_VERSION/$APP_NAME.rel /$APP_NAME/releases/$APP_VERSION/$APP_NAME.rel 16 | ADD rel/$APP_NAME/releases/$APP_VERSION/$APP_NAME.script /$APP_NAME/releases/$APP_VERSION/$APP_NAME.script 17 | ADD rel/$APP_NAME/releases/$APP_VERSION/$APP_NAME.sh /$APP_NAME/releases/$APP_VERSION/$APP_NAME.sh 18 | ADD rel/$APP_NAME/releases/$APP_VERSION/start.boot /$APP_NAME/releases/$APP_VERSION/start.boot 19 | ADD rel/$APP_NAME/releases/$APP_VERSION/sys.config /$APP_NAME/releases/$APP_VERSION/sys.config 20 | ADD rel/$APP_NAME/releases/$APP_VERSION/vm.args /$APP_NAME/releases/$APP_VERSION/vm.args 21 | 22 | RUN mkdir -p /$APP_NAME/routes 23 | ADD config/routes/route.exs /$APP_NAME/routes/route.exs 24 | 25 | EXPOSE $PORT 26 | 27 | CMD trap exit TERM; /$APP_NAME/bin/$APP_NAME foreground & wait -------------------------------------------------------------------------------- /lib/riemann.proto: -------------------------------------------------------------------------------- 1 | // From https://github.com/aphyr/riemann-java-client/blob/c6fe3537cd81341710fe27802641f34e8b639a5a/src/main/proto/riemann/proto.proto 2 | option java_package = "com.aphyr.riemann"; 3 | option java_outer_classname = "Proto"; 4 | 5 | // Deprecated; state was used by early versions of the protocol, but not any 6 | // more. 7 | message State { 8 | optional int64 time = 1; 9 | optional string state = 2; 10 | optional string service = 3; 11 | optional string host = 4; 12 | optional string description = 5; 13 | optional bool once = 6; 14 | repeated string tags = 7; 15 | optional float ttl = 8; 16 | } 17 | 18 | message Event { 19 | optional int64 time = 1; 20 | optional string state = 2; 21 | optional string service = 3; 22 | optional string host = 4; 23 | optional string description = 5; 24 | repeated string tags = 7; 25 | optional float ttl = 8; 26 | repeated Attribute attributes = 9; 27 | 28 | optional sint64 metric_sint64 = 13; 29 | optional double metric_d = 14; 30 | optional float metric_f = 15; 31 | } 32 | 33 | message Query { 34 | optional string string = 1; 35 | } 36 | 37 | message Msg { 38 | optional bool ok = 2; 39 | optional string error = 3; 40 | repeated State states = 4; 41 | optional Query query = 5; 42 | repeated Event events = 6; 43 | } 44 | 45 | message Attribute { 46 | required string key = 1; 47 | optional string value = 2; 48 | } 49 | -------------------------------------------------------------------------------- /lib/fast_ts/server.ex: -------------------------------------------------------------------------------- 1 | defmodule FastTS.Server do 2 | 3 | require Logger 4 | 5 | @doc """ 6 | Starts accepting connections on the give `port`. 7 | """ 8 | @spec accept(port :: integer) :: no_return 9 | def accept(port) do 10 | {:ok, socket} = :gen_tcp.listen(port, [:binary, packet: 4, active: false, reuseaddr: true]) 11 | Logger.info "Accepting connections on port #{port}" 12 | loop_acceptor(socket) 13 | end 14 | 15 | defp loop_acceptor(socket) do 16 | {:ok, client} = :gen_tcp.accept(socket) 17 | pid = spawn(fn -> serve(client) end) 18 | :ok = :gen_tcp.controlling_process(client, pid) 19 | loop_acceptor(socket) 20 | end 21 | 22 | def serve(socket) do 23 | Logger.debug "Client connected" 24 | result = socket |> read_message |> send_response 25 | case result do 26 | :stop -> 27 | :stop 28 | _ -> 29 | serve(socket) 30 | end 31 | end 32 | 33 | defp read_message(socket) do 34 | case :gen_tcp.recv(socket, 0) do 35 | {:ok, data} -> 36 | process_data(data) 37 | socket 38 | {:error, :closed} -> 39 | :stop 40 | {:error, reason} -> 41 | Logger.debug("Error reading message on socket: #{reason}") 42 | :stop 43 | end 44 | end 45 | 46 | defp send_response(:stop), do: :stop 47 | defp send_response(socket) do 48 | msg = RiemannProto.Msg.encode(RiemannProto.Msg.new(ok: true)) 49 | :gen_tcp.send(socket, msg) 50 | end 51 | 52 | defp process_data(data) do 53 | RiemannProto.Msg.decode(data) 54 | |> extract_events 55 | |> Enum.map(&ensure_timestamp/1) 56 | |> Enum.map(&stream_event/1) 57 | end 58 | 59 | defp extract_events(%RiemannProto.Msg{events: events}), do: events 60 | defp extract_events(_), do: [] 61 | 62 | defp ensure_timestamp(event = %RiemannProto.Event{time: nil}), do: %{event | time: System.system_time(:seconds)} 63 | defp ensure_timestamp(event = %RiemannProto.Event{}), do: event 64 | 65 | defp stream_event(event) do 66 | # TODO: 67 | # - Catch to avoid crash and report errors 68 | FastTS.Router.Modules.get 69 | |> Enum.each(fn(module) -> apply(module, :stream, [event]) end) 70 | end 71 | 72 | end 73 | 74 | # TODO: 75 | # - Ignore messages with outdated ttl 76 | # - Fix issue with parallel / spawn execution 77 | -------------------------------------------------------------------------------- /lib/fast_ts/router.ex: -------------------------------------------------------------------------------- 1 | defmodule FastTS.Router do 2 | require Logger 3 | 4 | defmacro __using__(_options) do 5 | quote do 6 | alias RiemannProto.Event 7 | import FastTS.Stream 8 | 9 | 10 | # Needed to be able to inject pipeline macro in the module using FastTS.Router: 11 | import unquote(__MODULE__), only: [pipeline: 2] 12 | 13 | # Define pipelines module attribute as accumulator: 14 | Module.register_attribute __MODULE__, :pipelines, accumulate: true 15 | 16 | # Delay the generation of some method to a last pass to allow getting the full result of the pipeline 17 | # accumulation (it will be done in macro __before_compile__/1 18 | @before_compile unquote(__MODULE__) 19 | 20 | FastTS.Router.register_router(__MODULE__) 21 | end 22 | end 23 | 24 | # list_pipelines is added to module and print the correct list of defined pipelines when called 25 | defmacro __before_compile__(_env) do 26 | quote do 27 | 28 | def streams do 29 | Enum.map(@pipelines, 30 | fn(name) -> 31 | {name, apply(__MODULE__, name, [])} 32 | end) 33 | end 34 | 35 | def stream(event) do 36 | streams |> 37 | Enum.each( fn({name, _pipeline}) -> send(name, event) end) 38 | end 39 | 40 | end 41 | end 42 | 43 | # To make sure, we generate properly the pipeline, a pipeline can 44 | # only use a sequence of operation defined in the stream API 45 | defmacro pipeline(description, do: pipeline_block) do 46 | pipeline_name = String.to_atom(description) 47 | 48 | # Transform the block call in a list of function calls 49 | block = case pipeline_block do 50 | nil -> 51 | nil 52 | {:__block__, [], block_sequence} -> 53 | block_sequence 54 | single_op -> 55 | [single_op] 56 | end 57 | 58 | case block do 59 | # 60 | nil -> 61 | Logger.info "Ignoring empty pipeline '#{description}'" 62 | _ -> 63 | quote do 64 | @pipelines unquote(pipeline_name) 65 | def unquote(pipeline_name)(), do: unquote(block) 66 | end 67 | end 68 | end 69 | 70 | def register_router(module) do 71 | Logger.info "Registering Router module: #{inspect module}" 72 | FastTS.Router.Modules.register(module) 73 | end 74 | 75 | end 76 | -------------------------------------------------------------------------------- /lib/fast_ts/stream/pipeline.ex: -------------------------------------------------------------------------------- 1 | defmodule FastTS.Stream.Pipeline do 2 | use GenServer 3 | 4 | @type pipeline :: list({:stateful, fun}|{:stateless, fun}) 5 | @spec start_link(name :: String.t, pipeline :: pipeline) :: :ignore | {:error, any} | {:ok, pid} 6 | def start_link(_name, []), do: :ignore 7 | def start_link(name, pipeline) do 8 | GenServer.start_link(__MODULE__, [name, pipeline]) 9 | end 10 | 11 | # Create a chained list of process to pass events through the pipeline 12 | # A pipeline is composed of a list of 'start' anonymous function. start function 13 | # They return the function to perform to add an event. 14 | # pipeline = [{:stateful, start_fun/2}|{:stateless, add_event_fun/1}] 15 | # Stateful start_fun takes parameters ets_table and pid of the next process in pipeline 16 | # stateless fun directly pass a basic add event fun receiving event as parameter. They bypass the ets table creation 17 | def init([name, pipeline]) do 18 | process = process_name(name) 19 | pipeline 20 | |> Enum.reverse 21 | |> Enum.reduce( 22 | nil, 23 | fn({:stateful, start_fun}, next_pid) -> 24 | spawn_link( fn -> set_loop_state(start_fun, next_pid) end ) 25 | ({:stateless, add_event_fun}, next_pid) -> 26 | spawn_link( fn -> do_loop(add_event_fun, next_pid) end) 27 | end) 28 | |> Process.register(process) 29 | 30 | state = [process, pipeline] 31 | {:ok, state} 32 | end 33 | 34 | def next(_result, nil), do: :nothing 35 | def next(:defer, _pid), do: :nothing 36 | def next(nil, _pid), do: :nothing 37 | def next(result, pid), do: send(pid, result) 38 | 39 | # We start one loop per steps in the pipeline. Each pipeline step is a process. 40 | # 41 | # TODO create a supervisor per pipeline, place each pipeline supervisor under a stream supervisor 42 | def set_loop_state(start_fun, next_pid) do 43 | {:ok, context} = FastTS.Stream.Context.start_link 44 | add_event_fun = start_fun.(context, next_pid) 45 | do_loop(add_event_fun, next_pid) 46 | end 47 | 48 | def do_loop(fun, next_pid) do 49 | receive do 50 | # TODO we can probably receive an event to reset state in the loop to avoid race condition between receive event / reset 51 | event -> 52 | event 53 | |> fun.() 54 | |> next(next_pid) 55 | end 56 | do_loop(fun, next_pid) 57 | end 58 | 59 | # Helper 60 | defp process_name(name) when is_atom(name), do: name 61 | defp process_name(name) when is_binary(name), do: String.to_atom(name) 62 | 63 | end 64 | -------------------------------------------------------------------------------- /config/routes/route.exs: -------------------------------------------------------------------------------- 1 | defmodule HelloFast.Router do 2 | use FastTS.Router 3 | 4 | pipeline "Basic pipeline" do 5 | # We only take functions under a given value 6 | over(12) 7 | email("mremond@test.com") 8 | stdout 9 | end 10 | 11 | # For now, we assume that we can only put pipeline function in the block 12 | # TODO We need to add more consistency checks on the content of the pipeline 13 | pipeline "Second pipeline" do 14 | rate(5) 15 | stdout 16 | end 17 | 18 | pipeline "Empty pipeline are ignored" do 19 | end 20 | 21 | pipeline "Scale metrics" do 22 | scale(15) 23 | stdout 24 | end 25 | 26 | pipeline "stabilize pipeline" do 27 | stable(5, fn %Event{state: s} -> s end) 28 | stdout 29 | end 30 | 31 | pipeline "tag" do 32 | tag("tag1") #add single tag 33 | tag(["tag2", "tag1", "tag3"]) #add multiple tags. Duplicates are removed 34 | tagged_all("tag3") 35 | tagged_all(["tag2", "test"]) 36 | tagged_any(["tag1", "apsdfgfy"]) 37 | stdout 38 | end 39 | 40 | pipeline "Generic filtering and mapping" do 41 | filter(fn %Event{service: "eth0" <> _} -> true end) #filter events with service starting with "eth0". 42 | map(fn x -> %{x | service: "net"} end) 43 | stdout 44 | end 45 | 46 | pipeline "State change detection" do 47 | changed_state("up") 48 | stdout 49 | end 50 | 51 | pipeline "stream reduce" do 52 | sreduce(fn(%Event{metric_f: f1}, %Event{metric_f: f2} = e) -> %{e | metric_f: max(f1, f2)} end) 53 | stdout 54 | end 55 | 56 | pipeline "stream reduce with initial val" do 57 | sreduce(fn(%Event{metric_f: f1}, %Event{metric_f: f2} = e) -> %{e | metric_f: max(f1, f2)} end, %Event{metric_f: 100}) 58 | stdout 59 | end 60 | 61 | pipeline "Generic change detection" do 62 | # This has the same effect than the State change detection pipeline 63 | changed(fn %Event{state: state} -> state end, "up", 64 | fn %Event{host: host, service: service} -> {host, service} end) 65 | stdout 66 | end 67 | 68 | pipeline "consecutive runs" do 69 | runs(3, fn %Event{state: state} -> state end) 70 | stdout 71 | end 72 | 73 | pipeline "throttle" do 74 | throttle(2,5) #at most 2 events each 5 seconds 75 | stdout 76 | end 77 | 78 | # TODO we need filter / matching 79 | # pipeline localhost(%Event{host: "localhost"}) do 80 | # rate(5) 81 | # stdout 82 | # end 83 | 84 | # pipeline "Calculate Rate and Broadcast" do 85 | # filter(is_server(%Event{host: "localhost"})) 86 | # IO.puts "We are running pipeline CRaB" 87 | # end 88 | 89 | 90 | end 91 | -------------------------------------------------------------------------------- /lib/fast_ts/supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule FastTS.Supervisor do 2 | use Supervisor 3 | 4 | require Logger 5 | require File 6 | require Path 7 | 8 | def start_link do 9 | set_routes 10 | Supervisor.start_link(__MODULE__, [], name: __MODULE__) 11 | end 12 | 13 | def init([]) do 14 | # TODO: We generate a spec id based on index, but pipeline id 15 | # should probably be generated when compiling the router 16 | modules = FastTS.Router.Modules.get |> Enum.reduce([], fn(module, acc) -> 17 | apply(module, :streams, []) ++ acc 18 | end) 19 | 20 | streams = Enum.map(Enum.with_index(modules), 21 | fn({{name, pipeline}, index}) -> 22 | worker(FastTS.Stream.Pipeline, [name, pipeline], id: index ) end) 23 | children = streams ++ 24 | [ 25 | # TODO: Make port configurable 26 | worker(Task, [FastTS.Server, :accept, [5555]]) 27 | ] 28 | supervise(children, strategy: :one_for_one) 29 | end 30 | 31 | # TODO set_routes should be part of the pipeline supervision process 32 | # -> Or maybe not, as of today, pipeline steps are linked process, the whole pipeline is restarted in case of crash, which is what we want 33 | defp set_routes do 34 | FastTS.Router.Modules.start_link() 35 | get_route_files |> Enum.map(fn(file) -> Code.load_file file end) 36 | end 37 | 38 | defp get_route_files do 39 | case get_route_dir do 40 | nil -> 41 | Logger.warning "No fast_ts route_dir configured: Using route file 'config/route.exs'" 42 | ["config/route.exs"] 43 | route_dir -> 44 | {:ok, files} = File.ls(route_dir) 45 | exs_files = Enum.reduce(files, [], 46 | fn(file, acc) -> 47 | cond do 48 | Path.extname(file) == ".exs" -> 49 | [Path.join(route_dir, file)|acc] 50 | true -> 51 | acc 52 | end 53 | end) 54 | case exs_files do 55 | [] -> 56 | Logger.error "No .exs file in route_dir" 57 | # TODO register a default dumb pipeline that output everything to logs 58 | [] 59 | _ -> 60 | exs_files 61 | end 62 | end 63 | end 64 | 65 | # First try to read route dir from FTS_ROUTE_DIR environment 66 | # variable, then try value from config file 67 | defp get_route_dir do 68 | env_route_dir = System.get_env("FTS_ROUTE_DIR") 69 | case env_route_dir do 70 | nil -> 71 | Application.get_env(:fast_ts, :route_dir) 72 | _ -> 73 | env_route_dir 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /test/fast_ts/stream/stateless_filters_test.exs: -------------------------------------------------------------------------------- 1 | defmodule StatelessFiltersTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias RiemannProto.Event 5 | import FastTSTestHelper 6 | 7 | doctest FastTS 8 | 9 | test "under" do 10 | {:stateless, f} = FastTS.Stream.under(2) 11 | ev3 = %{create(:event) | metric_f: 3} 12 | ev2 = %{create(:event) | metric_f: 2} 13 | ev0 = %{create(:event) | metric_f: 0} 14 | assert ev0 == f.(ev0) 15 | assert nil == f.(ev2) 16 | assert nil == f.(ev3) 17 | end 18 | 19 | test "over" do 20 | {:stateless, f} = FastTS.Stream.over(2) 21 | ev3 = %{create(:event) | metric_f: 3} 22 | ev2 = %{create(:event) | metric_f: 2} 23 | ev0 = %{create(:event) | metric_f: 0} 24 | assert nil == f.(ev0) 25 | assert nil == f.(ev2) 26 | assert ev3 == f.(ev3) 27 | end 28 | 29 | test "scale" do 30 | {:stateless, f} = FastTS.Stream.scale(2) 31 | ev2 = %{create(:event) | metric_f: 2} 32 | ev0 = %{create(:event) | metric_f: 0} 33 | assert %{ev2 | metric_f: 4} == f.(ev2) 34 | assert ev0 == f.(ev0) 35 | end 36 | 37 | test "filter" do 38 | ev = %{create(:event) | metric_f: 1, state: "ok"} 39 | 40 | # Function clause exceptions are handled as false 41 | {:stateless, down} = FastTS.Stream.filter(fn %Event{state: "down"} -> true end) 42 | assert nil == down.(ev) 43 | 44 | {:stateless, up} = FastTS.Stream.filter(fn %Event{state: "ok"} -> true end) 45 | assert ev == up.(ev) 46 | 47 | {:stateless, down} = FastTS.Stream.filter(fn %Event{state: "down"} -> true end) 48 | assert nil == down.(ev) 49 | end 50 | 51 | test "map" do 52 | ev = %{create(:event) | metric_f: 1, state: "ok"} 53 | {:stateless, m} = FastTS.Stream.map(fn ev -> %{ev | state: "down"} end) 54 | assert %{ev | state: "down"} == m.(ev) 55 | end 56 | 57 | test "tag" do 58 | ev = %{create(:event) | tags: []} 59 | {:stateless, t} = FastTS.Stream.tag("tag1") 60 | assert %{ev | tags: ["tag1"]} == t.(ev) 61 | {:stateless, t} = FastTS.Stream.tag(["tag1", "tag2"]) 62 | assert %{ev | tags: ["tag1", "tag2"]} == t.(ev) 63 | 64 | ev = %{create(:event) | tags: ["tag1"]} 65 | {:stateless, t} = FastTS.Stream.tag("tag1") 66 | assert ev == t.(ev) 67 | {:stateless, t} = FastTS.Stream.tag("tag2") 68 | assert %{ev | tags: ["tag1", "tag2"]} == t.(ev) 69 | end 70 | 71 | test "tagged_all" do 72 | ev = %{create(:event) | tags: []} 73 | ev1 = %{create(:event) | tags: ["tag1"]} 74 | ev2 = %{create(:event) | tags: ["tag1", "tag2", "tag3"]} 75 | {:stateless, t} = FastTS.Stream.tagged_all("tag1") 76 | assert nil == t.(ev) 77 | assert ev1 == t.(ev1) 78 | assert ev2 == t.(ev2) 79 | {:stateless, t} = FastTS.Stream.tagged_all(["tag1", "tag3"]) 80 | assert nil == t.(ev) 81 | assert nil == t.(ev1) 82 | assert ev2 == t.(ev2) 83 | end 84 | 85 | test "tagged_any" do 86 | ev = %{create(:event) | tags: []} 87 | ev1 = %{create(:event) | tags: ["tag1"]} 88 | ev2 = %{create(:event) | tags: ["tag1", "tag2", "tag3"]} 89 | {:stateless, t} = FastTS.Stream.tagged_any("tag1") 90 | assert nil == t.(ev) 91 | assert ev1 == t.(ev1) 92 | assert ev2 == t.(ev2) 93 | {:stateless, t} = FastTS.Stream.tagged_any(["tag1", "tag3"]) 94 | assert nil == t.(ev) 95 | assert ev1 == t.(ev1) 96 | assert ev2 == t.(ev2) 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /test/fast_ts/stream/stateful_filters_test.exs: -------------------------------------------------------------------------------- 1 | #{:stateful, f} 2 | defmodule StatefulFiltersTest do 3 | use ExUnit.Case, async: true 4 | alias RiemannProto.Event 5 | import FastTSTestHelper 6 | 7 | doctest FastTS 8 | 9 | setup do 10 | {:ok, context} = FastTS.Stream.Context.start_link 11 | {:ok, context: context} 12 | end 13 | 14 | test "sreduce with initial value", %{context: context} do 15 | {:stateful, cf} = FastTS.Stream.sreduce(fn prev, %Event{metric_f: n} -> prev + n end, 0) 16 | f = cf.(context, self()) 17 | assert 1 == f.(%{create(:event) | metric_f: 1}) 18 | assert 5 == f.(%{create(:event) | metric_f: 4}) 19 | assert 5 == f.(%{create(:event) | metric_f: 0}) 20 | end 21 | 22 | test "sreduce producing events, no initial value", %{context: context} do 23 | {:stateful, cf} = FastTS.Stream.sreduce(fn %Event{metric_f: n1}, %Event{metric_f: n}=e -> %{e | metric_f: max(n1, n)} end) 24 | f = cf.(context, self()) 25 | ev = create(:event) 26 | assert nil == f.(%{ev | metric_f: 1}) #first event is used to start the accumulator if not provided, it is not propagated. Same as riemann 27 | assert %{ev | metric_f: 4} == f.(%{ev | metric_f: 4}) 28 | assert %{ev | metric_f: 4} == f.(%{ev | metric_f: 2}) 29 | assert %{ev | metric_f: 6} == f.(%{ev | metric_f: 6}) 30 | assert %{ev | metric_f: 6} == f.(%{ev | metric_f: 4}) 31 | end 32 | 33 | test "sreduce producing events, initial value", %{context: context} do 34 | {:stateful, cf} = FastTS.Stream.sreduce(fn %Event{metric_f: n1}, %Event{metric_f: n}=e -> 35 | %{e | metric_f: max(n1, n)} end, %{create(:event) | metric_f: 4}) 36 | f = cf.(context, self()) 37 | ev = create(:event) 38 | assert %{ev | metric_f: 4} == f.(%{ev | metric_f: 1}) 39 | assert %{ev | metric_f: 5} == f.(%{ev | metric_f: 5}) 40 | assert %{ev | metric_f: 5} == f.(%{ev | metric_f: 2}) 41 | assert %{ev | metric_f: 6} == f.(%{ev | metric_f: 6}) 42 | assert %{ev | metric_f: 6} == f.(%{ev | metric_f: 4}) 43 | end 44 | 45 | 46 | test "changed_state" , %{context: context} do 47 | {:stateful, cf} = FastTS.Stream.changed_state("ok") 48 | f = cf.(context, self()) 49 | ev1 = %{create(:event) | service: "s1"} 50 | ev2 = %{create(:event) | service: "s2"} 51 | assert nil == f.(%{ev1 | state: "ok"}) 52 | assert nil == f.(%{ev1 | state: "ok"}) 53 | assert %{ev1 | state: "down"} == f.(%{ev1 | state: "down"}) 54 | assert nil == f.(%{ev2 | state: "ok"}) 55 | assert nil == f.(%{ev1 | state: "down"}) 56 | assert %{ev2 | state: "down"} == f.(%{ev2 | state: "down"}) 57 | assert %{ev2 | state: "ok"} == f.(%{ev2 | state: "ok"}) 58 | assert nil == f.(%{ev2 | state: "ok"}) 59 | assert nil == f.(%{ev1 | state: "down"}) 60 | end 61 | 62 | test "general change detector" , %{context: context} do 63 | {:stateful, cf} = FastTS.Stream.changed(fn %Event{metric_f: n} -> n end, 0, fn _ -> :some end) #just group all in together 64 | f = cf.(context, self()) 65 | ev1 = %{create(:event) | service: "s1"} 66 | ev2 = %{create(:event) | service: "s2"} 67 | assert nil == f.(%{ev1 | metric_f: 0}) 68 | assert nil == f.(%{ev2 | metric_f: 0}) 69 | assert %{ev1 | metric_f: 1} == f.(%{ev1 | metric_f: 1}) 70 | assert nil == f.(%{ev2 | metric_f: 1}) 71 | assert %{ev2 | metric_f: 2} == f.(%{ev2 | metric_f: 2}) 72 | end 73 | 74 | 75 | test "consecutive runs", %{context: context} do 76 | {:stateful, cf} = FastTS.Stream.runs(3, fn %Event{state: s} -> s end) 77 | f = cf.(context, self()) 78 | ev = create(:event) 79 | assert nil == f.(%{ev | state: "ok"}) 80 | assert nil == f.(%{ev | state: "ok"}) 81 | assert %{ev | state: "ok"} == f.(%{ev | state: "ok"}) 82 | assert %{ev | state: "ok"} == f.(%{ev | state: "ok"}) 83 | assert nil == f.(%{ev | state: "down"}) 84 | assert nil == f.(%{ev | state: "ok"}) 85 | assert nil == f.(%{ev | state: "down"}) 86 | assert nil == f.(%{ev | state: "down"}) 87 | assert %{ev | state: "down"} == f.(%{ev | state: "down"}) 88 | assert %{ev | state: "down"} == f.(%{ev | state: "down"}) 89 | end 90 | 91 | #TODO: see how to mock time/timeouts 92 | 93 | end 94 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FastTS 2 | 3 | FastTS is a fast Time Series Event Stream Processor. 4 | 5 | It is designed with two major use cases in mind: 6 | 7 | - Cloud platforms: coordination and monitoring of complex and large server infrastructure, 8 | - Internet of Things: monitor and control large machine network. 9 | 10 | FastTS is designed to be highly versatile and scalable. 11 | It is a framework: the platform is configurable with a high-level 12 | Elixir-base Domain Specific Language. 13 | 14 | The project is inspired by ideas introduced by Phoenix Framework and 15 | Riemann. 16 | 17 | The current version design has one process per pipeline step and pass 18 | data down the pipeline through message passing. Riemann use function 19 | calls between pipeline steps. We may offer such a option in a next 20 | version. 21 | 22 | ## Usage 23 | 24 | ### Prerequite 25 | 26 | Erlang R18+ and Elixir 1.1+. 27 | 28 | ### Building FastTS 29 | 30 | Here is how to build, prepare a first metrics routing script and deploy the tool: 31 | 32 | You can first checkout FastTS from Github: 33 | 34 | git clone https://github.com/processone/fast_ts.git 35 | 36 | Download dependencies and compile the project: 37 | 38 | mix deps.get 39 | MIX_ENV=prod mix compile 40 | 41 | You can then tweak the file in config directory. Most notably you can 42 | tweak the example time series route example file: `config/route.exs` 43 | 44 | You can then start the project with console attached with: 45 | 46 | iex -S mix 47 | 48 | ### Injecting metrics with Riemann client 49 | 50 | You can test injecting data with the Riemann protocol. 51 | 52 | You can for example try [Riemann Python client](https://github.com/borntyping/python-riemann-client): 53 | 54 | riemann-client send -h localhost -s web -t latency -t dev -m 120 55 | 56 | In that example: 57 | - host (-h) is `localhost` 58 | - service (-s) is `web` 59 | - tags (-t) are `latency` and `dev` 60 | - metric (-m) value is 120 61 | 62 | If you are using the default example route script, you should see in 63 | log file events printed to STDOUT: 64 | 65 | %RiemannProto.Event{attributes: [], description: nil, host: "localhost", metric_d: nil, metric_f: 4.0, metric_sint64: nil, service: "web", state: nil, tags: ["latency", "dev"], time: 1452510805, ttl: nil} 66 | 67 | ## Release / deployment 68 | 69 | For deployment, you can then prepare a release directory: 70 | 71 | MIX_ENV=prod mix release 72 | 73 | You can then test your release locally. Go to release dir: 74 | 75 | cd rel/fast_ts 76 | 77 | You can create a directory to store your route scripts: 78 | 79 | mkdir routes 80 | 81 | and then create your own time series route file: `routes/route.exs` 82 | 83 | When done you can start your release, for example with console attached: 84 | 85 | FTS_ROUTE_DIR=routes bin/fast_ts console 86 | 87 | `FTS_ROUTE_DIR` environment variable allows you to configure route 88 | scripts directory. 89 | 90 | For deploy you can simply compress the whole rel/ directory and 91 | uncompress it on server. It contains all needed code, including Erlang 92 | VM and Elixir environment. 93 | 94 | ## Build Docker container 95 | 96 | The following commands will prepare a container to run FastTS: 97 | 98 | mix deps.get 99 | MIX_ENV=prod mix compile 100 | MIX_ENV=prod mix release 101 | docker build -t fast_ts . 102 | 103 | You can then run FastTS container on a Docker host with the default 104 | with the default embedded FastTS route file: 105 | 106 | $ docker run --rm -p 5555:5555 fast_ts 107 | Using /fast_ts/releases/0.0.1/fast_ts.sh 108 | created directory: '/fast_ts/running-config' 109 | Exec: /usr/lib/erlang/erts-7.1/bin/erlexec -noshell -noinput +Bd -boot /fast_ts/releases/0.0.1/fast_ts -mode embedded -config /fast_ts/running-config/sys.config -boot_var ERTS_LIB_DIR /usr/lib/erlang/erts-7.1/../lib -env ERL_LIBS /fast_ts/lib -args_file /fast_ts/running-config/vm.args -- foreground 110 | Root: /fast_ts 111 | 112 | 18:18:22.875 [info] Ignoring empty pipeline 'Empty pipeline are ignored' 113 | 114 | 18:18:22.879 [info] Registering Router module: HelloFast.Router 115 | 116 | 18:18:22.886 [info] Accepting connections on port 5555 117 | 118 | If you want to start FastTS on Docker with your own route scripts, you 119 | can mount a route directory volume on your Docker host and pass it as 120 | `FTS_ROUTE_DIR` environment variable: 121 | 122 | docker run -v "$PWD/config/docker":/opt/routes -e "FTS_ROUTE_DIR=/opt/routes" --rm -p 5555:5555 fast_ts 123 | 124 | Note: The previous command assume that you are using it from docker 125 | host or that your local Docker Machine as the proper directory 126 | mounted. This is the case for example with Docker Machine on OSX, that 127 | grant access to Docker host to everything under `/Users`. That's why 128 | the previous command should work as is. 129 | 130 | ## Using the test client with your Docker container 131 | 132 | You need IP of your docker machine. If you do not have it, you can get 133 | environment of your docker-machine with: 134 | 135 | $ docker-machine env default 136 | export DOCKER_TLS_VERIFY="1" 137 | export DOCKER_HOST="tcp://192.168.99.100:2376" 138 | export DOCKER_CERT_PATH="/Users/mremond/.docker/machine/machines/default" 139 | export DOCKER_MACHINE_NAME="default" 140 | # Run this command to configure your shell: 141 | # eval "$(docker-machine env default)" 142 | 143 | You can this pass the IP addresse of the Docker host (or hostname if 144 | you have any set up) with `riemann-client -H` option: 145 | 146 | 147 | $ riemann-client -H 192.168.99.100 send -h localhost -s web -t latency -t dev -m 120 148 | { 149 | "host": "localhost", 150 | "metric_f": 120.0, 151 | "service": "web", 152 | "tags": [ 153 | "latency", 154 | "dev" 155 | ] 156 | } 157 | 158 | You should see the following log entry in your attached FastTS Docker container: 159 | 160 | %RiemannProto.Event{attributes: [], description: nil, host: "localhost", metric_d: nil, metric_f: 24.0, metric_sint64: nil, service: "web", state: nil, tags: ["latency", "dev"], time: 1452535525, ttl: nil} 161 | 162 | 163 | ## Embbed FastTS in your own app 164 | 165 | Embedding FastTS into your app will allow you to have your own metrics 166 | / alert dispatcher included into your system. 167 | 168 | 1. Add fast_ts to your list of dependencies in `mix.exs`: 169 | 170 | def deps do 171 | [{:fast_ts, github: "processone/fast_ts"}] 172 | end 173 | 174 | 2. Ensure fast_ts is started before your application: 175 | 176 | def application do 177 | [applications: [:fast_ts]] 178 | end 179 | 180 | ## Defining your metrics router 181 | 182 | Here is an example file showing a basic FastTS route script: 183 | 184 | defmodule HelloFast.Router do 185 | use FastTS.Router 186 | 187 | pipeline "Basic pipeline" do 188 | # We only take functions under a given value 189 | under(12) 190 | stdout 191 | end 192 | 193 | pipeline "Second pipeline" do 194 | # Buffer event for 5 second and calcule the rate of accumulate events per second 195 | rate(5) 196 | stdout 197 | end 198 | 199 | pipeline "Empty pipeline are ignored" do 200 | end 201 | 202 | end 203 | 204 | 205 | -------------------------------------------------------------------------------- /lib/fast_ts/stream.ex: -------------------------------------------------------------------------------- 1 | ## These are the components for the stream library 2 | defmodule FastTS.Stream do 3 | alias RiemannProto.Event 4 | 5 | # TODO: I think the block need to be able to receive the module 6 | # the pipeline is living in to read module attributes, to pass or 7 | # overload special configuration informations 8 | 9 | # == Output functions == 10 | 11 | @doc """ 12 | Prints event struct on stdout 13 | """ 14 | def stdout, do: {:stateless, &do_stdout/1} 15 | def do_stdout(event) do 16 | IO.puts "#{inspect event}" 17 | event 18 | end 19 | 20 | @doc """ 21 | Sends an email. Mailman module is used to send email. 22 | You can define email sending parameters in `config.exs` file as follow: 23 | 24 | # Configure MailMan to be able to send email 25 | config :mailman, 26 | relay: "localhost", 27 | port: 1025, 28 | auth: :never 29 | """ 30 | def email(to_address), do: {:stateless, &(do_email(&1, to_address))} 31 | def do_email(event = %Event{metric_f: _metric}, to_address) do 32 | # Compose email 33 | email = %Mailman.Email{ 34 | subject: "#{event.host} #{event.service} #{event.state}", 35 | from: "no-reply@process-one.net", 36 | to: [ to_address ], 37 | # TODO show "No Description" if there is no description 38 | text: "#{event.host} #{event.service} #{event.state} (#{event.metric_f})\nat #{Common.epoch_to_string(event.time)}\n\n#{event.description}" 39 | } 40 | Mailman.deliver(email, %Mailman.Context{}) 41 | event 42 | end 43 | 44 | # == Filter functions == 45 | 46 | @doc """ 47 | Passes on events only when their metric is smaller than x 48 | """ 49 | def under(value), do: {:stateless, &(do_under(&1, value))} 50 | def do_under(event = %Event{metric_f: metric}, value) do 51 | cond do 52 | metric < value -> 53 | event 54 | true -> 55 | nil 56 | end 57 | end 58 | 59 | @doc """ 60 | Passes on events only when their metric is greater than x 61 | """ 62 | def over(value), do: {:stateless, &(do_over(&1, value))} 63 | def do_over(event = %Event{metric_f: metric}, value) do 64 | cond do 65 | metric > value -> 66 | event 67 | true -> 68 | nil 69 | end 70 | end 71 | 72 | @doc """ 73 | Scale metrics by the given factor 74 | """ 75 | def scale(factor), do: {:stateless, &(do_scale(&1, factor))} 76 | def do_scale(event = %Event{metric_f: metric}, factor), do: %{event | metric_f: metric * factor} 77 | 78 | 79 | @doc """ 80 | Generic filtering. Filter events based on given function. 81 | The filtering function should no have side effects and must return true|false or fail with FunctionClauseError 82 | """ 83 | def filter(f), do: {:stateless, &(do_filter(&1, f))} 84 | def do_filter(event = %Event{}, f) do 85 | try do 86 | if f.(event) do 87 | event 88 | else 89 | nil 90 | end 91 | rescue 92 | FunctionClauseError -> nil 93 | # treat these as false. Allows easy filtering like fn %Event{service :"some"} -> true end 94 | # without having to provide the false clause 95 | end 96 | 97 | end 98 | 99 | @doc """ 100 | Generic map. Map (project) events using the given map function. 101 | The map function should no have side effects 102 | """ 103 | def map(f), do: {:stateless, &(do_map(&1, f))} 104 | def do_map(event = %Event{}, f), do: f.(event) 105 | 106 | 107 | def tag(tag), do: {:stateless, &do_tag(&1, (if is_list(tag), do: tag , else: [tag] ))} 108 | def do_tag(event = %Event{tags: tags}, new_tags) do 109 | %{event | tags: Enum.uniq(Enum.concat(tags, new_tags))} 110 | end 111 | 112 | def tagged_all(tag), do: {:stateless, &do_tagged_all(&1, (if is_list(tag), do: tag , else: [tag] ))} 113 | def do_tagged_all(event = %Event{tags: tagged}, tags) do 114 | if Enum.all?(tags, fn t -> Enum.member? tagged, t end), do: event, else: nil 115 | end 116 | 117 | def tagged_any(tag), do: {:stateless, &do_tagged_any(&1, (if is_list(tag), do: tag , else: [tag] ))} 118 | def do_tagged_any(event = %Event{tags: tagged}, tags) do 119 | if Enum.any?(tags, fn t -> Enum.member? tagged, t end), do: event, else: nil 120 | end 121 | 122 | 123 | # == Statefull processing functions == 124 | def sreduce(f), do: sreduce(f, :first_event) 125 | def sreduce(f, init), do: {:stateful, &(sreduce(&1, &2, f, init))} 126 | def sreduce(context, pid, f, init) do 127 | fn ev -> 128 | {new,output} = case {FastTS.Stream.Context.get(context, :sreduce), init} do 129 | {nil, :first_event} -> 130 | {f.(ev, ev), nil} 131 | {nil, val} -> 132 | r = f.(val, ev) 133 | {r, r} 134 | {prev, _} -> 135 | r = f.(prev, ev) 136 | {r, r} 137 | end 138 | FastTS.Stream.Context.put(context, :sreduce, new) 139 | output 140 | end 141 | end 142 | 143 | def throttle(n, secs), do: {:stateful, &(throttle(&1, &2, n, secs))} 144 | def throttle(context, pid, n, secs) do 145 | fn(ev) -> 146 | now = System.system_time(:seconds) 147 | case FastTS.Stream.Context.get(context, :throttle) do 148 | {start, count} when (now - start <= secs) and count >= n -> 149 | nil #these ones are to be discarded 150 | {start, count} when (now - start <= secs) and count < n -> 151 | FastTS.Stream.Context.put(context, :throttle, {start, count + 1}) 152 | ev 153 | _ -> 154 | # first time, or previous window already expired. Create new window. 155 | FastTS.Stream.Context.put(context, :throttle, {now, 1}) 156 | ev 157 | end 158 | end 159 | end 160 | 161 | @doc """ 162 | Stable output. Output stable events. It is defined as stable if for all events e_1,e_2,e_n 163 | in n sec, f(e_1) = f(e_2) = f(e_3) 164 | """ 165 | def stable(interval, f), do: {:stateful, &(stable(&1, &2, interval, f))} 166 | def stable(context, pid, interval, f) do 167 | FastTS.Stream.Context.put(context, :last, :undefined) 168 | fn(ev) -> 169 | now = System.system_time(:seconds) 170 | current = f.(ev) 171 | case FastTS.Stream.Context.get(context, :last) do 172 | :undefined -> 173 | FastTS.Stream.Context.put(context, :last, current) 174 | FastTS.Stream.Context.put(context, :last_ts, now) 175 | ^current -> 176 | if now - FastTS.Stream.Context.get(context, :last_ts) >= interval do 177 | ev 178 | else 179 | nil 180 | end 181 | other -> 182 | FastTS.Stream.Context.put(context, :last, current) 183 | FastTS.Stream.Context.put(context, :last_ts, now) 184 | nil 185 | end 186 | end 187 | end 188 | 189 | @doc """ 190 | Generic change detection. 191 | Detect changes in f(ev) for ev grouped in g(ev). 192 | Events that remains stable aren't propagated. 193 | """ 194 | def changed(f,init, g), do: {:stateful, &(changed(&1, &2, f, init, g))} 195 | def changed(context, pid, f, init, g) do 196 | fn(ev) -> 197 | group = g.(ev) 198 | current = f.(ev) 199 | prev = FastTS.Stream.Context.get(context, group) 200 | prev = if is_nil(prev), do: init, else: prev 201 | if prev == current do 202 | nil 203 | else 204 | FastTS.Stream.Context.put(context, group, current) 205 | ev 206 | end 207 | end 208 | end 209 | 210 | @doc """ 211 | Detect changes in state grouped by {host,service} pairs 212 | For a more general change detector use changed/3 instead 213 | """ 214 | def changed_state(init) do 215 | f = fn %Event{state: state} -> state end 216 | g = fn %Event{host: host, service: service} -> {host, service} end 217 | {:stateful, &(changed(&1, &2, f, init, g))} 218 | end 219 | 220 | @doc """ 221 | n events must have the same computed value. Pass the last one of the n 222 | """ 223 | def runs(n, f), do: {:stateful, &(runs(&1, &2, n, f))} 224 | def runs(context, pid, n, f) do 225 | fn(ev) -> 226 | current = f.(ev) 227 | runs = FastTS.Stream.Context.get(context, :runs) 228 | case runs do 229 | {^current, l} when l >= n -> 230 | ev 231 | {^current, l} -> 232 | FastTS.Stream.Context.put(context, :runs, {current, l + 1}) 233 | nil 234 | _ -> 235 | # This is the first one detected, next o one is expected to be number 2 236 | FastTS.Stream.Context.put(context, :runs, {current, 2}) 237 | nil 238 | end 239 | end 240 | end 241 | 242 | @doc """ 243 | Calculate rate of a given event per second, assuming metric is an occurence count 244 | 245 | Bufferize events for N second interval and divide total count by interval in second. 246 | On interval tick: 247 | - passes the last event down the pipe, with metric remplaced with rate per second. Note that if event types are 248 | not homogeneous they would need to be filter / sorted properly before. 249 | - Send a nil event down the pipe so that other pipe element can decide to generate one to fill the void 250 | """ 251 | def rate(interval), do: {:stateful, &(rate(&1, &2, interval))} 252 | def rate(context, pid, interval) do 253 | partition_time(context, interval, 254 | # Create : 255 | fn -> 256 | # TODO add wrapper around ets table operations: pass a list of key values (data), receive a list of key values (data) 257 | FastTS.Stream.Context.put(context, :count, 0) 258 | FastTS.Stream.Context.put(context, :state, nil) 259 | end, 260 | # Add event and defer passing to next pipeline stage : 261 | fn(event = %Event{metric_f: metric}) -> 262 | count = FastTS.Stream.Context.get(context, :count) 263 | FastTS.Stream.Context.put(context, :count, count + (metric||0)) 264 | FastTS.Stream.Context.put(context, :state, event) 265 | :defer 266 | end, 267 | # Finish = On interval, reset state event to next pipeline stage : 268 | fn(dataMap, startTS, endTS) -> 269 | case dataMap[:state] do 270 | nil -> # If no event were send during interval, do nothing 271 | FastTS.Stream.Pipeline.next(nil, pid) 272 | event -> # Otherwise calculate rate, replace metric with that value and pass last event down the pipeline 273 | count = dataMap[:count] 274 | rate = Float.round(count / (endTS - startTS), 3) 275 | FastTS.Stream.Pipeline.next(%{event | time: endTS, metric_f: rate}, pid) 276 | end 277 | end) 278 | end 279 | 280 | # TODO: when we want to stop stream, we need to stop timer 281 | defp partition_time(context, interval, create, add, finish) do 282 | create.() 283 | startTS = System.system_time(:seconds) 284 | :timer.apply_interval(interval * 1000, Kernel, :apply, [ __MODULE__, :switch_partition, [context, startTS, create, finish]]) 285 | add 286 | end 287 | 288 | # TODO: test against race condition between state reset and message that could arrive in-between 289 | # We can probably add a first clause in receive to reset state in the event receiving loop 290 | def switch_partition(context, startTS, create, finish) do 291 | dataMap = FastTS.Stream.Context.get_all(context) 292 | create.() 293 | endTS = System.system_time(:seconds) 294 | finish.(dataMap, startTS, endTS) 295 | end 296 | end 297 | 298 | # function: 299 | # info -> log object to file 300 | # Elixir: Several log levels 301 | 302 | # Event.time is unix time in seconds 303 | --------------------------------------------------------------------------------