├── .dockerignore ├── .editorconfig ├── .gitignore ├── .travis.yml ├── Dockerfile ├── README.md ├── config ├── banner.txt ├── config.exs └── travis.exs ├── docker-compose.yml ├── lib ├── config │ └── handler.ex ├── data │ ├── cassandra.ex │ ├── ecto.ex │ ├── repos │ │ └── test.ex │ └── schemas │ │ └── test.ex ├── demo.ex ├── main.ex ├── mix │ └── tasks │ │ └── docker.ex ├── ms │ ├── encoders │ │ └── poison.ex │ ├── log.ex │ ├── misc │ │ └── banner_gen.ex │ ├── plugs │ │ └── access_log.ex │ ├── registry.ex │ ├── request │ │ └── request.ex │ └── structs │ │ ├── access_log_msg.ex │ │ └── log_msg.ex ├── router │ ├── admin_router.ex │ ├── ecto_router.ex │ ├── metrics_router.ex │ └── router.ex ├── streams │ ├── kafka_consumer.ex │ └── redis.ex ├── structs │ └── res_dto.ex └── supervisor.ex ├── mix.exs ├── mix.lock ├── priv └── test │ └── migrations │ ├── 20170503150023_add_test_table.exs │ └── 20170508135901_add_test_change.exs ├── test ├── ms │ └── plugs │ │ └── access_log_test.exs └── test_helper.exs └── tools ├── compile-dependencies.sh ├── docker.sh ├── drun.sh ├── get-dependencies.sh ├── interactive.sh ├── lint.sh ├── run.sh └── static.sh /.dockerignore: -------------------------------------------------------------------------------- 1 | _build 2 | .idea 3 | deps -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | end_of_line = lf 10 | indent_style = space 11 | indent_size = 4 12 | quote_type = double 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | _build 3 | deps 4 | elixir-test.iml 5 | *.iml -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: elixir 2 | elixir: 3 | - 1.8.1 4 | services: 5 | - postgresql 6 | - redis 7 | env: 8 | global: 9 | - MIX_ENV=travis 10 | before_script: 11 | - mix ecto.create 12 | - mix ecto.migrate 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM msaraiva/elixir-dev 2 | LABEL authors="Chris Fröhlingsdorf " 3 | 4 | RUN apk update && apk add --no-cache bash 5 | 6 | WORKDIR /var/ex-test 7 | ADD . . 8 | 9 | RUN erl -version && elixir -v 10 | 11 | RUN mix local.hex --force && \ 12 | mix local.rebar --force 13 | 14 | RUN mix deps.get --force && \ 15 | mix clean --force && \ 16 | mix compile --force 17 | 18 | CMD ./tools/drun.sh 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Elixir-Logo](http://elixir-lang.org/images/logo/logo.png) 2 | 3 | # elixir-ms 4 | - elixir microservice base :ant::zap: 5 | - microservice skeleton from scratch aka "You dont need Phoenix" 6 | 7 | [![Build Status](https://travis-ci.org/krystianity/elixir-ms.svg?branch=master)](https://travis-ci.org/krystianity/elixir-ms) 8 | 9 | ## Ingredients :star: 10 | - `credo` for linting 11 | - `dialyxir` static analysis 12 | - strong `http server` basis using plug + cowboy 13 | - fast `json` encode/decode using poison 14 | - `http client` using httpoison 15 | - `metrics` using prometheus_ex 16 | - `healthchecks + config` setup using mix tools 17 | - (json) `access log` provided as Plug 18 | - ELK compliant `logger` with simple API 19 | - registry (in-memory cache) with simple API 20 | - `redis client` using redix (pub-sub) 21 | - `JIT Config` via weave 22 | - `RDBMS ORM` via ecto and postgrex 23 | - start-up banner 24 | - MIT License 25 | 26 | ## Requirements 27 | - Erlang/OTP >= 21 28 | - Elixir >= 1.8.1 29 | 30 | ## Installation 31 | - install Erlang 32 | - install Elixir 33 | - install Mix 34 | - http://elixir-lang.org/install.html#unix-and-unix-like // on mac simply `brew install elixir` 35 | - `git clone git@github.com:krystianity/elixir-ms.git` 36 | - run `mix deps.get` or `./tools/get-dependencies.sh` 37 | - start via `mix start` 38 | - (if you want to run this as is, you need a local postgres (see Other below) and 39 | a redis, otherwise you have to make adjustments to lib/demo.ex and mix.exs) 40 | 41 | ## Use as docker container via docker-compose 42 | build + run via `docker-compose up --build` 43 | 44 | ## Testing 45 | run `mix test` 46 | 47 | ## License 48 | MIT 49 | 50 | ## Other 51 | 52 | ### Database Setup 53 | - (requires a local postgres, with a user named "postgres" and a password "postgres" 54 | checkout config/config.exs to change these credentials 55 | - `mix ecto.create` 56 | - `mix ecto.migrate` 57 | 58 | ### Other Database Stuff 59 | - `mix ecto.gen.migration add_test_table -r ExTest.Repos.Test` 60 | - `mix ecto.rollback` 61 | - `docker run -it --rm --link postgres:postgres postgres:9.3 psql -h postgres -U postgres ex_test` 62 | -------------------------------------------------------------------------------- /config/banner.txt: -------------------------------------------------------------------------------- 1 | ,----. .=-.-. ,-.--, .=-.-. 2 | ,-.--` , \ _.-. /==/_ /.--.-. /=/, .'/==/_ /.-.,.---. 3 | |==|- _.-` .-,.'| |==|, | \==\ -\/=/- / |==|, |/==/ ` \ 4 | |==| `.-.|==|, | |==| | \==\ `-' ,/ |==| |==|-, .=., | 5 | /==/_ , /|==|- | |==|- | |==|, - | |==|- |==| '=' / 6 | |==| .-' |==|, | |==| ,| /==/ , \ |==| ,|==|- , .' 7 | |==|_ ,`-._|==|- `-._|==|- | /==/, .--, - \|==|- |==|_ . ,'. 8 | /==/ , //==/ - , ,/==/. / \==\- \/=/ , //==/. /==/ /\ , ) 9 | `--`-----`` `--`-----'`--`-` `--`-' `--` `--`-``--`-`--`--' 10 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :ex_test, 4 | 5 | port: 8080, 6 | 7 | log_level: "info", 8 | service_name: "ex-test", 9 | json_enabled: false, 10 | 11 | kafka_topic: "access-log", 12 | 13 | cassandra_host: "127.0.0.1", 14 | cassandra_port: 9042, 15 | 16 | redis_host: "127.0.0.1", 17 | redis_port: 6379 18 | 19 | config :kafka_ex, 20 | 21 | brokers: [{"127.0.0.1", 9092}], 22 | consumer_group: "kafka_ex", 23 | sync_timeout: 3000, 24 | max_restarts: 10, 25 | max_seconds: 60, 26 | use_ssl: false 27 | 28 | config :kafka_consumer, 29 | 30 | default_pool_size: 5, 31 | default_pool_max_overflow: 10, 32 | event_handlers: [ 33 | #{ 34 | #ExTest.KafkaConsumer, 35 | #[ 36 | # {"access-log", 0} 37 | #], 38 | #size: 5, 39 | #max_overflow: 5 40 | #} 41 | ] 42 | 43 | config :weave, 44 | #file_directory: "path/to/secrets", 45 | environment_prefix: "EXMS_", 46 | handler: Config.Handler 47 | 48 | config :ex_test, ecto_repos: [ExTest.Repos.Test] 49 | config :ex_test, ExTest.Repos.Test, 50 | adapter: Ecto.Adapters.Postgres, 51 | database: "ex_test", 52 | username: "postgres", 53 | password: "postgres" 54 | 55 | env_config = Path.expand("#{Mix.env}.exs", __DIR__) 56 | if File.exists?(env_config) do 57 | import_config(env_config) 58 | end 59 | -------------------------------------------------------------------------------- /config/travis.exs: -------------------------------------------------------------------------------- 1 | use Mix.Config 2 | 3 | config :ex_test, ExTest.Repos.Test, 4 | adapter: Ecto.Adapters.Postgres, 5 | database: "ex_test", 6 | username: "postgres", 7 | password: "" 8 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2' 2 | services: 3 | ex-test: 4 | build: . 5 | ports: 6 | - "8080:8080" 7 | -------------------------------------------------------------------------------- /lib/config/handler.ex: -------------------------------------------------------------------------------- 1 | defmodule Config.Handler do 2 | @moduledoc false 3 | 4 | def handle_configuration(key, value) do 5 | {:ex_test, String.to_atom(key), value} 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/data/cassandra.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Cassandra do 2 | use GenServer 3 | alias MSBase.Registry 4 | 5 | @process_key "cassandra-genserver" 6 | 7 | def start(_) do 8 | {:ok, pid} = GenServer.start_link(__MODULE__, nil) 9 | Registry.set(@process_key, pid) 10 | end 11 | 12 | def init(_init_arg) do 13 | {:ok, conn} = Xandra.start_link( 14 | host: Application.get_env(:ex_test, :cassandra_host), 15 | port: Application.get_env(:ex_test, :cassandra_port) 16 | ) 17 | 18 | Registry.set(@process_key, conn) 19 | {:ok, conn} 20 | end 21 | 22 | def void_query(statement, params) do 23 | {:ok, pid} = Registry.get(@process_key) 24 | GenServer.cast(pid, {:void_query, statement, params}) 25 | end 26 | 27 | def list_query(statement, params) do 28 | {:ok, pid} = Registry.get(@process_key) 29 | GenServer.call(pid, {:list_query, statement, params}) 30 | end 31 | 32 | def prepare(statement) do 33 | {:ok, pid} = Registry.get(@process_key) 34 | GenServer.call(pid, {:prepare, statement}) 35 | end 36 | 37 | def handle_cast({:void_query, statement, params}, state) do 38 | {:ok, conn} = Registry.get "cassandra-conn" 39 | {:ok, %Xandra.Void{}} = Xandra.execute(conn, statement, params) 40 | {:noreply, state} 41 | end 42 | 43 | def handle_call({:list_query, statement, params}, _, state) do 44 | {:ok, conn} = Registry.get "cassandra-conn" 45 | {:ok, %Xandra.Page{} = page} = Xandra.execute(conn, statement, params) 46 | {:reply, page, state} 47 | end 48 | 49 | def handle_call({:prepare, statement}, _, state) do 50 | {:ok, conn} = Registry.get "cassandra-conn" 51 | {:ok, prepared} = Xandra.prepare(conn, statement) 52 | {:reply, prepared, state} 53 | end 54 | 55 | end 56 | -------------------------------------------------------------------------------- /lib/data/ecto.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Ecto do 2 | import Ecto.Query 3 | 4 | def keyword_query do 5 | query = from w in ExTest.Schemas.Test, 6 | where: w.id > 0 or is_nil(w.bla), 7 | select: w 8 | res = ExTest.Repos.Test.all(query) 9 | IO.inspect res 10 | res 11 | end 12 | 13 | def pipe_query do 14 | ExTest.Schemas.Test 15 | |> where(id: 1) 16 | |> order_by(:id) 17 | |> limit(1) 18 | |> ExTest.Repos.Test.all 19 | |> IO.inspect 20 | end 21 | 22 | def any_query do 23 | query = from row in ExTest.Schemas.Test, 24 | where: not is_nil(row.id), 25 | select: row 26 | ExTest.Repos.Test.all(query) 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/data/repos/test.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Repos.Test do 2 | use Ecto.Repo, otp_app: :ex_test 3 | end 4 | -------------------------------------------------------------------------------- /lib/data/schemas/test.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Schemas.Test do 2 | use Ecto.Schema 3 | 4 | schema "test" do 5 | #field :id, :integer is always set by ecto 6 | field :bla, :string 7 | field :id_again, :integer 8 | field :blup, :float, default: 0.0 9 | field :mach_doch, :integer 10 | timestamps() 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/demo.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Demo do 2 | 3 | alias MSBase.Log 4 | alias ExTest.Redis 5 | # alias ExTest.Cassandra 6 | # alias ExTest.KafkaConsumer 7 | alias ExTest.Ecto 8 | alias MsBase.Request 9 | 10 | def run() do 11 | 12 | MsBase.BannerGen.read_banner() 13 | 14 | test_jit_config() 15 | run_logger() 16 | run_redis() 17 | #run_kafka() 18 | #run_cassandra() 19 | run_ecto() 20 | run_uuid() 21 | run_http_client() 22 | 23 | :ok 24 | end 25 | 26 | defp run_logger() do 27 | 28 | log_level = Application.get_env(:ex_test, :log_level) 29 | service_name = Application.get_env(:ex_test, :service_name) 30 | json_enabled = Application.get_env(:ex_test, :json_enabled) 31 | 32 | Log.init(log_level, service_name, json_enabled) 33 | Log.info "Application ready." 34 | end 35 | 36 | defp test_jit_config() do 37 | # start with EXMS_TEST=123 mix run --no-halt # to pass env variables to weave 38 | IO.inspect Application.get_env(:ex_test, :TEST) 39 | end 40 | 41 | defp run_redis() do 42 | channel = "ex-test-channel" 43 | Redis.start_link(channel) 44 | end 45 | 46 | # defp run_cassandra() do 47 | # Cassandra.init(nil) 48 | # end 49 | 50 | # defp run_kafka() do 51 | # KafkaConsumer.get_partition_config("access-log") 52 | # :ok 53 | # end 54 | 55 | defp run_ecto() do 56 | Ecto.keyword_query() 57 | Ecto.pipe_query() 58 | end 59 | 60 | defp run_uuid() do 61 | UUID.uuid4() |> IO.puts 62 | end 63 | 64 | defp run_http_client() do 65 | Request.start 66 | IO.inspect Request.get!("/users/krystianity").body[:public_repos] 67 | end 68 | 69 | end 70 | -------------------------------------------------------------------------------- /lib/main.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest do 2 | use Application 3 | 4 | # see mix.exs (defp applications) 5 | def start(_type, _args) do 6 | 7 | Weave.Loaders.Environment.load_configuration() 8 | link_res = ExTest.Supervisor.start_link() 9 | 10 | ExTest.Demo.run() 11 | 12 | link_res 13 | end 14 | 15 | end 16 | -------------------------------------------------------------------------------- /lib/mix/tasks/docker.ex: -------------------------------------------------------------------------------- 1 | defmodule Mix.Tasks.Docker do 2 | use Mix.Task 3 | 4 | def run(_) do 5 | Mix.Tasks.Cmd.run(~w(docker build -t ex-test:latest .)) 6 | Mix.Tasks.Cmd.run(~w(docker run -i --rm --name ex-test -p 8080:8080 ex-test:latest)) 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /lib/ms/encoders/poison.ex: -------------------------------------------------------------------------------- 1 | defimpl Poison.Encoder, for: Any do 2 | # this encoder is required when trying to json stringify 3 | # ecto query results, as they may contain private data 4 | # that will throw exceptions per default 5 | 6 | def encode(%{__struct__: _} = struct, options) do 7 | map = struct 8 | |> Map.from_struct 9 | |> sanitize_map 10 | Poison.Encoder.Map.encode(map, options) 11 | end 12 | 13 | defp sanitize_map(map) do 14 | Map.drop(map, [:__meta__, :__struct__]) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /lib/ms/log.ex: -------------------------------------------------------------------------------- 1 | defmodule MSBase.Log do 2 | 3 | @moduledoc """ 4 | Log 5 | """ 6 | 7 | alias MSBase.Registry 8 | alias MSBase.AccessLog 9 | 10 | def init(level, service_name, json_enabled) do 11 | 12 | {:ok, hostname} = :inet.gethostname() 13 | color = System.get_env("SERVICE_COLOR") 14 | pid = System.get_pid() 15 | 16 | Registry.set("log-options", %{ 17 | level: level, 18 | service: service_name, 19 | host: List.to_string(hostname), 20 | pid: pid, 21 | color: color, 22 | json: json_enabled 23 | }) 24 | 25 | :ok 26 | end 27 | 28 | defp get_options() do 29 | {:ok, options} = Registry.get("log-options") 30 | options 31 | end 32 | 33 | defp get_value_for_level(level) do 34 | case level do 35 | "TRACE" -> 36 | 10 37 | "DEBUG" -> 38 | 20 39 | "INFO" -> 40 | 30 41 | "WARN" -> 42 | 40 43 | "ERROR" -> 44 | 50 45 | "FATAL" -> 46 | 60 47 | _ -> 48 | 0 49 | end 50 | end 51 | 52 | defp get_iso_time do 53 | DateTime.to_iso8601 DateTime.utc_now() 54 | end 55 | 56 | defp get_json_log_msg(msg, level, options, corr_id) do 57 | 58 | {:ok, string} = Poison.encode(%MSBase.LogMsg{ 59 | msg: msg, 60 | loglevel: level, 61 | loglevel_value: get_value_for_level(level), 62 | "@timestamp": get_iso_time(), 63 | pid: options.pid, 64 | current_color: options.color, 65 | service: options.service, 66 | host: options.host, 67 | "correlation-id": corr_id, 68 | beam_pid: "#{:erlang.pid_to_list(self())}", 69 | node: "#{node()}" 70 | }) 71 | 72 | string 73 | end 74 | 75 | defp get_plain_log_msg(msg, level) do 76 | {:ok, string} = Poison.encode(msg) 77 | Enum.join([ 78 | get_iso_time(), 79 | "-", 80 | level <> ":", 81 | string 82 | ], " ") 83 | end 84 | 85 | defp write_log(msg, level, color) when is_binary(level) do 86 | options = get_options() 87 | 88 | string = if options.json do 89 | get_json_log_msg(msg, level, options, nil) 90 | else 91 | get_plain_log_msg(msg, level) 92 | end 93 | 94 | [color, string] 95 | |> Bunt.puts 96 | 97 | :ok 98 | end 99 | 100 | defp write_log(msg, level, color, conn) when is_binary(level) do 101 | options = get_options() 102 | 103 | corr_id = AccessLog.get_correlation_id(conn) 104 | 105 | string = if options.json do 106 | get_json_log_msg(msg, level, options, corr_id) 107 | else 108 | get_plain_log_msg(msg, level) 109 | end 110 | 111 | [color, string] 112 | |> Bunt.puts 113 | 114 | :ok 115 | end 116 | 117 | def trace(msg) do 118 | write_log(msg, "TRACE", :white) 119 | end 120 | 121 | def debug(msg) do 122 | write_log(msg, "DEBUG", :aqua) 123 | end 124 | 125 | def info(msg) do 126 | write_log(msg, "INFO", :green) 127 | end 128 | 129 | def warn(msg) do 130 | write_log(msg, "WARN", :yellow) 131 | end 132 | 133 | def error(msg) do 134 | write_log(msg, "ERROR", :red) 135 | end 136 | 137 | def fatal(msg) do 138 | write_log(msg, "FATAL", :darkmagenta) 139 | end 140 | 141 | def trace(msg, conn) do 142 | write_log(msg, "TRACE", :white, conn) 143 | end 144 | 145 | def debug(msg, conn) do 146 | write_log(msg, "DEBUG", :aqua, conn) 147 | end 148 | 149 | def info(msg, conn) do 150 | write_log(msg, "INFO", :green, conn) 151 | end 152 | 153 | def warn(msg, conn) do 154 | write_log(msg, "WARN", :yellow, conn) 155 | end 156 | 157 | def error(msg, conn) do 158 | write_log(msg, "ERROR", :red, conn) 159 | end 160 | 161 | def fatal(msg, conn) do 162 | write_log(msg, "FATAL", :darkmagenta, conn) 163 | end 164 | 165 | end 166 | -------------------------------------------------------------------------------- /lib/ms/misc/banner_gen.ex: -------------------------------------------------------------------------------- 1 | defmodule MsBase.BannerGen do 2 | #http://patorjk.com/software/taag/#p=display&f=Chiseled&t=elixir 3 | 4 | def read_banner() do 5 | 6 | banner_path = Path.expand("config/banner.txt", File.cwd!()) 7 | 8 | if File.exists?(banner_path) do 9 | File.stream!(banner_path) |> Stream.each(fn line -> IO.write(line) end) |> Stream.run 10 | end 11 | 12 | :ok 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/ms/plugs/access_log.ex: -------------------------------------------------------------------------------- 1 | defmodule MSBase.AccessLog do 2 | 3 | @moduledoc """ 4 | Access Log 5 | """ 6 | 7 | alias Plug.Conn 8 | @behaviour Plug 9 | 10 | def init(opts) do 11 | opts 12 | end 13 | 14 | def call(conn, _) do 15 | 16 | start = System.monotonic_time() 17 | 18 | Conn.register_before_send(conn, fn conn -> 19 | 20 | stop = System.monotonic_time() 21 | diff = System.convert_time_unit(stop - start, :native, :microsecond) 22 | 23 | status = Integer.to_string(conn.status) 24 | formatted_diff = Enum.join(formatted_diff(diff), " ") 25 | 26 | conn 27 | |> get_json_log(status, formatted_diff) 28 | |> write_log 29 | 30 | conn 31 | end) 32 | end 33 | 34 | def get_correlation_id(conn) do 35 | List.first Plug.Conn.get_req_header(conn, "correlation-id") 36 | end 37 | 38 | defp get_current_color do 39 | System.get_env("SERVICE_COLOR") 40 | end 41 | 42 | defp get_hostname do 43 | {:ok, hostname} = :inet.gethostname() 44 | hostname 45 | end 46 | 47 | defp get_iso_time do 48 | DateTime.to_iso8601 DateTime.utc_now() 49 | end 50 | 51 | defp get_server_name(conn) do 52 | (List.first Plug.Conn.get_req_header(conn, "host")) || "unknown" 53 | end 54 | 55 | defp get_remote_client_id(conn) do 56 | (List.first Plug.Conn.get_req_header(conn, "customer-uuid")) || 57 | (List.first Plug.Conn.get_req_header(conn, "auth-info-user-id")) || 58 | "unknown" 59 | end 60 | 61 | defp get_json_log(conn, status, response_time) do 62 | {:ok, string} = Poison.encode(%MSBase.AccessLogMsg{ 63 | status: status, 64 | response_time: response_time, 65 | service: "ex_test", 66 | request_method: conn.method, 67 | uri: "/" <> Enum.join(conn.path_info, "/"), 68 | query_string: conn.query_string, 69 | "@timestamp": get_iso_time(), 70 | "correlation-id": get_correlation_id(conn), 71 | beam_pid: "#{:erlang.pid_to_list(self())}", 72 | node: "#{node()}", 73 | host: get_hostname(), 74 | current_color: get_current_color(), 75 | server_name: get_server_name(conn), 76 | remote_client_id: get_remote_client_id(conn) 77 | }) 78 | 79 | string 80 | end 81 | 82 | defp write_log(string) do 83 | IO.puts string 84 | {:ok} 85 | end 86 | 87 | defp formatted_diff(diff) when diff > 1000, do: [diff |> div(1000) |> Integer.to_string, "ms"] 88 | defp formatted_diff(diff), do: [Integer.to_string(diff), "µs"] 89 | end 90 | -------------------------------------------------------------------------------- /lib/ms/registry.ex: -------------------------------------------------------------------------------- 1 | defmodule MSBase.Registry do 2 | use GenServer 3 | 4 | @moduledoc """ 5 | Registry 6 | """ 7 | 8 | def start_link do 9 | GenServer.start_link(__MODULE__, nil, name: :registry) 10 | end 11 | 12 | # "client-side" code 13 | 14 | # call awaits a return 15 | def get(key) when is_bitstring(key) do 16 | GenServer.call(:registry, {:lookup, key}) 17 | end 18 | 19 | # cast is fire & forget 20 | def set(key, value) when is_bitstring(key) do 21 | GenServer.cast(:registry, {:create, key, value}) 22 | end 23 | 24 | def all do 25 | GenServer.call(:registry, {:all}) 26 | end 27 | 28 | # "server-side" code 29 | 30 | # init is synchronous 31 | def init(_) do 32 | init_state = %{} 33 | {:ok, init_state} 34 | end 35 | 36 | def handle_call({:lookup, key}, _from, state) do 37 | {:reply, Map.fetch(state, key), state} 38 | end 39 | 40 | def handle_call({:all}, _from, state) do 41 | {:reply, state, state} 42 | end 43 | 44 | # map is immutable, Map.put will create new map 45 | # :noreply is sent back to genserver 46 | 47 | # newState is passed via genserver 48 | 49 | def handle_cast({:create, key, value}, state) do 50 | new_state = Map.put(state, key, value) 51 | {:noreply, new_state} 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/ms/request/request.ex: -------------------------------------------------------------------------------- 1 | defmodule MsBase.Request do 2 | use HTTPoison.Base 3 | 4 | @expected_fields ~w( 5 | login id avatar_url gravatar_id url html_url followers_url 6 | following_url gists_url starred_url subscriptions_url 7 | organizations_url repos_url events_url received_events_url type 8 | site_admin name company blog location email hireable bio 9 | public_repos public_gists followers following created_at updated_at 10 | ) 11 | 12 | def process_url(url) do 13 | "https://api.github.com" <> url 14 | end 15 | 16 | def process_response_body(body) do 17 | body 18 | |> Poison.decode! 19 | |> Map.take(@expected_fields) 20 | |> Enum.map(fn({k, v}) -> {String.to_atom(k), v} end) 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/ms/structs/access_log_msg.ex: -------------------------------------------------------------------------------- 1 | defmodule MSBase.AccessLogMsg do 2 | @derive [Poison.Encoder] 3 | defstruct [ 4 | "@timestamp": "", 5 | host: "", 6 | loglevel: "INFO", 7 | "correlation-id": "", 8 | application_type: "service", 9 | status: "", 10 | service: "", 11 | log_type: "access", 12 | request_method: "", 13 | uri: "", 14 | query_string: "", 15 | response_time: "", 16 | protocol: "HTTP", 17 | server_name: "", 18 | current_color: "", 19 | remote_client_id: "", 20 | bytes_received: "0", 21 | bytes_sent: "0", 22 | beam_pid: "", 23 | node: "" 24 | ] 25 | end 26 | -------------------------------------------------------------------------------- /lib/ms/structs/log_msg.ex: -------------------------------------------------------------------------------- 1 | defmodule MSBase.LogMsg do 2 | @derive [Poison.Encoder] 3 | defstruct [ 4 | "@timestamp": "", 5 | host: "", 6 | pid: "", 7 | loglevel: "", 8 | loglevel_value: "", 9 | log_type: "application", 10 | application_type: "service", 11 | service: "", 12 | current_color: "", 13 | msg: "", 14 | "correlation-id": "", 15 | beam_pid: "", 16 | node: "" 17 | ] 18 | end 19 | -------------------------------------------------------------------------------- /lib/router/admin_router.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.AdminRouter do 2 | use Plug.Router 3 | 4 | plug :match 5 | plug :dispatch 6 | 7 | get "/healthcheck" do 8 | conn 9 | |> put_resp_content_type("application/json") 10 | |> send_resp(200, "") 11 | end 12 | 13 | get "/health" do 14 | {:ok, string} = Poison.encode(%{ 15 | status: "UP", 16 | kafka: ExTest.KafkaConsumer.get_stats() 17 | }) 18 | conn 19 | |> put_resp_content_type("application/json") 20 | |> send_resp(200, string) 21 | end 22 | 23 | match _ do 24 | send_resp(conn, 404, "unavailable") 25 | end 26 | 27 | end 28 | -------------------------------------------------------------------------------- /lib/router/ecto_router.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.EctoRouter do 2 | use Plug.Router 3 | 4 | plug :match 5 | plug :dispatch 6 | 7 | post "/test" do 8 | changeset = Ecto.Changeset.cast(%ExTest.Schemas.Test{}, conn.body_params, [:mach_doch]) 9 | result = ExTest.Repos.Test.insert!(changeset) 10 | IO.inspect result 11 | 12 | conn 13 | |> put_resp_content_type("application/json") 14 | |> send_resp(200, "{}") 15 | end 16 | 17 | get "/test" do 18 | 19 | result = ExTest.Ecto.any_query() 20 | IO.inspect result 21 | 22 | {:ok, string} = Poison.encode(%{ 23 | result: result 24 | }) 25 | 26 | conn 27 | |> put_resp_content_type("application/json") 28 | |> send_resp(200, string) 29 | end 30 | 31 | match _ do 32 | send_resp(conn, 404, "unavailable") 33 | end 34 | 35 | end 36 | -------------------------------------------------------------------------------- /lib/router/metrics_router.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.MetricsRouter do 2 | use Plug.Router 3 | 4 | # require Prometheus.Format.Text 5 | 6 | plug :match 7 | plug :dispatch 8 | 9 | # checkout: https://github.com/deadtrickster/prometheus.ex/blob/master/test/format/text_test.exs 10 | # for custom metrics 11 | 12 | get "/" do 13 | # format = Prometheus.Format.Text.format(:default) 14 | conn 15 | |> put_resp_content_type("text/plain") 16 | |> send_resp(200, "prometheus_ex broken in elixir 1.8") 17 | end 18 | 19 | match _ do 20 | send_resp(conn, 404, "unavailable") 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/router/router.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Router do 2 | use Plug.Router 3 | 4 | alias MSBase.Log 5 | 6 | # plug pipeline 7 | 8 | plug MSBase.AccessLog 9 | plug :match 10 | plug Plug.Parsers, parsers: [:json], 11 | pass: ["application/json"], 12 | json_decoder: Poison 13 | plug :dispatch 14 | 15 | # sub router forwarding 16 | 17 | forward "/admin", to: ExTest.AdminRouter 18 | forward "/metrics", to: ExTest.MetricsRouter 19 | forward "/ecto", to: ExTest.EctoRouter 20 | 21 | # pattern matching 22 | 23 | get "/" do 24 | 25 | Log.debug("someone called index", conn) 26 | 27 | conn 28 | |> put_resp_content_type("text/plain") 29 | |> send_resp(200, "ExTest") 30 | end 31 | 32 | get "/example" do 33 | {:ok, string} = Poison.encode(%ExTest.ResDto{ 34 | a_string: "some text", 35 | a_number: 123, 36 | a_list: ["of", "items"] 37 | }) 38 | 39 | conn 40 | |> put_resp_content_type("application/json") 41 | |> send_resp(200, string) 42 | end 43 | 44 | post "/example" do 45 | case conn.body_params do 46 | %{"accepted" => value} -> 47 | send_resp(conn, 200, value) 48 | _ -> 49 | send_resp(conn, 400, "key 'accepted' not found in request body") 50 | end 51 | end 52 | 53 | match _ do 54 | send_resp(conn, 404, "unavailable") 55 | end 56 | 57 | end 58 | -------------------------------------------------------------------------------- /lib/streams/kafka_consumer.ex: -------------------------------------------------------------------------------- 1 | # this is just an example scheme of how a kafka json message could look like 2 | # you might want to adapt this 3 | defmodule ExTest.KafkaMessage do 4 | @derive [Poison.Encoder] 5 | defstruct [ 6 | :id, 7 | :payload, 8 | :key, 9 | :time, 10 | :type 11 | ] 12 | end 13 | 14 | defmodule ExTest.KafkaConsumer do 15 | use KafkaConsumer.EventHandler 16 | 17 | @moduledoc """ 18 | A Kafka Consumer Demo 19 | """ 20 | 21 | alias MSBase.Log 22 | 23 | def init(init_arg) do 24 | {:ok, init_arg} 25 | end 26 | 27 | def handle_call({topic, partition, message}, _from, state) do 28 | 29 | try do 30 | Log.debug "kafka msg: #{topic}:#{partition} message: #{message.key}" 31 | msg = Poison.decode!(message.value, as: %ExTest.KafkaMessage{}) 32 | Log.info msg.payload 33 | :ok 34 | rescue 35 | e in RuntimeError -> 36 | Log.error to_string(e) 37 | end 38 | 39 | {:reply, :ok, state} 40 | end 41 | 42 | # returns stats 43 | def get_stats(topic) when is_binary(topic) do 44 | KafkaEx.metadata(topic: topic) 45 | end 46 | 47 | def get_stats do 48 | topic = Application.get_env(:ex_test, :kafka_topic) 49 | get_stats topic 50 | end 51 | 52 | # returns a list of all partition ids for a given topic 53 | def get_partition_list(topic) do 54 | kafka_md = get_stats topic 55 | topic_md = List.first kafka_md.topic_metadatas 56 | part_md = topic_md.partition_metadatas 57 | Enum.map(part_md, fn(p) -> p.partition_id end) 58 | end 59 | 60 | # a list of all partitions of this topic 61 | # can be used to set config.exs easier 62 | def get_partition_config(topic) do 63 | partitions = get_partition_list topic 64 | Enum.map(partitions, fn(p) -> {topic, p} end) 65 | end 66 | 67 | end 68 | -------------------------------------------------------------------------------- /lib/streams/redis.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Redis do 2 | use GenServer 3 | 4 | alias MSBase.Log 5 | 6 | def start_link(channel) do 7 | GenServer.start_link(__MODULE__, channel) 8 | end 9 | 10 | def init(channel) do 11 | 12 | host = Application.get_env(:ex_test, :redis_host) 13 | port = Application.get_env(:ex_test, :redis_port) 14 | 15 | {:ok, conn} = Redix.PubSub.start_link(host: host, port: port) 16 | 17 | Redix.PubSub.subscribe(conn, channel, self()) 18 | 19 | # await subscription of channel 20 | receive do 21 | {:redix_pubsub, ^conn, :subscribed, %{channel: channel}} -> 22 | Log.info("redis subscribed to channel #{channel}.") 23 | {:ok, %{channel: channel}} 24 | after 25 | 200 -> 26 | Log.error("failed to subscribe to redis channel #{channel}.") 27 | {:stop, "redis subscription failed"} # :stop is a convention here 28 | end 29 | end 30 | 31 | def handle_info({:redix_pubsub, _, :message, %{channel: channel, payload: payload}}, state) do 32 | Log.debug("received redis message on channel #{channel} with payload #{payload}") 33 | #do something here 34 | {:noreply, state} 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/structs/res_dto.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.ResDto do 2 | @derive [Poison.Encoder] 3 | defstruct [ 4 | :a_string, 5 | :a_number, 6 | :a_list 7 | ] 8 | end 9 | -------------------------------------------------------------------------------- /lib/supervisor.ex: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Supervisor do 2 | use Supervisor 3 | 4 | alias MSBase.Registry 5 | 6 | def start_link do 7 | Supervisor.start_link(__MODULE__, []) 8 | end 9 | 10 | def init([]) do 11 | 12 | children = [ 13 | Plug.Adapters.Cowboy.child_spec(:http, ExTest.Router, [], [ 14 | port: Application.get_env(:ex_test, :port) 15 | ]), 16 | worker(Registry, []), 17 | supervisor(ExTest.Repos.Test, []) 18 | ] 19 | 20 | # a worker is a (erlang) process that has no other children 21 | 22 | supervise(children, strategy: :one_for_one) 23 | 24 | # (kills supervisor after 5 bad restarts of a worker) 25 | # one_for_one strategy keeps a single instance of the worker running 26 | # rest_for_one, once_for_all 27 | 28 | end 29 | 30 | end 31 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Mixfile do 2 | use Mix.Project 3 | 4 | def project do 5 | [ 6 | app: :ex_test, 7 | version: "2.0.0", 8 | elixir: "~> 1.8.1", 9 | build_embedded: Mix.env == :prod, 10 | start_permanent: Mix.env == :prod, 11 | deps: deps(), 12 | aliases: aliases() 13 | ] 14 | end 15 | 16 | def application do 17 | [ 18 | mod: {ExTest, []}, 19 | applications: [ 20 | #:prometheus_ex, 21 | :httpoison, 22 | :cowboy, 23 | :plug, 24 | :postgrex, 25 | #:kafka_consumer, 26 | :gproc, 27 | #:xandra, 28 | :postgrex 29 | ] 30 | ] 31 | end 32 | 33 | defp aliases do 34 | [ 35 | start: ["clean", "compile", "run --no-halt"] 36 | ] 37 | end 38 | 39 | defp deps do 40 | [ 41 | {:credo, "~> 0.7", only: [:dev, :test]}, # linting 42 | {:poison, "~> 3.1"}, # json parser 43 | {:httpoison, "~> 0.11"}, # http client 44 | #{:redix, "~> 0.5"}, # redis 45 | {:redix_pubsub, "~> 0.2.0"}, # redis pubsub actions (ships with redix) 46 | {:plug, "~> 1.3"}, # http server wrapper 47 | {:plug_cowboy, "~> 1.0"}, # http server 48 | {:bunt, "~> 0.2.0"}, # cli colors 49 | # {:prometheus_ex, "~> 1.1.0"}, # metrics, broken in elixir 1.8 50 | #{:kafka_ex, "~> 0.6.3"}, # kafka client 51 | {:kafka_consumer, "~> 1.2.0"}, # easier kafka consumer (ships with kafka_ex & poolboy) 52 | {:xandra, "~> 0.5.0"}, # cassandra driver 53 | {:weave, "~> 1.0.0"}, # JIT config 54 | {:dialyxir, "~> 0.5", only: [:dev], runtime: false}, # static analysis 55 | {:postgrex, "~> 0.13.2"}, #postgres driver 56 | {:ecto, "~> 2.1.0"}, #db orm 57 | { :uuid, "~> 1.1" }, 58 | ] 59 | end 60 | 61 | end 62 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, 3 | "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, 4 | "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, 5 | "cowboy": {:hex, :cowboy, "1.1.2", "61ac29ea970389a88eca5a65601460162d370a70018afe6f949a29dca91f3bb0", [:rebar3], [{:cowlib, "~> 1.0.2", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3.2", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"}, 6 | "cowlib": {:hex, :cowlib, "1.0.2", "9d769a1d062c9c3ac753096f868ca121e2730b9a377de23dec0f7e08b1df84ee", [:make], [], "hexpm"}, 7 | "credo": {:hex, :credo, "0.10.2", "03ad3a1eff79a16664ed42fc2975b5e5d0ce243d69318060c626c34720a49512", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, 8 | "db_connection": {:hex, :db_connection, "1.1.3", "89b30ca1ef0a3b469b1c779579590688561d586694a3ce8792985d4d7e575a61", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"}, 9 | "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"}, 10 | "dialyxir": {:hex, :dialyxir, "0.5.1", "b331b091720fd93e878137add264bac4f644e1ddae07a70bf7062c7862c4b952", [:mix], [], "hexpm"}, 11 | "ecto": {:hex, :ecto, "2.1.6", "29b45f393c2ecd99f83e418ea9b0a2af6078ecb30f401481abac8a473c490f84", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, repo: "hexpm", optional: true]}], "hexpm"}, 12 | "gproc": {:hex, :gproc, "0.6.1", "4579663e5677970758a05d8f65d13c3e9814ec707ad51d8dcef7294eda1a730c", [:rebar3], [], "hexpm"}, 13 | "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, 14 | "httpoison": {:hex, :httpoison, "0.13.0", "bfaf44d9f133a6599886720f3937a7699466d23bb0cd7a88b6ba011f53c6f562", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, 15 | "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, 16 | "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, 17 | "kafka_consumer": {:hex, :kafka_consumer, "1.2.0", "3251cbfa576a845f13467554a1eb3485c78378cc5f43315565606c909887595f", [:mix], [{:gproc, "~> 0.6.0", [hex: :gproc, repo: "hexpm", optional: false]}, {:kafka_ex, "0.6.3", [hex: :kafka_ex, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm"}, 18 | "kafka_ex": {:hex, :kafka_ex, "0.6.3", "26b9685b6a209fca8c861167b8d55e1d7943028f9c09661354e9a3be4e76d689", [:mix], [], "hexpm"}, 19 | "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, 20 | "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, 21 | "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, 22 | "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, 23 | "plug": {:hex, :plug, "1.7.2", "d7b7db7fbd755e8283b6c0a50be71ec0a3d67d9213d74422d9372effc8e87fd1", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}], "hexpm"}, 24 | "plug_cowboy": {:hex, :plug_cowboy, "1.0.0", "2e2a7d3409746d335f451218b8bb0858301c3de6d668c3052716c909936eb57a", [:mix], [{:cowboy, "~> 1.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, 25 | "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, 26 | "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, 27 | "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, 28 | "postgrex": {:hex, :postgrex, "0.13.5", "3d931aba29363e1443da167a4b12f06dcd171103c424de15e5f3fc2ba3e6d9c5", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 1.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm"}, 29 | "prometheus": {:hex, :prometheus, "3.5.1", "12139025b942743206a315f31c78d6557dd6848ffde1023f7e93f62ec93352db", [:mix, :rebar3], [], "hexpm"}, 30 | "prometheus_ex": {:hex, :prometheus_ex, "1.1.1", "410b268d492081d06abda59d14207f9cb97b95f082cdd51cbd50dc45444b65c5", [:mix], [{:prometheus, "~> 3.1", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, 31 | "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], [], "hexpm"}, 32 | "redix": {:hex, :redix, "0.5.2", "82a7b3cf9141a8d3de7d38d321717fe7ae6f998561cd87d374ff9012db87913a", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"}, 33 | "redix_pubsub": {:hex, :redix_pubsub, "0.2.0", "c42660d538698ca0b6a3bd671fac42e948402cdc3223e53db875891dd5446497", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:redix, "~> 0.5.0", [hex: :redix, repo: "hexpm", optional: false]}], "hexpm"}, 34 | "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, 35 | "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, 36 | "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], [], "hexpm"}, 37 | "weave": {:hex, :weave, "1.0.0", "5a8b8c0a49e05e782f701685c0a71eb7b5addd24cd2efc10e1547c8d4cda1df9", [:mix], [], "hexpm"}, 38 | "xandra": {:hex, :xandra, "0.5.1", "17c3f2f2b89dd417a9014233fda7345f138ff0ff0ebb893ee33f9d2dea449b96", [:mix], [{:db_connection, "~> 1.0", [hex: :db_connection, repo: "hexpm", optional: false]}], "hexpm"}, 39 | } 40 | -------------------------------------------------------------------------------- /priv/test/migrations/20170503150023_add_test_table.exs: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Repos.Test.Migrations.AddTestTable do 2 | use Ecto.Migration 3 | 4 | def up do 5 | 6 | create table(:test) do 7 | add :bla, :string 8 | add :id_again, :integer 9 | add :blup, :float, default: 0.0 10 | timestamps() 11 | end 12 | end 13 | 14 | def down do 15 | drop table(:test) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /priv/test/migrations/20170508135901_add_test_change.exs: -------------------------------------------------------------------------------- 1 | defmodule ExTest.Repos.Test.Migrations.AddTestChange do 2 | use Ecto.Migration 3 | 4 | def change do 5 | 6 | alter table(:test) do 7 | add :mach_doch, :integer 8 | end 9 | 10 | index(:test, [:mach_doch]) 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /test/ms/plugs/access_log_test.exs: -------------------------------------------------------------------------------- 1 | defmodule MSBase.AccessLogTest do 2 | use ExUnit.Case 3 | use Plug.Test 4 | 5 | import ExUnit.CaptureIO 6 | 7 | alias MSBase.AccessLog 8 | 9 | defp apply_request_headers(test_conn, request_headers) do 10 | Enum.reduce( 11 | request_headers, 12 | test_conn, 13 | fn({key, value}, c) -> 14 | Plug.Conn.put_req_header(c, key, value) 15 | end) 16 | end 17 | 18 | defp run_conn(url, request_headers \\ []) do 19 | run_plug = fn -> 20 | conn(:get, url, "") 21 | |> apply_request_headers(request_headers) 22 | |> AccessLog.call(%{}) 23 | |> resp(200, "response") 24 | |> send_resp 25 | end 26 | 27 | {:ok, log_msg} = Poison.decode(capture_io(run_plug), as: %MSBase.AccessLogMsg{}) 28 | log_msg 29 | end 30 | 31 | test "logs request information and access log fields" do 32 | log_msg = run_conn("/product?id=123") 33 | 34 | #TODO match against map 35 | assert log_msg.uri == "/product" 36 | assert log_msg.application_type == "service" 37 | assert log_msg.log_type == "access" 38 | assert log_msg.loglevel == "INFO" 39 | assert log_msg.request_method == "GET" 40 | assert log_msg.service == "ex_test" 41 | assert log_msg.status == "200" 42 | assert log_msg.protocol == "HTTP" 43 | assert log_msg.query_string == "id=123" 44 | end 45 | 46 | test "logs the timestamp" do 47 | unix_time_start = DateTime.utc_now() |> DateTime.to_unix(:microsecond) 48 | log_msg = run_conn("/") 49 | 50 | {:ok, time_log, _} = DateTime.from_iso8601(log_msg."@timestamp") 51 | unix_time_log = time_log |> DateTime.to_unix(:microsecond) 52 | diff = unix_time_log - unix_time_start 53 | 54 | assert_in_delta(diff, 1, 10000) 55 | end 56 | 57 | test "logs beam specific fields" do 58 | {:ok, hostname} = :inet.gethostname() 59 | log_msg = run_conn("/") 60 | 61 | assert log_msg.beam_pid == "#{:erlang.pid_to_list(self())}" 62 | assert log_msg.node == "#{node()}" 63 | assert log_msg.host == hostname 64 | end 65 | 66 | test "logs the current color" do 67 | System.put_env("SERVICE_COLOR", "green") 68 | log_msg = run_conn("/") 69 | 70 | assert log_msg.current_color == "green" 71 | end 72 | 73 | test "logs the request duration" do 74 | duration_regex = ~r/(?\d+) (?ms|µs)/ 75 | 76 | log_msg = run_conn("/") 77 | %{"duration" => duration_string, "unit" => unit} = Regex.named_captures(duration_regex, log_msg.response_time) 78 | duration = String.to_integer(duration_string) 79 | 80 | assert_in_delta(duration, 1, 999) 81 | assert unit == "ms" || unit == "µs" 82 | end 83 | 84 | test "logs the correlation-id" do 85 | correlation_id = "abc-123" 86 | log_msg = run_conn("/", [{"correlation-id", correlation_id}]) 87 | 88 | assert log_msg."correlation-id" == correlation_id 89 | end 90 | 91 | test "logs the customer-uuid as the remote_client_id if present" do 92 | customer_uuid = "007-customer" 93 | log_msg = run_conn( 94 | "/", 95 | [ 96 | {"customer-uuid", customer_uuid}, 97 | {"auth-info-user-id", "not_this"} 98 | ] 99 | ) 100 | 101 | assert log_msg.remote_client_id == customer_uuid 102 | end 103 | 104 | test "logs the auth-info-user-id as the remote_client_id as a fallback to customer-uuid" do 105 | auth_info_user_id = "auth-info-user-01" 106 | log_msg = run_conn("/", [{"auth-info-user-id", auth_info_user_id}]) 107 | 108 | assert log_msg.remote_client_id == auth_info_user_id 109 | end 110 | 111 | test "logs unknown as remote_client_id if no customer-uuid and auth-info-user-id header present" do 112 | log_msg = run_conn("/") 113 | 114 | assert log_msg.remote_client_id == "unknown" 115 | end 116 | 117 | test "logs host header as server name" do 118 | host = "127.0.0.1" 119 | log_msg = run_conn("/", [{"host", host}]) 120 | 121 | assert log_msg.server_name == host 122 | end 123 | 124 | test "logs unknown as server name if host header not present" do 125 | log_msg = run_conn("/") 126 | 127 | assert log_msg.server_name == "unknown" 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start() 2 | -------------------------------------------------------------------------------- /tools/compile-dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mix deps.compile -------------------------------------------------------------------------------- /tools/docker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | docker build -t ex-test:latest . 3 | docker run -it --rm --name ex-test -p 8080:8080 ex-test:latest 4 | -------------------------------------------------------------------------------- /tools/drun.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mix run --no-halt -------------------------------------------------------------------------------- /tools/get-dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mix deps.get -------------------------------------------------------------------------------- /tools/interactive.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | iex -S mix -------------------------------------------------------------------------------- /tools/lint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mix credo -------------------------------------------------------------------------------- /tools/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mix clean 3 | mix compile 4 | mix run --no-halt -------------------------------------------------------------------------------- /tools/static.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | mix dialyzer 3 | --------------------------------------------------------------------------------