├── .gitignore ├── .travis.yml ├── Emakefile ├── LICENSE ├── Makefile ├── README.md ├── bin ├── app ├── flow └── version ├── dev.config ├── docker-compose.yml ├── erlang.mk ├── log └── .gitignore ├── src ├── elastic.app.src ├── elastic.erl ├── elastic_application.erl ├── elastic_config.erl ├── elastic_http.erl ├── elastic_http_supervisor.erl └── elastic_supervisor.erl └── start-dev.sh /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.plt 3 | erl_crash.dump 4 | deps 5 | .eunit 6 | .DS_Store 7 | .erlang.mk.* 8 | ebin 9 | _rel 10 | relx 11 | .erlang.mk 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: erlang 3 | script: "make" 4 | otp_release: 5 | - 19.3 6 | -------------------------------------------------------------------------------- /Emakefile: -------------------------------------------------------------------------------- 1 | % -*- mode: erlang -*- 2 | {["src/*"], 3 | [{i, "include"}, 4 | {outdir, "ebin"}, 5 | debug_info] 6 | }. 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | #-*- mode: makefile-gmake -*- 2 | # Copyright (c) 2012-2015 Peter Morgan 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | PROJECT = elastic 17 | DEPS = gproc gun jsx 18 | include erlang.mk 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Elastic 2 | 3 | Elastic is a minimal [Erlang](http://erlang.org) API to 4 | [Elasticsearch](https://www.elastic.co/products/elasticsearch) 5 | packaged to [link](https://docs.docker.com/userguide/dockerlinks/) 6 | directly to the official 7 | [elasticsearch](https://registry.hub.docker.com/_/elasticsearch/) 8 | [docker](https://registry.hub.docker.com) image. 9 | 10 | ## Building 11 | 12 | Elastic uses [erlang.mk](https://github.com/ninenines/erlang.mk). To build run: 13 | 14 | ``` 15 | make 16 | ``` 17 | 18 | [![Build Status](https://travis-ci.org/shortishly/elastic.svg)](https://travis-ci.org/shortishly/elastic) 19 | 20 | ## Quick start 21 | 22 | ```shell 23 | docker-compose start 24 | ./start-dev.sh 25 | ``` 26 | 27 | To index a document: 28 | 29 | ```erlang 30 | elastic:index_document(hourly, "stats", #{timestamp => erlang:universaltime()}). 31 | {ok,#{<<"_id">> => <<"AU1cyBfR_TWT0pl8At7u">>, 32 | <<"_index">> => <<"logstash-2015.05.16.12">>, 33 | <<"_type">> => <<"stats">>, 34 | <<"_version">> => 1, 35 | <<"created">> => true}} 36 | ``` 37 | 38 | In Kibana, create a new index by selecting: "settings", followed by 39 | "add new" index pattern. Ensure that both "index contains time-based 40 | events" and also "use event times to create index names", with "daily" 41 | as the index pattern interval. The "time-field name" should populate 42 | with "timestamp" (otherwise a quick "refresh fields" should sort it). 43 | 44 | Once the index is created, you can navigate to "Discover" and view the 45 | indexed data in Kibana. 46 | 47 | 48 | ## Environment 49 | 50 | To support linking via Docker the following environment variables are 51 | used by default to connect the elasticsearch instance. 52 | 53 | |Parameter |Description|Default | 54 | |------------------------------------|-----------|----------------| 55 | |ELASTICSEARCH\_PORT\_9200\_TCP\_ADDR|hostname |$(bootdocker ip)| 56 | |ELASTICSEARCH\_PORT\_9200\_TCP\_PORT|port |9200 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /bin/app: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %% Copyright (c) 2012-2015 Peter Morgan 4 | %% 5 | %% Licensed under the Apache License, Version 2.0 (the "License"); 6 | %% you may not use this file except in compliance with the License. 7 | %% You may obtain a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, software 12 | %% distributed under the License is distributed on an "AS IS" BASIS, 13 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | %% See the License for the specific language governing permissions and 15 | %% limitations under the License. 16 | 17 | 18 | main([]) -> 19 | {application, Name, _} = application(), 20 | io:format("~w~n", [Name]). 21 | 22 | app_src() -> 23 | [Filename] = filelib:wildcard(filename:join([filename:dirname(escript:script_name()), "../src/*.app.src"])), 24 | Filename. 25 | 26 | application() -> 27 | {ok, [{application, _, _} = Application]} = file:consult(app_src()), 28 | Application. 29 | -------------------------------------------------------------------------------- /bin/flow: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright (c) 2012-2015 Peter Morgan 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Build a installable OS package whenever there are differences 18 | # between the develop and master branches. Following the git flow 19 | # process, this script creates a release branch to increment the patch 20 | # version number, merge the result onto master, and merge the final 21 | # back onto develop. A fresh build is created and an OS installable 22 | # package is created. 23 | # 24 | 25 | SCRIPT=$_ 26 | ROOT=$(dirname $0)/.. 27 | PATH=${PATH}:/usr/local/bin 28 | 29 | PREVIOUS_RELEASE=$(${ROOT}/bin/version app) 30 | 31 | # tidy up the working directory removing anything and everything 32 | # that isn't tracked (second -f removes dependencies that are 33 | # tracked by other external git repositories) 34 | # 35 | git clean -d -x -f -f 36 | git reset --hard 37 | 38 | # fetch from the origin 39 | # 40 | git pull 41 | 42 | # Checkout (and track if necessary from the origin) develop 43 | # 44 | if git show-ref --quiet --verify refs/heads/develop; 45 | then 46 | git checkout develop 47 | else 48 | git checkout --track refs/remotes/origin/develop 49 | fi 50 | 51 | # Only perform a build if there are differences between develop and 52 | # master. 53 | # 54 | git diff --quiet master && exit 0 55 | 56 | # If the build fails, we will rollback to develop prior to any 57 | # release branches or tags. 58 | # 59 | ROLLBACK=$(git show-ref --hash refs/heads/develop) 60 | 61 | # Create a release branch based off current develop branch. Branch 62 | # is named by incementing the patch version number (which isn't 63 | # updated on the filesystem yet - to ensure that it is still clean 64 | # to create the branch). 65 | # 66 | git checkout -b release/$(${ROOT}/bin/version app increment patch) develop 67 | 68 | # Release branch is now created, safely write the new release 69 | # number into the OTP application version manifest 70 | # 71 | RELEASE=$(${ROOT}/bin/version app write increment patch) 72 | 73 | # Commit the new release version number into the release branch. 74 | # 75 | git commit -a -m "version bump for release ${RELEASE} branch" 76 | 77 | # Checkout master, merge in the release branch. Tag the result 78 | # with the new version number. 79 | # 80 | git checkout master 81 | git merge --no-ff -m "release ${RELEASE}" release/${RELEASE} 82 | git tag -m "release ${RELEASE}" ${RELEASE} 83 | git branch -d release/${RELEASE} 84 | 85 | # Checkout develop and merge the new master in. 86 | # 87 | git checkout develop 88 | git merge -m "merging master into develop for release ${RELEASE}" master 89 | 90 | function rollback { 91 | # Something bad happened during the build, roll back 92 | # releases branch and tag. 93 | # 94 | echo "rolling back ${RELEASE} to ${ROLLBACK} at $(/bin/date)" 95 | 96 | git tag -d ${RELEASE} 97 | git reset --hard ${ROLLBACK} 98 | 99 | # tidy up the working directory removing anything and everything 100 | # that isn't tracked. 101 | # 102 | git clean -d -x -f -f 103 | } 104 | 105 | # exit immediately if a command exits with a non-zero exit status 106 | # 107 | set -e 108 | 109 | # trap and rollback any changes on error 110 | # 111 | trap rollback ERR 112 | 113 | # Clean the workspace and produce a new build for release. 114 | # 115 | make 116 | 117 | # Push the develop and master branches, together with any tags to 118 | # the origin. 119 | # 120 | git push --tags origin develop master 121 | 122 | echo "build completed at $(/bin/date)" 123 | -------------------------------------------------------------------------------- /bin/version: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env escript 2 | %% -*- erlang -*- 3 | %% Copyright (c) 2012-2015 Peter Morgan 4 | %% 5 | %% Licensed under the Apache License, Version 2.0 (the "License"); 6 | %% you may not use this file except in compliance with the License. 7 | %% You may obtain a copy of the License at 8 | %% 9 | %% http://www.apache.org/licenses/LICENSE-2.0 10 | %% 11 | %% Unless required by applicable law or agreed to in writing, software 12 | %% distributed under the License is distributed on an "AS IS" BASIS, 13 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | %% See the License for the specific language governing permissions and 15 | %% limitations under the License. 16 | 17 | 18 | -record(version, {major :: non_neg_integer(), 19 | minor :: non_neg_integer(), 20 | patch :: non_neg_integer()}). 21 | 22 | 23 | main(["app"]) -> 24 | display(version(application())); 25 | 26 | main(["app", "increment", Component]) -> 27 | display(bump(Component, version(application()))); 28 | 29 | main(["app", "write", "increment", Component]) -> 30 | {application, Name, Terms} = Application = application(), 31 | CurrentVersion = #version{} = version(Application), 32 | NewVersion = #version{} = bump(Component, CurrentVersion), 33 | 34 | %% write out the application configuration substituting the bumped 35 | %% version for the existing. 36 | %% 37 | ok = file:write_file(app_src(), io_lib:format("%%-*- mode: erlang -*-~n~p.~n", 38 | [{application, 39 | Name, 40 | lists:map(fun({vsn, _}) -> 41 | {vsn, stringify(NewVersion)}; 42 | (X) -> 43 | X 44 | end, Terms)}])), 45 | case file:consult(relx_config()) of 46 | {ok, Configuration} -> 47 | %% write out the release configuration substituting the bumped 48 | %% version for the existing. 49 | %% 50 | ok = file:write_file(relx_config(), ["%%-*- mode: erlang -*-", 51 | io_lib:nl(), 52 | lists:map(bump_release(stringify(NewVersion)), Configuration)]), 53 | display(NewVersion); 54 | 55 | {error, enoent} -> 56 | display(NewVersion) 57 | end. 58 | 59 | 60 | bump_release(NewVersion) -> 61 | fun({release, {Name, _}, Other}) -> 62 | io_lib:format("~p.~n", [{release, {Name, NewVersion}, Other}]); 63 | (X) -> 64 | io_lib:format("~p.~n", [X]) 65 | end. 66 | 67 | 68 | stringify(#version{major = Major, minor = Minor, patch = Patch}) -> 69 | lists:flatten(io_lib:format("~w.~w.~w", [Major, Minor, Patch])). 70 | 71 | %% there is a single readable "*.app.src" file contained in the 72 | %% "src" directory 73 | %% 74 | app_src() -> 75 | [Filename] = filelib:wildcard(filename:join([filename:dirname(escript:script_name()), "../src/*.app.src"])), 76 | Filename. 77 | 78 | application() -> 79 | {ok, [{application, _, _} = Application]} = file:consult(app_src()), 80 | Application. 81 | 82 | relx_config() -> 83 | filename:join([filename:dirname(escript:script_name()), "../relx.config"]). 84 | 85 | 86 | %% using semantic versioning "MAJOR.MINOR.PATCH", stored in the 87 | %% "vsn" attribute of the application. 88 | %% 89 | version({application, _, Terms}) -> 90 | [Major, Minor, Patch] = [list_to_integer(L) || L <- re:split(proplists:get_value(vsn, Terms), "\\.", [{return, list}])], 91 | #version{major = Major, minor = Minor, patch = Patch}. 92 | 93 | 94 | 95 | 96 | %% bumping the patch version will only increment the patch 97 | %% version 98 | %% 99 | bump("patch", #version{patch = Patch} = V) -> 100 | V#version{patch = Patch + 1}; 101 | 102 | %% bumping the minor version will reset the patch, and only increment 103 | %% the minor version. 104 | %% 105 | bump("minor", #version{minor = Minor} = V) -> 106 | V#version{minor = Minor + 1, patch = 0}; 107 | 108 | %% bumping the major version will reset both the patch and minor, 109 | %% and only increment the major version. 110 | %% 111 | bump("major", #version{major = Major} = V) -> 112 | V#version{major = Major + 1, minor = 0, patch = 0}. 113 | 114 | 115 | display(#version{major = Major, minor = Minor, patch = Patch}) -> 116 | io:format("~w.~w.~w\n", [Major, Minor, Patch]). 117 | 118 | -------------------------------------------------------------------------------- /dev.config: -------------------------------------------------------------------------------- 1 | %% -*- mode: erlang -*- 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | 15 | [ 16 | {kernel, [ 17 | {error_logger, {file, "log/kernel.log"}} 18 | ]}, 19 | 20 | {sasl, [ 21 | {sasl_error_logger, {file, "log/sasl.log"}}, 22 | {error_logger_mf_dir,"log"}, 23 | {error_logger_mf_maxbytes,10485760}, 24 | {error_logger_mf_maxfiles, 10}, 25 | {errlog_type, all} 26 | ]} 27 | ]. 28 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | elasticsearch: 2 | image: elasticsearch 3 | ports: 4 | - "9200:9200" 5 | 6 | kibana: 7 | image: kibana 8 | links: 9 | - elasticsearch 10 | ports: 11 | - "5601:5601" 12 | -------------------------------------------------------------------------------- /log/.gitignore: -------------------------------------------------------------------------------- 1 | [1-9]* 2 | *.log.[0-9]* 3 | *.idx 4 | *.siz 5 | index 6 | kernel.log 7 | sasl.log 8 | bet.terms 9 | -------------------------------------------------------------------------------- /src/elastic.app.src: -------------------------------------------------------------------------------- 1 | %%-*- mode: erlang -*- 2 | {application,elastic, 3 | [{description,"Elasticsearch client API"}, 4 | {vsn,"0.1.4"}, 5 | {id,"git"}, 6 | {modules,[]}, 7 | {registered,[]}, 8 | {applications,[kernel,stdlib,gproc,gun,jsx]}, 9 | {mod,{elastic_application,[]}}, 10 | {env,[{elasticsearch_port_9200_tcp_addr,"127.0.0.1"}, 11 | {elasticsearch_port_9200_tcp_port,"9200"}, 12 | {index_prefix,"logstash"}]}]}. 13 | -------------------------------------------------------------------------------- /src/elastic.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2012-2015 Peter Morgan 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | 15 | -module(elastic). 16 | -export([ 17 | bulk_index_documents/3, 18 | start/0, 19 | make/0, 20 | get_env/1, 21 | index_document/4, 22 | index_document/3 23 | ]). 24 | 25 | start() -> 26 | application:ensure_all_started(?MODULE). 27 | 28 | make() -> 29 | make:all([load]). 30 | 31 | get_env(Key) -> 32 | gproc:get_env(l, ?MODULE, Key, [os_env, app_env]). 33 | 34 | -type index() :: hourly | daily | monthly | yearly. 35 | 36 | -spec index_document(index(), iolist(), map() | iolist()) -> {ok, map()} | {error, binary()}. 37 | index_document(Index, Type, #{timestamp := {{_, _, _}, {_, _, _}} = DateTime} = Document) -> 38 | elastic_http:index(connection(), #{index => index(Index, DateTime), 39 | type => Type, 40 | document => jsx:encode(Document)}); 41 | 42 | index_document(Index, Type, #{} = Document) -> 43 | elastic_http:index(connection(), #{index => index(Index), 44 | type => Type, 45 | document => jsx:encode(Document)}); 46 | 47 | index_document(Index, Type, Document) -> 48 | elastic_http:index(connection(), #{index => index(Index), 49 | type => Type, 50 | document => Document}). 51 | 52 | -spec index_document(index(), iolist(), iolist(), map()) -> {ok, map()} | {error, binary()}. 53 | index_document(Index, Type, Id, #{timestamp := {{_, _, _}, {_, _, _}} = DateTime} = Document) -> 54 | elastic_http:index(connection(), #{index => index(Index, DateTime), type => Type, id => Id, document => jsx:encode(Document)}); 55 | index_document(Index, Type, Id, #{} = Document) -> 56 | elastic_http:index(connection(), #{index => index(Index), type => Type, id => Id, document => jsx:encode(Document)}); 57 | index_document(Index, Type, Id, Document) -> 58 | elastic_http:index(connection(), #{index => index(Index), type => Type, id => Id, document => Document}). 59 | 60 | -spec bulk_index_documents(index(), iolist(), list()) -> {ok, map()} | {error, binary()}. 61 | bulk_index_documents(Index, Type, Documents) -> 62 | elastic_http:bulk_index(connection(), #{index => index(Index), 63 | type => Type, 64 | documents => Documents}). 65 | 66 | connection() -> 67 | {ok, Connection} = elastic_http_supervisor:start_child(elastic_config:tcp_host(), 68 | elastic_config:tcp_port()), 69 | Connection. 70 | 71 | 72 | -spec index(index()) -> iolist(). 73 | index(Type) -> 74 | index(Type, erlang:universaltime()). 75 | 76 | -spec index(index(), calendar:datetime()) -> iolist(). 77 | index(Type, DateTime) -> 78 | [elastic_config:elastic_index(), "-", postfix(Type, DateTime)]. 79 | 80 | 81 | -spec postfix(index(), calendar:datetime()) -> iolist(). 82 | postfix(hourly, {{Year, Month, Date}, {Hour, _, _}}) -> 83 | io_lib:format("~4..0b.~2..0b.~2..0b.~2..0b", [Year, Month, Date, Hour]); 84 | postfix(daily, {{Year, Month, Date}, _}) -> 85 | io_lib:format("~4..0b.~2..0b.~2..0b", [Year, Month, Date]); 86 | postfix(monthly, {{Year, Month, _}, _}) -> 87 | io_lib:format("~4..0b.~2..0b", [Year, Month]); 88 | postfix(yearly, {{Year, _, _}, _}) -> 89 | io_lib:format("~4..0b", [Year]). 90 | -------------------------------------------------------------------------------- /src/elastic_application.erl: -------------------------------------------------------------------------------- 1 | -module(elastic_application). 2 | -behaviour(application). 3 | 4 | -export([ 5 | start/2, 6 | stop/1 7 | ]). 8 | 9 | start(_Type, _Args) -> 10 | elastic_supervisor:start_link(). 11 | 12 | stop(_State) -> 13 | ok. 14 | -------------------------------------------------------------------------------- /src/elastic_config.erl: -------------------------------------------------------------------------------- 1 | -module(elastic_config). 2 | -export([debug/1]). 3 | -export([tcp_host/0]). 4 | -export([tcp_port/0]). 5 | -export([elastic_user/0]). 6 | -export([elastic_pass/0]). 7 | -export([elastic_index/0]). 8 | 9 | debug(applications) -> 10 | envy(to_list, debug_applications, "betash"). 11 | 12 | envy(To, Name, Default) -> 13 | envy:To(betash, Name, default(Default)). 14 | 15 | default(Default) -> 16 | [os_env, app_env, {default, Default}]. 17 | 18 | tcp_host() -> 19 | % envy(to_list, elastic_host, "10.16.13.10"). 20 | envy(to_list, elastic_host, "76607d4ec9ba43d4411b7113025a2016.us-east-1.aws.found.io"). 21 | 22 | tcp_port() -> 23 | % envy(to_integer, elastic_port, 9200). 24 | envy(to_integer, elastic_port, 9243). 25 | 26 | elastic_user() -> 27 | envy(to_list, elastic_user, "elastic"). 28 | 29 | elastic_pass() -> 30 | envy(to_list, elastic_pass, "Bq4ESAXF9vuGRMqvxCWDhpox"). 31 | 32 | elastic_index() -> 33 | envy(to_list, elastic_index, "logstash"). 34 | -------------------------------------------------------------------------------- /src/elastic_http.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2012-2015 Peter Morgan 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | 15 | -module(elastic_http). 16 | -behaviour(gen_server). 17 | -export([ 18 | bulk_index/2, 19 | start_link/2, 20 | index/2 21 | ]). 22 | -export([ 23 | init/1, 24 | code_change/3, 25 | handle_call/3, 26 | handle_cast/2, 27 | handle_info/2, 28 | terminate/2, 29 | headers/0 30 | ]). 31 | 32 | start_link(Host, Port) -> 33 | gen_server:start_link(?MODULE, [Host, Port], [{spawn_opt, [{fullsweep_after, 10}]}]). 34 | 35 | -spec index(pid(), map()) -> iodata(). 36 | index(Elastic, Parameters) -> 37 | gen_server:call(Elastic, {index, Parameters}, infinity). 38 | 39 | -spec bulk_index(pid(), map()) -> iodata(). 40 | bulk_index(Elastic, Parameters) -> 41 | gen_server:call(Elastic, {bulk_index, Parameters}, infinity). 42 | 43 | init([Host, Port]) -> 44 | case gun:open(Host, Port, #{transport => ssl}) of 45 | {ok, Gun} -> 46 | {ok, #{gun => Gun, response_buffer => <<>>}}; 47 | 48 | {error, _} = Error -> 49 | {stop, Error, stateless} 50 | end. 51 | 52 | handle_call({index, #{index := Index, type := Type, id := Id, document := Document}}, Reply, #{gun := Gun} = S) -> 53 | {noreply, maps:put(gun:put(Gun, ["/", Index, "/", Type, "/", Id], headers(), Document), Reply, S)}; 54 | 55 | handle_call({index, #{index := Index, type := Type, document := Document}}, Reply, #{gun := Gun} = S) -> 56 | {noreply, maps:put(gun:post(Gun, ["/", Index, "/", Type, "/"], headers(), Document), Reply, S)}; 57 | 58 | handle_call({bulk_index, #{index := Index, type := Type, documents := Documents}}, Reply, #{gun := Gun} = S) -> 59 | 60 | BulkDocument = lists:foldl(fun(Document, Acc) -> create_action(Index, Type, Acc, Document) end, <<>>, Documents), 61 | {noreply, maps:put(gun:post(Gun, ["/", "_bulk"], headers(), BulkDocument), Reply, S)}. 62 | 63 | handle_cast(stop, S) -> 64 | {stop, normal, S}. 65 | 66 | handle_info({gun_up, Gun, http}, #{gun := Gun} = S) -> 67 | {noreply, S}; 68 | 69 | handle_info({gun_response, Gun, _, nofin, _Status, _Headers}, #{gun := Gun} = S) -> 70 | {noreply, S}; 71 | handle_info({gun_data, Gun, _, nofin, Data}, #{gun := Gun, response_buffer := RespBuffer} = S) -> 72 | {noreply, S#{response_buffer => << RespBuffer/binary, Data/binary>>}}; 73 | handle_info({gun_data, Gun, _, nofin, _Data}, #{gun := Gun} = S) -> 74 | {noreply, S}; 75 | 76 | handle_info({gun_data, Gun, Stream, fin, Body}, #{gun := Gun, response_buffer := RespBuffer} = S) -> 77 | case jsx:decode(<< RespBuffer/binary, Body/binary>>, [return_maps]) of 78 | #{<<"error">> := Reason} -> 79 | gen_server:reply(maps:get(Stream, S), {error, Reason}), 80 | {stop, normal, S}; 81 | 82 | Response -> 83 | gen_server:reply(maps:get(Stream, S), {ok, Response}), 84 | {stop, normal, S} 85 | end. 86 | 87 | code_change(_, State, _) -> 88 | {ok, State}. 89 | 90 | terminate(_, #{gun := Gun}) -> 91 | gun:close(Gun). 92 | 93 | create_action(Index, Type, BulkDocument, Document) -> 94 | JsonDoc = jsx:encode(Document), 95 | IndexAsBinary = erlang:list_to_binary(Index), 96 | TypeAsBinary = erlang:list_to_binary(Type), 97 | 98 | << BulkDocument/binary, 99 | <<"{\"index\":{\"_index\":\"">>/binary, 100 | IndexAsBinary/binary, 101 | <<"\",\"_type\":\"">>/binary, 102 | TypeAsBinary/binary, 103 | <<"\"}}\n">>/binary, 104 | JsonDoc/binary, 105 | <<"\n">>/binary >>. 106 | 107 | headers() -> 108 | Base64 = encode_basic_auth(elastic_config:elastic_user(), 109 | elastic_config:elastic_pass()), 110 | [ 111 | {"Authorization", binary_to_list(<<"Basic ", Base64/binary>>)} 112 | ]. 113 | 114 | encode_basic_auth(Username, Password) -> 115 | base64:encode(Username ++ [$: | Password]). 116 | -------------------------------------------------------------------------------- /src/elastic_http_supervisor.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2012-2015 Peter Morgan 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | 15 | -module(elastic_http_supervisor). 16 | -behaviour(supervisor). 17 | -export([ 18 | start_link/0, 19 | start_child/2 20 | ]). 21 | -export([ 22 | init/1 23 | ]). 24 | 25 | 26 | start_link() -> 27 | supervisor:start_link(ref(), ?MODULE, []). 28 | 29 | init([]) -> 30 | {ok, {{simple_one_for_one, 1, 5}, [elastic_supervisor:worker(elastic_http, temporary)]}}. 31 | 32 | start_child(Host, Port) -> 33 | supervisor:start_child(ref(), [Host, Port]). 34 | 35 | ref() -> 36 | {via, gproc, {n, l, ?MODULE}}. 37 | -------------------------------------------------------------------------------- /src/elastic_supervisor.erl: -------------------------------------------------------------------------------- 1 | %% Copyright (c) 2012-2015 Peter Morgan 2 | %% 3 | %% Licensed under the Apache License, Version 2.0 (the "License"); 4 | %% you may not use this file except in compliance with the License. 5 | %% You may obtain a copy of the License at 6 | %% 7 | %% http://www.apache.org/licenses/LICENSE-2.0 8 | %% 9 | %% Unless required by applicable law or agreed to in writing, software 10 | %% distributed under the License is distributed on an "AS IS" BASIS, 11 | %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | %% See the License for the specific language governing permissions and 13 | %% limitations under the License. 14 | 15 | -module(elastic_supervisor). 16 | -behaviour(supervisor). 17 | 18 | -export([ 19 | start_link/0, 20 | supervisor/1, 21 | supervisor/2, 22 | worker/1, 23 | worker/2, 24 | worker/3, 25 | worker/4 26 | ]). 27 | -export([ 28 | init/1 29 | ]). 30 | 31 | start_link() -> 32 | supervisor:start_link(ref(), ?MODULE, []). 33 | 34 | supervisor(Module) -> 35 | supervisor(Module, permanent). 36 | 37 | supervisor(Module, Restart) -> 38 | {Module, {Module, start_link, []}, Restart, infinity, supervisor, [Module]}. 39 | 40 | worker(Module) -> 41 | worker(Module, permanent). 42 | 43 | worker(Module, Restart) -> 44 | worker(Module, Restart, []). 45 | 46 | worker(Module, Restart, Parameters) -> 47 | worker(Module, Module, Restart, Parameters). 48 | 49 | worker(Id, Module, Restart, Parameters) -> 50 | {Id, {Module, start_link, Parameters}, Restart, 5000, worker, [Module]}. 51 | 52 | init([]) -> 53 | Procs = [supervisor(elastic_http_supervisor)], 54 | {ok, {{one_for_one, 1, 5}, Procs}}. 55 | 56 | ref() -> 57 | {via, gproc, {n, l, ?MODULE}}. 58 | -------------------------------------------------------------------------------- /start-dev.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Copyright (c) 2012-2015 Peter Morgan 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | cd $(dirname $0) 17 | APP=$(bin/app) 18 | 19 | export ELASTICSEARCH_PORT_9200_TCP_ADDR=$(boot2docker ip) 20 | export ELASTICSEARCH_PORT_9200_TCP_PORT=9200 21 | 22 | exec erl \ 23 | +K true \ 24 | -boot start_sasl \ 25 | -config dev.config \ 26 | -name ${APP} \ 27 | -pa deps/*/ebin \ 28 | -pa ebin \ 29 | -s rb \ 30 | -s ${APP} 31 | 32 | --------------------------------------------------------------------------------