├── .gitignore ├── LICENSE ├── README.md ├── TODO.md ├── bin ├── kiries └── lein ├── config └── riemann.config ├── profiles.clj ├── project.clj └── src └── kiries ├── elastic.clj └── redis.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /logs 2 | /data 3 | /target 4 | /lib 5 | /classes 6 | /checkouts 7 | pom.xml 8 | pom.xml.asc 9 | *.jar 10 | *.class 11 | .lein-* 12 | .*.sw? 13 | riemann.log* 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | ElasticSearch and Kibana are also distrbuted The Apache License v2.0 2 | 3 | Riemann is distributed under the Eclipse Public License v1.0 4 | 5 | Kiries code is distributed under the Eclipse Public License v1.0 6 | 7 | 8 | Eclipse Public License - v 1.0 9 | 10 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE 11 | PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF 12 | THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 13 | 14 | 1. DEFINITIONS 15 | 16 | "Contribution" means: 17 | 18 | a) in the case of the initial Contributor, the initial code and 19 | documentation distributed under this Agreement, and 20 | 21 | b) in the case of each subsequent Contributor: 22 | 23 | i) changes to the Program, and 24 | 25 | ii) additions to the Program; 26 | 27 | where such changes and/or additions to the Program originate from and 28 | are distributed by that particular Contributor. A Contribution 29 | 'originates' from a Contributor if it was added to the Program by such 30 | Contributor itself or anyone acting on such Contributor's 31 | behalf. Contributions do not include additions to the Program which: 32 | (i) are separate modules of software distributed in conjunction with 33 | the Program under their own license agreement, and (ii) are not 34 | derivative works of the Program. 35 | 36 | "Contributor" means any person or entity that distributes the Program. 37 | 38 | "Licensed Patents" mean patent claims licensable by a Contributor 39 | which are necessarily infringed by the use or sale of its Contribution 40 | alone or when combined with the Program. 41 | 42 | "Program" means the Contributions distributed in accordance with this 43 | Agreement. 44 | 45 | "Recipient" means anyone who receives the Program under this 46 | Agreement, including all Contributors. 47 | 48 | 2. GRANT OF RIGHTS 49 | 50 | a) Subject to the terms of this Agreement, each Contributor hereby 51 | grants Recipient a non-exclusive, worldwide, royalty-free copyright 52 | license to reproduce, prepare derivative works of, publicly display, 53 | publicly perform, distribute and sublicense the Contribution of such 54 | Contributor, if any, and such derivative works, in source code and 55 | object code form. 56 | 57 | b) Subject to the terms of this Agreement, each Contributor hereby 58 | grants Recipient a non-exclusive, worldwide, royalty-free patent 59 | license under Licensed Patents to make, use, sell, offer to sell, 60 | import and otherwise transfer the Contribution of such Contributor, if 61 | any, in source code and object code form. This patent license shall 62 | apply to the combination of the Contribution and the Program if, at 63 | the time the Contribution is added by the Contributor, such addition 64 | of the Contribution causes such combination to be covered by the 65 | Licensed Patents. The patent license shall not apply to any other 66 | combinations which include the Contribution. No hardware per se is 67 | licensed hereunder. 68 | 69 | c) Recipient understands that although each Contributor grants the 70 | licenses to its Contributions set forth herein, no assurances are 71 | provided by any Contributor that the Program does not infringe the 72 | patent or other intellectual property rights of any other entity. Each 73 | Contributor disclaims any liability to Recipient for claims brought by 74 | any other entity based on infringement of intellectual property rights 75 | or otherwise. As a condition to exercising the rights and licenses 76 | granted hereunder, each Recipient hereby assumes sole responsibility 77 | to secure any other intellectual property rights needed, if any. For 78 | example, if a third party patent license is required to allow 79 | Recipient to distribute the Program, it is Recipient's responsibility 80 | to acquire that license before distributing the Program. 81 | 82 | d) Each Contributor represents that to its knowledge it has sufficient 83 | copyright rights in its Contribution, if any, to grant the copyright 84 | license set forth in this Agreement. 85 | 86 | 3. REQUIREMENTS 87 | 88 | A Contributor may choose to distribute the Program in object code form 89 | under its own license agreement, provided that: 90 | 91 | a) it complies with the terms and conditions of this Agreement; and 92 | 93 | b) its license agreement: 94 | 95 | i) effectively disclaims on behalf of all Contributors all warranties 96 | and conditions, express and implied, including warranties or 97 | conditions of title and non-infringement, and implied warranties or 98 | conditions of merchantability and fitness for a particular purpose; 99 | 100 | ii) effectively excludes on behalf of all Contributors all liability 101 | for damages, including direct, indirect, special, incidental and 102 | consequential damages, such as lost profits; 103 | 104 | iii) states that any provisions which differ from this Agreement are 105 | offered by that Contributor alone and not by any other party; and 106 | 107 | iv) states that source code for the Program is available from such 108 | Contributor, and informs licensees how to obtain it in a reasonable 109 | manner on or through a medium customarily used for software exchange. 110 | 111 | When the Program is made available in source code form: 112 | 113 | a) it must be made available under this Agreement; and 114 | 115 | b) a copy of this Agreement must be included with each copy of the Program. 116 | 117 | Contributors may not remove or alter any copyright notices contained 118 | within the Program. 119 | 120 | Each Contributor must identify itself as the originator of its 121 | Contribution, if any, in a manner that reasonably allows subsequent 122 | Recipients to identify the originator of the Contribution. 123 | 124 | 4. COMMERCIAL DISTRIBUTION 125 | 126 | Commercial distributors of software may accept certain 127 | responsibilities with respect to end users, business partners and the 128 | like. While this license is intended to facilitate the commercial use 129 | of the Program, the Contributor who includes the Program in a 130 | commercial product offering should do so in a manner which does not 131 | create potential liability for other Contributors. Therefore, if a 132 | Contributor includes the Program in a commercial product offering, 133 | such Contributor ("Commercial Contributor") hereby agrees to defend 134 | and indemnify every other Contributor ("Indemnified Contributor") 135 | against any losses, damages and costs (collectively "Losses") arising 136 | from claims, lawsuits and other legal actions brought by a third party 137 | against the Indemnified Contributor to the extent caused by the acts 138 | or omissions of such Commercial Contributor in connection with its 139 | distribution of the Program in a commercial product offering. The 140 | obligations in this section do not apply to any claims or Losses 141 | relating to any actual or alleged intellectual property 142 | infringement. In order to qualify, an Indemnified Contributor must: a) 143 | promptly notify the Commercial Contributor in writing of such claim, 144 | and b) allow the Commercial Contributor tocontrol, and cooperate with 145 | the Commercial Contributor in, the defense and any related settlement 146 | negotiations. The Indemnified Contributor may participate in any such 147 | claim at its own expense. 148 | 149 | For example, a Contributor might include the Program in a commercial 150 | product offering, Product X. That Contributor is then a Commercial 151 | Contributor. If that Commercial Contributor then makes performance 152 | claims, or offers warranties related to Product X, those performance 153 | claims and warranties are such Commercial Contributor's responsibility 154 | alone. Under this section, the Commercial Contributor would have to 155 | defend claims against the other Contributors related to those 156 | performance claims and warranties, and if a court requires any other 157 | Contributor to pay any damages as a result, the Commercial Contributor 158 | must pay those damages. 159 | 160 | 5. NO WARRANTY 161 | 162 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS 163 | PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 164 | KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY 165 | WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY 166 | OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely 167 | responsible for determining the appropriateness of using and 168 | distributing the Program and assumes all risks associated with its 169 | exercise of rights under this Agreement , including but not limited to 170 | the risks and costs of program errors, compliance with applicable 171 | laws, damage to or loss of data, programs or equipment, and 172 | unavailability or interruption of operations. 173 | 174 | 6. DISCLAIMER OF LIABILITY 175 | 176 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR 177 | ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, 178 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING 179 | WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF 180 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 181 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR 182 | DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED 183 | HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 184 | 185 | 7. GENERAL 186 | 187 | If any provision of this Agreement is invalid or unenforceable under 188 | applicable law, it shall not affect the validity or enforceability of 189 | the remainder of the terms of this Agreement, and without further 190 | action by the parties hereto, such provision shall be reformed to the 191 | minimum extent necessary to make such provision valid and enforceable. 192 | 193 | If Recipient institutes patent litigation against any entity 194 | (including a cross-claim or counterclaim in a lawsuit) alleging that 195 | the Program itself (excluding combinations of the Program with other 196 | software or hardware) infringes such Recipient's patent(s), then such 197 | Recipient's rights granted under Section 2(b) shall terminate as of 198 | the date such litigation is filed. 199 | 200 | All Recipient's rights under this Agreement shall terminate if it 201 | fails to comply with any of the material terms or conditions of this 202 | Agreement and does not cure such failure in a reasonable period of 203 | time after becoming aware of such noncompliance. If all Recipient's 204 | rights under this Agreement terminate, Recipient agrees to cease use 205 | and distribution of the Program as soon as reasonably 206 | practicable. However, Recipient's obligations under this Agreement and 207 | any licenses granted by Recipient relating to the Program shall 208 | continue and survive. 209 | 210 | Everyone is permitted to copy and distribute copies of this Agreement, 211 | but in order to avoid inconsistency the Agreement is copyrighted and 212 | may only be modified in the following manner. The Agreement Steward 213 | reserves the right to publish new versions (including revisions) of 214 | this Agreement from time to time. No one other than the Agreement 215 | Steward has the right to modify this Agreement. The Eclipse Foundation 216 | is the initial Agreement Steward. The Eclipse Foundation may assign 217 | the responsibility to serve as the Agreement Steward to a suitable 218 | separate entity. Each new version of the Agreement will be given a 219 | distinguishing version number. The Program (including Contributions) 220 | may always be distributed subject to the version of the Agreement 221 | under which it was received. In addition, after a new version of the 222 | Agreement is published, Contributor may elect to distribute the 223 | Program (including its Contributions) under the new version. Except as 224 | expressly stated in Sections 2(a) and 2(b) above, Recipient receives 225 | no rights or licenses to the intellectual property of any Contributor 226 | under this Agreement, whether expressly, by implication, estoppel or 227 | otherwise. All rights in the Program not expressly granted under this 228 | Agreement are reserved. 229 | 230 | This Agreement is governed by the laws of the State of Washington and 231 | the intellectual property laws of the United States of America. No 232 | party to this Agreement will bring a legal action under this Agreement 233 | more than one year after the cause of action arose. Each party waives 234 | its rights to a jury trial in any resulting litigation. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kiries - KIbana, RIemann and ElasticSearch 2 | 3 | We wanted a point-and-shoot real-time trend analysis dashboard for our 4 | Riemann-based cluster monitoring, as well as for other generic 5 | time-series data sources. So, we glued our favorite tools together 6 | with Clojure. 7 | 8 | We include the following versions of the third-party components 9 | 10 | * [Kibana](http://www.elasticsearch.org/overview/kibana/) (3.1.0) 11 | * [Riemann](http://riemann.io) (0.2.5) 12 | * [Elastic Search](http://www.elasticsearch.org) (1.1.1) 13 | * [Elastic HQ](http://elastichq.org) (ea630c8) 14 | 15 | Those projects have all done incredible work, and Kiries is just a 16 | little glue and some documentation tying them together. 17 | 18 | Any clojure dependencies are described in the `project.clj`. 19 | 20 | Check out the [User Guide](resources/htdocs/index.md) for more details. 21 | 22 | # Quickstart 23 | 24 | __WARNING__ : Kiries will open publicly accessible ports on your host. 25 | Read the installation instructions below, or live dangerously 26 | 27 | Unpack an archive or checkout the repo and run: 28 | 29 | bin/kiries 30 | 31 | And point your browser at 32 | 33 | # Installation 34 | 35 | Unpack the archive, or open up the git repository, and you will see a 36 | directory structure like this: 37 | 38 | * `config` -- configuration files for riemann and ElasticSearch 39 | * `resources/htdocs` -- Web documents, including Kibana, and it configuration in `kibana/config.js` 40 | * `src` -- Kiries src files 41 | * `logs` -- log files 42 | * `bin` -- helper shell scripts 43 | * `lib` -- libraries and jars 44 | 45 | __WARNING__ : Kiries will open several publicly accessible ports on your host. 46 | 47 | * 9090 -- Webserver, serving up Kibana and docs 48 | * 9200,9300 -- ElasticSearch HTTP and Native APIs 49 | * 5555 -- Riemann listeners (tcp and udp) 50 | 51 | Read the following and take precautions as you see fit. 52 | 53 | By default, Kiries will start up Riemann tcp and udp servers on port 54 | `5555`. It will index all events it receives into ElasticSearch. To 55 | customize this behavior, edit the `config/riemann.config` file. 56 | 57 | Kiries will also start up ElasticSearch listening on the default 58 | ports. ES will store it's data in the `data` directory (relative to 59 | where the java runtime was started from. To customize the ES 60 | behavior, edit the `config/elasticsearch.yml` file. If you customize 61 | the http host and port, be sure to update the Riemann and Kibana 62 | configurations. 63 | 64 | Kibana is a series of HTML and Javascript files served up from 65 | `htdocs` and it's configuration is in `htdocs/kibana/config.js`. We 66 | serve up Kibana using an internal webserver, whose default port is 67 | `9090` and will listen on all interfaces. 68 | 69 | # Usage 70 | 71 | bin/kiries # call with -? for cmd line args 72 | 73 | And point your browser at 74 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | 2 | # PRERELEASE 3 | 4 | # Documentation 5 | - Integration Guide (expand index.md) 6 | - Care and Feeding of our ES time-series data 7 | 8 | # Mappings 9 | - enable ttl in the default mapping 10 | - modify es-index to take a ttl argument 11 | - kiries-jvm mappings 12 | - kiries-http mappings 13 | 14 | # POSTRELEASE 15 | 16 | # Import 17 | - CSV HTTP POST 18 | - JSON HTTP POST 19 | 20 | # ElasticSearch proxy 21 | - rewrite indexes 22 | - access control 23 | - query and result rewriting 24 | 25 | # CSS Layout 26 | - menu that hides/shows from side of screen 27 | 28 | # Integrate Friends for auth control 29 | - Allow for basic http auth 30 | - Admin and User roles 31 | - Admin has unrestricted access to ES 32 | - Users have read only access, accept for 'kibana-int' index 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /bin/kiries: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ### realpath and dependencies copyright Michael Kropat, used under the MIT license. 4 | ### See https://github.com/mkropat/sh-realpath 5 | realpath() { 6 | canonicalize_path "$(resolve_symlinks "$1")" 7 | } 8 | 9 | resolve_symlinks() { 10 | _resolve_symlinks "$1" 11 | } 12 | 13 | _resolve_symlinks() { 14 | _assert_no_path_cycles "$@" || return 15 | 16 | local dir_context path 17 | if path=$(readlink -- "$1"); then 18 | dir_context=$(dirname -- "$1") 19 | _resolve_symlinks "$(_prepend_dir_context_if_necessary "$dir_context" "$path")" "$@" 20 | else 21 | printf '%s\n' "$1" 22 | fi 23 | } 24 | 25 | _prepend_dir_context_if_necessary() { 26 | if [ "$1" = . ]; then 27 | printf '%s\n' "$2" 28 | else 29 | _prepend_path_if_relative "$1" "$2" 30 | fi 31 | } 32 | 33 | _prepend_path_if_relative() { 34 | case "$2" in 35 | /* ) printf '%s\n' "$2" ;; 36 | * ) printf '%s\n' "$1/$2" ;; 37 | esac 38 | } 39 | 40 | _assert_no_path_cycles() { 41 | local target path 42 | 43 | target=$1 44 | shift 45 | 46 | for path in "$@"; do 47 | if [ "$path" = "$target" ]; then 48 | return 1 49 | fi 50 | done 51 | } 52 | 53 | canonicalize_path() { 54 | if [ -d "$1" ]; then 55 | _canonicalize_dir_path "$1" 56 | else 57 | _canonicalize_file_path "$1" 58 | fi 59 | } 60 | 61 | _canonicalize_dir_path() { 62 | (cd "$1" 2>/dev/null && pwd -P) 63 | } 64 | 65 | _canonicalize_file_path() { 66 | local dir file 67 | dir=$(dirname -- "$1") 68 | file=$(basename -- "$1") 69 | (cd "$dir" 2>/dev/null && printf '%s/%s\n' "$(pwd -P)" "$file") 70 | } 71 | ### End content copyright Michael Kropat 72 | 73 | shopt -s nullglob 74 | 75 | # Avoid conflicts with plugins in the user's ~/.lein/profiles.clj 76 | export LEIN_HOME=${KIRIES_LEIN_HOME:-"$HOME/.lein-kiries"} 77 | 78 | # Find location of executable; move to base 79 | bin_dir=$(realpath "${BASH_SOURCE[0]}" 2>/dev/null) 80 | source_dir=${bin_dir%/bin/*} 81 | cd "$source_dir" || exit 82 | 83 | # Following ant conventions, -X* is a JVM argument as opposed to a kiries 84 | # argument. JVM arguments that need to start with -X should thus be passed 85 | # with -X-X... 86 | jvm_args=( ) 87 | args=( ) 88 | for arg; do 89 | if [[ $arg = -X* ]]; then 90 | jvm_args+=( "${arg#-X}" ) 91 | else 92 | args+=( "$arg" ) 93 | fi 94 | done 95 | 96 | if (( ${#jvm_args[@]} )); then 97 | printf -v JAVA_TOOL_OPTIONS '%q ' "${jvm_args[@]}" 98 | export JAVA_TOOL_OPTIONS 99 | fi 100 | 101 | # If a source directory exists, honor it 102 | if [[ -e src ]]; then 103 | exec bin/lein run -- "${args[@]}" 104 | fi 105 | 106 | # Otherwise, use the newest available uberjar 107 | potential_jars=( target/kiries-*standalone.jar ) 108 | if (( ${#potential_jars[@]} == 0 )); then 109 | echo "ERROR: No source directory, and no uberjar found" >&2 110 | exit 1 111 | fi 112 | 113 | newest_jar=${potential_jars[0]} 114 | for j in "${potential_jars[@]}"; do 115 | [[ $j -nt $newest_jar ]] && newest_jar=$j 116 | done 117 | 118 | exec java -cp "$source_dir/resources:$newest_jar" riemann.bin "${args[@]}" 119 | -------------------------------------------------------------------------------- /bin/lein: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Ensure this file is executable via `chmod a+x lein`, then place it 4 | # somewhere on your $PATH, like ~/bin. The rest of Leiningen will be 5 | # installed upon first run into the ~/.lein/self-installs directory. 6 | 7 | export LEIN_VERSION="2.5.1" 8 | 9 | case $LEIN_VERSION in 10 | *SNAPSHOT) SNAPSHOT="YES" ;; 11 | *) SNAPSHOT="NO" ;; 12 | esac 13 | 14 | if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then 15 | delimiter=";" 16 | else 17 | delimiter=":" 18 | fi 19 | 20 | if [[ "$OSTYPE" == "cygwin" ]]; then 21 | cygwin=true 22 | else 23 | cygwin=false 24 | fi 25 | 26 | function make_native_path { 27 | # ensure we have native paths 28 | if $cygwin && [[ "$1" == /* ]]; then 29 | echo -n "$(cygpath -wp "$1")" 30 | elif [[ "$OSTYPE" == "msys" && "$1" == /?/* ]]; then 31 | echo -n "$(sh -c "(cd $1 2 /dev/null 76 | download_failed_message "$LEIN_URL" "$exit_code" 77 | exit 1 78 | fi 79 | } 80 | 81 | if [ `id -u` -eq 0 ] && [ "$LEIN_ROOT" = "" ]; then 82 | echo "WARNING: You're currently running as root; probably by accident." 83 | echo "Press control-C to abort or Enter to continue as root." 84 | echo "Set LEIN_ROOT to disable this warning." 85 | read _ 86 | fi 87 | 88 | NOT_FOUND=1 89 | ORIGINAL_PWD="$PWD" 90 | while [ ! -r "$PWD/project.clj" ] && [ "$PWD" != "/" ] && [ $NOT_FOUND -ne 0 ] 91 | do 92 | cd .. 93 | if [ "$(dirname "$PWD")" = "/" ]; then 94 | NOT_FOUND=0 95 | cd "$ORIGINAL_PWD" 96 | fi 97 | done 98 | 99 | export LEIN_HOME="${LEIN_HOME:-"$HOME/.lein"}" 100 | 101 | for f in "$LEIN_HOME/leinrc" ".leinrc"; do 102 | if [ -e "$f" ]; then 103 | source "$f" 104 | fi 105 | done 106 | 107 | if $cygwin; then 108 | export LEIN_HOME=`cygpath -w "$LEIN_HOME"` 109 | fi 110 | 111 | LEIN_JAR="$LEIN_HOME/self-installs/leiningen-$LEIN_VERSION-standalone.jar" 112 | 113 | # normalize $0 on certain BSDs 114 | if [ "$(dirname "$0")" = "." ]; then 115 | SCRIPT="$(which $(basename "$0"))" 116 | if [ -z "$SCRIPT" ]; then 117 | SCRIPT="$0" 118 | fi 119 | else 120 | SCRIPT="$0" 121 | fi 122 | 123 | # resolve symlinks to the script itself portably 124 | while [ -h "$SCRIPT" ] ; do 125 | ls=`ls -ld "$SCRIPT"` 126 | link=`expr "$ls" : '.*-> \(.*\)$'` 127 | if expr "$link" : '/.*' > /dev/null; then 128 | SCRIPT="$link" 129 | else 130 | SCRIPT="$(dirname "$SCRIPT"$)/$link" 131 | fi 132 | done 133 | 134 | BIN_DIR="$(dirname "$SCRIPT")" 135 | 136 | export LEIN_JVM_OPTS="${LEIN_JVM_OPTS-"-XX:+TieredCompilation -XX:TieredStopAtLevel=1"}" 137 | 138 | # This needs to be defined before we call HTTP_CLIENT below 139 | if [ "$HTTP_CLIENT" = "" ]; then 140 | if type -p curl >/dev/null 2>&1; then 141 | if [ "$https_proxy" != "" ]; then 142 | CURL_PROXY="-x $https_proxy" 143 | fi 144 | HTTP_CLIENT="curl $CURL_PROXY -f -L -o" 145 | else 146 | HTTP_CLIENT="wget -O" 147 | fi 148 | fi 149 | 150 | 151 | # When :eval-in :classloader we need more memory 152 | grep -E -q '^\s*:eval-in\s+:classloader\s*$' project.clj 2> /dev/null && \ 153 | export LEIN_JVM_OPTS="$LEIN_JVM_OPTS -Xms64m -Xmx512m" 154 | 155 | if [ -r "$BIN_DIR/../src/leiningen/version.clj" ]; then 156 | # Running from source checkout 157 | LEIN_DIR="$(dirname "$BIN_DIR")" 158 | 159 | # Need to use lein release to bootstrap the leiningen-core library (for aether) 160 | if [ ! -r "$LEIN_DIR/leiningen-core/.lein-bootstrap" ]; then 161 | echo "Leiningen is missing its dependencies." 162 | echo "Please run \"lein bootstrap\" in the leiningen-core/ directory" 163 | echo "with a stable release of Leiningen. See CONTRIBUTING.md for details." 164 | exit 1 165 | fi 166 | 167 | # If project.clj for lein or leiningen-core changes, we must recalculate 168 | LAST_PROJECT_CHECKSUM=$(cat "$LEIN_DIR/.lein-project-checksum" 2> /dev/null) 169 | PROJECT_CHECKSUM=$(sum "$LEIN_DIR/project.clj" "$LEIN_DIR/leiningen-core/project.clj") 170 | if [ "$PROJECT_CHECKSUM" != "$LAST_PROJECT_CHECKSUM" ]; then 171 | if [ -r "$LEIN_DIR/.lein-classpath" ]; then 172 | rm "$LEIN_DIR/.lein-classpath" 173 | fi 174 | fi 175 | 176 | # Use bin/lein to calculate its own classpath. 177 | if [ ! -r "$LEIN_DIR/.lein-classpath" ] && [ "$1" != "classpath" ]; then 178 | echo "Recalculating Leiningen's classpath." 179 | ORIG_PWD="$PWD" 180 | cd "$LEIN_DIR" 181 | 182 | LEIN_NO_USER_PROFILES=1 $0 classpath .lein-classpath 183 | sum "$LEIN_DIR/project.clj" "$LEIN_DIR/leiningen-core/project.clj" > \ 184 | .lein-project-checksum 185 | cd "$ORIG_PWD" 186 | fi 187 | 188 | mkdir -p "$LEIN_DIR/target/classes" 189 | export LEIN_JVM_OPTS="$LEIN_JVM_OPTS -Dclojure.compile.path=$LEIN_DIR/target/classes" 190 | add_path CLASSPATH "$LEIN_DIR/leiningen-core/src/" "$LEIN_DIR/leiningen-core/resources/" \ 191 | "$LEIN_DIR/test:$LEIN_DIR/target/classes" "$LEIN_DIR/src" ":$LEIN_DIR/resources" 192 | 193 | if [ -r "$LEIN_DIR/.lein-classpath" ]; then 194 | add_path CLASSPATH "$(cat "$LEIN_DIR/.lein-classpath" 2> /dev/null)" 195 | else 196 | add_path CLASSPATH "$(cat "$LEIN_DIR/leiningen-core/.lein-bootstrap" 2> /dev/null)" 197 | fi 198 | else # Not running from a checkout 199 | add_path CLASSPATH "$LEIN_JAR" 200 | 201 | BOOTCLASSPATH="-Xbootclasspath/a:$LEIN_JAR" 202 | 203 | if [ ! -r "$LEIN_JAR" -a "$1" != "self-install" ]; then 204 | self_install 205 | fi 206 | fi 207 | 208 | # TODO: explain what to do when Java is missing 209 | export JAVA_CMD="${JAVA_CMD:-"java"}" 210 | export LEIN_JAVA_CMD="${LEIN_JAVA_CMD:-$JAVA_CMD}" 211 | 212 | if [[ -z "${DRIP_INIT+x}" && "$(basename "$LEIN_JAVA_CMD")" == *drip* ]]; then 213 | export DRIP_INIT="$(printf -- '-e\n(require (quote leiningen.repl))')" 214 | export DRIP_INIT_CLASS="clojure.main" 215 | fi 216 | 217 | # Support $JAVA_OPTS for backwards-compatibility. 218 | export JVM_OPTS="${JVM_OPTS:-"$JAVA_OPTS"}" 219 | 220 | # Handle jline issue with cygwin not propagating OSTYPE through java subprocesses: https://github.com/jline/jline2/issues/62 221 | cygterm=false 222 | if $cygwin; then 223 | case "$TERM" in 224 | rxvt* | xterm* | vt*) cygterm=true ;; 225 | esac 226 | fi 227 | 228 | if $cygterm; then 229 | LEIN_JVM_OPTS="$LEIN_JVM_OPTS -Djline.terminal=jline.UnixTerminal" 230 | stty -icanon min 1 -echo > /dev/null 2>&1 231 | fi 232 | 233 | # TODO: investigate http://skife.org/java/unix/2011/06/20/really_executable_jars.html 234 | # If you're packaging this for a package manager (.deb, homebrew, etc) 235 | # you need to remove the self-install and upgrade functionality or see lein-pkg. 236 | if [ "$1" = "self-install" ]; then 237 | if [ -r "$BIN_DIR/../src/leiningen/version.clj" ]; then 238 | echo "Running self-install from a checkout is not supported." 239 | echo "See CONTRIBUTING.md for SNAPSHOT-specific build instructions." 240 | exit 1 241 | fi 242 | echo "Manual self-install is deprecated; it will run automatically when necessary." 243 | self_install 244 | elif [ "$1" = "upgrade" ] || [ "$1" = "downgrade" ]; then 245 | if [ "$LEIN_DIR" != "" ]; then 246 | echo "The upgrade task is not meant to be run from a checkout." 247 | exit 1 248 | fi 249 | if [ $SNAPSHOT = "YES" ]; then 250 | echo "The upgrade task is only meant for stable releases." 251 | echo "See the \"Hacking\" section of the README." 252 | exit 1 253 | fi 254 | if [ ! -w "$SCRIPT" ]; then 255 | echo "You do not have permission to upgrade the installation in $SCRIPT" 256 | exit 1 257 | else 258 | TARGET_VERSION="${2:-stable}" 259 | echo "The script at $SCRIPT will be upgraded to the latest $TARGET_VERSION version." 260 | echo -n "Do you want to continue [Y/n]? " 261 | read RESP 262 | case "$RESP" in 263 | y|Y|"") 264 | echo 265 | echo "Upgrading..." 266 | TARGET="/tmp/lein-$$-upgrade" 267 | if $cygwin; then 268 | TARGET=`cygpath -w $TARGET` 269 | fi 270 | LEIN_SCRIPT_URL="https://github.com/technomancy/leiningen/raw/$TARGET_VERSION/bin/lein" 271 | $HTTP_CLIENT "$TARGET" "$LEIN_SCRIPT_URL" 272 | if [ $? == 0 ]; then 273 | cmp -s "$TARGET" "$SCRIPT" 274 | if [ $? == 0 ]; then 275 | echo "Leiningen is already up-to-date." 276 | fi 277 | mv "$TARGET" "$SCRIPT" && chmod +x "$SCRIPT" 278 | exec "$SCRIPT" version 279 | else 280 | download_failed_message "$LEIN_SCRIPT_URL" 281 | fi;; 282 | *) 283 | echo "Aborted." 284 | exit 1;; 285 | esac 286 | fi 287 | else 288 | if $cygwin; then 289 | # When running on Cygwin, use Windows-style paths for java 290 | ORIGINAL_PWD=`cygpath -w "$ORIGINAL_PWD"` 291 | fi 292 | 293 | # apply context specific CLASSPATH entries 294 | if [ -f .lein-classpath ]; then 295 | add_path CLASSPATH "$(cat .lein-classpath)" 296 | fi 297 | 298 | if [ $DEBUG ]; then 299 | echo "Leiningen's classpath: $CLASSPATH" 300 | fi 301 | 302 | if [ -r .lein-fast-trampoline ]; then 303 | export LEIN_FAST_TRAMPOLINE='y' 304 | fi 305 | 306 | if [ "$LEIN_FAST_TRAMPOLINE" != "" ] && [ -r project.clj ]; then 307 | INPUTS="$@ $(cat project.clj) $LEIN_VERSION $(test -f "$LEIN_HOME/profiles.clj" && cat "$LEIN_HOME/profiles.clj")" 308 | export INPUT_CHECKSUM=$(echo $INPUTS | shasum - | cut -f 1 -d " ") 309 | # Just don't change :target-path in project.clj, mkay? 310 | TRAMPOLINE_FILE="target/trampolines/$INPUT_CHECKSUM" 311 | else 312 | if hash mktemp 2>/dev/null; then 313 | # Check if mktemp is available before using it 314 | TRAMPOLINE_FILE="$(mktemp /tmp/lein-trampoline-XXXXXXXXXXXXX)" 315 | else 316 | TRAMPOLINE_FILE="/tmp/lein-trampoline-$$" 317 | fi 318 | trap "rm -f $TRAMPOLINE_FILE" EXIT 319 | fi 320 | 321 | if $cygwin; then 322 | TRAMPOLINE_FILE=`cygpath -w $TRAMPOLINE_FILE` 323 | fi 324 | 325 | if [ "$INPUT_CHECKSUM" != "" ] && [ -r "$TRAMPOLINE_FILE" ]; then 326 | if [ $DEBUG ]; then 327 | echo "Fast trampoline with $TRAMPOLINE_FILE." 328 | fi 329 | exec sh -c "exec $(cat $TRAMPOLINE_FILE)" 330 | else 331 | export TRAMPOLINE_FILE 332 | "$LEIN_JAVA_CMD" \ 333 | "${BOOTCLASSPATH[@]}" \ 334 | -Dfile.encoding=UTF-8 \ 335 | -Dmaven.wagon.http.ssl.easy=false \ 336 | -Dmaven.wagon.rto=10000 \ 337 | $LEIN_JVM_OPTS \ 338 | -Dleiningen.original.pwd="$ORIGINAL_PWD" \ 339 | -Dleiningen.script="$SCRIPT" \ 340 | -classpath "$CLASSPATH" \ 341 | clojure.main -m leiningen.core.main "$@" 342 | 343 | EXIT_CODE=$? 344 | 345 | if $cygterm ; then 346 | stty icanon echo > /dev/null 2>&1 347 | fi 348 | 349 | ## TODO: [ -r "$TRAMPOLINE_FILE" ] may be redundant? A trampoline file 350 | ## is always generated these days. 351 | if [ -r "$TRAMPOLINE_FILE" ] && [ "$LEIN_TRAMPOLINE_WARMUP" = "" ]; then 352 | TRAMPOLINE="$(cat $TRAMPOLINE_FILE)" 353 | if [ "$INPUT_CHECKSUM" = "" ]; then 354 | rm $TRAMPOLINE_FILE 355 | fi 356 | if [ "$TRAMPOLINE" = "" ]; then 357 | exit $EXIT_CODE 358 | else 359 | exec sh -c "exec $TRAMPOLINE" 360 | fi 361 | else 362 | exit $EXIT_CODE 363 | fi 364 | fi 365 | fi 366 | -------------------------------------------------------------------------------- /config/riemann.config: -------------------------------------------------------------------------------- 1 | ; -*- mode: clojure; -*- 2 | 3 | (require '[kiries.elastic :as elastic]) 4 | 5 | (logging/init {:file "riemann.log"}) 6 | 7 | (let [host "0.0.0.0"] 8 | (tcp-server {:host host}) 9 | (udp-server {:host host}) 10 | (ws-server {:host host})) 11 | 12 | (def index-name "metrics") 13 | 14 | (def elastic-addr (or (System/getenv "ELASTICSEARCH_URL") 15 | "http://localhost:9200")) 16 | 17 | (def connection (elastic/es-connect elastic-addr)) 18 | 19 | (periodically-expire 5) 20 | 21 | (let [index (index)] 22 | (streams 23 | (default :ttl 60 24 | index 25 | (expired 26 | (fn [event] (info "expired" event)))) 27 | (where (not (service #"^riemann")) 28 | (rollup 1 5 (elastic/es-index connection "metric" 29 | :timestamping :day 30 | :index index-name))))) 31 | -------------------------------------------------------------------------------- /profiles.clj: -------------------------------------------------------------------------------- 1 | {:dev {:dependencies [[org.clojure/tools.nrepl "0.2.10"]]}} 2 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject kiries "1.0.0-SNAPSHOT" 2 | :description "A bundled deployment of Riemann, and ElasticSearch" 3 | :dependencies [[clj-json "0.5.3"] 4 | [com.taoensso/carmine "2.14.0"] 5 | [clojurewerkz/elastisch "2.2.2" 6 | :exclusions [clj-http]] 7 | [com.taoensso/carmine "2.16.0"] 8 | [org.clojure/clojure "1.8.0"] 9 | [cheshire "5.7.0"] 10 | [org.clojure/tools.reader "1.0.3"] 11 | [org.clojars.jcsims/riemann "0.2.17"]] 12 | :repositories [["redhat" "https://maven.repository.redhat.com/ga"]] 13 | :main riemann.bin 14 | :profiles {:uberjar {:aot :all,}} 15 | :aliases {"server" ["trampoline" "run" "-m" "riemann.bin" "config/riemann.config"]}) 16 | -------------------------------------------------------------------------------- /src/kiries/elastic.clj: -------------------------------------------------------------------------------- 1 | (ns kiries.elastic 2 | (:require [cheshire.core :as json] 3 | [clj-http.client :as http] 4 | [clj-time.format] 5 | [clj-time.core] 6 | [clj-time.coerce] 7 | [clojure.edn :as edn] 8 | [clojure.java.io :as io] 9 | [clojure.string :as string] 10 | [clojure.tools.logging :as log] 11 | [clojurewerkz.elastisch.rest.bulk :as eb] 12 | [clojurewerkz.elastisch.rest :as esr] 13 | [riemann.streams :as streams])) 14 | 15 | (defn make-index-timestamper [index period] 16 | (let [formatter (clj-time.format/formatter (str "'" index "'" 17 | (cond 18 | (= period :day) 19 | "-YYYY.MM.dd" 20 | (= period :hour) 21 | "-YYYY.MM.dd.HH" 22 | (= period :week) 23 | "-YYYY.MM.dd.ww" 24 | (= period :month) 25 | "-YYYY.MM" 26 | (= period :year) 27 | "-YYYY")))] 28 | (fn [date] 29 | (clj-time.format/unparse formatter date)))) 30 | 31 | (def get-elasticsearch-version 32 | (memoize (fn [url] 33 | (-> (http/get url) 34 | :body 35 | (json/decode keyword) 36 | (get-in [:version :number]) 37 | (string/split #"\D+") 38 | (->> (map #(Integer/parseInt %))))))) 39 | 40 | (defn- get-major-version [connection] 41 | (first (get-elasticsearch-version (:uri connection)))) 42 | 43 | (def ^{:private true} format-iso8601 44 | (clj-time.format/with-zone (clj-time.format/formatters :date-time) 45 | clj-time.core/utc)) 46 | 47 | (defn ^{:private true} iso8601 [event-s] 48 | (clj-time.format/unparse format-iso8601 49 | (clj-time.coerce/from-long (* 1000 event-s)))) 50 | 51 | (defn ^{:private true} safe-iso8601 [event-s] 52 | (try (iso8601 event-s) 53 | (catch Exception e 54 | (log/warn "Unable to parse iso8601 input: " event-s) 55 | (clj-time.format/unparse format-iso8601 (clj-time.core/now))))) 56 | 57 | (defn ^{:private true} stashify-timestamp [event] 58 | (-> (if-not (get event "@timestamp") 59 | (assoc event "@timestamp" 60 | (if-let [isotime (:isotime event)] 61 | isotime 62 | (if-let [time (:time event)] 63 | (safe-iso8601 (long time)) 64 | (clj-time.format/unparse format-iso8601 (clj-time.core/now))))) 65 | event) 66 | (dissoc :isotime :time :ttl))) 67 | 68 | (defn ^{:private true} edn-safe-read [v] 69 | (try 70 | (edn/read-string v) 71 | (catch Exception e 72 | (log/warn "Unable to read supposed EDN form with value: " v) 73 | v))) 74 | 75 | (defn ^{:private true} massage-event [event] 76 | (into {} 77 | (for [[k v] event 78 | :when v] 79 | (cond 80 | (= (name k) "_id") [k v] 81 | (.startsWith (name k) "_") 82 | [(.substring (name k) 1) (edn-safe-read v)] 83 | :else 84 | [k v])))) 85 | 86 | (defn ^{:private true} elastic-event [event massage] 87 | (let [e (-> event 88 | stashify-timestamp)] 89 | (if massage 90 | (massage-event e) 91 | e))) 92 | 93 | (defn ^{:private true} riemann-to-elasticsearch [events massage] 94 | (->> [events] 95 | flatten 96 | (remove streams/expired?) 97 | (map #(elastic-event % massage)))) 98 | 99 | (defn es-connect 100 | "Connects to the ElasticSearch node. 101 | 102 | The first optional argument is a url for the node, which defaults to 103 | `http://localhost:9200`. The second optional arg is a map of 104 | options to be passed to clj-http. 105 | 106 | This must be called before any es-* functions can be used." 107 | ([] 108 | (es-connect "http://localhost:9200")) 109 | ([host] 110 | (es-connect host nil)) 111 | ([host http-opts] 112 | (esr/connect host 113 | (merge {:content-type "application/x-ndjson"} 114 | http-opts)))) 115 | 116 | (defn- parse-1x-bulk-results [res] 117 | {:total (count (:items res)) 118 | :succ (filter #(= 201 (get-in % [:create :status])) (:items res)) 119 | :failed (filter :error (:items res))}) 120 | 121 | (defn- parse-5x-bulk-results [res] 122 | {:total (count (:items res)) 123 | :succ (filter #(= 201 (get-in % [:index :status])) (:items res)) 124 | :failed (remove #(get-in % [:index "created"]) (:items res))}) 125 | 126 | (defn- parse-6x-bulk-results [res] 127 | {:total (count (:items res)) 128 | :succ (filter #(= 201 (get-in % [:index :status])) (:items res)) 129 | :failed (remove #(= "created" (get-in % [:index :result])) (:items res))}) 130 | 131 | (defn- parse-bulk-results [major-version res] 132 | (case major-version 133 | (1 2) (parse-1x-bulk-results res) 134 | 5 (parse-5x-bulk-results res) 135 | 6 (parse-6x-bulk-results res))) 136 | 137 | (defn es-index 138 | "A function which takes a sequence of events, and indexes them in 139 | ElasticSearch. It will set the `_type` field of the event to the 140 | value of `doc-type`, which tells ES which mapping to use for the 141 | event. 142 | 143 | The :index argument defaults to \"logstash\" for Kibana 144 | compatability. It's the root ES index name that this event will be 145 | indexed within. 146 | 147 | The :timestamping argument, default to :day and it controls the time 148 | range component of the index name. Acceptable values 149 | are :hour, :day, :week, :month and :year. 150 | 151 | Events will be massages to conform to Kibana expections. This means 152 | that the `@timestamp` field will be set if not found, based on the 153 | `time` field of the event. The `ttl` field will be removed, as it's 154 | internal to Riemann. Lastly, any fields starting with an `_` will 155 | have their value parsed as EDN. 156 | " 157 | [connection doc-type & {:keys [index timestamping massage] 158 | :or {index "logstash" 159 | massage true 160 | timestamping :day}}] 161 | (let [index-namer (make-index-timestamper index timestamping)] 162 | (fn [events] 163 | (let [esets (group-by (fn [e] 164 | (index-namer 165 | (clj-time.format/parse format-iso8601 166 | (get e "@timestamp")))) 167 | (riemann-to-elasticsearch events massage))] 168 | (doseq [index (keys esets)] 169 | (let [raw (get esets index) 170 | bulk-create-items 171 | (interleave (map #(if-let [id (get % "_id")] 172 | {:create {:_type doc-type :_id id}} 173 | {:index {:_type doc-type}}) 174 | raw) 175 | raw)] 176 | (when (seq bulk-create-items) 177 | (try 178 | (let [res (eb/bulk-with-index connection index bulk-create-items) 179 | {:keys [total succ failed]} 180 | (parse-bulk-results (get-major-version connection) res)] 181 | (log/info (str "elasticized " total "/" (count succ) "/" (- total (count succ)) 182 | " (total/succ/fail) items to index " index " in " (:took res) " ms")) 183 | (log/debug "Failed: " failed)) 184 | (catch Exception e 185 | (log/error "Unable to bulk index:" e)))))))))) 186 | -------------------------------------------------------------------------------- /src/kiries/redis.clj: -------------------------------------------------------------------------------- 1 | (ns kiries.redis 2 | (:require [clojure.tools.logging :as log] 3 | [taoensso.carmine :as car] 4 | [clj-json.core :as json])) 5 | 6 | (defn key-lengths 7 | "Given a redis connection and a pattern ('*' wildcard), return a map 8 | of key name to key length." 9 | [conn patt] 10 | (let [keys (car/wcar conn (car/keys patt))] 11 | (zipmap keys (car/wcar conn (doseq [key keys] (car/llen key)))))) 12 | 13 | (defn spublish 14 | "Returns a function which takes a sequence of events or other 15 | values, and publishes them as JSON, to the specified redis server 16 | and key. Suitable for using inside of rollup." 17 | [conn key] 18 | (fn [events] 19 | (let [jevents 20 | (remove nil? 21 | (map 22 | (fn [e] 23 | (try (json/generate-string e) 24 | (catch Exception err 25 | (log/error "Could not coerce event to json (" e "):" err) 26 | nil))) 27 | events))] 28 | (doseq [je jevents] 29 | (try 30 | (car/wcar conn 31 | (car/publish key je)) 32 | (catch Exception e 33 | (log/warn "Unable to publish" key "to" conn ":" e))))))) 34 | 35 | (defn publish 36 | "Returns a function which publishes a single event as JSON, to the 37 | spcified redis server and key." 38 | [conn key] 39 | (let [f (spublish conn key)] 40 | (fn [e] 41 | (f [e])))) 42 | --------------------------------------------------------------------------------