├── LICENSE ├── README.md ├── examples └── config │ └── example.config ├── project.clj ├── resources └── riemann_plugin │ └── riemann_elastic │ └── meta.edn └── src └── riemann └── elastic.clj /LICENSE: -------------------------------------------------------------------------------- 1 | Riemann Elastic is distributed under the Eclipse Public License v1.0 2 | 3 | Eclipse Public License - v 1.0 4 | 5 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE 6 | PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF 7 | THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 8 | 9 | 1. DEFINITIONS 10 | 11 | "Contribution" means: 12 | 13 | a) in the case of the initial Contributor, the initial code and 14 | documentation distributed under this Agreement, and 15 | 16 | b) in the case of each subsequent Contributor: 17 | 18 | i) changes to the Program, and 19 | 20 | ii) additions to the Program; 21 | 22 | where such changes and/or additions to the Program originate from and 23 | are distributed by that particular Contributor. A Contribution 24 | 'originates' from a Contributor if it was added to the Program by such 25 | Contributor itself or anyone acting on such Contributor's 26 | behalf. Contributions do not include additions to the Program which: 27 | (i) are separate modules of software distributed in conjunction with 28 | the Program under their own license agreement, and (ii) are not 29 | derivative works of the Program. 30 | 31 | "Contributor" means any person or entity that distributes the Program. 32 | 33 | "Licensed Patents" mean patent claims licensable by a Contributor 34 | which are necessarily infringed by the use or sale of its Contribution 35 | alone or when combined with the Program. 36 | 37 | "Program" means the Contributions distributed in accordance with this 38 | Agreement. 39 | 40 | "Recipient" means anyone who receives the Program under this 41 | Agreement, including all Contributors. 42 | 43 | 2. GRANT OF RIGHTS 44 | 45 | a) Subject to the terms of this Agreement, each Contributor hereby 46 | grants Recipient a non-exclusive, worldwide, royalty-free copyright 47 | license to reproduce, prepare derivative works of, publicly display, 48 | publicly perform, distribute and sublicense the Contribution of such 49 | Contributor, if any, and such derivative works, in source code and 50 | object code form. 51 | 52 | b) Subject to the terms of this Agreement, each Contributor hereby 53 | grants Recipient a non-exclusive, worldwide, royalty-free patent 54 | license under Licensed Patents to make, use, sell, offer to sell, 55 | import and otherwise transfer the Contribution of such Contributor, if 56 | any, in source code and object code form. This patent license shall 57 | apply to the combination of the Contribution and the Program if, at 58 | the time the Contribution is added by the Contributor, such addition 59 | of the Contribution causes such combination to be covered by the 60 | Licensed Patents. The patent license shall not apply to any other 61 | combinations which include the Contribution. No hardware per se is 62 | licensed hereunder. 63 | 64 | c) Recipient understands that although each Contributor grants the 65 | licenses to its Contributions set forth herein, no assurances are 66 | provided by any Contributor that the Program does not infringe the 67 | patent or other intellectual property rights of any other entity. Each 68 | Contributor disclaims any liability to Recipient for claims brought by 69 | any other entity based on infringement of intellectual property rights 70 | or otherwise. As a condition to exercising the rights and licenses 71 | granted hereunder, each Recipient hereby assumes sole responsibility 72 | to secure any other intellectual property rights needed, if any. For 73 | example, if a third party patent license is required to allow 74 | Recipient to distribute the Program, it is Recipient's responsibility 75 | to acquire that license before distributing the Program. 76 | 77 | d) Each Contributor represents that to its knowledge it has sufficient 78 | copyright rights in its Contribution, if any, to grant the copyright 79 | license set forth in this Agreement. 80 | 81 | 3. REQUIREMENTS 82 | 83 | A Contributor may choose to distribute the Program in object code form 84 | under its own license agreement, provided that: 85 | 86 | a) it complies with the terms and conditions of this Agreement; and 87 | 88 | b) its license agreement: 89 | 90 | i) effectively disclaims on behalf of all Contributors all warranties 91 | and conditions, express and implied, including warranties or 92 | conditions of title and non-infringement, and implied warranties or 93 | conditions of merchantability and fitness for a particular purpose; 94 | 95 | ii) effectively excludes on behalf of all Contributors all liability 96 | for damages, including direct, indirect, special, incidental and 97 | consequential damages, such as lost profits; 98 | 99 | iii) states that any provisions which differ from this Agreement are 100 | offered by that Contributor alone and not by any other party; and 101 | 102 | iv) states that source code for the Program is available from such 103 | Contributor, and informs licensees how to obtain it in a reasonable 104 | manner on or through a medium customarily used for software exchange. 105 | 106 | When the Program is made available in source code form: 107 | 108 | a) it must be made available under this Agreement; and 109 | 110 | b) a copy of this Agreement must be included with each copy of the Program. 111 | 112 | Contributors may not remove or alter any copyright notices contained 113 | within the Program. 114 | 115 | Each Contributor must identify itself as the originator of its 116 | Contribution, if any, in a manner that reasonably allows subsequent 117 | Recipients to identify the originator of the Contribution. 118 | 119 | 4. COMMERCIAL DISTRIBUTION 120 | 121 | Commercial distributors of software may accept certain 122 | responsibilities with respect to end users, business partners and the 123 | like. While this license is intended to facilitate the commercial use 124 | of the Program, the Contributor who includes the Program in a 125 | commercial product offering should do so in a manner which does not 126 | create potential liability for other Contributors. Therefore, if a 127 | Contributor includes the Program in a commercial product offering, 128 | such Contributor ("Commercial Contributor") hereby agrees to defend 129 | and indemnify every other Contributor ("Indemnified Contributor") 130 | against any losses, damages and costs (collectively "Losses") arising 131 | from claims, lawsuits and other legal actions brought by a third party 132 | against the Indemnified Contributor to the extent caused by the acts 133 | or omissions of such Commercial Contributor in connection with its 134 | distribution of the Program in a commercial product offering. The 135 | obligations in this section do not apply to any claims or Losses 136 | relating to any actual or alleged intellectual property 137 | infringement. In order to qualify, an Indemnified Contributor must: a) 138 | promptly notify the Commercial Contributor in writing of such claim, 139 | and b) allow the Commercial Contributor tocontrol, and cooperate with 140 | the Commercial Contributor in, the defense and any related settlement 141 | negotiations. The Indemnified Contributor may participate in any such 142 | claim at its own expense. 143 | 144 | For example, a Contributor might include the Program in a commercial 145 | product offering, Product X. That Contributor is then a Commercial 146 | Contributor. If that Commercial Contributor then makes performance 147 | claims, or offers warranties related to Product X, those performance 148 | claims and warranties are such Commercial Contributor's responsibility 149 | alone. Under this section, the Commercial Contributor would have to 150 | defend claims against the other Contributors related to those 151 | performance claims and warranties, and if a court requires any other 152 | Contributor to pay any damages as a result, the Commercial Contributor 153 | must pay those damages. 154 | 155 | 5. NO WARRANTY 156 | 157 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS 158 | PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 159 | KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY 160 | WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY 161 | OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely 162 | responsible for determining the appropriateness of using and 163 | distributing the Program and assumes all risks associated with its 164 | exercise of rights under this Agreement , including but not limited to 165 | the risks and costs of program errors, compliance with applicable 166 | laws, damage to or loss of data, programs or equipment, and 167 | unavailability or interruption of operations. 168 | 169 | 6. DISCLAIMER OF LIABILITY 170 | 171 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR 172 | ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, 173 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING 174 | WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF 175 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 176 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR 177 | DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED 178 | HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 179 | 180 | 7. GENERAL 181 | 182 | If any provision of this Agreement is invalid or unenforceable under 183 | applicable law, it shall not affect the validity or enforceability of 184 | the remainder of the terms of this Agreement, and without further 185 | action by the parties hereto, such provision shall be reformed to the 186 | minimum extent necessary to make such provision valid and enforceable. 187 | 188 | If Recipient institutes patent litigation against any entity 189 | (including a cross-claim or counterclaim in a lawsuit) alleging that 190 | the Program itself (excluding combinations of the Program with other 191 | software or hardware) infringes such Recipient's patent(s), then such 192 | Recipient's rights granted under Section 2(b) shall terminate as of 193 | the date such litigation is filed. 194 | 195 | All Recipient's rights under this Agreement shall terminate if it 196 | fails to comply with any of the material terms or conditions of this 197 | Agreement and does not cure such failure in a reasonable period of 198 | time after becoming aware of such noncompliance. If all Recipient's 199 | rights under this Agreement terminate, Recipient agrees to cease use 200 | and distribution of the Program as soon as reasonably 201 | practicable. However, Recipient's obligations under this Agreement and 202 | any licenses granted by Recipient relating to the Program shall 203 | continue and survive. 204 | 205 | Everyone is permitted to copy and distribute copies of this Agreement, 206 | but in order to avoid inconsistency the Agreement is copyrighted and 207 | may only be modified in the following manner. The Agreement Steward 208 | reserves the right to publish new versions (including revisions) of 209 | this Agreement from time to time. No one other than the Agreement 210 | Steward has the right to modify this Agreement. The Eclipse Foundation 211 | is the initial Agreement Steward. The Eclipse Foundation may assign 212 | the responsibility to serve as the Agreement Steward to a suitable 213 | separate entity. Each new version of the Agreement will be given a 214 | distinguishing version number. The Program (including Contributions) 215 | may always be distributed subject to the version of the Agreement 216 | under which it was received. In addition, after a new version of the 217 | Agreement is published, Contributor may elect to distribute the 218 | Program (including its Contributions) under the new version. Except as 219 | expressly stated in Sections 2(a) and 2(b) above, Recipient receives 220 | no rights or licenses to the intellectual property of any Contributor 221 | under this Agreement, whether expressly, by implication, estoppel or 222 | otherwise. All rights in the Program not expressly granted under this 223 | Agreement are reserved. 224 | 225 | This Agreement is governed by the laws of the State of Washington and 226 | the intellectual property laws of the United States of America. No 227 | party to this Agreement will bring a legal action under this Agreement 228 | more than one year after the cause of action arose. Each party waives 229 | its rights to a jury trial in any resulting litigation. 230 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Riemann Elastic 2 | 3 | Riemann Elastic is a Riemann plugin for indexing Riemann events in Elastic Search. 4 | 5 | Lein 6 | [org.tinnitus.dl/elastic-riemann "0.1.0-SNAPSHOT"] 7 | 8 | ## Installation from scratch. 9 | 10 | After cloning the repo, you can build the plugin using lein 11 | 12 | ```clj 13 | $ lein uberjar 14 | ``` 15 | 16 | This will create a plugin jar named `elastic-riemann-0.1.0-SNAPSHOT-standalone.jar` 17 | 18 | Now, you can start Riemann using the following command. 19 | 20 | ``` 21 | java -server -cp ${RIEMANN_HOME}/lib/riemann.jar:${RIEMANN_ELASTIC}/target/elastic-riemann-0.1.0-SNAPSHOT-standalone.jar clojure.main -m riemann.bin ${RIEMANN_HOME}/etc/riemann.config 22 | ``` 23 | 24 | This will start Riemann with the plugin available. You can load the plugin in riemann.config with the following line: 25 | 26 | ```clj 27 | (require '[riemann.elastic :as elastic]) 28 | ``` 29 | 30 | You can start pushing events to elastic search with the elastic/es-index function. Here's an snippet of a Riemann config that does the job. 31 | 32 | ```clk 33 | (def myindex (default :ttl 300 (index))) 34 | 35 | (def elastic-url "http://localhost:9200") 36 | (def elastic-conn (when (seq elastic-url) (elastic/es-connect elastic-url))) 37 | 38 | (def standard-sink 39 | (let [sinks (if elastic-conn 40 | [myindex (async-queue! :elastic-search 41 | {:queue-size 10000} 42 | (batch 200 10 (elastic/es-index "riemann-elastic")))] 43 | [myindex])] 44 | (fn [e] 45 | (call-rescue e sinks)))) 46 | 47 | ; send everything to stdout and standard-sink e.g. elastic and the index. 48 | (streams prn standard-sink) 49 | ``` 50 | 51 | ## Elastic and Kibana 52 | With the riemann.config above, events should be persisted to Elastic Search. You can see the last 10 events indexed with the following curl command. 53 | 54 | ``` 55 | $ curl http://localhost:9200/_search?query=* 56 | ``` 57 | 58 | If that works, Kibana will work too. You can find Kibana here: 59 | - http://www.elasticsearch.org/overview/kibana/ 60 | # Credit 61 | This project started as a fork of Kiries. 62 | - https://github.com/threatgrid/kiries 63 | 64 | Kiries bundles Riemann, Elastic Search and Kibana in single repository. It's awesome and really useful but proved difficult to integrate in my environment. I ripped out the Riemann Elastic bits and made it into a Riemann plugin. 65 | 66 | -------------------------------------------------------------------------------- /examples/config/example.config: -------------------------------------------------------------------------------- 1 | (require '[riemann.elastic :as elastic]) 2 | ; vim: filetype=clojure 3 | 4 | ; In this config, we'll start pushing data from Riemann into ElasticSearch 5 | ; This will give you the ability to store historical time series data 6 | ; for selected streams. 7 | 8 | ; We'll use the Riemann Elastic Search plugin from tnn1t1s 9 | ; https://github.com/tnn1t1s/riemann-elastic 10 | (def elastic-url "http://localhost:9200") 11 | (def elastic-conn (when (seq elastic-url) (elastic/es-connect elastic-url))) 12 | 13 | ; A generic indexing function with default ttl 14 | (def default-index (default :ttl 300 (index))) 15 | 16 | ; define a 'sink' that writes to elastic search 17 | (def elastic-sink 18 | "write events to elastic search and the Riemann index" 19 | (let [sinks [(async-queue! :elastic-search 20 | {:queue-size 10000} 21 | (batch 200 10 (elastic/es-index "riemann-elastic")))]] 22 | (fn [e] 23 | (call-rescue e sinks)))) 24 | 25 | ; Listen on the local interface over TCP (5555), UDP (5555), and websockets 26 | ; (5556) 27 | (logging/init {:file "riemann.log"}) 28 | (let [host "127.0.0.1"] 29 | (tcp-server {:host host}) 30 | (udp-server {:host host}) 31 | (ws-server {:host host})) 32 | 33 | ; Expire old events from the index every 5 seconds. 34 | (periodically-expire 5) 35 | 36 | (let [index (index)] 37 | (streams 38 | ; send 'foo' to the default index and elastic 39 | (with :service "foo" default-index elastic-sink) 40 | ; send 'bar' to the default index only 41 | (with :service "bar" default-index))) 42 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject org.tinnitus.dl/elastic-riemann "0.1.1-SNAPSHOT" 2 | :description "A plugin for using ElasticSearch and Kibana with Riemann" 3 | :dependencies [[org.clojure/clojure "1.5.1"] 4 | [org.clojure/tools.cli "0.3.5"] 5 | [cheshire "5.6.1"] 6 | [clj-logging-config "1.9.12"] 7 | [riemann "0.2.11" :exclusions [joda-time org.slf4j/slf4j-log4j12]] 8 | [clojurewerkz/elastisch "1.2.0"] 9 | [org.elasticsearch/elasticsearch "2.3.2"] 10 | ] 11 | :main elastic-riemann.core) 12 | 13 | -------------------------------------------------------------------------------- /resources/riemann_plugin/riemann_elastic/meta.edn: -------------------------------------------------------------------------------- 1 | {:plugin "riemann-elastic" 2 | :title "A plugin to allow riemann to output to elastic search" 3 | :git-repo "https://github.com/tnn1t1s/riemann-elastic" 4 | :require riemann.elastic} 5 | -------------------------------------------------------------------------------- /src/riemann/elastic.clj: -------------------------------------------------------------------------------- 1 | (ns riemann.elastic 2 | (:use [clojure.tools.logging :only (info error debug warn)]) 3 | (:require [cheshire.core :as json] 4 | [clj-time.format] 5 | [clj-time.core] 6 | [clj-time.coerce] 7 | [clojure.edn :as edn] 8 | [clojure.java.io :as io] 9 | [clojurewerkz.elastisch.rest.bulk :as eb] 10 | [clojurewerkz.elastisch.rest :as esr] 11 | [riemann.streams :as streams])) 12 | 13 | (defn make-index-timestamper [index period] 14 | (let [formatter (clj-time.format/formatter (str "'" index "'" 15 | (cond 16 | (= period :day) 17 | "-YYYY.MM.dd" 18 | (= period :hour) 19 | "-YYYY.MM.dd.HH" 20 | (= period :week) 21 | "-YYYY.MM.dd.ww" 22 | (= period :month) 23 | "-YYYY.MM" 24 | (= period :year) 25 | "-YYYY")))] 26 | (fn [date] 27 | (clj-time.format/unparse formatter date)))) 28 | 29 | (def ^{:private true} format-iso8601 30 | (clj-time.format/with-zone (clj-time.format/formatters :date-time-no-ms) 31 | clj-time.core/utc)) 32 | 33 | (defn ^{:private true} iso8601 [event-s] 34 | (clj-time.format/unparse format-iso8601 35 | (clj-time.coerce/from-long (* 1000 event-s)))) 36 | 37 | (defn ^{:private true} safe-iso8601 [event-s] 38 | (try (iso8601 event-s) 39 | (catch Exception e 40 | (warn "Unable to parse iso8601 input: " event-s) 41 | (clj-time.format/unparse format-iso8601 (clj-time.core/now))))) 42 | 43 | (defn ^{:private true} stashify-timestamp [event] 44 | (-> (if-not (get event "@timestamp") 45 | (let [time (:time event)] 46 | (assoc event "@timestamp" (safe-iso8601 (long time)))) 47 | event) 48 | (dissoc :time) 49 | (dissoc :ttl))) 50 | 51 | (defn ^{:private true} edn-safe-read [v] 52 | (try 53 | (edn/read-string v) 54 | (catch Exception e 55 | (warn "Unable to read supposed EDN form with value: " v) 56 | v))) 57 | 58 | (defn ^{:private true} massage-event [event] 59 | (into {} 60 | (for [[k v] event 61 | :when v] 62 | (cond 63 | (= (name k) "_id") [k v] 64 | (.startsWith (name k) "_") [(subs (name k) 1) (edn-safe-read v)] 65 | :else [k v])))) 66 | 67 | (defn ^{:private true} elastic-event [event massage] 68 | (let [e (-> event 69 | stashify-timestamp)] 70 | (if massage 71 | (massage-event e) 72 | e))) 73 | 74 | (defn ^{:private true} riemann-to-elasticsearch [events massage] 75 | (->> [events] 76 | flatten 77 | (remove streams/expired?) 78 | (map #(elastic-event % massage)))) 79 | 80 | (defn es-connect 81 | "Connects to the ElasticSearch node. The optional argument is a url 82 | for the node, which defaults to `http://localhost:9200`. This must 83 | be called before any es-* functions can be used." 84 | [& argv] 85 | (esr/connect! (or (first argv) "http://localhost:9200"))) 86 | 87 | (defn es-index 88 | "A function which takes a sequence of events, and indexes them in 89 | ElasticSearch. It will set the `_type` field of the event to the 90 | value of `doc-type`, which tells ES which mapping to use for the 91 | event. 92 | 93 | The :index argument defaults to \"logstash\" for Kibana 94 | compatability. It's the root ES index name that this event will be 95 | indexed within. 96 | 97 | The :timestamping argument, default to :day and it controls the time 98 | range component of the index name. Acceptable values 99 | are :hour, :day, :week, :month and :year. 100 | 101 | Events will be massages to conform to Kubana expections. This means 102 | that the `@timestamp` field will be set if not found, based on the 103 | `time` field of the event. The `ttl` field will be removed, as it's 104 | internal to Riemann. Lastly, any fields starting with an `_` will 105 | have their value parsed as EDN. 106 | " 107 | [doc-type & {:keys [index timestamping massage] 108 | :or {index "logstash" 109 | massage true 110 | timestamping :day}}] 111 | (let [index-namer (make-index-timestamper index timestamping)] 112 | (fn [events] 113 | (let [esets (group-by (fn [e] 114 | (index-namer 115 | (clj-time.format/parse format-iso8601 116 | (get e "@timestamp")))) 117 | (riemann-to-elasticsearch events massage))] 118 | (doseq [index (keys esets)] 119 | (let [raw (get esets index) 120 | bulk-create-items 121 | (interleave (map #(if-let [id (get % "_id")] 122 | {:create {:_type doc-type :_id id}} 123 | {:create {:_type doc-type}} 124 | ) 125 | raw) 126 | raw)] 127 | (when (seq bulk-create-items) 128 | (try 129 | (let [res (eb/bulk-with-index index bulk-create-items) 130 | total (count (:items res)) 131 | succ (filter :ok (:items res)) 132 | failed (filter :error (:items res))] 133 | 134 | (info "elasticized" total "/" (count succ) "/" (count failed) " (total/succ/fail) items to index " index "in " (:took res) "ms") 135 | (debug "Failed: " failed)) 136 | (catch Exception e 137 | (error "Unable to bulk index:" e)))))))))) 138 | 139 | (defn ^{:private true} resource-as-json [resource-name] 140 | (json/parse-string (slurp (io/resource resource-name)))) 141 | 142 | 143 | (defn ^{:private true} file-as-json [file-name] 144 | (try 145 | (json/parse-string (slurp file-name)) 146 | (catch Exception e 147 | (error "Exception while reading JSON file: " file-name) 148 | (throw e)))) 149 | 150 | 151 | (defn load-index-template 152 | "Loads the file into ElasticSearch as an index template." 153 | [template-name mapping-file] 154 | (esr/put (esr/index-template-url template-name) 155 | :body (file-as-json mapping-file))) 156 | --------------------------------------------------------------------------------