├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── project.clj ├── resources └── riemann_plugin │ └── kafka │ └── meta.edn └── src └── riemann └── plugin └── kafka.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: clojure 3 | lein: lein2 4 | jdk: 5 | - oraclejdk8 6 | - oraclejdk7 7 | - openjdk7 8 | branches: 9 | except: 10 | - gh-pages 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013 Pierre-Yves Ritschard 2 | 3 | Permission to use, copy, modify, and distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | riemann-kafka 2 | ============= 3 | 4 | [![Build Status](https://secure.travis-ci.org/pyr/riemann-kafka.png)](http://travis-ci.org/pyr/riemann-kafka) 5 | 6 | 7 | kafka consumer and producer support for riemann 8 | 9 | ## Synopsis 10 | 11 | In riemann.config 12 | 13 | ```clojure 14 | 15 | (load-plugins) 16 | 17 | (kafka/kafka-consumer {:topic "events" 18 | :zookeeper.connect "localhost:181" 19 | :group.id "riemann.consumer" 20 | :auto.offset.reset "smallest" 21 | :auto.commit.enable "false"}) 22 | 23 | 24 | (let [producer (kafka/kafka-producer 25 | {:topic "expired" 26 | :metadata.broker.list "localhost:9091"})] 27 | (streams 28 | prn 29 | (expired producer))) 30 | ``` 31 | 32 | ### Customized Encoder and Decoder 33 | 34 | The riemann-kafka producer supports customized encoder to encode events. 35 | A encoder is a function that expects a sequence of events and returns a Bytes object. 36 | If you don't specify a :encoder, riemann.common/encode is used, which turns events into a protobuf. 37 | 38 | ```clojure 39 | 40 | (defn my-json-encoder 41 | "Encode events into json, then toBytes" 42 | [events] 43 | (.getBytes (cheshire.core/encode events))) 44 | 45 | (let [producer (kafka/kafka-producer 46 | {:topic "expired" 47 | :metadata.broker.list "localhost:9091" 48 | :encoder my-json-encoder})] 49 | (streams 50 | prn 51 | (expired producer))) 52 | 53 | ``` 54 | 55 | The riemann-kafka consumer supports customized decoder to decode messages. 56 | A decoder gets a message from kafka as a Byte object, then returns a sequence of events. 57 | If you don't specify a :decoder, the default decoder expects the incoming message to be a riemann protobuf object. 58 | 59 | ```clojure 60 | 61 | (defn my-json-decoder 62 | "Decode kafka message into a riemann event" 63 | [input] 64 | ; input is a single kafka message in Bytes 65 | ; If the payload is a string, it needs to be reverted by `(String. input)` 66 | ; Return SHOULD be a seq of riemann events 67 | (let [decoded-msg (cheshire.core/parse-string (String. input) true)] 68 | (map riemann.common/event decoded-msg))) 69 | 70 | (kafka/kafka-consumer {:topic "events" 71 | :zookeeper.connect "localhost:181" 72 | :group.id "riemann.consumer" 73 | :auto.offset.reset "smallest" 74 | :auto.commit.enable "false" 75 | :decoder my-json-decoder}) 76 | ``` 77 | 78 | ## Installing 79 | 80 | You will need to build this module for now and push it on riemann's classpath, for this 81 | you will need a working JDK, JRE and [leiningen](http://leiningen.org). 82 | 83 | First build the project: 84 | 85 | ``` 86 | lein uberjar 87 | ``` 88 | 89 | The resulting artifact will be in `target/riemann-kafka-standalone-0.1.0.jar`. 90 | You will need to push that jar on the machine(s) where riemann runs, for instance, in 91 | `/usr/lib/riemann/riemann-kafka.jar`. 92 | 93 | If you have installed riemann from a stock package you will only need to tweak 94 | `/etc/default/riemann` and change 95 | the line `EXTRA_CLASSPATH` to read: 96 | 97 | ``` 98 | EXTRA_CLASSPATH=/usr/lib/riemann/riemann-kafka.jar 99 | ``` 100 | 101 | You can then use exposed functions, provided you have loaded the plugin in your configuration. 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject spootnik/riemann-kafka "0.1.4" 2 | :description "riemann producer and consumer for kafka queues" 3 | :url "https://github.com/pyr/riemann-kafka" 4 | :license {:name "MIT License"} 5 | :dependencies [[org.clojure/clojure "1.7.0"] 6 | [riemann "0.2.9"] 7 | [clj-kafka "0.3.2" 8 | :exclusions [org.slf4j/slf4j-log4j12 9 | org.slf4j/slf4j-simple]]]) 10 | -------------------------------------------------------------------------------- /resources/riemann_plugin/kafka/meta.edn: -------------------------------------------------------------------------------- 1 | {:plugin "kafka" 2 | :title "A plugin for consuming & producing events to and from kafka queues" 3 | :git-repo "https://github.com/pyr/riemann-kafka" 4 | :require riemann.plugin.kafka} 5 | -------------------------------------------------------------------------------- /src/riemann/plugin/kafka.clj: -------------------------------------------------------------------------------- 1 | (ns riemann.plugin.kafka 2 | "A riemann plugin to consume and produce from and to a kafka queue" 3 | (:import com.aphyr.riemann.Proto$Msg 4 | kafka.consumer.KafkaStream 5 | kafka.producer.KeyedMessage) 6 | (:require [riemann.core :refer [stream!]] 7 | [riemann.common :refer [decode-msg encode]] 8 | [clj-kafka.core :refer [to-clojure]] 9 | [clj-kafka.consumer.zk :refer [consumer]] 10 | [clj-kafka.producer :refer [send-message producer]] 11 | [riemann.service :refer [Service ServiceEquiv]] 12 | [riemann.config :refer [service!]] 13 | [clojure.tools.logging :refer [debug info error]])) 14 | 15 | (defn protobuf-decoder 16 | "Decode protobuf object to riemann events" 17 | [value] 18 | (:events (decode-msg (Proto$Msg/parseFrom value)))) 19 | 20 | (defn safe-decode 21 | "Do not let a bad payload break our consumption" 22 | [decoder input] 23 | (try 24 | (let [{:keys [value]} (to-clojure input)] 25 | (decoder value)) 26 | (catch Exception e 27 | (error e "could not decode msg")))) 28 | 29 | (defn stringify 30 | "Prepare a map to be converted to properties" 31 | [props] 32 | (let [input (dissoc props :topic :decoder :encoder :commit.per.msg) 33 | skeys (map (juxt (comp name key) val) input)] 34 | (reduce merge {} skeys))) 35 | 36 | (defn start-kafka-thread 37 | "Start a kafka thread which will pop messages off of the queue as long 38 | as running? is true" 39 | [running? core {:keys [topic] :as config}] 40 | (let [inq (consumer (stringify config))] 41 | (future 42 | (info "in consumption thread with consumer: " inq) 43 | (try 44 | (let [stream-map (.createMessageStreams inq {topic (int 1)}) 45 | [stream & _] (get stream-map topic) 46 | msg-seq (iterator-seq (.iterator ^KafkaStream stream)) 47 | decoder (:decoder config protobuf-decoder) 48 | commit-per-msg (:commit.per.msg config true)] 49 | (when (not commit-per-msg) 50 | (info "Commission from riemann-kafka consumer is disabled." 51 | "DO NOT disable :auto.commit.enable in your consumer config.")) 52 | (doseq [msg msg-seq :while @running? :when @core] 53 | (doseq [event (safe-decode decoder msg)] 54 | (debug "got input event: " event) 55 | (stream! @core event)) 56 | (when commit-per-msg (.commitOffsets inq))) 57 | (info "was instructed to stop, BYE!")) 58 | (catch Exception e 59 | (error e "interrupted consumption")) 60 | (finally 61 | (.shutdown inq)))))) 62 | 63 | (defn kafka-consumer 64 | "Yield a kafka consumption service" 65 | [config] 66 | (service! 67 | (let [running? (atom true) 68 | core (atom nil)] 69 | (reify 70 | clojure.lang.ILookup 71 | (valAt [this k not-found] 72 | (or (.valAt this k) not-found)) 73 | (valAt [this k] 74 | (info "looking up: " k) 75 | (if (= (name k) "config") config)) 76 | ServiceEquiv 77 | (equiv? [this other] 78 | (= config (:config other))) 79 | Service 80 | (conflict? [this other] 81 | (= config (:config other))) 82 | (start! [this] 83 | (info "starting kafka consumer running for topics: " 84 | (:topic config)) 85 | (start-kafka-thread running? core 86 | (merge {:topic "riemann"} config))) 87 | (reload! [this new-core] 88 | (info "reload called, setting new core value") 89 | (reset! core new-core)) 90 | (stop! [this] 91 | (reset! running? false) 92 | (info "kafka consumer stopping")))))) 93 | 94 | (defn kafka-producer 95 | "Yield a kafka producer" 96 | [{:keys [topic] :as config}] 97 | (let [p (producer (stringify config))] 98 | (fn [event] 99 | (let [events (if (sequential? event) event [event]) 100 | encoder (or (:encoder config) encode)] 101 | (send-message p (KeyedMessage. topic (encoder events))))))) 102 | --------------------------------------------------------------------------------