├── .gitignore ├── test └── clojurewerkz │ └── eep │ ├── stats_test.clj │ ├── test_utils.clj │ ├── throughput_test.clj │ ├── windows_test.clj │ └── emitter_test.clj ├── project.clj ├── src └── clojurewerkz │ └── eep │ ├── stats.clj │ ├── visualization.clj │ ├── clocks.clj │ ├── windows.clj │ └── emitter.clj ├── Changelog.md └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /lib 3 | /classes 4 | /checkouts 5 | pom.xml* 6 | *.jar 7 | *.class 8 | .lein-* 9 | doc/* 10 | .nrepl* 11 | -------------------------------------------------------------------------------- /test/clojurewerkz/eep/stats_test.clj: -------------------------------------------------------------------------------- 1 | (ns clojurewerkz.eep.stats-test 2 | (:require [clojure.test :refer :all] 3 | [clojurewerkz.eep.stats :refer :all])) 4 | 5 | (deftest sum-test 6 | (is (= 6 (sum [1 2 3])))) 7 | 8 | (deftest mean-test 9 | (is (= 2 (mean [1 2 3])))) 10 | -------------------------------------------------------------------------------- /test/clojurewerkz/eep/test_utils.clj: -------------------------------------------------------------------------------- 1 | (ns clojurewerkz.eep.test-utils 2 | (:use clojure.test) 3 | (:import [java.util.concurrent CountDownLatch TimeUnit])) 4 | 5 | (defn wrap-countdown 6 | "Countdown latch before executing function, proxy fn" 7 | [latch f] 8 | (fn [& values] 9 | (let [res (apply f values)] 10 | (.countDown @latch) 11 | res))) 12 | 13 | (defn make-latch 14 | "Creates a new latch" 15 | [i] 16 | (atom (CountDownLatch. i))) 17 | 18 | (defn reset-latch 19 | "Resets latch count to i" 20 | [latch i] 21 | (reset! latch (CountDownLatch. i))) 22 | 23 | (defn await-latch 24 | "Awaits for latch for 500ms" 25 | [latch] 26 | (.await @latch 1000 TimeUnit/MILLISECONDS)) 27 | 28 | (defmacro after-latch 29 | "Awaits for latch for 500ms" 30 | [latch & body] 31 | `(do 32 | (assert (.await (deref ~latch) 3 TimeUnit/SECONDS) 33 | (str "Timed out waiting on a latch... Still " 34 | (.getCount (deref ~latch)) " to go.")) 35 | ~@body)) 36 | 37 | (defmacro with-latch 38 | [countdown-from & body] 39 | `(let [latch# (CountDownLatch. ~countdown-from) 40 | ~'latch latch#] 41 | ~@body 42 | (.await latch# 1 TimeUnit/SECONDS) 43 | (is (= 0 (.getCount latch#))))) 44 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject clojurewerkz/eep "1.0.0-beta2-SNAPSHOT" 2 | :description "Embedded Event Processing in Clojure" 3 | :license {:name "Eclipse Public License" 4 | :url "http://www.eclipse.org/legal/epl-v10.html"} 5 | :dependencies [[org.clojure/clojure "1.6.0"] 6 | [com.ifesdjeen/utils "0.4.0"] 7 | [clojurewerkz/meltdown "1.1.0"] 8 | [rhizome "0.2.0"]] 9 | :profiles {:1.7 {:dependencies [[org.clojure/clojure "1.7.0"]]} 10 | :master {:dependencies [[org.clojure/clojure "1.8.0-master-SNAPSHOT"]]} 11 | :dev {:plugins [[codox "0.8.10"]] 12 | :codox {:sources ["src/clojure"] 13 | :output-dir "doc/api"}}} 14 | :aliases {"all" ["with-profile" "dev:dev,1.7:dev,master"]} 15 | :repositories {"sonatype" {:url "http://oss.sonatype.org/content/repositories/releases" 16 | :snapshots false 17 | :releases {:checksum :fail}} 18 | "springsource-milestone" {:url "http://repo.springsource.org/libs-milestone" 19 | :releases {:checksum :fail :update :always}} 20 | "springsource-snapshots" {:url "http://repo.springsource.org/libs-snapshot" 21 | :snapshots true 22 | :releases {:checksum :fail :update :always}} 23 | "sonatype-snapshots" {:url "http://oss.sonatype.org/content/repositories/snapshots" 24 | :snapshots true 25 | :releases {:checksum :fail :update :always}}} 26 | :test-selectors {:default (fn [m] (not (:performance m))) 27 | :performance :performance 28 | :focus :focus 29 | :all (constantly true)}) 30 | -------------------------------------------------------------------------------- /src/clojurewerkz/eep/stats.clj: -------------------------------------------------------------------------------- 1 | ;; This source code is dual-licensed under the Apache License, version 2 | ;; 2.0, and the Eclipse Public License, version 1.0. 3 | ;; 4 | ;; The APL v2.0: 5 | ;; 6 | ;; ---------------------------------------------------------------------------------- 7 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team 8 | ;; 9 | ;; Licensed under the Apache License, Version 2.0 (the "License"); 10 | ;; you may not use this file except in compliance with the License. 11 | ;; You may obtain a copy of the License at 12 | ;; 13 | ;; http://www.apache.org/licenses/LICENSE-2.0 14 | ;; 15 | ;; Unless required by applicable law or agreed to in writing, software 16 | ;; distributed under the License is distributed on an "AS IS" BASIS, 17 | ;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | ;; See the License for the specific language governing permissions and 19 | ;; limitations under the License. 20 | ;; ---------------------------------------------------------------------------------- 21 | ;; 22 | ;; The EPL v1.0: 23 | ;; 24 | ;; ---------------------------------------------------------------------------------- 25 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team. 26 | ;; All rights reserved. 27 | ;; 28 | ;; This program and the accompanying materials are made available under the terms of 29 | ;; the Eclipse Public License Version 1.0, 30 | ;; which accompanies this distribution and is available at 31 | ;; http://www.eclipse.org/legal/epl-v10.html. 32 | ;; ---------------------------------------------------------------------------------- 33 | 34 | (ns clojurewerkz.eep.stats) 35 | 36 | (defn sum 37 | "Calculates sum" 38 | [buffer] 39 | (apply + buffer)) 40 | 41 | (defn mean 42 | "Calculates mean" 43 | [vals] 44 | (let [non-nil (keep identity vals) 45 | cnt (count non-nil)] 46 | (when (pos? cnt) 47 | (/ (reduce + non-nil) cnt)))) 48 | 49 | (defn variance 50 | "Calculates variance, deviation from mean value" 51 | [arr] 52 | (let [mean (/ (reduce + arr) (count arr)) 53 | sqr #(* % %)] 54 | (/ 55 | (reduce + (map #(sqr (- % mean)) arr)) 56 | (- (count arr) 1)))) 57 | 58 | (defn percentage 59 | "Calculates percentage of value from total" 60 | [total value] 61 | (* (/ value total) 100.0)) 62 | -------------------------------------------------------------------------------- /test/clojurewerkz/eep/throughput_test.clj: -------------------------------------------------------------------------------- 1 | (ns clojurewerkz.eep.throughput-test 2 | (:require [clojure.test :refer :all] 3 | [clojurewerkz.eep.emitter :refer :all] 4 | [clojurewerkz.eep.test-utils :refer :all]) 5 | (:import [reactor.event.dispatch RingBufferDispatcher] 6 | [com.lmax.disruptor.dsl ProducerType] 7 | [com.lmax.disruptor YieldingWaitStrategy])) 8 | 9 | (defn throughput-test 10 | [emitter iterations] 11 | (let [latch (make-latch (/ iterations 2))] 12 | (defsplitter emitter :entry (fn [e] 13 | (if (even? e) :even :countdown))) 14 | 15 | (defn incrementer [acc _] 16 | (inc acc)) 17 | 18 | (defaggregator emitter :even incrementer 0) 19 | (defobserver emitter :countdown (fn [_] 20 | (.countDown @latch))) 21 | 22 | (let [start (System/currentTimeMillis)] 23 | (doseq [i (range iterations)] 24 | (notify emitter :entry i)) 25 | 26 | (after-latch latch 27 | (let [end (System/currentTimeMillis) 28 | elapsed (- end start)] 29 | (is (= (/ iterations 2) (state (get-handler emitter :even)))) 30 | (println 31 | (str 32 | "Iterations: " 33 | iterations 34 | " " 35 | (-> emitter 36 | (.reactor) 37 | (.getDispatcher) 38 | (.getClass) 39 | (.getSimpleName)) 40 | " throughput (" elapsed "ms): " (Math/round (float (/ iterations (/ elapsed 1000)))) "/sec"))))) 41 | 42 | (stop emitter))) 43 | 44 | (deftest ^:performance dispatcher-throughput-test 45 | (doseq [i [10000 100000 1000000]] 46 | (testing "Event Loop" 47 | (throughput-test (create :dispatcher-type :event-loop) i)) 48 | (testing "Thread Pool Executor" 49 | (throughput-test (create :dispatcher-type :thread-pool) i)) 50 | (testing "Ring Buffer" 51 | (throughput-test (create :dispatcher-type :ring-buffer) i)) 52 | (testing "Ring Buffer" 53 | (throughput-test (create :dispatcher (RingBufferDispatcher. "dispatcher-name" 4096 nil ProducerType/MULTI (YieldingWaitStrategy.))) i)))) 54 | -------------------------------------------------------------------------------- /test/clojurewerkz/eep/windows_test.clj: -------------------------------------------------------------------------------- 1 | (ns clojurewerkz.eep.windows-test 2 | (:require [clojurewerkz.eep.emitter :as e] 3 | [clojurewerkz.eep.stats :as s] 4 | [clojurewerkz.eep.clocks :as c] 5 | [clojure.test :refer :all] 6 | [clojurewerkz.eep.windows :refer :all] 7 | [clojurewerkz.eep.test-utils :refer :all])) 8 | 9 | (defn sum 10 | [buffer] 11 | (apply + buffer)) 12 | 13 | (def timespan 100) 14 | 15 | (deftest simple-sliding-window-test 16 | (let [last-val (atom nil) 17 | window (sliding-window-simple 5 18 | sum 19 | #(reset! last-val %))] 20 | (window 1) 21 | (window 2) 22 | (window 3) 23 | (window 4) 24 | (window 5) 25 | (is (= 15 @last-val)) 26 | (window 6) 27 | (is (= 20 @last-val)) 28 | (window 7) 29 | (is (= 25 @last-val)))) 30 | 31 | (deftest simple-tumbling-window-test 32 | (let [last-val (atom nil) 33 | window (tumbling-window-simple 5 34 | sum 35 | #(reset! last-val %))] 36 | (is (nil? @last-val)) 37 | (window 1) 38 | (window 2) 39 | (window 3) 40 | (window 4) 41 | (window 5) 42 | (is (= 15 @last-val)) 43 | 44 | (window 6) 45 | (is (= 15 @last-val)) 46 | (window 7) 47 | (window 8) 48 | (window 9) 49 | (window 10) 50 | (is (= 40 @last-val)))) 51 | 52 | (deftest simple-monotonic-window-test 53 | (let [last-val (atom nil) 54 | window (monotonic-window-simple (c/make-counting-clock 5) 55 | sum 56 | #(reset! last-val %))] 57 | (is (nil? @last-val)) 58 | (window 1) 59 | (is (nil? @last-val)) 60 | (window 1) 61 | (window 1) 62 | (window 1) 63 | (window 1) 64 | (is (= nil @last-val)) 65 | (window 1) 66 | (is (= 5 @last-val))) 67 | 68 | (let [last-val (atom nil) 69 | window (monotonic-window-simple (c/make-wall-clock timespan) 70 | sum 71 | #(reset! last-val %))] 72 | (is (nil? @last-val)) 73 | (window 1) 74 | (is (nil? @last-val)) 75 | (window 1) 76 | (window 1) 77 | (window 1) 78 | (Thread/sleep timespan) 79 | (window 1) 80 | (is (= 4 @last-val)) 81 | (window 1) 82 | (is (= 4 @last-val)) 83 | 84 | (Thread/sleep timespan) 85 | (window 1) 86 | (is (= 2 @last-val)))) 87 | -------------------------------------------------------------------------------- /src/clojurewerkz/eep/visualization.clj: -------------------------------------------------------------------------------- 1 | ;; This source code is dual-licensed under the Apache License, version 2 | ;; 2.0, and the Eclipse Public License, version 1.0. 3 | ;; 4 | ;; The APL v2.0: 5 | ;; 6 | ;; ---------------------------------------------------------------------------------- 7 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team 8 | ;; 9 | ;; Licensed under the Apache License, Version 2.0 (the "License"); 10 | ;; you may not use this file except in compliance with the License. 11 | ;; You may obtain a copy of the License at 12 | ;; 13 | ;; http://www.apache.org/licenses/LICENSE-2.0 14 | ;; 15 | ;; Unless required by applicable law or agreed to in writing, software 16 | ;; distributed under the License is distributed on an "AS IS" BASIS, 17 | ;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | ;; See the License for the specific language governing permissions and 19 | ;; limitations under the License. 20 | ;; ---------------------------------------------------------------------------------- 21 | ;; 22 | ;; The EPL v1.0: 23 | ;; 24 | ;; ---------------------------------------------------------------------------------- 25 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team. 26 | ;; All rights reserved. 27 | ;; 28 | ;; This program and the accompanying materials are made available under the terms of 29 | ;; the Eclipse Public License Version 1.0, 30 | ;; which accompanies this distribution and is available at 31 | ;; http://www.eclipse.org/legal/epl-v10.html. 32 | ;; ---------------------------------------------------------------------------------- 33 | 34 | (ns clojurewerkz.eep.visualization 35 | (:use rhizome.viz 36 | clojurewerkz.eep.emitter)) 37 | 38 | (defn shape-by-type 39 | [emitter key] 40 | (let [handler-type (last (clojure.string/split (str (type (get-handler emitter key))) #"\."))] 41 | (case handler-type 42 | "Transformer" "box" 43 | "Aggregator" "box3d" 44 | "CommutativeAggregator" "hexagon" 45 | "Multicast" "diamond" 46 | "Buffer" "folder" 47 | "Observer" "Mcircle" 48 | "Rollup" "doubleoctagon" 49 | "Filter" "triangle" 50 | "Splitter" "pentagon" 51 | :else "ellipse"))) 52 | 53 | (defn visualise-graph 54 | [emitter] 55 | (let [g (into {} 56 | (for [[k v] (get-handler emitter)] 57 | [k (downstream v)]))] 58 | 59 | (println (filter (fn [k] (not (or (= :splitter k) 60 | (= :rebroadcast k)))) 61 | (keys g))) 62 | (view-graph (keys g) 63 | g 64 | :node->descriptor (fn [n] 65 | {:label (name n) :shape (shape-by-type emitter n)})))) 66 | -------------------------------------------------------------------------------- /Changelog.md: -------------------------------------------------------------------------------- 1 | ## Changes between 1.0.0-beta1 and 1.0.0-beta2 2 | 3 | No changes yet. 4 | 5 | 6 | ## Changes between 1.0.0-alpha5 and 1.0.0-beta1 7 | 8 | `beta1` has **breaking public API changes**. 9 | 10 | ### Emitter Creation API Change 11 | 12 | `clojurewerkz.eep.emitter/create` no longer uses pseudo-kwargs. So, instead of 13 | 14 | ``` clojure 15 | (require '[clojurewerkz.eep.emitter :as eem]) 16 | 17 | (let [me (eem/create :dispatcher-type rtype :env env)] 18 | ) 19 | ``` 20 | 21 | use the function like so 22 | 23 | ``` clojure 24 | (require '[clojurewerkz.eep.emitter :as eem]) 25 | 26 | (let [me (eem/create {:dispatcher-type rtype :env env})] 27 | ) 28 | ``` 29 | 30 | 31 | ### Clojure 1.6 32 | 33 | EEP now depends on `org.clojure/clojure` version `1.6.0`. It is 34 | still compatible with Clojure 1.4 and if your `project.clj` depends on 35 | a different version, it will be used, but 1.6 is the default now. 36 | 37 | ## Changes between 1.0.0-alpha4 and 1.0.0-alpha5 38 | 39 | ### Fixed a problem with repeated emitter evaluation 40 | 41 | `build-topology` had a bug that caused emitter given in the form of `(create)` to be 42 | re-evaluated each time the topology was updated. The bug does not affect codebases 43 | that use a single emitter instance bound to an existing var. 44 | 45 | ### Fixed a problem with `add-handler` not returining an instance of emitter 46 | 47 | Usually an emitter is stored in a var, but if you use a threading 48 | macro such as `->` to build topologies, `add-handler` failed because 49 | it returned a caching registry. Thew new version returns the emitter, 50 | allowing for threading macros to work. 51 | 52 | ### Optional `downstreams` argument for properly visualising splitters. 53 | 54 | Because splitters only receives a function that's responsible for the 55 | routing, it's impossible for EEP to know where the events are routed 56 | after split. You can define a splitter with an array of all possible 57 | splits to make data flow visualisation possible. 58 | 59 | For exmaple, following splitter will split events to even and odd ones. Along with 60 | splitter function, pass an vector of `[:even :odd]` so that visualiser would catch it. 61 | 62 | ```clj 63 | (defsplitter *emitter* :entrypoint (fn [i] (if (even? i) :even :odd)) [:even :odd]) 64 | ``` 65 | 66 | ## Changes between 1.0.0-alpha3 and 1.0.0-alpha4 67 | 68 | ### Meltdown is updated to 1.0.0-aplha3 69 | 70 | Meltown alpha3 is a release with minor API additions. 71 | 72 | ### Fixed problem with RingBuffer dispatcher overflow 73 | 74 | RingBuffer operates in it's own pool, adding notifications blocks RingBuffer's yielding, 75 | therefore `notify` function block forever. 76 | 77 | EEP now has realistic throughput tests that verify that the issue is gone. 78 | 79 | ### Added more options to emitter constructor 80 | 81 | It is now possible to pass backing Dispatcher for reactor that's handling routing for the 82 | Emitter and an environment. 83 | 84 | ## Changes between 1.0.0-alpha2 and 1.0.0-alpha3 85 | 86 | ### Added an option to pass timer to timed window 87 | 88 | Previously, timed window would keep emitting tuples to dead processing graph. Now you can 89 | take control of your timer and stop timer together with emitter. 90 | 91 | ### Meltdown is updated to 1.0.0-aplha3 92 | 93 | Meltown alpha3 is a release with minor bugfixes 94 | 95 | ## Changes between 1.0.0-alpha1 and 1.0.0-alpha2 96 | 97 | ### Meltdown is updated to 1.0.0-aplha2 98 | 99 | ## 1.0.0-alpha1 100 | 101 | Initial release 102 | -------------------------------------------------------------------------------- /src/clojurewerkz/eep/clocks.clj: -------------------------------------------------------------------------------- 1 | ;; This source code is dual-licensed under the Apache License, version 2 | ;; 2.0, and the Eclipse Public License, version 1.0. 3 | ;; 4 | ;; The APL v2.0: 5 | ;; 6 | ;; ---------------------------------------------------------------------------------- 7 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team 8 | ;; 9 | ;; Licensed under the Apache License, Version 2.0 (the "License"); 10 | ;; you may not use this file except in compliance with the License. 11 | ;; You may obtain a copy of the License at 12 | ;; 13 | ;; http://www.apache.org/licenses/LICENSE-2.0 14 | ;; 15 | ;; Unless required by applicable law or agreed to in writing, software 16 | ;; distributed under the License is distributed on an "AS IS" BASIS, 17 | ;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | ;; See the License for the specific language governing permissions and 19 | ;; limitations under the License. 20 | ;; ---------------------------------------------------------------------------------- 21 | ;; 22 | ;; The EPL v1.0: 23 | ;; 24 | ;; ---------------------------------------------------------------------------------- 25 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team. 26 | ;; All rights reserved. 27 | ;; 28 | ;; This program and the accompanying materials are made available under the terms of 29 | ;; the Eclipse Public License Version 1.0, 30 | ;; which accompanies this distribution and is available at 31 | ;; http://www.eclipse.org/legal/epl-v10.html. 32 | ;; ---------------------------------------------------------------------------------- 33 | 34 | (ns clojurewerkz.eep.clocks 35 | "Generic implementation clocks to be used in windowed oprations" 36 | (:refer-clojure :exclude [time])) 37 | 38 | (defprotocol Clock 39 | (time [this] "Returns current clock time.") 40 | (elapsed? [this] "Returns wether clock is elapsed") 41 | (reset [_] "Resets clock") 42 | (tick [_] "Makes a tick within clock, updating internal counter")) 43 | 44 | (deftype CountingClock [initial period current] 45 | Clock 46 | (time [_] 47 | current) 48 | 49 | (elapsed? [_] 50 | (> (- current initial) period)) 51 | 52 | (tick [_] 53 | (CountingClock. initial period (inc current))) 54 | 55 | (reset [_] 56 | (CountingClock. current period current)) 57 | 58 | Object 59 | (toString [_] 60 | (str "Initial: " initial ", Period: " period ", Current:" current))) 61 | 62 | (defn- now 63 | "java.util.Date resolution is not enough for the Clock, as enqueue that's fired exactly after clock creation will 64 | produce a tick that yields same exact time." 65 | [] 66 | (System/currentTimeMillis)) 67 | 68 | (deftype WallClock [initial period current] 69 | Clock 70 | (time [_] 71 | current) 72 | 73 | (elapsed? [_] 74 | (>= (- current initial) period)) 75 | 76 | (tick [this] 77 | (WallClock. initial period (now))) 78 | 79 | (reset [_] 80 | (WallClock. (now) period (now))) 81 | 82 | Object 83 | (toString [_] 84 | (str "Initial: " initial ", Period: " period ", Current:" current))) 85 | 86 | 87 | (defn make-counting-clock 88 | "Simplest clock implementation that increments counter on each clock tick." 89 | [period] 90 | (CountingClock. 0 period 0)) 91 | 92 | (defn make-wall-clock 93 | "Wall clock, using System/currentTimeMillis to check wether timer is elapsed" 94 | [period] 95 | (WallClock. (now) period (now))) 96 | 97 | (def period 98 | {:second 1000 99 | :seconds 1000 100 | :minute (* 60 1000) 101 | :minutes (* 60 1000) 102 | :hour (* 60 60 1000) 103 | :hours (* 60 60 1000) 104 | :day (* 24 60 60 1000) 105 | :days (* 24 60 60 1000) 106 | :weeks (* 24 60 60 1000)}) 107 | -------------------------------------------------------------------------------- /src/clojurewerkz/eep/windows.clj: -------------------------------------------------------------------------------- 1 | ;; This source code is dual-licensed under the Apache License, version 2 | ;; 2.0, and the Eclipse Public License, version 1.0. 3 | ;; 4 | ;; The APL v2.0: 5 | ;; 6 | ;; ---------------------------------------------------------------------------------- 7 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team 8 | ;; 9 | ;; Licensed under the Apache License, Version 2.0 (the "License"); 10 | ;; you may not use this file except in compliance with the License. 11 | ;; You may obtain a copy of the License at 12 | ;; 13 | ;; http://www.apache.org/licenses/LICENSE-2.0 14 | ;; 15 | ;; Unless required by applicable law or agreed to in writing, software 16 | ;; distributed under the License is distributed on an "AS IS" BASIS, 17 | ;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | ;; See the License for the specific language governing permissions and 19 | ;; limitations under the License. 20 | ;; ---------------------------------------------------------------------------------- 21 | ;; 22 | ;; The EPL v1.0: 23 | ;; 24 | ;; ---------------------------------------------------------------------------------- 25 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team. 26 | ;; All rights reserved. 27 | ;; 28 | ;; This program and the accompanying materials are made available under the terms of 29 | ;; the Eclipse Public License Version 1.0, 30 | ;; which accompanies this distribution and is available at 31 | ;; http://www.eclipse.org/legal/epl-v10.html. 32 | ;; ---------------------------------------------------------------------------------- 33 | 34 | (ns clojurewerkz.eep.windows 35 | "Generic implementation of windowed operations" 36 | (:require [clojurewerkz.eep.clocks :as clocks] 37 | [com.ifesdjeen.utils.circular-buffer :as cb]) 38 | (:import [java.util Timer TimerTask Date])) 39 | 40 | ;; 41 | ;; Implementation 42 | ;; 43 | 44 | (defn sliding-window-simple 45 | "A simple sliding window. Sliding windows (here) have a fixed a-priori known size. 46 | 47 | Example: Sliding window of size 2 computing sum of values. 48 | 49 | t0 t1 (emit) t2 (emit) tN 50 | +---+ +---+---+ -...-...- 51 | | 1 | | 2 | 1 | <3> : x : x : 52 | +---+ +---+---+ _...+---+---+ ... 53 | | 2 | | 2 | 3 | <5> 54 | +---+ +---+---+ 55 | | 4 | 56 | +---+ 57 | 58 | Useful to hold last `size` elements. 59 | " 60 | [size aggregate emit-fn] 61 | (let [buffer (atom (cb/circular-buffer size))] 62 | (fn [value] 63 | (swap! buffer conj value) 64 | (when (cb/full? @buffer) 65 | (emit-fn (aggregate @buffer)))))) 66 | 67 | (defn tumbling-window-simple 68 | "A tumbling window. Tumbling windows (here) have a fixed a-priori known size. 69 | 70 | Example: Tumbling window of size 2 computing sum of values. 71 | 72 | t0 t1 (emit) t2 t3 (emit) t4 73 | +---+ +---+---+ -...-...- 74 | | 1 | | 2 | 1 | <3> : x : x : 75 | +---+ +---+---+ -...+---+---+ +---+---+ ... 76 | | 3 | | 4 | 3 | <7> 77 | +---+ +---+---+ 78 | 79 | Useful to accumulate `size` elements and aggreagate on overflow. 80 | " 81 | [size aggregate emit-fn] 82 | (let [buffer (atom (cb/circular-buffer size))] 83 | (fn [value] 84 | (swap! buffer conj value) 85 | (when (cb/full? @buffer) 86 | (emit-fn (aggregate @buffer)) 87 | (reset! buffer (cb/circular-buffer size)))))) 88 | 89 | (defn monotonic-window-simple 90 | "A simple monotonic window, that makes a clock tick on every call. Whenever 91 | clock is elapsed, runs `emit-fn`. 92 | 93 | In essence, it's an alternative implementation of tumbling-window that allows 94 | to use custom emission control rather than having a buffer overflow check. 95 | 96 | Useful for cases when emission should be controlled by arbitrary function, 97 | possibly unrelated to window contents." 98 | [clock-orig aggregate emit-fn] 99 | (let [clock (atom clock-orig) 100 | buffer (atom [])] 101 | (fn [value] 102 | (swap! clock clocks/tick) 103 | (when (clocks/elapsed? @clock) 104 | (emit-fn (aggregate @buffer)) 105 | (reset! buffer []) 106 | (swap! clock clocks/reset)) 107 | (swap! buffer conj value)))) 108 | 109 | (defn timed-window-simple 110 | "A simple timed window, that runs on wall clock. Receives events and stores them 111 | until clock is elapsed, runs `emit-fn` for aggregation after that. 112 | 113 | In essence, it's an alternative implementation of tumbling-window or monotonic-window 114 | that allows wall clock control. 115 | 116 | Useful for accumulating events for time-bound events processing, accumulates events 117 | for a certain period of time (for example, 1 minute), and aggregates them." 118 | ([clock-orig tick-period aggregate emit-fn] 119 | (timed-window-simple clock-orig tick-period aggregate emit-fn (Timer. true))) 120 | ([clock-orig tick-period aggregate emit-fn timer] 121 | (let [clock (atom clock-orig) 122 | buffer (atom []) 123 | task (proxy [TimerTask] [] 124 | (run [] 125 | (swap! clock clocks/tick) 126 | (when (clocks/elapsed? @clock) 127 | (emit-fn (aggregate @buffer)) 128 | (reset! buffer []) 129 | (swap! clock clocks/reset))))] 130 | (.scheduleAtFixedRate timer task 0 tick-period) 131 | (fn [value] 132 | (swap! buffer conj value))))) 133 | -------------------------------------------------------------------------------- /test/clojurewerkz/eep/emitter_test.clj: -------------------------------------------------------------------------------- 1 | (ns clojurewerkz.eep.emitter-test 2 | (:require [clojurewerkz.meltdown.reactor :as reactor] 3 | [clojurewerkz.meltdown.env :as me] 4 | [clojurewerkz.eep.emitter :refer :all] 5 | [clojurewerkz.eep.stats :as stats] 6 | [clojurewerkz.eep.windows :as windows] 7 | [clojurewerkz.eep.clocks :as clocks] 8 | [clojure.test :refer :all] 9 | [clojurewerkz.eep.test-utils :refer :all])) 10 | 11 | (defn new-emitter 12 | [] 13 | (let [rtype (rand-nth [:event-loop :thread-pool :ring-buffer]) 14 | env (me/create)] 15 | (create {:dispatcher-type rtype :env env}))) 16 | 17 | (deftest test-no-argument-form 18 | (create)) 19 | 20 | (deftest test-aggregator 21 | (let [em (new-emitter) 22 | latch (make-latch 3)] 23 | (defaggregator em :count (wrap-countdown latch +) 100) 24 | 25 | (dotimes [i 3] 26 | (notify em :count 1)) 27 | 28 | (after-latch latch 29 | (is (= 103 (state (get-handler em :count))))) 30 | (stop em))) 31 | 32 | (deftest test-caggregator 33 | (let [em (new-emitter) 34 | latch (make-latch 3)] 35 | (defcaggregator em :count (wrap-countdown latch +) 100) 36 | 37 | (dotimes [i 3] 38 | (notify em :count 1)) 39 | 40 | (Thread/sleep 10) 41 | (after-latch latch 42 | (is (= 103 (state (get-handler em :count))))) 43 | (stop em))) 44 | 45 | (deftest test-defobserver 46 | (let [em (new-emitter) 47 | latch (make-latch 1)] 48 | (defaggregator em :count (wrap-countdown latch +) 100) 49 | (is (= 100 (state (get-handler em :count)))) 50 | 51 | (notify em :count 1) 52 | 53 | (after-latch latch 54 | (is (= 101 (state (get-handler em :count))))) 55 | (stop em)) 56 | 57 | (let [em (new-emitter) 58 | latch (make-latch 5)] 59 | (defobserver em :countdown (fn [_] 60 | (.countDown @latch))) 61 | (dotimes [i 5] 62 | (notify em :countdown 1)) 63 | 64 | (is (.await @latch 500 java.util.concurrent.TimeUnit/MILLISECONDS)) 65 | (stop em))) 66 | 67 | (deftest test-filter-pipe 68 | (let [em (new-emitter) 69 | latch (make-latch 2)] 70 | (deffilter em :entrypoint even? :summarizer) 71 | (defaggregator em :summarizer (wrap-countdown latch +) 0) 72 | (notify em :entrypoint 1) 73 | (notify em :entrypoint 2) 74 | (notify em :entrypoint 3) 75 | (notify em :entrypoint 4) 76 | (notify em :entrypoint 5) 77 | (after-latch latch 78 | (is (= 6 (state (get-handler em :summarizer))))) 79 | (stop em))) 80 | 81 | (deftest test-multicast 82 | (testing "Basic multicast abilities" 83 | (let [em (new-emitter) 84 | latch (make-latch 9) 85 | f (wrap-countdown latch +)] 86 | (defmulticast em :entrypoint [:multicast1 :multicast2 :multicast3]) 87 | (defaggregator em :multicast1 f 0) 88 | (defaggregator em :multicast2 f 0) 89 | (defaggregator em :multicast3 f 0) 90 | (notify em :entrypoint 1) 91 | (notify em :entrypoint 2) 92 | (notify em :entrypoint 3) 93 | 94 | (Thread/sleep 200) 95 | (after-latch latch 96 | (is (= 6 (state (get-handler em :multicast1)))) 97 | (is (= 6 (state (get-handler em :multicast2)))) 98 | (is (= 6 (state (get-handler em :multicast3))))) 99 | (stop em))) 100 | 101 | (testing "Re-adding multicast" 102 | (let [em (new-emitter) 103 | latch (make-latch 3) 104 | f (wrap-countdown latch +)] 105 | (defmulticast em :entrypoint [:multicast1]) 106 | (defmulticast em :entrypoint [:multicast2]) 107 | (defmulticast em :entrypoint [:multicast3]) 108 | (defaggregator em :multicast1 f 0) 109 | (defaggregator em :multicast2 f 0) 110 | (defaggregator em :multicast3 f 0) 111 | 112 | (notify em :entrypoint 1) 113 | (notify em :entrypoint 2) 114 | (notify em :entrypoint 3) 115 | 116 | (Thread/sleep 200) 117 | (after-latch latch 118 | (is (= 6 (state (get-handler em :multicast1)))) 119 | (is (= 6 (state (get-handler em :multicast2)))) 120 | (is (= 6 (state (get-handler em :multicast3))))) 121 | (stop em)))) 122 | 123 | (deftest test-transform 124 | (let [em (new-emitter) 125 | latch (make-latch 5)] 126 | (deftransformer em :entrypoint (partial * 2) :summarizer) 127 | (defaggregator em :summarizer (wrap-countdown latch +) 0) 128 | (notify em :entrypoint 1) 129 | (notify em :entrypoint 2) 130 | (notify em :entrypoint 3) 131 | (notify em :entrypoint 4) 132 | (notify em :entrypoint 5) 133 | 134 | (Thread/sleep 10) 135 | (after-latch latch 136 | (is (= 30 (state (get-handler em :summarizer))))) 137 | (stop em))) 138 | 139 | (deftest test-splitter 140 | (let [em (new-emitter) 141 | latch (make-latch 5) 142 | f (wrap-countdown latch +)] 143 | (defsplitter em :entrypoint (fn [i] (if (even? i) :even :odd))) 144 | (defaggregator em :even f 0) 145 | (defaggregator em :odd f 0) 146 | (notify em :entrypoint 1) 147 | (notify em :entrypoint 2) 148 | (notify em :entrypoint 3) 149 | (notify em :entrypoint 4) 150 | (notify em :entrypoint 5) 151 | 152 | (Thread/sleep 10) 153 | (after-latch latch 154 | (is (= 6 (state (get-handler em :even)))) 155 | (is (= 9 (state (get-handler em :odd))))) 156 | (stop em))) 157 | 158 | (deftest test-carefully 159 | (let [em (new-emitter)] 160 | (defaggregator em :failure-is-expected-here (wrap-carefully em :failure-is-expected-here +) 0) 161 | (notify em :failure-is-expected-here "a") 162 | (Thread/sleep 100) 163 | (is (not (nil? (:failure-is-expected-here (.errors em))))) 164 | (stop em))) 165 | 166 | (deftest test-threading-dsl 167 | (let [em (new-emitter) 168 | latch (make-latch 5) 169 | f (wrap-countdown latch +)] 170 | 171 | (-> em 172 | (defsplitter :entrypoint (fn [i] (if (even? i) :even :odd))) 173 | (defaggregator :even f 0) 174 | (defaggregator :odd f 0)) 175 | 176 | (notify em :entrypoint 1) 177 | (notify em :entrypoint 2) 178 | (notify em :entrypoint 3) 179 | (notify em :entrypoint 4) 180 | (notify em :entrypoint 5) 181 | 182 | (after-latch latch 183 | (is (= 6 (state (get-handler em :even)))) 184 | (is (= 9 (state (get-handler em :odd))))) 185 | (stop em))) 186 | 187 | (deftest test-splitter-dsl 188 | (let [em (new-emitter) 189 | latch (make-latch 5) 190 | f (wrap-countdown latch +)] 191 | 192 | (build-topology em 193 | :entrypoint (defsplitter (fn [i] (if (even? i) :even :odd))) 194 | :even (defaggregator f 0) 195 | :odd (defaggregator f 0)) 196 | 197 | (notify em :entrypoint 1) 198 | (notify em :entrypoint 2) 199 | (notify em :entrypoint 3) 200 | (notify em :entrypoint 4) 201 | (notify em :entrypoint 5) 202 | 203 | (after-latch latch 204 | (is (= 6 (state (get-handler em :even)))) 205 | (is (= 9 (state (get-handler em :odd))))) 206 | (stop em))) 207 | 208 | (deftest test-rollup 209 | (let [em (new-emitter)] 210 | (defrollup em :entrypoint 100 :buffer) 211 | (defaggregator em :buffer keep-last nil) 212 | (notify em :entrypoint 1) 213 | (notify em :entrypoint 2) 214 | (notify em :entrypoint 3) 215 | (is (nil? (state (get-handler em :buffer)))) 216 | (Thread/sleep 150) 217 | (is (= [1 2 3] (state (get-handler em :buffer)))))) 218 | 219 | (deftest test-group-aggregate 220 | (is (= {:a 4 :b 6} (group-aggregate stats/sum [[:a 1] [:b 2] [:a 3] [:b 4]])))) 221 | 222 | (deftest test-alive 223 | (let [em (new-emitter)] 224 | (is (alive? em)) 225 | (stop em) 226 | (is (not (alive? em))))) 227 | -------------------------------------------------------------------------------- /src/clojurewerkz/eep/emitter.clj: -------------------------------------------------------------------------------- 1 | ;; This source code is dual-licensed under the Apache License, version 2 | ;; 2.0, and the Eclipse Public License, version 1.0. 3 | ;; 4 | ;; The APL v2.0: 5 | ;; 6 | ;; ---------------------------------------------------------------------------------- 7 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team 8 | ;; 9 | ;; Licensed under the Apache License, Version 2.0 (the "License"); 10 | ;; you may not use this file except in compliance with the License. 11 | ;; You may obtain a copy of the License at 12 | ;; 13 | ;; http://www.apache.org/licenses/LICENSE-2.0 14 | ;; 15 | ;; Unless required by applicable law or agreed to in writing, software 16 | ;; distributed under the License is distributed on an "AS IS" BASIS, 17 | ;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 18 | ;; See the License for the specific language governing permissions and 19 | ;; limitations under the License. 20 | ;; ---------------------------------------------------------------------------------- 21 | ;; 22 | ;; The EPL v1.0: 23 | ;; 24 | ;; ---------------------------------------------------------------------------------- 25 | ;; Copyright (c) 2014-2014 Michael S. Klishin, Alex Petrov, and the ClojureWerkz Team. 26 | ;; All rights reserved. 27 | ;; 28 | ;; This program and the accompanying materials are made available under the terms of 29 | ;; the Eclipse Public License Version 1.0, 30 | ;; which accompanies this distribution and is available at 31 | ;; http://www.eclipse.org/legal/epl-v10.html. 32 | ;; ---------------------------------------------------------------------------------- 33 | 34 | (ns clojurewerkz.eep.emitter 35 | "Generic event emitter implementation heavily inspired by gen_event in Erlang/OTP" 36 | (:require clojure.pprint 37 | [clojure.set :as s] 38 | [clojurewerkz.meltdown.reactor :as mr] 39 | [clojurewerkz.meltdown.consumers :as mc] 40 | [clojurewerkz.meltdown.selectors :as ms :refer [$]] 41 | [clojurewerkz.eep.windows :as ws] 42 | [clojurewerkz.eep.clocks :as cl] 43 | [com.ifesdjeen.utils.circular-buffer :as cb]) 44 | (:import [java.util.concurrent ConcurrentHashMap Executors ExecutorService] 45 | [reactor.function Consumer] 46 | [reactor.event Event])) 47 | 48 | (defn pprint-to-str 49 | [& objs] 50 | (let [w (java.io.StringWriter.)] 51 | (clojure.pprint/pprint objs w) 52 | (.toString w))) 53 | 54 | (def ^{:doc "Default thread pool size, calculated as # available processors + 1"} 55 | pool-size (-> (Runtime/getRuntime) 56 | .availableProcessors 57 | inc)) 58 | 59 | (defonce notify-pool (Executors/newFixedThreadPool (int pool-size))) 60 | 61 | (defn sync-submit 62 | [f] 63 | (.submit notify-pool ^Callable f)) 64 | 65 | (defprotocol IHandler 66 | (state [_]) 67 | (downstream [_])) 68 | 69 | (defprotocol IEmitter 70 | (add-handler [_ event-type handler] "Registers a handler on given emitter") 71 | (delete-handler [_ t] "Removes the handler `f` from the current emitter, that's used for event 72 | type `t`. ") 73 | (get-handler [_] [_ t] "Returns all currently registered Handlers for Emitter") 74 | (notify-in-pool [_ type args] "Asynchronous event dispatch function. Should be used for all cases when 75 | notification is done from Handler") 76 | (notify [_ type args] "Synchronous (default) event dispatch function. All the Handlers (both 77 | stateful and stateless). Pretty much direct routing.") 78 | (notify-some [_ type-checker args] "Asynchronous notification, with function that matches an event type. 79 | Pretty much topic routing.") 80 | (! [_ type args] "Erlang-style alias for `notify`") 81 | (swap-handler [_ t new-f] "Replaces typed event handler with `new-f` event handler.") 82 | (stop [_] "Cancels all pending tasks, stops event emission.") 83 | (alive? [_] "Returns wether the current emitter is alive or no") 84 | (register-exception [_ t e])) 85 | 86 | (defn- add-handler-intern 87 | [handlers event-type handler] 88 | (swap! handlers assoc event-type handler)) 89 | 90 | (defn- delete-handler-intern 91 | [handlers event-type] 92 | (swap! handlers dissoc event-type)) 93 | 94 | (deftype Emitter [handlers errors reactor] 95 | IEmitter 96 | (add-handler [this event-type handler] 97 | 98 | (when (nil? (get handler event-type)) 99 | (add-handler-intern handlers event-type handler) 100 | (mr/register-consumer reactor ($ event-type) handler) 101 | (.select (.getConsumerRegistry reactor) event-type)) 102 | this) 103 | 104 | (delete-handler [this event-type] 105 | (when-let [old-handler (get-handler this event-type)] 106 | (try 107 | (.unregister (.getConsumerRegistry reactor) event-type) 108 | (catch Exception e)) 109 | (swap! handlers dissoc event-type) 110 | old-handler)) 111 | 112 | (swap-handler [this event-type f] 113 | (let [old (delete-handler this event-type)] 114 | (add-handler this event-type f) 115 | old)) 116 | 117 | (notify [_ t args] 118 | (mr/notify-raw ^Reactor reactor t (Event. args))) 119 | 120 | (notify-in-pool [_ t args] 121 | (sync-submit 122 | #(mr/notify-raw ^Reactor reactor t (Event. args)))) 123 | 124 | (! [this t args] 125 | (notify this t args)) 126 | 127 | (get-handler [_] 128 | @handlers) 129 | 130 | (get-handler [_ t] 131 | (get @handlers t)) 132 | 133 | (stop [_] 134 | (-> reactor 135 | (.getDispatcher) 136 | (.shutdown))) 137 | 138 | (alive? [_] 139 | (-> reactor 140 | (.getDispatcher) 141 | (.alive))) 142 | 143 | (register-exception [_ t e] 144 | (.put errors t e)) 145 | 146 | (toString [_] 147 | (pprint-to-str "\n" (mapv #(.toString %) @handlers)))) 148 | 149 | (defn create 150 | "Creates a fresh Event Emitter with the default executor." 151 | ([{:keys [dispatcher-type dispatcher env] 152 | :or [env (me/create)]}] 153 | (let [reactor (mr/create :dispatcher-type dispatcher-type :dispatcher dispatcher :env env)] 154 | (Emitter. (atom {}) (ConcurrentHashMap.) reactor))) 155 | ([] (create {}))) 156 | 157 | ;; 158 | ;; Operations 159 | ;; 160 | 161 | (deftype Aggregator [emitter f state_] 162 | IHandler 163 | (state [_] 164 | @state_) 165 | 166 | (downstream [_] nil) 167 | 168 | Consumer 169 | (accept [_ payload] 170 | (swap! state_ f (.getData payload))) 171 | 172 | Object 173 | (toString [_] 174 | (pprint-to-str f @state_))) 175 | 176 | (deftype CommutativeAggregator [emitter f state_] 177 | IHandler 178 | (state [_] 179 | @state_) 180 | 181 | (downstream [_] nil) 182 | 183 | Consumer 184 | (accept [_ payload] 185 | (dosync 186 | (commute state_ f (.getData payload)))) 187 | 188 | Object 189 | (toString [_] 190 | (str "Handler: " f ", state: " @state_) )) 191 | 192 | (deftype Observer [emitter f] 193 | IHandler 194 | (state [_] 195 | nil) 196 | 197 | (downstream [_] nil) 198 | 199 | Consumer 200 | (accept [_ payload] 201 | (f (.getData payload)))) 202 | 203 | (deftype Rollup [emitter f redistribute-t] 204 | IHandler 205 | (state [_] 206 | nil) 207 | 208 | (downstream [_] [redistribute-t]) 209 | 210 | Consumer 211 | (accept [_ payload] 212 | (f (.getData payload))) 213 | 214 | Object 215 | (toString [_] 216 | (str f ", " redistribute-t))) 217 | 218 | (deftype Filter [emitter filter-fn rebroadcast] 219 | IHandler 220 | (state [_] nil) 221 | 222 | (downstream [_] [rebroadcast]) 223 | 224 | Consumer 225 | (accept [_ payload] 226 | (let [data (.getData payload)] 227 | (when (filter-fn data) 228 | (notify-in-pool emitter rebroadcast data)))) 229 | 230 | Object 231 | (toString [_] 232 | (str filter-fn ", " rebroadcast))) 233 | 234 | (deftype Multicast [emitter rebroadcast-types] 235 | IHandler 236 | (state [_] nil) 237 | 238 | (downstream [_] rebroadcast-types) 239 | 240 | Consumer 241 | (accept [_ payload] 242 | (let [data (.getData payload)] 243 | (doseq [t rebroadcast-types] 244 | (notify-in-pool emitter t data)))) 245 | 246 | Object 247 | (toString [_] 248 | (clojure.string/join ", " rebroadcast-types))) 249 | 250 | (deftype Splitter [emitter split-fn downstreams] 251 | IHandler 252 | (state [_] nil) 253 | 254 | (downstream [_] downstreams) 255 | 256 | Consumer 257 | (accept [_ payload] 258 | (let [data (.getData payload)] 259 | (notify-in-pool emitter (split-fn data) data))) 260 | 261 | Object 262 | (toString [_] 263 | (clojure.string/join ", " [split-fn]))) 264 | 265 | (deftype Transformer [emitter transform-fn rebroadcast] 266 | IHandler 267 | (state [_] nil) 268 | 269 | (downstream [_] 270 | (if (sequential? rebroadcast) 271 | rebroadcast 272 | [rebroadcast])) 273 | 274 | Consumer 275 | (accept [_ payload] 276 | (let [data (.getData payload)] 277 | (if (sequential? rebroadcast) 278 | (doseq [t rebroadcast] 279 | (notify-in-pool emitter t (transform-fn data))) 280 | (notify-in-pool emitter rebroadcast (transform-fn data))))) 281 | 282 | Object 283 | (toString [_] 284 | (clojure.string/join ", " [transform-fn rebroadcast]))) 285 | 286 | (deftype Buffer [emitter buf] 287 | IHandler 288 | (state [_] (cb/to-vec @buf)) 289 | 290 | (downstream [_] nil) 291 | 292 | Consumer 293 | (accept [_ payload] 294 | (swap! buf conj (.getData payload)))) 295 | 296 | ;; 297 | ;; Builder fns 298 | ;; 299 | 300 | (defn deffilter 301 | "Defines a filter operation, that receives events of a type `t`, and rebroadcasts ones 302 | for which `filter-fn` returns true" 303 | [emitter t filter-fn rebroadcast] 304 | (add-handler emitter t (Filter. emitter filter-fn rebroadcast))) 305 | 306 | (defn deftransformer 307 | "Defines a transformer, that gets tuples events of a type `t`, transforms them with `transform-fn` 308 | and rebroadcasts them to `rebroadcast` handlers." 309 | [emitter t transform-fn rebroadcast] 310 | (add-handler emitter t (Transformer. emitter transform-fn rebroadcast))) 311 | 312 | (def defmap deftransformer) 313 | 314 | (defn defaggregator 315 | "Defines an aggregator, that is initialized with `initial-state`, then gets events of a type `t` 316 | and aggregates state by applying `aggregate-fn` to current state and incoming event." 317 | [emitter t aggregate-fn initial-state] 318 | (add-handler emitter t (Aggregator. emitter aggregate-fn (atom initial-state)))) 319 | 320 | (def defreduce defaggregator) 321 | 322 | (defn defcaggregator 323 | "Defines a commutative aggregator, that is initialized with `initial-state`, then gets of 324 | a type `t` and aggregates state by applying `aggregate-fn` to current state and tuple." 325 | [emitter t aggregate-fn initial-state] 326 | (add-handler emitter t (CommutativeAggregator. emitter aggregate-fn (ref initial-state)))) 327 | 328 | (defn defmulticast 329 | "Defines a multicast, that receives events of a type `t`, and rebroadcasts them to several other handlers." 330 | [emitter t m] 331 | (let [h (delete-handler emitter t)] 332 | (add-handler emitter t 333 | (Multicast. emitter 334 | (if (isa? Multicast (type h)) 335 | (set (concat (.rebroadcast-types h) m)) 336 | (set m)))))) 337 | 338 | (defn undefmulticast 339 | "Unregisters a multicast. If there're no downstreams for multicast, deregisters handler completely." 340 | [emitter t m] 341 | (let [multicast-types (s/difference 342 | (.rebroadcast-types (get-handler emitter t)) 343 | (set m))] 344 | (if (empty? multicast-types) 345 | (delete-handler emitter t) 346 | (add-handler emitter t (Multicast. emitter multicast-types))))) 347 | 348 | (defn defsplitter 349 | ([emitter t split-fn] 350 | (defsplitter emitter t split-fn nil)) 351 | ([emitter t split-fn downstreams] 352 | (add-handler emitter t (Splitter. emitter split-fn downstreams)))) 353 | 354 | (defn defobserver 355 | "Defines an observer, that runs (potentially with side-effects) f for tuples of given type." 356 | [emitter t f] 357 | (add-handler emitter t (Observer. emitter f))) 358 | 359 | (defn defrollup 360 | "Rollup is a timed window, that accumulates entries until it times out, and emits them 361 | to the next processing part afterwards. Rollup resolution should not be less than 10 milliseconds." 362 | [emitter t period redistribute-t] 363 | (let [window (ws/timed-window-simple 364 | (cl/make-wall-clock period) 365 | 10 identity 366 | #(notify emitter redistribute-t %))] 367 | (add-handler emitter t (Rollup. emitter window redistribute-t)))) 368 | 369 | (defn defbuffer 370 | "Defines a circular buffer with given `capacity`" 371 | [emitter t capacity] 372 | (add-handler emitter t (Buffer. emitter (atom (cb/circular-buffer capacity))))) 373 | 374 | ;; 375 | ;; Debug utils 376 | ;; 377 | 378 | (defmacro carefully 379 | "Test macro, should only be used internally" 380 | [emitter handler-type & body] 381 | `(try 382 | ~@body 383 | (catch Exception e# 384 | (println "Exception occured while processing " ~handler-type ": " (.getMessage e#)) 385 | (.printStackTrace e#) 386 | (register-exception ~emitter ~handler-type e#)))) 387 | 388 | (defn wrap-carefully 389 | "Helper method to help with debugging of complex flows, when something is failing and you don't really see why" 390 | [emitter handler-type f] 391 | (fn [a b] 392 | (carefully emitter handler-type 393 | (f a b)))) 394 | 395 | (defn wrap-debug 396 | "Helper method to help with debugging of complex flows, when something is failing and you don't really see why" 397 | [emitter handler-type f] 398 | (fn [a b] 399 | (let [res (f a b)] 400 | (println (format "%s - %s: Input: [%s, %s], Output: %s" 401 | (.getName (Thread/currentThread)) 402 | handler-type 403 | a b 404 | res)) 405 | res))) 406 | 407 | (defmacro build-topology 408 | "Builds aggregation topology from the given `hander-type` and handler builder." 409 | ([emitter a [first & rest]] 410 | `(let [emitter# ~emitter] 411 | (~first emitter# ~a ~@rest))) 412 | ([emitter a b & more] 413 | `(let [emitter# ~emitter] 414 | (build-topology emitter# ~a ~b) 415 | (build-topology emitter# ~@more)))) 416 | 417 | ;; 418 | ;; 419 | ;; 420 | 421 | (defn keep-last 422 | "Aggregator helper function, always keeps only the last value" 423 | [_ last] 424 | last) 425 | 426 | (defn group-aggregate 427 | "Wrapper function for aggregators" 428 | [aggregate-fn tuples] 429 | (into {} 430 | (for [[k vals] (group-by first tuples)] 431 | [k (aggregate-fn (map second vals))]))) 432 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # EEP, Embedded Event Processing in Clojure 2 | 3 | EEP is a Clojure library for embedded event processing. 4 | It combines a lightweight generic event handling system, 5 | and with multiple windowed stream operations. 6 | 7 | eep-clj is heavily influenced by other EEP projects: 8 | 9 | * [eep-js (JavaScript)](https://github.com/darach/eep-js) 10 | * [eep-erl (Erlang)](https://github.com/darach/eep-erl) 11 | * [eep-php (PHP)](https://github.com/ianbarber/eep-php) 12 | 13 | 14 | ## Project Maturity 15 | 16 | EEP is a *young* and evolving project. The API may change 17 | significantly in the near future, so use it at your own discretion. 18 | 19 | This section will be update as the project matures. 20 | 21 | 22 | ## Maven Artifacts 23 | 24 | ### Most Recent Release 25 | 26 | With Leiningen: 27 | 28 | [clojurewerkz/eep "1.0.0-beta1"] 29 | 30 | With Maven: 31 | 32 | 33 | clojurewerkz 34 | eep 35 | 1.0.0-beta1 36 | 37 | 38 | 39 | ## Documentation & Examples 40 | 41 | ### Quickstart 42 | 43 | In order to create an emitter, use `clojurewerkz.eep.emitter/create` function: 44 | 45 | ```clj 46 | (ns user 47 | (:require [clojurewerkz.eep.emitter :as eem])) 48 | 49 | (def emitter (eem/create {})) 50 | ``` 51 | 52 | You can register event handlers on an emitter by using handler helper 53 | functions. For example, in order to calculate sums for even and odd 54 | numbers, you can first define a `splitter` and then two `aggregators`, 55 | one for even and one for odd ones: 56 | 57 | ```clj 58 | (eem/defsplitter emitter :entrypoint (fn [i] (if (even? i) :even :odd))) 59 | 60 | (eem/defaggregator emitter :even (fn [acc i] (+ acc i)) 0) 61 | (eem/defaggregator emitter :odd (fn [acc i] (+ acc i)) 0) 62 | ``` 63 | 64 | Here, `:entrypoint`, `:even` and `:odd` are event types, unique event 65 | identifiers. 66 | 67 | In order to push data to emitter, use `clojurewerkz.eep.emitter/notify`, 68 | which takes an emitter, event type and payload: 69 | 70 | ```clj 71 | (eem/notify emitter :entrypoint 1) 72 | (eem/notify emitter :entrypoint 1) 73 | (eem/notify emitter :entrypoint 1) 74 | (eem/notify emitter :entrypoint 4) 75 | ``` 76 | 77 | You can then view the state of an aggregator like so: 78 | 79 | ```clj 80 | (eem/state (eem/get-handler emitter :odd)) ;; 3 81 | (eem/state (eem/get-handler emitter :even)) ;; 4 82 | ``` 83 | 84 | ## Core Concepts 85 | 86 | * `Emitter` is responsible for handler registration and event 87 | routing. It holds everything together. 88 | 89 | * `Event`s are dispatched by user code. An event is an 90 | arbitrary tuple of user-defined structure. There's no validation 91 | provided for it. 92 | 93 | * `Event Type` is a unique event type identifier, used for routing. It can 94 | be a number, a symbol, a keyword, a string or anything else. All the events 95 | coming into `Emitter` have a type. 96 | 97 | * `Handler` is a function and optional state attached to it. The function 98 | acts as a callback, executed whenever an event is matched on the type. 99 | The same handler can be used for multiple event types, but 100 | an event type can only have one handler at most. 101 | 102 | ## Handler types 103 | 104 | Each handler is attached to emitter with a `type`, which uniquely 105 | identifies it within an emitter. You can only attach a single handler 106 | for any given `type`. However, you can attach a single Handler to 107 | multiple `types`. 108 | 109 | Handlers may be stateful and stateless. `filter`, `splitter`, 110 | `transformer`, `multicast` and `observer` are __stateless__. On the 111 | other hand, `aggregator`, `buffer` and `rollup` are __stateful__. 112 | 113 | ### Stateful Handlers 114 | 115 | `aggregator` is initialized with initial value, then gets events of 116 | a certain type and aggregates state by applying aggregate function to 117 | current state and an incoming event. It's similar to `reduce` 118 | function in Clojure, except for it's applied to the stream of data. 119 | 120 | ```clj 121 | (def emitter (eem/create {})) ;; create the emitter 122 | (eem/defaggregator 123 | emitter ;; the emitter 124 | :accumulator ;; the event type to attach to 125 | (fn [acc i] (+ acc i)) ;; the function to apply to the stream 126 | 0) ;; the initial state 127 | ;; send 0-9 down the stream 128 | (doseq [i (range 10)] 129 | (eem/notify emitter :accumulator i)) 130 | 131 | ;; state is 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 132 | (eem/state (eem/get-handler emitter :accumulator)) ;; 45 133 | ``` 134 | 135 | `buffer` receives events of a certain type and stores them in a 136 | circular buffer with given capacity. As soon as capacity is 137 | reached, it drops events (first in, first out). 138 | 139 | ```clj 140 | (def emitter (eem/create {})) 141 | (eem/defbuffer 142 | emitter ;; the emitter 143 | :entry ;; the event type to attach to 144 | 5) ; the maximum values. 145 | ;; send 0-9 down the stream 146 | (doseq [i (range 10)] 147 | (eem/notify emitter :entry i)) 148 | 149 | ;; 0 1 2 3 4 were dropped, in that order. 150 | (eem/state (eem/get-handler emitter :entry)) ; [5 6 7 8 9] 151 | ``` 152 | 153 | `rollup` acts in a manner similar to buffer, except for it's 154 | time-bound but not capacity-bound, so whenever a time period is 155 | reached, it dispatches all the events to several other handlers. 156 | 157 | ```clj 158 | ;; Aggregates events for 100 milliseconds and emits them to :summarizer 159 | ;; as soon as timespan elapsed 160 | (eem/defrollup emitter :rollup-entry 100 :summarizer) 161 | ``` 162 | 163 | ### Stateless Handlers 164 | 165 | Note: calling `state` on a stateless handler will return `nil`. 166 | 167 | `filter` receives events of a certain type, and forwards ones for which 168 | `filter-fn` returns `true` to one or more other handlers: 169 | 170 | ```clj 171 | ;; Filters events going through the stream, allowing only even ones 172 | ;; to go through 173 | (def emitter (eem/create {})) 174 | (eem/deffilter 175 | emitter ;; the emitter 176 | :filtered ;; the event type to attach to 177 | number? ;; function to evaluate input 178 | :only-numbers) ;; event-type to forward input that evaluates to true 179 | ;; buffer to receive filtered input for example 180 | (eem/defbuffer emitter :only-numbers 5) 181 | ;; send some test data down the stream 182 | (doseq [i [1 "a" 5 "b" 9 "c" "d" 32 "eep" 58]] 183 | (eem/notify emitter :filtered i)) 184 | 185 | ;; all items where (number? item) was false were not forwarded 186 | (eem/state (eem/get-handler emitter :only-numbers)) ;; [1 5 9 32 58] 187 | ``` 188 | 189 | `splitter` receives events of a certain type, and dispatches them to 190 | type returned by predicate function. For example, you can split stream 191 | of integers to even and odd ones and process them down the pipeline 192 | differently. 193 | 194 | ```clj 195 | ;; Splits event stream to two parts, routing even events with :even 196 | ;; type and odd ones with :odd. 197 | (def emitter (eem/create {})) 198 | (eem/defsplitter 199 | emitter ;; the emitter 200 | :entry ;; the event type to attach to 201 | (fn [i] (if (number? i) :numbers :non-numbers))) ;; function evaluates input and returns which event type to forward to. 202 | ;; aggregator to receive the numbers for example 203 | (eem/defaggregator emitter :numbers + 0) 204 | ;; buffer to receive the numbers for example 205 | (eem/defbuffer emitter :non-numbers 5) 206 | ;; send some test data down the stream 207 | (doseq [i [1 "a" 5 "b" 9 "c" "d" 32 "eep" 58]] 208 | (eem/notify emitter :entry i)) 209 | 210 | ;; all numbers are sent to :numbers, all strings are sent to :not-numbers 211 | (eem/state (eem/get-handler emitter :numbers)) ;; 105, which is 1 + 5 + 9 +32 + 58 212 | (eem/state (eem/get-handler emitter :non-numbers)) ;; ["a" "b" "c" "d" "eep"], which is all the non-numbers 213 | ``` 214 | 215 | `transformer` defines a transformer that gets typed tuples, applies 216 | transformation function to each one of them and forwards them to 217 | one or more other handlers. It's similar to applying `map` to 218 | elements of a list, except for function is applied to stream of data. 219 | 220 | ```clj 221 | ;; Transforms event stream by multiplying each event to 2 222 | (def emitter (eem/create {})) 223 | 224 | ;; Define the transformer function 225 | (defn fizzbuzzer [i] 226 | (cond 227 | (zero? (mod i 15)) "FizzBuzz" 228 | (zero? (mod i 5)) "Buzz" 229 | (zero? (mod i 3)) "Fizz" 230 | :else i)) 231 | 232 | (eem/deftransformer 233 | emitter ;; the emitter 234 | :entry ;; the event type to attach to 235 | fizzbuzzer ;; the transformer function 236 | :fizzbuzz) ;; the new event type to forward to 237 | ;; a buffer to receive output for example 238 | (eem/defbuffer emitter :fizzbuzz 5) 239 | 240 | ;; send some test data down the stream 241 | (doseq [i (range 10)] 242 | (eem/notify emitter :entry i)) 243 | 244 | ;; Anything divided by 3 is "Fizz", anything divided by 5 is "Buzz", and anything divided by 15 is "FizzBuzz" 245 | (eem/state (eem/get-handler emitter :fizzbuzz)) ;; ["Buzz" "Fizz" 7 8 "Fizz"] 246 | ``` 247 | 248 | 249 | `multicast` receives events of a certain type and broadcasts them 250 | to several handlers with different types. For example, whenever an 251 | alert is received, you may want to send notifications via email, 252 | IRC, Jabber and append event to the log file. 253 | 254 | ```clj 255 | ;; Redistributes incoming events, routing them to multiple other event types 256 | (def emitter (eem/create {})) 257 | (eem/defmulticast 258 | emitter ;; the emitter 259 | :entry ;; the event type to attach to 260 | [:accumulator :incrementer :multiplier]) ;; vector of event types to forward to 261 | 262 | ;; set up aggregators for example 263 | (eem/defaggregator emitter :accumulator (fn [acc i] (+ acc i)) 0) 264 | (eem/defaggregator emitter :incrementer (fn [acc i] (+ acc 1)) 0) 265 | (eem/defaggregator emitter :multiplier (fn [acc i] (* acc i)) 1) 266 | 267 | ;; send test data down the stream 268 | (doseq [i [2 3 4]] 269 | (eem/notify emitter :entry i)) 270 | 271 | (eem/state (eem/get-handler emitter :accumulator)) ;; 9, 2 + 3 + 4 272 | (eem/state (eem/get-handler emitter :incrementer)) ;; 3, 1 + 1 + 1 273 | (eem/state (eem/get-handler emitter :multiplier)) ;; 24, 2 * 3 * 4 274 | 275 | ;; It's also possible to attach additional multicast entries. This will 276 | ;; append :subtractor to the list of streams broadcasted by :entry from that point forward 277 | (eem/defmulticast emitter :entry [:subtractor]) 278 | (eem/defaggregator emitter :subtractor (fn [acc i] (- acc i)) 0) 279 | (eem/notify emitter :entry 2) 280 | 281 | (eem/state (eem/get-handler emitter :accumulator)) ;; 11, 2 + 3 + 4 + 2 282 | (eem/state (eem/get-handler emitter :incrementer)) ;; 4, 1 + 1 + 1 + 1 283 | (eem/state (eem/get-handler emitter :multiplier)) ;; 48, 2 * 3 * 4 * 2 284 | (eem/state (eem/get-handler emitter :subtractor)) ;; -2 285 | ``` 286 | 287 | `observer` receives events of a certain type and runs function 288 | (potentially with side-effects) on each one of them. 289 | 290 | ```clj 291 | (def emitter (eem/create {})) 292 | 293 | ;; our function with side effects 294 | (defn announcer [item] 295 | (println (str "I would like to announce: " item))) 296 | 297 | (eem/defobserver emitter :announce announcer) 298 | 299 | (eem/notify emitter :announce "This Item") 300 | ;; prints "I would like to announce: This Item" 301 | ``` 302 | 303 | ## Topology DSL 304 | 305 | There's a DSL that threads emitter through all handler declarations, 306 | in order to create aggregation topologies in a more concise and obvious 307 | way: 308 | 309 | ```clj 310 | (def emitter 311 | (eem/build-topology (eem/create {}) 312 | :entry (eem/defsplitter (fn [i] (if (even? i) :even :odd))) 313 | :even (eem/defbuffer 5) 314 | :odd (eem/defbuffer 5))) 315 | 316 | (doseq [i (range 10)] 317 | (eem/notify emitter :entry i)) 318 | 319 | (eem/state (eem/get-handler emitter :even)) ;; [0 2 4 6 8] 320 | (eem/state (eem/get-handler emitter :odd)) ;; [1 3 5 7 9] 321 | ``` 322 | 323 | Alternatively, you can use Clojure `->` for creating concise topologies: 324 | 325 | ```clj 326 | (def emitter 327 | (-> (eem/create {}) 328 | (eem/defsplitter :entry (fn [i] (if (even? i) :even :odd))) 329 | (eem/defbuffer :even 5) 330 | (eem/defbuffer :odd 5))) 331 | 332 | (doseq [i (range 10)] 333 | (eem/notify emitter :entry i)) 334 | 335 | (eem/state (eem/get-handler emitter :even)) ;; [0 2 4 6 8] 336 | (eem/state (eem/get-handler emitter :odd)) ;; [1 3 5 7 9] 337 | ``` 338 | 339 | ## Topology visualization 340 | 341 | You can also visualize your topology by calling `clojurewerkz.eep.visualization/visualise-graph` 342 | and giving it an emitter. You'll get an image like this one: 343 | 344 | [![Topology Visualization Example](http://coffeenco.de/assets/images/topology_example.png)](http://coffeenco.de/assets/images/topology_example.png) 345 | 346 | ## Windows 347 | 348 | Windows and buffers are an essential part of event processing. 349 | We've added the most important implementations of windowed 350 | operations, such as sliding, tumbling, monotonic and timed windows 351 | to EEP to allow you to use them within topologies. 352 | 353 | ### Sliding window 354 | 355 | Sliding windows have a fixed a-priori known size. 356 | 357 | Example: Sliding window of size 2 computing sum of values. 358 | 359 | ``` 360 | t0 t1 (emit) t2 (emit) tN 361 | +---+ +---+---+ -...-...- 362 | | 1 | | 2 | 1 | <3> : x : x : 363 | +---+ +---+---+ _...+---+---+ ... 364 | | 2 | | 2 | 3 | <5> 365 | +---+ +---+---+ 366 | | 4 | 367 | +---+ 368 | ``` 369 | 370 | Useful to hold last `size` elements. 371 | 372 | ### Tumbling window 373 | 374 | Tumbling windows (here) have a fixed a-priori known size. 375 | 376 | Example: Tumbling window of size 2 computing sum of values. 377 | 378 | ``` 379 | t0 t1 (emit) t2 t3 (emit) t4 380 | +---+ +---+---+ -...-...- 381 | | 1 | | 2 | 1 | <3> : x : x : 382 | +---+ +---+---+ -...+---+---+ +---+---+ ... 383 | | 3 | | 4 | 3 | <7> 384 | +---+ +---+---+ 385 | ``` 386 | 387 | Useful to accumulate `size` elements and aggregate on overflow. 388 | 389 | ### Monotonic window 390 | 391 | Makes a clock tick on every call. Whenever clock is elapsed, emits to aggregator. 392 | 393 | In essence, it's an alternative implementation of tumbling-window that allows 394 | to use custom emission control rather than having a buffer overflow check. 395 | 396 | Useful for cases when emission should be controlled by arbitrary function, 397 | possibly unrelated to window contents. 398 | 399 | ### Timed window 400 | 401 | A simple timed window, that runs on wall clock. Receives events and stores them 402 | until clock is elapsed, emits for aggregation after that. 403 | 404 | In essence, it's an alternative implementation of tumbling-window or monotonic-window 405 | that allows wall clock control. 406 | 407 | Useful for accumulating events for time-bound events processing, accumulates events 408 | for a certain period of time (for example, 1 minute), and aggregates them. 409 | 410 | ## Busy-spin 411 | 412 | Whenever you create an emitter, you may notice that one of your cores is 100% busy. You should 413 | not worry about it, since all dispatchers use a tight loop for dispatch, without sleeping, therefore 414 | not yielding control back to OS, so OS defines that as 100% processor load. 415 | 416 | ## Supported Clojure Versions 417 | 418 | EEP requires Clojure 1.6+. 419 | 420 | 421 | ## Development 422 | 423 | EEP uses [Leiningen 2](https://github.com/technomancy/leiningen/blob/master/doc/TUTORIAL.md). Make 424 | sure you have it installed and then run tests against all supported Clojure versions using 425 | 426 | lein all test 427 | 428 | Then create a branch and make your changes on it. Once you are done with your changes and all 429 | tests pass, submit a pull request on Github. 430 | 431 | ## License 432 | 433 | Copyright © 2014-2016 Michael Klishin, Alex Petrov, and the ClojureWerkz team. 434 | 435 | Double licensed under the [Eclipse Public License](http://www.eclipse.org/legal/epl-v10.html) (the same as Clojure) or 436 | the [Apache Public License 2.0](http://www.apache.org/licenses/LICENSE-2.0.html). 437 | --------------------------------------------------------------------------------