├── .nrepl-history ├── tests.edn ├── .dir-locals.el ├── bin └── kaocha ├── doc └── intro.md ├── .gitignore ├── test └── cddr │ └── ksml │ ├── core_test.clj │ └── eval_test.clj ├── examples ├── resources │ └── logging.edn └── ksml │ └── examples │ └── anomaly_detection.clj ├── deps.edn ├── CHANGELOG.md ├── .circleci └── config.yml ├── src └── cddr │ └── ksml │ ├── core.clj │ ├── ring.clj │ └── eval.clj ├── README.md └── LICENSE /.nrepl-history: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests.edn: -------------------------------------------------------------------------------- 1 | #kaocha/v1 {} 2 | -------------------------------------------------------------------------------- /.dir-locals.el: -------------------------------------------------------------------------------- 1 | ((nil . ((cider-boot-parameters . "dev")))) 2 | -------------------------------------------------------------------------------- /bin/kaocha: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | clojure -M:test -m kaocha.runner "$@" 3 | -------------------------------------------------------------------------------- /doc/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction to streamin 2 | 3 | TODO: write [great documentation](http://jacobian.org/writing/what-to-write/) 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .hg/ 11 | .cpcache/ 12 | .mach/ 13 | .hgignore 14 | 15 | -------------------------------------------------------------------------------- /test/cddr/ksml/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns cddr.ksml.core-test 2 | (:require 3 | [clojure.test :refer :all] 4 | [cddr.ksml.core :refer [ksml* ksml v->]])) 5 | 6 | (deftest test-v-> 7 | (is [:bar [:baz [:foo]]] 8 | (v-> [:foo] 9 | [:bar] 10 | [:baz]))) 11 | -------------------------------------------------------------------------------- /examples/resources/logging.edn: -------------------------------------------------------------------------------- 1 | {:level :info 2 | :console true 3 | :encoder :pattern 4 | :pattern "%p [%d] %t - %c %m%n" 5 | ;; kafka/zookeeper are super verbose which is great when something goes 6 | ;; wrong but a bit too much noise for dev mode 7 | :overrides {"org.apache.kafka" :error 8 | "org.apache.zookeeper" :error}} 9 | -------------------------------------------------------------------------------- /deps.edn: -------------------------------------------------------------------------------- 1 | {:deps 2 | {org.clojure/clojure {:mvn/version "RELEASE"} 3 | org.apache.kafka/kafka-streams {:mvn/version "3.1.0"}} 4 | 5 | :paths 6 | ["src" "resources" "test"] 7 | 8 | :aliases 9 | {:test {:extra-paths ["test"] 10 | :extra-deps {org.apache.kafka/kafka-streams-test-utils {:mvn/version "3.1.0"} 11 | lambdaisland/kaocha {:mvn/version "1.60.977"}}}}} 12 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | All notable changes to this project will be documented in this file. This change log follows the conventions of [keepachangelog.com](http://keepachangelog.com/). 3 | 4 | ## [Unreleased] 5 | ### Changed 6 | - Add a new arity to `make-widget-async` to provide a different widget shape. 7 | 8 | ## [0.1.1] - 2017-08-20 9 | ### Changed 10 | - Documentation on how to make the widgets. 11 | 12 | ### Removed 13 | - `make-widget-sync` - we're all async, all the time. 14 | 15 | ### Fixed 16 | - Fixed widget maker to keep working when daylight savings switches over. 17 | 18 | ## 0.1.0 - 2017-08-20 19 | ### Added 20 | - Files from the new template. 21 | - Widget maker public API - `make-widget-sync`. 22 | 23 | [Unreleased]: https://github.com/your-name/streamin/compare/0.1.1...HEAD 24 | [0.1.1]: https://github.com/your-name/streamin/compare/0.1.0...0.1.1 25 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Clojure CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-clojure/ for more details 4 | # 5 | version: 2.1 6 | jobs: 7 | build: 8 | docker: 9 | # specify the version you desire here 10 | - image: circleci/clojure:tools-deps-1.10.1.727 11 | 12 | # Specify service dependencies here if necessary 13 | # CircleCI maintains a library of pre-built images 14 | # documented at https://circleci.com/docs/2.0/circleci-images/ 15 | # - image: circleci/postgres:9.4 16 | 17 | working_directory: ~/repo 18 | 19 | environment: 20 | # Customize the JVM maximum heap limit 21 | JVM_OPTS: -Xmx3200m 22 | 23 | steps: 24 | - checkout 25 | 26 | # Download and cache dependencies 27 | - restore_cache: 28 | keys: 29 | - v2-dependencies-{{ checksum "deps.edn" }} 30 | # fallback to using the latest cache if no exact match is found 31 | - v2-dependencies- 32 | 33 | - run: clojure -P -M:test 34 | 35 | - save_cache: 36 | paths: 37 | - ~/.m2 38 | key: v2-dependencies-{{ checksum "deps.edn" }} 39 | 40 | # run tests! 41 | - run: bin/kaocha 42 | -------------------------------------------------------------------------------- /src/cddr/ksml/core.clj: -------------------------------------------------------------------------------- 1 | (ns cddr.ksml.core 2 | (:require 3 | [cddr.ksml.eval :as ksml.eval]) 4 | (:import 5 | (org.apache.kafka.streams StreamsBuilder KafkaStreams StreamsConfig) 6 | (org.apache.kafka.streams.kstream Predicate))) 7 | 8 | (defn builder [] 9 | (StreamsBuilder.)) 10 | 11 | (defn kafka-config 12 | [config] 13 | (if (instance? StreamsConfig config) 14 | config 15 | (StreamsConfig. config))) 16 | 17 | ;; v1 18 | (defn ksml* 19 | [expr] 20 | (eval 21 | `(binding [ksml.eval/*builder* (builder)] 22 | ~(ksml.eval/eval expr) 23 | ksml.eval/*builder*))) 24 | 25 | ;; v2 26 | ;; (defn ksml* 27 | ;; [expr] 28 | ;; (eval 29 | ;; `(binding [*builder* (builder)] 30 | ;; (.. *builder* ~expr) 31 | ;; *builder*))) 32 | 33 | (defmacro ksml 34 | [expr] 35 | `(binding [ksml.eval/*builder* (StreamsBuilder.)] 36 | ~(ksml.eval/eval expr) 37 | ksml.eval/*builder*)) 38 | 39 | (defn streams 40 | [builder config] 41 | (KafkaStreams. builder (kafka-config config))) 42 | 43 | (defmacro v-> 44 | "Like Clojure's `->` but expects the 'forms' to be vectors" 45 | [x & forms] 46 | (loop [x x, forms forms] 47 | (if forms 48 | (let [form (first forms) 49 | threaded (if (vector? form) 50 | `[~(first form) ~x ~@(next form)] 51 | (vector form x))] 52 | (recur threaded (next forms))) 53 | x))) 54 | -------------------------------------------------------------------------------- /examples/ksml/examples/anomaly_detection.clj: -------------------------------------------------------------------------------- 1 | (ns ksml.examples.anomaly-detection 2 | (:require 3 | [cddr.ksml.core :as k :refer [v->]] 4 | [clojure.edn :as edn] 5 | [clojure.java.io :as io] 6 | [clojure.tools.logging :as log] 7 | [unilog.config :refer [start-logging!]]) 8 | (:import 9 | (org.apache.kafka.streams KafkaStreams) 10 | (org.apache.kafka.streams StreamsConfig) 11 | (org.apache.kafka.common.serialization Serdes))) 12 | 13 | (let [logging (-> "logging.edn" 14 | (io/resource) 15 | (slurp) 16 | (edn/read-string))] 17 | (start-logging! logging)) 18 | 19 | (def string-serde (-> (Serdes/String) 20 | (.getClass) 21 | (.getName))) 22 | 23 | (defn many-clicks? [_ click-count] 24 | (> click-count 3)) 25 | 26 | (defn value-nil? [_ val] 27 | (nil? val)) 28 | 29 | (defn anomalous-users 30 | [clicks] 31 | (v-> clicks 32 | [:map [:key-value-mapper 33 | (fn [_ username] 34 | [username username])]] 35 | [:group-by-key] 36 | [:count [:time-window (* 60 1000)] "UserCountStore"] 37 | [:filter [:predicate many-clicks?]])) 38 | 39 | (defn anomalous-users-for-console 40 | [users] 41 | (v-> users 42 | [:to-stream] 43 | [:filter [:predicate value-nil?]])) 44 | 45 | (def config 46 | {"application.id" "anomaly-detection-lambda-example" 47 | "client.id" "anomaly-detection-lambda-example-client" 48 | "bootstrap.servers" "10.11.12.13:9092" 49 | "default.key.serde" string-serde 50 | "default.value.serde" string-serde 51 | "commit.interval" 500}) 52 | 53 | (defn -main [& args] 54 | (let [clicks [:stream [:strs "UserClicks"]] 55 | builder (k/ksml* (-> (anomalous-users clicks) 56 | (anomalous-users-for-console))) 57 | streams (k/streams builder config)] 58 | 59 | (doto streams 60 | (.cleanUp) 61 | (.start)) 62 | 63 | (log/info "Started anomaly detection streams") 64 | 65 | (doto (Runtime/getRuntime) 66 | (.addShutdownHook (Thread. #(.close streams)))))) 67 | 68 | -------------------------------------------------------------------------------- /src/cddr/ksml/ring.clj: -------------------------------------------------------------------------------- 1 | (ns cddr.ksml.ring 2 | "A Kafka Streams application consists of a set of self-coordinating processes 3 | that distribute the work of processing one or more input topics. Each process 4 | typically only has access to data pertaining to a sub-set of the entire dataset. 5 | 6 | The KafkaStreams object keeps track of the mapping between the host of each 7 | process and it's corresponding keys in local storage. This lets us discover 8 | at request-time the host that is capable of handling some HTTP request that 9 | is backed by the distributed local storage of a kafka streams application. 10 | 11 | The handler defined in this namespace either handles the request locally if it 12 | can, or if necessary forwards the request on to the remote host and responds 13 | passes the response onto the original requestor") 14 | 15 | (defn remote? 16 | [location self] 17 | (= (select-keys location [:server-name :server-port]) 18 | (select-keys self [:server-name :server-port]))) 19 | 20 | (defn handler 21 | "Return a ring handler that locates the state required to serve a response 22 | and delegates the request to a handler with access to the the state. 23 | 24 | To support a wide variety of use-cases, this handler takes a map with the 25 | following keyword parameters 26 | 27 | :find-host Finds the location of the state required to fulfill this 28 | request. The returned location should be a map with the keys 29 | :server-name and :server-port 30 | 31 | :remote If the state is determined to be located elsewhere, this 32 | function will be invoked to serve a response. It accepts 33 | a location, and the original request 34 | 35 | 36 | :local If the state is determined to be located on this instance, 37 | the `local` function will be invoked to serve a response. It 38 | accepts the original `request`, and the specified `streams` 39 | object." 40 | [{:keys [find-host 41 | remote 42 | local 43 | streams]}] 44 | (fn [request] 45 | (let [location (find-host streams request)] 46 | (if (remote? location request) 47 | (-> request 48 | (assoc :location location) 49 | (remote)) 50 | 51 | (-> request 52 | (assoc :streams streams) 53 | (local)))))) 54 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ksml 2 | 3 | Ksml is a library for representing kafka streams topologies as data 4 | 5 | ## Overview 6 | 7 | Kafka Streams is a client library for building mission-critical 8 | real-time applications and microservices, where the input and/or 9 | output data is stored in Kafka clusters. Kafka Streams combines the 10 | simplicity of writing and deploying standard Java and Scala 11 | applications on the client side with the benefits of Kafka's 12 | server-side cluster technology to make these applications highly 13 | scalable, elastic, fault-tolerant, distributed, and much more. 14 | 15 | A stream processing application is any program that makes use of the 16 | Kafka Streams library. It defines its computational logic through one 17 | or more processor topologies, where a processor topology is a graph of 18 | stream processors (nodes) that are connected by streams (edges). 19 | 20 | KSML is a library for representing these topologies as data. It 21 | uses vectors to represent distributed streaming primitives like map, 22 | join, filter etc, and plain old Clojure functions for representing the 23 | processing logic to be performed by each node. 24 | 25 | ## Rationale 26 | 27 | The clojure community generally places a high value on data. Expressing 28 | the computation as data provides a number of advantages 29 | 30 | * it becomes easy to compose topology fragments using only the 31 | standard collection manipulation tools like map, concat, merge etc 32 | 33 | * you can report on it, analyze it, graph it etc, limited 34 | only by your imagination 35 | 36 | * you can transform it for the purposes of instrumentation to produce 37 | standardized metrics, logging, and exception handling 38 | 39 | ## Usage 40 | 41 | watch out. code has just been heavily refactored and still need to review 42 | the examples below and make more extensive docs 43 | 44 | ```clojure 45 | (ns com.ksml.wordcount 46 | (:require 47 | [clojure.string :as str] 48 | [cddr.ksml.core :refer [ksml* v->]]) 49 | (:import 50 | (org.apache.kafka.streams.kstream KStreamBuilder) 51 | (org.apache.kafka.streams KafkaStreams))) 52 | 53 | (defn- split-line 54 | [line] 55 | (-> line 56 | (str/lower-case) 57 | (str/split #"\\W+"))) 58 | 59 | (defn wordcount 60 | [lines] 61 | (v-> lines 62 | [:flat-map-values [:value-mapper split-line]] 63 | [:group-by 64 | [:key-value-mapper (fn [k word] word)]] 65 | [:count "Counts"] 66 | [:to! [:serde 'String] 67 | [:serde 'Long] 68 | "WordsWithCountsTopic"])) 69 | 70 | (defn -main [& args] 71 | (let [lines [:stream "TextLinesTopic"]] 72 | (doto (KafkaStreams. (ksml* (wordcount lines))) 73 | (.start)))) 74 | ``` 75 | 76 | ## Bugs 77 | 78 | While there is a decent [testsuite](https://github.com/cddr/ksml/blob/master/test/cddr/ksml/eval_test.clj) 79 | this library is still in the early stages. I might need to tweak the 80 | data structures as we use it to build programs and discover more 81 | optimal ways of describing topologies. 82 | 83 | Credit to @ztellman for the idea and hiccup, and SICP for implementation 84 | ideas. 85 | 86 | ## License 87 | 88 | Copyright © 2017 Andy Chambers 89 | 90 | Distributed under the Eclipse Public License either version 1.0 or (at 91 | your option) any later version. 92 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor to control, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of New York and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /src/cddr/ksml/eval.clj: -------------------------------------------------------------------------------- 1 | (ns cddr.ksml.eval 2 | "ksml is a clojure/data based representation of a kafka streams application. 3 | 4 | This namespace defines the evaluator for ksml. At a high-level though, there 5 | are only a few types of expressions in a ksml program 6 | 7 | * self-evaluating things like e.g. 8 | - numbers, strings, regexes 9 | - clojure functions 10 | 11 | * special operators (the kafka streams DSL) 12 | - :stream, :table, :branch, :map etc 13 | 14 | For each special operator, the code in here knows how to convert that operator 15 | (together with it's optional args) into the underlying java interop." 16 | (:refer-clojure :exclude [eval reducer]) 17 | (:require 18 | [clojure.string :as str] 19 | [clojure.spec.alpha :as s]) 20 | (:import 21 | (java.util.regex Pattern) 22 | (java.time Duration) 23 | (org.apache.kafka.common.serialization Serializer Deserializer Serdes) 24 | (org.apache.kafka.streams KeyValue StreamsBuilder 25 | Topology Topology$AutoOffsetReset) 26 | (org.apache.kafka.streams.state Stores) 27 | (org.apache.kafka.streams.processor Processor TimestampExtractor ProcessorSupplier 28 | StreamPartitioner 29 | FailOnInvalidTimestamp, LogAndSkipOnInvalidTimestamp, 30 | UsePartitionTimeOnInvalidTimestamp, WallclockTimestampExtractor) 31 | (org.apache.kafka.streams.kstream Named Grouped Consumed Materialized 32 | KStream JoinWindows TimeWindows 33 | Predicate 34 | Initializer Aggregator Reducer Merger 35 | ForeachAction 36 | ValueMapper KeyValueMapper 37 | ValueJoiner 38 | Transformer TransformerSupplier))) 39 | 40 | (declare eval) 41 | 42 | (def ^:dynamic *builder*) 43 | 44 | (defmacro defsyntax [name rules] 45 | `(do 46 | (def ~name ~rules) 47 | (defn ~(symbol (str name "?")) [expr#] 48 | (when (vector? expr#) 49 | (contains? ~name (first expr#)))))) 50 | 51 | (defn camelize 52 | [in] 53 | (str/replace in #"-(\w)" 54 | #(str/upper-case (second %1)))) 55 | 56 | (defn build-op 57 | [op & args] 58 | `(.. *builder* (~op ~@args))) 59 | 60 | (defn apply-args-to-op 61 | [op & args] 62 | `(~op ~@args)) 63 | 64 | (defn key-value 65 | [k v] 66 | (KeyValue. k v)) 67 | 68 | (def builders 69 | { 70 | ;; builders 71 | :stream build-op 72 | :table build-op 73 | :global-table build-op 74 | :merge (fn [op & args] 75 | `(.. *builder* (~op (into-array KStream 76 | (vector ~@args))))) 77 | :named (fn [op & args] 78 | `(Named/as ~(first args))) 79 | :strs (fn [op & args] 80 | `(into-array String (vector ~@args))) 81 | 82 | :topics (fn [op & args] 83 | `(vector ~@args)) 84 | 85 | :duration (fn [op & args] 86 | `(Duration/parse ~(first args))) 87 | 88 | :offset-reset (fn [op & args] 89 | `(Topology$AutoOffsetReset/valueOf 90 | ~(first args))) 91 | 92 | :timestamp-extractor (fn [op & args] 93 | `(new ~(first args)))}) 94 | 95 | (def mappers 96 | { 97 | :to-stream (fn [op stream & args] 98 | `(.. ~stream 99 | (~op ~@args))) 100 | :branch (fn [op stream & args] 101 | (let [arg1 (first args)] 102 | (if (= 'org.apache.kafka.streams.kstream.Named/as (first arg1)) 103 | `(.. ~stream 104 | (~op ~arg1 (into-array Predicate (vector ~@(rest args))))) 105 | `(.. ~stream 106 | (~op (into-array Predicate (vector ~@args))))))) 107 | :filter (fn [op stream predicate-fn & args] 108 | `(.. ~stream 109 | (~op ~predicate-fn ~@args))) 110 | :filter-not (fn [op stream predicate-fn & args] 111 | `(.. ~stream 112 | (~op ~predicate-fn ~@args))) 113 | :flat-map (fn [op stream map-fn & args] 114 | `(.. ~stream 115 | (~op ~map-fn))) 116 | :flat-map-values (fn [op stream map-fn & args] 117 | `(.. ~stream 118 | (~op ~map-fn))) 119 | :foreach (fn [op stream each-fn & args] 120 | `(.. ~stream 121 | (~op ~each-fn))) 122 | :map (fn [op stream map-fn & args] 123 | `(.. ~stream 124 | (~op ~map-fn))) 125 | :map-values (fn [op stream map-fn & args] 126 | `(.. ~stream 127 | (~op ~map-fn))) 128 | :select-key (fn [op stream key-fn & args] 129 | `(.. ~stream 130 | (~op ~key-fn)))}) 131 | 132 | (def joiners 133 | {:join (fn [op left right join-fn & args] 134 | `(.. ~left 135 | (~op ~right ~join-fn ~@args))) 136 | 137 | :left-join (fn [op left right join-fn & args] 138 | `(.. ~left 139 | (~op ~right ~join-fn ~@args))) 140 | 141 | :outer-join (fn [op left right join-fn & args] 142 | `(.. ~left 143 | (~op ~right ~join-fn ~@args))) 144 | }) 145 | 146 | (def aggregators 147 | { 148 | :group-by (fn [op stream group-fn & optional] 149 | `(.. ~stream 150 | (~op ~group-fn ~@optional))) 151 | 152 | :group-by-key (fn [op stream & optional] 153 | `(.. ~stream 154 | (~op ~@optional))) 155 | 156 | 157 | :aggregate (fn [op stream & args] 158 | `(.. ~stream 159 | (~op ~@args))) 160 | :count (fn [op stream & args] 161 | `(.. ~stream 162 | (~op ~@args))) 163 | :reduce (fn [op stream & args] 164 | `(.. ~stream 165 | (~op ~@args))) 166 | }) 167 | 168 | (def io-syntax 169 | {:materialized (fn [op init & modifiers] 170 | (let [m `(. Materialized ~init)] 171 | (if (empty? modifiers) 172 | m 173 | `(.. ~m ~@modifiers)))) 174 | 175 | ;; consumed 176 | :consumed (fn [op init & modifiers] 177 | (let [c `(. Consumed ~init)] 178 | (if (empty? modifiers) 179 | c 180 | `(.. ~c ~@modifiers)))) 181 | :grouped (fn [op init & modifiers] 182 | (let [g `(. Grouped ~init)] 183 | (if (empty? modifiers) 184 | g 185 | `(.. ~g ~@modifiers)))) 186 | 187 | :repartitioned (fn [op init & modifiers] 188 | (let [r `(. Repartitioned ~init)] 189 | (if (empty? modifiers) 190 | r 191 | `(.. ~r ~@modifiers)))) 192 | 193 | :with apply-args-to-op 194 | :as apply-args-to-op 195 | :numberOfPartitions apply-args-to-op 196 | :streamPartitioner apply-args-to-op 197 | :withNumberOfPartitions apply-args-to-op 198 | :withStreamPartitioner apply-args-to-op 199 | :withKeySerde apply-args-to-op 200 | :withValueSerde apply-args-to-op 201 | :withName apply-args-to-op 202 | :withTimestampExtractor apply-args-to-op 203 | :withOffsetResetPolicy apply-args-to-op 204 | :withCachingDisabled apply-args-to-op 205 | :withCachingEnabled apply-args-to-op 206 | :withLoggingDisabled apply-args-to-op 207 | :withLoggingEnabled apply-args-to-op 208 | :withRetention apply-args-to-op}) 209 | 210 | (def serde-syntax 211 | {:serde (fn [op & args] 212 | `(.. Serdes ~@args)) 213 | :serde-from (fn [op & args] 214 | `(~op ~@args))}) 215 | 216 | (def store-syntax 217 | {:stores (fn [op & args] 218 | `(.. Stores ~@args)) 219 | :inMemoryKeyValueStore apply-args-to-op 220 | :inMemorySessionStore apply-args-to-op 221 | :inMemoryWindowStore apply-args-to-op 222 | :lruMap apply-args-to-op 223 | :persistentKeyValueStore apply-args-to-op 224 | :persistentSessionStore apply-args-to-op 225 | :persistentTimestampedKeyValueStore apply-args-to-op 226 | :persistentWindowStore apply-args-to-op}) 227 | 228 | ;; ksml syntax rules 229 | 230 | (defsyntax application 231 | (merge builders 232 | mappers 233 | joiners 234 | aggregators 235 | io-syntax 236 | serde-syntax 237 | store-syntax)) 238 | 239 | (defsyntax lambda 240 | {:predicate (fn [pred-fn] 241 | (reify Predicate 242 | (test [_ k v] 243 | (boolean (pred-fn k v))))) 244 | 245 | :key-value-mapper (fn [map-fn] 246 | (reify KeyValueMapper 247 | (apply [_ k v] 248 | (apply key-value (map-fn k v))))) 249 | 250 | :value-mapper (fn [vmap-fn] 251 | (reify ValueMapper 252 | (apply [_ v] 253 | (vmap-fn v)))) 254 | 255 | :value-joiner (fn [join-fn] 256 | (reify ValueJoiner 257 | (apply [_ left right] 258 | (join-fn left right)))) 259 | 260 | :foreach-action (fn [each-fn] 261 | (reify ForeachAction 262 | (apply [_ k v] 263 | (each-fn k v)))) 264 | 265 | :initializer (fn [init-fn] 266 | (reify Initializer 267 | (apply [_] 268 | (init-fn)))) 269 | 270 | :aggregator (fn [agg-fn] 271 | (reify Aggregator 272 | (apply [_ k v agg] 273 | (agg-fn agg [k v])))) 274 | 275 | :merger (fn [merge-fn] 276 | (reify Merger 277 | (apply [_ k agg1 agg2] 278 | (merge-fn k agg1 agg2)))) 279 | 280 | :reducer (fn [reduce-fn] 281 | (reify Reducer 282 | (apply [_ v1 v2] 283 | (reduce-fn v1 v2)))) 284 | 285 | :partitioner (fn [partition-fn] 286 | (reify StreamPartitioner 287 | (partition [this topic k v i] 288 | (partition-fn topic k v i)))) 289 | 290 | :serializer (fn serializer 291 | [ser-fn] 292 | (reify Serializer 293 | (close [_]) 294 | (configure [this cfg is-key?]) 295 | (serialize [this topic headers data] 296 | (ser-fn topic headers data)) 297 | (serialize [this topic data] 298 | (ser-fn topic data)))) 299 | 300 | :deserializer (fn deserializer 301 | [deser-fn] 302 | (reify Deserializer 303 | (close [_]) 304 | (configure [this cfg is-key?]) 305 | (deserialize [this topic headers data] 306 | (deser-fn topic headers data)) 307 | (deserialize [this topic data] 308 | (deser-fn topic data)))) 309 | 310 | :processor-supplier (fn processor-supplier 311 | ([process-fn] 312 | (processor-supplier process-fn (constantly nil))) 313 | ([process-fn init-fn] 314 | (reify ProcessorSupplier 315 | (get [_] 316 | (let [ctx (atom nil)] 317 | (reify Processor 318 | (init [_ context] 319 | (reset! ctx context) 320 | (when init-fn 321 | (init-fn ctx))) 322 | (process [_ k v] 323 | (process-fn @ctx k v)))))))) 324 | 325 | :transformer-supplier (fn transformer-supplier 326 | ([transform-fn] 327 | (transformer-supplier transform-fn nil)) 328 | ([transform-fn init-fn] 329 | (reify TransformerSupplier 330 | (get [_] 331 | (let [ctx (atom nil)] 332 | (reify Transformer 333 | (init [_ context] 334 | (reset! ctx context) 335 | (when init-fn 336 | (init-fn ctx))) 337 | (transform [_ k v] 338 | (transform-fn @ctx k v)))))))) 339 | }) 340 | 341 | ;; Super simple evaluator. In ksml, there are only a few types of 342 | ;; expression 343 | ;; 344 | ;; * self-evaluating 345 | ;; e.g. strings, numbers, functions, raw clojure code 346 | ;; 347 | ;; * lambdas 348 | ;; Tend to take a function argument and evaluate to an instance 349 | ;; satisfying a Kafka Streams interface like Predicate, or KeyValueMapper. 350 | ;; When evaluating lambda expressions, we skip evaluating the 351 | ;; the provided function and instead wrap it in the corresponding Kafka 352 | ;; Streams object. 353 | ;; 354 | ;; * application 355 | ;; Application expressions consist of an operator (e.g. :stream, :filter, 356 | ;; :branch etc). The evaluator evaluates the arguments and passes the 357 | ;; evaluated expressions to the operator specific function for expansion 358 | ;; into code that can be evaluated by standard Clojure `eval`. 359 | 360 | (defn clojure? 361 | "In ksml, a `clojure?` expression is a list whose first element 362 | is a symbol. Expressions like this will be passed through the 363 | evaluator as-is" 364 | [expr] 365 | (symbol? (first expr))) 366 | 367 | (defn self-evaluating? 368 | [expr] 369 | (or 370 | (string? expr) 371 | (class? expr) 372 | (map? expr) 373 | (instance? Pattern expr) 374 | (number? expr) 375 | (and (list? expr) 376 | (symbol? (first expr))))) 377 | 378 | (defn values 379 | [exprs] 380 | (map eval exprs)) 381 | 382 | (defn eval 383 | [expr] 384 | (cond 385 | (self-evaluating? expr) expr 386 | (lambda? expr) (let [[op & functions] expr] 387 | `(apply (lambda ~op) 388 | ~(first functions) 389 | ~(rest functions))) 390 | (application? expr) (let [[op & args] expr] 391 | (apply (application op) 392 | (-> op name camelize symbol) 393 | (values args))) 394 | 395 | :else (throw (ex-info "unknown expression: " {:expr expr})))) 396 | -------------------------------------------------------------------------------- /test/cddr/ksml/eval_test.clj: -------------------------------------------------------------------------------- 1 | (ns cddr.ksml.eval-test 2 | (:require 3 | [clojure.test :refer :all] 4 | [cddr.ksml.eval :as k] 5 | [cddr.ksml.core :refer [ksml* v->]]) 6 | (:import 7 | (java.io ByteArrayInputStream) 8 | (java.time Duration) 9 | (org.apache.kafka.common.serialization Serde Serdes Serializer Deserializer) 10 | (org.apache.kafka.streams StreamsBuilder) 11 | (org.apache.kafka.streams.kstream KStream Consumed) 12 | (org.apache.kafka.streams.processor FailOnInvalidTimestamp))) 13 | 14 | (def topics 15 | [:strs "a" "b" "c"]) 16 | 17 | (defn builder 18 | [] 19 | (StreamsBuilder.)) 20 | 21 | (defn keval 22 | [expr] 23 | (cddr.ksml.eval/eval expr)) 24 | 25 | (defn kpprint 26 | [expr] 27 | (clojure.pprint/pprint (keval expr))) 28 | 29 | 30 | 31 | (def extractor 32 | {:fail FailOnInvalidTimestamp}) 33 | 34 | (def keySerde [:serde '(ByteArray)]) 35 | (def valSerde [:serde '(ByteArray)]) 36 | (def topicPattern #"p") 37 | (def topic "foo") 38 | (def topics ["foo" "bar"]) 39 | (def offset [:offset-reset "EARLIEST"]) 40 | (def join-window [:join-window 1000]) 41 | (def timestampExtractor [:timestamp-extractor (extractor :fail)]) 42 | 43 | (def state-store 44 | [:store "log" {:with-keys keySerde 45 | :with-values valSerde 46 | :factory '.inMemory 47 | :logging-disabled? true}]) 48 | (def state-store-name "foo-store") 49 | (def queryable-store-name "foo-store") 50 | (def partitioner [:partitioner (fn [k v i] 51 | 0)]) 52 | (def grouped-stream 53 | (v-> [:stream topicPattern] 54 | [:group-by-key])) 55 | 56 | (defn consumes-pattern? 57 | [b] 58 | (= (str topicPattern) 59 | (str (.sourceTopicPattern b)))) 60 | 61 | (defn consumes-topics? 62 | [b] 63 | (let [topic-group (get (.topicGroups b) (int 0))] 64 | (= #{"foo" "bar"} 65 | (.sourceTopics topic-group)))) 66 | 67 | (defn serde? 68 | [expr] 69 | (instance? Serde (eval 70 | (k/eval expr)))) 71 | 72 | (defn valid-ksml? 73 | [expr] 74 | (-> expr keval eval)) 75 | 76 | (deftest test-eval-serde 77 | (is (valid-ksml? [:serde '(Integer)])) 78 | (is (valid-ksml? [:serde 79 | [:serde-from Integer]])) 80 | (is (valid-ksml? [:serde 81 | [:serde-from [:serializer (fn [])] 82 | [:deserializer (fn [])]]]))) 83 | 84 | (deftest test-eval-stream 85 | (testing "from pattern" 86 | (let [ksb (ksml* [:stream topicPattern])] 87 | (is (instance? StreamsBuilder ksb)))) 88 | 89 | (testing "from pattern with serdes" 90 | (let [ksb (ksml* [:stream topicPattern 91 | [:consumed [:with keySerde valSerde]]])] 92 | (is (instance? StreamsBuilder ksb)))) 93 | 94 | (testing "from topics with serdes" 95 | (let [ksb (ksml* `[:stream [:topics ~@topics] 96 | [:consumed [:with ~keySerde ~valSerde]]])] 97 | (is (instance? StreamsBuilder ksb)))) 98 | 99 | (testing "from topics" 100 | (let [ksb (ksml* `[:stream [:topics ~@topics]])] 101 | (is (instance? StreamsBuilder ksb)))) 102 | 103 | (testing "from pattern with serdes and timestamp extractor" 104 | (let [ksb (ksml* `[:stream ~topicPattern 105 | [:consumed [:with ~keySerde ~valSerde] 106 | [:withTimestampExtractor ~timestampExtractor]]])] 107 | (is (instance? StreamsBuilder ksb)))) 108 | 109 | (testing "from topics with serdes and timestamp extractor" 110 | (let [ksb (ksml* `[:stream [:topics ~@topics] 111 | [:consumed [:with ~keySerde ~valSerde] 112 | [:withTimestampExtractor ~timestampExtractor]]])] 113 | (is (instance? StreamsBuilder ksb)))) 114 | 115 | (testing "from pattern and offset" 116 | (is (ksml* `[:stream ~topicPattern 117 | [:consumed [:with ~offset]]]))) 118 | 119 | (testing "from pattern with serdes and offset" 120 | (is (ksml* `[:stream ~topicPattern 121 | [:consumed [:with ~keySerde ~valSerde] 122 | [:withOffsetResetPolicy ~offset]]]))) 123 | 124 | (testing "from topics with serde and offset" 125 | (is (ksml* `[:stream [:topics ~@topics] 126 | [:consumed [:with ~keySerde ~valSerde] 127 | [:withOffsetResetPolicy ~offset]]]))) 128 | 129 | (testing "from topics with offset" 130 | (is (ksml* `[:stream [:topics ~@topics] 131 | [:consumed [:with ~offset]]]))) 132 | 133 | (testing "from pattern with offset and timestamp-extractor and serdes" 134 | (is (ksml* `[:stream ~topicPattern 135 | [:consumed 136 | [:with ~keySerde ~valSerde ~timestampExtractor ~offset]]]))) 137 | 138 | (testing "from topics with offset and timestamp-extractor and serdes" 139 | (is (ksml* `[:stream [:topics ~@topics] 140 | [:consumed 141 | [:with ~keySerde ~valSerde ~timestampExtractor ~offset]]])))) 142 | 143 | (deftest test-eval-stores 144 | (is (valid-ksml? [:stores [:inMemoryKeyValueStore "yolo"]])) 145 | (is (valid-ksml? [:stores [:inMemorySessionStore "yolo" 146 | [:duration "PT1H"]]]))) 147 | 148 | (deftest test-eval-materialized 149 | (is (valid-ksml? [:materialized [:as "foo"] 150 | [:withCachingDisabled]])) 151 | (is (valid-ksml? [:materialized [:as "foo"] 152 | [:withCachingEnabled]])) 153 | (is (valid-ksml? [:materialized [:as "foo"] 154 | [:withLoggingDisabled]])) 155 | (is (valid-ksml? [:materialized [:as "foo"] 156 | [:withLoggingEnabled {"foo" "bar"}]])) 157 | (is (valid-ksml? [:materialized [:as "foo"] 158 | [:withRetention [:duration "PT24H"]]])) 159 | (is (valid-ksml? [:materialized [:with keySerde valSerde]]))) 160 | 161 | (deftest test-eval-table 162 | (testing "from topic with serdes" 163 | (is (ksml* `[:table ~topic 164 | [:consumed 165 | [:with ~keySerde ~valSerde]]]))) 166 | 167 | (testing "from topic with serdes, and materialized" 168 | (is (ksml* `[:table ~topic 169 | [:consumed 170 | [:with ~keySerde ~valSerde]] 171 | [:materialized [:with ~keySerde ~valSerde]]]))) 172 | 173 | (testing "from topic with serdes and state-store name" 174 | (is (ksml* `[:table ~topic 175 | [:consumed 176 | [:with ~keySerde ~valSerde]] 177 | [:materialized [:as "store-name"]]]))) 178 | 179 | (testing "from topic" 180 | (is (ksml* [:table topic]))) 181 | 182 | (testing "from topic with state store supplier" 183 | (is (ksml* [:table topic 184 | [:materialized [:with keySerde valSerde]]]))) 185 | 186 | (testing "from topic with state-store name" 187 | (is (ksml* [:table topic 188 | [:materialized [:as state-store-name]]]))) 189 | 190 | (testing "from topic with timestamp extractor and serdes and state-store name" 191 | (is (ksml* `[:table ~topic 192 | [:consumed [:with ~keySerde ~valSerde] 193 | [:withTimestampExtractor ~timestampExtractor]] 194 | [:materialized [:as ~state-store-name]]]))) 195 | 196 | (testing "from topic with timestamp extractor and state-store name" 197 | (is (ksml* `[:table ~topic 198 | [:consumed [:with ~timestampExtractor]] 199 | [:materialized [:as ~state-store-name]]]))) 200 | 201 | (testing "from topic with offset and serdes" 202 | (is (ksml* `[:table ~topic 203 | [:consumed [:with ~keySerde ~valSerde] 204 | [:withOffsetResetPolicy ~offset]]]))) 205 | 206 | (testing "from topic with offset and state-store name" 207 | (is (ksml* `[:table ~topic 208 | [:consumed [:with ~offset]] 209 | [:materialized [:as ~state-store-name]]]))) 210 | 211 | (testing "from topic with offset" 212 | (is (ksml* `[:table ~topic 213 | [:consumed [:with ~offset]]]))) 214 | 215 | (testing "from topic with offset and state-store supplier" 216 | (is (ksml* `[:table ~topic 217 | [:consumed [:with ~offset]] 218 | [:materialized 219 | [:as [:stores [:inMemoryKeyValueStore "yolo"]]]]]))) 220 | 221 | (testing "from topic with offset and timestamp extractor and serdes" 222 | (is (ksml* [:table topic 223 | [:consumed [:with keySerde valSerde] 224 | [:withOffsetResetPolicy offset] 225 | [:withTimestampExtractor timestampExtractor]] 226 | [:materialized 227 | [:as [:stores [:inMemoryKeyValueStore "yolo"]]]]]))) 228 | 229 | (testing "from topic with offset and timestamp extractor and serdes and state store-name" 230 | (is (ksml* [:table topic 231 | [:consumed [:with keySerde valSerde] 232 | [:withOffsetResetPolicy offset] 233 | [:withTimestampExtractor timestampExtractor]] 234 | [:materialized [:as state-store-name]]]))) 235 | 236 | (testing "from topic with offset and timestamp extractor and state-store-name" 237 | (is (ksml* [:table topic 238 | [:consumed [:with offset] 239 | [:withTimestampExtractor timestampExtractor]] 240 | [:materialized [:as state-store-name]]])))) 241 | 242 | (deftest test-global-stream 243 | (testing "from topic with serdes" 244 | (is (ksml* [:global-table topic 245 | [:consumed [:with keySerde valSerde]]]))) 246 | 247 | (testing "from topic with serdes and state store supplier" 248 | (is (ksml* [:global-table topic 249 | [:consumed [:with keySerde valSerde]] 250 | [:materialized 251 | [:as [:stores [:inMemoryKeyValueStore "yolo"]]]]]))) 252 | 253 | (testing "from topic with serde and queryableStoreName" 254 | (is (ksml* [:global-table topic 255 | [:consumed [:with keySerde valSerde]] 256 | [:materialized [:as queryable-store-name]]]))) 257 | 258 | (testing "from topic with serde and timestamp extractor and queryable-store-name" 259 | (is (ksml* [:global-table topic 260 | [:consumed [:with keySerde valSerde] 261 | [:withTimestampExtractor timestampExtractor]] 262 | [:materialized [:as queryable-store-name]]]))) 263 | 264 | (testing "from topic" 265 | (is (ksml* [:global-table topic]))) 266 | 267 | (testing "from topic with queryable-store-name" 268 | (is (ksml* [:global-table topic 269 | [:materialized [:as queryable-store-name]]])))) 270 | 271 | (def allow-all [:predicate (fn [k v] true)]) 272 | (def allow-none [:predicate (fn [k v] false)]) 273 | (def kv-map [:key-value-mapper (fn [k v] 274 | [k v])]) 275 | (def vmap [:value-mapper (fn [v] v)]) 276 | (def side-effect! [:foreach-action (fn [k1 v1])]) 277 | (def xform [:transformer (fn [k v] [k v])]) 278 | (def group-fn [:key-value-mapper (fn [v] 279 | (:part-id v))]) 280 | 281 | ;; (def edn-serde 282 | ;; [:serde 283 | ;; [:serializer (fn [this topic data] 284 | ;; (when data 285 | ;; (.getBytes (pr-str data))))] 286 | ;; [:deserializer (fn [this topic data] 287 | ;; (when data 288 | ;; (clojure.edn/read (ByteArrayInputStream. data))))]]) 289 | 290 | (deftest test-ktable-ops 291 | (let [this-table [:table "left"] 292 | other-stream [:stream #"right"] 293 | other-global-table [:global-table keySerde valSerde "lookup"] 294 | other-table [:table "right"] 295 | join-fn [:value-joiner (fn [l r] 296 | (= (:id l) (:id r)))]] 297 | 298 | (testing "filter" 299 | (is (ksml* [:filter [:table topic] allow-all])) 300 | (is (ksml* [:filter [:table topic] allow-all 301 | [:named "filtered-topic"]])) 302 | (is (ksml* [:filter [:table topic] allow-all 303 | [:materialized 304 | [:as [:stores [:inMemoryKeyValueStore "yolo"]]]]]))) 305 | 306 | (testing "filter-not" 307 | (is (ksml* [:filter-not [:table topic] allow-none])) 308 | (is (ksml* [:filter-not [:table topic] allow-all 309 | [:named "filtered-topic"]])) 310 | (is (ksml* [:filter-not [:table topic] allow-all 311 | [:materialized 312 | [:as [:stores [:inMemoryKeyValueStore "yolo"]]]]]))) 313 | 314 | (testing "group-by" 315 | (is (ksml* [:group-by [:table topic] 316 | kv-map]))) 317 | 318 | (testing "group by, grouped" 319 | (is (ksml* [:group-by [:table topic] kv-map 320 | [:grouped [:with keySerde valSerde]]]))) 321 | 322 | (testing "join" 323 | (is (ksml* [:join this-table other-table join-fn])) 324 | (is (ksml* [:join this-table other-table join-fn 325 | [:named "join-store"] 326 | [:materialized [:with keySerde valSerde]]])) 327 | (is (ksml* [:join this-table other-table join-fn 328 | [:materialized 329 | [:as [:stores [:inMemoryKeyValueStore "yolo"]]]]]))) 330 | 331 | (testing "left join" 332 | (is (ksml* [:left-join this-table other-table join-fn])) 333 | (is (ksml* [:left-join this-table other-table join-fn 334 | [:named "join-store"] 335 | [:materialized [:with keySerde valSerde]]])) 336 | (is (ksml* [:left-join this-table other-table join-fn 337 | [:materialized 338 | [:as [:stores [:inMemoryKeyValueStore "yolo"]]]]]))) 339 | 340 | (testing "map values" 341 | (is (ksml* [:map-values this-table vmap])) 342 | (is (ksml* [:map-values this-table vmap 343 | [:named "yolo"]])) 344 | (is (ksml* [:map-values this-table vmap 345 | [:materialized 346 | [:as [:stores [:inMemoryKeyValueStore "yolo"]]]]]))) 347 | 348 | (testing "outer join" 349 | (is (ksml* [:outer-join this-table other-table join-fn])) 350 | (is (ksml* [:outer-join this-table other-table join-fn 351 | [:named "outer-join-store"] 352 | [:materialized [:with keySerde valSerde]]])) 353 | (is (ksml* [:outer-join this-table other-table join-fn 354 | [:materialized 355 | [:as [:stores [:inMemoryKeyValueStore "yolo"]]]]]))) 356 | 357 | (testing "to-stream" 358 | (is (ksml* [:to-stream this-table])) 359 | (is (ksml* [:to-stream this-table 360 | [:key-value-mapper (fn [k v] 361 | [k v])]])) 362 | (is (ksml* [:to-stream this-table 363 | [:key-value-mapper (fn [k v] 364 | [k v])] 365 | [:named "yolo"]])) 366 | (is (ksml* [:to-stream this-table 367 | [:named "yolo"]]))))) 368 | 369 | 370 | (deftest test-kstream-ops 371 | (testing "branch" 372 | (is (ksml* [:branch [:stream topicPattern] 373 | allow-all 374 | allow-none])) 375 | (is (ksml* [:branch [:stream topicPattern] 376 | [:named "yolo"] 377 | allow-all 378 | allow-none])) 379 | (testing "filter" 380 | (is (ksml* [:filter [:stream topicPattern] allow-all])) 381 | (is (ksml* [:filter [:stream topicPattern] allow-all 382 | [:named "yolo"]]))) 383 | 384 | (testing "filter-not" 385 | (is (ksml* [:filter-not [:stream topicPattern] allow-none])) 386 | (is (ksml* [:filter-not [:stream topicPattern] allow-none 387 | [:named "yolo"]]))) 388 | 389 | (testing "flat-map" 390 | (is (ksml* [:flat-map [:stream #"foos"] kv-map])) 391 | (is (ksml* [:flat-map [:stream #"foos"] kv-map 392 | [:named "yolo"]]))) 393 | 394 | (testing "flat-map-values" 395 | (is (ksml* [:flat-map-values [:stream #"foos"] vmap])) 396 | (is (ksml* [:flat-map-values [:stream #"foos"] vmap 397 | [:named "yolo"]]))) 398 | 399 | (testing "foreach" 400 | (is (ksml* [:foreach [:stream topicPattern] side-effect!])) 401 | (is (ksml* [:foreach [:stream topicPattern] side-effect! 402 | [:named "yolo"]]))) 403 | 404 | (testing "group-by" 405 | (is (ksml* [:group-by [:stream topicPattern] group-fn])) 406 | (is (ksml* [:group-by [:stream topicPattern] group-fn 407 | [:grouped [:as "foo"]]]))))) 408 | 409 | ;; (is (ksml* [:group-by [:stream topicPattern] 410 | ;; group-fn 411 | ;; keySerde 412 | ;; valSerde]))) 413 | 414 | ;; (let [this-stream [:stream #"left"] 415 | ;; other-stream [:stream #"right"] 416 | ;; other-global-table [:global-table keySerde valSerde "lookup"] 417 | ;; other-table [:table "right"] 418 | ;; join-fn [:value-joiner (fn [l r] 419 | ;; (= (:id l) (:id r)))]] 420 | 421 | ;; (testing "process" 422 | ;; (is (ksml* [:process! this-stream 423 | ;; [:processor-supplier (fn [context k v] 424 | ;; v)] 425 | ;; [:strs]]))) 426 | 427 | 428 | ;; (testing "join global table" 429 | ;; (is (ksml* [:join-global this-stream other-global-table 430 | ;; kv-map 431 | ;; join-fn]))) 432 | 433 | ;; (testing "join stream with window" 434 | ;; (is (ksml* [:join this-stream other-stream 435 | ;; join-fn 436 | ;; join-window]))) 437 | 438 | ;; (testing "join stream with window and serdes" 439 | ;; (is (ksml* [:join this-stream other-stream 440 | ;; join-fn 441 | ;; join-window 442 | ;; keySerde 443 | ;; valSerde 444 | ;; valSerde]))) 445 | 446 | ;; (testing "join table" 447 | ;; (is (ksml* [:join this-stream other-table join-fn]))) 448 | 449 | ;; (testing "join table with serdes" 450 | ;; (is (ksml* [:join this-stream other-table join-fn keySerde valSerde]))) 451 | 452 | ;; (testing "left join global table" 453 | ;; (is (ksml* [:left-join-global this-stream other-global-table 454 | ;; kv-map 455 | ;; join-fn]))) 456 | 457 | ;; (testing "left join stream with window" 458 | ;; (is (ksml* [:left-join this-stream other-stream 459 | ;; join-fn 460 | ;; join-window]))) 461 | 462 | 463 | ;; (testing "left join stream with window and serdes" 464 | ;; (is (ksml* [:left-join this-stream other-stream 465 | ;; join-fn 466 | ;; join-window 467 | ;; keySerde 468 | ;; valSerde 469 | ;; valSerde]))) 470 | 471 | ;; (testing "left join table" 472 | ;; (is (ksml* [:left-join this-stream other-table 473 | ;; join-fn]))) 474 | 475 | ;; (testing "left join table with serdes" 476 | ;; (is (ksml* [:left-join this-stream other-table 477 | ;; join-fn 478 | ;; keySerde 479 | ;; valSerde]))) 480 | 481 | ;; (testing "map" 482 | ;; (is (ksml* [:map [:stream #"words"] kv-map]))) 483 | 484 | ;; (testing "map values" 485 | ;; (is (ksml* [:map-values [:stream topicPattern] vmap]))) 486 | 487 | ;; (testing "outer join" 488 | ;; (is (ksml* [:outer-join this-stream other-stream 489 | ;; join-fn 490 | ;; join-window])) 491 | ;; (is (ksml* [:outer-join this-stream other-stream 492 | ;; join-fn join-window 493 | ;; keySerde valSerde valSerde]))) 494 | 495 | ;; (testing "peek" 496 | ;; (is (ksml* [:peek! this-stream 497 | ;; [:foreach-action (fn [k v] 498 | ;; "yolo")]]))) 499 | 500 | ;; (testing "print!" 501 | ;; (is (ksml* [:print! [:stream #"foo"]])) 502 | ;; (is (ksml* [:print! this-stream keySerde valSerde])) 503 | ;; (is (ksml* [:print! this-stream keySerde valSerde "stream-name"])) 504 | ;; (is (ksml* [:print! this-stream "stream-name"]))))) 505 | --------------------------------------------------------------------------------