├── .circleci └── config.yml ├── .github └── workflows │ └── clojure.yml ├── .gitignore ├── LICENSE ├── README.md ├── project.clj ├── src └── kafka_component │ ├── config.clj │ ├── core.clj │ └── mock.clj └── test └── kafka_component ├── core_test.clj └── mock_test.clj /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | build: 5 | branches: 6 | ignore: 7 | - gh-pages 8 | docker: 9 | - image: circleci/clojure:lein-2.7.1 10 | environment: 11 | LEIN_ROOT: nbd 12 | JVM_OPTS: -Xmx320m 13 | steps: 14 | - checkout 15 | - restore_cache: # restores saved cache if checksum hasn't changed since the last run 16 | key: kafka-component--clojure-{{ checksum "project.clj" }} 17 | - run: lein deps 18 | - save_cache: # generate and store cache in the .m2 directory using a key template 19 | paths: 20 | - ~/.m2 21 | key: kafka-component--clojure-{{ checksum "project.clj" }} 22 | - run: lein do test 23 | -------------------------------------------------------------------------------- /.github/workflows/clojure.yml: -------------------------------------------------------------------------------- 1 | name: Clojure CI 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | 9 | steps: 10 | - uses: actions/checkout@v1 11 | - name: Install dependencies 12 | run: lein deps 13 | - name: Run tests 14 | run: lein test 15 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .DS_Store 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor tocontrol, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of New York and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-component 2 | 3 | Provides a Kafka consumption pool component to work with Stuart Sierra's 4 | component. 5 | 6 | The consumer component will manually commit offsets after each batch of messages 7 | has been processed, instead of relying on `auto.commit.enable`. This library takes 8 | special care to make sure that messages have actually been processed before committing 9 | which the default `auto.commit.enable` doesn't have as strong semantics. To avoid a 10 | thread committing another thread's offsets, each thread receives its own kafka consumer. 11 | 12 | # Installation 13 | 14 | Add to your dependencies in your `project.clj`: 15 | 16 | [![Clojars Project](https://img.shields.io/clojars/v/kafka-component.svg)](https://clojars.org/kafka-component) 17 | 18 | # Usage 19 | 20 | Use `KafkaWriter` and `KafkaReader` in a system: 21 | 22 | ```clojure 23 | (ns myapp 24 | (:require [kafka-component.core :as kafka-component] 25 | [com.stuartsierra.component :as component]) 26 | 27 | (def config 28 | {:writer-config {:native-producer-overrides {"bootstrap.servers" "localhost:9092"}} 29 | :reader-config {:shutdown-timeout 4 ; default 30 | :concurrency-level 2 31 | :topics ["topic_one" "or_more"] 32 | :native-consumer-overrides {"bootstrap.servers" "localhost:9092" 33 | "group.id" "myapp" 34 | "auto.offset.reset" "largest"}}}) 35 | 36 | (defn create-system [config] 37 | (component/system-using 38 | (component/system-map 39 | :logger println 40 | :exception-handler (fn [e] (.printStackTrace e)) 41 | :record-processor {:process (fn [{:keys [key value]}] (println "Received message: " value))} 42 | :reader (kafka-component/map->KafkaReader (config :reader-config)) 43 | :writer (kafka-component/map->KafkaWriter (config :writer-config))))) 44 | {:reader [:logger :exception-handler :record-processor]})) 45 | ``` 46 | 47 | ## Produce a message 48 | 49 | ```clojure 50 | (let [{:keys [writer]} (component/start (create-system config))] 51 | (kafka-component/write writer "topic_one" "message-key" "message-body")) 52 | ``` 53 | 54 | It is also possible to `kakfka-component/write-async`. 55 | 56 | ## Consume a mesage 57 | 58 | Because reader is listening to "topic_one", it will deliver a message to the record-processor: 59 | 60 | ``` 61 | Received message: message-body 62 | ``` 63 | 64 | # Testing 65 | 66 | You can use specically designed mock implementations for tests. The mocks 67 | emulate kafka, in-memory without the large startup overhead. 68 | 69 | ```clojure 70 | (ns myapp.tests 71 | (:require [kafka-component.mock :as kafka-mock] 72 | [clojure.test :refer :all])) 73 | 74 | (deftest basic-produce-consume 75 | (kafka-mock/with-test-producer-consumer producer consumer 76 | ;; tell mock producer to send a message on a kafka queue 77 | (kafka-mock/send producer "topic" "key" "value") 78 | 79 | ;; tell mock consumer to subscribe to topic 80 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0}] 81 | (kafka-mock/accumulate-messages consumer "topic" {:timeout 1000}))))) 82 | 83 | (deftest transforming-messages 84 | (kafka-mock/with-test-producer-consumer producer consumer 85 | (dotimes [n 100] 86 | (kafka-mock/send producer "topic" "key" (str n))) 87 | 88 | ;; transform and filter messages 89 | (is (= [1 3] 90 | (kafka-mock/txfm-messages consumer "topic" 91 | (comp (map :value) 92 | (map #(Integer/parseInt %)) 93 | (filter odd?) 94 | (take 2)) 95 | {:timeout 1000}))))) 96 | ``` 97 | 98 | It is also possible to `kafka-mock/send-async`. 99 | 100 | The producer and consumer created by `with-test-producer-consumer` run outside of 101 | a system. To use the mocks *inside* a system, modify the system config: 102 | 103 | ```clojure 104 | (def test-config 105 | (-> myapp/config 106 | (assoc-in [:writer-config :native-producer-type] :mock) 107 | (assoc-in [:reader-config :native-consumer-type] :mock) 108 | (assoc-in [:reader-config :native-consumer-overrides "auto.offset.reset"] "earliest"))) 109 | ``` 110 | 111 | Usually, you will want to do both. 112 | * If your system has a `reader`, create messages outside of the system with a 113 | `producer`, then test that the system reads and processes the messages 114 | correctly. 115 | * If your system has a `writer`, poke your system to produce a message, then 116 | test that an external `consumer` can read the message. 117 | 118 | # More 119 | 120 | See the [documentation](http://mayvenn.github.io/kafka-component) for more. 121 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject kafka-component "0.8.1-SNAPSHOT" 2 | :description "A kafka component to consume from Kafka" 3 | :url "https://github.com/Mayvenn/kafka-component" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :deploy-repositories [["releases" :clojars]] 7 | :dependencies [[org.clojure/clojure "1.9.0"] 8 | [com.stuartsierra/component "0.2.2"] 9 | [io.weft/gregor "0.7.0"] 10 | [org.clojure/core.async "0.4.474"]] 11 | :codox {:source-paths ["src"] 12 | :source-uri "http://github.com/Mayvenn/kafka-component/blob/master/{filepath}#L{line}" 13 | :metadata {:doc/format :markdown} 14 | :doc-files ["README.md"]} 15 | :profiles 16 | {:uberjar {:aot :all} 17 | :dev {:source-paths ["dev"] 18 | :dependencies [[diff-eq "0.2.2"] 19 | [org.clojure/tools.namespace "0.2.9"] 20 | [embedded-kafka "0.6.0"]] 21 | :plugins [[lein-cljfmt "0.3.0"] 22 | [lein-codox "0.10.2"]] 23 | :injections [(require 'diff-eq.core) 24 | (diff-eq.core/diff!)]}}) 25 | -------------------------------------------------------------------------------- /src/kafka_component/config.clj: -------------------------------------------------------------------------------- 1 | (ns kafka-component.config) 2 | 3 | (def default-consumer-config 4 | {"enable.auto.commit" "false" 5 | "max.poll.records" "1"}) 6 | 7 | (def default-producer-config 8 | {"acks" "all" 9 | "retries" "3" 10 | "max.in.flight.requests.per.connection" "1"}) 11 | 12 | (defn assert-non-nil-values [m] 13 | (doseq [[k v] m] 14 | (assert v (format "%s cannot be nil in the config" k)))) 15 | 16 | (defn assert-contains-keys [m & ks] 17 | (doseq [k ks] 18 | (assert (contains? m k) (format "\"%s\" must be provided in the config" k)))) 19 | 20 | (defn assert-string-values [m] 21 | (doseq [[k v] m] 22 | (assert (string? v) (format "%s must be a string" k)))) 23 | 24 | (defn assert-request-timeout-valid [opts] 25 | (let [request-timeout (. Integer parseInt (get opts "request.timeout.ms" "40000")) 26 | session-timeout (. Integer parseInt (get opts "session.timeout.ms" "30000")) 27 | fetch-max-wait (. Integer parseInt (get opts "fetch.max.wait.ms" "500"))] 28 | (assert (and (> request-timeout session-timeout) 29 | (> request-timeout fetch-max-wait)) 30 | "\"request.timeout.ms\" must be greater than both \"session.timeout.ms\" and \"fetch.max.wait.ms\""))) 31 | 32 | (defn assert-consumer-opts [opts] 33 | (assert opts "Kafka consumer options cannot be nil") 34 | (assert (#{"latest" "earliest" "none"} (get opts "auto.offset.reset")) 35 | "\"auto.offset.reset\" should be set to one of #{\"latest\" \"earliest\" \"none\"}") 36 | (assert-contains-keys opts "bootstrap.servers" "group.id") 37 | (assert-non-nil-values opts) 38 | (assert-string-values opts) 39 | (assert-request-timeout-valid opts)) 40 | 41 | (defn assert-producer-opts [opts] 42 | (assert opts "Kafka producer options cannot be nil") 43 | (assert-non-nil-values opts)) 44 | -------------------------------------------------------------------------------- /src/kafka_component/core.clj: -------------------------------------------------------------------------------- 1 | (ns kafka-component.core 2 | (:require [com.stuartsierra.component :as component] 3 | [gregor.core :as gregor] 4 | [kafka-component.config :as config] 5 | [clojure.string :as string] 6 | [clojure.string :as str]) 7 | (:import [java.util.concurrent Executors TimeUnit] 8 | org.apache.kafka.clients.consumer.CommitFailedException 9 | org.apache.kafka.common.errors.WakeupException)) 10 | 11 | (defn- latest-offsets [records] 12 | (->> records 13 | (reduce (fn [key->offset {:keys [topic partition offset]}] 14 | (update key->offset [topic partition] (fnil max 0) offset)) 15 | {}) 16 | (mapv (fn [[[topic partition] offset]] 17 | {:topic topic 18 | :partition partition 19 | :offset (inc offset)})))) 20 | 21 | (defmulti make-consumer 22 | (fn [type topics overrides] 23 | (config/assert-consumer-opts overrides) 24 | type)) 25 | 26 | (defmulti make-producer 27 | (fn [type overrides] 28 | (config/assert-producer-opts overrides) 29 | type)) 30 | 31 | (defmethod make-consumer :default [_ topics overrides] 32 | (gregor/consumer (overrides "bootstrap.servers") 33 | (overrides "group.id") 34 | topics 35 | (merge config/default-consumer-config overrides))) 36 | 37 | (defmethod make-producer :default [_ overrides] 38 | (gregor/producer (overrides "bootstrap.servers") 39 | (merge config/default-producer-config overrides))) 40 | 41 | (defmacro with-err-str [& body] 42 | `(let [wr# (java.io.StringWriter.)] 43 | (binding [*err* wr#] 44 | ~@body) 45 | (str wr#))) 46 | 47 | (defn panic! [] 48 | (System/exit 63)) 49 | 50 | (defmacro ^{:style/indent 2} try-or-panic [task-id panic-msg & body] 51 | `(try 52 | ~@body 53 | (catch Throwable t# 54 | (println (str "source=kafka-consumer action=exception notice=a restart may be required to continue processing kafka partitions msg=" ~panic-msg " task-id=" ~task-id " exception=" (with-err-str (.printStackTrace t#)))) 55 | (panic!)))) 56 | 57 | (def report-health-interval (* 1 60 1000)) ;; Wait 10 minutes between reporting health 58 | 59 | (defn make-task [logger exception-handler process-record poll-interval make-kafka-consumer task-id] 60 | (let [kafka-consumer (make-kafka-consumer) 61 | log-exception (fn log-exception [e message] 62 | (try-or-panic task-id "failed to use logger to log exception" 63 | (logger :event-error message)) 64 | (try-or-panic task-id "failed to use exception-handler to log exception" 65 | (exception-handler e))) 66 | log (fn log [level msg] 67 | (try-or-panic task-id (format "failed to log %s %s" level (pr-str msg)) 68 | (logger level msg))) 69 | poll-counter (atom 0)] 70 | (reify 71 | java.lang.Runnable 72 | (run [_] 73 | (try 74 | (log :event {:event {:name :kafka.consumer.task/started}}) 75 | 76 | (while true 77 | (swap! poll-counter inc) 78 | (when (= (mod @poll-counter 79 | (/ report-health-interval (or poll-interval 100))) 80 | 0) 81 | (reset! poll-counter 0) 82 | (log :event {:event {:name :kafka.consumer.task/healthy 83 | :data {:task-id task-id}}})) 84 | (let [records (gregor/poll kafka-consumer (or poll-interval 100))] 85 | (doseq [{:keys [topic partition key] :as record} records] 86 | (log :event {:event {:name :kafka.consumer.task/received 87 | :data {:task-id task-id 88 | :topic topic 89 | :partition partition 90 | :key key}}}) 91 | (try 92 | (process-record record) 93 | (catch WakeupException e (throw e)) 94 | (catch Throwable e 95 | (log-exception e {:event {:name :kafka.consumer.task/erred-while-processing-message 96 | :data {:task-id task-id 97 | :topic topic 98 | :partition partition 99 | :key key 100 | :error e}}})))) 101 | (try 102 | (gregor/commit-offsets! kafka-consumer (latest-offsets records)) 103 | (catch CommitFailedException e 104 | (log-exception e {:event {:name :kafka.consumer.task/erred-saving-offsets 105 | :data {:task-id task-id 106 | :error e}}}))))) 107 | (log :event {:event {:name :kafka.consumer.task/exiting 108 | :data {:task-id task-id}}}) 109 | (catch WakeupException _ 110 | (log :event {:event {:name :kafka.consumer.task/woke-up 111 | :data {:task-id task-id}}})) 112 | (catch Throwable e 113 | (log-exception e {:event {:name :kafka.consumer.task/erred-in-task-runnable 114 | :data {:error e 115 | :task-id task-id}}})) 116 | (finally 117 | (log :event {:event {:name :kafka.consumer.task/closed 118 | :data {:task-id task-id}}}) 119 | (gregor/close kafka-consumer)))) 120 | java.io.Closeable 121 | (close [_] 122 | (gregor/wakeup kafka-consumer))))) 123 | 124 | (defn init-and-start-task-pool [make-task pool-id concurrency-level] 125 | (let [native-pool (Executors/newFixedThreadPool concurrency-level) 126 | task-ids (map (partial str pool-id "-") (range concurrency-level)) 127 | tasks (map make-task task-ids)] 128 | (doseq [t tasks] (.submit native-pool t)) 129 | {:native-pool native-pool 130 | :tasks tasks})) 131 | 132 | (defn stop-task-pool [{:keys [native-pool tasks]} shutdown-timeout] 133 | (doseq [t tasks] (.close t)) 134 | (when native-pool 135 | (.shutdown native-pool) 136 | (.awaitTermination native-pool shutdown-timeout TimeUnit/SECONDS))) 137 | 138 | (defn kw-name 139 | "Returns a keyword as a string with namespace preserved." 140 | [kw-or-str] 141 | (-> kw-or-str 142 | keyword 143 | str 144 | (subs 1))) 145 | 146 | (defn unstructure-log-message [message] 147 | (if (contains? message :event) 148 | (let [event (:event message)] 149 | (->> (:data event) 150 | (into [["action" (-> event :name kw-name)]] 151 | (map (fn [[key value]] 152 | [(kw-name key) 153 | (cond 154 | (nil? value) "nil" 155 | (keyword? value) (kw-name value) 156 | :else value)]))) 157 | (map (partial string/join "=")) 158 | (string/join " "))) 159 | (prn-str message))) 160 | 161 | (defrecord KafkaReader [logger 162 | structured-logging? 163 | exception-handler 164 | record-processor 165 | concurrency-level 166 | poll-interval 167 | shutdown-timeout 168 | topics 169 | native-consumer-type 170 | native-consumer-overrides] 171 | component/Lifecycle 172 | (start [this] 173 | (assert (not= shutdown-timeout 0) "\"shutdown-timeout\" must not be zero") 174 | (assert (ifn? (:process record-processor)) "record-processor does not have a function :process") 175 | (let [logger (if structured-logging? 176 | logger 177 | (fn [level data] 178 | (logger (case level 179 | :event :debug 180 | :event-error :error 181 | :info) 182 | (unstructure-log-message data)))) 183 | make-native-consumer (partial make-consumer native-consumer-type topics native-consumer-overrides) ; a thunk, does not need more args 184 | make-task (partial make-task logger exception-handler (:process record-processor) poll-interval make-native-consumer) 185 | ;will get task-id when pool is started 186 | pool-id (pr-str topics) 187 | log-data {:concurrency-level concurrency-level 188 | :shutdown-timeout shutdown-timeout 189 | :topics topics 190 | :poll-interval poll-interval 191 | :native-consumer-overrides native-consumer-overrides 192 | :native-consumer-type native-consumer-type 193 | :pool-id pool-id}] 194 | 195 | (logger :event {:event {:name :kafka.consumer/initialized 196 | :data log-data}}) 197 | (let [running-pool (init-and-start-task-pool make-task pool-id concurrency-level)] 198 | (logger :event {:event {:name :kafka.consumer/started 199 | :data log-data}}) 200 | (merge this 201 | {:pool running-pool 202 | :log-data log-data})))) 203 | (stop [{:keys [pool log-data logger] :as this}] 204 | (let [logger (if structured-logging? 205 | logger 206 | (fn [level data] 207 | (logger (case level 208 | :event :debug 209 | :event-error :error 210 | :info) 211 | (unstructure-log-message data))))] 212 | (when pool 213 | (logger :event {:event {:name :kafka.consumer/started-to-stop 214 | :data log-data}}) 215 | (stop-task-pool pool (or shutdown-timeout 4)) 216 | (logger :event {:event {:name :kafka.consumer/stopped 217 | :data log-data}})) 218 | (dissoc this :pool :logger)))) 219 | 220 | (defrecord KafkaWriter [logger native-producer-type native-producer-overrides] 221 | component/Lifecycle 222 | (start [this] 223 | (assoc this :native-producer (make-producer native-producer-type native-producer-overrides))) 224 | (stop [this] 225 | (when-let [p (:native-producer this)] 226 | (gregor/close p 2)) ;; 2 seconds to wait to send remaining messages, should this be configurable? 227 | (dissoc this :native-producer))) 228 | 229 | (defn write-async [writer topic key val] 230 | (gregor/send (:native-producer writer) topic key val)) 231 | 232 | (defn write [writer topic key val] 233 | (try 234 | @(write-async writer topic key val) 235 | (catch Throwable t 236 | (let [outer-exception (ex-info "Unable to write to kafka" 237 | {:cause t 238 | :topic topic 239 | :key key} 240 | t)] 241 | (when (:logger writer) 242 | ((:logger writer) :event-error {:event {:name :kafka.producer/erred 243 | :data {:error outer-exception 244 | :topic topic 245 | :key key}}})) 246 | (throw outer-exception))))) 247 | -------------------------------------------------------------------------------- /src/kafka_component/mock.clj: -------------------------------------------------------------------------------- 1 | (ns kafka-component.mock 2 | (:refer-clojure :exclude [send]) 3 | (:require [clojure.core.async 4 | :refer 5 | [! >!! alt! alt!! chan close! go poll! sliding-buffer timeout]] 6 | [gregor.core :as gregor] 7 | [com.stuartsierra.component :as component] 8 | [kafka-component.core :as core] 9 | [kafka-component.config :as config] 10 | [clojure.set :as set]) 11 | (:import java.lang.Integer 12 | java.util.Collection 13 | java.util.concurrent.TimeUnit 14 | java.util.regex.Pattern 15 | [org.apache.kafka.clients.consumer Consumer ConsumerRebalanceListener ConsumerRecord ConsumerRecords] 16 | [org.apache.kafka.clients.producer Callback Producer ProducerRecord RecordMetadata] 17 | [org.apache.kafka.common.errors InvalidOffsetException WakeupException] 18 | org.apache.kafka.common.TopicPartition)) 19 | 20 | ;; TODO: update README for new consumer config/constructors 21 | (def default-mock-consumer-opts 22 | {"auto.offset.reset" "earliest" 23 | "group.id" "test" 24 | "max.poll.records" "1000"}) 25 | 26 | (def standalone-mock-consumer-opts (assoc default-mock-consumer-opts "bootstrap.servers" "localhost.fake")) 27 | 28 | (def debug (atom false)) 29 | 30 | ;; structure of broker-state: 31 | ;; {"sample-topic" [partition-state *]} 32 | ;; where partition-state is: 33 | ;; {:messages [first-msg second-msg] :watchers chan-of-interested-consumers} 34 | ;; TODO: where should all the random comm chans go, they are siblings of topics in broker state right now, weird 35 | (def broker-state (atom {})) 36 | 37 | ;; structure of committed-offsets: 38 | ;; {[group-id topic-partition] 10} 39 | (def committed-offsets (atom {})) 40 | 41 | (def buffer-size 20) 42 | (def default-num-partitions 2) 43 | (def consumer-backoff 20) 44 | (def rebalance-participants-timeout 2000) 45 | (def consumer-rebalance-timeout 4000) 46 | (def consumer-unsubscribe-timeout 5000) 47 | 48 | (defn logger [& args] 49 | (when @debug 50 | (locking println 51 | (apply println args)))) 52 | 53 | (defn reset-state! [] 54 | ;; TODO: wait until everyone is shutdown before clearing these 55 | (reset! broker-state {}) 56 | (reset! committed-offsets {})) 57 | 58 | (defn ->topic-partition [topic partition] 59 | (TopicPartition. topic partition)) 60 | 61 | (defn record->topic-partition [record] 62 | (TopicPartition. (.topic record) (.partition record))) 63 | 64 | (defn broker-create-topic 65 | ([] (broker-create-topic default-num-partitions)) 66 | ([num-partitions] 67 | (into [] (repeatedly num-partitions (constantly {:messages [] :watchers (chan (sliding-buffer buffer-size))}))))) 68 | 69 | (defn broker-ensure-topic [broker-state topic] 70 | (if (broker-state topic) 71 | broker-state 72 | (assoc broker-state topic (broker-create-topic)))) 73 | 74 | (defn producer-record->consumer-record [offset record] 75 | (ConsumerRecord. (.topic record) (.partition record) offset (.key record) (.value record))) 76 | 77 | (defn add-record-to-topic [state producer-record] 78 | (let [topic (.topic producer-record) 79 | offset (count (get-in state [topic (.partition producer-record) :messages])) 80 | consumer-record (producer-record->consumer-record offset producer-record)] 81 | (update-in state [topic (.partition consumer-record) :messages] conj consumer-record))) 82 | 83 | (defn add-record-in-broker-state [state producer-record] 84 | (let [topic (.topic producer-record)] 85 | (-> state 86 | (broker-ensure-topic topic) 87 | (add-record-to-topic producer-record)))) 88 | 89 | (defmacro goe 90 | {:style/indent 0} 91 | [& body] 92 | `(go 93 | (try ~@body 94 | (catch Exception e# 95 | (do (logger e#) 96 | (throw e#)))))) 97 | 98 | (defmacro goe-loop 99 | {:style/indent 1} 100 | [& body] 101 | `(goe (loop ~@body))) 102 | 103 | (defn close-all-from [ch] 104 | ;; the consumers back off to avoid flooding this channel but in some malicious 105 | ;; scenarios this could loop forever. Could add a max watchers param if we wanted 106 | ;; keep this under control or send over some sort of communication as to which 107 | ;; tick this occurred in to try to know when to stop? Or an entirely different design. 108 | ;; It is less of a problem now that the watchers are a sliding buffer, though 109 | ;; technically they could fill faster than we can drain 110 | (loop [] 111 | (when-let [o (poll! ch)] 112 | (close! o) 113 | (recur)))) 114 | 115 | (defn committed-record-metadata [record] 116 | (RecordMetadata. (record->topic-partition record) 0 (.offset record) 117 | (.timestamp record) (.checksum record) 118 | (.serializedKeySize record) (.serializedValueSize record))) 119 | 120 | (defn broker-save-record! [state record] 121 | (let [topic (.topic record) 122 | record-with-partition (ProducerRecord. topic (or (.partition record) (int 0)) (.key record) (.value record)) 123 | state-with-record (swap! state add-record-in-broker-state record-with-partition) 124 | partition (get-in state-with-record [topic (.partition record-with-partition)])] 125 | (close-all-from (:watchers partition)) 126 | (committed-record-metadata (last (:messages partition))))) 127 | 128 | (defn broker-receive-messages [state msg-ch] 129 | (goe-loop [] 130 | (alt! 131 | msg-ch ([[res-ch msg]] 132 | (when res-ch 133 | (>! res-ch (broker-save-record! state msg)) 134 | (close! res-ch) 135 | (recur)))))) 136 | 137 | (defprotocol IRebalance 138 | (all-topics [this]) 139 | (apply-pending-topics [this topics]) 140 | (clean-up-subscriptions [this])) 141 | 142 | (defn assign-partitions [broker-state consumers participants participants-ch complete-ch] 143 | (let [broker-state @broker-state 144 | topics (distinct (mapcat all-topics consumers)) 145 | 146 | participants->assignments 147 | (apply merge-with concat {} 148 | (for [topic topics 149 | :let [subscribed-participants (filterv #(contains? (set (all-topics %)) topic) participants)] 150 | partition (range (count (broker-state topic))) 151 | :let [participant (get subscribed-participants (mod partition (max (count participants) 1)))] 152 | :when participant] 153 | {participant [(->topic-partition topic partition)]}))] 154 | (logger "-- [consumer-coordinator] topics" (pr-str topics)) 155 | (logger "-- [consumer-coordinator] consumers" (pr-str consumers)) 156 | (logger "-- [consumer-coordinator] assignments" (pr-str participants->assignments)) 157 | (doseq [consumer consumers] 158 | (let [assignments (participants->assignments consumer)] 159 | (.assign consumer (or assignments [])))) 160 | (close! participants-ch) 161 | (close! complete-ch))) 162 | 163 | (defn rebalance-participants 164 | "Try to get all the consumers to participate in the rebalance, but if they 165 | don't all check in, continue without some of them." 166 | [broker-state consumers participants-ch complete-ch] 167 | (doseq [c consumers] 168 | (>!! (:rebalance-control-ch c) [participants-ch complete-ch])) 169 | (goe-loop [participants []] 170 | (alt! 171 | participants-ch ([participant] 172 | (when participant ; else, closed 173 | (let [participants' (conj participants participant)] 174 | (if (>= (count participants') (count consumers)) 175 | (assign-partitions broker-state consumers participants' participants-ch complete-ch) 176 | (recur participants'))))) 177 | (timeout rebalance-participants-timeout) (assign-partitions broker-state consumers participants participants-ch complete-ch)))) 178 | 179 | (defn rebalance-consumers [relevant-consumers broker-state] 180 | (let [rebalance-participants-ch (chan buffer-size) 181 | ;; The complete ch tells each participant when they can resume polling, 182 | ;; and allows the coordinator to start another rebalance 183 | rebalance-complete-ch (chan)] 184 | (rebalance-participants broker-state relevant-consumers rebalance-participants-ch rebalance-complete-ch) 185 | (consumers (swap! state update group-id (fnil conj #{}) consumer) 201 | consumers (consumers-with-topic-overlap (get group->consumers group-id) topics)] 202 | (rebalance-consumers consumers broker-state) 203 | (recur)))) 204 | leave-ch ([consumer] 205 | (when consumer ; else, closed 206 | (clean-up-subscriptions consumer) 207 | (let [group-id (get-in consumer [:config "group.id"] "") 208 | topics (all-topics consumer) 209 | group->consumers (swap! state update group-id disj consumer) 210 | consumers (consumers-with-topic-overlap (get group->consumers group-id) topics)] 211 | (when (seq consumers) 212 | (rebalance-consumers consumers broker-state)) 213 | (recur))))))) 214 | 215 | (defn start! [] 216 | (let [msg-ch (chan buffer-size) 217 | join-ch (chan) 218 | leave-ch (chan) 219 | ;; {"group1" #{consumer-1 consumer-2}} 220 | consumer-coordinator-state (atom {})] 221 | (broker-receive-messages broker-state msg-ch) 222 | (consumer-coordinator consumer-coordinator-state broker-state join-ch leave-ch) 223 | (reset! broker-state {:msg-ch msg-ch :join-ch join-ch :leave-ch leave-ch}))) 224 | 225 | (defn debug! [enable] 226 | (reset! debug enable)) 227 | 228 | (defn shutdown! [] 229 | (let [{:keys [msg-ch join-ch leave-ch]} @broker-state] 230 | (close! msg-ch) 231 | (close! join-ch) 232 | (close! leave-ch) 233 | (reset-state!))) 234 | 235 | (defn fixture-restart-broker! [f] 236 | (start!) 237 | (f) 238 | (shutdown!)) 239 | 240 | (defn close-mock [state] 241 | (assoc state :conn-open? false)) 242 | 243 | (defn read-offsets [grouped-messages] 244 | (into {} (for [[topic-partition msg-list] grouped-messages] 245 | [topic-partition (inc (.offset (last msg-list)))]))) 246 | 247 | (defn max-poll-records [config] 248 | (if-let [max-poll-records-str (config "max.poll.records")] 249 | (do 250 | (assert String (type max-poll-records-str)) 251 | (Integer/parseInt max-poll-records-str)) 252 | Integer/MAX_VALUE)) 253 | 254 | (defn get-offset [broker-state topic partition config] 255 | (let [group-id (config "group.id" "")] 256 | (if-let [committed-offset (get @committed-offsets [group-id (->topic-partition topic partition)])] 257 | committed-offset 258 | (case (config "auto.offset.reset") 259 | "earliest" 0 260 | "latest" (count (get-in broker-state [topic partition :messages])) 261 | "none" (throw (InvalidOffsetException. (str "auto.offset.reset=none, no existing offset for group " group-id " topic " topic " partition " partition))))))) 262 | 263 | (defn ^:private desires-repoll [state subscribed-topic-partitions] 264 | (let [poll-chan (chan buffer-size)] 265 | (doseq [[topic-partition _] subscribed-topic-partitions] 266 | (>!! (get-in state [(.topic topic-partition) (.partition topic-partition) :watchers]) poll-chan)) 267 | poll-chan)) 268 | 269 | (defn ^:private read-messages [state subscribed-topic-partitions config] 270 | ;; TODO: round robin across topic-partitions? seems not that necessary right now 271 | ;; TODO: what happens if you try to read partitions you don't "own" 272 | (let [messages (mapcat (fn unread-messages [[topic-partition read-offset]] 273 | (let [topic (.topic topic-partition) 274 | partition (.partition topic-partition) 275 | messages (get-in state [topic partition :messages])] 276 | (logger (format "-- [consumer %s] read-offset=%d max-offset=%d topic=%s" 277 | (get config "group.id") 278 | read-offset 279 | (count messages) 280 | topic)) 281 | (when (< read-offset (count messages)) 282 | (subvec messages read-offset)))) 283 | subscribed-topic-partitions)] 284 | (->> messages 285 | (take (max-poll-records config)) 286 | (group-by record->topic-partition)))) 287 | 288 | ;; TODO: implement missing methods 289 | ;; TODO: validate config? 290 | (defrecord MockConsumer [consumer-state wakeup-ch rebalance-control-ch join-ch leave-ch logger config] 291 | IRebalance 292 | (all-topics [_] (:subscribed-topics @consumer-state)) 293 | (apply-pending-topics [_ topics] 294 | (swap! consumer-state #(assoc % :subscribed-topics topics))) 295 | (clean-up-subscriptions [_] 296 | (swap! consumer-state #(assoc % :subscribed-topics [] :subscribed-topic-partitions {}))) 297 | Consumer 298 | (assign [_ partitions] 299 | (logger (format "-- [consumer %s] subscribe to partitions: %s" 300 | (get config "group.id") 301 | (pr-str partitions))) 302 | (let [broker-state @broker-state] 303 | (swap! consumer-state #(assoc % :subscribed-topic-partitions 304 | (reduce (fn [m topic-partition] 305 | (assoc m topic-partition 306 | (get-offset broker-state (.topic topic-partition) (.partition topic-partition) config))) 307 | {} partitions))))) 308 | (close [this] 309 | (swap! consumer-state close-mock) 310 | (.unsubscribe this)) 311 | (commitAsync [_] (throw (UnsupportedOperationException.))) 312 | (commitAsync [_ offsets cb] (throw (UnsupportedOperationException.))) 313 | (commitAsync [_ cb] (throw (UnsupportedOperationException.))) 314 | (commitSync [_] (throw (UnsupportedOperationException.))) 315 | (commitSync [_ offsets] 316 | (let [new-commits (reduce (fn [m [topic-partition offset-and-metadata]] 317 | (assoc m [(config "group.id" "") topic-partition] 318 | (.offset offset-and-metadata))) 319 | {} 320 | offsets)] 321 | (swap! committed-offsets merge new-commits))) 322 | (committed [_ partition] (throw (UnsupportedOperationException.))) 323 | (listTopics [_] (throw (UnsupportedOperationException.))) 324 | (metrics [_] (throw (UnsupportedOperationException.))) 325 | (partitionsFor [_ topic] (throw (UnsupportedOperationException.))) 326 | (pause [_ partitions] (throw (UnsupportedOperationException.))) 327 | (paused [_] (throw (UnsupportedOperationException.))) 328 | (poll [this max-timeout] 329 | ;; TODO: assert not closed 330 | (alt!! 331 | rebalance-control-ch ([[rebalance-participants-ch rebalance-complete-ch]] 332 | (>!! rebalance-participants-ch this) 333 | (alt!! 334 | rebalance-complete-ch nil 335 | (timeout consumer-rebalance-timeout) (throw (Exception. "dead waiting for rebalance"))) 336 | (.poll this max-timeout)) 337 | 338 | ;; Somebody outside needs to shutdown quickly, aborting the poll loop 339 | wakeup-ch (throw (WakeupException.)) 340 | 341 | :default 342 | (let [{:keys [subscribed-topic-partitions]} @consumer-state 343 | ;; Tell broker that if it doesn't have messages now, but gets them 344 | ;; while we're waiting for the timeout, we'd like to be interupted. 345 | ;; This prevents excessive waiting and handles the case of an 346 | ;; infinite max-timeout. 347 | poll-chan (desires-repoll @broker-state subscribed-topic-partitions) 348 | ;; Need to re-read broker state immediately after setting watchers, 349 | ;; so that we see any messages created between the time the poll 350 | ;; started and when we registered. The first read of the broker 351 | ;; state was just to find out where to put poll-chan 352 | topic-partition->messages (read-messages @broker-state subscribed-topic-partitions config)] 353 | (if (seq topic-partition->messages) 354 | (do 355 | (swap! consumer-state #(update % :subscribed-topic-partitions merge (read-offsets topic-partition->messages))) 356 | (ConsumerRecords. topic-partition->messages)) 357 | ;; Maybe we didn't actually have any messages to read 358 | (alt!! 359 | ;; We've waited too long for messages, give up 360 | (timeout max-timeout) ([_] 361 | ;; TODO: on timeout is it empty ConsumerRecords or nil? assuming nil for now 362 | ;; TODO: what does kafka do if not subscribed to any topics? currently assuming nil 363 | ;; TODO: read one last time, maybe with (.poll this 0), 364 | ;; but avoiding an infinite loop somehow? 365 | nil) 366 | ;; But, before the timeout, broker got new messsages on some 367 | ;; topic+partition that this consumer is interested in. It is 368 | ;; possible through race conditions that this signal was a 369 | ;; lie, that is, that we already read the messages the broker 370 | ;; is trying to tell us about, but it is harmless to retry as 371 | ;; long as we back off a little bit to avoid flooding the 372 | ;; watchers channel 373 | poll-chan ([_] 374 | (Thread/sleep consumer-backoff) 375 | (.poll this max-timeout)) 376 | ;; Somebody outside needs to shutdown quickly, and is aborting 377 | ;; the poll loop 378 | wakeup-ch (throw (WakeupException.))))))) 379 | (position [_ partition] 380 | ;; Not hard, but not valuable 381 | (throw (UnsupportedOperationException.))) 382 | (resume [_ partitions] (throw (UnsupportedOperationException.))) 383 | (seek [_ partition offset] (throw (UnsupportedOperationException.))) 384 | (seekToBeginning [_ partitions] (throw (UnsupportedOperationException.))) 385 | (seekToEnd [_ partitions] (throw (UnsupportedOperationException.))) 386 | (^void subscribe [^Consumer this ^Collection topics] 387 | (logger (format "-- [consumer %s] subscribe to topics: %s" 388 | (get config "group.id") 389 | (pr-str (seq topics)))) 390 | ;; TODO: what if already subscribed, what does Kafka do? 391 | (swap! broker-state #(reduce (fn [state topic] (broker-ensure-topic state topic)) % topics)) 392 | (>!! join-ch [this topics])) 393 | (^void subscribe [^Consumer this ^Collection topics ^ConsumerRebalanceListener listener] 394 | (throw (UnsupportedOperationException.))) 395 | (^void subscribe [^Consumer this ^Pattern pattern ^ConsumerRebalanceListener listener] 396 | (throw (UnsupportedOperationException.))) 397 | (unsubscribe [this] 398 | (alt!! 399 | [[leave-ch this]] :wrote 400 | (timeout consumer-unsubscribe-timeout) ([_] (logger "dead waiting to unsubscribe") nil))) 401 | (wakeup [_] 402 | (close! wakeup-ch))) 403 | 404 | (defn mock-consumer 405 | ([config] (mock-consumer [] config)) 406 | ([auto-subscribe-topics config] 407 | (assert (:join-ch @broker-state) "Broker is not running! Did you mean to call 'start!' first?") 408 | (config/assert-consumer-opts config) 409 | (let [{:keys [join-ch leave-ch]} @broker-state 410 | mock-consumer (->MockConsumer (atom {:subscribed-topic-partitions {}}) 411 | (chan) 412 | (chan buffer-size) 413 | join-ch 414 | leave-ch 415 | logger 416 | (merge config/default-consumer-config config))] 417 | (when (seq auto-subscribe-topics) 418 | (.subscribe mock-consumer auto-subscribe-topics)) 419 | mock-consumer))) 420 | 421 | ;; TODO: assertions 422 | (defn assert-proper-record [record] 423 | (assert (string? (.value record)) (str "Message record value should be a string. Got: " (type (.value record))))) 424 | (defn assert-producer-not-closed 425 | "Checks conn-open? in producer state" 426 | [producer-state]) 427 | 428 | (def noop-cb 429 | (reify 430 | Callback 431 | (onCompletion [this record-metadata e]))) 432 | 433 | (defrecord MockProducer [producer-state msg-ch config] 434 | Producer 435 | (close [_] (swap! producer-state close-mock)) 436 | (close [_ timeout time-unit] (swap! producer-state close-mock)) 437 | (flush [_] (throw (UnsupportedOperationException.))) 438 | (metrics [_] (throw (UnsupportedOperationException.))) 439 | (partitionsFor [_ topic] (throw (UnsupportedOperationException.))) 440 | (send [this record] 441 | (.send this record noop-cb)) 442 | (send [_ producer-record cb] 443 | (assert-proper-record producer-record) 444 | (assert-producer-not-closed producer-state) 445 | (logger "--MockProducer send" (pr-str producer-record)) 446 | (let [res-ch (chan 1) 447 | rtn-promise (promise)] 448 | (goe 449 | (>! msg-ch [res-ch producer-record]) 450 | (let [committed-record-metadata (MockProducer (atom nil) msg-ch (merge config/default-producer-config config)))) 475 | 476 | (defmethod core/make-consumer :mock [_ topics overrides] 477 | (mock-consumer topics overrides)) 478 | 479 | (defmethod core/make-producer :mock [_ overrides] 480 | (mock-producer overrides)) 481 | 482 | ;;;;; Test Helpers 483 | (defn record->clj [record] 484 | {:value (.value record) 485 | :key (.key record) 486 | :partition (.partition record) 487 | :topic (.topic record) 488 | :offset (.offset record)}) 489 | 490 | (defn records->clj 491 | ([consumer-records] 492 | (if (seq consumer-records) 493 | (map record->clj (iterator-seq (.iterator consumer-records))) 494 | []))) 495 | 496 | (defn get-messages 497 | "DEPRECATED: use `txfm-messages` instead. 498 | Poll until `consumer` receives some messages on `topic`. If `timeout` (in ms) 499 | expires first, return an empty vector." 500 | {:deprecated "0.5.10"} 501 | ([consumer topic timeout] 502 | (.subscribe consumer [topic]) 503 | (get-messages consumer timeout)) 504 | ([consumer timeout] 505 | (loop [i (int (Math/ceil (/ timeout 100)))] 506 | (if (> i 0) 507 | (if-let [consumer-records (seq (records->clj (.poll consumer 100)))] 508 | consumer-records 509 | (recur (dec i))) 510 | [])))) 511 | 512 | (defn accumulate-subscribed-messages 513 | "Helper to consume messages off any existing subscriptions. Will return as 514 | soon as `at-least-n` messages have been fetched. May return more, since an 515 | individual poll can fetch more. If `timeout` (in ms) expires first, returns 516 | any messages fetched so far. Mesages may be reformatted with `format-fn` then 517 | filtered with `filter-fn`. Messages excluded by `filter-fn` do not count 518 | towards `at-least-n`." 519 | [consumer {:keys [timeout at-least-n format-fn filter-fn] 520 | :or {at-least-n 1, format-fn identity, filter-fn identity}}] 521 | {:pre [(and (number? timeout) (pos? timeout))]} 522 | (loop [messages [] 523 | attempts (Math/ceil (/ timeout 100))] 524 | (if (or (>= (count messages) at-least-n) 525 | (neg? attempts)) 526 | messages 527 | (recur 528 | (concat messages 529 | (->> (seq (records->clj (.poll consumer 100))) 530 | (map format-fn) 531 | (filter filter-fn))) 532 | (dec attempts))))) 533 | 534 | (defn accumulate-messages 535 | "Like `accumulate-subscribed-messages`, but subscribes to the `topic` first." 536 | [consumer topic options] 537 | (.subscribe consumer [topic]) 538 | (accumulate-subscribed-messages consumer options)) 539 | 540 | (defn txfm-subscribed-messages 541 | "Helper to txfm messages sent to the `consumer` on any existing subscriptions. 542 | Messages will be transformed by the transducing function `xf`. Will return as 543 | soon as `xf` indicates it has enough records, or the `timeout` (in ms, by 544 | default 1000) expires. Thus, to avoid waiting for the `timeout`, include e.g. 545 | `(take 3)` in `xf`. If the `timeout` expires, will return any messages 546 | transformed so far. 547 | 548 | For example, 549 | 550 | ``` clojure 551 | (kafka-mock/send producer \"topic\" \"key\" \"5\") 552 | (kafka-mock/send producer \"topic\" \"key\" \"6\") 553 | (kafka-mock/send producer \"topic\" \"key\" \"7\") 554 | (kafka-mock/send producer \"topic\" \"key\" \"8\") 555 | (.subscribe consumer [\"topic\"]) 556 | (kafka-mock/txfm-subscribed-messages consumer (comp (map :value) 557 | (map #(Integer/parseInt %)) 558 | (filter even?) 559 | (take 1))) 560 | ;; => [6] 561 | ``` 562 | 563 | By default messages retrieved from `(.poll consumer)` will be collected as by 564 | `clojure.core/cat` and fed one-by-one through `xf`. This behvaior can be 565 | altered by providing a different `ixf`. For example, `clojure.core/conj` will 566 | feed each batch through in its entirety. This can be combined with 567 | `max.poll.records` to control batch size. 568 | 569 | Also by default, the transformed messages will be accumulated as by 570 | `clojure.core/conj`, and returned. A different reducing function `rf` can be 571 | supplied. For example, if `xf` produces a stream of numbers, the reducing 572 | function `clojure.core/+` will return their sum." 573 | 574 | ([consumer xf] (txfm-subscribed-messages consumer xf {})) 575 | ([consumer xf {:keys [timeout ixf rf] :or {timeout 1000, ixf cat, rf conj}}] 576 | {:pre [(number? timeout) 577 | (pos? timeout)]} 578 | (let [f ((comp ixf xf) rf)] 579 | (loop [acc (f) 580 | attempts (dec (quot timeout 100))] 581 | (let [result (f acc (records->clj (.poll consumer 100)))] 582 | (cond 583 | (reduced? result) @result 584 | (not (pos? attempts)) result 585 | :else (recur result (dec attempts)))))))) 586 | 587 | (defn txfm-messages 588 | "Like `txfm-subscribed-messages`, but subscribes to the `topic` first." 589 | ([consumer topic xf] (txfm-messages consumer topic xf {})) 590 | ([consumer topic xf options] 591 | (.subscribe consumer [topic]) 592 | (txfm-subscribed-messages consumer xf options))) 593 | 594 | (defn send-async [producer topic k v] 595 | (gregor/send producer topic k v)) 596 | 597 | (defn send [producer topic k v] 598 | @(send-async producer topic k v)) 599 | 600 | (defmacro with-test-broker [& body] 601 | `(do 602 | (start!) 603 | (try 604 | ~@body 605 | (finally (shutdown!))))) 606 | 607 | (defmacro with-test-producer-consumer [producer-name consumer-name & body] 608 | `(with-test-broker 609 | (let [~producer-name (mock-producer {}) 610 | ~consumer-name (mock-consumer standalone-mock-consumer-opts)] 611 | ~@body))) 612 | -------------------------------------------------------------------------------- /test/kafka_component/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns kafka-component.core-test 2 | (:require [clojure.test :refer :all] 3 | [com.stuartsierra.component :as component] 4 | [embedded-kafka.core :as ek] 5 | [kafka-component.core :refer :all] 6 | [gregor.core :as gregor])) 7 | 8 | (def test-config {:kafka-reader-config {:concurrency-level 1 9 | :topics ["test_events"] 10 | :native-consumer-overrides ek/kafka-config} 11 | :kafka-writer-config {:structured-logging? true 12 | :native-producer-overrides ek/kafka-config}}) 13 | 14 | (defn poorly-implemented-processor [state-atom] 15 | {:process (juxt (partial swap! state-atom conj) 16 | (partial prn "Message consumed: ") 17 | (fn [m] (throw (ex-info "poorly implemented to test failures while processing a message" {}))))}) 18 | 19 | (defn single-delivery-processor [msg-promise] 20 | {:process (juxt (partial deliver msg-promise) 21 | (partial prn "Message consumed: "))}) 22 | 23 | (defn test-system 24 | ([config] 25 | (test-system config identity)) 26 | ([config transform] 27 | (let [messages (promise)] 28 | (component/system-using 29 | (transform (component/system-map 30 | :logger println 31 | :exception-handler println 32 | :messages messages 33 | :test-event-record-processor (single-delivery-processor messages) 34 | :test-event-reader (map->KafkaReader (:kafka-reader-config config)) 35 | :writer (map->KafkaWriter (:kafka-writer-config config)))) 36 | {:test-event-reader {:logger :logger 37 | :exception-handler :exception-handler 38 | :record-processor :test-event-record-processor}})))) 39 | 40 | (defmacro with-resource 41 | [bindings close-fn & body] 42 | `(let ~bindings 43 | (try 44 | ~@body 45 | (finally 46 | (~close-fn ~(bindings 0)))))) 47 | 48 | (defmacro with-test-system 49 | [config sys & body] 50 | `(with-resource [system# (component/start (test-system ~config))] 51 | component/stop 52 | (let [~sys system#] 53 | ~@body))) 54 | 55 | (defmacro with-transformed-test-system 56 | [config transform sys & body] 57 | `(with-resource [system# (component/start (test-system ~config ~transform))] 58 | component/stop 59 | (let [~sys system#] 60 | ~@body))) 61 | 62 | (defn- now [] 63 | (.getTime (java.util.Date.))) 64 | 65 | (defn wait-until [pred timeout] 66 | (let [start-time (now) 67 | timed-out? (fn [start-time duration] 68 | (let [elapsed (- (now) start-time)] 69 | (> elapsed duration)))] 70 | (loop [res (pred)] 71 | (cond res res 72 | (timed-out? start-time timeout) nil 73 | :else (do 74 | (Thread/sleep 10) 75 | (recur (pred))))))) 76 | 77 | (deftest sending-and-receiving-messages-using-kafka-with-message-commits 78 | (ek/with-test-broker producer consumer 79 | (with-test-system test-config {:keys [messages writer]} 80 | (write writer "test_events" "key" "yolo") 81 | (is (= {:topic "test_events" :partition 0 :key "key" :offset 0 :value "yolo"} 82 | (deref messages 10000 {}))) 83 | (testing "it should commit offsets to message offset + 1" 84 | (is (wait-until #(= 1 (:offset (gregor/committed consumer "test_events" 0))) 85 | 10000)))))) 86 | 87 | (deftest when-provided-exception-handler-throws-an-exception-component-can-still-read-messages 88 | (testing "recovering from a poorly implemented exception handler" 89 | (with-redefs [panic! (fn [])] 90 | (let [messages (atom [])] 91 | (ek/with-test-broker producer consumer 92 | (with-transformed-test-system test-config 93 | (fn [sys] (assoc sys 94 | :exception-handler (fn [e] (throw (ex-info "Fail Whale" {}))) 95 | :test-event-record-processor (poorly-implemented-processor messages))) 96 | {:keys [writer]} 97 | (write writer "test_events" "key" "yolo") 98 | (write writer "test_events" "key" "yolo") 99 | (is (wait-until (fn [] (= 2 (count @messages))) 10000)) 100 | (testing "it should commit offsets to message offset + 1" 101 | (is (wait-until #(= 2 (:offset (gregor/committed consumer "test_events" 0))) 102 | 10000))))))))) 103 | 104 | (deftest when-provided-logger-handler-throws-an-exception-component-can-still-read-messages 105 | (testing "recovering from a poorly implemented logger" 106 | (with-redefs [panic! (fn [])] 107 | (let [messages (atom [])] 108 | (ek/with-test-broker producer consumer 109 | (with-transformed-test-system test-config 110 | (fn [sys] (assoc sys 111 | :logger (fn [level err] (when (not (#{:info :debug} level)) 112 | (throw (ex-info "Fail Whale" {})))) 113 | :test-event-record-processor (poorly-implemented-processor messages))) 114 | {:keys [writer]} 115 | (write writer "test_events" "key" "yolo") 116 | (write writer "test_events" "key" "yolo") 117 | (is (wait-until (fn [] (= 2 (count @messages))) 10000)) 118 | (testing "it should commit offsets to message offset + 1" 119 | (is (wait-until #(= 2 (:offset (gregor/committed consumer "test_events" 0))) 120 | 10000))))))))) 121 | 122 | (deftest reader-fail-when-auto-offset-reset-is-invalid 123 | (let [test-config (assoc-in test-config [:kafka-reader-config :native-consumer-overrides "auto.offset.reset"] "smallest")] 124 | (is (thrown? Exception 125 | (with-test-system test-config sys))))) 126 | 127 | (deftest reader-fail-when-bootstrap-servers-is-missing 128 | (let [test-config (update-in test-config [:kafka-reader-config :native-consumer-overrides] dissoc "bootstrap.servers")] 129 | (is (thrown? Exception 130 | (with-test-system test-config sys))))) 131 | 132 | (deftest reader-fail-when-group-id-is-missing 133 | (let [test-config (update-in test-config [:kafka-reader-config :native-consumer-overrides] dissoc "group.id")] 134 | (is (thrown? Exception 135 | (with-test-system test-config sys))))) 136 | 137 | (deftest reader-fail-when-shutdown-grace-period-is-zero 138 | (let [test-config (assoc-in test-config [:kafka-reader-config :shutdown-timeout] 0)] 139 | (is (thrown? Exception 140 | (with-test-system test-config sys))))) 141 | 142 | (deftest reader-fails-when-given-a-record-processor-without-process 143 | (let [test-transform (fn [system] 144 | (assoc system 145 | :test-event-record-processor 146 | {:not-the-right-key (fn [record])}))] 147 | (is (thrown? Exception 148 | (with-transformed-test-system test-config test-transform sys))))) 149 | 150 | (deftest reader-fails-when-given-a-record-processor-with-bad-process 151 | (let [test-transform (fn [system] 152 | (assoc system 153 | :test-event-record-processor 154 | {:process "not a function"}))] 155 | (is (thrown? Exception 156 | (with-transformed-test-system test-config test-transform sys))))) 157 | 158 | (deftest reader-can-be-stopped-before-being-started 159 | (let [system (test-system test-config)] 160 | (component/stop system) 161 | (is true "The real test is that the above does not throw an exception. This is just to appease clojure.test, which expects at least one 'is'."))) 162 | -------------------------------------------------------------------------------- /test/kafka_component/mock_test.clj: -------------------------------------------------------------------------------- 1 | (ns kafka-component.mock-test 2 | (:require [clojure.test :refer :all] 3 | [com.stuartsierra.component :as component] 4 | [kafka-component 5 | [core :as core] 6 | [core-test :as core-test :refer [with-resource]] 7 | [mock :as mock]]) 8 | (:import [org.apache.kafka.clients.producer Callback ProducerRecord RecordMetadata] 9 | org.apache.kafka.common.errors.WakeupException)) 10 | 11 | (use-fixtures :each mock/fixture-restart-broker!) 12 | 13 | (defn deep-merge 14 | [& maps] 15 | (if (every? map? maps) 16 | (apply merge-with deep-merge maps) 17 | (last maps))) 18 | 19 | (def mock-config (deep-merge core-test/test-config 20 | {:kafka-writer-config {:native-producer-type :mock} 21 | :kafka-reader-config {:native-consumer-type :mock 22 | :native-consumer-overrides mock/default-mock-consumer-opts}})) 23 | 24 | (defmacro with-test-system 25 | [config sys & body] 26 | `(with-resource [system# (component/start (core-test/test-system (deep-merge mock-config ~config)))] 27 | component/stop 28 | (let [~sys system#] 29 | ~@body))) 30 | 31 | (deftest sending-and-receiving-messages-using-mock 32 | (with-test-system {} {:keys [messages writer]} 33 | (core/write writer "test_events" "key" "yolo") 34 | (is (= {:topic "test_events" :partition 0 :key "key" :offset 0 :value "yolo"} 35 | (deref messages 500 []))))) 36 | 37 | (def timeout 500) 38 | 39 | (defn mock-consumer [overrides] 40 | (core/make-consumer :mock [] (merge mock/standalone-mock-consumer-opts overrides))) 41 | 42 | (defn mock-producer [overrides] 43 | (core/make-producer :mock overrides)) 44 | 45 | (defn producer-record 46 | ([topic k v] (ProducerRecord. topic k v)) 47 | ([topic k v partition] (ProducerRecord. topic (int partition) k v))) 48 | 49 | (defn reify-send-callback [cb] 50 | (reify Callback 51 | (onCompletion [this metadata ex] 52 | (cb metadata ex)))) 53 | 54 | (deftest send-on-producer-returns-a-future-of-RecordMetadata 55 | (let [producer (mock-producer {}) 56 | res (mock/send producer "topic" "key" "value")] 57 | (is (= RecordMetadata (type res))) 58 | (is (= "topic" (.topic res))) 59 | (is (= 0 (.partition res))) 60 | (is (= 0 (.offset res))))) 61 | 62 | (deftest send-non-string-to-producer-raises-exception 63 | (let [producer (mock-producer {})] 64 | (try 65 | (mock/send producer "topic" "key" {"hello" "world"}) 66 | (is false "Expected exception thrown") 67 | (catch AssertionError e 68 | (is (.contains (.getMessage e) "Message record value should be a string. Got:") 69 | (.getMessage e)))))) 70 | 71 | (deftest send-on-producer-increments-offset 72 | (let [producer (mock-producer {}) 73 | res (repeatedly 2 #(mock/send-async producer "topic" "key" "value"))] 74 | (is (= [0 1] (map (comp #(.offset %) deref) res))))) 75 | 76 | (deftest send-on-producer-with-callback-calls-the-callback 77 | (let [producer (mock-producer {}) 78 | cb-res (promise) 79 | cb #(deliver cb-res [%1 %2]) 80 | _ (.send producer (producer-record "topic" "key" "value") 81 | (reify-send-callback cb)) 82 | [res ex] (deref cb-res timeout [])] 83 | (is (= RecordMetadata (type res))) 84 | (is (= "topic" (.topic res))) 85 | (is (= 0 (.partition res))) 86 | (is (= 0 (.offset res))))) 87 | 88 | (deftest consumer-can-receive-message-sent-after-subscribing 89 | (mock/shutdown!) 90 | (mock/with-test-producer-consumer producer consumer 91 | (.subscribe consumer ["topic"]) 92 | (mock/send producer "topic" "key" "value") 93 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0}] 94 | (mock/txfm-subscribed-messages consumer (take 1) {:timeout timeout})))) 95 | (mock/start!)) 96 | 97 | (deftest consumer-can-receive-message-from-different-partitions 98 | (let [producer (mock-producer {}) 99 | consumer (mock-consumer {"max.poll.records" "2"})] 100 | @(.send producer (producer-record "topic" "key" "value" 0)) 101 | @(.send producer (producer-record "topic" "key" "value" 1)) 102 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0} 103 | {:value "value" :key "key" :partition 1 :topic "topic" :offset 0}] 104 | (sort-by :partition (mock/txfm-messages consumer "topic" (take 2) {:timeout timeout})))))) 105 | 106 | (deftest consumer-can-limit-number-of-messages-polled 107 | (let [producer (mock-producer {}) 108 | consumer (mock-consumer {"max.poll.records" "1"})] 109 | (mock/send producer "topic" "key" "value") 110 | (mock/send producer "topic" "key2" "value2") 111 | (is (= [[{:value "value" :key "key" :partition 0 :topic "topic" :offset 0}] 112 | [{:value "value2" :key "key2" :partition 0 :topic "topic" :offset 1}]] 113 | (mock/txfm-messages consumer "topic" (take 2) {:ixf conj 114 | :timeout timeout}))))) 115 | 116 | (deftest consumer-can-receive-message-sent-before-subscribing 117 | (let [producer (mock-producer {}) 118 | consumer (mock-consumer {"auto.offset.reset" "earliest"})] 119 | (mock/send producer "topic" "key" "value") 120 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0}] 121 | (mock/txfm-messages consumer "topic" (take 1) {:timeout timeout}))))) 122 | 123 | (deftest consumer-can-use-latest-auto-offset-reset-to-skip-earlier-messages 124 | (let [producer (mock-producer {}) 125 | consumer (mock-consumer {"auto.offset.reset" "latest"})] 126 | (mock/send producer "topic" "key" "value") 127 | (is (= [] (mock/txfm-messages consumer "topic" identity {:timeout timeout}))))) 128 | 129 | (deftest consumer-can-receive-messages-from-multiple-topics 130 | (let [producer (mock-producer {}) 131 | consumer (mock-consumer {"max.poll.records" "2"})] 132 | (.subscribe consumer ["topic" "topic2"]) 133 | (mock/send producer "topic" "key" "value") 134 | (mock/send producer "topic2" "key2" "value2") 135 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0} 136 | {:value "value2" :key "key2" :partition 0 :topic "topic2" :offset 0}] 137 | (sort-by :topic (mock/txfm-subscribed-messages consumer (take 2) {:timeout timeout})))))) 138 | 139 | (deftest consumer-waits-for-new-messages-to-arrive 140 | (mock/shutdown!) 141 | (mock/with-test-producer-consumer producer consumer 142 | (let [msg-promise (promise)] 143 | (future (deliver msg-promise (mock/txfm-messages consumer "topic" (take 1) {:timeout (* 4 timeout)}))) 144 | (mock/send producer "topic" "key" "value") 145 | (is (= 1 (count (deref msg-promise (* 8 timeout) [])))))) 146 | (mock/start!)) 147 | 148 | (deftest consumer-can-unsubscribe-from-topics 149 | (let [producer (mock-producer {}) 150 | consumer (mock-consumer {"max.poll.records" "2"})] 151 | (.subscribe consumer ["topic" "topic2"]) 152 | (mock/send producer "topic" "key" "value") 153 | (mock/send producer "topic2" "key2" "value2") 154 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0} 155 | {:value "value2" :key "key2" :partition 0 :topic "topic2" :offset 0}] 156 | (sort-by :partition (mock/txfm-subscribed-messages consumer (take 2) {:timeout timeout})))) 157 | 158 | (.unsubscribe consumer) 159 | 160 | (mock/send producer "topic" "key" "value") 161 | (mock/send producer "topic2" "key2" "value2") 162 | (is (= [] (mock/txfm-subscribed-messages consumer identity {:timeout timeout}))))) 163 | 164 | (deftest consumer-can-be-woken-up 165 | (let [consumer (mock-consumer {}) 166 | woken (promise)] 167 | (.subscribe consumer ["topic"]) 168 | (future 169 | (try 170 | (let [res (.poll consumer (* 2 timeout))] 171 | (println res)) 172 | (catch WakeupException e 173 | (deliver woken "I'm awake!")))) 174 | (.wakeup consumer) 175 | (is (= "I'm awake!" (deref woken timeout nil))))) 176 | 177 | (deftest consumer-can-be-woken-up-outside-of-poll-and-poll-still-throws-wakeup-exception 178 | (let [consumer (mock-consumer {}) 179 | woken (promise)] 180 | (.subscribe consumer ["topic"]) 181 | (.wakeup consumer) 182 | (future 183 | (try 184 | (let [res (.poll consumer timeout)] 185 | (println res)) 186 | (catch WakeupException e 187 | (deliver woken "I'm awake!")))) 188 | (is (= "I'm awake!" (deref woken timeout nil))))) 189 | 190 | (defn consume-messages [expected-message-count messages messages-promise msg] 191 | (locking expected-message-count 192 | (let [updated-messages (swap! messages conj msg)] 193 | (when (>= (count updated-messages) expected-message-count) 194 | (deliver messages-promise @messages))))) 195 | 196 | (deftest mock-system-can-be-started-to-consume-messages 197 | (let [received-messages (promise)] 198 | (core-test/with-transformed-test-system 199 | (deep-merge mock-config 200 | {:kafka-reader-config {:topics ["topic"] 201 | :concurrency-level 1 202 | :poll-interval 10 203 | :native-consumer-overrides {"auto.offset.reset" "earliest" 204 | "group.id" "test"}}}) 205 | (fn [system-map] 206 | (assoc-in system-map [:test-event-record-processor :process] 207 | (partial consume-messages 2 (atom []) received-messages))) 208 | sys 209 | (let [producer (mock-producer {})] 210 | @(.send producer (producer-record "topic" "key" "value" 0)) 211 | @(.send producer (producer-record "topic" "key2" "value2" 1)) 212 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0} 213 | {:value "value2" :key "key2" :partition 1 :topic "topic" :offset 0}] 214 | (sort-by :partition (deref received-messages 5000 [])))))))) 215 | 216 | (deftest multiple-consumers-in-the-same-group-share-the-messages 217 | (let [received-messages (promise)] 218 | (core-test/with-transformed-test-system 219 | (deep-merge mock-config 220 | {:kafka-reader-config {:topics ["topic"] 221 | :concurrency-level 2 222 | :poll-interval 10 223 | :native-consumer-overrides {"auto.offset.reset" "earliest" 224 | "group.id" "test"}}}) 225 | (fn [system-map] 226 | (assoc-in system-map [:test-event-record-processor :process] 227 | (partial consume-messages 2 (atom []) received-messages))) 228 | sys 229 | (let [producer (mock-producer {})] 230 | @(.send producer (producer-record "topic" "key" "value" 0)) 231 | @(.send producer (producer-record "topic" "key2" "value2" 1)) 232 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0} 233 | {:value "value2" :key "key2" :partition 1 :topic "topic" :offset 0}] 234 | (sort-by :partition (deref received-messages 5000 [])))))))) 235 | 236 | (deftest multiple-consumers-in-the-same-group-process-each-message-only-once 237 | (let [message-count (atom 0)] 238 | (core-test/with-transformed-test-system 239 | (deep-merge mock-config 240 | {:kafka-reader-config {:topics ["topic"] 241 | :concurrency-level 2 242 | :poll-interval 10 243 | :native-consumer-overrides {"auto.offset.reset" "earliest" 244 | "group.id" "test-group"}}}) 245 | (fn [system-map] 246 | (assoc-in system-map [:test-event-record-processor :process] 247 | (fn [msg] (swap! message-count inc)))) 248 | sys 249 | (let [producer (mock-producer {})] 250 | @(.send producer (producer-record "topic" "key" "value" 0)) 251 | @(.send producer (producer-record "topic" "key2" "value2" 1)) 252 | (Thread/sleep (* 4 timeout)) 253 | (is (= 2 @message-count)))))) 254 | 255 | (deftest multiple-consumers-in-multiple-groups-share-the-messages-appropriately 256 | (let [group-1-received-messages (promise) 257 | group-2-received-messages (promise)] 258 | (core-test/with-transformed-test-system 259 | (deep-merge mock-config 260 | {:kafka-reader-config {:topics ["topic"] 261 | :concurrency-level 2 262 | :poll-interval 10 263 | :native-consumer-overrides {"auto.offset.reset" "earliest" 264 | "group.id" "group1"}}}) 265 | (fn [system-map] 266 | (assoc-in system-map [:test-event-record-processor :process] 267 | (partial consume-messages 2 (atom []) group-1-received-messages))) 268 | sys1 269 | (core-test/with-transformed-test-system 270 | (deep-merge mock-config 271 | {:kafka-reader-config {:topics ["topic"] 272 | :concurrency-level 2 273 | :poll-interval 10 274 | :native-consumer-overrides {"auto.offset.reset" "earliest" 275 | "group.id" "group2"}}}) 276 | (fn [system-map] 277 | (assoc-in system-map [:test-event-record-processor :process] 278 | (partial consume-messages 2 (atom []) group-2-received-messages))) 279 | sys2 280 | (let [producer (mock-producer {})] 281 | @(.send producer (producer-record "topic" "key" "value" 0)) 282 | @(.send producer (producer-record "topic" "key2" "value2" 1)) 283 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0} 284 | {:value "value2" :key "key2" :partition 1 :topic "topic" :offset 0}] 285 | (sort-by :partition (deref group-1-received-messages 5000 [])))) 286 | (is (= [{:value "value" :key "key" :partition 0 :topic "topic" :offset 0} 287 | {:value "value2" :key "key2" :partition 1 :topic "topic" :offset 0}] 288 | (sort-by :partition (deref group-2-received-messages 5000 []))))))))) 289 | 290 | (deftest producers-can-be-closed 291 | (let [writer (mock-producer {})] 292 | (component/start writer) 293 | (component/stop writer) 294 | (is true "true to avoid cider's no assertion error"))) 295 | 296 | (deftest producers-fail-when-broker-is-not-started 297 | (mock/shutdown!) 298 | (try 299 | (mock-producer {}) 300 | (is false "expected exception to be raised") 301 | (catch Throwable e 302 | (is (.contains (.getMessage e) "Broker is not running! Did you mean to call 'start!' first?") 303 | (str "Got: " (.getMessage e))))) 304 | (mock/start!)) 305 | 306 | (deftest consumers-fail-when-broker-is-not-started 307 | (mock/shutdown!) 308 | (try 309 | (mock-consumer {}) 310 | (is false "expected exception to be raised") 311 | (catch Throwable e 312 | (is (.contains (.getMessage e) "Broker is not running! Did you mean to call 'start!' first?") 313 | (str "Got: " (.getMessage e))))) 314 | (mock/start!)) 315 | 316 | (deftest consumers-fail-when-auto-offset-reset-is-invalid 317 | (try 318 | (mock-consumer {"auto.offset.reset" "a-cat"}) 319 | (is false "expected exception to be raised") 320 | (catch Throwable e 321 | (is (.contains (.getMessage e) "\"auto.offset.reset\" should be set to one of #{\"latest\" \"earliest\" \"none\"}") 322 | (str "Got: " (.getMessage e)))))) 323 | 324 | (deftest consumers-fail-when-bootstrap-servers-is-missing 325 | (try 326 | (core/make-consumer :mock [] (dissoc mock/standalone-mock-consumer-opts "bootstrap.servers")) 327 | (is false "expected exception to be raised") 328 | (catch Throwable e 329 | (is (.contains (.getMessage e) "\"bootstrap.servers\" must be provided in the config") 330 | (str "Got: " (.getMessage e)))))) 331 | 332 | (deftest consumers-fail-when-group-id-is-missing 333 | (try 334 | (core/make-consumer :mock [] (dissoc mock/standalone-mock-consumer-opts "group.id")) 335 | 336 | (is false "expected exception to be raised") 337 | (catch Throwable e 338 | (is (.contains (.getMessage e) "\"group.id\" must be provided in the config") 339 | (str "Got: " (.getMessage e)))))) 340 | 341 | (deftest reader-fail-when-request-timeout-invalid 342 | (let [mock-config (assoc-in mock-config [:kafka-reader-config :native-consumer-overrides "request.timeout.ms"] "12")] 343 | (is (thrown? Exception 344 | (with-test-system mock-config sys))))) 345 | 346 | (deftest reader-fail-when-given-non-string-values 347 | (let [mock-config (assoc-in mock-config [:kafka-reader-config :native-consumer-overrides "request.timeout.ms"] 30000)] 348 | (is (thrown? Exception 349 | (with-test-system mock-config sys))))) 350 | 351 | --------------------------------------------------------------------------------